cld 96 third_party/cld/encodings/compact_lang_det/cldutil.cc uint32 cld::BiHashV25(const char* word_ptr, int bytecount) { cld 184 third_party/cld/encodings/compact_lang_det/cldutil.cc uint32 cld::QuadHashV25(const char* word_ptr, int bytecount) { cld 198 third_party/cld/encodings/compact_lang_det/cldutil.cc uint32 cld::QuadHashV25Underscore(const char* word_ptr, int bytecount) { cld 338 third_party/cld/encodings/compact_lang_det/cldutil.cc uint64 cld::OctaHash40(const char* word_ptr, int bytecount) { cld 356 third_party/cld/encodings/compact_lang_det/cldutil.cc uint64 cld::OctaHash40underscore(const char* word_ptr, int bytecount) { cld 386 third_party/cld/encodings/compact_lang_det/cldutil.cc void cld::ProcessProbV25UniTote(int propval, Tote* tote) { cld 389 third_party/cld/encodings/compact_lang_det/cldutil.cc if (pa->probs[0] > 0) {tote->Add(cld::PackLanguage(CHINESE), pa->probs[0]);} cld 390 third_party/cld/encodings/compact_lang_det/cldutil.cc if (pa->probs[1] > 0) {tote->Add(cld::PackLanguage(CHINESE_T), pa->probs[1]);} cld 391 third_party/cld/encodings/compact_lang_det/cldutil.cc if (pa->probs[2] > 0) {tote->Add(cld::PackLanguage(JAPANESE), pa->probs[2]);} cld 392 third_party/cld/encodings/compact_lang_det/cldutil.cc if (pa->probs[3] > 0) {tote->Add(cld::PackLanguage(KOREAN), pa->probs[3]);} cld 393 third_party/cld/encodings/compact_lang_det/cldutil.cc if (pa->probs[4] > 0) {tote->Add(cld::PackLanguage(VIETNAMESE), pa->probs[4]);} cld 394 third_party/cld/encodings/compact_lang_det/cldutil.cc if (pa->probs[5] > 0) {tote->Add(cld::PackLanguage(ZHUANG), pa->probs[5]);} cld 401 third_party/cld/encodings/compact_lang_det/cldutil.cc void cld::ProcessProbV25Tote(uint32 probs, Tote* tote) { cld 404 third_party/cld/encodings/compact_lang_det/cldutil.cc const uint8* prob123_entry = cld::LgProb2TblEntry(prob123); cld 407 third_party/cld/encodings/compact_lang_det/cldutil.cc if (top1 > 0) {tote->Add(top1, cld::LgProb3(prob123_entry, 0));} cld 409 third_party/cld/encodings/compact_lang_det/cldutil.cc if (top2 > 0) {tote->Add(top2, cld::LgProb3(prob123_entry, 1));} cld 411 third_party/cld/encodings/compact_lang_det/cldutil.cc if (top3 > 0) {tote->Add(top3, cld::LgProb3(prob123_entry, 2));} cld 424 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::DoUniScoreV3(const UTF8PropObj* unigram_obj, cld 503 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::DoBigramScoreV3(const cld::CLDTableSummary* bigram_obj, cld 561 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::DoQuadScoreV3(const cld::CLDTableSummary* quadgram_obj, cld 677 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::DoOctaScoreV3(const cld::CLDTableSummary* octagram_obj, cld 764 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::ReliabilityDelta(int value1, int value2, int gramcount) { cld 779 third_party/cld/encodings/compact_lang_det/cldutil.cc return cld::minint(max_reliability_percent, cld 789 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::ReliabilityMainstream(int topscore, int len, int mean_score) { cld 811 third_party/cld/encodings/compact_lang_det/cldutil.cc double cld::GetNormalizedScore(Language lang, UnicodeLScript lscript, cld 839 third_party/cld/encodings/compact_lang_det/cldutil.cc int cld::GetReliability(int len, UnicodeLScript lscript, cld 868 third_party/cld/encodings/compact_lang_det/cldutil.cc cld::DbgLangName3(UnpackLanguage(chunk_tote->Key(0)), temp1); cld 870 third_party/cld/encodings/compact_lang_det/cldutil.cc cld::DbgLangName3(UnpackLanguage(chunk_tote->Key(1)), temp2); cld 898 third_party/cld/encodings/compact_lang_det/cldutil.cc void cld::DemoteNotTop40(Tote* chunk_tote, int packed_plus_one) { cld 15 third_party/cld/encodings/compact_lang_det/cldutil.h namespace cld { cld 1039 third_party/cld/encodings/compact_lang_det/cldutil.h inline const uint32 QuadHashV3Lookup4(const cld::CLDTableSummary* gram_obj, cld 1089 third_party/cld/encodings/compact_lang_det/cldutil.h inline const uint32 OctaHashV3Lookup4(const cld::CLDTableSummary* gram_obj, cld 1147 third_party/cld/encodings/compact_lang_det/cldutil.h int DoBigramScoreV3(const cld::CLDTableSummary* bigram_obj, cld 1153 third_party/cld/encodings/compact_lang_det/cldutil.h int DoQuadScoreV3(const cld::CLDTableSummary* quadgram_obj, cld 1160 third_party/cld/encodings/compact_lang_det/cldutil.h int DoOctaScoreV3(const cld::CLDTableSummary* octagram_obj, cld 21 third_party/cld/encodings/compact_lang_det/cldutil_dbg.h namespace cld { cld 54 third_party/cld/encodings/compact_lang_det/compact_lang_det.h namespace cld { cld 103 third_party/cld/encodings/compact_lang_det/compact_lang_det.h const cld::CLDTableSummary* quadgram_obj; cld 30 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc extern const cld::CLDTableSummary kCjkBiTable_obj; cld 31 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc extern const cld::CLDTableSummary kQuadTable_obj; cld 32 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc extern const cld::CLDTableSummary kLongWord8Table_obj; cld 770 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc limit = cld::minint(limit, kMaxSpaceScan); cld 783 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc limit = cld::minint(limit, kMaxSpaceScan); cld 989 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int len = cld::minint(chunksize, remaining_bytes); cld 1108 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (cld::UnpackLanguage(doc_tote->Key(sub)) >= NUM_LANGUAGES) { cld 1112 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc ExtLanguageCode(cld::UnpackLanguage(doc_tote->Key(sub)))); cld 1146 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language lang = cld::UnpackLanguage(plang); cld 1162 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int altsub = doc_tote->Find(cld::PackLanguage(altlang)); cld 1184 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int newpercent = cld::maxint(reliable_percent, reliable_percent2); cld 1185 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc newpercent = cld::maxint(newpercent, kMinReliableKeepPercent); cld 1215 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language lang = cld::UnpackLanguage(plang); cld 1220 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc bool is_tier3 = (cld::kIsPackedTop40[plang] == 0); cld 1269 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc from_lang = cld::UnpackLanguage(close_packedlang); cld 1270 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc to_lang = cld::UnpackLanguage(close_packedlang2); cld 1274 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc from_lang = cld::UnpackLanguage(close_packedlang2); cld 1275 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc to_lang = cld::UnpackLanguage(close_packedlang); cld 1320 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc ExtLanguageCode(cld::UnpackLanguage(lang_sub)), cld 1367 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language cur_lang = cld::UnpackLanguage(chunk_tote->Key(0)); cld 1379 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int reliability = cld::GetReliability((len * 2) / advance_by, cld 1382 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cur_unreliable = (reliability < cld::kMinReliable); cld 1421 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc ExtLanguageCode(cld::UnpackLanguage(chunk_tote->Key(0))), cld 1425 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc ExtLanguageCode(cld::UnpackLanguage(chunk_tote->Key(1))), cld 1436 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc ExtLanguageCode(cld::UnpackLanguage(chunk_tote->Key(0))), cld 1445 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::PrintLang(stderr, chunk_tote, cld 1458 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::PrintText(stderr, cur_lang, temp); cld 1545 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int len = cld::DoUniScoreV3(unigram_obj, cld 1550 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::DoBigramScoreV3(&kCjkBiTable_obj, cld 1560 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::DemoteNotTop40(chunk_tote, cld::PackLanguage(plus_one)); cld 1571 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language top_lang = cld::UnpackLanguage(chunk_tote->CurrentTopKey()); cld 1573 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::PrintText(stderr, top_lang, temp); cld 1593 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc static void ScoreQuadgrams(const cld::CLDTableSummary* quadgram_obj, cld 1613 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int len = cld::DoQuadScoreV3(quadgram_obj, cld 1618 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::DoOctaScoreV3(&kLongWord8Table_obj, cld 1627 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::DemoteNotTop40(chunk_tote, cld::PackLanguage(plus_one)); cld 1639 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language top_lang = cld::UnpackLanguage(chunk_tote->CurrentTopKey()); cld 1641 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::PrintText(stderr, top_lang, temp); cld 1673 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language defaultlang = cld::kDefaultLanguagePerLScript[lscript]; cld 1674 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc script_tote->Add(cld::PackLanguage(defaultlang), 1); cld 1678 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::PrintLang(stderr, script_tote, cld 1683 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc cld::PrintText(stderr, defaultlang, temp); cld 1795 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc const uint8* prob123_entry = cld::LgProb2TblEntry(prob123); cld 1797 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top1 > 0) {lang_hint_boost[top1] += cld::LgProb3(prob123_entry, 0);} cld 1799 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top2 > 0) {lang_hint_boost[top2] += cld::LgProb3(prob123_entry, 1);} cld 1801 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top3 > 0) {lang_hint_boost[top3] += cld::LgProb3(prob123_entry, 2);} cld 1817 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc const uint8* prob123_entry = cld::LgProb2TblEntry(prob123); cld 1819 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top1 > 0) {lang_hint_boost[top1] += cld::LgProb3(prob123_entry, 0);} cld 1821 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top2 > 0) {lang_hint_boost[top2] += cld::LgProb3(prob123_entry, 1);} cld 1823 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top3 > 0) {lang_hint_boost[top3] += cld::LgProb3(prob123_entry, 2);} cld 1838 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc const uint8* prob123_entry = cld::LgProb2TblEntry(prob123); cld 1840 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top1 > 0) {lang_hint_boost[top1] += cld::LgProb3(prob123_entry, 0);} cld 1842 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top2 > 0) {lang_hint_boost[top2] += cld::LgProb3(prob123_entry, 1);} cld 1844 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (top3 > 0) {lang_hint_boost[top3] += cld::LgProb3(prob123_entry, 2);} cld 1875 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc language3[0] = cld::UnpackLanguage(lang1); cld 1879 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc normalized_score3[0] = cld::GetNormalizedScore(language3[0], cld 1887 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc language3[1] = cld::UnpackLanguage(lang2); cld 1891 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc normalized_score3[1] = cld::GetNormalizedScore(language3[1], cld 1899 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc language3[2] = cld::UnpackLanguage(lang3); cld 1903 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc normalized_score3[2] = cld::GetNormalizedScore(language3[2], cld 1918 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int total_text_bytes_div = cld::maxint(1, total_text_bytes); // Avoid zdiv cld 1944 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc *is_reliable = reliable_percent >= cld::kMinReliable; cld 2027 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc int plang_second = cld::PackLanguage(language3[active_slot[1]]); cld 2028 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc bool is_tier3 = (cld::kIsPackedTop40[plang_second] == 0); cld 2184 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc chunksizequads = cld::minint(cld::maxint(chunksizequads, kMinChunkSizeQuads), cld 2271 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc Language onlylang = cld::kOnlyLanguagePerLScript[lscript]; cld 2275 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc ScoreNilgrams(&scriptspan, cld::PackLanguage(onlylang), &doc_tote, cld 2277 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc } else if (cld::kScoreUniPerLScript[lscript] != 0) { cld 2456 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc fprintf(stderr, "%s ", ExtLanguageCode(cld::UnpackLanguage(subseq[i]))); cld 2509 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc if (cld::kIsPackedTop40[cld::PackLanguage(language3[0])] == 0) { cld 2511 third_party/cld/encodings/compact_lang_det/compact_lang_det_impl.cc } else if (cld::kIsPackedTop40[cld::PackLanguage(language3[1])] == 0) { cld 26 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc static const cld::IndirectProbBucket4 kCjkBiTable[kCjkBiTableSize] = { cld 39 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc extern const cld::CLDTableSummary kCjkBiTable_obj = { cld 114 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc static const cld::IndirectProbBucket4 kDeltaOctaChrome[kDeltaOctaChromeSize] = { cld 1254 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc extern const cld::CLDTableSummary kLongWord8Table_obj = { cld 31 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc static const cld::IndirectProbBucket4 kLongWord8Table[kLongWord8TableSize] = { cld 44 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc extern const cld::CLDTableSummary kLongWord8Table_obj = { cld 30 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc static const cld::IndirectProbBucket4 kQuadTable[kQuadTableSize] = { cld 41 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc extern const cld::CLDTableSummary kQuadTable_obj = { cld 114 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc static const cld::IndirectProbBucket4 kQuadChrome[kQuadChromeSize] = { cld 70926 third_party/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc extern const cld::CLDTableSummary kQuadTable_obj = { cld 199 third_party/cld/encodings/compact_lang_det/getonescriptspan.cc cld::UTF8GenericScan(&utf8scannotjustletterspecial_obj, src, len, cld 10 third_party/cld/encodings/compact_lang_det/win/cld_utf8utils.h namespace cld {