root/third_party/cld/encodings/compact_lang_det/cldutil.h

INCLUDED FROM

// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_
#define ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_

#include <string>
#include "encodings/compact_lang_det/ext_lang_enc.h"
#include "encodings/compact_lang_det/tote.h"
#include "encodings/compact_lang_det/win/cld_basictypes.h"
#include "encodings/compact_lang_det/win/cld_commandlineflags.h"
#include "encodings/compact_lang_det/win/cld_utf8statetable.h"

namespace cld {

  // Hash bucket for four-way associative lookup with < 64K buckets
  // 32 bytes per bucket, 8-byte entries
  typedef struct {
    uint32 key[4];        // hashed word to look up
    uint32 value[4];      // packed three lang numbers and probability subscript
  } SmallWordProbBucket4;

  // Hash bucket for fouro-way associative lookup with >= 64K buckets
  // 24 bytes per bucket, 6-byte entries
  typedef struct {
    uint16 key[4];        // Half of hashed word to look up; other
                          //  half is used to pick the bucket
    uint32 value[4];      // packed three lang numbers and probability subscript
  } LargeQuadProbBucket4;

  // Hash bucket for four-way associative lookup, indirect probabilities
  // 16 bytes per bucket, 4-byte entries
  typedef struct {
    uint32 keyvalue[4];   // Upper part of word is hash, lower is indirect prob
  } IndirectProbBucket4;


  // This describes a complete CLD table, consisting of
  // a main lookup table, an indirect language/probability table, and
  // three constants.
  // The main table key is a quadgram, bigram, or longword hash, with
  // part of the key used to select a bucket modulo kCLDTableSize,
  // and the rest matched against the key portion of four entries in a bucket,
  // defined by kCLDTableKeyMask. The remaining bits of an entry, defined
  // by ~kCLDTableKeyMask, are usually a subscript in the indirect table.
  //
  // By using part of the key to select a bucket, those key bits do not need
  // to be stored in the main table entries, saving space (typically 2 bytes).
  //
  // By using an indirect table for lang/prob triples, only the subscript needs
  // to be stored in the main table entires, saving space (typically 2 bytes).
  //
  // Each entry in the indirect table has three languages and three
  // corresponding probabilities, packed into four bytes.
  //
  // The build date constant is included just for version tracking and is not
  // otherwise used.
  //
  // Different-size tables can be linked in for different production
  // environments. By going indirect through this struct, the runtime code is
  // insensitive to the actual sizes.
  //
  // An empty placeholder table can be described by a table size of 1
  // bucket, a keymask of 0xffffffff, a degenerate bucket of four no-match
  // entries, and a degenerate indirect table of one no-languages entry.
  //
  //
  struct CLDTableSummary {
    const IndirectProbBucket4* kCLDTable;
                                        // Each bucket has four entries, part
                                        //  key and part indirect subscript
    const uint32* kCLDTableInd;         // Each entry is three packed lang/prob
    const int kCLDTableSize;            // Bucket count
    const int kCLDTableIndSize;         // Entries count
    const int kCLDTableKeyMask;         // Mask hash key
    const int kCLDTableBuildDate;       // yyyymmdd
  };


  // Keeps per-character 0-12 language probabilities for CTJKVZ-- in that order.
  // Chinese ChineseT Japanese Korean Vietnamese Zhuang
  // (2 bytes unused, for alignment padding and future)
  typedef struct {
    uint8 probs[8];
  } UnigramProbArray;

  // Map 8-bit subscript to CTJKVZ probabilities
  // Target runtime probabilities for CTJK + VZ
  // Hand-generated to cover a reasonable range of choices
  static const int kTargetCTJKVZProbsSize = 242;
  static const UnigramProbArray kTargetCTJKVZProbs[kTargetCTJKVZProbsSize] = {
    {{0,0,0,0,0,0,0,0}},
    {{0,0,0,0,0,12,0,0}},
    {{0,0,0,0,12,0,0,0}},
    {{0,0,0,12,0,0,0,0}},
    {{0,0,12,0,0,0,0,0}},
    {{0,12,0,0,0,0,0,0}},
    {{12,0,0,0,0,0,0,0}},

    {{8,0,0,0,4,0,0,0}},
    {{8,0,0,4,0,0,0,0}},
    {{8,0,4,0,0,0,0,0}},
    {{8,4,0,0,0,0,0,0}},
    {{8,2,0,2,0,0,0,0}},
    {{0,0,0,0,0,8,0,0}},
    {{0,4,8,0,0,0,0,0}},
    {{4,0,0,0,0,8,0,0}},
    {{0,0,8,0,0,0,0,0}},
    {{8,2,2,0,0,0,0,0}},
    {{0,8,4,0,0,0,0,0}},
    {{8,0,0,0,0,4,0,0}},
    {{0,8,2,0,0,0,0,0}},
    {{4,8,0,0,0,0,0,0}},
    {{2,8,0,2,0,0,0,0}},
    {{2,2,8,0,0,0,0,0}},
    {{0,8,0,0,0,0,0,0}},
    {{0,2,8,0,0,0,0,0}},
    {{2,8,2,0,0,0,0,0}},
    {{8,0,0,0,0,0,0,0}},
    {{2,8,0,0,0,0,0,0}},
    {{8,2,0,0,0,0,0,0}},

    {{0,6,2,0,2,0,0,0}},
    {{2,0,0,0,6,0,0,0}},
    {{4,0,0,0,6,0,0,0}},
    {{4,6,0,0,4,0,0,0}},
    {{4,6,2,0,2,0,0,0}},
    {{4,6,4,0,2,0,0,0}},
    {{5,4,6,0,0,0,0,0}},
    {{6,0,0,0,4,0,0,0}},
    {{6,0,2,0,4,0,0,0}},
    {{6,0,4,0,4,0,0,0}},
    {{6,2,0,0,4,0,0,0}},
    {{6,2,2,0,4,0,0,0}},
    {{6,2,4,0,2,0,0,0}},
    {{6,4,0,0,2,0,0,0}},
    {{6,4,2,0,2,0,0,0}},
    {{0,0,6,2,0,0,0,0}},
    {{0,6,2,0,0,2,0,0}},
    {{2,2,2,0,0,6,0,0}},
    {{2,2,6,4,0,0,0,0}},
    {{2,4,0,0,0,6,0,0}},
    {{2,6,0,4,0,0,0,0}},
    {{2,6,2,4,0,0,0,0}},
    {{2,6,4,4,0,0,0,0}},
    {{4,0,2,0,0,6,0,0}},
    {{4,2,6,2,0,0,0,0}},
    {{4,4,2,0,0,6,0,0}},
    {{4,6,4,0,0,2,0,0}},
    {{6,0,2,0,0,2,0,0}},
    {{6,2,0,0,0,2,0,0}},
    {{6,2,2,0,0,4,0,0}},
    {{6,2,4,0,0,2,0,0}},
    {{4,6,2,0,0,4,0,0}},
    {{6,4,2,0,0,4,0,0}},
    {{2,0,0,0,0,6,0,0}},
    {{6,2,0,2,0,0,0,0}},
    {{2,2,0,0,0,6,0,0}},
    {{6,2,6,0,0,0,0,0}},
    {{6,4,2,0,0,2,0,0}},
    {{6,4,2,2,0,0,0,0}},
    {{4,6,4,2,0,0,0,0}},
    {{6,0,2,0,0,4,0,0}},
    {{6,0,4,0,0,2,0,0}},
    {{6,0,6,0,0,0,0,0}},
    {{6,2,2,0,0,0,0,0}},
    {{6,4,0,0,0,2,0,0}},
    {{6,4,5,0,0,0,0,0}},
    {{0,6,0,2,0,0,0,0}},
    {{0,6,2,2,0,0,0,0}},
    {{2,6,0,2,0,0,0,0}},
    {{2,6,2,2,0,0,0,0}},
    {{4,2,0,0,0,6,0,0}},
    {{6,4,0,0,0,4,0,0}},
    {{6,4,0,2,0,0,0,0}},
    {{6,6,0,2,0,0,0,0}},
    {{6,0,4,0,0,4,0,0}},
    {{6,2,0,0,0,4,0,0}},
    {{6,6,2,2,0,0,0,0}},
    {{4,6,0,0,0,2,0,0}},
    {{2,6,6,0,0,0,0,0}},
    {{4,5,6,0,0,0,0,0}},
    {{4,6,0,2,0,0,0,0}},
    {{6,2,0,0,0,6,0,0}},
    {{0,6,4,2,0,0,0,0}},
    {{4,0,6,0,0,0,0,0}},
    {{2,6,4,2,0,0,0,0}},
    {{4,6,0,0,0,4,0,0}},
    {{6,2,2,0,0,0,0,0}},
    {{4,6,2,2,0,0,0,0}},
    {{4,6,5,0,0,0,0,0}},
    {{6,0,2,0,0,0,0,0}},
    {{6,4,4,0,0,0,0,0}},
    {{4,2,6,0,0,0,0,0}},
    {{2,0,6,0,0,0,0,0}},
    {{4,4,0,0,0,6,0,0}},
    {{4,4,6,0,0,0,0,0}},
    {{4,6,2,0,0,2,0,0}},
    {{2,2,6,0,0,0,0,0}},
    {{2,4,6,0,0,0,0,0}},
    {{0,6,6,0,0,0,0,0}},
    {{6,2,4,0,0,0,0,0}},
    {{0,4,6,0,0,0,0,0}},
    {{4,0,0,0,0,6,0,0}},
    {{4,6,4,0,0,0,0,0}},
    {{6,0,0,0,0,6,0,0}},
    {{6,0,0,0,0,2,0,0}},
    {{6,0,4,0,0,0,0,0}},
    {{6,5,4,0,0,0,0,0}},
    {{0,2,6,0,0,0,0,0}},
    {{0,0,6,0,0,0,0,0}},
    {{6,6,2,0,0,0,0,0}},
    {{2,6,4,0,0,0,0,0}},
    {{6,4,2,0,0,0,0,0}},
    {{2,6,2,0,0,0,0,0}},
    {{2,6,0,0,0,0,0,0}},
    {{6,0,0,0,0,4,0,0}},
    {{6,4,0,0,0,0,0,0}},
    {{6,6,0,0,0,0,0,0}},
    {{5,6,4,0,0,0,0,0}},
    {{0,6,0,0,0,0,0,0}},
    {{6,2,0,0,0,0,0,0}},
    {{0,6,2,0,0,0,0,0}},
    {{4,6,2,0,0,0,0,0}},
    {{0,6,4,0,0,0,0,0}},
    {{4,6,0,0,0,0,0,0}},
    {{6,0,0,0,0,0,0,0}},
    {{6,6,5,0,0,0,0,0}},
    {{6,5,6,0,0,0,0,0}},
    {{5,6,6,0,0,0,0,0}},
    {{5,5,6,0,0,0,0,0}},
    {{5,6,5,0,0,0,0,0}},
    {{6,5,5,0,0,0,0,0}},
    {{6,6,6,0,0,0,0,0}},
    {{6,5,0,0,0,0,0,0}},
    {{6,0,5,0,0,0,0,0}},
    {{0,6,5,0,0,0,0,0}},
    {{5,6,0,0,0,0,0,0}},
    {{5,0,6,0,0,0,0,0}},
    {{0,5,6,0,0,0,0,0}},

    {{0,0,0,0,4,0,0,0}},
    {{0,0,0,4,0,0,0,0}},
    {{2,2,0,0,4,0,0,0}},
    {{2,2,2,0,4,0,0,0}},
    {{2,4,0,0,2,0,0,0}},
    {{2,4,2,0,2,0,0,0}},
    {{2,4,4,0,2,0,0,0}},
    {{4,0,2,0,4,0,0,0}},
    {{4,0,4,0,2,0,0,0}},
    {{4,2,0,0,2,0,0,0}},
    {{4,2,2,0,2,0,0,0}},
    {{4,4,0,0,2,0,0,0}},
    {{4,4,2,0,2,0,0,0}},
    {{4,4,4,0,2,0,0,0}},
    {{0,2,2,4,0,0,0,0}},
    {{2,2,4,2,0,0,0,0}},
    {{2,4,4,0,0,2,0,0}},
    {{2,4,4,2,0,0,0,0}},
    {{4,0,4,0,0,2,0,0}},
    {{4,0,4,0,0,4,0,0}},
    {{4,2,2,4,0,0,0,0}},
    {{4,4,0,2,0,0,0,0}},
    {{2,2,0,4,0,0,0,0}},
    {{2,4,2,2,0,0,0,0}},
    {{4,4,2,2,0,0,0,0}},
    {{4,0,4,0,0,0,0,0}},
    {{4,4,4,0,0,4,0,0}},
    {{0,4,0,2,0,0,0,0}},
    {{0,4,2,2,0,0,0,0}},
    {{4,0,2,0,0,2,0,0}},
    {{4,2,0,0,0,4,0,0}},
    {{2,2,2,0,0,4,0,0}},
    {{4,0,0,2,0,0,0,0}},
    {{4,4,4,0,0,2,0,0}},
    {{4,0,0,0,0,4,0,0}},
    {{4,0,2,0,0,4,0,0}},
    {{4,2,0,0,0,2,0,0}},
    {{4,2,2,0,0,2,0,0}},
    {{2,4,0,2,0,0,0,0}},
    {{2,2,0,0,0,4,0,0}},
    {{2,4,0,0,0,4,0,0}},
    {{2,4,2,0,0,4,0,0}},
    {{4,2,4,0,0,0,0,0}},
    {{2,0,4,0,0,0,0,0}},
    {{4,0,2,0,0,0,0,0}},
    {{4,4,0,0,0,4,0,0}},
    {{4,4,2,0,0,4,0,0}},
    {{0,4,4,0,0,0,0,0}},
    {{4,4,0,0,0,2,0,0}},
    {{2,4,0,0,0,2,0,0}},
    {{2,2,4,0,0,0,0,0}},
    {{0,2,4,0,0,0,0,0}},
    {{4,2,2,0,0,0,0,0}},
    {{2,4,2,0,0,2,0,0}},
    {{4,4,4,0,0,0,0,0}},
    {{2,4,4,0,0,0,0,0}},
    {{0,0,4,0,0,0,0,0}},
    {{0,4,2,0,0,0,0,0}},
    {{4,4,2,0,0,2,0,0}},
    {{2,4,2,0,0,0,0,0}},
    {{4,2,0,0,0,0,0,0}},
    {{4,4,0,0,0,0,0,0}},
    {{4,4,2,0,0,0,0,0}},
    {{2,4,0,0,0,0,0,0}},
    {{0,4,0,0,0,0,0,0}},
    {{4,0,0,0,0,0,0,0}},
    {{0,0,0,4,4,0,0,0}},
    {{0,0,4,0,4,0,0,0}},
    {{0,0,4,4,0,0,0,0}},
    {{0,4,0,0,4,0,0,0}},
    {{0,4,0,4,0,0,0,0}},
    {{4,0,0,0,4,0,0,0}},
    {{4,0,0,4,0,0,0,0}},

    {{2,0,0,0,0,0,0,0}},
    {{0,2,0,0,0,0,0,0}},
    {{0,2,0,2,2,0,0,0}},
    {{0,2,2,0,2,0,0,0}},
    {{2,0,0,2,2,0,0,0}},
    {{2,0,2,0,2,0,0,0}},
    {{2,0,2,2,0,0,0,0}},
    {{2,2,0,0,2,0,0,0}},
    {{2,2,2,2,0,0,0,0}},
    {{2,2,0,2,0,0,0,0}},
    {{2,2,0,0,0,0,0,0}},
    {{0,0,2,0,0,0,0,0}},
    {{0,2,2,0,0,0,0,0}},
    {{2,2,2,0,0,0,0,0}},
    {{0,0,0,2,0,0,0,0}},
    {{2,0,2,0,0,0,0,0}},
    {{0,2,0,2,0,0,0,0}},
    {{0,0,2,2,0,0,0,0}},
    {{0,2,2,2,0,0,0,0}},
  };




  // 1 to skip ASCII space, vowels AEIOU aeiou and UTF-8 continuation bytes 80-BF
  static const uint8 kSkipSpaceVowelContinue[256] = {
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,
    0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,

    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  };

  // 1 to skip ASCII space, and UTF-8 continuation bytes 80-BF
  static const uint8 kSkipSpaceContinue[256] = {
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,

    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  };


  // If != UNKNOWN, use nilgrams to determine language of this script
  static const Language kOnlyLanguagePerLScript[] = {
    ENGLISH,            // ULScript_Common, [no words should be in this script]
    UNKNOWN_LANGUAGE,   // ULScript_Latin,
    //UNKNOWN_LANGUAGE,   // ULScript_Greek,  Jan 2009: change so we can score quads
    GREEK,              // ULScript_Greek,  Mar 2009: change back; do gibberish separately
    UNKNOWN_LANGUAGE,   // ULScript_Cyrillic,
    ARMENIAN,           // ULScript_Armenian,
    UNKNOWN_LANGUAGE,   // ULScript_Hebrew,
    UNKNOWN_LANGUAGE,   // ULScript_Arabic,
    SYRIAC,             // ULScript_Syriac,
    DHIVEHI,            // ULScript_Thaana,
    UNKNOWN_LANGUAGE,   // ULScript_Devanagari,
    UNKNOWN_LANGUAGE,   // ULScript_Bengali,
    PUNJABI,            // ULScript_Gurmukhi,
    GUJARATI,           // ULScript_Gujarati,
    ORIYA,              // ULScript_Oriya,
    TAMIL,              // ULScript_Tamil,
    TELUGU,             // ULScript_Telugu,
    KANNADA,            // ULScript_Kannada,
    MALAYALAM,          // ULScript_Malayalam,
    SINHALESE,          // ULScript_Sinhala,
    THAI,               // ULScript_Thai,
    LAOTHIAN,           // ULScript_Lao,
    UNKNOWN_LANGUAGE,   // ULScript_Tibetan,
    BURMESE,            // ULScript_Myanmar,
    GEORGIAN,           // ULScript_Georgian,
    UNKNOWN_LANGUAGE,   // ULScript_HanCJK,
    UNKNOWN_LANGUAGE,   // ULScript_Ethiopic,
    CHEROKEE,           // ULScript_Cherokee,
    INUKTITUT,          // ULScript_Canadian_Aboriginal,
    X_OGHAM,            // ULScript_Ogham,
    X_RUNIC,            // ULScript_Runic,
    KHMER,              // ULScript_Khmer,
    MONGOLIAN,          // ULScript_Mongolian,
    X_YI,               // ULScript_Yi,
    X_OLD_ITALIC,       // ULScript_Old_Italic,
    X_GOTHIC,           // ULScript_Gothic,
    X_DESERET,          // ULScript_Deseret,
    ENGLISH,            // ULScript_Inherited, [no words should be in this script]
    TAGALOG,            // ULScript_Tagalog,
    X_HANUNOO,          // ULScript_Hanunoo,
    X_BUHID,            // ULScript_Buhid,
    X_TAGBANWA,         // ULScript_Tagbanwa,
    LIMBU,              // ULScript_Limbu,
    X_TAI_LE,           // ULScript_Tai_Le,
    X_LINEAR_B,         // ULScript_Linear_B,
    X_UGARITIC,         // ULScript_Ugaritic,
    X_SHAVIAN,          // ULScript_Shavian,
    X_OSMANYA,          // ULScript_Osmanya,
    X_CYPRIOT,          // ULScript_Cypriot,
    X_BUGINESE,         // ULScript_Buginese,
    X_COPTIC,           // ULScript_Coptic,
    X_NEW_TAI_LUE,      // ULScript_New_Tai_Lue,
    X_GLAGOLITIC,       // ULScript_Glagolitic,
    X_TIFINAGH,         // ULScript_Tifinagh,
    X_SYLOTI_NAGRI,     // ULScript_Syloti_Nagri,
    X_OLD_PERSIAN,      // ULScript_Old_Persian,
    X_KHAROSHTHI,       // ULScript_Kharoshthi,
    X_BALINESE,         // ULScript_Balinese,
    X_CUNEIFORM,        // ULScript_Cuneiform,
    X_PHOENICIAN,       // ULScript_Phoenician,
    X_PHAGS_PA,         // ULScript_Phags_Pa,
    X_NKO,              // ULScript_Nko,

    // Unicode 5.1
    X_SUDANESE,         // ULScript_Sundanese,
    X_LEPCHA,           // ULScript_Lepcha,
    X_OL_CHIKI,         // ULScript_Ol_Chiki,
    X_VAI,              // ULScript_Vai,
    X_SAURASHTRA,       // ULScript_Saurashtra,
    X_KAYAH_LI,         // ULScript_Kayah_Li,
    X_REJANG,           // ULScript_Rejang,
    X_LYCIAN,           // ULScript_Lycian,
    X_CARIAN,           // ULScript_Carian,
    X_LYDIAN,           // ULScript_Lydian,
    X_CHAM,             // ULScript_Cham,
  };

  COMPILE_ASSERT(arraysize(kOnlyLanguagePerLScript) == ULScript_NUM_SCRIPTS,
       kOnlyLanguagePerLScript_has_incorrect_length);


  // This is, in a sense, the complement of the table above
  // If != UNKNOWN, determines a default language of this script
  static const Language kDefaultLanguagePerLScript[] = {
    UNKNOWN_LANGUAGE,            // ULScript_Common, [no words should be in this script]
    ENGLISH,   // ULScript_Latin,
    UNKNOWN_LANGUAGE,              // ULScript_Greek,
    RUSSIAN,   // ULScript_Cyrillic,
    UNKNOWN_LANGUAGE,           // ULScript_Armenian,
    HEBREW,   // ULScript_Hebrew,
    ARABIC,   // ULScript_Arabic,
    UNKNOWN_LANGUAGE,             // ULScript_Syriac,
    UNKNOWN_LANGUAGE,            // ULScript_Thaana,
    HINDI,   // ULScript_Devanagari,
    BENGALI,   // ULScript_Bengali,
    UNKNOWN_LANGUAGE,            // ULScript_Gurmukhi,
    UNKNOWN_LANGUAGE,           // ULScript_Gujarati,
    UNKNOWN_LANGUAGE,              // ULScript_Oriya,
    UNKNOWN_LANGUAGE,              // ULScript_Tamil,
    UNKNOWN_LANGUAGE,             // ULScript_Telugu,
    UNKNOWN_LANGUAGE,            // ULScript_Kannada,
    UNKNOWN_LANGUAGE,          // ULScript_Malayalam,
    UNKNOWN_LANGUAGE,          // ULScript_Sinhala,
    UNKNOWN_LANGUAGE,               // ULScript_Thai,
    UNKNOWN_LANGUAGE,           // ULScript_Lao,
    TIBETAN,   // ULScript_Tibetan,
    UNKNOWN_LANGUAGE,            // ULScript_Myanmar,
    UNKNOWN_LANGUAGE,           // ULScript_Georgian,
    CHINESE,   // ULScript_HanCJK,
    AMHARIC,   // ULScript_Ethiopic,
    UNKNOWN_LANGUAGE,           // ULScript_Cherokee,
    UNKNOWN_LANGUAGE,          // ULScript_Canadian_Aboriginal,
    UNKNOWN_LANGUAGE,            // ULScript_Ogham,
    UNKNOWN_LANGUAGE,            // ULScript_Runic,
    UNKNOWN_LANGUAGE,              // ULScript_Khmer,
    UNKNOWN_LANGUAGE,          // ULScript_Mongolian,
    UNKNOWN_LANGUAGE,               // ULScript_Yi,
    UNKNOWN_LANGUAGE,       // ULScript_Old_Italic,
    UNKNOWN_LANGUAGE,           // ULScript_Gothic,
    UNKNOWN_LANGUAGE,          // ULScript_Deseret,
    UNKNOWN_LANGUAGE,            // ULScript_Inherited, [no words should be in this script]
    UNKNOWN_LANGUAGE,            // ULScript_Tagalog,
    UNKNOWN_LANGUAGE,          // ULScript_Hanunoo,
    UNKNOWN_LANGUAGE,            // ULScript_Buhid,
    UNKNOWN_LANGUAGE,         // ULScript_Tagbanwa,
    UNKNOWN_LANGUAGE,              // ULScript_Limbu,
    UNKNOWN_LANGUAGE,           // ULScript_Tai_Le,
    UNKNOWN_LANGUAGE,         // ULScript_Linear_B,
    UNKNOWN_LANGUAGE,         // ULScript_Ugaritic,
    UNKNOWN_LANGUAGE,          // ULScript_Shavian,
    UNKNOWN_LANGUAGE,          // ULScript_Osmanya,
    UNKNOWN_LANGUAGE,          // ULScript_Cypriot,
    UNKNOWN_LANGUAGE,         // ULScript_Buginese,
    UNKNOWN_LANGUAGE,           // ULScript_Coptic,
    UNKNOWN_LANGUAGE,      // ULScript_New_Tai_Lue,
    UNKNOWN_LANGUAGE,       // ULScript_Glagolitic,
    UNKNOWN_LANGUAGE,         // ULScript_Tifinagh,
    UNKNOWN_LANGUAGE,     // ULScript_Syloti_Nagri,
    UNKNOWN_LANGUAGE,      // ULScript_Old_Persian,
    UNKNOWN_LANGUAGE,       // ULScript_Kharoshthi,
    UNKNOWN_LANGUAGE,         // ULScript_Balinese,
    UNKNOWN_LANGUAGE,        // ULScript_Cuneiform,
    UNKNOWN_LANGUAGE,       // ULScript_Phoenician,
    UNKNOWN_LANGUAGE,         // ULScript_Phags_Pa,
    UNKNOWN_LANGUAGE,              // ULScript_Nko,

    // Unicode 5.1
    UNKNOWN_LANGUAGE,         // ULScript_Sundanese,
    UNKNOWN_LANGUAGE,           // ULScript_Lepcha,
    UNKNOWN_LANGUAGE,         // ULScript_Ol_Chiki,
    UNKNOWN_LANGUAGE,              // ULScript_Vai,
    UNKNOWN_LANGUAGE,       // ULScript_Saurashtra,
    UNKNOWN_LANGUAGE,         // ULScript_Kayah_Li,
    UNKNOWN_LANGUAGE,           // ULScript_Rejang,
    UNKNOWN_LANGUAGE,           // ULScript_Lycian,
    UNKNOWN_LANGUAGE,           // ULScript_Carian,
    UNKNOWN_LANGUAGE,           // ULScript_Lydian,
    UNKNOWN_LANGUAGE,             // ULScript_Cham,
  };

  COMPILE_ASSERT(arraysize(kDefaultLanguagePerLScript) == ULScript_NUM_SCRIPTS,
       kDefaultLanguagePerLScript_has_incorrect_length);


  // True for standalone languages (only lang in a script)
  // Subscripted by packed language number
  // If 1, we will use nilgrams to determine language
  static const uint8 kIsStandaloneLang[EXT_NUM_LANGUAGES + 1] = {
     0,
     0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,1,0,    // GREEK
     0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
     0,1,0,0,1, 0,1,0,0,0, 0,0,1,1,0, 0,0,0,0,1,    // MALAYALAM..KANNADA
     1,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 1,0,0,0,1,    // PUNJABI..SINHALESE
     0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,1,1,0,    // ARMENIAN..LAOTHIAN

     0,0,0,0,1, 0,1,1,1,0, 1,0,0,0,0, 0,0,0,0,0,    // KHMER..ORIYA
     0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
     0,1,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    // INUKTITUT

     0,0,0,0,0,                                     // [160..164]
    // Add new language standalone bit just before here
     0,0,0,0,0, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1,
     1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1,

     1,1,1,1,
   };

   // True for ULScript_HanCJK
   // (Vietnamese and Zhuang also have Latin script quadgrams)
   // Subscripted by packed language number
   static const uint8 kIsUnigramLang[EXT_NUM_LANGUAGES + 1] = {
      0,
      0,0,0,0,0, 0,0,0,1,1, 0,0,0,0,0, 0,1,0,0,0,    // JAPANESE KOREAN CHINESE
      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //
      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //
      0,0,0,0,0, 0,1,0,0,1, 0,0,0,0,0, 0,0,0,0,0,    // VIETNAMESE CHINESE_T
      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //

      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //
      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //
      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 1,0,0,0,0,    // ZHUANG

      0,0,0,0,0,                                     // [160..164]
     // Add new language unigram bit just before here

      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //
      0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,    //

      0,0,0,0,
   };


  // True for ULScript_HanCJK
  // Subscripted by lscript number
  static const uint8 kScoreUniPerLScript[] = {
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,
  };

  COMPILE_ASSERT(arraysize(kScoreUniPerLScript) == ULScript_NUM_SCRIPTS,
       kScoreUniPerLScript_has_incorrect_length);


  // Defines Top40 packed languages

  // Tier 0/1 Language enum list (16)
  //   ENGLISH, /*no en_GB,*/ FRENCH, ITALIAN, GERMAN, SPANISH,    // E - FIGS
  //   DUTCH, CHINESE, CHINESE_T, JAPANESE, KOREAN,
  //   PORTUGUESE, RUSSIAN, POLISH, TURKISH, THAI,
  //   ARABIC,
  //
  // Tier 2 Language enum list (22)
  //   SWEDISH, FINNISH, DANISH, /*no pt-PT,*/ ROMANIAN, HUNGARIAN,
  //   HEBREW, INDONESIAN, CZECH, GREEK, NORWEGIAN,
  //   VIETNAMESE, BULGARIAN, CROATIAN, LITHUANIAN, SLOVAK,
  //   TAGALOG, SLOVENIAN, SERBIAN, CATALAN, LATVIAN,
  //   UKRAINIAN, HINDI,
  //
  //   use SERBO_CROATIAN instead of BOSNIAN, SERBIAN, CROATIAN, MONTENEGRIN(21)
  //
  // Include IgnoreMe (TG_UNKNOWN_LANGUAGE, 25+1) as a top 40

  // NOTE: packed, i.e. Language enum + 1
  static const uint8 kIsPackedTop40[EXT_NUM_LANGUAGES + 1] = {
    0,
    1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,1, 1,1,1,1,0,
    1,1,1,1,0, 1,0,1,0,0, 0,0,1,1,1, 1,0,0,1,0,
    0,0,0,0,0, 0,0,0,0,0, 0,0,0,1,1, 1,0,0,0,0,
    0,0,0,1,0, 0,1,0,1,1, 0,0,0,0,0, 0,0,0,0,0,
    0,0,0,0,0, 0,0,0,0,0, 0,0,1,0,0, 0,0,0,0,0,

    0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
    0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
    0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,

    0,0,0,0,0,                                    // [160..164]
    // Add new language top40 bit just before here

    0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,
    0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0,

    0,0,0,0,
  };



  // Table has 234 eight-byte entries. Each entry has a five-byte array and
  // a three-byte array of log base 2 probabilities in the range 0..11.
  // The intended use is to express five or three probabilities in a single-byte
  // subscript, then decode via this table. These probabilities are
  // intended to go with an array of five or three language numbers.
  //
  // The corresponding language numbers will have to be sorted by descending
  // probability, then the actual probability subscript chosen to match the
  // closest available entry in this table.
  //
  // Pattern of probability values:
  // hi 3/4 1/2 1/4 lo    hi mid lo
  // where "3/4" is (hi*3+lo)/4, "1/2" is (hi+lo)/2, and "1/4" is (hi+lo*3)/4 and
  // mid is one of 3/4 1/2 or 1/4.
  // There are three groups of 78 (=12*13/2) entries, with hi running 0..11 and
  // lo running 0..hi. Only the first group is used for five-entry lookups.
  // The mid value in the first group is 1/2, the second group 3/4, and the
  // third group 1/4. For three-entry lookups, this allows the mid entry to be
  // somewhat higher or lower than the midpoint, to allow a better match to the
  // original probabilities.
  static const int kLgProbV2TblSize = 234;
  static const uint8 kLgProbV2Tbl[kLgProbV2TblSize * 8] = {
    1,1,1,1,1, 1,1,1,     // [0]
    2,2,2,1,1, 2,2,1,     // [1]
    2,2,2,2,2, 2,2,2,
    3,3,2,2,1, 3,2,1,     // [3]
    3,3,3,2,2, 3,3,2,
    3,3,3,3,3, 3,3,3,
    4,3,3,2,1, 4,3,1,     // [6]
    4,4,3,3,2, 4,3,2,
    4,4,4,3,3, 4,4,3,
    4,4,4,4,4, 4,4,4,
    5,4,3,2,1, 5,3,1,     // [10]
    5,4,4,3,2, 5,4,2,
    5,5,4,4,3, 5,4,3,
    5,5,5,4,4, 5,5,4,
    5,5,5,5,5, 5,5,5,
    6,5,4,2,1, 6,4,1,     // [15]
    6,5,4,3,2, 6,4,2,
    6,5,5,4,3, 6,5,3,
    6,6,5,5,4, 6,5,4,
    6,6,6,5,5, 6,6,5,
    6,6,6,6,6, 6,6,6,
    7,6,4,3,1, 7,4,1,     // [21]
    7,6,5,3,2, 7,5,2,
    7,6,5,4,3, 7,5,3,
    7,6,6,5,4, 7,6,4,
    7,7,6,6,5, 7,6,5,
    7,7,7,6,6, 7,7,6,
    7,7,7,7,7, 7,7,7,
    8,6,5,3,1, 8,5,1,     // [28]
    8,7,5,4,2, 8,5,2,
    8,7,6,4,3, 8,6,3,
    8,7,6,5,4, 8,6,4,
    8,7,7,6,5, 8,7,5,
    8,8,7,7,6, 8,7,6,
    8,8,8,7,7, 8,8,7,
    8,8,8,8,8, 8,8,8,
    9,7,5,3,1, 9,5,1,     // [36]
    9,7,6,4,2, 9,6,2,
    9,8,6,5,3, 9,6,3,
    9,8,7,5,4, 9,7,4,
    9,8,7,6,5, 9,7,5,
    9,8,8,7,6, 9,8,6,
    9,9,8,8,7, 9,8,7,
    9,9,9,8,8, 9,9,8,
    9,9,9,9,9, 9,9,9,
    10,8,6,3,1, 10,6,1,   // [45]
    10,8,6,4,2, 10,6,2,
    10,8,7,5,3, 10,7,3,
    10,9,7,6,4, 10,7,4,
    10,9,8,6,5, 10,8,5,
    10,9,8,7,6, 10,8,6,
    10,9,9,8,7, 10,9,7,
    10,10,9,9,8, 10,9,8,
    10,10,10,9,9, 10,10,9,
    10,10,10,10,10, 10,10,10,
    11,9,6,4,1, 11,6,1,   // [55]
    11,9,7,4,2, 11,7,2,
    11,9,7,5,3, 11,7,3,
    11,9,8,6,4, 11,8,4,
    11,10,8,7,5, 11,8,5,
    11,10,9,7,6, 11,9,6,
    11,10,9,8,7, 11,9,7,
    11,10,10,9,8, 11,10,8,
    11,11,10,10,9, 11,10,9,
    11,11,11,10,10, 11,11,10,
    11,11,11,11,11, 11,11,11,
    12,9,7,4,1, 12,7,1,   // [66]
    12,10,7,5,2, 12,7,2,
    12,10,8,5,3, 12,8,3,
    12,10,8,6,4, 12,8,4,
    12,10,9,7,5, 12,9,5,
    12,11,9,8,6, 12,9,6,
    12,11,10,8,7, 12,10,7,
    12,11,10,9,8, 12,10,8,
    12,11,11,10,9, 12,11,9,
    12,12,11,11,10, 12,11,10,
    12,12,12,11,11, 12,12,11,
    12,12,12,12,12, 12,12,12,

    1,1,1,1,1, 1,1,1,
    2,2,2,1,1, 2,2,1,
    2,2,2,2,2, 2,2,2,
    3,3,2,2,1, 3,3,1,
    3,3,3,2,2, 3,3,2,
    3,3,3,3,3, 3,3,3,
    4,3,3,2,1, 4,3,1,
    4,4,3,3,2, 4,4,2,
    4,4,4,3,3, 4,4,3,
    4,4,4,4,4, 4,4,4,
    5,4,3,2,1, 5,4,1,
    5,4,4,3,2, 5,4,2,
    5,5,4,4,3, 5,5,3,
    5,5,5,4,4, 5,5,4,
    5,5,5,5,5, 5,5,5,
    6,5,4,2,1, 6,5,1,
    6,5,4,3,2, 6,5,2,
    6,5,5,4,3, 6,5,3,
    6,6,5,5,4, 6,6,4,
    6,6,6,5,5, 6,6,5,
    6,6,6,6,6, 6,6,6,
    7,6,4,3,1, 7,6,1,
    7,6,5,3,2, 7,6,2,
    7,6,5,4,3, 7,6,3,
    7,6,6,5,4, 7,6,4,
    7,7,6,6,5, 7,7,5,
    7,7,7,6,6, 7,7,6,
    7,7,7,7,7, 7,7,7,
    8,6,5,3,1, 8,6,1,
    8,7,5,4,2, 8,7,2,
    8,7,6,4,3, 8,7,3,
    8,7,6,5,4, 8,7,4,
    8,7,7,6,5, 8,7,5,
    8,8,7,7,6, 8,8,6,
    8,8,8,7,7, 8,8,7,
    8,8,8,8,8, 8,8,8,
    9,7,5,3,1, 9,7,1,
    9,7,6,4,2, 9,7,2,
    9,8,6,5,3, 9,8,3,
    9,8,7,5,4, 9,8,4,
    9,8,7,6,5, 9,8,5,
    9,8,8,7,6, 9,8,6,
    9,9,8,8,7, 9,9,7,
    9,9,9,8,8, 9,9,8,
    9,9,9,9,9, 9,9,9,
    10,8,6,3,1, 10,8,1,
    10,8,6,4,2, 10,8,2,
    10,8,7,5,3, 10,8,3,
    10,9,7,6,4, 10,9,4,
    10,9,8,6,5, 10,9,5,
    10,9,8,7,6, 10,9,6,
    10,9,9,8,7, 10,9,7,
    10,10,9,9,8, 10,10,8,
    10,10,10,9,9, 10,10,9,
    10,10,10,10,10, 10,10,10,
    11,9,6,4,1, 11,9,1,
    11,9,7,4,2, 11,9,2,
    11,9,7,5,3, 11,9,3,
    11,9,8,6,4, 11,9,4,
    11,10,8,7,5, 11,10,5,
    11,10,9,7,6, 11,10,6,
    11,10,9,8,7, 11,10,7,
    11,10,10,9,8, 11,10,8,
    11,11,10,10,9, 11,11,9,
    11,11,11,10,10, 11,11,10,
    11,11,11,11,11, 11,11,11,
    12,9,7,4,1, 12,9,1,
    12,10,7,5,2, 12,10,2,
    12,10,8,5,3, 12,10,3,
    12,10,8,6,4, 12,10,4,
    12,10,9,7,5, 12,10,5,
    12,11,9,8,6, 12,11,6,
    12,11,10,8,7, 12,11,7,
    12,11,10,9,8, 12,11,8,
    12,11,11,10,9, 12,11,9,
    12,12,11,11,10, 12,12,10,
    12,12,12,11,11, 12,12,11,
    12,12,12,12,12, 12,12,12,

    1,1,1,1,1, 1,1,1,
    2,2,2,1,1, 2,1,1,
    2,2,2,2,2, 2,2,2,
    3,3,2,2,1, 3,2,1,
    3,3,3,2,2, 3,2,2,
    3,3,3,3,3, 3,3,3,
    4,3,3,2,1, 4,2,1,
    4,4,3,3,2, 4,3,2,
    4,4,4,3,3, 4,3,3,
    4,4,4,4,4, 4,4,4,
    5,4,3,2,1, 5,2,1,
    5,4,4,3,2, 5,3,2,
    5,5,4,4,3, 5,4,3,
    5,5,5,4,4, 5,4,4,
    5,5,5,5,5, 5,5,5,
    6,5,4,2,1, 6,2,1,
    6,5,4,3,2, 6,3,2,
    6,5,5,4,3, 6,4,3,
    6,6,5,5,4, 6,5,4,
    6,6,6,5,5, 6,5,5,
    6,6,6,6,6, 6,6,6,
    7,6,4,3,1, 7,3,1,
    7,6,5,3,2, 7,3,2,
    7,6,5,4,3, 7,4,3,
    7,6,6,5,4, 7,5,4,
    7,7,6,6,5, 7,6,5,
    7,7,7,6,6, 7,6,6,
    7,7,7,7,7, 7,7,7,
    8,6,5,3,1, 8,3,1,
    8,7,5,4,2, 8,4,2,
    8,7,6,4,3, 8,4,3,
    8,7,6,5,4, 8,5,4,
    8,7,7,6,5, 8,6,5,
    8,8,7,7,6, 8,7,6,
    8,8,8,7,7, 8,7,7,
    8,8,8,8,8, 8,8,8,
    9,7,5,3,1, 9,3,1,
    9,7,6,4,2, 9,4,2,
    9,8,6,5,3, 9,5,3,
    9,8,7,5,4, 9,5,4,
    9,8,7,6,5, 9,6,5,
    9,8,8,7,6, 9,7,6,
    9,9,8,8,7, 9,8,7,
    9,9,9,8,8, 9,8,8,
    9,9,9,9,9, 9,9,9,
    10,8,6,3,1, 10,3,1,
    10,8,6,4,2, 10,4,2,
    10,8,7,5,3, 10,5,3,
    10,9,7,6,4, 10,6,4,
    10,9,8,6,5, 10,6,5,
    10,9,8,7,6, 10,7,6,
    10,9,9,8,7, 10,8,7,
    10,10,9,9,8, 10,9,8,
    10,10,10,9,9, 10,9,9,
    10,10,10,10,10, 10,10,10,
    11,9,6,4,1, 11,4,1,
    11,9,7,4,2, 11,4,2,
    11,9,7,5,3, 11,5,3,
    11,9,8,6,4, 11,6,4,
    11,10,8,7,5, 11,7,5,
    11,10,9,7,6, 11,7,6,
    11,10,9,8,7, 11,8,7,
    11,10,10,9,8, 11,9,8,
    11,11,10,10,9, 11,10,9,
    11,11,11,10,10, 11,10,10,
    11,11,11,11,11, 11,11,11,
    12,9,7,4,1, 12,4,1,
    12,10,7,5,2, 12,5,2,
    12,10,8,5,3, 12,5,3,
    12,10,8,6,4, 12,6,4,
    12,10,9,7,5, 12,7,5,
    12,11,9,8,6, 12,8,6,
    12,11,10,8,7, 12,8,7,
    12,11,10,9,8, 12,9,8,
    12,11,11,10,9, 12,10,9,
    12,12,11,11,10, 12,11,10,
    12,12,12,11,11, 12,11,11,
    12,12,12,12,12, 12,12,12,
  };

  // Backmap a single desired probability into an entry in kLgProbV2Tbl
  static const uint8 kLgProbV2TblBackmap[13] = {
    0,
    0, 1, 3, 6,   10, 15, 21, 28,   36, 45, 55, 66,
  };


  // Always advances one UTF-8 character
  static const uint8 kAdvanceOneChar[256] = {
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,

    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
    3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
  };

  // Does not advance past space or cr/lf/nul
  static const uint8 kAdvanceOneCharButSpace[256] = {
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,

    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
    3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
  };

  // Advances *only* on space or ASCII vowel (or illegal byte)
  static const uint8 kAdvanceOneCharSpaceVowel[256] = {
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,
    0,1,0,0,0,1,0,0, 0,1,0,0,0,0,0,1, 0,0,0,0,0,1,0,0, 0,0,0,0,0,0,0,0,

    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  };

  // Advances *only* on space (or illegal byte)
  static const uint8 kAdvanceOneCharSpace[256] = {
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,

    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  };


//------------------------------------------------------------------------------
// General
//------------------------------------------------------------------------------
  static inline int minint(int a, int b) {return (a < b) ? a: b;}
  static inline int maxint(int a, int b) {return (a > b) ? a: b;}

  // Here to make available for debugging
  int ReliabilityDelta(int value1, int value2, int count);
  int ReliabilityMainstream(int topscore, int len, int mean_score);

  // Returns "0" for too small
  inline const char* MyExtLanguageCode(Language lang) {
    return ExtLanguageCode(lang);
  }

  // Map script into Latin, Cyrillic, Arabic, Other. Used in keeping track of
  // amount of training data for language-script combinations
  inline int LScript4(UnicodeLScript lscript) {
    if (lscript == ULScript_Latin) {return 0;}
    if (lscript == ULScript_Cyrillic) {return 1;}
    if (lscript == ULScript_Arabic) {return 2;}
    return 3;
  }


  // Routines to access 3 or 5 log probabilities in a single byte.

  // Return address of 8-byte entry[i]
  inline const uint8* LgProb2TblEntry(int i) {
    return &kLgProbV2Tbl[i * 8];
  }

  // Return one of five probabilities in an entry
  // CURRENTLY UNUSED
  inline uint8 LgProb5(const uint8* entry, int j) {
    return entry[j];
  }

  // Return one of three probabilities in an entry
  inline uint8 LgProb3(const uint8* entry, int j) {
    return entry[j + 5];
  }



//------------------------------------------------------------------------------
// Hashing groups of 1/2/4/8 letters, perhaps with spaces or underscores
//------------------------------------------------------------------------------

  // Pick up 1..12 bytes and hash them via mask/shift/add. NO pre/post
  // OVERSHOOTS up to 3 bytes
  uint32 BiHashV25(const char* word_ptr, int bytecount);

  // Pick up 1..12 bytes plus pre/post space and hash them via mask/shift/add
  // UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
  uint32 QuadHashV25(const char* word_ptr, int bytecount);

  // Pick up 1..12 bytes plus pre/post '_' and hash them via mask/shift/add
  // OVERSHOOTS up to 3 bytes
  uint32 QuadHashV25Underscore(const char* word_ptr, int bytecount);


  // Pick up 1..24 bytes plus pre/post space and hash them via mask/shift/add
  // UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
  // For runtime use of tables V3
  uint64 OctaHash40(const char* word_ptr, int bytecount);

  uint64 OctaHash40underscore(const char* word_ptr, int bytecount);


  // From 32-bit gram FP, return hash table subscript and remaining key
  inline void QuadFPJustHash(uint32 quadhash,
                                  uint32 keymask,
                                  int bucketcount,
                                  uint32* subscr, uint32* hashkey) {
    *subscr = (quadhash + (quadhash >> 12)) & (bucketcount - 1);
    *hashkey = quadhash & keymask;
  }

  // Look up 32-bit gram FP in caller-passed table
  // Typical size 256K entries (1.5MB)
  // Two-byte hashkey
  inline const uint32 QuadHashV3Lookup4(const cld::CLDTableSummary* gram_obj,
                                        uint32 quadhash) {

    uint32 subscr, hashkey;
    const IndirectProbBucket4* quadtable = gram_obj->kCLDTable;
    uint32 keymask = gram_obj->kCLDTableKeyMask;
    int bucketcount = gram_obj->kCLDTableSize;
    QuadFPJustHash(quadhash, keymask, bucketcount, &subscr, &hashkey);
    const IndirectProbBucket4* bucket_ptr = &quadtable[subscr];
    // Four-way associative, 4 compares
    if (((hashkey ^ bucket_ptr->keyvalue[0]) & keymask) == 0) {
      return bucket_ptr->keyvalue[0];
    }
    if (((hashkey ^ bucket_ptr->keyvalue[1]) & keymask) == 0) {
      return bucket_ptr->keyvalue[1];
    }
    if (((hashkey ^ bucket_ptr->keyvalue[2]) & keymask) == 0) {
      return bucket_ptr->keyvalue[2];
    }
    if (((hashkey ^ bucket_ptr->keyvalue[3]) & keymask) == 0) {
      return bucket_ptr->keyvalue[3];
    }
    return 0;
  }


  // Map 40 bits to subscript, hashkey, expected 18-22 bit subscript (min 16)
  //     wwwwwwww xxxxxxxx xxxxxxxx yyyyyyyy yyyyyyyy
  //   + ........ ....wwww wwwwxxxx xxxxxxxx xxxxyyyy
  //     00000000 00000000 00000011 11111111 11111111 (18-bit bucketcount-1)
  //
  // hashkey:
  //              wwwwxxxx xxxxxxxx xxxx.... ........ (20-bit keymask)
  // 12-bit shift in subscript mixes in ~4 letters x 4 bits each

  // From 40-bit gram FP, return hash table subscript and remaining key
  inline void OctaFPJustHash(uint64 longwordhash,
                                    uint32 keymask,
                                    int bucketcount,
                                    uint32* subscr, uint32* hashkey) {
    uint32 temp = (longwordhash + (longwordhash >> 12)) & (bucketcount - 1);
    *subscr = temp;
    temp = longwordhash >> 4;
    *hashkey = temp & keymask;
  }

  // Look up 40-bit gram FP in caller-passed table
  // Typical size 256K-4M entries (1-16MB)
  // 24-12 bit hashkey packed with 8-20 bit indirect lang/probs
  // keymask is 0xfffff000 for 20-bit hashkey and 12-bit indirect
  inline const uint32 OctaHashV3Lookup4(const cld::CLDTableSummary* gram_obj,
                                            uint64 longwordhash) {
    uint32 subscr, hashkey;
    const IndirectProbBucket4* octatable = gram_obj->kCLDTable;
    uint32 keymask = gram_obj->kCLDTableKeyMask;
    int bucketcount = gram_obj->kCLDTableSize;
    OctaFPJustHash(longwordhash, keymask, bucketcount,
                          &subscr, &hashkey);
    const IndirectProbBucket4* bucket_ptr = &octatable[subscr];
    // Four-way associative, 4 compares
    if (((hashkey ^ bucket_ptr->keyvalue[0]) & keymask) == 0) {
      return bucket_ptr->keyvalue[0];
    }
    if (((hashkey ^ bucket_ptr->keyvalue[1]) & keymask) == 0) {
      return bucket_ptr->keyvalue[1];
    }
    if (((hashkey ^ bucket_ptr->keyvalue[2]) & keymask) == 0) {
      return bucket_ptr->keyvalue[2];
    }
    if (((hashkey ^ bucket_ptr->keyvalue[3]) & keymask) == 0) {
      return bucket_ptr->keyvalue[3];
    }
    return 0;
  }



//------------------------------------------------------------------------------
// Scoring single groups of letters
//------------------------------------------------------------------------------

  // UNIGRAM score one => tote
  // Input: 1-byte entry of subscript into unigram probs, plus
  //  an accumulator tote.
  // Output: running sums in tote updated
  void ProcessProbV25UniTote(int propval, Tote* tote);

  // BIGRAM, QUADGRAM, OCTAGRAM score one => tote
  // Input: 4-byte entry of 3 language numbers and one probability subscript,
  //  plus an accumulator tote. (language 0 means unused entry)
  // Output: running sums in tote updated
  void ProcessProbV25Tote(uint32 probs, Tote* tote);


//------------------------------------------------------------------------------
// Routines to accumulate probabilities
//------------------------------------------------------------------------------

  // Score up to n=gram_limit unigrams, returning number of bytes consumed
  // Caller supplies table, such as compact_lang_det_generated_ctjkvz_b1_obj
  int DoUniScoreV3(const UTF8PropObj* unigram_obj,
                   const char* isrc, int srclen, int advance_by,
                   int* tote_grams, int gram_limit, Tote* chunk_tote);


  // Score all words in isrc, using languages that have bigrams (CJK)
  // Caller supplies table, such as &kCjkBiTable_obj or &kGibberishTable_obj
  // Return number of bigrams that hit in the hash table
  int DoBigramScoreV3(const cld::CLDTableSummary* bigram_obj,
                      const char* isrc, int srclen, Tote* chunk_tote);


  // Score up to n=gram_limit quadgrams, returning number of bytes consumed
  // Caller supplies table, such as &kQuadTable_obj or &kGibberishTable_obj
  int DoQuadScoreV3(const cld::CLDTableSummary* quadgram_obj,
                    const char* isrc, int srclen, int advance_by,
                    int* tote_grams, int gram_limit, Tote* chunk_tote);

  // Score all octagrams (words) in isrc, using languages that have quadgrams
  // Caller supplies table, such as &kLongWord8Table_obj
  // Return number of words that hit in the hash table
  int DoOctaScoreV3(const cld::CLDTableSummary* octagram_obj,
                    const char* isrc, int srclen, Tote* chunk_tote);

//------------------------------------------------------------------------------
// Reliability calculations, for single language and between languages
//------------------------------------------------------------------------------

  // Reliability = 0..100
  static const int kMinReliable = 75;

  // Calculate ratio of score per 1KB vs. expected score per 1KB
  double GetNormalizedScore(Language lang, UnicodeLScript lscript,
                          int bytes, int score);

  // Calculate reliablity of len bytes of script lscript with chunk_tote
  int GetReliability(int len, UnicodeLScript lscript, const Tote* chunk_tote);


//------------------------------------------------------------------------------
// Miscellaneous
//------------------------------------------------------------------------------

  // Make languages packed into uint32 values non-zero
  // These routines later could remap so languages not in QuadHash tables are not
  // represented, and so that any thrashing in accumulation is eliminated
  uint8 inline PackLanguage(Language lang) {
    return static_cast<uint8>(lang + 1);}

  Language inline UnpackLanguage(int ilang) {
    return static_cast<Language>(ilang - 1);}

  // Useful single-byte tests
  bool inline IsUTF8ContinueByte(char c) {
    return static_cast<signed char>(c) < -64;}
  bool inline IsUTF8HighByte(char c) {
    return static_cast<signed char>(c) < 0;}


  // Demote all languages except Top40 and plus_one
  // Do this just before sorting
  void DemoteNotTop40(Tote* chunk_tote, int packed_plus_one);

}       // End namespace cld


#endif  // ENCODINGS_COMPACT_LANG_DET_CLDUTIL_H_