diff --git a/.github/workflows/test-build.js b/.github/workflows/test-build.js index febbf05..90d89d0 100644 --- a/.github/workflows/test-build.js +++ b/.github/workflows/test-build.js @@ -8,15 +8,28 @@ assert(typeof chardet.detectFile, 'function'); assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ - { confidence: 100, name: 'ASCII', lang: undefined }, - { confidence: 98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 98, name: 'ISO-8859-2', lang: 'hu' }, - { confidence: 10, name: 'UTF-8', lang: undefined }, - { confidence: 10, name: 'Shift_JIS', lang: 'ja' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'EUC-JP', lang: 'ja' }, - { confidence: 10, name: 'EUC-KR', lang: 'ko' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, + { confidence: 1, name: 'ASCII', lang: undefined }, + { name: 'ISO-8859-2', lang: 'hu', confidence: 0.6 }, + { name: 'windows-1250', lang: 'hu', confidence: 0.6 }, + { name: 'ISO-8859-1', lang: 'en', confidence: 0.4 }, + { name: 'windows-1252', lang: 'en', confidence: 0.4 }, + { name: 'ISO-8859-10', lang: 'is', confidence: 0.4 }, + { name: 'windows-1257', lang: 'lv', confidence: 0.2 }, + { name: 'macintosh', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-3', lang: 'mt', confidence: 0.2 }, + { name: 'ISO-8859-4', lang: 'lv', confidence: 0.2 }, + { name: 'ISO-8859-13', lang: 'lt', confidence: 0.2 }, + { name: 'ISO-8859-14', lang: 'cy', confidence: 0.2 }, + { name: 'ISO-8859-15', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-16', lang: 'ro', confidence: 0.2 }, + { name: 'CP850', lang: 'de', confidence: 0.2 }, + { name: 'CP852', lang: 'pl', confidence: 0.2 }, + { confidence: 0.1, name: 'UTF-8', lang: undefined }, + { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'EUC-JP', lang: 'ja' }, + { confidence: 0.1, name: 'EUC-KR', lang: 'ko' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, ]); console.log(' > test-build.js OK'); diff --git a/.github/workflows/test-build.ts b/.github/workflows/test-build.ts index ae04c30..d074a2a 100644 --- a/.github/workflows/test-build.ts +++ b/.github/workflows/test-build.ts @@ -11,15 +11,28 @@ const main = async () => { assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ - { confidence: 100, name: 'ASCII', lang: undefined }, - { confidence: 98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 98, name: 'ISO-8859-2', lang: 'hu' }, - { confidence: 10, name: 'UTF-8', lang: undefined }, - { confidence: 10, name: 'Shift_JIS', lang: 'ja' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'EUC-JP', lang: 'ja' }, - { confidence: 10, name: 'EUC-KR', lang: 'ko' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, + { confidence: 1, name: 'ASCII', lang: undefined }, + { name: 'ISO-8859-2', lang: 'hu', confidence: 0.6 }, + { name: 'windows-1250', lang: 'hu', confidence: 0.6 }, + { name: 'ISO-8859-1', lang: 'en', confidence: 0.4 }, + { name: 'windows-1252', lang: 'en', confidence: 0.4 }, + { name: 'ISO-8859-10', lang: 'is', confidence: 0.4 }, + { name: 'windows-1257', lang: 'lv', confidence: 0.2 }, + { name: 'macintosh', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-3', lang: 'mt', confidence: 0.2 }, + { name: 'ISO-8859-4', lang: 'lv', confidence: 0.2 }, + { name: 'ISO-8859-13', lang: 'lt', confidence: 0.2 }, + { name: 'ISO-8859-14', lang: 'cy', confidence: 0.2 }, + { name: 'ISO-8859-15', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-16', lang: 'ro', confidence: 0.2 }, + { name: 'CP850', lang: 'de', confidence: 0.2 }, + { name: 'CP852', lang: 'pl', confidence: 0.2 }, + { confidence: 0.1, name: 'UTF-8', lang: undefined }, + { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'EUC-JP', lang: 'ja' }, + { confidence: 0.1, name: 'EUC-KR', lang: 'ko' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, ]); }; diff --git a/.gitignore b/.gitignore index 6c52297..5358bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ lib TODO.md package-lock.json .vscode +BASELINE.md diff --git a/BASELINE.md b/BASELINE.md new file mode 100644 index 0000000..c2790dc --- /dev/null +++ b/BASELINE.md @@ -0,0 +1,115 @@ +# Generated model migration baseline + +Recorded on 2026-07-12 at commit +`df1e8dd0088f5d4b71790a54ee2fc1f2afe6cb0a` (`v3`), using Node.js v24.10.0 +and npm 11.6.1. + +`TODO.md` is locally present but ignored by the repository. There were no +tracked or unignored changes when this baseline was recorded. Generated `lib/` +and coverage output are also ignored. + +## Checks + +| Check | Result | +| --------------------------------- | ------------------------------------------------------------------ | +| `npm test` | Pass: 8 files, 48 tests passed, 8 todo | +| TypeScript | Pass | +| Coverage | 93.04% statements, 85.44% branches, 95.86% functions, 93.13% lines | +| `npm run models:verify` | Pass; models and evaluation are reproducible | +| `npm run validate -- --split=all` | Completes; 0/138 encoding and language predictions correct | +| `npm run benchmark` | 7,848.13 files/sec; 1.42 MiB/sec | +| `npm run build` | Pass | +| Built `lib/` | 341,661 bytes across files; 448 KiB allocated on disk | +| `npm pack --dry-run` | 42,359 bytes packed; 348,964 bytes unpacked; 42 entries | + +The benchmark processed 13,800 calls and 2,622,400 bytes in 1,758.38 ms, +producing 100,200 matches. These figures are a local performance reference, not +a hard CI threshold; comparisons should use the same machine and runtime. + +## Runtime corpus validation + +The validation corpus currently contains only newly generated models that are +not registered with the runtime. The 0% result is therefore the expected +pre-migration baseline, not a model-evaluator failure. Every input still +received at least one prediction. + +### By split + +| Split | Files | Encoding correct | Language correct | +| ---------- | ------: | ---------------: | ---------------: | +| train | 92 | 0 (0.00%) | 0 (0.00%) | +| validation | 23 | 0 (0.00%) | 0 (0.00%) | +| test | 23 | 0 (0.00%) | 0 (0.00%) | +| **Total** | **138** | **0 (0.00%)** | **0 (0.00%)** | + +### By expected encoding + +| Expected encoding | Files | Encoding correct | Most common current prediction | +| ----------------- | ----: | ---------------: | ------------------------------ | +| CP850 | 18 | 0 | windows-1252 | +| CP852 | 6 | 0 | windows-1250 | +| CP949 | 6 | 0 | EUC-KR | +| IBM855 | 12 | 0 | windows-1252 | +| IBM866 | 6 | 0 | windows-1252 | +| ISO-8859-10 | 6 | 0 | ISO-8859-1 | +| ISO-8859-13 | 6 | 0 | windows-1257 | +| ISO-8859-14 | 6 | 0 | ISO-8859-1 | +| ISO-8859-15 | 18 | 0 | ISO-8859-1 | +| ISO-8859-16 | 6 | 0 | ISO-8859-2 | +| ISO-8859-3 | 6 | 0 | ISO-8859-1 | +| ISO-8859-4 | 6 | 0 | windows-1257 | +| KOI8-U | 6 | 0 | KOI8-R | +| macintosh | 18 | 0 | windows-1252 | +| x-mac-cyrillic | 12 | 0 | windows-1251 | + +The most common predictions above are useful collision targets, but do not +describe every document. Notably, weak IBM855 input can also be classified as +Shift_JIS or windows-1255. + +## Public naming decisions + +Existing names remain exactly as declared by `EncodingName`; the migration must +not silently recase or rename them. The new SBCS canonical result names are: + +```text +KOI8-U +IBM866 +IBM855 +macintosh +x-mac-cyrillic +ISO-8859-3 +ISO-8859-4 +ISO-8859-10 +ISO-8859-13 +ISO-8859-14 +ISO-8859-15 +ISO-8859-16 +CP850 +CP852 +``` + +CP949 is reserved as the canonical future result name for the separate +multibyte follow-up. + +The current public API does not accept encoding-name filters and does not expose +alias normalization. Consequently, the initial migration adds only these +canonical `EncodingName` values. Iconv/compiler input labels such as `CP866`, +`CP855`, `MACINTOSH`, and `MAC-CYRILLIC` are implementation aliases and must not +be returned by `detect` or `analyse`. + +## Migration scope decision + +All existing SBCS recognizers will move to generated models. The legacy byte +maps and n-gram tables in `src/encoding/sbcs.ts` will not remain as a parallel +model system. Existing encodings must first be added to the corpus/model inputs, +generated, and validated before their hard-coded tables are removed. + +## Known pre-migration limitations + +- Generated SBCS models are not consumed or registered by the detector. +- New encodings therefore lose to the closest existing recognizer. +- Confidence values are not yet calibrated between generated SBCS and existing + UTF, ISO-2022, or MBCS recognizers. +- The existing test suite has eight explicitly pending tests. +- CP949 is represented in the corpus but intentionally excluded from the SBCS + model compiler and requires separate MBCS work. diff --git a/README.md b/README.md index 6c10081..932ebb4 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ Returned value is an array of objects sorted by confidence value in descending o ```javascript [ - { confidence: 90, name: 'UTF-8' }, - { confidence: 20, name: 'windows-1252', lang: 'fr' }, + { confidence: 0.9, name: 'UTF-8' }, + { confidence: 0.2, name: 'windows-1252', lang: 'fr' }, ]; ``` @@ -89,16 +89,29 @@ Note on [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncod - ISO-2022-CN - ISO-8859-1 - ISO-8859-2 +- ISO-8859-3 +- ISO-8859-4 - ISO-8859-5 - ISO-8859-6 - ISO-8859-7 - ISO-8859-8 - ISO-8859-9 +- ISO-8859-10 +- ISO-8859-13 +- ISO-8859-14 +- ISO-8859-15 +- ISO-8859-16 - Big5 +- CP850 +- CP852 - EUC-JP - EUC-KR - GB18030 +- IBM855 +- IBM866 - KOI8-R +- KOI8-U +- macintosh - Shift_JIS - UTF-8 - UTF-16BE @@ -115,6 +128,7 @@ Note on [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncod - windows-1256 - windows-1257 - windows-1258 +- x-mac-cyrillic Currently only these encodings are supported. @@ -129,11 +143,4 @@ Yes. Type definitions are included. ### TODO -- [ ] KOI8-U for Ukrainian -- [ ] IBM866 for DOS Cyrillic -- [ ] macintosh and x-mac-cyrillic - [ ] CP949 / UHC support within the Korean recognizer -- [ ] ISO-8859-3, ISO-8859-4 -- [ ] ISO-8859-10, ISO-8859-13 -- [ ] ISO-8859-14, ISO-8859-15, ISO-8859-16 -- [ ] DOS CP850, CP852, CP855 diff --git a/corpus/README.md b/corpus/README.md index 2878450..d1a1fba 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -4,8 +4,9 @@ This directory contains a deterministic seed corpus for developing additional encoding recognisers. The prose is synthetic and should not be treated as a replacement for a large, independently sourced natural-language corpus. -`sources.json` stores UTF-8 originals. `manifest.json` maps languages to iconv -encodings. Generated byte samples and statistics live under `generated/`. +`sources.json` stores UTF-8 originals. Held-out material is kept separately in +`test-sources.json`. `manifest.json` maps languages to iconv encodings. +Generated byte samples and statistics live under `generated/`. Build and verify the corpus with: @@ -17,8 +18,21 @@ npm run models:evaluate npm run models:verify ``` -The build script generates the corpus and performs an exact iconv round trip for -every document. The check script independently rebuilds into a temporary +Benchmark all inputs or isolate a stable recognizer family workload with: + +```sh +npm run benchmark +npm run benchmark -- --family=singlebyte +npm run benchmark -- --family=multibyte +``` + +Benchmark output includes the input bytes per iteration and a workload SHA-256. +Performance results are comparable only when the family, split, and workload +hash match. + +The build script generates the corpus and performs an iconv round trip for every +document, allowing canonically equivalent Unicode composition where an encoding +such as windows-1258 stores combining marks. The check script independently rebuilds into a temporary directory and byte-compares the result with `generated/`. Four documents per language are assigned to training, one to validation, and one independent document to testing. Statistics are calculated from training documents only. @@ -34,9 +48,20 @@ The model compiler derives each single-byte encoding's case-folding byte map through iconv, applies the detector's whitespace normalization, and emits 64 sorted detector-ready trigrams into `src/encoding/models/generated.ts`. It also learns Laplace-smoothed high-byte distributions from the training split. The -evaluator uses byte likelihood to rank candidates whose trigram hit rates are -statistically competitive at a 95% margin. `models:evaluate` reports every -held-out test document, while `models:verify` checks reproducibility and fails if -any encoding or language result is incorrect. CP949 is excluded because its -multibyte recogniser requires character frequency data rather than byte -trigrams. +compiler evaluator and library runtime share the normalization, trigram, +byte-likelihood, confidence, candidate-prefiltering, and tie-breaking +implementation in `src/encoding/sbcs-scoring.ts`. Candidates whose trigram hit +rates are statistically competitive at a 95% margin are ranked by byte +likelihood and deterministic encoding-family preferences. `models:evaluate` +reports every held-out test document, while `models:verify` checks +reproducibility and fails if any encoding or language result is incorrect. A +different encoding label is accepted only when both encodings decode the actual +held-out bytes identically; the report distinguishes these byte-equivalent +results from exact matches. + +For multibyte encodings, the structural parsers identify valid raw code units +and the compiler selects the 128 most frequent units from the training split. +Runtime confidence is the structurally valid proportion of input code units +covered by that generated model. Shift_JIS, EUC-JP, Big5, EUC-KR, and GB18030 +use this path. CP949 remains in the corpus but is excluded from the runtime +models until its extended byte sequences have a dedicated parser. diff --git a/corpus/generated/Big5/zh/test/community-garden.bin b/corpus/generated/Big5/zh/test/community-garden.bin new file mode 100644 index 0000000..594af6e --- /dev/null +++ b/corpus/generated/Big5/zh/test/community-garden.bin @@ -0,0 +1,3 @@ +社區花園 + +鄰居們把空院子改造成綠色花園，種下鮮花、蔬菜和小樹。孩子們畫了彩色標牌，各家每週一起給植物澆水。 diff --git a/corpus/generated/Big5/zh/train/city-morning.bin b/corpus/generated/Big5/zh/train/city-morning.bin new file mode 100644 index 0000000..71e4cc7 --- /dev/null +++ b/corpus/generated/Big5/zh/train/city-morning.bin @@ -0,0 +1,3 @@ +城市的早晨 + +城市慢慢醒來，車站聚集了上班的人，咖啡店打開大門，第一班公共汽車開始穿過安靜的街道。 diff --git a/corpus/generated/Big5/zh/train/library.bin b/corpus/generated/Big5/zh/train/library.bin new file mode 100644 index 0000000..82396af --- /dev/null +++ b/corpus/generated/Big5/zh/train/library.bin @@ -0,0 +1,3 @@ +安靜的圖書館 + +社區圖書館裡只有翻書的聲音。讀者尋找歷史、科學和旅行方面的書，管理員幫助學生查找資料。 diff --git a/corpus/generated/Big5/zh/train/market.bin b/corpus/generated/Big5/zh/train/market.bin new file mode 100644 index 0000000..97ee3e1 --- /dev/null +++ b/corpus/generated/Big5/zh/train/market.bin @@ -0,0 +1,3 @@ +星期六的市場 + +廣場上的市場擺滿新鮮蔬菜、水果、蜂蜜和麵包。顧客向農民詢問收成，並交流家常菜的做法。 diff --git a/corpus/generated/Big5/zh/train/workshop.bin b/corpus/generated/Big5/zh/train/workshop.bin new file mode 100644 index 0000000..a6e6c69 --- /dev/null +++ b/corpus/generated/Big5/zh/train/workshop.bin @@ -0,0 +1,3 @@ +數位工作室 + +一個小團隊正在製作博物館指南。設計師整理頁面，開發者檢查程式，編輯為不同年齡的參觀者撰寫清楚的說明。 diff --git a/corpus/generated/Big5/zh/validation/river-trip.bin b/corpus/generated/Big5/zh/validation/river-trip.bin new file mode 100644 index 0000000..c52a8b5 --- /dev/null +++ b/corpus/generated/Big5/zh/validation/river-trip.bin @@ -0,0 +1,3 @@ +河邊旅行 + +朋友們帶著地圖和熱茶穿過樹林。走了很久以後，他們坐在安靜的河邊休息，聽傍晚的鳥鳴。 diff --git a/corpus/generated/EUC-JP/ja/test/community-garden.bin b/corpus/generated/EUC-JP/ja/test/community-garden.bin new file mode 100644 index 0000000..d9ad4af --- /dev/null +++ b/corpus/generated/EUC-JP/ja/test/community-garden.bin @@ -0,0 +1,3 @@ +譁啗及瓔 + +嗤赭及諦凶切反塢中化中凶瓔毛幫尹﹜累支沓窪支潦中枆毛縈尹凶﹝閡升手凶切反縉蟒支井卅誑毛綜曰﹜笮簡反咍蔥縈坁卞踹毛支月﹝ diff --git a/corpus/generated/EUC-JP/ja/train/city-morning.bin b/corpus/generated/EUC-JP/ja/train/city-morning.bin new file mode 100644 index 0000000..72a9c8a --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/city-morning.bin @@ -0,0 +1,3 @@ +章及蘋 + +蘋及章反斗勻仁曰杴毛創引允﹝斛卞反騷剷允月諦互螂引曰﹜脾辭饕互滮繷咫情ⅡХ擗峊苭馱珊聾咫岉怳磥臗鬗篕洃嶀諢 diff --git a/corpus/generated/EUC-JP/ja/train/library.bin b/corpus/generated/EUC-JP/ja/train/library.bin new file mode 100644 index 0000000..f15e8f5 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/library.bin @@ -0,0 +1,3 @@ +斃井卅豭踏欺 + +譁啗及豭踏欺匹反呿及矢□斥毛戶仁月祥互妏仇尹月﹝づ樊反昳銖支粗喀支庣墊及呿毛繭仄﹜銘踏反喀戲及譬屯坁毛澎鰻丹﹝ diff --git a/corpus/generated/EUC-JP/ja/train/market.bin b/corpus/generated/EUC-JP/ja/train/market.bin new file mode 100644 index 0000000..4169791 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/market.bin @@ -0,0 +1,3 @@ +蘿豖ゥ及閩樺 + +僮樺及閩樺卞反蕙蟒卅沓窪﹜絆坁﹜邞怐﹜嬴五凶化及由件互岈少﹝ъ中坁菸反Ш笮午筐濡及恔咥卞勾中化狪允﹝ diff --git a/corpus/generated/EUC-JP/ja/train/workshop.bin b/corpus/generated/EUC-JP/ja/train/workshop.bin new file mode 100644 index 0000000..20c2244 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/workshop.bin @@ -0,0 +1,3 @@ +犯斥正伙鼎佷 + +凝今卅民□丞互⑥坁欺及副ま毛綜勻化中月﹝澀煌樊互脰昍毛幫尹﹜釩紡啎洛部憧氻繷恉坐楚〨埣蜈啎畎洶咫磥鉹馱分獑壑羷韙砥 diff --git a/corpus/generated/EUC-JP/ja/validation/river-trip.bin b/corpus/generated/EUC-JP/ja/validation/river-trip.bin new file mode 100644 index 0000000..a0faed9 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/validation/river-trip.bin @@ -0,0 +1,3 @@ +燠尺及凝庣墊 + +竻諦凶切反譁豭午硃井中云辭毛儅勻化蕨毛忷中凶﹝贏中辣忷及詨﹜斃井卅燠彶匹菱心卅互日芞杅及躉及擘毛妏中凶﹝ diff --git a/corpus/generated/EUC-KR/ko/test/community-garden.bin b/corpus/generated/EUC-KR/ko/test/community-garden.bin new file mode 100644 index 0000000..d5f8130 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/test/community-garden.bin @@ -0,0 +1,3 @@ +葆擊 奢翕 薑錳 + +輿團菟擎 綠橫 氈湍 葆渡擊 薑葬ж堅 粒婁 瓣模, 橫萼 釭鼠蒂 褕歷棻. 嬴檜菟擎 憲煙殖煙и 檜葷ル蒂 虜菟歷堅 檜醒菟擎 衙輿 賅罹 僭擊 輿堅 紫掘蒂 薑絲и棻. diff --git a/corpus/generated/EUC-KR/ko/train/city-morning.bin b/corpus/generated/EUC-KR/ko/train/city-morning.bin new file mode 100644 index 0000000..d2976f1 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/city-morning.bin @@ -0,0 +1,3 @@ +紫衛曖 嬴藹 + +紫衛朝 繭繭 嬴藹擊 蜃朝棻. 蘋む陛 僥擊 翮堅 幗蝶陛 羅 遴ч擊 衛濛ж貊 餌塋菟擎 瑒蹀風 景蝶蒂 檗朝棻. 勒僭 餌檜煎 嫩擎 х維檜 廓霞棻. diff --git a/corpus/generated/EUC-KR/ko/train/library.bin b/corpus/generated/EUC-KR/ko/train/library.bin new file mode 100644 index 0000000..6c5b886 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/library.bin @@ -0,0 +1,3 @@ +褻辨и 紫憮婦 + +翕啻 紫憮婦縑朝 疇濰 剩晦朝 模葬陛 濤濤ж啪 菟萼棻. 絮濠菟擎 羲餌諦 婁з, 罹ч縑 婦и 疇擊 瓊朝棻. 餌憮朝 з儅菟擊 嬪и 濛擎 瞪衛蒂 遽綠и棻. diff --git a/corpus/generated/EUC-KR/ko/train/market.bin b/corpus/generated/EUC-KR/ko/train/market.bin new file mode 100644 index 0000000..3789568 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/market.bin @@ -0,0 +1,3 @@ +饜蹂橾 衛濰 + +惜濰縑 翮萼 衛濰縑朝 餌婁諦 羚, 纂鍔諦 鬥 掘遴 誧檜 場檣棻. 槳椒菟擎 瓣模蒂 堅腦堅 堯睡諦 啗瞰 檜撿晦蒂 釭援貊 餵縑憮 虜萇 蹂葬徹擊 模偃и棻. diff --git a/corpus/generated/EUC-KR/ko/train/workshop.bin b/corpus/generated/EUC-KR/ko/train/workshop.bin new file mode 100644 index 0000000..c1ed449 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/workshop.bin @@ -0,0 +1,3 @@ +蛤雖瓚 濛機褒 + +濛擎 ぜ檜 夢僭婦 寰頂 Щ煎斜極擊 虜萇棻. 蛤濠檜傘朝 詭景蒂 薑葬ж堅 偃嫦濠朝 囀萄蒂 挫恉牉 ら餵濠朝 寞僥偌縑啪 蓮遴 撲貲擊 噹棻. 挫 旋濠 旋婁 濠紫 匐餌и棻. diff --git a/corpus/generated/EUC-KR/ko/validation/river-trip.bin b/corpus/generated/EUC-KR/ko/validation/river-trip.bin new file mode 100644 index 0000000..c8a9dce --- /dev/null +++ b/corpus/generated/EUC-KR/ko/validation/river-trip.bin @@ -0,0 +1,3 @@ +鬼陛煎 集陪 罹ч + +橾蹂橾縑 耀掘菟擎 紫衛 夤曖 褻辨и 鬼戲煎 勘橫鬲棻. 雖紫諦 評嗆и 離, 餌霞晦蒂 穫啣棻. 曹 骯疇 菴縑朝 鬼陛縑 導嬴 盪喔 億模葬蒂 菟歷棻. diff --git a/corpus/generated/GB18030/zh/test/community-garden.bin b/corpus/generated/GB18030/zh/test/community-garden.bin new file mode 100644 index 0000000..ab73969 --- /dev/null +++ b/corpus/generated/GB18030/zh/test/community-garden.bin @@ -0,0 +1,3 @@ +扦^豪@ + +懈參諾埏赽蜊婖傖鄭伎豪@ㄛ意狟鰎豪﹜忣粕睿苤銦ㄩＩ賸粗伎侜ゞ爰鷐疰貓L珨れ緻眵昜粕恣 diff --git a/corpus/generated/GB18030/zh/train/city-morning.bin b/corpus/generated/GB18030/zh/train/city-morning.bin new file mode 100644 index 0000000..9759d58 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/city-morning.bin @@ -0,0 +1,3 @@ +傑庈腔婌鹵 + +傑庈鞣鞣倳瞨眉桴擄摩賸奻啤腔芄玷孚虛湖嶱湮嬡ㄛ菴珨啤鼠僕イ嶱宎援綎假髺腔誰耋﹝ diff --git a/corpus/generated/GB18030/zh/train/library.bin b/corpus/generated/GB18030/zh/train/library.bin new file mode 100644 index 0000000..01c0725 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/library.bin @@ -0,0 +1,3 @@ +假髺腔D斸^ + +扦^D斸^孬硐衄楹礸鐃秞﹝莮氪孔v妢﹜褪W睿藏俴源醱腔齮牲傱T邳W汜脤梑揃蹋﹝ diff --git a/corpus/generated/GB18030/zh/train/market.bin b/corpus/generated/GB18030/zh/train/market.bin new file mode 100644 index 0000000..a2c8003 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/market.bin @@ -0,0 +1,3 @@ +陎ぶ鞠腔庈 + +V鹺炵騫[M陔鰎忣粕﹜阨彆﹜瑚蹲睿I婦﹝諦砃煢鏍彶傖ㄛK蝠霜模都粕腔酕楊﹝ diff --git a/corpus/generated/GB18030/zh/train/workshop.bin b/corpus/generated/GB18030/zh/train/workshop.bin new file mode 100644 index 0000000..819d65e --- /dev/null +++ b/corpus/generated/GB18030/zh/train/workshop.bin @@ -0,0 +1,3 @@ +舒遜尤鼳 + +珨苤F淏婓庨釬痔昜禭硌鰍﹝偞淕燴醱ㄛ嶱追氪z脤最宒ㄛ斢銓珀蕡g腔Ｅ^氪蚴ь奠腔挲隴﹝ diff --git a/corpus/generated/GB18030/zh/validation/river-trip.bin b/corpus/generated/GB18030/zh/validation/river-trip.bin new file mode 100644 index 0000000..fe7365d --- /dev/null +++ b/corpus/generated/GB18030/zh/validation/river-trip.bin @@ -0,0 +1,3 @@ +碩藏俴 + +攬衭快礸D睿幘韐拋^靾痋軗賸竭壅眕摽ㄛ坻釴婓假髺腔碩倎洘ㄛ奢俀腔讋頀﹝ diff --git a/corpus/generated/ISO-8859-1/da/test/community-garden.bin b/corpus/generated/ISO-8859-1/da/test/community-garden.bin new file mode 100644 index 0000000..bbb5980 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/test/community-garden.bin @@ -0,0 +1,3 @@ +F熞leshaven + +Naboerne gjorde en tom g緳d til en gr鷢 have, b鷨nene malede skilte, og familierne vander planterne hver uge. diff --git a/corpus/generated/ISO-8859-1/da/train/city-morning.bin b/corpus/generated/ISO-8859-1/da/train/city-morning.bin new file mode 100644 index 0000000..aafe557 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen v嶓ner langsomt, caf嶪rne 嶙ner, og folk venter p de f鷨ste busser. diff --git a/corpus/generated/ISO-8859-1/da/train/library.bin b/corpus/generated/ISO-8859-1/da/train/library.bin new file mode 100644 index 0000000..dfcd24b --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/library.bin @@ -0,0 +1,3 @@ +Det stille bibliotek + +L犘ere finder b鷤er om historie og videnskab, mens bibliotekaren hj熞per eleverne. diff --git a/corpus/generated/ISO-8859-1/da/train/market.bin b/corpus/generated/ISO-8859-1/da/train/market.bin new file mode 100644 index 0000000..e156a58 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/market.bin @@ -0,0 +1,3 @@ +L鷨dagsmarked + +B鷢der s熞ger frisk br鷣, 熲ler, ost og honning p den travle plads. diff --git a/corpus/generated/ISO-8859-1/da/train/workshop.bin b/corpus/generated/ISO-8859-1/da/train/workshop.bin new file mode 100644 index 0000000..04a0b0c --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt v熳ksted + +Et lille hold bygger en museumsguide, unders鷤er koden og skriver tydelige tekster. diff --git a/corpus/generated/ISO-8859-1/da/validation/river-trip.bin b/corpus/generated/ISO-8859-1/da/validation/river-trip.bin new file mode 100644 index 0000000..386c495 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til floden + +Vennerne tog et kort og varm te med p en s鷢dag og hvilede bagefter ved floden. diff --git a/corpus/generated/ISO-8859-1/de/test/community-garden.bin b/corpus/generated/ISO-8859-1/de/test/community-garden.bin new file mode 100644 index 0000000..df4bbec --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/test/community-garden.bin @@ -0,0 +1,3 @@ +Der Gemeinschaftsgarten + +Die Nachbarn haben einen leeren Hof begrnt und dort Kr酳ter, Gemse und junge B酳me gepflanzt. Die Schler malten bunte Schilder, w鄣rend 鄟tere Bewohner regelm魠ig gie絽n und Ger酹e aufr酳men. diff --git a/corpus/generated/ISO-8859-1/de/train/city-morning.bin b/corpus/generated/ISO-8859-1/de/train/city-morning.bin new file mode 100644 index 0000000..3ccc201 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen in der Stadt + +Die Stadt erwacht langsam. B踄kereien 鐪fnen ihre Tren, Busse beginnen ihre Fahrt und viele Menschen lesen unterwegs die Nachrichten. 鈁er den D踄hern wird der Himmel allm鄣lich heller. diff --git a/corpus/generated/ISO-8859-1/de/train/library.bin b/corpus/generated/ISO-8859-1/de/train/library.bin new file mode 100644 index 0000000..992cca1 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/library.bin @@ -0,0 +1,3 @@ +Die ruhige Bibliothek + +In der kleinen Bibliothek rascheln leise die Seiten. Besucher suchen Bcher ber Geschichte, Technik und ferne L鄚der. Eine Mitarbeiterin bereitet eine Ausstellung fr Schler vor. diff --git a/corpus/generated/ISO-8859-1/de/train/market.bin b/corpus/generated/ISO-8859-1/de/train/market.bin new file mode 100644 index 0000000..db29233 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/market.bin @@ -0,0 +1,3 @@ +Markt am Samstag + +Auf dem Platz bieten Bauern 癥fel, K酲e, Honig und frisches Brot an. Die Kunden vergleichen Gemse, sprechen ber die Ernte und tauschen einfache Rezepte aus. diff --git a/corpus/generated/ISO-8859-1/de/train/workshop.bin b/corpus/generated/ISO-8859-1/de/train/workshop.bin new file mode 100644 index 0000000..17ea61c --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale Werkstatt + +Ein kleines Team entwickelt einen digitalen Museumsfhrer. Designer ordnen das Men, Entwickler prfen den Code und Redakteure schreiben verst鄚dliche Texte fr G酲te. diff --git a/corpus/generated/ISO-8859-1/de/validation/river-trip.bin b/corpus/generated/ISO-8859-1/de/validation/river-trip.bin new file mode 100644 index 0000000..09bc118 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ausflug an den Fluss + +Am Sonntag wandern Freunde zu einem ruhigen Fluss. Sie nehmen eine Karte, hei絽n Tee und eine Kamera mit. Nach dem langen Weg sitzen sie am Ufer und h顤en den V鐷eln zu. diff --git a/corpus/generated/ISO-8859-1/en/test/community-garden.bin b/corpus/generated/ISO-8859-1/en/test/community-garden.bin new file mode 100644 index 0000000..549b374 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/test/community-garden.bin @@ -0,0 +1,3 @@ +The community garden + +Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small caf table. diff --git a/corpus/generated/ISO-8859-1/en/train/city-morning.bin b/corpus/generated/ISO-8859-1/en/train/city-morning.bin new file mode 100644 index 0000000..f7199ae --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/city-morning.bin @@ -0,0 +1,3 @@ +Morning in the city + +The city wakes slowly; caf廥 open their doors and commuters read news beside a fa蓷de. diff --git a/corpus/generated/ISO-8859-1/en/train/library.bin b/corpus/generated/ISO-8859-1/en/train/library.bin new file mode 100644 index 0000000..20400f3 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/library.bin @@ -0,0 +1,3 @@ +The quiet library + +Readers browse books about history, science, and na鴳e art while the librarian prepares a r廥um. diff --git a/corpus/generated/ISO-8859-1/en/train/market.bin b/corpus/generated/ISO-8859-1/en/train/market.bin new file mode 100644 index 0000000..c760513 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/market.bin @@ -0,0 +1,3 @@ +Saturday market + +Farmers sell bread, apples, jalape隳s, and p漮 while neighbours exchange recipes. diff --git a/corpus/generated/ISO-8859-1/en/train/workshop.bin b/corpus/generated/ISO-8859-1/en/train/workshop.bin new file mode 100644 index 0000000..788536f --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/workshop.bin @@ -0,0 +1,3 @@ +Digital workshop + +A small team designs the museum guide and discusses its r犨e, code, and clear descriptions. diff --git a/corpus/generated/ISO-8859-1/en/validation/river-trip.bin b/corpus/generated/ISO-8859-1/en/validation/river-trip.bin new file mode 100644 index 0000000..b686334 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/validation/river-trip.bin @@ -0,0 +1,3 @@ +Journey to the river + +Friends carried a map and a thermos, walked past the caf, and rested by the river. diff --git a/corpus/generated/ISO-8859-1/es/test/community-garden.bin b/corpus/generated/ISO-8859-1/es/test/community-garden.bin new file mode 100644 index 0000000..3c82a6f --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/test/community-garden.bin @@ -0,0 +1,3 @@ +El jard璯 comunitario + +Los vecinos convirtieron un patio vac甐 en un jard璯 lleno de 嫫boles, hierbas y verduras. Los ni隳s pintaron carteles de colores y cada s墎ado varias familias riegan las plantas. diff --git a/corpus/generated/ISO-8859-1/es/train/city-morning.bin b/corpus/generated/ISO-8859-1/es/train/city-morning.bin new file mode 100644 index 0000000..518bc97 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/city-morning.bin @@ -0,0 +1,3 @@ +La ma鎙na en la ciudad + +La ciudad despierta poco a poco. Las cafeter燰s abren sus puertas, los autobuses inician la ruta y los vecinos leen las noticias. El cielo se vuelve m嫳 claro sobre los tejados. diff --git a/corpus/generated/ISO-8859-1/es/train/library.bin b/corpus/generated/ISO-8859-1/es/train/library.bin new file mode 100644 index 0000000..4d6010f --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/library.bin @@ -0,0 +1,3 @@ +Una biblioteca tranquila + +En la biblioteca del barrio se oyen p墔inas y pasos suaves. Los lectores buscan libros de historia, ciencia y viajes. La bibliotecaria prepara una exposici鏮 para los alumnos. diff --git a/corpus/generated/ISO-8859-1/es/train/market.bin b/corpus/generated/ISO-8859-1/es/train/market.bin new file mode 100644 index 0000000..31af767 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/market.bin @@ -0,0 +1,3 @@ +El mercado del s墎ado + +El mercado llena la plaza de colores. Los agricultores venden manzanas, miel, queso y pan reci幯 hecho. Los clientes comparan las verduras y comparten recetas de sus familias. diff --git a/corpus/generated/ISO-8859-1/es/train/workshop.bin b/corpus/generated/ISO-8859-1/es/train/workshop.bin new file mode 100644 index 0000000..92a2f60 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/workshop.bin @@ -0,0 +1,3 @@ +Taller digital + +Un equipo peque隳 crea el cat嫮ogo digital del museo. Los dise鎙dores ordenan el men, los programadores revisan el c鏚igo y los editores escriben textos claros para el pblico. diff --git a/corpus/generated/ISO-8859-1/es/validation/river-trip.bin b/corpus/generated/ISO-8859-1/es/validation/river-trip.bin new file mode 100644 index 0000000..022d6f6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/validation/river-trip.bin @@ -0,0 +1,3 @@ +Paseo junto al r甐 + +El domingo unos amigos caminan hacia un r甐 tranquilo. Llevan un mapa, t caliente y una c嫥ara. Despu廥 del paseo descansan en la orilla y escuchan a los p奫aros. diff --git a/corpus/generated/ISO-8859-1/fr/test/community-garden.bin b/corpus/generated/ISO-8859-1/fr/test/community-garden.bin new file mode 100644 index 0000000..2dce62d --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/test/community-garden.bin @@ -0,0 +1,3 @@ +Le jardin partag + +Les habitants ont transform une cour oubli嶪 en jardin fleuri. Les enfants ont pr廧ar des 彋iquettes color嶪s, tandis que les voisins se r徼nissent chaque semaine pour arroser les l嶲umes et les jeunes arbres. diff --git a/corpus/generated/ISO-8859-1/fr/train/city-morning.bin b/corpus/generated/ISO-8859-1/fr/train/city-morning.bin new file mode 100644 index 0000000..411550d --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/city-morning.bin @@ -0,0 +1,3 @@ +Le matin en ville + +La ville se r憝eille lentement. Les caf廥 ouvrent leurs portes, les autobus commencent leur trajet et les passants consultent les nouvelles. Chacun pr廧are sa journ嶪 sous un ciel encore p滎e. diff --git a/corpus/generated/ISO-8859-1/fr/train/library.bin b/corpus/generated/ISO-8859-1/fr/train/library.bin new file mode 100644 index 0000000..e06a9ef --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/library.bin @@ -0,0 +1,3 @@ +La biblioth鋂ue calme + +Dans la biblioth鋂ue du quartier, les pages tournent en silence. Les lecteurs cherchent des livres sur l'histoire, les sciences et les voyages. Une biblioth嶰aire pr廧are une exposition pour les 幨鋦es. diff --git a/corpus/generated/ISO-8859-1/fr/train/market.bin b/corpus/generated/ISO-8859-1/fr/train/market.bin new file mode 100644 index 0000000..9717faa --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/market.bin @@ -0,0 +1,3 @@ +Le march du samedi + +Le march remplit la place de couleurs et de parfums. Les producteurs proposent des pommes, du miel, du fromage et du pain frais. Les clients discutent des saisons et 嶰hangent des recettes familiales. diff --git a/corpus/generated/ISO-8859-1/fr/train/workshop.bin b/corpus/generated/ISO-8859-1/fr/train/workshop.bin new file mode 100644 index 0000000..4dea00e --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/workshop.bin @@ -0,0 +1,3 @@ +Un atelier num廨ique + +Une petite 廦uipe construit le catalogue num廨ique du mus嶪. Les graphistes organisent le menu, les d憝eloppeurs v廨ifient le code et les r嶮acteurs pr廧arent des textes clairs pour le public. diff --git a/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin b/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin new file mode 100644 index 0000000..48f3a71 --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Une promenade pr鋊 de la rivi鋨e + +Dimanche, des amis partent vers une rivi鋨e tranquille. Ils emportent une carte, du th chaud et un appareil photo. Apr鋊 une longue marche, ils se reposent sur la rive et 嶰outent les oiseaux. diff --git a/corpus/generated/ISO-8859-1/it/test/community-garden.bin b/corpus/generated/ISO-8859-1/it/test/community-garden.bin new file mode 100644 index 0000000..76a54ea --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/test/community-garden.bin @@ -0,0 +1,3 @@ +Il giardino comune + +I vicini hanno trasformato un cortile vuoto in un giardino della citt, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante. diff --git a/corpus/generated/ISO-8859-1/it/train/city-morning.bin b/corpus/generated/ISO-8859-1/it/train/city-morning.bin new file mode 100644 index 0000000..a8faca7 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/city-morning.bin @@ -0,0 +1,3 @@ +Mattina in citt + +La citt si sveglia lentamente, i caff aprono e la gente aspetta i primi autobus. diff --git a/corpus/generated/ISO-8859-1/it/train/library.bin b/corpus/generated/ISO-8859-1/it/train/library.bin new file mode 100644 index 0000000..e9b65ab --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/library.bin @@ -0,0 +1,3 @@ +La biblioteca tranquilla + +I lettori cercano pi libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti pi giovani. diff --git a/corpus/generated/ISO-8859-1/it/train/market.bin b/corpus/generated/ISO-8859-1/it/train/market.bin new file mode 100644 index 0000000..780f149 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/market.bin @@ -0,0 +1,3 @@ +Mercato del sabato + +Gli agricoltori vendono pane fresco, caff, mele, formaggio e miele nella piazza della citt. diff --git a/corpus/generated/ISO-8859-1/it/train/workshop.bin b/corpus/generated/ISO-8859-1/it/train/workshop.bin new file mode 100644 index 0000000..21734d6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/workshop.bin @@ -0,0 +1,3 @@ +Laboratorio digitale + +Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perch siano utili a pi persone. diff --git a/corpus/generated/ISO-8859-1/it/validation/river-trip.bin b/corpus/generated/ISO-8859-1/it/validation/river-trip.bin new file mode 100644 index 0000000..0ff2d62 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/validation/river-trip.bin @@ -0,0 +1,3 @@ +Gita al fiume + +Gli amici portano una mappa e t caldo, camminano nel bosco e riposano sulla riva. diff --git a/corpus/generated/ISO-8859-1/nl/test/community-garden.bin b/corpus/generated/ISO-8859-1/nl/test/community-garden.bin new file mode 100644 index 0000000..89526b9 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/test/community-garden.bin @@ -0,0 +1,3 @@ +De buurttuin + +Buren veranderden een lege binnenplaats naast een caf in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water. diff --git a/corpus/generated/ISO-8859-1/nl/train/city-morning.bin b/corpus/generated/ISO-8859-1/nl/train/city-morning.bin new file mode 100644 index 0000000..13f5ea7 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/city-morning.bin @@ -0,0 +1,3 @@ +Ochtend in de stad + +De stad ontwaakt langzaam, caf廥 openen hun deuren en mensen wachten op de eerste bussen. diff --git a/corpus/generated/ISO-8859-1/nl/train/library.bin b/corpus/generated/ISO-8859-1/nl/train/library.bin new file mode 100644 index 0000000..cdf34ac --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/library.bin @@ -0,0 +1,3 @@ +De stille bibliotheek + +Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris ge鮦teresseerde leerlingen en 澭n leraar helpt. diff --git a/corpus/generated/ISO-8859-1/nl/train/market.bin b/corpus/generated/ISO-8859-1/nl/train/market.bin new file mode 100644 index 0000000..9c584bb --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/market.bin @@ -0,0 +1,3 @@ +Zaterdagmarkt + +Boeren verkopen vers brood, appels, kaas en honing naast twee caf廥 op het drukke plein. diff --git a/corpus/generated/ISO-8859-1/nl/train/workshop.bin b/corpus/generated/ISO-8859-1/nl/train/workshop.bin new file mode 100644 index 0000000..53ff667 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale werkplaats + +Een klein team maakt een museumgids, controleert de code en schrijft duidelijke idee螚 over po褯ie en teksten. diff --git a/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin b/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin new file mode 100644 index 0000000..27db29e --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Reis naar de rivier + +Vrienden namen een kaart en warme thee mee en rustten na 澭n lange wandeling bij de rivier. diff --git a/corpus/generated/ISO-8859-1/no/test/community-garden.bin b/corpus/generated/ISO-8859-1/no/test/community-garden.bin new file mode 100644 index 0000000..b82814c --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/test/community-garden.bin @@ -0,0 +1,3 @@ +Felleshagen + +Naboene gjorde en tom g緳d til en gr鷢n hage, barna malte skilt og familiene vanner plantene hver uke. diff --git a/corpus/generated/ISO-8859-1/no/train/city-morning.bin b/corpus/generated/ISO-8859-1/no/train/city-morning.bin new file mode 100644 index 0000000..972847d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen v嶡ner langsomt, kafeene 幠ner d鷨ene og folk venter p de f鷨ste bussene n熳 bryggen og sj鷫n. diff --git a/corpus/generated/ISO-8859-1/no/train/library.bin b/corpus/generated/ISO-8859-1/no/train/library.bin new file mode 100644 index 0000000..f972c3d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/library.bin @@ -0,0 +1,3 @@ +Det stille biblioteket + +Lesere finner b鷦er om historie, 熳lige fortellinger og vitenskap mens bibliotekaren hjelper elevene. diff --git a/corpus/generated/ISO-8859-1/no/train/market.bin b/corpus/generated/ISO-8859-1/no/train/market.bin new file mode 100644 index 0000000..c020f8d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/market.bin @@ -0,0 +1,3 @@ +L鷨dagsmarked + +B鷢der selger ferskt br鷣, epler, ost og honning p det travle torget. diff --git a/corpus/generated/ISO-8859-1/no/train/workshop.bin b/corpus/generated/ISO-8859-1/no/train/workshop.bin new file mode 100644 index 0000000..954e228 --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt verksted + +Et lite lag lager en museumsguide, unders鷦er koden og skriver tydelige tekster. diff --git a/corpus/generated/ISO-8859-1/no/validation/river-trip.bin b/corpus/generated/ISO-8859-1/no/validation/river-trip.bin new file mode 100644 index 0000000..1489859 --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til elven + +Vennene tok med kart og varm te p s鷢dag og hvilte senere ved elven. diff --git a/corpus/generated/ISO-8859-1/pt/test/community-garden.bin b/corpus/generated/ISO-8859-1/pt/test/community-garden.bin new file mode 100644 index 0000000..65848b6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/test/community-garden.bin @@ -0,0 +1,3 @@ +A horta comunit嫫ia + +Os vizinhos transformaram um p嫢io vazio em jardim, as crian蓷s pintaram placas e as fam璱ias regam as plantas toda semana. diff --git a/corpus/generated/ISO-8859-1/pt/train/city-morning.bin b/corpus/generated/ISO-8859-1/pt/train/city-morning.bin new file mode 100644 index 0000000..18b861f --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/city-morning.bin @@ -0,0 +1,3 @@ +Manh na cidade + +A cidade acorda devagar, os caf廥 abrem e as pessoas esperam os primeiros 獼ibus. diff --git a/corpus/generated/ISO-8859-1/pt/train/library.bin b/corpus/generated/ISO-8859-1/pt/train/library.bin new file mode 100644 index 0000000..50b8034 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/library.bin @@ -0,0 +1,3 @@ +A biblioteca tranquila + +Os leitores procuram livros de hist鏎ia e ci瘽cia enquanto a bibliotec嫫ia ajuda os alunos. diff --git a/corpus/generated/ISO-8859-1/pt/train/market.bin b/corpus/generated/ISO-8859-1/pt/train/market.bin new file mode 100644 index 0000000..3810190 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/market.bin @@ -0,0 +1,3 @@ +Mercado de s墎ado + +Os agricultores vendem p緌 fresco, ma誽s, queijo e mel na pra蓷 movimentada. diff --git a/corpus/generated/ISO-8859-1/pt/train/workshop.bin b/corpus/generated/ISO-8859-1/pt/train/workshop.bin new file mode 100644 index 0000000..ac78c4e --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/workshop.bin @@ -0,0 +1,3 @@ +Oficina digital + +Uma pequena equipe prepara o guia do museu, verifica o c鏚igo e escreve descri踥es claras. diff --git a/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin b/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin new file mode 100644 index 0000000..de2590a --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Viagem ao rio + +Os amigos levaram um mapa e ch quente, caminharam pela mata e descansaram junto ao rio. diff --git a/corpus/generated/ISO-8859-1/sv/test/community-garden.bin b/corpus/generated/ISO-8859-1/sv/test/community-garden.bin new file mode 100644 index 0000000..0acb233 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/test/community-garden.bin @@ -0,0 +1,3 @@ +Gemensam tr輐g緳d + +Grannarna gjorde en tom g緳d till en gr霵 tr輐g緳d, barnen m嶚ade skyltar och familjerna vattnar v鉶terna varje vecka. diff --git a/corpus/generated/ISO-8859-1/sv/train/city-morning.bin b/corpus/generated/ISO-8859-1/sv/train/city-morning.bin new file mode 100644 index 0000000..f0ba24b --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgon i staden + +Staden vaknar l幩gsamt, kaf嶪rna 鞿pnar och m鄚niskor v鄚tar p de f顤sta bussarna. diff --git a/corpus/generated/ISO-8859-1/sv/train/library.bin b/corpus/generated/ISO-8859-1/sv/train/library.bin new file mode 100644 index 0000000..a7319ac --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/library.bin @@ -0,0 +1,3 @@ +Det tysta biblioteket + +L酲are s闥er b鐼ker om historia och vetenskap medan bibliotekarien hj鄟per eleverna. diff --git a/corpus/generated/ISO-8859-1/sv/train/market.bin b/corpus/generated/ISO-8859-1/sv/train/market.bin new file mode 100644 index 0000000..0bd6cfb --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/market.bin @@ -0,0 +1,3 @@ +L顤dagsmarknad + +B霵der s鄟jer f酺skt br鐰, 鄡plen, ost och honung p det livliga torget. diff --git a/corpus/generated/ISO-8859-1/sv/train/workshop.bin b/corpus/generated/ISO-8859-1/sv/train/workshop.bin new file mode 100644 index 0000000..81db320 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/workshop.bin @@ -0,0 +1,3 @@ +Digital verkstad + +Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter f顤 bes闥are. diff --git a/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin b/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin new file mode 100644 index 0000000..4704973 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Utflykt till floden + +V鄚nerna tog med karta och varmt te och vilade sedan vid floden. diff --git a/corpus/generated/ISO-8859-16/ro/test/community-garden.bin b/corpus/generated/ISO-8859-16/ro/test/community-garden.bin index 54f8405..c763657 100644 --- a/corpus/generated/ISO-8859-16/ro/test/community-garden.bin +++ b/corpus/generated/ISO-8859-16/ro/test/community-garden.bin @@ -1,3 +1,3 @@ Gr緂ina comunit裻ii -Vecinii au transformat o curte goal 螽tr-o gr緂in cu flori, legume 槐 pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se 螽t滎nesc s綯t緆滱al pentru a uda plantele. +Vecinii au transformat o curte goal 螽tr-o gr緂in cu flori, legume 槐 pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se 螽t滎nesc s綯t緆滱al pentru a uda plantele. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/city-morning.bin b/corpus/generated/ISO-8859-16/ro/train/city-morning.bin index 08d7469..2986bf8 100644 --- a/corpus/generated/ISO-8859-16/ro/train/city-morning.bin +++ b/corpus/generated/ISO-8859-16/ro/train/city-morning.bin @@ -1,3 +1,3 @@ Dimineaa 螽 ora -Ora滾l se treze演e 螽cet. Cafenelele deschid u槐le, autobuzele pornesc pe traseu, iar oamenii citesc 演irile. Cerul devine mai luminos deasupra cl緂irilor. +Ora滾l se treze演e 螽cet. Cafenelele deschid u槐le, autobuzele pornesc pe traseu, iar oamenii citesc 演irile. Cerul devine mai luminos deasupra cl緂irilor. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/library.bin b/corpus/generated/ISO-8859-16/ro/train/library.bin index 1afc415..6517980 100644 --- a/corpus/generated/ISO-8859-16/ro/train/library.bin +++ b/corpus/generated/ISO-8859-16/ro/train/library.bin @@ -1,3 +1,3 @@ Biblioteca lini演it -姷 biblioteca de cartier se aud pagini 螽toarse 螽cet. Cititorii caut c綖i despre istorie, 演iin 槐 c緄綟orii. Bibliotecara preg綟e演e o expoziie pentru elevi. +姷 biblioteca de cartier se aud pagini 螽toarse 螽cet. Cititorii caut c綖i despre istorie, 演iin 槐 c緄綟orii. Bibliotecara preg綟e演e o expoziie pentru elevi. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/market.bin b/corpus/generated/ISO-8859-16/ro/train/market.bin index 15a0338..7178006 100644 --- a/corpus/generated/ISO-8859-16/ro/train/market.bin +++ b/corpus/generated/ISO-8859-16/ro/train/market.bin @@ -1,3 +1,3 @@ Piaa de s滵b綟 -Piaa umple centrul cu mirosuri 槐 culori. Fermierii aduc mere, miere, br滱z 槐 p毃ne proasp綟. Cump綖綟orii aleg legume 槐 schimb reete de familie. +Piaa umple centrul cu mirosuri 槐 culori. Fermierii aduc mere, miere, br滱z 槐 p毃ne proasp綟. Cump綖綟orii aleg legume 槐 schimb reete de familie. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/workshop.bin b/corpus/generated/ISO-8859-16/ro/train/workshop.bin index 132f010..9fdeac1 100644 --- a/corpus/generated/ISO-8859-16/ro/train/workshop.bin +++ b/corpus/generated/ISO-8859-16/ro/train/workshop.bin @@ -1,3 +1,3 @@ Atelier digital -O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. +O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin b/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin index c125be0..60dfedf 100644 --- a/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin +++ b/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin @@ -1,3 +1,3 @@ Plimbare l滱g r漉 -Duminic, prietenii au mers spre un r漉 lini演it din afara ora滾lui. Au luat o hart, ceai cald 槐 un aparat foto. Dup plimbare s-au odihnit pe mal. +Duminic, prietenii au mers spre un r漉 lini演it din afara ora滾lui. Au luat o hart, ceai cald 槐 un aparat foto. Dup plimbare s-au odihnit pe mal. Preul este 10 . diff --git a/corpus/generated/ISO-8859-2/cs/test/community-garden.bin b/corpus/generated/ISO-8859-2/cs/test/community-garden.bin new file mode 100644 index 0000000..61bfe36 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/test/community-garden.bin @@ -0,0 +1,3 @@ +Sousedsk zahrada + +Soused prom髶ili pr嫙dn dv鑴 v zelenou zahradu, d鮅i namalovaly barevn zna鋘y a rodiny zal憝aj rostliny. diff --git a/corpus/generated/ISO-8859-2/cs/train/city-morning.bin b/corpus/generated/ISO-8859-2/cs/train/city-morning.bin new file mode 100644 index 0000000..6efcbfe --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/city-morning.bin @@ -0,0 +1,3 @@ +R嫕o ve m鮀t + +M鮀to se pomalu probouz, kav嫫ny otev甏aj dve鷫 a lid 鋀kaj na prvn autobusy. diff --git a/corpus/generated/ISO-8859-2/cs/train/library.bin b/corpus/generated/ISO-8859-2/cs/train/library.bin new file mode 100644 index 0000000..31bd1a9 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/library.bin @@ -0,0 +1,3 @@ +Tich knihovna + +ten暟i hledaj knihy o historii a v駤, zat璥co knihovnice rad 擔k籯. diff --git a/corpus/generated/ISO-8859-2/cs/train/market.bin b/corpus/generated/ISO-8859-2/cs/train/market.bin new file mode 100644 index 0000000..f274a31 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/market.bin @@ -0,0 +1,3 @@ +Sobotn trh + +Farm暟i prod嫛aj 鋀rstv chl嶵, med, ovoce a vo瀡v byliny. diff --git a/corpus/generated/ISO-8859-2/cs/train/workshop.bin b/corpus/generated/ISO-8859-2/cs/train/workshop.bin new file mode 100644 index 0000000..d1dd133 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/workshop.bin @@ -0,0 +1,3 @@ +Digit嫮n d璱na + +Mal tm p鷡pravuje pr驠odce muzeem, kontroluje k鏚 a p竁e jasn popisy. diff --git a/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin b/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin new file mode 100644 index 0000000..09eba90 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin @@ -0,0 +1,3 @@ +Vlet k 鷫ce + +P鸅tel vzali mapu a tepl 醀j, pro雍i lesem a odpo鴈vali u 鷫ky. diff --git a/corpus/generated/ISO-8859-2/hu/test/community-garden.bin b/corpus/generated/ISO-8859-2/hu/test/community-garden.bin new file mode 100644 index 0000000..251d1d3 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/test/community-garden.bin @@ -0,0 +1,3 @@ +K驆飉s嶲i kert + +A szomsz嶮ok z闤d kertt alak癃ott嫜 az res udvart, a gyerekek t墎l嫜at festettek, a csal墂ok pedig 霵t驆ik a n饘幯yeket. diff --git a/corpus/generated/ISO-8859-2/hu/train/city-morning.bin b/corpus/generated/ISO-8859-2/hu/train/city-morning.bin new file mode 100644 index 0000000..74c3adc --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/city-morning.bin @@ -0,0 +1,3 @@ +Reggel a v嫫osban + +A v嫫os lassan 嶵red, a k嫛憴鏦 kinyitnak, az emberek pedig buszra v嫫nak. diff --git a/corpus/generated/ISO-8859-2/hu/train/library.bin b/corpus/generated/ISO-8859-2/hu/train/library.bin new file mode 100644 index 0000000..4bbeea1 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/library.bin @@ -0,0 +1,3 @@ +A csendes k霵yvt嫫 + +Az olvas鏦 t顤t幯elmi 廥 tudom嫕yos k霵yveket keresnek, a k霵yvt嫫os seg癃 a di嫜oknak. diff --git a/corpus/generated/ISO-8859-2/hu/train/market.bin b/corpus/generated/ISO-8859-2/hu/train/market.bin new file mode 100644 index 0000000..b95088b --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/market.bin @@ -0,0 +1,3 @@ +Szombati piac + +A gazd嫜 friss kenyeret, alm嫢, sajtot 廥 m憴et 嫫ulnak a t廨en. diff --git a/corpus/generated/ISO-8859-2/hu/train/workshop.bin b/corpus/generated/ISO-8859-2/hu/train/workshop.bin new file mode 100644 index 0000000..819901d --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/workshop.bin @@ -0,0 +1,3 @@ +Digit嫮is mhely + +Egy kis csapat mzeumi tmutat鏒 k廥z癃, k鏚ot ellen鰎iz 廥 vil墔os le甏嫳okat 甏. diff --git a/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin b/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin new file mode 100644 index 0000000..f807c95 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kir嫕dul嫳 a foly鏌oz + +A bar嫢ok t廨k廧et 廥 meleg te嫢 vittek, majd megpihentek a foly partj嫕. diff --git a/corpus/generated/ISO-8859-2/pl/test/community-garden.bin b/corpus/generated/ISO-8859-2/pl/test/community-garden.bin new file mode 100644 index 0000000..804f020 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/test/community-garden.bin @@ -0,0 +1,3 @@ +Ogr鏚 s御iedzki + +Mieszka鎍y zmienili pusty dziedziniec w zielony ogr鏚 z kwiatami, warzywami i m這dymi drzewami. Dzieci zrobi造 kolorowe tabliczki, a rodziny co tydzie podlewaj ro郵iny. diff --git a/corpus/generated/ISO-8859-2/pl/train/city-morning.bin b/corpus/generated/ISO-8859-2/pl/train/city-morning.bin new file mode 100644 index 0000000..a1106b8 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/city-morning.bin @@ -0,0 +1,3 @@ +Poranek w mie軼ie + +Miasto budzi si powoli. Kawiarnie otwieraj drzwi, autobusy rozpoczynaj pierwsze kursy, a ludzie czytaj wiadomo軼i. Nad dachami robi si coraz ja郾iej. diff --git a/corpus/generated/ISO-8859-2/pl/train/library.bin b/corpus/generated/ISO-8859-2/pl/train/library.bin new file mode 100644 index 0000000..dc98317 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/library.bin @@ -0,0 +1,3 @@ +Cicha biblioteka + +W osiedlowej bibliotece s造cha szelest kartek. Czytelnicy szukaj ksi捫ek o historii, nauce i podr騜ach. Bibliotekarka przygotowuje wystaw dla uczni闚. diff --git a/corpus/generated/ISO-8859-2/pl/train/market.bin b/corpus/generated/ISO-8859-2/pl/train/market.bin new file mode 100644 index 0000000..fcebf79 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/market.bin @@ -0,0 +1,3 @@ +Sobotni targ + +Targ wype軟ia rynek kolorami i zapachami. Rolnicy sprzedaj jab趾a, mi鏚, ser i 鈍ie篡 chleb. Klienci wybieraj warzywa i wymieniaj rodzinne przepisy. diff --git a/corpus/generated/ISO-8859-2/pl/train/workshop.bin b/corpus/generated/ISO-8859-2/pl/train/workshop.bin new file mode 100644 index 0000000..9de1523 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/workshop.bin @@ -0,0 +1,3 @@ +Cyfrowa pracownia + +Ma造 zesp馧 tworzy cyfrowy przewodnik po muzeum. Projektanci uk豉daj menu, programi軼i sprawdzaj kod, a redaktorzy pisz jasne opisy dla go軼i. diff --git a/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin b/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin new file mode 100644 index 0000000..b715445 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Spacer nad rzek + +W niedziel przyjaciele poszli nad spokojn rzek za miastem. Zabrali map, ciep陰 herbat i aparat. Po d逝gim spacerze odpoczywali na brzegu. diff --git a/corpus/generated/ISO-8859-2/ro/test/community-garden.bin b/corpus/generated/ISO-8859-2/ro/test/community-garden.bin new file mode 100644 index 0000000..54f8405 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/test/community-garden.bin @@ -0,0 +1,3 @@ +Gr緂ina comunit裻ii + +Vecinii au transformat o curte goal 螽tr-o gr緂in cu flori, legume 槐 pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se 螽t滎nesc s綯t緆滱al pentru a uda plantele. diff --git a/corpus/generated/ISO-8859-2/ro/train/city-morning.bin b/corpus/generated/ISO-8859-2/ro/train/city-morning.bin new file mode 100644 index 0000000..08d7469 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/city-morning.bin @@ -0,0 +1,3 @@ +Dimineaa 螽 ora + +Ora滾l se treze演e 螽cet. Cafenelele deschid u槐le, autobuzele pornesc pe traseu, iar oamenii citesc 演irile. Cerul devine mai luminos deasupra cl緂irilor. diff --git a/corpus/generated/ISO-8859-2/ro/train/library.bin b/corpus/generated/ISO-8859-2/ro/train/library.bin new file mode 100644 index 0000000..1afc415 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/library.bin @@ -0,0 +1,3 @@ +Biblioteca lini演it + +姷 biblioteca de cartier se aud pagini 螽toarse 螽cet. Cititorii caut c綖i despre istorie, 演iin 槐 c緄綟orii. Bibliotecara preg綟e演e o expoziie pentru elevi. diff --git a/corpus/generated/ISO-8859-2/ro/train/market.bin b/corpus/generated/ISO-8859-2/ro/train/market.bin new file mode 100644 index 0000000..15a0338 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/market.bin @@ -0,0 +1,3 @@ +Piaa de s滵b綟 + +Piaa umple centrul cu mirosuri 槐 culori. Fermierii aduc mere, miere, br滱z 槐 p毃ne proasp綟. Cump綖綟orii aleg legume 槐 schimb reete de familie. diff --git a/corpus/generated/ISO-8859-2/ro/train/workshop.bin b/corpus/generated/ISO-8859-2/ro/train/workshop.bin new file mode 100644 index 0000000..132f010 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/workshop.bin @@ -0,0 +1,3 @@ +Atelier digital + +O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. diff --git a/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin b/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin new file mode 100644 index 0000000..c125be0 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin @@ -0,0 +1,3 @@ +Plimbare l滱g r漉 + +Duminic, prietenii au mers spre un r漉 lini演it din afara ora滾lui. Au luat o hart, ceai cald 槐 un aparat foto. Dup plimbare s-au odihnit pe mal. diff --git a/corpus/generated/ISO-8859-5/ru/test/community-garden.bin b/corpus/generated/ISO-8859-5/ru/test/community-garden.bin new file mode 100644 index 0000000..1ed1849 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/test/community-garden.bin @@ -0,0 +1,3 @@ +憾橞 慛 + +嗟甀訹 砵涫甀 葮寋殏 葠黽婝 摵冓醐 婞睔 葠慛婥訹 稹涴, 赹觛椷 嗍詅婑 婭雎祑. 椎畽 慡殏倯 櫡稃 瑱桉崱稃, 摶慖婥 婝豇砨雸訹摐 硞笪蜦 楙婭趀 葠訹眓碪 閛摵殎崶. diff --git a/corpus/generated/ISO-8859-5/ru/train/city-morning.bin b/corpus/generated/ISO-8859-5/ru/train/city-morning.bin new file mode 100644 index 0000000..e329ef5 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/city-morning.bin @@ -0,0 +1,3 @@ +轎頏 豇頏婭 + +喂頏 葮睙錈俷皸 葠摵氪殎棰. 請 睙瑱棰砬凎 葠攄趉鍒摬 葎摽倗嵑, 栜詅誾厞 睭筄遻俷 婞淀, 蒎隓鄳 倷瘑栜摠 祒槧孲 楅 嗝骭飹. 魂婥 諑瑱鍒 摶痶橩楉 痶摦笪匎 葾倎 楅 棰祒 婭椷. diff --git a/corpus/generated/ISO-8859-5/ru/train/library.bin b/corpus/generated/ISO-8859-5/ru/train/library.bin new file mode 100644 index 0000000..a05a46f --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/library.bin @@ -0,0 +1,3 @@ +檗殔楥厞 朓桉崿甀硞 + + 朓桉崿甀祰 畽槧 髳觛摵檺 摵閛楉緦. п瑱甀訹 祒朓閛鍒 竤寔 痶 嵎瘑雸, 楅蜮 蜄甀髳摵眧櫇. 醣瘔虡楉稃 豇瘑秠 祒摵倷筎, 睭眙覣鍒 楅 砨葮睙 葠嗍衄鍒 魧睒徾嵃倱 楅殽 楯笣鄳 嗝甀雸倯. diff --git a/corpus/generated/ISO-8859-5/ru/train/market.bin b/corpus/generated/ISO-8859-5/ru/train/market.bin new file mode 100644 index 0000000..3ff323b --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/market.bin @@ -0,0 +1,3 @@ +擔殎楑 櫡嗝靳 + +請 葾碕俶 葮碚痽嵕 睙殎楑 櫡嗝靳. 齣靸淀 葮寋欳訹 擿詅稃, 塱, 摠 愻欶崷 槮桻. 豭筎葎甀訹 摳倷楉眓鍒 葮痽蜮碲, 晇慖媎鍒 葮痽倷禛嗏 豝萶摠眓鍒 嵋甀雎摴鄳 婝嗝魦寊 雎窳蜋. diff --git a/corpus/generated/ISO-8859-5/ru/train/workshop.bin b/corpus/generated/ISO-8859-5/ru/train/workshop.bin new file mode 100644 index 0000000..54ff5e6 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/workshop.bin @@ -0,0 +1,3 @@ +檗栻覣 嗝摵淀摍厞 + +調栻貵駗 筊嗝楗 摶赻哢 繌殑瘔睚楋 硞瑱詅 塨赹. 楮豝欼淀 痶摦笪匎 喿楏, 葮痵閛嗈嵎碲 葮痭淀瀁 筊, 雎婕筌睔 豇瘑秠 櫆楋 睧嵎倎崶 婤 葠慖畽甀觛 閛軝錼 砨軠凊瘑. diff --git a/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin b/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin new file mode 100644 index 0000000..1c97742 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ +豭欳婓 雎祰 + + 砨摍雎慖椷 婄蜨斀 睭葮倷嵫嵎 摲睠睟棰 雎祰 豝 豇頏婝. 擂 眢曛 硞飶, 豇黽諑 覣 鬿瘑冔葎閛. 豭摛 婝觝睟 葮痵蜞稃 蜄甀髳摵眙楥嵃 睭婑槱訹 楅 晇雎赶 摛蜼倯 眙諆靲嵒 蜋崽. diff --git a/corpus/generated/ISO-8859-6/ar/test/community-garden.bin b/corpus/generated/ISO-8859-6/ar/test/community-garden.bin new file mode 100644 index 0000000..d2a9d0c --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/test/community-garden.bin @@ -0,0 +1,3 @@ +迍罺 ы俇 + +眒 ы殀捚 ы祛駌 駍鞀堄и 摓 蚡虭 瑳耪优 髳箬 ы譙慳 鞁慲в 槼鳽 駍忯 ы极縏 毲 貣 ы磃вв. diff --git a/corpus/generated/ISO-8859-6/ar/train/city-morning.bin b/corpus/generated/ISO-8859-6/ar/train/city-morning.bin new file mode 100644 index 0000000..50c1ce8 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/city-morning.bin @@ -0,0 +1,3 @@ +桱е ы樕耩 + +弝抏畽 ы樕耩 覤餑 駔慲 ы槻ю 蠍駍褘 耩暷 耩忐 ы磈 ы虰撦в 鴀瑵殲 ы襞и. diff --git a/corpus/generated/ISO-8859-6/ar/train/library.bin b/corpus/generated/ISO-8859-6/ar/train/library.bin new file mode 100644 index 0000000..ddd9db9 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/library.bin @@ -0,0 +1,3 @@ +ы樍岓 ы褘玅 + +糔豖 ы瑵Щ 氰 蒘 ы岆栴 駍駇儰 鴀蚡棝 邋耩 ы樍岓 ы逑а 摓 ё抏и ы槤д ы樉к. diff --git a/corpus/generated/ISO-8859-6/ar/train/market.bin b/corpus/generated/ISO-8859-6/ar/train/market.bin new file mode 100644 index 0000000..67ef9f6 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/market.bin @@ -0,0 +1,3 @@ +軑 ы蚧 + +縏晑 ы樠и湆 ы恛 駍駇赸 駍魟ъ褆 鴀岓ж ы祏и ы髳慳 駍鞀虰砆 ы駋糗. diff --git a/corpus/generated/ISO-8859-6/ar/train/workshop.bin b/corpus/generated/ISO-8859-6/ar/train/workshop.bin new file mode 100644 index 0000000..27001c3 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/workshop.bin @@ -0,0 +1,3 @@ +駌埩 栱樏 + +縕槫 慥罺 殑縠 眊翯 鳵樀俉 鴀捚枘 ы暪栫枔 ы婛挸 鴀蒘 ы槥旄鳽 髳慳 駍笴. diff --git a/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin b/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin new file mode 100644 index 0000000..2d416cb --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin @@ -0,0 +1,3 @@ +敊韎 饜 ы緗 + +俜 ы譏眄Щ 恮縢 髫② 蚡拰 鳺婰 虮 ы緗 呬 к忴湆 饜 郰駌 ы槿Щ. diff --git a/corpus/generated/ISO-8859-7/el/test/community-garden.bin b/corpus/generated/ISO-8859-7/el/test/community-garden.bin new file mode 100644 index 0000000..2e8196c --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/test/community-garden.bin @@ -0,0 +1,3 @@ + 膬澼濻濋 縤豵 + +矨 蜲裎瀍殣 椻摿摿 擩 塤樥 斠錞 貘嫇澼 縤豵, 酃 謯毈橎 翦蜤媺澞摿 謮罅罃麧 罼 毉 毉膬蜙翴毇 豵轚緮罍 酃 皭轗. diff --git a/corpus/generated/ISO-8859-7/el/train/city-morning.bin b/corpus/generated/ISO-8859-7/el/train/city-morning.bin new file mode 100644 index 0000000..f58abba --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/city-morning.bin @@ -0,0 +1,3 @@ +哢 黀賝 蹞錣 + + 蹞錣 隰謺 摙蜡, 酃 罼巕 摿檽蜧罍 罼 毉 塥黓豵 謥齌徾臩罍 酃 錼獿鼫葐. diff --git a/corpus/generated/ISO-8859-7/el/train/library.bin b/corpus/generated/ISO-8859-7/el/train/library.bin new file mode 100644 index 0000000..6724218 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/library.bin @@ -0,0 +1,3 @@ +幛蘙 睾碲濈魶聬 + +矨 摿摦薞黀殣 鸂籦瀦 睾碲葐 澞鐐麌摥 罼 歑澞轘懞 罼 睾碲濈鳿縔齌濿 碨賥 鐐蘪 憼鳿轕. diff --git a/corpus/generated/ISO-8859-7/el/train/market.bin b/corpus/generated/ISO-8859-7/el/train/market.bin new file mode 100644 index 0000000..40995f7 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/market.bin @@ -0,0 +1,3 @@ +貤瞁敳橎鏻聬 摦瀎 + +矨 摦嬿醲 豵纆燽 矙懃, 徾錁, 瓕燽酃 罼 檁夒僸縔 黀賝 謻敳橀. diff --git a/corpus/generated/ISO-8859-7/el/train/workshop.bin b/corpus/generated/ISO-8859-7/el/train/workshop.bin new file mode 100644 index 0000000..4f44381 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/workshop.bin @@ -0,0 +1,3 @@ +崱灖摓 殥蝁黀碖濈 + +欥 擩膲 瀌塤 殦毉幪緪 檷誽 斀蘦橀瀦, 槶楯灚 鐐 蕘僸罼 罼 蜤媺樥 罼魨鵘 耪蜉樇. diff --git a/corpus/generated/ISO-8859-7/el/validation/river-trip.bin b/corpus/generated/ISO-8859-7/el/validation/river-trip.bin new file mode 100644 index 0000000..dcea70f --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/validation/river-trip.bin @@ -0,0 +1,3 @@ +髑儊瀌 黀 豵轗擩 + +矨 廲錝 謰鼁 攩嚪 罼 緪黀 鏵塕 罼 鍷膬蘘嫇鐋罼 魠謻 黀 豵轗擩. diff --git a/corpus/generated/ISO-8859-8/he/test/community-garden.bin b/corpus/generated/ISO-8859-8/he/test/community-garden.bin new file mode 100644 index 0000000..28d54b2 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/test/community-garden.bin @@ -0,0 +1,3 @@ +鳱潞 噂僸檡澬 + +嘵鋺澼 勱錼 踤 鼞灛 懧潞 澮毿, 僸懠澼 灖澮 擯澼 槼霘鐋漦 霘灚 僬 噈鍡澼 摠 摎. diff --git a/corpus/generated/ISO-8859-8/he/train/city-morning.bin b/corpus/generated/ISO-8859-8/he/train/city-morning.bin new file mode 100644 index 0000000..90ce4a0 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/city-morning.bin @@ -0,0 +1,3 @@ +摎蠲 摥澮 + +劋澮 霝蠈矘 憵, 朄 噂醵 贄賟 樈蹠澼 鍡錁 憵樘槦殥澼 噌僓歑澼. diff --git a/corpus/generated/ISO-8859-8/he/train/library.bin b/corpus/generated/ISO-8859-8/he/train/library.bin new file mode 100644 index 0000000..d4abd33 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/library.bin @@ -0,0 +1,3 @@ +儊鐖澭 嘵玁 + +灚鸆澼 鍡鐒澼 嚪鼞 僸鼨氂毈 槴蜚 槼嚪斸澬 蠈翬 檡斁殫澼 憼谾 霠氂漦. diff --git a/corpus/generated/ISO-8859-8/he/train/market.bin b/corpus/generated/ISO-8859-8/he/train/market.bin new file mode 100644 index 0000000..d4e1672 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/market.bin @@ -0,0 +1,3 @@ +毿 朄 + +踮憵澼 鍷鋿澼 懞 儑, 蝁, 鏻鷿 槻摋謱 摠澣 劋鍷鼥. diff --git a/corpus/generated/ISO-8859-8/he/train/workshop.bin b/corpus/generated/ISO-8859-8/he/train/workshop.bin new file mode 100644 index 0000000..9120483 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/workshop.bin @@ -0,0 +1,3 @@ +鼖謱 蜒睾鳻澬 + +欈漦 玁 鎀濈 鍏鼞 斁樉歖樖, 摎裱 僬 噂樍 槶漦 歖氂澼 暟氂澼. diff --git a/corpus/generated/ISO-8859-8/he/validation/river-trip.bin b/corpus/generated/ISO-8859-8/he/validation/river-trip.bin new file mode 100644 index 0000000..f2a8f51 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/validation/river-trip.bin @@ -0,0 +1,3 @@ +鴇樦 鳭 僽噌 + +儃暟澼 檉谾 隮 漦 賝, 僶錼 摋譊 歑谾 擩 僽噌. diff --git a/corpus/generated/ISO-8859-9/tr/test/community-garden.bin b/corpus/generated/ISO-8859-9/tr/test/community-garden.bin new file mode 100644 index 0000000..831e0ea --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/test/community-garden.bin @@ -0,0 +1,3 @@ +Mahalle bah蔒si + +Komular bo avluyu yeil bir bah蔒ye 蔒virdi, 蔞cuklar renkli tabelalar yapt ve aileler bitkileri her hafta suluyor. diff --git a/corpus/generated/ISO-8859-9/tr/train/city-morning.bin b/corpus/generated/ISO-8859-9/tr/train/city-morning.bin new file mode 100644 index 0000000..e8458f6 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/city-morning.bin @@ -0,0 +1,3 @@ +溳hirde sabah + +溳hir yava蓷 uyanr, kafeler a踦lr ve insanlar ilk otobsleri bekler. diff --git a/corpus/generated/ISO-8859-9/tr/train/library.bin b/corpus/generated/ISO-8859-9/tr/train/library.bin new file mode 100644 index 0000000..20c465c --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/library.bin @@ -0,0 +1,3 @@ +Sessiz ktphane + +Okurlar tarih ve bilim kitaplar ararken ktphaneci 瓘rencilere yardmc olur. diff --git a/corpus/generated/ISO-8859-9/tr/train/market.bin b/corpus/generated/ISO-8859-9/tr/train/market.bin new file mode 100644 index 0000000..0856260 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/market.bin @@ -0,0 +1,3 @@ +Cumartesi pazar + +パft蓾ler kalabalk meydanda taze ekmek, elma, peynir ve bal satar. diff --git a/corpus/generated/ISO-8859-9/tr/train/workshop.bin b/corpus/generated/ISO-8859-9/tr/train/workshop.bin new file mode 100644 index 0000000..844ca1e --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/workshop.bin @@ -0,0 +1,3 @@ +Dijital at闤ye + +K踙k bir ekip mze rehberi hazrlar, kodu denetler ve a踦k a踦klamalar yazar. diff --git a/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin b/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin new file mode 100644 index 0000000..2568000 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Nehir gezisi + +Arkadalar harita ve scak 蓷y alp ormanda yrd, sonra nehir kysnda dinlendi. diff --git a/corpus/generated/KOI8-R/ru/test/community-garden.bin b/corpus/generated/KOI8-R/ru/test/community-garden.bin new file mode 100644 index 0000000..17de462 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/test/community-garden.bin @@ -0,0 +1,3 @@ +屩楦 蚆 + +鹺埧旻 谹籣埧 倷屺攫 俵眒鰍 衲轄椋 儷玾 俵蚆馨旻 譙籥, 硠旼挎 迍昑囁 霰疻趹. 靻埩 蚅攫賻 旂侂 埡蟯弚侂, 衃蚥馨 鰍ж豜疶旻衯 佹秺梏 恓霰抸 俵旻袬婥 畛衲曬奷. diff --git a/corpus/generated/KOI8-R/ru/train/city-morning.bin b/corpus/generated/KOI8-R/ru/train/city-morning.bin new file mode 100644 index 0000000..2bf62de --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/city-morning.bin @@ -0,0 +1,3 @@ +灄眐 ж眐霰 + +覤眐 倷珃棞豁娸 俵衲瓚曬恘. 轀 珃埡恘訬賺 俵朒曶殮衭 釓衶輿奿, 謬昑痸蹊 珆刳椕豁 儷籤, 陓眢棸 避婘謬袃 趿狦晟 恔 芩砢眙. 嚓馨 瘃埡殮 衃狟暙恌 狟袀秺謊 倢蹉 恔 恘趿 霰挎. diff --git a/corpus/generated/KOI8-R/ru/train/library.bin b/corpus/generated/KOI8-R/ru/train/library.bin new file mode 100644 index 0000000..5e2b1e4 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/library.bin @@ -0,0 +1,3 @@ +臗庌恄蹊 薺蟯奻埧佹 + + 薺蟯奻埧侜 埩 袹旼衲晟 衲畛恌贊. 尕轂攫 趿薺畛殮 冼囟 狟 孖婘疶, 恔梖 俷埧袹衲訰捑. 鯔婞桫恌侂 ж婘豽 趿衲避劼, 珆覂瓾殮 恔 豜倷珃 俵迍Щ殮 袷玶媓夼趨 恔彸 挍笥棸 芩埧疶賻. diff --git a/corpus/generated/KOI8-R/ru/train/market.bin b/corpus/generated/KOI8-R/ru/train/market.bin new file mode 100644 index 0000000..689aa9b --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/market.bin @@ -0,0 +1,3 @@ +擽曬恮 旂芩皊 + +轀 倢疪謐 倷珂狦尕 珃曬恮 旂芩皊. 磎眝籤 倷屺蘿旻 捔昑侂, 礿, 袃 衿纖圴 旼. 蟴劼釓埧旻 衵避恌袬殮 倷狦梖婬, 藍蚥齣殮 倷狦避藪虭 睋倅袃袬殮 妅埧疻蚔棸 鰍芩褁吇 疻藷俶. diff --git a/corpus/generated/KOI8-R/ru/train/workshop.bin b/corpus/generated/KOI8-R/ru/train/workshop.bin new file mode 100644 index 0000000..9133544 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/workshop.bin @@ -0,0 +1,3 @@ +臗蟠瓾 芩衲籤蚎蹊 + +醚蟠杻衖 冾芩恅 衃矬薜 嗌攣婞玴挃 佹埡昑 邯硠. 韎睋庉籤 狟袀秺謊 芶怹, 倷珌畛迋孖婬 倷瓬籤挴 冾, 疻鐘刱玾 ж婘豽 晊挃 玿孖蹉奷 騷 俵蚥埩埧旼 畛硩棼 豜祴輾婘. diff --git a/corpus/generated/KOI8-R/ru/validation/river-trip.bin b/corpus/generated/KOI8-R/ru/validation/river-trip.bin new file mode 100644 index 0000000..cb9b8fa --- /dev/null +++ b/corpus/generated/KOI8-R/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ +蟴蘿騰 疻侜 + + 豜蚎疻蚥挎 黨殑寍 珆倷避妀孖 衄玹珅恘 疻侜 睋 ж眐鰍. 懫 跁捈 佹眕, ж眒瘃 瓾 に婘蹈釓畛. 蟴蚝 鰍昒珅 倷珌梋侂 俷埧袹衲覂恄夼 珆囁賻 恔 藍疻м 蚝殏賻 覂畷眛圮 俶卍. diff --git a/corpus/generated/Shift_JIS/ja/test/community-garden.bin b/corpus/generated/Shift_JIS/ja/test/community-garden.bin new file mode 100644 index 0000000..e110411 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/test/community-garden.bin @@ -0,0 +1,3 @@ +n + +lAABqFNDATAB diff --git a/corpus/generated/Shift_JIS/ja/train/city-morning.bin b/corpus/generated/Shift_JIS/ja/train/city-morning.bin new file mode 100644 index 0000000..f31111e --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/city-morning.bin @@ -0,0 +1,3 @@ +X + +XoBwlWAiXJAoXnB diff --git a/corpus/generated/Shift_JIS/ja/train/library.bin b/corpus/generated/Shift_JIS/ja/train/library.bin new file mode 100644 index 0000000..258556e --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/library.bin @@ -0,0 +1,3 @@ +} + +n}{y[WBjws{TAiw`B diff --git a/corpus/generated/Shift_JIS/ja/train/market.bin b/corpus/generated/Shift_JIS/ja/train/market.bin new file mode 100644 index 0000000..a71af01 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/market.bin @@ -0,0 +1,3 @@ +yjs + +LsVNAAIApBq_GbB diff --git a/corpus/generated/Shift_JIS/ja/train/workshop.bin b/corpus/generated/Shift_JIS/ja/train/workshop.bin new file mode 100644 index 0000000..8f8126c --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/workshop.bin @@ -0,0 +1,3 @@ +fW^H[ + +`[BvAJR[hmFAWB diff --git a/corpus/generated/Shift_JIS/ja/validation/river-trip.bin b/corpus/generated/Shift_JIS/ja/validation/river-trip.bin new file mode 100644 index 0000000..2724ab8 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/validation/river-trip.bin @@ -0,0 +1,3 @@ +s + +Fln}XBUAx[B diff --git a/corpus/generated/index.json b/corpus/generated/index.json index cc7e572..7885b13 100644 --- a/corpus/generated/index.json +++ b/corpus/generated/index.json @@ -1,4 +1,76 @@ [ + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "test", + "document": "community-garden", + "path": "Big5/zh/test/community-garden.bin", + "bytes": 107, + "highBytes": 79, + "uniqueHighBytes": 39, + "sha256": "4602b4204cff37f57af8cc21fa4751c6e33e56c4078927b211a14e00657fc8bb" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "city-morning", + "path": "Big5/zh/train/city-morning.bin", + "bytes": 97, + "highBytes": 70, + "uniqueHighBytes": 33, + "sha256": "f141f927f2a8b8e6540a117ccee6678e8f4db6f81cd5c0ebad699de8e9d05b7c" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "library", + "path": "Big5/zh/train/library.bin", + "bytes": 101, + "highBytes": 82, + "uniqueHighBytes": 40, + "sha256": "b20e84ac072e488aa6114e2ad862b39d127774244f26304e9fdf9716c9937ac9" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "market", + "path": "Big5/zh/train/market.bin", + "bytes": 99, + "highBytes": 74, + "uniqueHighBytes": 35, + "sha256": "3fa6ec765b1c5ca517ec35b276defa25d9505cfb06d89f5b8bc09079165ac75b" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "workshop", + "path": "Big5/zh/train/workshop.bin", + "bytes": 113, + "highBytes": 83, + "uniqueHighBytes": 42, + "sha256": "0673960c008a34a445c94870c643cf40e55beaec03c290bab0dcbaa86c52b128" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "validation", + "document": "river-trip", + "path": "Big5/zh/validation/river-trip.bin", + "bytes": 93, + "highBytes": 71, + "uniqueHighBytes": 39, + "sha256": "d23ba0d2a924cfc8881a734d32b20c591791610270bb5fe0c5b44d2aa8257827" + }, { "encoding": "CP850", "iconv": "CP850", @@ -359,6 +431,222 @@ "uniqueHighBytes": 56, "sha256": "f6399d3705fc3e6c2a6427516ce6c8b9585a0a1e035b53a9efb7ef0811dd6a00" }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "test", + "document": "community-garden", + "path": "EUC-JP/ja/test/community-garden.bin", + "bytes": 129, + "highBytes": 126, + "uniqueHighBytes": 48, + "sha256": "6654d80ff3f372a76c64cff40e76361bd50fddfad6745d9a39f0b265bebbec27" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "city-morning", + "path": "EUC-JP/ja/train/city-morning.bin", + "bytes": 117, + "highBytes": 114, + "uniqueHighBytes": 44, + "sha256": "5c1bc169f52cbbc621feafdd6956fa4be9ed66211a385da85e1364ad0cb9c0fe" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "library", + "path": "EUC-JP/ja/train/library.bin", + "bytes": 125, + "highBytes": 122, + "uniqueHighBytes": 49, + "sha256": "9b8da9e80124276f5d3a46c82fac0bb35489164b972e529f0ee03df6568e1549" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "market", + "path": "EUC-JP/ja/train/market.bin", + "bytes": 115, + "highBytes": 112, + "uniqueHighBytes": 45, + "sha256": "861116732268050cb5c7bdc19c0ceff1aa1fb80a99f172a95f0926a5aaafd9a2" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "workshop", + "path": "EUC-JP/ja/train/workshop.bin", + "bytes": 131, + "highBytes": 128, + "uniqueHighBytes": 50, + "sha256": "1e952f775f77207dbf5faed31865416068b03f82009e6e86b0517cf564fc1d68" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "validation", + "document": "river-trip", + "path": "EUC-JP/ja/validation/river-trip.bin", + "bytes": 119, + "highBytes": 116, + "uniqueHighBytes": 43, + "sha256": "b09785c8f0852a1239d2283d7fc8d531a85fa57443894bd9ec14a8f9cd364103" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "test", + "document": "community-garden", + "path": "EUC-KR/ko/test/community-garden.bin", + "bytes": 166, + "highBytes": 138, + "uniqueHighBytes": 49, + "sha256": "15d58dc224354498565689b014ef393a0c6d9bcf0d536235997785c0c864e336" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "city-morning", + "path": "EUC-KR/ko/train/city-morning.bin", + "bytes": 152, + "highBytes": 126, + "uniqueHighBytes": 45, + "sha256": "54eadc94abafcb5446f54f61029b48a6f7d080fbd3f80219416537fbcf469ee9" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "library", + "path": "EUC-KR/ko/train/library.bin", + "bytes": 157, + "highBytes": 130, + "uniqueHighBytes": 48, + "sha256": "1aca0feb504c558b985ec91d05a54eb6c61f06399cb276e82e08551d4dffbeac" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "market", + "path": "EUC-KR/ko/train/market.bin", + "bytes": 155, + "highBytes": 128, + "uniqueHighBytes": 54, + "sha256": "2de80f2cde96c8a97d8cca91d0d85e47ec37d8362458829797a2576c9f11a880" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "workshop", + "path": "EUC-KR/ko/train/workshop.bin", + "bytes": 176, + "highBytes": 148, + "uniqueHighBytes": 53, + "sha256": "e9f1924b0b9e410049817e0355f167d012b3a1b24ee91074acd08306102a65a6" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "validation", + "document": "river-trip", + "path": "EUC-KR/ko/validation/river-trip.bin", + "bytes": 154, + "highBytes": 126, + "uniqueHighBytes": 56, + "sha256": "f6399d3705fc3e6c2a6427516ce6c8b9585a0a1e035b53a9efb7ef0811dd6a00" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "test", + "document": "community-garden", + "path": "GB18030/zh/test/community-garden.bin", + "bytes": 107, + "highBytes": 96, + "uniqueHighBytes": 55, + "sha256": "0b2447d40ee94fc0f54193e4a0657fb7d524db9a1456dfbf161db4e2d66a238f" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "city-morning", + "path": "GB18030/zh/train/city-morning.bin", + "bytes": 97, + "highBytes": 89, + "uniqueHighBytes": 52, + "sha256": "f831e80d10c0cc072648d19ea815faa0ece4ed763cea6925b18f38e487d8b2ea" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "library", + "path": "GB18030/zh/train/library.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 51, + "sha256": "1e0c934cfe5346ae0e1dd54f7355e8de90fcb47013d9edbd842db64f510abad8" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "market", + "path": "GB18030/zh/train/market.bin", + "bytes": 99, + "highBytes": 89, + "uniqueHighBytes": 56, + "sha256": "4b7df06fad0d5ad72c7c3aea20f843409d5176839813ade77ed773537863918f" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "workshop", + "path": "GB18030/zh/train/workshop.bin", + "bytes": 113, + "highBytes": 100, + "uniqueHighBytes": 61, + "sha256": "3209cbd7a69500b9825d7226b0065c9a7ae6206b113608c94b7108a89dca4a11" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "validation", + "document": "river-trip", + "path": "GB18030/zh/validation/river-trip.bin", + "bytes": 93, + "highBytes": 85, + "uniqueHighBytes": 48, + "sha256": "2d4ab1b56be3899eb86a28c8f46ccecb392e4e39ecdba918ae0e4cc2ed4f94ab" + }, { "encoding": "IBM855", "iconv": "CP855", @@ -576,940 +864,4180 @@ "sha256": "7f39f7fb36cbabea98997b9c390d71e0742ef2fad5c7deca432ee6444e164c49" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "test", "document": "community-garden", - "path": "ISO-8859-10/is/test/community-garden.bin", - "bytes": 174, - "highBytes": 19, - "uniqueHighBytes": 6, - "sha256": "11bbdafb4917a3935f98940af20e169c4a7eef5642d4a159297b44cd8b565d78" + "path": "ISO-8859-1/da/test/community-garden.bin", + "bytes": 123, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "d742c49869d5f97ff0dc9bed58a7e819cf5a9f3cde419ff785f2201591fa7991" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "city-morning", - "path": "ISO-8859-10/is/train/city-morning.bin", - "bytes": 161, - "highBytes": 11, - "uniqueHighBytes": 8, - "sha256": "3c8a3358a8195de52877343b34b5a71606732b3df110594862cebeb9c76aa3cf" + "path": "ISO-8859-1/da/train/city-morning.bin", + "bytes": 89, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "75320357a51c6716244970d7c802f42af40a6a65fa1602e4f38e239e16ccd887" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "library", - "path": "ISO-8859-10/is/train/library.bin", - "bytes": 166, - "highBytes": 18, - "uniqueHighBytes": 7, - "sha256": "a78d39ae9c97ed64273f11792d1e5a353080658d28ab6a85cc248f28b91d18a0" + "path": "ISO-8859-1/da/train/library.bin", + "bytes": 105, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "4b0f38943f7eb950af8b04b16ac6ce581c48f70e933f66bd03725842d9e1bd58" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "market", - "path": "ISO-8859-10/is/train/market.bin", - "bytes": 168, - "highBytes": 10, - "uniqueHighBytes": 5, - "sha256": "c15c77b0cb233941ca1b038ddd269d4d866e4b5b48d61989220169b6cd851c9b" + "path": "ISO-8859-1/da/train/market.bin", + "bytes": 84, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "61d9285e5b354f3cffb320f3bb62af7d4ce86fe34ba0e5067d5bc26aa9cd010c" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "workshop", - "path": "ISO-8859-10/is/train/workshop.bin", - "bytes": 167, - "highBytes": 15, - "uniqueHighBytes": 6, - "sha256": "1bdd4e4aa76ecbd47f82533ec096fe3ee5c637debdefad8bc10251e8f3867af0" + "path": "ISO-8859-1/da/train/workshop.bin", + "bytes": 103, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "aca6cded29cf164fc9445de75292a45f007c971ea9407d7ef0d794413a7cd975" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "validation", "document": "river-trip", - "path": "ISO-8859-10/is/validation/river-trip.bin", - "bytes": 168, - "highBytes": 17, - "uniqueHighBytes": 9, - "sha256": "ce28fe8d7fe2ee466f03495c48f00a6da6fa7d3df530c7052873c444e1dd6f59" + "path": "ISO-8859-1/da/validation/river-trip.bin", + "bytes": 97, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "0edb6dd9323283e842dea2112213cf7b86608b257a36cfd318b8c71b3c01c8e0" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "de", "split": "test", "document": "community-garden", - "path": "ISO-8859-13/lt/test/community-garden.bin", - "bytes": 177, - "highBytes": 17, - "uniqueHighBytes": 6, - "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + "path": "ISO-8859-1/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "de", "split": "train", "document": "city-morning", - "path": "ISO-8859-13/lt/train/city-morning.bin", - "bytes": 156, - "highBytes": 11, + "path": "ISO-8859-1/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, "uniqueHighBytes": 4, - "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "de", "split": "train", "document": "library", - "path": "ISO-8859-13/lt/train/library.bin", - "bytes": 163, - "highBytes": 8, - "uniqueHighBytes": 5, - "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" + "path": "ISO-8859-1/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "de", "split": "train", "document": "market", - "path": "ISO-8859-13/lt/train/market.bin", - "bytes": 171, - "highBytes": 12, - "uniqueHighBytes": 8, - "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" + "path": "ISO-8859-1/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "de", "split": "train", "document": "workshop", - "path": "ISO-8859-13/lt/train/workshop.bin", - "bytes": 170, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" + "path": "ISO-8859-1/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "de", "split": "validation", "document": "river-trip", - "path": "ISO-8859-13/lt/validation/river-trip.bin", - "bytes": 165, - "highBytes": 16, - "uniqueHighBytes": 5, - "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "test", - "document": "community-garden", - "path": "ISO-8859-14/cy/test/community-garden.bin", - "bytes": 194, + "path": "ISO-8859-1/de/validation/river-trip.bin", + "bytes": 192, "highBytes": 3, - "uniqueHighBytes": 3, - "sha256": "11475514e6d6789bb154d27be8a381507f5c136b1ec88219820a77d3f11a352a" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "city-morning", - "path": "ISO-8859-14/cy/train/city-morning.bin", - "bytes": 195, - "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "5310db3607ef3fcadf53799acdd8836356af51cf840f1f391c4c31acdb3188ce" + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "library", - "path": "ISO-8859-14/cy/train/library.bin", - "bytes": 194, + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-1/en/test/community-garden.bin", + "bytes": 136, "highBytes": 1, "uniqueHighBytes": 1, - "sha256": "c8445d5b31a86bb8a0f295792afe5d4632e9c30182460cb0bc590c6fa6156afb" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "market", - "path": "ISO-8859-14/cy/train/market.bin", - "bytes": 183, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "c2b02e1a6e22d7f21483955ebf4d00e4b0500c3ee91dcb67e67cb07f015932e3" + "sha256": "8574d38336d96e1a48fd42b012b3924bc36ee57550bc58a28ed6d5c17d514c1d" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", "split": "train", - "document": "workshop", - "path": "ISO-8859-14/cy/train/workshop.bin", - "bytes": 173, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "2fe7199c4f96f4c5941e8f6b95aa44ae6f8d85863552550993b3b019acdeee6f" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "validation", - "document": "river-trip", - "path": "ISO-8859-14/cy/validation/river-trip.bin", - "bytes": 161, + "document": "city-morning", + "path": "ISO-8859-1/en/train/city-morning.bin", + "bytes": 108, "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "318d863b3f7efa4a716e75ba4cd138fe9b9c337fb6f88eafe7ee1c671629e054" - }, - { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", - "language": "de", - "split": "test", - "document": "community-garden", - "path": "ISO-8859-15/de/test/community-garden.bin", - "bytes": 220, - "highBytes": 12, - "uniqueHighBytes": 3, - "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" - }, - { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", - "language": "de", - "split": "train", - "document": "city-morning", - "path": "ISO-8859-15/de/train/city-morning.bin", - "bytes": 208, - "highBytes": 6, - "uniqueHighBytes": 4, - "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + "sha256": "ff22d70d940981c1ea141576fb34c74d841dbf2e2140d80ee55f51dd63bd9ef0" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", "split": "train", "document": "library", - "path": "ISO-8859-15/de/train/library.bin", - "bytes": 204, - "highBytes": 5, + "path": "ISO-8859-1/en/train/library.bin", + "bytes": 117, + "highBytes": 3, "uniqueHighBytes": 2, - "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + "sha256": "12ea1dc5452af3ca116753189e184ef0a10645eee17f33211dc5f7d182318f38" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", "split": "train", "document": "market", - "path": "ISO-8859-15/de/train/market.bin", - "bytes": 177, - "highBytes": 4, + "path": "ISO-8859-1/en/train/market.bin", + "bytes": 100, + "highBytes": 3, "uniqueHighBytes": 3, - "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + "sha256": "cbaa356df57d8557b3d2c3ca4d6ec5845077db27583e102aebdaedcacf9f2aee" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", "split": "train", "document": "workshop", - "path": "ISO-8859-15/de/train/workshop.bin", - "bytes": 188, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + "path": "ISO-8859-1/en/train/workshop.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "157d773566f7cd87f7bafb87780ba98dcc090adbdaaea2c03483e7a3b6a93a70" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/de/validation/river-trip.bin", - "bytes": 192, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + "path": "ISO-8859-1/en/validation/river-trip.bin", + "bytes": 106, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c4e405fd6466194b2ddc91f07f8bcb624c904531d6d96f37e35d04847c6abbb9" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/es/test/community-garden.bin", + "path": "ISO-8859-1/es/test/community-garden.bin", "bytes": 203, "highBytes": 6, "uniqueHighBytes": 3, "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/es/train/city-morning.bin", + "path": "ISO-8859-1/es/train/city-morning.bin", "bytes": 202, "highBytes": 3, "uniqueHighBytes": 3, "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "library", - "path": "ISO-8859-15/es/train/library.bin", + "path": "ISO-8859-1/es/train/library.bin", "bytes": 202, "highBytes": 2, "uniqueHighBytes": 2, "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "market", - "path": "ISO-8859-15/es/train/market.bin", + "path": "ISO-8859-1/es/train/market.bin", "bytes": 199, "highBytes": 2, "uniqueHighBytes": 2, "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "workshop", - "path": "ISO-8859-15/es/train/workshop.bin", + "path": "ISO-8859-1/es/train/workshop.bin", "bytes": 194, "highBytes": 6, "uniqueHighBytes": 4, "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/es/validation/river-trip.bin", + "path": "ISO-8859-1/es/validation/river-trip.bin", "bytes": 184, "highBytes": 6, "uniqueHighBytes": 3, "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/fr/test/community-garden.bin", + "path": "ISO-8859-1/fr/test/community-garden.bin", "bytes": 231, "highBytes": 9, "uniqueHighBytes": 1, "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/fr/train/city-morning.bin", + "path": "ISO-8859-1/fr/train/city-morning.bin", "bytes": 212, "highBytes": 5, "uniqueHighBytes": 2, "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "library", - "path": "ISO-8859-15/fr/train/library.bin", + "path": "ISO-8859-1/fr/train/library.bin", "bytes": 226, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "market", - "path": "ISO-8859-15/fr/train/market.bin", + "path": "ISO-8859-1/fr/train/market.bin", "bytes": 223, "highBytes": 3, "uniqueHighBytes": 1, "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "workshop", - "path": "ISO-8859-15/fr/train/workshop.bin", + "path": "ISO-8859-1/fr/train/workshop.bin", "bytes": 215, "highBytes": 8, "uniqueHighBytes": 1, "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/fr/validation/river-trip.bin", + "path": "ISO-8859-1/fr/validation/river-trip.bin", "bytes": 228, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "test", "document": "community-garden", - "path": "ISO-8859-16/ro/test/community-garden.bin", - "bytes": 206, - "highBytes": 13, - "uniqueHighBytes": 5, - "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + "path": "ISO-8859-1/it/test/community-garden.bin", + "bytes": 161, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "53bb7b070795cd69fba5bcc79a4958b2ac5bb23677ebdac99c80623f96d501dd" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "city-morning", - "path": "ISO-8859-16/ro/train/city-morning.bin", - "bytes": 175, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + "path": "ISO-8859-1/it/train/city-morning.bin", + "bytes": 101, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "7c59ad9c3c9ac0b84c02c4829bffabdc83817989f08836d9ee44afd8a392e43d" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "library", - "path": "ISO-8859-16/ro/train/library.bin", - "bytes": 187, - "highBytes": 17, - "uniqueHighBytes": 5, - "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + "path": "ISO-8859-1/it/train/library.bin", + "bytes": 141, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "bb28d91b3f10f0208886078bf0fafbc85c7e03148069ddf5a9762809f28f8981" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "market", - "path": "ISO-8859-16/ro/train/market.bin", - "bytes": 171, - "highBytes": 17, - "uniqueHighBytes": 4, - "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + "path": "ISO-8859-1/it/train/market.bin", + "bytes": 114, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "40ce81cfc5ec4c5b655c2fb9ba519022a77da2c319950c868bc16c7f69278b11" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "workshop", - "path": "ISO-8859-16/ro/train/workshop.bin", - "bytes": 170, - "highBytes": 6, + "path": "ISO-8859-1/it/train/workshop.bin", + "bytes": 152, + "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + "sha256": "5b5704c6191b4179ee6966a2302452eaaf99c348b34e26bc210879c77def38bc" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "validation", "document": "river-trip", - "path": "ISO-8859-16/ro/validation/river-trip.bin", - "bytes": 170, - "highBytes": 10, - "uniqueHighBytes": 3, - "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + "path": "ISO-8859-1/it/validation/river-trip.bin", + "bytes": 98, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8fdde80f1933a9dfb82cc0181c6770e9947ccf5bfd745b8251ee02f54e6a19a7" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "test", "document": "community-garden", - "path": "ISO-8859-3/mt/test/community-garden.bin", - "bytes": 186, - "highBytes": 13, - "uniqueHighBytes": 4, - "sha256": "d6874f4f630507a619f841cad55a2046be0ee930fb9d450e20bcf0e9525759d5" + "path": "ISO-8859-1/nl/test/community-garden.bin", + "bytes": 153, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "26b1ccb49c443abe7580037a8a836bf99e9d5a5135b1751b77a73e3f61c1a9bf" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "city-morning", - "path": "ISO-8859-3/mt/train/city-morning.bin", - "bytes": 171, - "highBytes": 7, - "uniqueHighBytes": 3, - "sha256": "e40f845da038bcae4f7c6edb026ca823788410f8895cd7ae2765ca4afde49bfb" + "path": "ISO-8859-1/nl/train/city-morning.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "201fbe0db619c5d0d4ccecd723c27c68e0807437531553e8c17ba84cfc4335b2" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "library", - "path": "ISO-8859-3/mt/train/library.bin", - "bytes": 181, - "highBytes": 6, + "path": "ISO-8859-1/nl/train/library.bin", + "bytes": 150, + "highBytes": 3, "uniqueHighBytes": 2, - "sha256": "b16045c76dede00ff236e27cd647ea45e25a98d4cf04530c9baf947dbb1a9cb2" + "sha256": "5d9cdae67a5a7cf6cb319750006ea3cf6afaf379b11c35aa241ef63b203cb17b" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "market", - "path": "ISO-8859-3/mt/train/market.bin", - "bytes": 172, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "be0fbd5745a025d769bb128a55d8c80bb1397590edde4d79d28db1310589af8e" + "path": "ISO-8859-1/nl/train/market.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "15466fb33051e25a441363879415fe1c6ec949d753105ad851a75b2054d8c0da" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "workshop", - "path": "ISO-8859-3/mt/train/workshop.bin", - "bytes": 164, - "highBytes": 15, - "uniqueHighBytes": 5, - "sha256": "433e386a7ccaeaeef6e9fbd1daf1f586cc61788cd13973523c2d679026964420" + "path": "ISO-8859-1/nl/train/workshop.bin", + "bytes": 132, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "4fa51f6d8f9543c9df0acef5a2711a5d805fd18286c1164d786bd8ac715c183b" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "validation", "document": "river-trip", - "path": "ISO-8859-3/mt/validation/river-trip.bin", - "bytes": 176, - "highBytes": 9, - "uniqueHighBytes": 2, - "sha256": "9c73d965fbad70cf830b5a3db04e12c4f0b3bd50ead049f7d6703b5c19cf4eb1" + "path": "ISO-8859-1/nl/validation/river-trip.bin", + "bytes": 113, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "d9448db92c3fe1270814d7798759eca44290f94fd76c0ff9012dc2401b426253" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "test", "document": "community-garden", - "path": "ISO-8859-4/lv/test/community-garden.bin", - "bytes": 174, - "highBytes": 17, - "uniqueHighBytes": 8, - "sha256": "c0382f3b40081b8a5d30ca7d26699a2f9e9eb2e453edac9beb97b4a4a9c3051b" + "path": "ISO-8859-1/no/test/community-garden.bin", + "bytes": 116, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b86618ac6687d3769cca67ec9cb49a0d4d1dea45476592a316ae6eef40d576db" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "city-morning", - "path": "ISO-8859-4/lv/train/city-morning.bin", - "bytes": 163, - "highBytes": 15, - "uniqueHighBytes": 7, - "sha256": "d16cbb8016fdba887337226f08a30e96f2752b657dacbabec3a00b36d5538f60" + "path": "ISO-8859-1/no/train/city-morning.bin", + "bytes": 116, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "445f262b0f2ea6f724271b8486b0cfc861cf4e081e4617db895bc36a33dea2ee" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "library", - "path": "ISO-8859-4/lv/train/library.bin", - "bytes": 161, - "highBytes": 15, - "uniqueHighBytes": 5, - "sha256": "c6757b519e310d300ab70b574b32056a85d7bdf140afd265d5e44101cca573e0" + "path": "ISO-8859-1/no/train/library.bin", + "bytes": 126, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "96c146e3d2b86817371ba6ca5d07361d31519cb0d16fbbecfaf7bba36f0ff2eb" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "market", - "path": "ISO-8859-4/lv/train/market.bin", - "bytes": 170, - "highBytes": 12, - "uniqueHighBytes": 5, - "sha256": "33999e249def18a8469c731d110dbdb9614103b3dce69d8005bc6993b878fc88" + "path": "ISO-8859-1/no/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "a0279dbf28e14a6d9841789f50751ec9cdb09c2743f4ac1c61e72f310e916732" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "workshop", - "path": "ISO-8859-4/lv/train/workshop.bin", - "bytes": 163, - "highBytes": 10, - "uniqueHighBytes": 4, - "sha256": "1429abf14fb2e4144c64dd59657aec2615da7795c92f4e6f5e5f5534ee4be9a0" + "path": "ISO-8859-1/no/train/workshop.bin", + "bytes": 100, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "9c70c92214b0f0ac9000a131773f1ed1756d630037293a759982edcb621e056a" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "validation", "document": "river-trip", - "path": "ISO-8859-4/lv/validation/river-trip.bin", - "bytes": 156, - "highBytes": 14, - "uniqueHighBytes": 4, - "sha256": "282b08484ad284a0a069c2d8113e09bc80de8163fee60568042c538b0541fc3f" + "path": "ISO-8859-1/no/validation/river-trip.bin", + "bytes": 85, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "931a303e54cbfe3ccddf45aa0dd3956be07e4cf16bf1fe5ee0874bd4ad2bab9a" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "test", "document": "community-garden", - "path": "KOI8-U/uk/test/community-garden.bin", - "bytes": 181, - "highBytes": 152, - "uniqueHighBytes": 29, - "sha256": "25d71bdf3678270bf3a539771f8e429faaef3157c5649c0f2edc4ec06e0c80fa" + "path": "ISO-8859-1/pt/test/community-garden.bin", + "bytes": 145, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "a8e56da45e10d94745a21dd36a2721410e1b0d04b5bfee15b2c451a27c8ba62d" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "city-morning", - "path": "KOI8-U/uk/train/city-morning.bin", - "bytes": 205, - "highBytes": 168, - "uniqueHighBytes": 31, - "sha256": "853b422f64ef06d7a64528db49d4eca0f11240852392e51cb2700b1b8b9fd122" + "path": "ISO-8859-1/pt/train/city-morning.bin", + "bytes": 99, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "c9a43be7ac9a1af0be31146446c7cea5142ba97545c304e9fdfef166c0dfabe1" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "library", - "path": "KOI8-U/uk/train/library.bin", - "bytes": 182, - "highBytes": 151, - "uniqueHighBytes": 32, - "sha256": "42455a09e6cafcd694b38198b1f172e44e63774cde6c9e760181334974536fe5" + "path": "ISO-8859-1/pt/train/library.bin", + "bytes": 116, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "634c4f0d24e7f1a8bbf9b1fd68d0fe414376d2efeff0fbfc7637d921709f2a62" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "market", - "path": "KOI8-U/uk/train/market.bin", - "bytes": 199, - "highBytes": 165, - "uniqueHighBytes": 33, - "sha256": "0d8b40da8ad30cc4d6d849f2439f472db8092d622e428747aa55e1fa1797800b" + "path": "ISO-8859-1/pt/train/market.bin", + "bytes": 96, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "79f397f7b2ac2cca51091f006462be7777befb52d36c54237a8cfb0cb20cd924" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "workshop", - "path": "KOI8-U/uk/train/workshop.bin", - "bytes": 189, - "highBytes": 161, - "uniqueHighBytes": 30, - "sha256": "b4b92fa64c4b9252511ed0240a7e1cb565a0b1158f07b50fefb93fab1bd5f0f7" + "path": "ISO-8859-1/pt/train/workshop.bin", + "bytes": 108, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "fbb52707258ca57000887397d2bd8220907fc178ed36b929c62e72bf211848e1" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "validation", "document": "river-trip", - "path": "KOI8-U/uk/validation/river-trip.bin", - "bytes": 195, - "highBytes": 160, - "uniqueHighBytes": 31, - "sha256": "1b015aa68443a92670b3e73e37fa96d70d1e2dea1a9ed736bebfcf077bde9dfe" + "path": "ISO-8859-1/pt/validation/river-trip.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "db583dead964fc7fe8a257ebd69f9dd7388c89384bac132955a76b02150c753f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "test", "document": "community-garden", - "path": "macintosh/de/test/community-garden.bin", - "bytes": 220, - "highBytes": 12, + "path": "ISO-8859-1/sv/test/community-garden.bin", + "bytes": 138, + "highBytes": 8, "uniqueHighBytes": 3, - "sha256": "9a35ac96816b79920d3076818a106719d95169d017b89832ff735652fdbc45d5" + "sha256": "7229bf29e0adc2b37cdb6663bb696e500e13000633665f0f226915f5f74c4853" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "city-morning", - "path": "macintosh/de/train/city-morning.bin", - "bytes": 208, - "highBytes": 6, + "path": "ISO-8859-1/sv/train/city-morning.bin", + "bytes": 101, + "highBytes": 7, "uniqueHighBytes": 4, - "sha256": "d826a59c9fe0ea77efe5554f688a96a25ba31edd3b502177f88b229237bfafff" + "sha256": "5ac24bf0fe3bfb1d64aa066ff78e3af67bcf0b4a6f8d142fefeff0b8ad4ae72c" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "library", - "path": "macintosh/de/train/library.bin", - "bytes": 204, - "highBytes": 5, + "path": "ISO-8859-1/sv/train/library.bin", + "bytes": 108, + "highBytes": 4, "uniqueHighBytes": 2, - "sha256": "29829a1ac13b7f7ad2ecd8abd7d0f2454b3beb4ede31af4ab1a8901cb73103b9" + "sha256": "fc05c9ad957334217feaa333f6cfe7cd206b250fbae9ea51a19d9c69f72158e7" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "market", - "path": "macintosh/de/train/market.bin", - "bytes": 177, - "highBytes": 4, + "path": "ISO-8859-1/sv/train/market.bin", + "bytes": 89, + "highBytes": 7, "uniqueHighBytes": 3, - "sha256": "f135969c32b723540e69570512179a361f12aca33782130567cafc8b8cb40974" + "sha256": "9f4f10ac20699e493cdd99b8c1ee6caa29d4a9359962aab15ac23bb773f1d26f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "workshop", - "path": "macintosh/de/train/workshop.bin", - "bytes": 188, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "a77aa45de75ac7b572174e1c99aa8aab3d7a3cb0278b2f68932d1c7fe4598218" + "path": "ISO-8859-1/sv/train/workshop.bin", + "bytes": 110, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "989f41c9b83ba4e3475a6902c33340fa7b07bc965776a09e9db165321bdcb3cb" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "validation", "document": "river-trip", - "path": "macintosh/de/validation/river-trip.bin", - "bytes": 192, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "b2edf27b09f5aeaf742baed4ebc17e0a890da6a9f0aa52a40adbcb16104e5538" + "path": "ISO-8859-1/sv/validation/river-trip.bin", + "bytes": 86, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "84cc4d3ab5abaa2d2575d868f0056375043c7e17d6c34ecd4112fca4632bc321" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "test", "document": "community-garden", - "path": "macintosh/es/test/community-garden.bin", - "bytes": 203, - "highBytes": 6, - "uniqueHighBytes": 3, - "sha256": "b675b0218b3db70d7c6f6f43cd7eb00fe7b45713f6e23e3fbac03a176e95935a" + "path": "ISO-8859-10/is/test/community-garden.bin", + "bytes": 174, + "highBytes": 19, + "uniqueHighBytes": 6, + "sha256": "11bbdafb4917a3935f98940af20e169c4a7eef5642d4a159297b44cd8b565d78" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "city-morning", - "path": "macintosh/es/train/city-morning.bin", - "bytes": 202, - "highBytes": 3, - "uniqueHighBytes": 3, - "sha256": "15f13077700d48818a5a5050c47683e1c14260cc22ce956abd29bd07eddf10eb" + "path": "ISO-8859-10/is/train/city-morning.bin", + "bytes": 161, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "3c8a3358a8195de52877343b34b5a71606732b3df110594862cebeb9c76aa3cf" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "library", - "path": "macintosh/es/train/library.bin", - "bytes": 202, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "a489cbb2f2f276564faf85fd6f86795a1ec5a1e5b1cfeda35874e2302e9d3d1b" + "path": "ISO-8859-10/is/train/library.bin", + "bytes": 166, + "highBytes": 18, + "uniqueHighBytes": 7, + "sha256": "a78d39ae9c97ed64273f11792d1e5a353080658d28ab6a85cc248f28b91d18a0" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", - "split": "train", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", + "split": "train", "document": "market", - "path": "macintosh/es/train/market.bin", - "bytes": 199, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "80770be98c0f1e40ee59e9de08615620aa8a0af1ff73ef8b6a3cc483b14a3a4e" + "path": "ISO-8859-10/is/train/market.bin", + "bytes": 168, + "highBytes": 10, + "uniqueHighBytes": 5, + "sha256": "c15c77b0cb233941ca1b038ddd269d4d866e4b5b48d61989220169b6cd851c9b" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "workshop", - "path": "macintosh/es/train/workshop.bin", - "bytes": 194, - "highBytes": 6, - "uniqueHighBytes": 4, - "sha256": "5b25ecfaa0df015d5c4c1fb52fce66af137588ac8677acd50c468b65b5112018" + "path": "ISO-8859-10/is/train/workshop.bin", + "bytes": 167, + "highBytes": 15, + "uniqueHighBytes": 6, + "sha256": "1bdd4e4aa76ecbd47f82533ec096fe3ee5c637debdefad8bc10251e8f3867af0" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "validation", "document": "river-trip", - "path": "macintosh/es/validation/river-trip.bin", - "bytes": 184, - "highBytes": 6, - "uniqueHighBytes": 3, - "sha256": "d68dda4671b8158cbff67ab7b476d2b819ac45ad00719d4dbe48f22ba1862baa" + "path": "ISO-8859-10/is/validation/river-trip.bin", + "bytes": 168, + "highBytes": 17, + "uniqueHighBytes": 9, + "sha256": "ce28fe8d7fe2ee466f03495c48f00a6da6fa7d3df530c7052873c444e1dd6f59" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "test", "document": "community-garden", - "path": "macintosh/fr/test/community-garden.bin", - "bytes": 231, + "path": "ISO-8859-13/lt/test/community-garden.bin", + "bytes": 177, + "highBytes": 17, + "uniqueHighBytes": 6, + "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + }, + { + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-13/lt/train/city-morning.bin", + "bytes": 156, + "highBytes": 11, + "uniqueHighBytes": 4, + "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + }, + { + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", + "split": "train", + "document": "library", + "path": "ISO-8859-13/lt/train/library.bin", + "bytes": 163, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" + }, + { + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", + "split": "train", + "document": "market", + "path": "ISO-8859-13/lt/train/market.bin", + "bytes": 171, + "highBytes": 12, + "uniqueHighBytes": 8, + "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" + }, + { + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", + "split": "train", + "document": "workshop", + "path": "ISO-8859-13/lt/train/workshop.bin", + "bytes": 170, "highBytes": 9, - "uniqueHighBytes": 1, - "sha256": "50363a67b8df69b6fbdb3b7a5a25c22c40cdac6ccca71d3bfafeff490e122c58" + "uniqueHighBytes": 4, + "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-13/lt/validation/river-trip.bin", + "bytes": 165, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-14/cy/test/community-garden.bin", + "bytes": 194, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "11475514e6d6789bb154d27be8a381507f5c136b1ec88219820a77d3f11a352a" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", "split": "train", "document": "city-morning", - "path": "macintosh/fr/train/city-morning.bin", - "bytes": 212, - "highBytes": 5, + "path": "ISO-8859-14/cy/train/city-morning.bin", + "bytes": 195, + "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "5f8dbd5bcfe437962ef5943c707e1695edce736abace97d5b29ef68945c42415" + "sha256": "5310db3607ef3fcadf53799acdd8836356af51cf840f1f391c4c31acdb3188ce" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", "split": "train", "document": "library", - "path": "macintosh/fr/train/library.bin", - "bytes": 226, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "66ecc14482e9af6c7d2c813d11462760e1a70b98a6d6c0c966821cd03f07eee3" + "path": "ISO-8859-14/cy/train/library.bin", + "bytes": 194, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c8445d5b31a86bb8a0f295792afe5d4632e9c30182460cb0bc590c6fa6156afb" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", "split": "train", "document": "market", - "path": "macintosh/fr/train/market.bin", - "bytes": 223, + "path": "ISO-8859-14/cy/train/market.bin", + "bytes": 183, "highBytes": 3, - "uniqueHighBytes": 1, - "sha256": "5e81aa00b93ef4299cde155e0d215906411a1b0566a676ea307403ef658274d5" + "uniqueHighBytes": 2, + "sha256": "c2b02e1a6e22d7f21483955ebf4d00e4b0500c3ee91dcb67e67cb07f015932e3" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", "split": "train", "document": "workshop", - "path": "macintosh/fr/train/workshop.bin", - "bytes": 215, - "highBytes": 8, - "uniqueHighBytes": 1, - "sha256": "ee23ebed42809949b51365120200f135cb051471430331b7c6ccf8282841075a" + "path": "ISO-8859-14/cy/train/workshop.bin", + "bytes": 173, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "2fe7199c4f96f4c5941e8f6b95aa44ae6f8d85863552550993b3b019acdeee6f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", "split": "validation", "document": "river-trip", - "path": "macintosh/fr/validation/river-trip.bin", - "bytes": 228, + "path": "ISO-8859-14/cy/validation/river-trip.bin", + "bytes": 161, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "318d863b3f7efa4a716e75ba4cd138fe9b9c337fb6f88eafe7ee1c671629e054" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/de/train/city-morning.bin", + "bytes": 208, "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "library", + "path": "ISO-8859-15/de/train/library.bin", + "bytes": 204, + "highBytes": 5, "uniqueHighBytes": 2, - "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "market", + "path": "ISO-8859-15/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "library", + "path": "ISO-8859-15/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "market", + "path": "ISO-8859-15/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "library", + "path": "ISO-8859-15/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "market", + "path": "ISO-8859-15/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-16/ro/test/community-garden.bin", + "bytes": 224, + "highBytes": 15, + "uniqueHighBytes": 6, + "sha256": "cbb06341ae7b36086fa1cc8e9ce8df8a66cf37ac24a4558200e71f037eaf6bc5" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-16/ro/train/city-morning.bin", + "bytes": 193, + "highBytes": 11, + "uniqueHighBytes": 5, + "sha256": "47d8c1bf653e17a058fd3c8a3141eaa08652db1c8e27a46be5b177a8770b1006" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "library", + "path": "ISO-8859-16/ro/train/library.bin", + "bytes": 205, + "highBytes": 19, + "uniqueHighBytes": 6, + "sha256": "fbd051c8abf29897208b0f3199e4504b8d40677a58edea5d541b37bfea98b459" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "market", + "path": "ISO-8859-16/ro/train/market.bin", + "bytes": 189, + "highBytes": 19, + "uniqueHighBytes": 5, + "sha256": "8584a99429d049f70c73ea6c50208b23faef446f1c50b9833b7e31e2ef3adbc5" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "ISO-8859-16/ro/train/workshop.bin", + "bytes": 188, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "d3dbc5ecf59a9705bc6dd1d7ed97222304426d439ee87084d3729d221a00145c" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-16/ro/validation/river-trip.bin", + "bytes": 188, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "d130ff9bdb9431000be39f0d1f6cd5d7a0aa805e8bd4eca9723c71761b110020" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/cs/test/community-garden.bin", + "bytes": 128, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "1eba2d29ae0f73a2f12f43469971a65f81c58779530b1b753285cb2d8c29200d" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/cs/train/city-morning.bin", + "bytes": 98, + "highBytes": 13, + "uniqueHighBytes": 6, + "sha256": "a0f0dfa252459d39f725fee0b0a13818681efd1c28e854e4eaf32509e03f5a55" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "library", + "path": "ISO-8859-2/cs/train/library.bin", + "bytes": 88, + "highBytes": 12, + "uniqueHighBytes": 7, + "sha256": "cd1af77dd042d83bb4106f82c6a213eb32babb8c8be59d4b3b07ece358a651a4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "market", + "path": "ISO-8859-2/cs/train/market.bin", + "bytes": 74, + "highBytes": 10, + "uniqueHighBytes": 7, + "sha256": "083f02c5caabf56bf78ab98fc83cabb38dfc79337e33d3a0f694de80c2403e0e" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/cs/train/workshop.bin", + "bytes": 90, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "05d2ab8033b5d71b2c863a14f4c54ca237af31f71158a2370622417e783788cf" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/cs/validation/river-trip.bin", + "bytes": 80, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "3eba80db3fa617e843e8165ca7dd7ee1332d362e9024b85ffa3f9dcedf4f4651" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/hu/test/community-garden.bin", + "bytes": 139, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "d55e643af71274694d74cff7dbf12589bfd13a2b86f2ddcfb53476578a3132b8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/hu/train/city-morning.bin", + "bytes": 94, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "97945a679eae0390b0b3c65608b9cffe4cd8ba6196b8af4be0cec9272bf25ae4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "library", + "path": "ISO-8859-2/hu/train/library.bin", + "bytes": 108, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "bb34de899f6968d695ac39ed18a9f72077d309da97d9721d4e6551562d644427" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "market", + "path": "ISO-8859-2/hu/train/market.bin", + "bytes": 80, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a6a508b4927acd6751a091826a4cc4945330021699819d1c956c8ea6b440a56b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/hu/train/workshop.bin", + "bytes": 101, + "highBytes": 14, + "uniqueHighBytes": 7, + "sha256": "2a294d4fef2aeeee691b74de2293bdf7a7bae7854b3021607fc7df7d17a1a3a4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/hu/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "ce35a0b3ccaa4b07727c0b9e925c958ad050a5d774d91c3f25962c3b7291ffab" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/pl/test/community-garden.bin", + "bytes": 188, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "f9e88a60cbcc42a798c28f8e9b56ee86c8f657f6c8bf8487b8ce2c61c1f02a8b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/pl/train/city-morning.bin", + "bytes": 174, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "7aa1f6ba90080099f3a458933c75a4d32aa5d1f7687cbbc2900c9da738361402" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "library", + "path": "ISO-8859-2/pl/train/library.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 6, + "sha256": "4533cb3ea7bd91bf69478c029c0d4e71755df177c96a18d5363289a80aa08df8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "market", + "path": "ISO-8859-2/pl/train/market.bin", + "bytes": 166, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "14e939af68d4b46232b52c9527d0ec87ab4a5bfb2d178fb4f7a58f8fdcd45f6a" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/pl/train/workshop.bin", + "bytes": 165, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "f749532f6c64c2ff857f54f2e4ef7a0b42ac1dfb6a60bfeeb518fdc75d966c6a" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/pl/validation/river-trip.bin", + "bytes": 161, + "highBytes": 9, + "uniqueHighBytes": 3, + "sha256": "9b6685ebb982c76027a1f70a1fb36ad3913d4400c76dae5f54b460f61704e2d7" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "library", + "path": "ISO-8859-2/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "market", + "path": "ISO-8859-2/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-3/mt/test/community-garden.bin", + "bytes": 186, + "highBytes": 13, + "uniqueHighBytes": 4, + "sha256": "d6874f4f630507a619f841cad55a2046be0ee930fb9d450e20bcf0e9525759d5" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-3/mt/train/city-morning.bin", + "bytes": 171, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "e40f845da038bcae4f7c6edb026ca823788410f8895cd7ae2765ca4afde49bfb" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "library", + "path": "ISO-8859-3/mt/train/library.bin", + "bytes": 181, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "b16045c76dede00ff236e27cd647ea45e25a98d4cf04530c9baf947dbb1a9cb2" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "market", + "path": "ISO-8859-3/mt/train/market.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "be0fbd5745a025d769bb128a55d8c80bb1397590edde4d79d28db1310589af8e" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "workshop", + "path": "ISO-8859-3/mt/train/workshop.bin", + "bytes": 164, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "433e386a7ccaeaeef6e9fbd1daf1f586cc61788cd13973523c2d679026964420" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-3/mt/validation/river-trip.bin", + "bytes": 176, + "highBytes": 9, + "uniqueHighBytes": 2, + "sha256": "9c73d965fbad70cf830b5a3db04e12c4f0b3bd50ead049f7d6703b5c19cf4eb1" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-4/lv/test/community-garden.bin", + "bytes": 174, + "highBytes": 17, + "uniqueHighBytes": 8, + "sha256": "c0382f3b40081b8a5d30ca7d26699a2f9e9eb2e453edac9beb97b4a4a9c3051b" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-4/lv/train/city-morning.bin", + "bytes": 163, + "highBytes": 15, + "uniqueHighBytes": 7, + "sha256": "d16cbb8016fdba887337226f08a30e96f2752b657dacbabec3a00b36d5538f60" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "library", + "path": "ISO-8859-4/lv/train/library.bin", + "bytes": 161, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "c6757b519e310d300ab70b574b32056a85d7bdf140afd265d5e44101cca573e0" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "market", + "path": "ISO-8859-4/lv/train/market.bin", + "bytes": 170, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "33999e249def18a8469c731d110dbdb9614103b3dce69d8005bc6993b878fc88" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "workshop", + "path": "ISO-8859-4/lv/train/workshop.bin", + "bytes": 163, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "1429abf14fb2e4144c64dd59657aec2615da7795c92f4e6f5e5f5534ee4be9a0" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-4/lv/validation/river-trip.bin", + "bytes": 156, + "highBytes": 14, + "uniqueHighBytes": 4, + "sha256": "282b08484ad284a0a069c2d8113e09bc80de8163fee60568042c538b0541fc3f" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-5/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "1bb1e574a38f791fa40a6e3742960a95e6576dc9016efa256a7a9d7e869d3fb4" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-5/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "aeb268c03cb92a112177352ea91fa6f13691d0bbd28729397e25ca9e36a63b30" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "library", + "path": "ISO-8859-5/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "61e10db337d7b99d3290cc4d0e83412621abf39f97bfa5f28a9e1249117d74f0" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "market", + "path": "ISO-8859-5/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "8d252284135141f156b8483f0eaea527549221996fdd2034b53337d8fe37782a" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "ISO-8859-5/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "7f22de320a26b19ac6054b59dbe79cc7e8e82857d0529c227c948ca33ef309b4" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-5/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "9bbfd60e8c09ca445265bae5c320ca4565c3800acb17f3fe18af74f0d4b53f66" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-6/ar/test/community-garden.bin", + "bytes": 112, + "highBytes": 92, + "uniqueHighBytes": 25, + "sha256": "e49c5c7a78e9e802de85f4e3b075486c68fa32625c0682b0bc19267616009b42" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-6/ar/train/city-morning.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 22, + "sha256": "e5a89c48b2108cf6a4e864621eb2a7894e83566bceeeac7ea4458904751cb4d4" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "library", + "path": "ISO-8859-6/ar/train/library.bin", + "bytes": 107, + "highBytes": 89, + "uniqueHighBytes": 25, + "sha256": "f91cf3a719aa42bb5d2c16831a0e01a37fbf699e3d394b0b417b43a03494c26f" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "market", + "path": "ISO-8859-6/ar/train/market.bin", + "bytes": 89, + "highBytes": 75, + "uniqueHighBytes": 24, + "sha256": "08be46278b902f4cf10072e2506c7d2d29cb075c4bc087fb632cdd29553233bc" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "workshop", + "path": "ISO-8859-6/ar/train/workshop.bin", + "bytes": 91, + "highBytes": 75, + "uniqueHighBytes": 21, + "sha256": "dccb5be7cce959474487e4c395c238144116f43076dc650e4e3be5a56a38dc02" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-6/ar/validation/river-trip.bin", + "bytes": 91, + "highBytes": 73, + "uniqueHighBytes": 24, + "sha256": "b9afd70647f0be078fd19fa1040fddfe01cd90170a944ffeaf4e529112b45684" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-7/el/test/community-garden.bin", + "bytes": 136, + "highBytes": 112, + "uniqueHighBytes": 25, + "sha256": "8f7531c39aa8efa5f4e38e8eb948f713d69cbe72badea15391e5c62870179370" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-7/el/train/city-morning.bin", + "bytes": 93, + "highBytes": 74, + "uniqueHighBytes": 25, + "sha256": "80e97194d91d40aeae9020cd96d22ac6ad01e3c39020377e32bdf63c2b9622e0" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "library", + "path": "ISO-8859-7/el/train/library.bin", + "bytes": 111, + "highBytes": 94, + "uniqueHighBytes": 27, + "sha256": "eb0448b06be644629054c0f94e0c6dccb7458d80a6c477eda1ab321f378c51bb" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "market", + "path": "ISO-8859-7/el/train/market.bin", + "bytes": 85, + "highBytes": 69, + "uniqueHighBytes": 28, + "sha256": "ee94d165652f333d57c5be996825a8723cfa76aba286f1f22a0b0d40312eece9" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "workshop", + "path": "ISO-8859-7/el/train/workshop.bin", + "bytes": 108, + "highBytes": 90, + "uniqueHighBytes": 27, + "sha256": "2466d051c231cbb22d9fa5a962bb180a88dbe805ad1b609467bd06afe5de3b02" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-7/el/validation/river-trip.bin", + "bytes": 92, + "highBytes": 75, + "uniqueHighBytes": 25, + "sha256": "4dc6ccfd555071ba0df444d4f2fd5b0eb96cf44e389dc872279e8ac1096b359e" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-8/he/test/community-garden.bin", + "bytes": 104, + "highBytes": 84, + "uniqueHighBytes": 21, + "sha256": "6044aa4ee9d7f7517ca965d0a83029c5569a83e8efad7b9f39974fa95fd61298" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-8/he/train/city-morning.bin", + "bytes": 79, + "highBytes": 64, + "uniqueHighBytes": 19, + "sha256": "5c0eddaa74015d9d135cd9c9d80b117cd82a3aa3effe6298deaca2c06ac1305b" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "library", + "path": "ISO-8859-8/he/train/library.bin", + "bytes": 86, + "highBytes": 72, + "uniqueHighBytes": 20, + "sha256": "b9b2605035a9017f18f9de847e03e623581ffed30ea1623fe332efcac420637b" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "market", + "path": "ISO-8859-8/he/train/market.bin", + "bytes": 64, + "highBytes": 49, + "uniqueHighBytes": 21, + "sha256": "ed2975da284ad08b0261f536f69344981747bdd3b37d0ae6a4ec0a36ed1d57ca" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "workshop", + "path": "ISO-8859-8/he/train/workshop.bin", + "bytes": 80, + "highBytes": 64, + "uniqueHighBytes": 21, + "sha256": "763e4f5501fa3308c2862f48f5173d29d5fd1ccf730664a9bc4c40a714dfcbc6" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-8/he/validation/river-trip.bin", + "bytes": 63, + "highBytes": 47, + "uniqueHighBytes": 18, + "sha256": "5319026034eb165464faad8f2de9c2e6c97f01c92e71d2fc3441310417d70b28" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-9/tr/test/community-garden.bin", + "bytes": 136, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "a2224b89fee790662cf9c3cac7200da0f8c5842b4ce5d26512d76e522c6db437" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-9/tr/train/city-morning.bin", + "bytes": 87, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "a1231eabb26c3d7d770c885972062ee78365f871d1f62be5977332949e8b1a80" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "library", + "path": "ISO-8859-9/tr/train/library.bin", + "bytes": 98, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "432b50510bf2a5ab8b7b51aaf608341bea95afbf43edff4fa4d404f4a2d87a70" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "market", + "path": "ISO-8859-9/tr/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "25354256b7d0e3f936eb224f6343e5d3d73f78a504d5b2d3c260a2015a62e356" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "workshop", + "path": "ISO-8859-9/tr/train/workshop.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "0bc87de2045407c5c4db2589ab89cb9faf40cf5556d17caf0bfa916a1b4a0b03" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-9/tr/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "b94d965d212deb815bd73eabf712bb673f632c88c0a0c9f772202109e4d85e8d" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "KOI8-R/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "7be1a487f774fe73d3dd7a96f9c443d194edf77b1316e89483096c6e0c815e6c" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "KOI8-R/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "42e7be1a39fd943782012cc3c291fa4cf0d922c51b300ec7b5a3d7f3cfc5cc51" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "library", + "path": "KOI8-R/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "9ec8dd417b12bf60f2141ea177caf67a444bbd9b53cb031596df8fe60ea398c6" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "market", + "path": "KOI8-R/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "6096b9d90fa7ab1e9ec13727c3e8d92d5c57841066b4c795b1609cee03dc3110" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "KOI8-R/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "23a3947cc61c1418df7b89c74fe7eb940ef773ab0d04723cdd3c6857cc09a448" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "KOI8-R/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "893c13722e2ba6f81610cbebf9abb72940b3ef07a639613b2ca87cd993c925d8" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "test", + "document": "community-garden", + "path": "KOI8-U/uk/test/community-garden.bin", + "bytes": 181, + "highBytes": 152, + "uniqueHighBytes": 29, + "sha256": "25d71bdf3678270bf3a539771f8e429faaef3157c5649c0f2edc4ec06e0c80fa" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "city-morning", + "path": "KOI8-U/uk/train/city-morning.bin", + "bytes": 205, + "highBytes": 168, + "uniqueHighBytes": 31, + "sha256": "853b422f64ef06d7a64528db49d4eca0f11240852392e51cb2700b1b8b9fd122" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "library", + "path": "KOI8-U/uk/train/library.bin", + "bytes": 182, + "highBytes": 151, + "uniqueHighBytes": 32, + "sha256": "42455a09e6cafcd694b38198b1f172e44e63774cde6c9e760181334974536fe5" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "market", + "path": "KOI8-U/uk/train/market.bin", + "bytes": 199, + "highBytes": 165, + "uniqueHighBytes": 33, + "sha256": "0d8b40da8ad30cc4d6d849f2439f472db8092d622e428747aa55e1fa1797800b" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "workshop", + "path": "KOI8-U/uk/train/workshop.bin", + "bytes": 189, + "highBytes": 161, + "uniqueHighBytes": 30, + "sha256": "b4b92fa64c4b9252511ed0240a7e1cb565a0b1158f07b50fefb93fab1bd5f0f7" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "validation", + "document": "river-trip", + "path": "KOI8-U/uk/validation/river-trip.bin", + "bytes": 195, + "highBytes": 160, + "uniqueHighBytes": 31, + "sha256": "1b015aa68443a92670b3e73e37fa96d70d1e2dea1a9ed736bebfcf077bde9dfe" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "macintosh/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "9a35ac96816b79920d3076818a106719d95169d017b89832ff735652fdbc45d5" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "macintosh/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "d826a59c9fe0ea77efe5554f688a96a25ba31edd3b502177f88b229237bfafff" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "library", + "path": "macintosh/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "29829a1ac13b7f7ad2ecd8abd7d0f2454b3beb4ede31af4ab1a8901cb73103b9" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "market", + "path": "macintosh/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "f135969c32b723540e69570512179a361f12aca33782130567cafc8b8cb40974" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "workshop", + "path": "macintosh/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a77aa45de75ac7b572174e1c99aa8aab3d7a3cb0278b2f68932d1c7fe4598218" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "macintosh/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "b2edf27b09f5aeaf742baed4ebc17e0a890da6a9f0aa52a40adbcb16104e5538" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "macintosh/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "b675b0218b3db70d7c6f6f43cd7eb00fe7b45713f6e23e3fbac03a176e95935a" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "macintosh/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "15f13077700d48818a5a5050c47683e1c14260cc22ce956abd29bd07eddf10eb" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "library", + "path": "macintosh/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "a489cbb2f2f276564faf85fd6f86795a1ec5a1e5b1cfeda35874e2302e9d3d1b" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "market", + "path": "macintosh/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "80770be98c0f1e40ee59e9de08615620aa8a0af1ff73ef8b6a3cc483b14a3a4e" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "workshop", + "path": "macintosh/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "5b25ecfaa0df015d5c4c1fb52fce66af137588ac8677acd50c468b65b5112018" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "macintosh/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "d68dda4671b8158cbff67ab7b476d2b819ac45ad00719d4dbe48f22ba1862baa" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "macintosh/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "50363a67b8df69b6fbdb3b7a5a25c22c40cdac6ccca71d3bfafeff490e122c58" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "macintosh/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "5f8dbd5bcfe437962ef5943c707e1695edce736abace97d5b29ef68945c42415" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "library", + "path": "macintosh/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "66ecc14482e9af6c7d2c813d11462760e1a70b98a6d6c0c966821cd03f07eee3" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "market", + "path": "macintosh/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "5e81aa00b93ef4299cde155e0d215906411a1b0566a676ea307403ef658274d5" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "macintosh/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "ee23ebed42809949b51365120200f135cb051471430331b7c6ccf8282841075a" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "macintosh/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "test", + "document": "community-garden", + "path": "Shift_JIS/ja/test/community-garden.bin", + "bytes": 129, + "highBytes": 113, + "uniqueHighBytes": 43, + "sha256": "a58ecbafb06d5f88d10f2d056a96eb0e826d2dfe50360c9da6f591ece6ec45e0" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "city-morning", + "path": "Shift_JIS/ja/train/city-morning.bin", + "bytes": 117, + "highBytes": 98, + "uniqueHighBytes": 37, + "sha256": "7848fc97e017d4eebb1dcc1ad7755a035330bb9fd5cc01fbe3df7db505745e34" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "library", + "path": "Shift_JIS/ja/train/library.bin", + "bytes": 125, + "highBytes": 104, + "uniqueHighBytes": 43, + "sha256": "e0d3b6fec3a86651a59ca899d3f9df16529da913ffdd4d4e1b3da9cc0fce6517" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "market", + "path": "Shift_JIS/ja/train/market.bin", + "bytes": 115, + "highBytes": 94, + "uniqueHighBytes": 39, + "sha256": "2c442382b1bc33652f62e4086143582114db0e2adde38f7cda22cf94e29c9f8b" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "workshop", + "path": "Shift_JIS/ja/train/workshop.bin", + "bytes": 131, + "highBytes": 109, + "uniqueHighBytes": 45, + "sha256": "e78003a71014de82d26e954ca441ce084fd1f07dc39f5f045596d91e776116a2" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "validation", + "document": "river-trip", + "path": "Shift_JIS/ja/validation/river-trip.bin", + "bytes": 119, + "highBytes": 104, + "uniqueHighBytes": 41, + "sha256": "9d7d47ab703aab26e1ecb264cb5cb838224a32f7992bb54af5fe10d27f954809" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "test", + "document": "community-garden", + "path": "windows-1250/cs/test/community-garden.bin", + "bytes": 128, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "1eba2d29ae0f73a2f12f43469971a65f81c58779530b1b753285cb2d8c29200d" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "city-morning", + "path": "windows-1250/cs/train/city-morning.bin", + "bytes": 98, + "highBytes": 13, + "uniqueHighBytes": 6, + "sha256": "a0f0dfa252459d39f725fee0b0a13818681efd1c28e854e4eaf32509e03f5a55" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "library", + "path": "windows-1250/cs/train/library.bin", + "bytes": 88, + "highBytes": 12, + "uniqueHighBytes": 7, + "sha256": "1469e1d9d57e6f147a56fb9caf1f5b0eab61dedd7c5482c19199de6853ade2ef" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "market", + "path": "windows-1250/cs/train/market.bin", + "bytes": 74, + "highBytes": 10, + "uniqueHighBytes": 7, + "sha256": "083f02c5caabf56bf78ab98fc83cabb38dfc79337e33d3a0f694de80c2403e0e" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "workshop", + "path": "windows-1250/cs/train/workshop.bin", + "bytes": 90, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "98ad6215f5c5e7ebaa4324bd133c34bd89e63abc0fcffaac9bfa73ecd3175f17" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/cs/validation/river-trip.bin", + "bytes": 80, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "7934c52e77661f3ba94150b262a68510c2da7c9a6d493fe1d72c2ea7658a97b5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "test", + "document": "community-garden", + "path": "windows-1250/hu/test/community-garden.bin", + "bytes": 139, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "d55e643af71274694d74cff7dbf12589bfd13a2b86f2ddcfb53476578a3132b8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "city-morning", + "path": "windows-1250/hu/train/city-morning.bin", + "bytes": 94, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "97945a679eae0390b0b3c65608b9cffe4cd8ba6196b8af4be0cec9272bf25ae4" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "library", + "path": "windows-1250/hu/train/library.bin", + "bytes": 108, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "bb34de899f6968d695ac39ed18a9f72077d309da97d9721d4e6551562d644427" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "market", + "path": "windows-1250/hu/train/market.bin", + "bytes": 80, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a6a508b4927acd6751a091826a4cc4945330021699819d1c956c8ea6b440a56b" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "workshop", + "path": "windows-1250/hu/train/workshop.bin", + "bytes": 101, + "highBytes": 14, + "uniqueHighBytes": 7, + "sha256": "2a294d4fef2aeeee691b74de2293bdf7a7bae7854b3021607fc7df7d17a1a3a4" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/hu/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "ce35a0b3ccaa4b07727c0b9e925c958ad050a5d774d91c3f25962c3b7291ffab" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "test", + "document": "community-garden", + "path": "windows-1250/pl/test/community-garden.bin", + "bytes": 188, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "bfb9b6bd673324ae75dba20653fe290162c78cc26d13be88fa4fd57ee3a7b101" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "city-morning", + "path": "windows-1250/pl/train/city-morning.bin", + "bytes": 174, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "809b42296a766532c1dedb56aaf287db6538397f3f221c8164a2c627871187d5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "library", + "path": "windows-1250/pl/train/library.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 6, + "sha256": "d9f13cd5121d5f68350e48835968eee29e76da5406a7e6de2b3ed5481c7f51f0" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "market", + "path": "windows-1250/pl/train/market.bin", + "bytes": 166, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "bda4333ebf0ad7c3f37cf924b23be19ab53ddc3117f6031d8075c846a76299b5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "workshop", + "path": "windows-1250/pl/train/workshop.bin", + "bytes": 165, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "3344ae802a8cccebcb4561ccc3320991aa1c3c789bcf6172b7427d9bbf7fa6a0" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/pl/validation/river-trip.bin", + "bytes": 161, + "highBytes": 9, + "uniqueHighBytes": 3, + "sha256": "aac1576aea489691f339c2cce058bb908effa44befab53e4f04a82698896fc54" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "windows-1250/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "windows-1250/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "library", + "path": "windows-1250/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "market", + "path": "windows-1250/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "windows-1250/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "windows-1251/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "b02d0279ed09a186f1a89564cc9e00cf02fd68b0db74a1be7529ee85c8481256" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "windows-1251/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "74c61cbdff726b61e50b34c19a5777c7c13fef80f82ff10677136da50064ada8" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "library", + "path": "windows-1251/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "52bcdfd21b84cf735a555d8a7669e5dc36c12509459df325f3eb3ce5b5891377" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "market", + "path": "windows-1251/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "ca18ba887b386031b0a5dc1fc1731c94314ad760c063947f18513da039200773" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "windows-1251/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "cc7dfbf50476621ae34c5f6de6eae7ba3e13fc650f507a17fc6c07e77bf45ee8" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "windows-1251/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "78219367c687aac640cc10f65cb672bcc274a0418acb42669d050d81c849c329" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "test", + "document": "community-garden", + "path": "windows-1252/da/test/community-garden.bin", + "bytes": 123, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "d742c49869d5f97ff0dc9bed58a7e819cf5a9f3cde419ff785f2201591fa7991" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "city-morning", + "path": "windows-1252/da/train/city-morning.bin", + "bytes": 89, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "75320357a51c6716244970d7c802f42af40a6a65fa1602e4f38e239e16ccd887" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "library", + "path": "windows-1252/da/train/library.bin", + "bytes": 105, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "4b0f38943f7eb950af8b04b16ac6ce581c48f70e933f66bd03725842d9e1bd58" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "market", + "path": "windows-1252/da/train/market.bin", + "bytes": 84, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "61d9285e5b354f3cffb320f3bb62af7d4ce86fe34ba0e5067d5bc26aa9cd010c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "workshop", + "path": "windows-1252/da/train/workshop.bin", + "bytes": 103, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "aca6cded29cf164fc9445de75292a45f007c971ea9407d7ef0d794413a7cd975" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/da/validation/river-trip.bin", + "bytes": 97, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "0edb6dd9323283e842dea2112213cf7b86608b257a36cfd318b8c71b3c01c8e0" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "windows-1252/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "windows-1252/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "library", + "path": "windows-1252/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "market", + "path": "windows-1252/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "workshop", + "path": "windows-1252/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "test", + "document": "community-garden", + "path": "windows-1252/en/test/community-garden.bin", + "bytes": 136, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8574d38336d96e1a48fd42b012b3924bc36ee57550bc58a28ed6d5c17d514c1d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "city-morning", + "path": "windows-1252/en/train/city-morning.bin", + "bytes": 108, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "ff22d70d940981c1ea141576fb34c74d841dbf2e2140d80ee55f51dd63bd9ef0" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "library", + "path": "windows-1252/en/train/library.bin", + "bytes": 117, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "12ea1dc5452af3ca116753189e184ef0a10645eee17f33211dc5f7d182318f38" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "market", + "path": "windows-1252/en/train/market.bin", + "bytes": 100, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "cbaa356df57d8557b3d2c3ca4d6ec5845077db27583e102aebdaedcacf9f2aee" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "workshop", + "path": "windows-1252/en/train/workshop.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "157d773566f7cd87f7bafb87780ba98dcc090adbdaaea2c03483e7a3b6a93a70" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/en/validation/river-trip.bin", + "bytes": 106, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c4e405fd6466194b2ddc91f07f8bcb624c904531d6d96f37e35d04847c6abbb9" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "windows-1252/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "windows-1252/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "library", + "path": "windows-1252/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "market", + "path": "windows-1252/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "workshop", + "path": "windows-1252/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "windows-1252/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "windows-1252/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "library", + "path": "windows-1252/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "market", + "path": "windows-1252/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "windows-1252/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "test", + "document": "community-garden", + "path": "windows-1252/it/test/community-garden.bin", + "bytes": 161, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "53bb7b070795cd69fba5bcc79a4958b2ac5bb23677ebdac99c80623f96d501dd" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "city-morning", + "path": "windows-1252/it/train/city-morning.bin", + "bytes": 101, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "7c59ad9c3c9ac0b84c02c4829bffabdc83817989f08836d9ee44afd8a392e43d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "library", + "path": "windows-1252/it/train/library.bin", + "bytes": 141, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "bb28d91b3f10f0208886078bf0fafbc85c7e03148069ddf5a9762809f28f8981" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "market", + "path": "windows-1252/it/train/market.bin", + "bytes": 114, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "40ce81cfc5ec4c5b655c2fb9ba519022a77da2c319950c868bc16c7f69278b11" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "workshop", + "path": "windows-1252/it/train/workshop.bin", + "bytes": 152, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "5b5704c6191b4179ee6966a2302452eaaf99c348b34e26bc210879c77def38bc" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/it/validation/river-trip.bin", + "bytes": 98, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8fdde80f1933a9dfb82cc0181c6770e9947ccf5bfd745b8251ee02f54e6a19a7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "test", + "document": "community-garden", + "path": "windows-1252/nl/test/community-garden.bin", + "bytes": 153, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "26b1ccb49c443abe7580037a8a836bf99e9d5a5135b1751b77a73e3f61c1a9bf" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "city-morning", + "path": "windows-1252/nl/train/city-morning.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "201fbe0db619c5d0d4ccecd723c27c68e0807437531553e8c17ba84cfc4335b2" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "library", + "path": "windows-1252/nl/train/library.bin", + "bytes": 150, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "5d9cdae67a5a7cf6cb319750006ea3cf6afaf379b11c35aa241ef63b203cb17b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "market", + "path": "windows-1252/nl/train/market.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "15466fb33051e25a441363879415fe1c6ec949d753105ad851a75b2054d8c0da" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "workshop", + "path": "windows-1252/nl/train/workshop.bin", + "bytes": 132, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "4fa51f6d8f9543c9df0acef5a2711a5d805fd18286c1164d786bd8ac715c183b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/nl/validation/river-trip.bin", + "bytes": 113, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "d9448db92c3fe1270814d7798759eca44290f94fd76c0ff9012dc2401b426253" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "test", + "document": "community-garden", + "path": "windows-1252/no/test/community-garden.bin", + "bytes": 116, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b86618ac6687d3769cca67ec9cb49a0d4d1dea45476592a316ae6eef40d576db" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "city-morning", + "path": "windows-1252/no/train/city-morning.bin", + "bytes": 116, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "445f262b0f2ea6f724271b8486b0cfc861cf4e081e4617db895bc36a33dea2ee" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "library", + "path": "windows-1252/no/train/library.bin", + "bytes": 126, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "96c146e3d2b86817371ba6ca5d07361d31519cb0d16fbbecfaf7bba36f0ff2eb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "market", + "path": "windows-1252/no/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "a0279dbf28e14a6d9841789f50751ec9cdb09c2743f4ac1c61e72f310e916732" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "workshop", + "path": "windows-1252/no/train/workshop.bin", + "bytes": 100, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "9c70c92214b0f0ac9000a131773f1ed1756d630037293a759982edcb621e056a" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/no/validation/river-trip.bin", + "bytes": 85, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "931a303e54cbfe3ccddf45aa0dd3956be07e4cf16bf1fe5ee0874bd4ad2bab9a" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "test", + "document": "community-garden", + "path": "windows-1252/pt/test/community-garden.bin", + "bytes": 145, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "a8e56da45e10d94745a21dd36a2721410e1b0d04b5bfee15b2c451a27c8ba62d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "city-morning", + "path": "windows-1252/pt/train/city-morning.bin", + "bytes": 99, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "c9a43be7ac9a1af0be31146446c7cea5142ba97545c304e9fdfef166c0dfabe1" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "library", + "path": "windows-1252/pt/train/library.bin", + "bytes": 116, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "634c4f0d24e7f1a8bbf9b1fd68d0fe414376d2efeff0fbfc7637d921709f2a62" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "market", + "path": "windows-1252/pt/train/market.bin", + "bytes": 96, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "79f397f7b2ac2cca51091f006462be7777befb52d36c54237a8cfb0cb20cd924" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "workshop", + "path": "windows-1252/pt/train/workshop.bin", + "bytes": 108, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "fbb52707258ca57000887397d2bd8220907fc178ed36b929c62e72bf211848e1" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/pt/validation/river-trip.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "db583dead964fc7fe8a257ebd69f9dd7388c89384bac132955a76b02150c753f" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "test", + "document": "community-garden", + "path": "windows-1252/sv/test/community-garden.bin", + "bytes": 138, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "7229bf29e0adc2b37cdb6663bb696e500e13000633665f0f226915f5f74c4853" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "city-morning", + "path": "windows-1252/sv/train/city-morning.bin", + "bytes": 101, + "highBytes": 7, + "uniqueHighBytes": 4, + "sha256": "5ac24bf0fe3bfb1d64aa066ff78e3af67bcf0b4a6f8d142fefeff0b8ad4ae72c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "library", + "path": "windows-1252/sv/train/library.bin", + "bytes": 108, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "fc05c9ad957334217feaa333f6cfe7cd206b250fbae9ea51a19d9c69f72158e7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "market", + "path": "windows-1252/sv/train/market.bin", + "bytes": 89, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "9f4f10ac20699e493cdd99b8c1ee6caa29d4a9359962aab15ac23bb773f1d26f" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "workshop", + "path": "windows-1252/sv/train/workshop.bin", + "bytes": 110, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "989f41c9b83ba4e3475a6902c33340fa7b07bc965776a09e9db165321bdcb3cb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/sv/validation/river-trip.bin", + "bytes": 86, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "84cc4d3ab5abaa2d2575d868f0056375043c7e17d6c34ecd4112fca4632bc321" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "test", + "document": "community-garden", + "path": "windows-1253/el/test/community-garden.bin", + "bytes": 136, + "highBytes": 112, + "uniqueHighBytes": 25, + "sha256": "8f7531c39aa8efa5f4e38e8eb948f713d69cbe72badea15391e5c62870179370" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "city-morning", + "path": "windows-1253/el/train/city-morning.bin", + "bytes": 93, + "highBytes": 74, + "uniqueHighBytes": 25, + "sha256": "80e97194d91d40aeae9020cd96d22ac6ad01e3c39020377e32bdf63c2b9622e0" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "library", + "path": "windows-1253/el/train/library.bin", + "bytes": 111, + "highBytes": 94, + "uniqueHighBytes": 27, + "sha256": "eb0448b06be644629054c0f94e0c6dccb7458d80a6c477eda1ab321f378c51bb" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "market", + "path": "windows-1253/el/train/market.bin", + "bytes": 85, + "highBytes": 69, + "uniqueHighBytes": 28, + "sha256": "ee94d165652f333d57c5be996825a8723cfa76aba286f1f22a0b0d40312eece9" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "workshop", + "path": "windows-1253/el/train/workshop.bin", + "bytes": 108, + "highBytes": 90, + "uniqueHighBytes": 27, + "sha256": "2466d051c231cbb22d9fa5a962bb180a88dbe805ad1b609467bd06afe5de3b02" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "validation", + "document": "river-trip", + "path": "windows-1253/el/validation/river-trip.bin", + "bytes": 92, + "highBytes": 75, + "uniqueHighBytes": 25, + "sha256": "4dc6ccfd555071ba0df444d4f2fd5b0eb96cf44e389dc872279e8ac1096b359e" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "test", + "document": "community-garden", + "path": "windows-1254/tr/test/community-garden.bin", + "bytes": 136, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "a2224b89fee790662cf9c3cac7200da0f8c5842b4ce5d26512d76e522c6db437" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "city-morning", + "path": "windows-1254/tr/train/city-morning.bin", + "bytes": 87, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "a1231eabb26c3d7d770c885972062ee78365f871d1f62be5977332949e8b1a80" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "library", + "path": "windows-1254/tr/train/library.bin", + "bytes": 98, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "432b50510bf2a5ab8b7b51aaf608341bea95afbf43edff4fa4d404f4a2d87a70" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "market", + "path": "windows-1254/tr/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "25354256b7d0e3f936eb224f6343e5d3d73f78a504d5b2d3c260a2015a62e356" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "workshop", + "path": "windows-1254/tr/train/workshop.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "0bc87de2045407c5c4db2589ab89cb9faf40cf5556d17caf0bfa916a1b4a0b03" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "validation", + "document": "river-trip", + "path": "windows-1254/tr/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "b94d965d212deb815bd73eabf712bb673f632c88c0a0c9f772202109e4d85e8d" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "test", + "document": "community-garden", + "path": "windows-1255/he/test/community-garden.bin", + "bytes": 104, + "highBytes": 84, + "uniqueHighBytes": 21, + "sha256": "6044aa4ee9d7f7517ca965d0a83029c5569a83e8efad7b9f39974fa95fd61298" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "city-morning", + "path": "windows-1255/he/train/city-morning.bin", + "bytes": 79, + "highBytes": 64, + "uniqueHighBytes": 19, + "sha256": "5c0eddaa74015d9d135cd9c9d80b117cd82a3aa3effe6298deaca2c06ac1305b" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "library", + "path": "windows-1255/he/train/library.bin", + "bytes": 86, + "highBytes": 72, + "uniqueHighBytes": 20, + "sha256": "b9b2605035a9017f18f9de847e03e623581ffed30ea1623fe332efcac420637b" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "market", + "path": "windows-1255/he/train/market.bin", + "bytes": 64, + "highBytes": 49, + "uniqueHighBytes": 21, + "sha256": "ed2975da284ad08b0261f536f69344981747bdd3b37d0ae6a4ec0a36ed1d57ca" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "workshop", + "path": "windows-1255/he/train/workshop.bin", + "bytes": 80, + "highBytes": 64, + "uniqueHighBytes": 21, + "sha256": "763e4f5501fa3308c2862f48f5173d29d5fd1ccf730664a9bc4c40a714dfcbc6" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "validation", + "document": "river-trip", + "path": "windows-1255/he/validation/river-trip.bin", + "bytes": 63, + "highBytes": 47, + "uniqueHighBytes": 18, + "sha256": "5319026034eb165464faad8f2de9c2e6c97f01c92e71d2fc3441310417d70b28" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "test", + "document": "community-garden", + "path": "windows-1256/ar/test/community-garden.bin", + "bytes": 112, + "highBytes": 92, + "uniqueHighBytes": 25, + "sha256": "a36cd96bbd0a9597105cb4292371e4c2998aabd844ba3c9b6cd7a0d85dfe23e8" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "city-morning", + "path": "windows-1256/ar/train/city-morning.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 22, + "sha256": "fd5a2fe9debabe95e8ca2e20fc6a95db0d4f7ed301bfb285a49f3c4e84e7409c" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "library", + "path": "windows-1256/ar/train/library.bin", + "bytes": 107, + "highBytes": 89, + "uniqueHighBytes": 25, + "sha256": "7d22becffc494a5a0cca03a06f66167f909ceb67d914125e0668bb5db170ec9e" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "market", + "path": "windows-1256/ar/train/market.bin", + "bytes": 89, + "highBytes": 75, + "uniqueHighBytes": 24, + "sha256": "153ade153ba47b62ef5a64e3b1215b54f3cd01607fd44708a288eb78889faf82" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "workshop", + "path": "windows-1256/ar/train/workshop.bin", + "bytes": 91, + "highBytes": 75, + "uniqueHighBytes": 21, + "sha256": "3aa1c63ed62f9fd08039dcc50fa8705a239a1b133a96fdb81c7340d5d4cd9a79" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "validation", + "document": "river-trip", + "path": "windows-1256/ar/validation/river-trip.bin", + "bytes": 91, + "highBytes": 73, + "uniqueHighBytes": 24, + "sha256": "dd2d2c110d7753296192722e41de23e67c3350eea9217a1f5935829742bd2c69" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "test", + "document": "community-garden", + "path": "windows-1257/et/test/community-garden.bin", + "bytes": 122, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b09cf914e826b9230579833e56b483b71e75865e26b0cfdd921b9f386dafb448" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "city-morning", + "path": "windows-1257/et/train/city-morning.bin", + "bytes": 94, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "68d9d20af9ad1104bdfa39df3ce31e24acc520754c942351d037bc9378a9c5bc" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "library", + "path": "windows-1257/et/train/library.bin", + "bytes": 101, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "60b30c22898450c95e27d36443d3984c00987bbd4933d796c92a6e3ca3a38d5c" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "market", + "path": "windows-1257/et/train/market.bin", + "bytes": 92, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "fd5fc0e9d16f48c3a3f750929689afe78e5da298455fce90960e402427a64d9d" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "workshop", + "path": "windows-1257/et/train/workshop.bin", + "bytes": 107, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "731b2b0230faadc87c3a981fc8ae34ec858d99970a1f73dbf0aefdcb4b58f3d0" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/et/validation/river-trip.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 2, + "sha256": "81df0820697d38b422b3a2e8954782c17d2e93d39be7111c48e0d6f7f9b4c07e" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "test", + "document": "community-garden", + "path": "windows-1257/lt/test/community-garden.bin", + "bytes": 177, + "highBytes": 17, + "uniqueHighBytes": 6, + "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "city-morning", + "path": "windows-1257/lt/train/city-morning.bin", + "bytes": 156, + "highBytes": 11, + "uniqueHighBytes": 4, + "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "library", + "path": "windows-1257/lt/train/library.bin", + "bytes": 163, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "market", + "path": "windows-1257/lt/train/market.bin", + "bytes": 171, + "highBytes": 12, + "uniqueHighBytes": 8, + "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "workshop", + "path": "windows-1257/lt/train/workshop.bin", + "bytes": 170, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/lt/validation/river-trip.bin", + "bytes": 165, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "test", + "document": "community-garden", + "path": "windows-1257/lv/test/community-garden.bin", + "bytes": 174, + "highBytes": 17, + "uniqueHighBytes": 8, + "sha256": "a42bdb992cc4b4c21108bc9a5850cb2ddff8d6b568c17a7505a69efdd8a815ba" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "city-morning", + "path": "windows-1257/lv/train/city-morning.bin", + "bytes": 163, + "highBytes": 15, + "uniqueHighBytes": 7, + "sha256": "bbd09e02d22432ac8e2295607379bdd17cf7ec9ea40d8f86c44d7bcce144c221" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "library", + "path": "windows-1257/lv/train/library.bin", + "bytes": 161, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "05ef86295a393e71f052f9932d4c2492f44fae597764429f188ab3ddcd0be5bd" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "market", + "path": "windows-1257/lv/train/market.bin", + "bytes": 170, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "1fbdc5105ca4e06c0c87edfe63a807d1543944a170f8e41f3bd50c0523753ba4" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "workshop", + "path": "windows-1257/lv/train/workshop.bin", + "bytes": 163, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "14a636582f0cdc34bcb262c63e52f58d31a81b04c91504839032ec88764f82df" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/lv/validation/river-trip.bin", + "bytes": 156, + "highBytes": 14, + "uniqueHighBytes": 4, + "sha256": "58b262baaa8f97950fb68db83d3a55a43ad3f140bc74010a9bb2a57fb1b86cc4" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "test", + "document": "community-garden", + "path": "windows-1258/vi/test/community-garden.bin", + "bytes": 128, + "highBytes": 36, + "uniqueHighBytes": 14, + "sha256": "9e50ec1534c1b7fc362908307bc6890705ecf794c0bb1e159917ec107fd77875" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "city-morning", + "path": "windows-1258/vi/train/city-morning.bin", + "bytes": 132, + "highBytes": 37, + "uniqueHighBytes": 13, + "sha256": "c2568706a878d17ba6f98ffd6881f24da76d012ee67d8469aed7556fadcb92bb" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "library", + "path": "windows-1258/vi/train/library.bin", + "bytes": 111, + "highBytes": 25, + "uniqueHighBytes": 11, + "sha256": "f319f96fe6fcbca8b0d120b30065f27299f9a356733c406bca99fa1e0d8a4d60" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "market", + "path": "windows-1258/vi/train/market.bin", + "bytes": 94, + "highBytes": 24, + "uniqueHighBytes": 12, + "sha256": "6d5fad1cf1c98c236cbe21b8ce770cf07998b21eb7a0cb84fc2723f432388a18" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "workshop", + "path": "windows-1258/vi/train/workshop.bin", + "bytes": 105, + "highBytes": 30, + "uniqueHighBytes": 11, + "sha256": "f4c388a881bcb3bde3a1c02e1b371de703b0d2c75425be1206428cb828c9d7a1" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "validation", + "document": "river-trip", + "path": "windows-1258/vi/validation/river-trip.bin", + "bytes": 93, + "highBytes": 23, + "uniqueHighBytes": 12, + "sha256": "5379d3a9a9d3a72910233391382978a455ef5c80afb3cd31931bfe08a112fcd8" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "test", + "document": "community-garden", + "path": "windows-874/th/test/community-garden.bin", + "bytes": 103, + "highBytes": 97, + "uniqueHighBytes": 35, + "sha256": "77d8715caa7248f93f1e2a9518680d6026d3688a1711b6fe5e65722fc030771a" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "city-morning", + "path": "windows-874/th/train/city-morning.bin", + "bytes": 75, + "highBytes": 69, + "uniqueHighBytes": 32, + "sha256": "69a85d6da187e6944204dc80897bd3ecfe407193e3bb7964acc21b964bc0f58e" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "library", + "path": "windows-874/th/train/library.bin", + "bytes": 99, + "highBytes": 95, + "uniqueHighBytes": 36, + "sha256": "5e22a173b7c6cc9a867f0b13cecf42da7fae43ad4a15e0affade46786443bcd6" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "market", + "path": "windows-874/th/train/market.bin", + "bytes": 66, + "highBytes": 60, + "uniqueHighBytes": 30, + "sha256": "9299c007d348b83f096f7ceca669b27be5bc82f8bf2dd5f006604fb8302cc2ce" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "workshop", + "path": "windows-874/th/train/workshop.bin", + "bytes": 82, + "highBytes": 77, + "uniqueHighBytes": 38, + "sha256": "6ece49322ff6115b207f13799d415787aeaa84d251b8a8b7f5e318d7acca9530" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "validation", + "document": "river-trip", + "path": "windows-874/th/validation/river-trip.bin", + "bytes": 72, + "highBytes": 69, + "uniqueHighBytes": 27, + "sha256": "257c08098491bde6fa4830723d074c608082f28073699548f1242c9e0565366e" }, { "encoding": "x-mac-cyrillic", diff --git a/corpus/generated/ngrams.mjs b/corpus/generated/ngrams.mjs index 8fb8a41..d6be5ed 100644 --- a/corpus/generated/ngrams.mjs +++ b/corpus/generated/ngrams.mjs @@ -1,4 +1,30 @@ export default [ + { + encoding: 'Big5', + language: 'zh', + documents: 4, + trigrams: [ + 0xa1430a, 0x52aaba, 0x77c052, 0xa141a8, 0xa143c5, 0xa5abb3, 0xa677c0, + 0xaabaa5, 0xaabab0, 0xabb0a5, 0xabb3f5, 0xaed1c0, 0xb0a5ab, 0xb27aad, + 0xb9cfae, 0xbaa5ab, 0xc052aa, 0xcfaed1, 0xd1c05d, 0x0a0aa4, 0x0a0aaa, + 0x0a0aab, 0x0a0abc, 0x0aa440, 0x0aaac0, 0x0aabb0, 0x0abc73, 0x40a854, + 0x40abc7, 0x40add3, 0x40af5a, 0x40b0d8, 0x40b3d5, 0x41a5c1, 0x41a8ae, + 0x41a8c3, 0x41a940, 0x41b2c4, 0x41b67d, 0x41bade, 0x41bd73, 0x41bdad, + 0x42a4f4, 0x42acec, 0x42b8c1, 0x43b35d, 0x43ba43, 0x43bff4, 0x43c555, + 0x43c5aa, 0x44a143, 0x45b6b0, 0x46a457, 0x47a142, 0x48a141, 0x4ca677, + 0x4da7e4, 0x4daec8, 0x4db7a1, 0x4dc4d1, 0x50a67e, 0x50b4c1, 0x54a8ae, + 0x55abc8, 0x55bec7, 0x56b941, 0x57aaba, 0x57af5a, 0x5aa4bd, 0x5aaaba, + 0x5baacc, 0x5cbaa1, 0x5d0a0a, 0x5da143, 0x5dabfc, 0x5dad70, 0x5db8cc, + 0x60b5e6, 0x61b160, 0x62bb73, 0x64a7e4, 0x64b57b, 0x65a94d, 0x67b24d, + 0x6aaaf9, 0x6ba143, 0x6cacef, 0x6ea143, 0x6eadb5, 0x6faacc, 0x70ae76, + 0x70b9ce, 0x73a740, 0x73b3f5, 0x73bfe8, 0x73c241, 0x75a6b3, 0x75a740, + 0x76a142, 0x76bee3, 0x79ae61, 0x7aadb6, 0x7aadfb, 0x7ba6a1, 0x7da46a, + 0x7da96c, 0x7db56f, 0x7ec4d6, 0xa141a9, 0xa141b2, 0xa141b6, 0xa141ba, + 0xa141bd, 0xa142a4, 0xa142ac, 0xa142b8, 0xa143b3, 0xa1a141, 0xa1a9fa, + 0xa1aaba, 0xa1b773, 0xa3a650, 0xa440ad, 0xa440af, 0xa446a4, 0xa448a1, + 0xa457aa, 0xa457af, + ], + }, { encoding: 'CP850', language: 'de', @@ -129,6 +155,84 @@ export default [ 0x20c4a1, 0x20c4ab, ], }, + { + encoding: 'EUC-JP', + language: 'ja', + documents: 4, + trigrams: [ + 0xa1a30a, 0xbcd4a4, 0xa1bca5, 0xa4aba4, 0xa4acca, 0xa4cba4, 0xa4cea5, + 0xa4eba1, 0xa4f2b3, 0xd4a4ac, 0xeba1a3, 0xa4a4a4, 0xa4afa4, 0xa4b7a1, + 0xa4b9a1, 0xa4b9a4, 0xa4c3a4, 0xa4c6a4, 0xa4cebb, 0xa4cec4, 0xa4dea4, + 0xa4e1a4, 0xa4eaa4, 0xaaa1a2, 0xaba4ca, 0xb3b9a4, 0xb4dba4, 0xb7a1a2, + 0xb9a1a3, 0xb9a4ac, 0xbbd4be, 0xbdf1a4, 0xbdf1b4, 0xbeeca4, 0xbfdebd, + 0xc0c5a4, 0xc5a4ab, 0xcba4cf, 0xcbdca4, 0xcebbd4, 0xd4beec, 0xdca4f2, + 0xdebdf1, 0xe2a4f2, 0xf1b4db, 0x0a0ab9, 0x0a0abe, 0x0a0ac3, 0x0a0ac4, + 0x0ab9ad, 0x0abeae, 0x0ac3cf, 0x0ac4ab, 0xa1a2b2, 0xa1a2b3, 0xa1a2b5, + 0xa1a2ba, 0xa1a2bb, 0xa1a2be, 0xa1a2ca, 0xa1a2cb, 0xa1a3b1, 0xa1a3c0, + 0xa1a3c6, 0xa1a3c7, 0xa2b2cc, 0xa2b3ab, 0xa2b5ca, 0xa2bac7, 0xa2bbca, + 0xa2bec6, 0xa2cad4, 0xa2cbaa, 0xa3b1d8, 0xa3c0df, 0xa3c6c9, 0xa3c7e3, + 0xa4a4c0, 0xa4a4c6, 0xa4a4ca, 0xa4a4eb, 0xa4a6a1, 0xa4a8a1, 0xa4a8a4, + 0xa4aca5, 0xa4acb2, 0xa4acbd, 0xa4acc0, 0xa4acc7, 0xa4acc8, 0xa4ada4, + 0xa4afa1, 0xa4b1a1, 0xa4b3a4, 0xa4b5a4, 0xa4bfa4, 0xa4c0e2, 0xa4c4a4, + 0xa4c6cf, 0xa4c7a4, 0xa4c8b5, 0xa4caa5, 0xa4caaa, 0xa4cabf, 0xa4cac4, + 0xa4cacc, 0xa4ceb0, 0xa4ceb3, 0xa4cebf, 0xa4cecb, 0xa4cece, 0xa4cfa4, + 0xa4cfb3, 0xa4cfbf, 0xa4cfc4, 0xa4cfc7, 0xa4cfcb, 0xa4cfce, 0xa4d6a1, + 0xa4d9ca, 0xa4e4a4, 0xa4e4b2, 0xa4e4ce, 0xa4e6a4, 0xa4eaa1, 0xa4eabb, + 0xa4eacc, 0xa4ebb2, + ], + }, + { + encoding: 'EUC-KR', + language: 'ko', + documents: 4, + trigrams: [ + 0xb4d92e, 0xb4c220, 0xc0bb20, 0xd92e20, 0xb8a620, 0xc0ba20, 0x20bbe7, + 0xb5e9c0, 0xbdc3c0, 0xbfcd20, 0xd92e0a, 0x20bdc3, 0xb0a120, 0xb0ed20, + 0xb4c2b4, 0xb8e720, 0xc0cc20, 0xc2b4d9, 0xc7d120, 0xc7d1b4, 0xcd20b0, + 0xd1b4d9, 0xe9c0ba, 0x0a0ab5, 0x20b1db, 0x20b5b5, 0x20b8b8, 0x20bcd2, + 0x20bec6, 0x20bfad, 0x20c0db, 0x20c3a5, 0x20c8ae, 0x2e20b5, 0xa1b4c2, + 0xa5c0bb, 0xa620c1, 0xadb0fc, 0xb0d420, 0xb5b5bc, 0xb5b5bd, 0xb5bcad, + 0xb5bdc3, 0xb7ce20, 0xb8b5e7, 0xb8b8b5, 0xbab8a6, 0xbb20b8, 0xbcadb0, + 0xbcd2b8, 0xbec6c4, 0xbfa120, 0xbfa1b4, 0xbfee20, 0xc0dab4, 0xc0dab5, + 0xc0dbc0, 0xc0e520, 0xc0e5bf, 0xc220c3, 0xc3a5c0, 0xc3c0e5, 0xc6c4a7, + 0xc7cfb0, 0xc7cfb8, 0xc8aec0, 0xcc20b9, 0xcfb8e7, 0xdab4c2, 0xdbc0ba, + 0xe5bfa1, 0x0a0ab1, 0x0a0ac0, 0x0ab1a4, 0x0ab5b5, 0x0ab5bf, 0x0ac0db, + 0x20b0ab, 0x20b0b3, 0x20b0c7, 0x20b0cb, 0x20b0e8, 0x20b0ed, 0x20b0fa, + 0x20b0fc, 0x20b1b8, 0x20b2dc, 0x20b3aa, 0x20b3d1, 0x20b3f3, 0x20b3f5, + 0x20b4ba, 0x20b5b6, 0x20b5e9, 0x20b5f0, 0x20b8c2, 0x20b8de, 0x20b9ae, + 0x20b9da, 0x20b9e0, 0x20b9e6, 0x20b9f6, 0x20b9f8, 0x20bba7, 0x20bcb3, + 0x20bcd5, 0x20bdac, 0x20beb4, 0x20bec8, 0x20bfa9, 0x20bfaa, 0x20bfe4, + 0x20bfee, 0x20c0a7, 0x20c0cc, 0x20c0d0, 0x20c0da, 0x20c0dc, 0x20c0fc, + 0x20c1a4, 0x20c1d8, 0x20c1fd, 0x20c3a3, 0x20c3a4, 0x20c3b5, 0x20c3b9, + 0x20c4a1, 0x20c4ab, + ], + }, + { + encoding: 'GB18030', + language: 'zh', + documents: 4, + trigrams: [ + 0xa1a30a, 0x4495f8, 0x6fb5c4, 0x884495, 0x95f8f0, 0xb0b2ec, 0xb2ec6f, + 0xb3c7ca, 0xb5c4ca, 0xc4cad0, 0xc7cad0, 0xcad088, 0xcfa1a3, 0xd088f6, + 0xec6fb5, 0xf8f05e, 0x0a0a8f, 0x0a0ab3, 0x0a0ac9, 0x0a0ad2, 0x0a8f56, + 0x0ab3c7, 0x0ac9e7, 0x0ad2bb, 0x46eaa0, 0x49b0fc, 0x4bbdbb, 0x4dd0c2, + 0x4fd38b, 0x548ecd, 0x54a3ac, 0x5688f6, 0x57bacd, 0x57c9fa, 0x59c1cf, + 0x5b9d4d, 0x5e0a0a, 0x5e8844, 0x5eb0b2, 0x5ed165, 0x5ed5df, 0x5ed6b8, + 0x5fb06c, 0x5fb4f3, 0x5fcabc, 0x65d6bb, 0x66c3f7, 0x67b5c4, 0x6cd5df, + 0x72c3f1, 0x72cadf, 0x75d7f7, 0x76cab7, 0x78d5df, 0x7ab2e9, 0x80d0a1, + 0x814bbd, 0x81eda3, 0x8280d0, 0x838696, 0x855e88, 0x85a2d3, 0x86548e, + 0x8696ca, 0x87d5be, 0x87e95f, 0x8846ea, 0x88f60a, 0x88f694, 0x88f6c9, + 0x8b8e9f, 0x8b9ee9, 0x8c57ba, 0x8c57c9, 0x8c91c7, 0x8ca4d5, 0x8e9fd5, + 0x8ecdd6, 0x8edd8b, 0x8f5688, 0x91c7e5, 0x93c3e6, 0x945b9d, 0x94b5ce, + 0x95d2f4, 0x95f8a3, 0x95f8b5, 0x96cad5, 0x997ab2, 0x99bfcd, 0x9a76ca, + 0x9d4dd0, 0x9ee9b2, 0x9fd5fb, 0xa0d5fd, 0xa18846, 0xa1a2b7, 0xa1a2bf, + 0xa1a2cb, 0xa1a3d4, 0xa1a3d7, 0xa1a3ee, 0xa2b7e4, 0xa2bfc6, 0xa2cbae, + 0xa2d35e, 0xa3ac81, 0xa3acb5, 0xa3acb9, 0xa3acbe, 0xa3acbf, 0xa3acdc, + 0xa3ace9, 0xa3b2cb, 0xa3d44f, 0xa3d778, 0xa3ee99, 0xa4d5d2, 0xa4d7f7, + 0xa7b7c8, 0xa8a1a3, 0xa9ceef, 0xa9df5e, 0xab8c91, 0xabb9b2, 0xac814b, + 0xacb5da, 0xacb9dc, + ], + }, { encoding: 'IBM855', language: 'ru', @@ -207,6 +311,266 @@ export default [ 0x0a83ae, 0x0a8da0, ], }, + { + encoding: 'ISO-8859-1', + language: 'da', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x676572, 0x6f6720, 0x64656e, 0x646572, + 0x6c6520, 0x6e6465, 0x737465, 0x74656b, 0x0a0a42, 0x206269, 0x206279, + 0x206465, 0x20686f, 0x2070e5, 0x2c206f, 0x626962, 0x626c69, 0x640a0a, + 0x652062, 0x652066, 0x65640a, 0x656e73, 0x65722c, 0x65722e, 0x65726e, + 0x69626c, 0x696465, 0x696c6c, 0x696f74, 0x6b7374, 0x6c696f, 0x6c6c65, + 0x6e6572, 0x6f7465, 0x70e520, 0x722065, 0x722c20, 0x722e0a, 0x726e65, + 0x736572, 0x746572, 0x766572, 0x79656e, 0xe52064, 0xf86765, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4ce6, 0x206272, 0x206275, + 0x2062f8, 0x206361, 0x20656c, 0x20656e, 0x206669, 0x20666f, 0x206672, + 0x2066f8, 0x206869, 0x20686a, 0x206920, 0x206b6f, 0x206c61, 0x206c69, + 0x206d65, 0x206d75, 0x206f6d, 0x206f73, 0x20706c, 0x20736b, 0x207374, + 0x2073e6, 0x207465, 0x207472, 0x207479, 0x20756e, 0x207665, 0x207669, + 0x2076e5, 0x2076e6, 0x20e562, 0x20e662, 0x2c2063, 0x2c206d, 0x2c2075, + 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, 0x457420, 0x4ce673, + 0x4cf872, 0x4d6f72, 0x61622c, 0x616473, 0x6166e9, 0x616773, 0x616c74, + 0x616e67, 0x617265, 0x61726b, 0x61766c, 0x622c20, 0x626c65, 0x626e65, + 0x6272f8, 0x627573, 0x627965, 0x627967, 0x62f867, 0x636166, 0x642062, + 0x642c20, 0x646167, 0x646520, 0x64652c, 0x64656c, 0x64732e, 0x652068, + 0x65206f, 0x652070, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'de', + documents: 4, + trigrams: [ + 0x656e20, 0x636865, 0x657220, 0x206465, 0x20756e, 0x696520, 0x736368, + 0x756e64, 0x65696e, 0x68656e, 0x696368, 0x696e65, 0x6e2064, 0x6e6420, + 0x6e656e, 0x206469, 0x626572, 0x646572, 0x687265, 0x6c6569, 0x205465, + 0x206569, 0x446965, 0x616368, 0x636874, 0x64656e, 0x646965, 0x652c20, + 0x656974, 0x65722e, 0x65726e, 0x686572, 0x696e20, 0x697461, 0x697465, + 0x6b6c65, 0x6c6572, 0x6e2042, 0x6e2069, 0x6e2e20, 0x6e6520, 0x726520, + 0x726569, 0x742065, 0x746520, 0x74656e, 0x204269, 0x204765, 0x204d65, + 0x205265, 0x205374, 0x206265, 0x2066fc, 0x206968, 0x206b6c, 0x206c65, + 0x207665, 0x20fc62, 0x2e2042, 0x2e2044, 0x426962, 0x45696e, 0x4d656e, + 0x537461, 0x616474, 0x616c65, 0x616d20, 0x617573, 0x626c69, 0x636872, + 0x636b65, 0x642066, 0x65204d, 0x652053, 0x652054, 0x652075, 0x656368, + 0x656c6c, 0x656e2e, 0x657265, 0x657277, 0x657320, 0x66fc72, 0x676974, + 0x686520, 0x68656b, 0x68656c, 0x687465, 0x69626c, 0x69636b, 0x696769, + 0x696872, 0x696f74, 0x6c6520, 0x6c6963, 0x6c696f, 0x6e206c, 0x6e2076, + 0x6e6465, 0x6e7465, 0x6e7477, 0x6f7468, 0x722047, 0x722053, 0x722064, + 0x722e0a, 0x722e20, 0x726e20, 0x736520, 0x73652c, 0x737461, 0x737465, + 0x737563, 0x740a0a, 0x742061, 0x746164, 0x74616c, 0x746572, 0x746865, + 0x747769, 0x756368, 0x757373, 0x766572, 0x776963, 0xe46e64, 0xfc6265, + 0xfc7220, 0x0a0a41, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'en', + documents: 4, + trigrams: [ + 0x20616e, 0x616e64, 0x686520, 0x6e6420, 0x727320, 0x207468, 0x2c2061, + 0x746865, 0x652061, 0x652c20, 0x656164, 0x657273, 0x657320, 0x732061, + 0x206120, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206c69, 0x206e65, + 0x207265, 0x207768, 0x546865, 0x616465, 0x626f75, 0x627261, 0x636974, + 0x642063, 0x64206e, 0x646520, 0x646573, 0x652063, 0x657369, 0x68696c, + 0x696272, 0x696465, 0x696c65, 0x697479, 0x6c6520, 0x6c6962, 0x6c6c20, + 0x6e2074, 0x722064, 0x726172, 0x726561, 0x732062, 0x732072, 0x732073, + 0x732c20, 0x732e0a, 0x776869, 0x790a0a, 0x0a0a41, 0x0a0a46, 0x0a0a52, + 0x0a0a54, 0x0a4120, 0x0a4661, 0x0a5265, 0x0a5468, 0x206162, 0x206170, + 0x206172, 0x206265, 0x20626f, 0x206361, 0x20636c, 0x206469, 0x20646f, + 0x206578, 0x206661, 0x206775, 0x206869, 0x20696e, 0x206974, 0x206a61, + 0x206d61, 0x206d75, 0x206e61, 0x206f70, 0x207072, 0x2070e2, 0x207175, + 0x2072e9, 0x2072f4, 0x207363, 0x207365, 0x20736c, 0x20736d, 0x207465, + 0x207761, 0x20776f, 0x2c2063, 0x2c206a, 0x2c2073, 0x3b2063, 0x412073, + 0x446967, 0x466172, 0x4d6f72, 0x526561, 0x536174, 0x612066, 0x612072, + 0x61626f, 0x616420, 0x61642c, 0x6166e9, 0x616b65, 0x616c20, 0x616c61, + 0x616c6c, 0x616d20, 0x616e20, 0x616e67, 0x617065, 0x617070, 0x617220, + 0x617265, 0x617269, 0x61726b, 0x61726d, 0x617274, 0x617279, 0x617475, + 0x617920, 0x61e761, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'es', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x617320, 0x656c20, 0x656e20, 0x657320, 0x206c61, + 0x206c6f, 0x207920, 0x2e204c, 0x61646f, 0x616e20, 0x6c6f73, 0x6e206c, + 0x6f7265, 0x726573, 0x706172, 0x732063, 0x204c6f, 0x206369, 0x20656c, + 0x207061, 0x4c6f73, 0x612065, 0x612070, 0x617261, 0x626c69, 0x6c206d, + 0x6c6120, 0x6e6120, 0x732061, 0x732e20, 0x746f72, 0x206269, 0x20636c, + 0x20636f, 0x206469, 0x206d65, 0x207265, 0x207375, 0x207665, 0x456c20, + 0x4c6120, 0x612062, 0x626962, 0x636961, 0x646520, 0x64656c, 0x646967, + 0x656361, 0x65732e, 0x69626c, 0x696369, 0x696f74, 0x6c2063, 0x6c696f, + 0x6e2065, 0x6f2073, 0x6f2e20, 0x6f7465, 0x726120, 0x732064, 0x732070, + 0x732076, 0x732e0a, 0x746563, 0x0a0a45, 0x204c61, 0x206361, 0x206c65, + 0x206d61, 0x20706f, 0x207072, 0x207365, 0x207465, 0x2c206c, 0x612063, + 0x612064, 0x61206c, 0x612079, 0x616e61, 0x61726f, 0x61732c, 0x61732e, + 0x627265, 0x627573, 0x636120, 0x636164, 0x636965, 0x636975, 0x636c61, + 0x636f2e, 0x636f6d, 0x646164, 0x64656e, 0x646f20, 0x646f72, 0x656369, + 0x656e61, 0x657175, 0x657263, 0x657274, 0x676974, 0x676f20, 0x696120, + 0x696173, 0x69656c, 0x69656e, 0x696769, 0x697461, 0x697564, 0x6c6172, + 0x6c6173, 0x6c6c65, 0x6d6572, 0x6d7061, 0x6e2070, 0x6e2072, 0x6e6173, + 0x6e6f73, 0x6f2064, 0x6f2079, 0x6f636f, 0x6f6d70, 0x6f732e, 0x706f63, + 0x717565, 0x717569, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'fr', + documents: 4, + trigrams: [ + 0x657320, 0x206c65, 0x656e74, 0x6c6573, 0x6e7420, 0x74206c, 0x732070, + 0x206465, 0x657420, 0x657572, 0x727320, 0x742064, 0x206475, 0x206574, + 0x647520, 0x6c6520, 0x732063, 0x757273, 0x204c65, 0x20636f, 0x207072, + 0x2e204c, 0x4c6573, 0x646573, 0x652063, 0x652070, 0x207061, 0x20706f, + 0x2c206c, 0x626c69, 0x65206d, 0x656e63, 0x65732e, 0x6c6c65, 0x6f7572, + 0x706172, 0x717565, 0x726520, 0x732e20, 0x746573, 0x756520, 0x757220, + 0x206269, 0x206361, 0x20656e, 0x206d61, 0x207361, 0x4c6520, 0x616765, + 0x617265, 0x626962, 0x637465, 0x646520, 0x650a0a, 0x652064, 0x652065, + 0x652073, 0x69626c, 0x69656e, 0x696c6c, 0x696f74, 0x6c6575, 0x6c696f, + 0x6d656e, 0x6e6365, 0x6e6520, 0x6f6e73, 0x6f7468, 0x7072e9, 0x72206c, + 0x726368, 0x72e970, 0x732065, 0x73206c, 0x732073, 0x746575, 0xe97061, + 0xe97269, 0x0a0a4c, 0x20636c, 0x206672, 0x206c61, 0x206e75, 0x207265, + 0x2072e9, 0x20756e, 0x207669, 0x2c2064, 0x4c6120, 0x556e65, 0x612062, + 0x616972, 0x616973, 0x617263, 0x636865, 0x6368e9, 0x636965, 0x636f6e, + 0x652e20, 0x656e20, 0x65732c, 0x676573, 0x686973, 0x68e871, 0x68e920, + 0x69656c, 0x696572, 0x696e20, 0x697175, 0x697265, 0x697374, 0x697420, + 0x6c6120, 0x6c656e, 0x6c6965, 0x6d6172, 0x6d6d65, 0x6de972, 0x6e2070, + 0x6e7320, 0x6e7473, 0x6e756d, 0x6f6d6d, 0x6f7576, 0x706f73, 0x706f75, + 0x70726f, 0x72656e, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'it', + documents: 4, + trigrams: [ + 0x6c6120, 0x207069, 0x656e74, 0x206465, 0x206520, 0x612070, 0x6c6c61, + 0x6e6f20, 0x6f7269, 0x746f72, 0x206369, 0x206c61, 0x612067, 0x612069, + 0x61746f, 0x636974, 0x64656c, 0x657263, 0x692061, 0x692063, 0x692073, + 0x696120, 0x697474, 0x69f920, 0x6c6920, 0x7069f9, 0x726920, 0x7474e0, + 0x206269, 0x206361, 0x20636f, 0x206469, 0x206765, 0x206920, 0x206c65, + 0x206d65, 0x207065, 0x207072, 0x207363, 0x207369, 0x207374, 0x2c2063, + 0x4c6120, 0x612062, 0x612063, 0x612064, 0x61206c, 0x616666, 0x616e6f, + 0x626962, 0x626c69, 0x636166, 0x636f6c, 0x637269, 0x65206c, 0x652073, + 0x652c20, 0x656361, 0x656c20, 0x656c65, 0x656c6c, 0x657363, 0x657474, + 0x6666e8, 0x67696f, 0x676c69, 0x692070, 0x69626c, 0x696f20, 0x696f74, + 0x6c696f, 0x6d656e, 0x6e6120, 0x6e7465, 0x6e7472, 0x6f2064, 0x6f2065, + 0x6f2070, 0x6f2c20, 0x6f6e6f, 0x6f7465, 0x706572, 0x726120, 0x726361, + 0x726520, 0x726961, 0x736372, 0x746120, 0x746563, 0x7a6120, 0x0a0a47, + 0x0a0a49, 0x0a0a4c, 0x0a0a55, 0x0a476c, 0x0a4920, 0x0a4c61, 0x0a556e, + 0x206120, 0x206167, 0x206169, 0x206170, 0x206173, 0x206175, 0x206365, + 0x206368, 0x20666f, 0x206672, 0x206769, 0x20676c, 0x206775, 0x20696c, + 0x20696e, 0x206c69, 0x206d69, 0x206d75, 0x206e65, 0x207061, 0x207361, + 0x207371, 0x207376, 0x207472, 0x207574, 0x207665, 0x2c2066, 0x2c2067, + 0x2c2069, 0x2c206d, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'nl', + documents: 4, + trigrams: [ + 0x656e20, 0x20656e, 0x646520, 0x206465, 0x656572, 0x74656e, 0x766572, + 0x206f70, 0x207374, 0x207465, 0x652062, 0x652065, 0x652073, 0x657273, + 0x6e206f, 0x6e2e0a, 0x736368, 0x742064, 0x746572, 0x206269, 0x206361, + 0x20636f, 0x206565, 0x206765, 0x206865, 0x206c65, 0x206f76, 0x207665, + 0x207765, 0x2c2063, 0x446520, 0x61616b, 0x616173, 0x6166e9, 0x616b74, + 0x626962, 0x626c69, 0x636166, 0x636874, 0x652063, 0x65696e, 0x656b65, + 0x656e2e, 0x656e73, 0x657220, 0x657264, 0x657265, 0x65726b, 0x66e973, + 0x687465, 0x69626c, 0x696465, 0x696e20, 0x696e67, 0x696f74, 0x697320, + 0x6b6520, 0x6b656e, 0x6b7420, 0x6c6520, 0x6c6565, 0x6c6569, 0x6c696f, + 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206d, 0x6e2074, 0x6e2076, 0x6e2077, + 0x6f656b, 0x6f6e74, 0x6f7020, 0x6f7065, 0x6f7468, 0x6f7665, 0x70656e, + 0x72656e, 0x727320, 0x732065, 0x73206f, 0x732c20, 0x73656e, 0x737365, + 0x737461, 0x737465, 0x746164, 0x746865, 0xe97320, 0x0a0a42, 0x0a0a44, + 0x0a0a45, 0x0a0a4c, 0x0a426f, 0x0a4465, 0x0a4565, 0x0a4c65, 0x206170, + 0x20626f, 0x206272, 0x206275, 0x206472, 0x206475, 0x20686f, 0x206875, + 0x206964, 0x20696e, 0x206b61, 0x206b6c, 0x206c61, 0x206d61, 0x206d65, + 0x206d75, 0x206e61, 0x206f6e, 0x20706c, 0x20706f, 0x207363, 0x207477, + 0x207761, 0x207a6f, 0x20e9e9, 0x2c2061, 0x2c206b, 0x426f65, 0x446967, + 0x45656e, 0x4c657a, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'no', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x6f6720, 0x656e65, 0x206c61, 0x652066, + 0x676572, 0x6e6520, 0x6e6572, 0x737465, 0x74656b, 0x0a0a42, 0x206269, + 0x206272, 0x206465, 0x20666f, 0x2070e5, 0x207665, 0x626962, 0x626c69, + 0x640a0a, 0x646572, 0x652062, 0x652074, 0x652c20, 0x65640a, 0x656e73, + 0x657273, 0x657420, 0x672073, 0x676520, 0x67656e, 0x69626c, 0x696765, + 0x696e67, 0x696f74, 0x697465, 0x6b6572, 0x6b7374, 0x6c6167, 0x6c6520, + 0x6c6967, 0x6c696f, 0x6e206f, 0x6e6465, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x72206f, 0x72656e, 0x726765, 0x746520, 0x746572, + 0x746f72, 0x76656e, 0x766572, 0x79656e, 0xe52064, 0xf86b65, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4c65, 0x206275, 0x206279, + 0x2062f8, 0x2064f8, 0x20656c, 0x20656e, 0x206570, 0x206665, 0x206669, + 0x2066f8, 0x206869, 0x20686a, 0x20686f, 0x206920, 0x206b61, 0x206b6f, + 0x206c69, 0x206d65, 0x206d75, 0x206ee6, 0x206f6d, 0x206f73, 0x207365, + 0x20736a, 0x20736b, 0x207374, 0x207465, 0x20746f, 0x207472, 0x207479, + 0x20756e, 0x207669, 0x2076e5, 0x20e570, 0x20e672, 0x2c2065, 0x2c206b, + 0x2c206f, 0x2c2075, 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, + 0x457420, 0x4c6573, 0x4cf872, 0x4d6f72, 0x616665, 0x616720, 0x616765, + 0x616773, 0x616c74, 0x616e67, 0x617020, 0x617265, 0x61726b, 0x61766c, + 0x627279, 0x6272f8, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'pt', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x206520, 0x207072, 0x696120, 0x6e6120, 0x206369, + 0x206f73, 0x612064, 0x612065, 0x61206f, 0x646520, 0x657320, 0x657363, + 0x726573, 0x732061, 0x732070, 0x732e0a, 0x0a0a4f, 0x0a4f73, 0x206269, + 0x206573, 0x206e61, 0x206f20, 0x207065, 0x207665, 0x4f7320, 0x612070, + 0x616465, 0x61646f, 0x616d20, 0x617261, 0x617320, 0x626962, 0x626c69, + 0x636120, 0x636964, 0x646120, 0x646164, 0x646967, 0x646f20, 0x652061, + 0x656d20, 0x657175, 0x666963, 0x69626c, 0x696461, 0x696d65, 0x696f74, + 0x6c696f, 0x6e7175, 0x6f2065, 0x6f7265, 0x6f7465, 0x717565, 0x717569, + 0x72616d, 0x726961, 0x726f73, 0x732063, 0x736372, 0x746563, 0x746f72, + 0x0a0a41, 0x0a0a55, 0x0a4120, 0x0a556d, 0x206120, 0x206162, 0x206163, + 0x206167, 0x20616a, 0x20616c, 0x206173, 0x206361, 0x20636c, 0x2063f3, + 0x206469, 0x20646f, 0x20656e, 0x206571, 0x206672, 0x206775, 0x206869, + 0x206c65, 0x206c69, 0x206d61, 0x206d65, 0x206d6f, 0x206d75, 0x2070e3, + 0x207175, 0x2073e1, 0x207472, 0x20f46e, 0x2c206d, 0x2c206f, 0x2c2071, + 0x2c2076, 0x412062, 0x412063, 0x4d616e, 0x4d6572, 0x4f6669, 0x556d61, + 0x610a0a, 0x612061, 0x612062, 0x612063, 0x61206d, 0x612074, 0x612e0a, + 0x616272, 0x61636f, 0x616461, 0x6166e9, 0x616761, 0x616772, 0x616a75, + 0x616c0a, 0x616c75, 0x616e68, 0x616e71, 0x616e74, 0x61722c, 0x61732e, + 0x61e761, 0x61e7e3, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'sv', + documents: 4, + trigrams: [ + 0x657220, 0x617220, 0x206f63, 0x636820, 0x656e20, 0x6f6368, 0x737461, + 0x64656e, 0x657420, 0x6b6172, 0x726e61, 0x736b61, 0x74206c, 0x746164, + 0x766572, 0x206269, 0x206465, 0x2066f6, 0x206c69, 0x2070e5, 0x20736b, + 0x207479, 0x207665, 0x612062, 0x612074, 0x612e0a, 0x61640a, 0x616465, + 0x617265, 0x626962, 0x626c69, 0x640a0a, 0x65726e, 0x66f672, 0x676120, + 0x69626c, 0x696761, 0x696f74, 0x6b6170, 0x6b6572, 0x6b6e61, 0x6c6967, + 0x6c696f, 0x6e612e, 0x6e6172, 0x6e736b, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x722066, 0x72206f, 0x736172, 0x73f66b, 0x746120, + 0x74656b, 0x746f72, 0xe52064, 0x0a0a42, 0x0a0a45, 0x0a0a4c, 0x0a0a53, + 0x0a42f6, 0x0a4574, 0x0a4ce4, 0x0a5374, 0x206265, 0x206272, 0x206275, + 0x2062f6, 0x20656c, 0x20656e, 0x2066e4, 0x206772, 0x206869, 0x20686a, + 0x20686f, 0x206920, 0x206b61, 0x206b6f, 0x206c61, 0x206ce5, 0x206d65, + 0x206d75, 0x206de4, 0x206f6d, 0x206f73, 0x207374, 0x2073e4, 0x2073f6, + 0x207465, 0x20746f, 0x207661, 0x2076e4, 0x20e470, 0x20f670, 0x2c2067, + 0x2c206b, 0x2c206f, 0x2c20e4, 0x42f66e, 0x446574, 0x446967, 0x457474, + 0x4ce473, 0x4cf672, 0x4d6f72, 0x537461, 0x61206f, 0x6120f6, 0x6166e9, + 0x616720, 0x616773, 0x616b6e, 0x616c20, 0x616d74, 0x616e20, 0x616e73, + 0x617020, 0x617061, 0x617269, 0x61726b, 0x61726e, 0x626573, 0x6272f6, + 0x627573, 0x62f663, + ], + }, { encoding: 'ISO-8859-10', language: 'is', @@ -367,6 +731,110 @@ export default [ encoding: 'ISO-8859-16', language: 'ro', documents: 4, + trigrams: [ + 0x696920, 0x756c20, 0x206465, 0x6f7269, 0x726969, 0x746520, 0x746f72, + 0x692063, 0x7265fe, 0x203130, 0x205072, 0x206573, 0x20a42e, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x2e2050, 0x3020a4, 0x313020, 0x507265, 0x652031, + 0x652c20, 0x656c65, 0x657374, 0x65fe75, 0x692e20, 0x696572, 0x6c2065, + 0x737465, 0xa42e0a, 0xba6920, 0xfe756c, 0x206d69, 0x207072, 0x207265, + 0x61fe61, 0x626c69, 0x636869, 0x646520, 0x652070, 0x652e20, 0x656361, + 0x657269, 0x657363, 0x69626c, 0x696e65, 0x696f74, 0x6c6520, 0x6c696f, + 0x6f7465, 0x736520, 0x746563, 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, + 0x20616c, 0x206175, 0x206361, 0x20636c, 0x206375, 0x2063e3, 0x206469, + 0x206578, 0x206961, 0x206d65, 0x206f72, 0x207065, 0x207363, 0x207365, + 0x207472, 0x20ba74, 0x2c2069, 0x426962, 0x506961, 0x612064, 0x616c20, + 0x617220, 0x617574, 0x617ae3, 0x636120, 0x636172, 0x636574, 0x63e320, + 0x646573, 0x646967, 0x652063, 0x652064, 0x652074, 0x6520ee, 0x65617a, + 0x656e69, 0x656e74, 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, + 0x65ba74, 0x676974, 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, + 0x6963e3, 0x696769, 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, + 0x6c2064, 0x6c6567, 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, + 0x6d696e, 0x6e6365, 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, + 0x7261ba, 0x72652c, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'cs', + documents: 4, + trigrams: [ + 0x206120, 0x207072, 0x616aed, 0x6aed20, 0x206b6e, 0x636520, 0x6b6e69, + 0x6e6968, 0x6eed20, 0x792e0a, 0x0a0a4d, 0x20706f, 0x20e865, 0x2c206b, + 0x610a0a, 0x612070, 0x612076, 0x652061, 0x65206d, 0x652070, 0x686f76, + 0x69686f, 0x6a6520, 0x6e610a, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, + 0x73792e, 0x756a65, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xf86920, + 0x0a0a46, 0x0a0ac8, 0x0a4661, 0x0a4d61, 0x0a4dec, 0x0ac874, 0x206175, + 0x206279, 0x206368, 0x206476, 0x2064ed, 0x206869, 0x20686c, 0x206a61, + 0x206b61, 0x206b6f, 0x206bf3, 0x206c69, 0x206d65, 0x206d75, 0x206dec, + 0x206e61, 0x206f20, 0x206f74, 0x206f76, 0x2070ed, 0x2070f8, 0x207261, + 0x207365, 0x207472, 0x2074fd, 0x207665, 0x20766f, 0x2076ec, 0x207a61, + 0x20bee1, 0x2c206d, 0x2c206f, 0x2c207a, 0x446967, 0x466172, 0x4d616c, + 0x4dec73, 0x52e16e, 0x536f62, 0x546963, 0x61206c, 0x6164ed, 0x616c75, + 0x616cfd, 0x61726d, 0x61736e, 0x6174ed, 0x617574, 0x617675, 0x6176e1, + 0x6176e9, 0x622c20, 0x626f74, 0x626f75, 0x627573, 0x62796c, 0x63686c, + 0x6368e1, 0x636f20, 0x642061, 0x642c20, 0x64616a, 0x646365, 0x647665, + 0x64e176, 0x64e920, 0x64ec2c, 0x64ed20, 0x64ed6c, 0x65206a, 0x65206b, + 0x652072, 0x65642c, 0x656461, 0x65656d, 0x656b61, 0x656d2c, 0x656ee1, + 0x657273, 0x6576ed, 0x65f865, 0x676974, 0x680a0a, 0x686973, 0x686c65, + 0x686ce9, 0x687920, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'hu', + documents: 4, + trigrams: [ + 0x206120, 0x2c2061, 0x6e616b, 0x6f7320, 0x0a0a41, 0x206bf6, 0x2076e1, + 0x20e973, 0x657265, 0x6bf66e, 0x6e7976, 0x726f73, 0x73206b, 0x742c20, + 0x76e172, 0xe1726f, 0xe97320, 0xf66e79, 0x0a4120, 0x206373, 0x206b65, + 0x206b69, 0x61206b, 0x612076, 0x616b2e, 0x617420, 0x657420, 0x697320, + 0x6b2c20, 0x6b2e0a, 0x6d6920, 0x6f7420, 0x73206c, 0x73206d, 0x74206b, + 0x74e172, 0x7674e1, 0x797674, 0xf36b20, 0x0a0a45, 0x0a417a, 0x0a4567, + 0x20616c, 0x20617a, 0x206275, 0x206469, 0x20656c, 0x20656d, 0x206672, + 0x206761, 0x206be1, 0x206be9, 0x206bf3, 0x206c61, 0x206c65, 0x206de9, + 0x206dfa, 0x206dfb, 0x206f6c, 0x207065, 0x207069, 0x207361, 0x207365, + 0x207475, 0x2074e9, 0x2074f6, 0x207669, 0x20e172, 0x20e962, 0x20ed72, + 0x20fa74, 0x2c206b, 0x2c2073, 0x412063, 0x412067, 0x412076, 0x417a20, + 0x446967, 0x456779, 0x526567, 0x537a6f, 0x612064, 0x612074, 0x61630a, + 0x616a74, 0x616b20, 0x616b2c, 0x616c6d, 0x616e0a, 0x616e20, 0x617061, + 0x617373, 0x6173f3, 0x617469, 0x6174f3, 0x617a20, 0x617a64, 0x62616e, + 0x626174, 0x626572, 0x627265, 0x627573, 0x630a0a, 0x637361, 0x637365, + 0x642c20, 0x646573, 0x646967, 0x6469e1, 0x646f6d, 0x646f74, 0x64e16b, + 0x65642c, 0x656469, 0x656767, 0x6567ed, 0x656b20, 0x656b2c, 0x656b65, + 0x656c20, 0x656c6c, 0x656c6d, 0x656c79, 0x656d62, 0x656e2e, 0x656e64, + 0x656e79, 0x656ef5, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'pl', + documents: 4, + trigrams: [ + 0x616ab1, 0x6ab120, 0x207072, 0x206920, 0x207779, 0x616d69, 0x636861, + 0x70727a, 0x727a79, 0xb66369, 0x206a61, 0x20706f, 0x20726f, 0x2c2061, + 0x616368, 0x626c69, 0x636920, 0x647a69, 0x656b20, 0x692073, 0x692e20, + 0x69626c, 0x696572, 0x696f74, 0x6c696f, 0x6e6961, 0x6f7261, 0x6f7465, + 0x706973, 0x727a65, 0x74656b, 0x0a0a4d, 0x206120, 0x206269, 0x20646c, + 0x206b6f, 0x206d69, 0x207069, 0x207369, 0x207370, 0x20737a, 0x2e204b, + 0x610a0a, 0x612070, 0x612072, 0x617267, 0x626962, 0x636520, 0x63692e, + 0x637920, 0x637a79, 0x642c20, 0x64616a, 0x646c61, 0x65206f, 0x656461, + 0x656b61, 0x657261, 0x66726f, 0x68616d, 0x692077, 0x692c20, 0x696379, + 0x696520, 0x69656e, 0x697379, 0x69ea20, 0x6b6172, 0x6c6120, 0x6c6e69, + 0x6d6920, 0x6d6965, 0x6e6369, 0x6e6520, 0x6e656b, 0x6e6963, 0x6e6965, + 0x6f727a, 0x6fb663, 0x707261, 0x72616a, 0x72616d, 0x726f77, 0x7369ea, + 0x737072, 0x73746f, 0x737920, 0x737a65, 0x746f72, 0x75647a, 0x776120, + 0x776961, 0x776965, 0x792063, 0x792070, 0x792073, 0x796672, 0x7a7920, + 0x7a7974, 0xb1206a, 0xb1206b, 0xb12077, 0x0a0a54, 0x0a0a57, 0x0a4d61, + 0x0a4d69, 0x0a5461, 0x0a5720, 0x204269, 0x20437a, 0x204b61, 0x204b6c, + 0x204e61, 0x205072, 0x20526f, 0x206175, 0x206275, 0x206368, 0x20636f, + 0x206379, 0x20637a, 0x206461, 0x206472, 0x20676f, 0x206869, 0x206b61, + 0x206b73, 0x206b75, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'ro', + documents: 4, trigrams: [ 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, @@ -441,6 +909,162 @@ export default [ 0x206b6c, 0x206b72, ], }, + { + encoding: 'ISO-8859-5', + language: 'ru', + documents: 4, + trigrams: [ + 0xeee220, 0xd0eee2, 0x20dfe0, 0xdfe0de, 0x20d820, 0x20ddd0, 0x20dfde, + 0xd0ef20, 0xe0ded4, 0xebd520, 0xefe220, 0x20d2eb, 0x20d3de, 0x20dcd0, + 0x20ded1, 0xd0e2d5, 0xd5dddd, 0xd820df, 0xd82c20, 0xdbd820, 0xdcd0e0, + 0xddd020, 0xddebd5, 0xe0eb20, 0xe220e1, 0xe2d5db, 0xe2d5e0, 0xe2ded2, + 0xef20df, 0x0a0abd, 0x20d020, 0x20d1d8, 0x20d2de, 0x20dade, 0x20dee1, + 0x20dee2, 0x20dfdb, 0x20e0d5, 0x20e1de, 0x20efe0, 0x2c20d0, 0x2c20d1, + 0xbdd020, 0xd00a0a, 0xd020df, 0xd0d5e2, 0xd0ddd8, 0xd0e0da, 0xd0e1e2, + 0xd1d8d1, 0xd1dbd8, 0xd1e1e3, 0xd2d0ee, 0xd2d5e0, 0xd2efe2, 0xd3dee2, + 0xd42c20, 0xd4d0ee, 0xd4d820, 0xd5dbd5, 0xd5dbd8, 0xd5e0d8, 0xd5e0eb, + 0xd5e1e2, 0xd6d4d0, 0xd820de, 0xd820e1, 0xd8d1db, 0xd8dee2, 0xd8e1e2, + 0xd8e2d0, 0xd8ef20, 0xdad00a, 0xdad520, 0xdbd8de, 0xddd0ef, 0xddd8da, + 0xddd8ef, 0xddddef, 0xddded2, 0xddefef, 0xded1e1, 0xded2ef, 0xdedbec, + 0xdedcd0, 0xdedddd, 0xdee0de, 0xdee1d5, 0xdee1e2, 0xdee1eb, 0xdee2d5, + 0xdee2de, 0xdfd8e1, 0xdfdee1, 0xe0dad0, 0xe0dcd0, 0xe0dee1, 0xe1d5dd, + 0xe1ddeb, 0xe1e2d0, 0xe1e2d5, 0xe1e2de, 0xe1e3d6, 0xe1eb20, 0xe1ef20, + 0xe220da, 0xe220dd, 0xe220df, 0xe2d5da, 0xe2dee0, 0xe2e0de, 0xe2e1ef, + 0xe3d6d4, 0xe5ded4, 0xeb20df, 0xeb2c20, 0xeb2e0a, 0xebd2d0, 0xebd920, + 0xef20ef, 0xefe0dc, 0xefeee2, 0xefef20, 0x0a0ab2, 0x0a0ab3, 0x0ab220, + 0x0ab3de, 0x0abdd0, + ], + }, + { + encoding: 'ISO-8859-6', + language: 'ar', + documents: 4, + trigrams: [ + 0x20c7e4, 0xc7e4e5, 0x20e8c7, 0x20e8ea, 0xe3cac8, 0xe620c7, 0xe8c7e4, + 0xe8e620, 0x0a0aea, 0xc90a0a, 0xd120c7, 0xc7ca20, 0xc7ccd9, 0xc7d120, + 0xc7e4c3, 0xc7e4d9, 0xc7e4e8, 0xc820c7, 0xc8c920, 0xc920c7, 0xc920e8, + 0xc92e0a, 0xca20e8, 0xcac820, 0xcac8c9, 0xcb20c7, 0xccd920, 0xcd20c7, + 0xcfeae6, 0xd1c7cc, 0xd5e1c7, 0xd920c7, 0xe4e5cf, 0xe4e5e3, 0xe5cfea, + 0xe5e3ca, 0xe6c7d3, 0xe8d5e1, 0xeae6c9, 0x0a0aca, 0x0acad3, 0x0aeac8, + 0x0aead5, 0x0aead9, 0x20c3c8, 0x20c3e5, 0x20c7ce, 0x20c8e7, 0x20c8ea, + 0x20cfe4, 0x20d1e2, 0x20d5da, 0x20d9e6, 0x20e1d1, 0x20e1ea, 0x20e3ca, + 0x20e4e4, 0x20e8ca, 0x20e8d5, 0x20eae6, 0xc120d9, 0xc120e8, 0xc3c8e8, + 0xc3cdc7, 0xc3cec8, 0xc3e5ea, 0xc4e8e6, 0xc6c90a, 0xc720c8, 0xc720e4, + 0xc720e8, 0xc720ea, 0xc72e0a, 0xc7c120, 0xc7c820, 0xc7c8e7, 0xc7cd20, + 0xc7ceca, 0xc7cfc6, 0xc7cfe4, 0xc7cfea, 0xc7d12e, 0xc7d1d9, 0xc7d1ea, + 0xc7d320, 0xc7d3c8, 0xc7d6cd, 0xc7d9cf, 0xc7e1e4, 0xc7e3e7, 0xc7e4ca, + 0xc7e4cd, 0xc7e4ce, 0xc7e4d2, 0xc7e4d3, 0xc7e4d4, 0xc7e4d7, 0xc7e4e1, + 0xc7e4e2, 0xc7e4e6, 0xc7e4e7, 0xc7e7ea, 0xc820e1, 0xc8c7cd, 0xc8c7cf, + 0xc8c7d1, 0xc8c92e, 0xc8ca0a, 0xc8cdcb, 0xc8d1e5, 0xc8d220, 0xc8e7c7, + 0xc8e7cf, 0xc8e8c7, 0xc8eae6, 0xc920c8, 0xc920d1, 0xca0a0a, 0xcac7d1, + 0xcac8c7, 0xcacd20, 0xcacde1, 0xcad3ca, 0xcad8d1, 0xcae1ca, 0xcaeac7, + 0xcaeae2, 0xcce8e6, + ], + }, + { + encoding: 'ISO-8859-7', + language: 'el', + documents: 4, + trigrams: [ + 0x20eae1, 0xe1e920, 0xeae1e9, 0x20e2e9, 0xe120ea, 0xe12e0a, 0xe2e9e2, + 0xe5e920, 0xe9e2eb, 0xeff5ed, 0xf4e120, 0xf5ed20, 0x0a0acf, 0x0acfe9, + 0x20e1e3, 0x20e1ed, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xcfe920, + 0xdfe12e, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, 0xe5f220, 0xe70a0a, 0xe720e2, + 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9efe8, 0xe9f3f4, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xf0fceb, 0xf1e9ef, 0xf220ea, 0xf3f4de, + 0xf3f4e7, 0xf4e5f2, 0xf4e7ed, 0xfcebe7, 0x0a0ac7, 0x0a0acc, 0x0ac720, + 0x0acce9, 0x20dced, 0x20e1f1, 0x20e2ef, 0x20e3f1, 0x20e5eb, 0x20e5f0, + 0x20e5f1, 0x20e5f4, 0x20e720, 0x20e9f3, 0x20eae5, 0x20eafe, 0x20ebe5, + 0x20ecdd, 0x20ece1, 0x20ece9, 0x20ecef, 0x20ecf5, 0x20eef5, 0x20efe4, + 0x20efe9, 0x20efec, 0x20f0e5, 0x20f0eb, 0x20f0ef, 0x20f6f1, 0x20f8dc, + 0x20f8f9, 0x2c20e5, 0x2c20ec, 0x2c20f4, 0x2c20f6, 0xb9f3f5, 0xc720f0, + 0xcce9e1, 0xd0f1f9, 0xd3e1e2, 0xd8e7f6, 0xdc0a0a, 0xdc20e1, 0xdc20ea, + 0xdc20f3, 0xdc20f4, 0xdc2c20, 0xdce4e1, 0xdce6e5, 0xdcede8, 0xdcf1e9, + 0xdcf4e9, 0xdcf6e5, 0xdcf7ed, 0xdd20e1, 0xdde3f7, 0xddebe9, 0xddedef, + 0xddf22e, 0xde20ef, 0xdeeae7, 0xdeece7, 0xdef1e9, 0xdf20f3, 0xdf2c20, + 0xdfe120, 0xdfe1f2, 0xdfe3ef, 0xdfece5, 0xdfeff5, 0xe120e5, 0xe120e9, + 0xe120eb, 0xe120ec, 0xe1e2e2, 0xe1e3ed, 0xe1e3ef, 0xe1e3f1, 0xe1e8e1, + 0xe1e8e7, 0xe1eafc, + ], + }, + { + encoding: 'ISO-8859-8', + language: 'he', + documents: 4, + trigrams: [ + 0xe9ed20, 0xe5f8e9, 0xed20ec, 0xed20ee, 0xf1f4f8, 0xf8e9ed, 0x20e4f7, + 0x20ece0, 0x20eee7, 0x2c20e1, 0xe0e9ed, 0xe4f1f4, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed2e, 0xed2e0a, 0xef20ee, 0xf0e420, 0xf2e9f8, + 0xf4f8e9, 0xf7e5f8, 0xf820ee, 0xf8e5fa, 0xf8fa20, 0xf9e9ed, 0xfa0a0a, + 0xfa20ec, 0x0a0ae4, 0x0a0ae7, 0x0a0af6, 0x0a0af7, 0x0ae4f2, 0x0ae7f7, + 0x0af6e5, 0x0af7e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, + 0x20e1fa, 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e8f8, 0x20ece1, + 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, 0x20eeeb, 0x20eef7, + 0x20eefa, 0x20f0f4, 0x20f1f4, 0x20f2e5, 0x20f4e9, 0x20f7e8, 0x20f9e1, + 0x20fae9, 0x2c20e3, 0x2c20f4, 0xe0e5e8, 0xe0e5ef, 0xe0e5f8, 0xe0e82c, + 0xe0f0f9, 0xe0f9e5, 0xe0fa20, 0xe120fa, 0xe1e5e3, 0xe1e5f1, 0xe1e5f7, + 0xe1e7e5, 0xe1e9f0, 0xe1ebe9, 0xe1f2e9, 0xe1f8e5, 0xe1f92c, 0xe1fa0a, + 0xe1fae9, 0xe2e1e9, 0xe2e9e8, 0xe320e5, 0xe3e1f9, 0xe3e9e2, 0xe3e9ed, + 0xe3f0e4, 0xe3f220, 0xe3f720, 0xe3f8e9, 0xe40a0a, 0xe420e1, 0xe420e3, + 0xe420e4, 0xe420e5, 0xe420f0, 0xe42e0a, 0xe4e9f1, 0xe4f2e9, 0xe4f2ee, + 0xe4f7e5, 0xe4f7f4, 0xe4f8e0, 0xe4f9f7, 0xe5e0f0, 0xe5e1e5, 0xe5e2e1, + 0xe5e320, 0xe5e3f7, 0xe5e4f1, 0xe5e5fa, 0xe5e6e9, 0xe5e6f8, 0xe5e8e5, + 0xe5ebe5, 0xe5ebf8, + ], + }, + { + encoding: 'ISO-8859-9', + language: 'tr', + documents: 4, + trigrams: [ + 0x6c6572, 0x6c6172, 0x207665, 0x722e0a, 0x766520, 0x2061e7, 0x207961, + 0x617220, 0x61e7fd, 0x657220, 0x697220, 0x722076, 0x206269, 0x20656b, + 0x206b61, 0x206bfc, 0x207361, 0x207461, 0x2c206b, 0x616261, 0x616c20, + 0x616c61, 0x616e65, 0x61722e, 0x6172fd, 0x617a61, 0x62616c, 0x650a0a, + 0x652062, 0x656869, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697461, + 0x6bfc74, 0x706861, 0x722079, 0x722c20, 0x726920, 0x726c61, 0x746172, + 0x74fc70, 0x7a6172, 0x7a6520, 0xde6568, 0xe7fd6b, 0xfc7068, 0xfc74fc, + 0xfd6b20, 0x0a0a4b, 0x0a0a4f, 0x0a0ac7, 0x0a0ade, 0x0a4bfc, 0x0a4f6b, + 0x0ac769, 0x0ade65, 0x206172, 0x206174, 0x206261, 0x206265, 0x206465, + 0x20656c, 0x206861, 0x20696c, 0x20696e, 0x206b69, 0x206b6f, 0x206d65, + 0x206dfc, 0x206f6c, 0x206f74, 0x207061, 0x207065, 0x207265, 0x207579, + 0x20f6f0, 0x2c2065, 0x2c2070, 0x43756d, 0x44696a, 0x4bfce7, 0x4f6b75, + 0x536573, 0x612074, 0x612075, 0x612c20, 0x616665, 0x61680a, 0x616cfd, + 0x616d61, 0x616e64, 0x616e6c, 0x616efd, 0x61706c, 0x61722c, 0x617261, + 0x617264, 0x617269, 0x61726b, 0x617274, 0x617461, 0x6174f6, 0x617661, + 0x617a65, 0x617afd, 0x61fee7, 0x626168, 0x62656b, 0x626572, 0x62696c, + 0x626972, 0x62fc73, 0x636920, 0x63696c, 0x63fd20, 0x646120, 0x64616e, + 0x646520, 0x64656e, 0x647520, 0x64fd6d, 0x652061, 0x652065, 0x652069, + 0x652072, 0x652073, + ], + }, + { + encoding: 'KOI8-R', + language: 'ru', + documents: 4, + trigrams: [ + 0xc0d420, 0xc1c0d4, 0x20d0d2, 0xd0d2cf, 0x20c920, 0x20cec1, 0x20d0cf, + 0xc1d120, 0xd1d420, 0xd2cfc4, 0xd9c520, 0x20c7cf, 0x20cdc1, 0x20cfc2, + 0x20d7d9, 0xc1d4c5, 0xc5cece, 0xc920d0, 0xc92c20, 0xccc920, 0xcdc1d2, + 0xcec120, 0xced9c5, 0xd120d0, 0xd2d920, 0xd420d3, 0xd4c5cc, 0xd4c5d2, + 0xd4cfd7, 0x0a0aee, 0x20c120, 0x20c2c9, 0x20cbcf, 0x20cfd3, 0x20cfd4, + 0x20d0cc, 0x20d1d2, 0x20d2c5, 0x20d3cf, 0x20d7cf, 0x2c20c1, 0x2c20c2, + 0xc10a0a, 0xc120d0, 0xc1c5d4, 0xc1cec9, 0xc1d2cb, 0xc1d3d4, 0xc2c9c2, + 0xc2ccc9, 0xc2d3d5, 0xc42c20, 0xc4c1c0, 0xc4c920, 0xc5ccc5, 0xc5ccc9, + 0xc5d2c9, 0xc5d2d9, 0xc5d3d4, 0xc7cfd4, 0xc8cfc4, 0xc920cf, 0xc920d3, + 0xc9c2cc, 0xc9cfd4, 0xc9d120, 0xc9d3d4, 0xc9d4c1, 0xcbc10a, 0xcbc520, + 0xccc9cf, 0xcec1d1, 0xcec9cb, 0xcec9d1, 0xceced1, 0xcecfd7, 0xced1d1, + 0xcfc2d3, 0xcfccd8, 0xcfcdc1, 0xcfcece, 0xcfd2cf, 0xcfd3c5, 0xcfd3d4, + 0xcfd3d9, 0xcfd4c5, 0xcfd4cf, 0xcfd7d1, 0xd0c9d3, 0xd0cfd3, 0xd120d1, + 0xd1c0d4, 0xd1d120, 0xd1d2cd, 0xd2cbc1, 0xd2cdc1, 0xd2cfd3, 0xd3c5ce, + 0xd3ced9, 0xd3d120, 0xd3d4c1, 0xd3d4c5, 0xd3d4cf, 0xd3d5d6, 0xd3d920, + 0xd420cb, 0xd420ce, 0xd420d0, 0xd4c5cb, 0xd4cfd2, 0xd4d2cf, 0xd4d3d1, + 0xd5d6c4, 0xd6c4c1, 0xd7c1c0, 0xd7c5d2, 0xd7d1d4, 0xd920d0, 0xd92c20, + 0xd92e0a, 0xd9ca20, 0xd9d7c1, 0xeec120, 0x0a0ae7, 0x0a0af7, 0x0ae7cf, + 0x0aeec1, 0x0aeec5, + ], + }, { encoding: 'KOI8-U', language: 'uk', @@ -545,6 +1169,656 @@ export default [ 0x70726f, 0x72656e, ], }, + { + encoding: 'Shift_JIS', + language: 'ja', + documents: 4, + trigrams: [ + 0x81420a, 0x8ed282, 0x815b83, 0x82a982, 0x82aa95, 0x82c982, 0x82cc83, + 0x82e981, 0x82f08a, 0xd282aa, 0xe98142, 0x0a0a92, 0x5882aa, 0x738fea, + 0x7d8f91, 0x82a282, 0x82aa94, 0x82ad82, 0x82b581, 0x82b781, 0x82b782, + 0x82c182, 0x82c482, 0x82cc8e, 0x82cc92, 0x82dc82, 0x82df82, 0x82e882, + 0x8a5882, 0x8ad982, 0x8e738f, 0x8f9182, 0x8f918a, 0x8fea82, 0x907d8f, + 0x90c382, 0x918ad9, 0x967b82, 0xa88141, 0xa982c8, 0xb58141, 0xb78142, + 0xc382a9, 0xc982cd, 0xcc8e73, 0xe082f0, 0x0a0a8d, 0x0a0a8f, 0x0a8d4c, + 0x0a8fac, 0x0a926e, 0x0a92a9, 0x4189ca, 0x418a4a, 0x418b69, 0x418dc5, + 0x418e69, 0x418fc4, 0x4195d2, 0x419649, 0x428977, 0x4290dd, 0x4293c7, + 0x429483, 0x4682b5, 0x4790df, 0x48965b, 0x4996a8, 0x4a82af, 0x4a94ad, + 0x4c8fea, 0x4e82c8, 0x52815b, 0x5482b5, 0x56914e, 0x5782dc, 0x5782f0, + 0x57835e, 0x578ed2, 0x5882cc, 0x5882cd, 0x5b0a0a, 0x5b8357, 0x5b8368, + 0x5b8380, 0x5e838b, 0x5f89c6, 0x60815b, 0x6082a4, 0x6282b7, 0x668357, + 0x6882f0, 0x698f91, 0x699283, 0x6a82e2, 0x6a93fa, 0x6c82aa, 0x6d9446, + 0x6e82df, 0x6e88e6, 0x6f82dc, 0x6f8358, 0x708393, 0x7182cd, 0x7382cc, + 0x768ed2, 0x7782c9, 0x7782e2, 0x7790b6, 0x79815b, 0x79976a, 0x7b82cc, + 0x7b82f0, 0x8082aa, 0x814189, 0x81418a, 0x81418b, 0x81418d, 0x81418e, + 0x81418f, 0x814195, 0x814196, 0x814289, 0x814290, 0x814293, 0x814294, + 0x82a290, 0x82a295, + ], + }, + { + encoding: 'windows-1250', + language: 'cs', + documents: 4, + trigrams: [ + 0x206120, 0x207072, 0x616aed, 0x6aed20, 0x206b6e, 0x636520, 0x6b6e69, + 0x6e6968, 0x6eed20, 0x792e0a, 0x0a0a4d, 0x20706f, 0x20e865, 0x2c206b, + 0x610a0a, 0x612070, 0x612076, 0x652061, 0x65206d, 0x652070, 0x686f76, + 0x69686f, 0x6a6520, 0x6e610a, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, + 0x73792e, 0x756a65, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xf86920, + 0x0a0a46, 0x0a0ac8, 0x0a4661, 0x0a4d61, 0x0a4dec, 0x0ac874, 0x206175, + 0x206279, 0x206368, 0x206476, 0x2064ed, 0x206869, 0x20686c, 0x206a61, + 0x206b61, 0x206b6f, 0x206bf3, 0x206c69, 0x206d65, 0x206d75, 0x206dec, + 0x206e61, 0x206f20, 0x206f74, 0x206f76, 0x2070ed, 0x2070f8, 0x207261, + 0x207365, 0x207472, 0x2074fd, 0x207665, 0x20766f, 0x2076ec, 0x207a61, + 0x209ee1, 0x2c206d, 0x2c206f, 0x2c207a, 0x446967, 0x466172, 0x4d616c, + 0x4dec73, 0x52e16e, 0x536f62, 0x546963, 0x61206c, 0x6164ed, 0x616c75, + 0x616cfd, 0x61726d, 0x61736e, 0x6174ed, 0x617574, 0x617675, 0x6176e1, + 0x6176e9, 0x622c20, 0x626f74, 0x626f75, 0x627573, 0x62796c, 0x63686c, + 0x6368e1, 0x636f20, 0x642061, 0x642c20, 0x64616a, 0x646365, 0x647665, + 0x64e176, 0x64e920, 0x64ec2c, 0x64ed20, 0x64ed6c, 0x65206a, 0x65206b, + 0x652072, 0x65642c, 0x656461, 0x65656d, 0x656b61, 0x656d2c, 0x656ee1, + 0x657273, 0x6576ed, 0x65f865, 0x676974, 0x680a0a, 0x686973, 0x686c65, + 0x686ce9, 0x687920, + ], + }, + { + encoding: 'windows-1250', + language: 'hu', + documents: 4, + trigrams: [ + 0x206120, 0x2c2061, 0x6e616b, 0x6f7320, 0x0a0a41, 0x206bf6, 0x2076e1, + 0x20e973, 0x657265, 0x6bf66e, 0x6e7976, 0x726f73, 0x73206b, 0x742c20, + 0x76e172, 0xe1726f, 0xe97320, 0xf66e79, 0x0a4120, 0x206373, 0x206b65, + 0x206b69, 0x61206b, 0x612076, 0x616b2e, 0x617420, 0x657420, 0x697320, + 0x6b2c20, 0x6b2e0a, 0x6d6920, 0x6f7420, 0x73206c, 0x73206d, 0x74206b, + 0x74e172, 0x7674e1, 0x797674, 0xf36b20, 0x0a0a45, 0x0a417a, 0x0a4567, + 0x20616c, 0x20617a, 0x206275, 0x206469, 0x20656c, 0x20656d, 0x206672, + 0x206761, 0x206be1, 0x206be9, 0x206bf3, 0x206c61, 0x206c65, 0x206de9, + 0x206dfa, 0x206dfb, 0x206f6c, 0x207065, 0x207069, 0x207361, 0x207365, + 0x207475, 0x2074e9, 0x2074f6, 0x207669, 0x20e172, 0x20e962, 0x20ed72, + 0x20fa74, 0x2c206b, 0x2c2073, 0x412063, 0x412067, 0x412076, 0x417a20, + 0x446967, 0x456779, 0x526567, 0x537a6f, 0x612064, 0x612074, 0x61630a, + 0x616a74, 0x616b20, 0x616b2c, 0x616c6d, 0x616e0a, 0x616e20, 0x617061, + 0x617373, 0x6173f3, 0x617469, 0x6174f3, 0x617a20, 0x617a64, 0x62616e, + 0x626174, 0x626572, 0x627265, 0x627573, 0x630a0a, 0x637361, 0x637365, + 0x642c20, 0x646573, 0x646967, 0x6469e1, 0x646f6d, 0x646f74, 0x64e16b, + 0x65642c, 0x656469, 0x656767, 0x6567ed, 0x656b20, 0x656b2c, 0x656b65, + 0x656c20, 0x656c6c, 0x656c6d, 0x656c79, 0x656d62, 0x656e2e, 0x656e64, + 0x656e79, 0x656ef5, + ], + }, + { + encoding: 'windows-1250', + language: 'pl', + documents: 4, + trigrams: [ + 0x616ab9, 0x6ab920, 0x207072, 0x206920, 0x207779, 0x616d69, 0x636861, + 0x70727a, 0x727a79, 0x9c6369, 0x206a61, 0x20706f, 0x20726f, 0x2c2061, + 0x616368, 0x626c69, 0x636920, 0x647a69, 0x656b20, 0x692073, 0x692e20, + 0x69626c, 0x696572, 0x696f74, 0x6c696f, 0x6e6961, 0x6f7261, 0x6f7465, + 0x706973, 0x727a65, 0x74656b, 0x0a0a4d, 0x206120, 0x206269, 0x20646c, + 0x206b6f, 0x206d69, 0x207069, 0x207369, 0x207370, 0x20737a, 0x2e204b, + 0x610a0a, 0x612070, 0x612072, 0x617267, 0x626962, 0x636520, 0x63692e, + 0x637920, 0x637a79, 0x642c20, 0x64616a, 0x646c61, 0x65206f, 0x656461, + 0x656b61, 0x657261, 0x66726f, 0x68616d, 0x692077, 0x692c20, 0x696379, + 0x696520, 0x69656e, 0x697379, 0x69ea20, 0x6b6172, 0x6c6120, 0x6c6e69, + 0x6d6920, 0x6d6965, 0x6e6369, 0x6e6520, 0x6e656b, 0x6e6963, 0x6e6965, + 0x6f727a, 0x6f9c63, 0x707261, 0x72616a, 0x72616d, 0x726f77, 0x7369ea, + 0x737072, 0x73746f, 0x737920, 0x737a65, 0x746f72, 0x75647a, 0x776120, + 0x776961, 0x776965, 0x792063, 0x792070, 0x792073, 0x796672, 0x7a7920, + 0x7a7974, 0xb9206a, 0xb9206b, 0xb92077, 0x0a0a54, 0x0a0a57, 0x0a4d61, + 0x0a4d69, 0x0a5461, 0x0a5720, 0x204269, 0x20437a, 0x204b61, 0x204b6c, + 0x204e61, 0x205072, 0x20526f, 0x206175, 0x206275, 0x206368, 0x20636f, + 0x206379, 0x20637a, 0x206461, 0x206472, 0x20676f, 0x206869, 0x206b61, + 0x206b73, 0x206b75, + ], + }, + { + encoding: 'windows-1250', + language: 'ro', + documents: 4, + trigrams: [ + 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, + 0x206d69, 0x207072, 0x207265, 0x61fe61, 0x626c69, 0x636869, 0x646520, + 0x652070, 0x656361, 0x657269, 0x657363, 0x692e20, 0x69626c, 0x696e65, + 0x696f74, 0x6c6520, 0x6c696f, 0x6f7465, 0x736520, 0x746520, 0x746563, + 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, 0x20616c, 0x206175, 0x206361, + 0x20636c, 0x206375, 0x2063e3, 0x206469, 0x206578, 0x206961, 0x206d65, + 0x206f72, 0x207065, 0x207363, 0x207365, 0x207472, 0x20ba74, 0x2c2069, + 0x426962, 0x506961, 0x612064, 0x616c20, 0x617220, 0x617574, 0x617ae3, + 0x636120, 0x636172, 0x636574, 0x63e320, 0x646573, 0x646967, 0x652063, + 0x652064, 0x652074, 0x6520ee, 0x652e0a, 0x65617a, 0x656e69, 0x656e74, + 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, 0x65ba74, 0x676974, + 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, 0x6963e3, 0x696769, + 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, 0x6c2064, 0x6c6567, + 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, 0x6d696e, 0x6e6365, + 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, 0x7261ba, 0x72652c, + 0x72696c, 0x72756c, 0x736320, 0x736368, 0x737572, 0x742e20, 0x74616c, + 0x746974, 0x747275, 0x74e30a, 0x752065, 0x756d70, 0x757a65, 0x7a6561, + 0xba7465, 0xe30a0a, + ], + }, + { + encoding: 'windows-1251', + language: 'ru', + documents: 4, + trigrams: [ + 0xfef220, 0xe0fef2, 0x20eff0, 0xeff0ee, 0x20e820, 0x20ede0, 0x20efee, + 0xe0ff20, 0xf0eee4, 0xfbe520, 0xfff220, 0x20e2fb, 0x20e3ee, 0x20ece0, + 0x20eee1, 0xe0f2e5, 0xe5eded, 0xe820ef, 0xe82c20, 0xebe820, 0xece0f0, + 0xede020, 0xedfbe5, 0xf0fb20, 0xf220f1, 0xf2e5eb, 0xf2e5f0, 0xf2eee2, + 0xff20ef, 0x0a0acd, 0x20e020, 0x20e1e8, 0x20e2ee, 0x20eaee, 0x20eef1, + 0x20eef2, 0x20efeb, 0x20f0e5, 0x20f1ee, 0x20fff0, 0x2c20e0, 0x2c20e1, + 0xcde020, 0xe00a0a, 0xe020ef, 0xe0e5f2, 0xe0ede8, 0xe0f0ea, 0xe0f1f2, + 0xe1e8e1, 0xe1ebe8, 0xe1f1f3, 0xe2e0fe, 0xe2e5f0, 0xe2fff2, 0xe3eef2, + 0xe42c20, 0xe4e0fe, 0xe4e820, 0xe5ebe5, 0xe5ebe8, 0xe5f0e8, 0xe5f0fb, + 0xe5f1f2, 0xe6e4e0, 0xe820ee, 0xe820f1, 0xe8e1eb, 0xe8eef2, 0xe8f1f2, + 0xe8f2e0, 0xe8ff20, 0xeae00a, 0xeae520, 0xebe8ee, 0xede0ff, 0xede8ea, + 0xede8ff, 0xededff, 0xedeee2, 0xedffff, 0xeee1f1, 0xeee2ff, 0xeeebfc, + 0xeeece0, 0xeeeded, 0xeef0ee, 0xeef1e5, 0xeef1f2, 0xeef1fb, 0xeef2e5, + 0xeef2ee, 0xefe8f1, 0xefeef1, 0xf0eae0, 0xf0ece0, 0xf0eef1, 0xf1e5ed, + 0xf1edfb, 0xf1f2e0, 0xf1f2e5, 0xf1f2ee, 0xf1f3e6, 0xf1fb20, 0xf1ff20, + 0xf220ea, 0xf220ed, 0xf220ef, 0xf2e5ea, 0xf2eef0, 0xf2f0ee, 0xf2f1ff, + 0xf3e6e4, 0xf5eee4, 0xfb20ef, 0xfb2c20, 0xfb2e0a, 0xfbe2e0, 0xfbe920, + 0xff20ff, 0xfff0ec, 0xfffef2, 0xffff20, 0x0a0ac2, 0x0a0ac3, 0x0ac220, + 0x0ac3ee, 0x0acde0, + ], + }, + { + encoding: 'windows-1252', + language: 'da', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x676572, 0x6f6720, 0x64656e, 0x646572, + 0x6c6520, 0x6e6465, 0x737465, 0x74656b, 0x0a0a42, 0x206269, 0x206279, + 0x206465, 0x20686f, 0x2070e5, 0x2c206f, 0x626962, 0x626c69, 0x640a0a, + 0x652062, 0x652066, 0x65640a, 0x656e73, 0x65722c, 0x65722e, 0x65726e, + 0x69626c, 0x696465, 0x696c6c, 0x696f74, 0x6b7374, 0x6c696f, 0x6c6c65, + 0x6e6572, 0x6f7465, 0x70e520, 0x722065, 0x722c20, 0x722e0a, 0x726e65, + 0x736572, 0x746572, 0x766572, 0x79656e, 0xe52064, 0xf86765, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4ce6, 0x206272, 0x206275, + 0x2062f8, 0x206361, 0x20656c, 0x20656e, 0x206669, 0x20666f, 0x206672, + 0x2066f8, 0x206869, 0x20686a, 0x206920, 0x206b6f, 0x206c61, 0x206c69, + 0x206d65, 0x206d75, 0x206f6d, 0x206f73, 0x20706c, 0x20736b, 0x207374, + 0x2073e6, 0x207465, 0x207472, 0x207479, 0x20756e, 0x207665, 0x207669, + 0x2076e5, 0x2076e6, 0x20e562, 0x20e662, 0x2c2063, 0x2c206d, 0x2c2075, + 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, 0x457420, 0x4ce673, + 0x4cf872, 0x4d6f72, 0x61622c, 0x616473, 0x6166e9, 0x616773, 0x616c74, + 0x616e67, 0x617265, 0x61726b, 0x61766c, 0x622c20, 0x626c65, 0x626e65, + 0x6272f8, 0x627573, 0x627965, 0x627967, 0x62f867, 0x636166, 0x642062, + 0x642c20, 0x646167, 0x646520, 0x64652c, 0x64656c, 0x64732e, 0x652068, + 0x65206f, 0x652070, + ], + }, + { + encoding: 'windows-1252', + language: 'de', + documents: 4, + trigrams: [ + 0x656e20, 0x636865, 0x657220, 0x206465, 0x20756e, 0x696520, 0x736368, + 0x756e64, 0x65696e, 0x68656e, 0x696368, 0x696e65, 0x6e2064, 0x6e6420, + 0x6e656e, 0x206469, 0x626572, 0x646572, 0x687265, 0x6c6569, 0x205465, + 0x206569, 0x446965, 0x616368, 0x636874, 0x64656e, 0x646965, 0x652c20, + 0x656974, 0x65722e, 0x65726e, 0x686572, 0x696e20, 0x697461, 0x697465, + 0x6b6c65, 0x6c6572, 0x6e2042, 0x6e2069, 0x6e2e20, 0x6e6520, 0x726520, + 0x726569, 0x742065, 0x746520, 0x74656e, 0x204269, 0x204765, 0x204d65, + 0x205265, 0x205374, 0x206265, 0x2066fc, 0x206968, 0x206b6c, 0x206c65, + 0x207665, 0x20fc62, 0x2e2042, 0x2e2044, 0x426962, 0x45696e, 0x4d656e, + 0x537461, 0x616474, 0x616c65, 0x616d20, 0x617573, 0x626c69, 0x636872, + 0x636b65, 0x642066, 0x65204d, 0x652053, 0x652054, 0x652075, 0x656368, + 0x656c6c, 0x656e2e, 0x657265, 0x657277, 0x657320, 0x66fc72, 0x676974, + 0x686520, 0x68656b, 0x68656c, 0x687465, 0x69626c, 0x69636b, 0x696769, + 0x696872, 0x696f74, 0x6c6520, 0x6c6963, 0x6c696f, 0x6e206c, 0x6e2076, + 0x6e6465, 0x6e7465, 0x6e7477, 0x6f7468, 0x722047, 0x722053, 0x722064, + 0x722e0a, 0x722e20, 0x726e20, 0x736520, 0x73652c, 0x737461, 0x737465, + 0x737563, 0x740a0a, 0x742061, 0x746164, 0x74616c, 0x746572, 0x746865, + 0x747769, 0x756368, 0x757373, 0x766572, 0x776963, 0xe46e64, 0xfc6265, + 0xfc7220, 0x0a0a41, + ], + }, + { + encoding: 'windows-1252', + language: 'en', + documents: 4, + trigrams: [ + 0x20616e, 0x616e64, 0x686520, 0x6e6420, 0x727320, 0x207468, 0x2c2061, + 0x746865, 0x652061, 0x652c20, 0x656164, 0x657273, 0x657320, 0x732061, + 0x206120, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206c69, 0x206e65, + 0x207265, 0x207768, 0x546865, 0x616465, 0x626f75, 0x627261, 0x636974, + 0x642063, 0x64206e, 0x646520, 0x646573, 0x652063, 0x657369, 0x68696c, + 0x696272, 0x696465, 0x696c65, 0x697479, 0x6c6520, 0x6c6962, 0x6c6c20, + 0x6e2074, 0x722064, 0x726172, 0x726561, 0x732062, 0x732072, 0x732073, + 0x732c20, 0x732e0a, 0x776869, 0x790a0a, 0x0a0a41, 0x0a0a46, 0x0a0a52, + 0x0a0a54, 0x0a4120, 0x0a4661, 0x0a5265, 0x0a5468, 0x206162, 0x206170, + 0x206172, 0x206265, 0x20626f, 0x206361, 0x20636c, 0x206469, 0x20646f, + 0x206578, 0x206661, 0x206775, 0x206869, 0x20696e, 0x206974, 0x206a61, + 0x206d61, 0x206d75, 0x206e61, 0x206f70, 0x207072, 0x2070e2, 0x207175, + 0x2072e9, 0x2072f4, 0x207363, 0x207365, 0x20736c, 0x20736d, 0x207465, + 0x207761, 0x20776f, 0x2c2063, 0x2c206a, 0x2c2073, 0x3b2063, 0x412073, + 0x446967, 0x466172, 0x4d6f72, 0x526561, 0x536174, 0x612066, 0x612072, + 0x61626f, 0x616420, 0x61642c, 0x6166e9, 0x616b65, 0x616c20, 0x616c61, + 0x616c6c, 0x616d20, 0x616e20, 0x616e67, 0x617065, 0x617070, 0x617220, + 0x617265, 0x617269, 0x61726b, 0x61726d, 0x617274, 0x617279, 0x617475, + 0x617920, 0x61e761, + ], + }, + { + encoding: 'windows-1252', + language: 'es', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x617320, 0x656c20, 0x656e20, 0x657320, 0x206c61, + 0x206c6f, 0x207920, 0x2e204c, 0x61646f, 0x616e20, 0x6c6f73, 0x6e206c, + 0x6f7265, 0x726573, 0x706172, 0x732063, 0x204c6f, 0x206369, 0x20656c, + 0x207061, 0x4c6f73, 0x612065, 0x612070, 0x617261, 0x626c69, 0x6c206d, + 0x6c6120, 0x6e6120, 0x732061, 0x732e20, 0x746f72, 0x206269, 0x20636c, + 0x20636f, 0x206469, 0x206d65, 0x207265, 0x207375, 0x207665, 0x456c20, + 0x4c6120, 0x612062, 0x626962, 0x636961, 0x646520, 0x64656c, 0x646967, + 0x656361, 0x65732e, 0x69626c, 0x696369, 0x696f74, 0x6c2063, 0x6c696f, + 0x6e2065, 0x6f2073, 0x6f2e20, 0x6f7465, 0x726120, 0x732064, 0x732070, + 0x732076, 0x732e0a, 0x746563, 0x0a0a45, 0x204c61, 0x206361, 0x206c65, + 0x206d61, 0x20706f, 0x207072, 0x207365, 0x207465, 0x2c206c, 0x612063, + 0x612064, 0x61206c, 0x612079, 0x616e61, 0x61726f, 0x61732c, 0x61732e, + 0x627265, 0x627573, 0x636120, 0x636164, 0x636965, 0x636975, 0x636c61, + 0x636f2e, 0x636f6d, 0x646164, 0x64656e, 0x646f20, 0x646f72, 0x656369, + 0x656e61, 0x657175, 0x657263, 0x657274, 0x676974, 0x676f20, 0x696120, + 0x696173, 0x69656c, 0x69656e, 0x696769, 0x697461, 0x697564, 0x6c6172, + 0x6c6173, 0x6c6c65, 0x6d6572, 0x6d7061, 0x6e2070, 0x6e2072, 0x6e6173, + 0x6e6f73, 0x6f2064, 0x6f2079, 0x6f636f, 0x6f6d70, 0x6f732e, 0x706f63, + 0x717565, 0x717569, + ], + }, + { + encoding: 'windows-1252', + language: 'fr', + documents: 4, + trigrams: [ + 0x657320, 0x206c65, 0x656e74, 0x6c6573, 0x6e7420, 0x74206c, 0x732070, + 0x206465, 0x657420, 0x657572, 0x727320, 0x742064, 0x206475, 0x206574, + 0x647520, 0x6c6520, 0x732063, 0x757273, 0x204c65, 0x20636f, 0x207072, + 0x2e204c, 0x4c6573, 0x646573, 0x652063, 0x652070, 0x207061, 0x20706f, + 0x2c206c, 0x626c69, 0x65206d, 0x656e63, 0x65732e, 0x6c6c65, 0x6f7572, + 0x706172, 0x717565, 0x726520, 0x732e20, 0x746573, 0x756520, 0x757220, + 0x206269, 0x206361, 0x20656e, 0x206d61, 0x207361, 0x4c6520, 0x616765, + 0x617265, 0x626962, 0x637465, 0x646520, 0x650a0a, 0x652064, 0x652065, + 0x652073, 0x69626c, 0x69656e, 0x696c6c, 0x696f74, 0x6c6575, 0x6c696f, + 0x6d656e, 0x6e6365, 0x6e6520, 0x6f6e73, 0x6f7468, 0x7072e9, 0x72206c, + 0x726368, 0x72e970, 0x732065, 0x73206c, 0x732073, 0x746575, 0xe97061, + 0xe97269, 0x0a0a4c, 0x20636c, 0x206672, 0x206c61, 0x206e75, 0x207265, + 0x2072e9, 0x20756e, 0x207669, 0x2c2064, 0x4c6120, 0x556e65, 0x612062, + 0x616972, 0x616973, 0x617263, 0x636865, 0x6368e9, 0x636965, 0x636f6e, + 0x652e20, 0x656e20, 0x65732c, 0x676573, 0x686973, 0x68e871, 0x68e920, + 0x69656c, 0x696572, 0x696e20, 0x697175, 0x697265, 0x697374, 0x697420, + 0x6c6120, 0x6c656e, 0x6c6965, 0x6d6172, 0x6d6d65, 0x6de972, 0x6e2070, + 0x6e7320, 0x6e7473, 0x6e756d, 0x6f6d6d, 0x6f7576, 0x706f73, 0x706f75, + 0x70726f, 0x72656e, + ], + }, + { + encoding: 'windows-1252', + language: 'it', + documents: 4, + trigrams: [ + 0x6c6120, 0x207069, 0x656e74, 0x206465, 0x206520, 0x612070, 0x6c6c61, + 0x6e6f20, 0x6f7269, 0x746f72, 0x206369, 0x206c61, 0x612067, 0x612069, + 0x61746f, 0x636974, 0x64656c, 0x657263, 0x692061, 0x692063, 0x692073, + 0x696120, 0x697474, 0x69f920, 0x6c6920, 0x7069f9, 0x726920, 0x7474e0, + 0x206269, 0x206361, 0x20636f, 0x206469, 0x206765, 0x206920, 0x206c65, + 0x206d65, 0x207065, 0x207072, 0x207363, 0x207369, 0x207374, 0x2c2063, + 0x4c6120, 0x612062, 0x612063, 0x612064, 0x61206c, 0x616666, 0x616e6f, + 0x626962, 0x626c69, 0x636166, 0x636f6c, 0x637269, 0x65206c, 0x652073, + 0x652c20, 0x656361, 0x656c20, 0x656c65, 0x656c6c, 0x657363, 0x657474, + 0x6666e8, 0x67696f, 0x676c69, 0x692070, 0x69626c, 0x696f20, 0x696f74, + 0x6c696f, 0x6d656e, 0x6e6120, 0x6e7465, 0x6e7472, 0x6f2064, 0x6f2065, + 0x6f2070, 0x6f2c20, 0x6f6e6f, 0x6f7465, 0x706572, 0x726120, 0x726361, + 0x726520, 0x726961, 0x736372, 0x746120, 0x746563, 0x7a6120, 0x0a0a47, + 0x0a0a49, 0x0a0a4c, 0x0a0a55, 0x0a476c, 0x0a4920, 0x0a4c61, 0x0a556e, + 0x206120, 0x206167, 0x206169, 0x206170, 0x206173, 0x206175, 0x206365, + 0x206368, 0x20666f, 0x206672, 0x206769, 0x20676c, 0x206775, 0x20696c, + 0x20696e, 0x206c69, 0x206d69, 0x206d75, 0x206e65, 0x207061, 0x207361, + 0x207371, 0x207376, 0x207472, 0x207574, 0x207665, 0x2c2066, 0x2c2067, + 0x2c2069, 0x2c206d, + ], + }, + { + encoding: 'windows-1252', + language: 'nl', + documents: 4, + trigrams: [ + 0x656e20, 0x20656e, 0x646520, 0x206465, 0x656572, 0x74656e, 0x766572, + 0x206f70, 0x207374, 0x207465, 0x652062, 0x652065, 0x652073, 0x657273, + 0x6e206f, 0x6e2e0a, 0x736368, 0x742064, 0x746572, 0x206269, 0x206361, + 0x20636f, 0x206565, 0x206765, 0x206865, 0x206c65, 0x206f76, 0x207665, + 0x207765, 0x2c2063, 0x446520, 0x61616b, 0x616173, 0x6166e9, 0x616b74, + 0x626962, 0x626c69, 0x636166, 0x636874, 0x652063, 0x65696e, 0x656b65, + 0x656e2e, 0x656e73, 0x657220, 0x657264, 0x657265, 0x65726b, 0x66e973, + 0x687465, 0x69626c, 0x696465, 0x696e20, 0x696e67, 0x696f74, 0x697320, + 0x6b6520, 0x6b656e, 0x6b7420, 0x6c6520, 0x6c6565, 0x6c6569, 0x6c696f, + 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206d, 0x6e2074, 0x6e2076, 0x6e2077, + 0x6f656b, 0x6f6e74, 0x6f7020, 0x6f7065, 0x6f7468, 0x6f7665, 0x70656e, + 0x72656e, 0x727320, 0x732065, 0x73206f, 0x732c20, 0x73656e, 0x737365, + 0x737461, 0x737465, 0x746164, 0x746865, 0xe97320, 0x0a0a42, 0x0a0a44, + 0x0a0a45, 0x0a0a4c, 0x0a426f, 0x0a4465, 0x0a4565, 0x0a4c65, 0x206170, + 0x20626f, 0x206272, 0x206275, 0x206472, 0x206475, 0x20686f, 0x206875, + 0x206964, 0x20696e, 0x206b61, 0x206b6c, 0x206c61, 0x206d61, 0x206d65, + 0x206d75, 0x206e61, 0x206f6e, 0x20706c, 0x20706f, 0x207363, 0x207477, + 0x207761, 0x207a6f, 0x20e9e9, 0x2c2061, 0x2c206b, 0x426f65, 0x446967, + 0x45656e, 0x4c657a, + ], + }, + { + encoding: 'windows-1252', + language: 'no', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x6f6720, 0x656e65, 0x206c61, 0x652066, + 0x676572, 0x6e6520, 0x6e6572, 0x737465, 0x74656b, 0x0a0a42, 0x206269, + 0x206272, 0x206465, 0x20666f, 0x2070e5, 0x207665, 0x626962, 0x626c69, + 0x640a0a, 0x646572, 0x652062, 0x652074, 0x652c20, 0x65640a, 0x656e73, + 0x657273, 0x657420, 0x672073, 0x676520, 0x67656e, 0x69626c, 0x696765, + 0x696e67, 0x696f74, 0x697465, 0x6b6572, 0x6b7374, 0x6c6167, 0x6c6520, + 0x6c6967, 0x6c696f, 0x6e206f, 0x6e6465, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x72206f, 0x72656e, 0x726765, 0x746520, 0x746572, + 0x746f72, 0x76656e, 0x766572, 0x79656e, 0xe52064, 0xf86b65, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4c65, 0x206275, 0x206279, + 0x2062f8, 0x2064f8, 0x20656c, 0x20656e, 0x206570, 0x206665, 0x206669, + 0x2066f8, 0x206869, 0x20686a, 0x20686f, 0x206920, 0x206b61, 0x206b6f, + 0x206c69, 0x206d65, 0x206d75, 0x206ee6, 0x206f6d, 0x206f73, 0x207365, + 0x20736a, 0x20736b, 0x207374, 0x207465, 0x20746f, 0x207472, 0x207479, + 0x20756e, 0x207669, 0x2076e5, 0x20e570, 0x20e672, 0x2c2065, 0x2c206b, + 0x2c206f, 0x2c2075, 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, + 0x457420, 0x4c6573, 0x4cf872, 0x4d6f72, 0x616665, 0x616720, 0x616765, + 0x616773, 0x616c74, 0x616e67, 0x617020, 0x617265, 0x61726b, 0x61766c, + 0x627279, 0x6272f8, + ], + }, + { + encoding: 'windows-1252', + language: 'pt', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x206520, 0x207072, 0x696120, 0x6e6120, 0x206369, + 0x206f73, 0x612064, 0x612065, 0x61206f, 0x646520, 0x657320, 0x657363, + 0x726573, 0x732061, 0x732070, 0x732e0a, 0x0a0a4f, 0x0a4f73, 0x206269, + 0x206573, 0x206e61, 0x206f20, 0x207065, 0x207665, 0x4f7320, 0x612070, + 0x616465, 0x61646f, 0x616d20, 0x617261, 0x617320, 0x626962, 0x626c69, + 0x636120, 0x636964, 0x646120, 0x646164, 0x646967, 0x646f20, 0x652061, + 0x656d20, 0x657175, 0x666963, 0x69626c, 0x696461, 0x696d65, 0x696f74, + 0x6c696f, 0x6e7175, 0x6f2065, 0x6f7265, 0x6f7465, 0x717565, 0x717569, + 0x72616d, 0x726961, 0x726f73, 0x732063, 0x736372, 0x746563, 0x746f72, + 0x0a0a41, 0x0a0a55, 0x0a4120, 0x0a556d, 0x206120, 0x206162, 0x206163, + 0x206167, 0x20616a, 0x20616c, 0x206173, 0x206361, 0x20636c, 0x2063f3, + 0x206469, 0x20646f, 0x20656e, 0x206571, 0x206672, 0x206775, 0x206869, + 0x206c65, 0x206c69, 0x206d61, 0x206d65, 0x206d6f, 0x206d75, 0x2070e3, + 0x207175, 0x2073e1, 0x207472, 0x20f46e, 0x2c206d, 0x2c206f, 0x2c2071, + 0x2c2076, 0x412062, 0x412063, 0x4d616e, 0x4d6572, 0x4f6669, 0x556d61, + 0x610a0a, 0x612061, 0x612062, 0x612063, 0x61206d, 0x612074, 0x612e0a, + 0x616272, 0x61636f, 0x616461, 0x6166e9, 0x616761, 0x616772, 0x616a75, + 0x616c0a, 0x616c75, 0x616e68, 0x616e71, 0x616e74, 0x61722c, 0x61732e, + 0x61e761, 0x61e7e3, + ], + }, + { + encoding: 'windows-1252', + language: 'sv', + documents: 4, + trigrams: [ + 0x657220, 0x617220, 0x206f63, 0x636820, 0x656e20, 0x6f6368, 0x737461, + 0x64656e, 0x657420, 0x6b6172, 0x726e61, 0x736b61, 0x74206c, 0x746164, + 0x766572, 0x206269, 0x206465, 0x2066f6, 0x206c69, 0x2070e5, 0x20736b, + 0x207479, 0x207665, 0x612062, 0x612074, 0x612e0a, 0x61640a, 0x616465, + 0x617265, 0x626962, 0x626c69, 0x640a0a, 0x65726e, 0x66f672, 0x676120, + 0x69626c, 0x696761, 0x696f74, 0x6b6170, 0x6b6572, 0x6b6e61, 0x6c6967, + 0x6c696f, 0x6e612e, 0x6e6172, 0x6e736b, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x722066, 0x72206f, 0x736172, 0x73f66b, 0x746120, + 0x74656b, 0x746f72, 0xe52064, 0x0a0a42, 0x0a0a45, 0x0a0a4c, 0x0a0a53, + 0x0a42f6, 0x0a4574, 0x0a4ce4, 0x0a5374, 0x206265, 0x206272, 0x206275, + 0x2062f6, 0x20656c, 0x20656e, 0x2066e4, 0x206772, 0x206869, 0x20686a, + 0x20686f, 0x206920, 0x206b61, 0x206b6f, 0x206c61, 0x206ce5, 0x206d65, + 0x206d75, 0x206de4, 0x206f6d, 0x206f73, 0x207374, 0x2073e4, 0x2073f6, + 0x207465, 0x20746f, 0x207661, 0x2076e4, 0x20e470, 0x20f670, 0x2c2067, + 0x2c206b, 0x2c206f, 0x2c20e4, 0x42f66e, 0x446574, 0x446967, 0x457474, + 0x4ce473, 0x4cf672, 0x4d6f72, 0x537461, 0x61206f, 0x6120f6, 0x6166e9, + 0x616720, 0x616773, 0x616b6e, 0x616c20, 0x616d74, 0x616e20, 0x616e73, + 0x617020, 0x617061, 0x617269, 0x61726b, 0x61726e, 0x626573, 0x6272f6, + 0x627573, 0x62f663, + ], + }, + { + encoding: 'windows-1253', + language: 'el', + documents: 4, + trigrams: [ + 0x20eae1, 0xe1e920, 0xeae1e9, 0x20e2e9, 0xe120ea, 0xe12e0a, 0xe2e9e2, + 0xe5e920, 0xe9e2eb, 0xeff5ed, 0xf4e120, 0xf5ed20, 0x0a0acf, 0x0acfe9, + 0x20e1e3, 0x20e1ed, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xcfe920, + 0xdfe12e, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, 0xe5f220, 0xe70a0a, 0xe720e2, + 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9efe8, 0xe9f3f4, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xf0fceb, 0xf1e9ef, 0xf220ea, 0xf3f4de, + 0xf3f4e7, 0xf4e5f2, 0xf4e7ed, 0xfcebe7, 0x0a0ac7, 0x0a0acc, 0x0ac720, + 0x0acce9, 0x20dced, 0x20e1f1, 0x20e2ef, 0x20e3f1, 0x20e5eb, 0x20e5f0, + 0x20e5f1, 0x20e5f4, 0x20e720, 0x20e9f3, 0x20eae5, 0x20eafe, 0x20ebe5, + 0x20ecdd, 0x20ece1, 0x20ece9, 0x20ecef, 0x20ecf5, 0x20eef5, 0x20efe4, + 0x20efe9, 0x20efec, 0x20f0e5, 0x20f0eb, 0x20f0ef, 0x20f6f1, 0x20f8dc, + 0x20f8f9, 0x2c20e5, 0x2c20ec, 0x2c20f4, 0x2c20f6, 0xb9f3f5, 0xc720f0, + 0xcce9e1, 0xd0f1f9, 0xd3e1e2, 0xd8e7f6, 0xdc0a0a, 0xdc20e1, 0xdc20ea, + 0xdc20f3, 0xdc20f4, 0xdc2c20, 0xdce4e1, 0xdce6e5, 0xdcede8, 0xdcf1e9, + 0xdcf4e9, 0xdcf6e5, 0xdcf7ed, 0xdd20e1, 0xdde3f7, 0xddebe9, 0xddedef, + 0xddf22e, 0xde20ef, 0xdeeae7, 0xdeece7, 0xdef1e9, 0xdf20f3, 0xdf2c20, + 0xdfe120, 0xdfe1f2, 0xdfe3ef, 0xdfece5, 0xdfeff5, 0xe120e5, 0xe120e9, + 0xe120eb, 0xe120ec, 0xe1e2e2, 0xe1e3ed, 0xe1e3ef, 0xe1e3f1, 0xe1e8e1, + 0xe1e8e7, 0xe1eafc, + ], + }, + { + encoding: 'windows-1254', + language: 'tr', + documents: 4, + trigrams: [ + 0x6c6572, 0x6c6172, 0x207665, 0x722e0a, 0x766520, 0x2061e7, 0x207961, + 0x617220, 0x61e7fd, 0x657220, 0x697220, 0x722076, 0x206269, 0x20656b, + 0x206b61, 0x206bfc, 0x207361, 0x207461, 0x2c206b, 0x616261, 0x616c20, + 0x616c61, 0x616e65, 0x61722e, 0x6172fd, 0x617a61, 0x62616c, 0x650a0a, + 0x652062, 0x656869, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697461, + 0x6bfc74, 0x706861, 0x722079, 0x722c20, 0x726920, 0x726c61, 0x746172, + 0x74fc70, 0x7a6172, 0x7a6520, 0xde6568, 0xe7fd6b, 0xfc7068, 0xfc74fc, + 0xfd6b20, 0x0a0a4b, 0x0a0a4f, 0x0a0ac7, 0x0a0ade, 0x0a4bfc, 0x0a4f6b, + 0x0ac769, 0x0ade65, 0x206172, 0x206174, 0x206261, 0x206265, 0x206465, + 0x20656c, 0x206861, 0x20696c, 0x20696e, 0x206b69, 0x206b6f, 0x206d65, + 0x206dfc, 0x206f6c, 0x206f74, 0x207061, 0x207065, 0x207265, 0x207579, + 0x20f6f0, 0x2c2065, 0x2c2070, 0x43756d, 0x44696a, 0x4bfce7, 0x4f6b75, + 0x536573, 0x612074, 0x612075, 0x612c20, 0x616665, 0x61680a, 0x616cfd, + 0x616d61, 0x616e64, 0x616e6c, 0x616efd, 0x61706c, 0x61722c, 0x617261, + 0x617264, 0x617269, 0x61726b, 0x617274, 0x617461, 0x6174f6, 0x617661, + 0x617a65, 0x617afd, 0x61fee7, 0x626168, 0x62656b, 0x626572, 0x62696c, + 0x626972, 0x62fc73, 0x636920, 0x63696c, 0x63fd20, 0x646120, 0x64616e, + 0x646520, 0x64656e, 0x647520, 0x64fd6d, 0x652061, 0x652065, 0x652069, + 0x652072, 0x652073, + ], + }, + { + encoding: 'windows-1255', + language: 'he', + documents: 4, + trigrams: [ + 0xe9ed20, 0xe5f8e9, 0xed20ec, 0xed20ee, 0xf1f4f8, 0xf8e9ed, 0x20e4f7, + 0x20ece0, 0x20eee7, 0x2c20e1, 0xe0e9ed, 0xe4f1f4, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed2e, 0xed2e0a, 0xef20ee, 0xf0e420, 0xf2e9f8, + 0xf4f8e9, 0xf7e5f8, 0xf820ee, 0xf8e5fa, 0xf8fa20, 0xf9e9ed, 0xfa0a0a, + 0xfa20ec, 0x0a0ae4, 0x0a0ae7, 0x0a0af6, 0x0a0af7, 0x0ae4f2, 0x0ae7f7, + 0x0af6e5, 0x0af7e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, + 0x20e1fa, 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e8f8, 0x20ece1, + 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, 0x20eeeb, 0x20eef7, + 0x20eefa, 0x20f0f4, 0x20f1f4, 0x20f2e5, 0x20f4e9, 0x20f7e8, 0x20f9e1, + 0x20fae9, 0x2c20e3, 0x2c20f4, 0xe0e5e8, 0xe0e5ef, 0xe0e5f8, 0xe0e82c, + 0xe0f0f9, 0xe0f9e5, 0xe0fa20, 0xe120fa, 0xe1e5e3, 0xe1e5f1, 0xe1e5f7, + 0xe1e7e5, 0xe1e9f0, 0xe1ebe9, 0xe1f2e9, 0xe1f8e5, 0xe1f92c, 0xe1fa0a, + 0xe1fae9, 0xe2e1e9, 0xe2e9e8, 0xe320e5, 0xe3e1f9, 0xe3e9e2, 0xe3e9ed, + 0xe3f0e4, 0xe3f220, 0xe3f720, 0xe3f8e9, 0xe40a0a, 0xe420e1, 0xe420e3, + 0xe420e4, 0xe420e5, 0xe420f0, 0xe42e0a, 0xe4e9f1, 0xe4f2e9, 0xe4f2ee, + 0xe4f7e5, 0xe4f7f4, 0xe4f8e0, 0xe4f9f7, 0xe5e0f0, 0xe5e1e5, 0xe5e2e1, + 0xe5e320, 0xe5e3f7, 0xe5e4f1, 0xe5e5fa, 0xe5e6e9, 0xe5e6f8, 0xe5e8e5, + 0xe5ebe5, 0xe5ebf8, + ], + }, + { + encoding: 'windows-1256', + language: 'ar', + documents: 4, + trigrams: [ + 0x20c7e1, 0xc7e1e3, 0x20e6c7, 0x20e6ed, 0xdfcac8, 0xe420c7, 0xe6c7e1, + 0xe6e420, 0x0a0aed, 0xc90a0a, 0xd120c7, 0xc7ca20, 0xc7ccda, 0xc7d120, + 0xc7e1c3, 0xc7e1da, 0xc7e1e6, 0xc820c7, 0xc8c920, 0xc920c7, 0xc920e6, + 0xc92e0a, 0xca20e6, 0xcac820, 0xcac8c9, 0xcb20c7, 0xccda20, 0xcd20c7, + 0xcfede4, 0xd1c7cc, 0xd5ddc7, 0xda20c7, 0xe1e3cf, 0xe1e3df, 0xe3cfed, + 0xe3dfca, 0xe4c7d3, 0xe6d5dd, 0xede4c9, 0x0a0aca, 0x0acad3, 0x0aedc8, + 0x0aedd5, 0x0aedda, 0x20c3c8, 0x20c3e3, 0x20c7ce, 0x20c8e5, 0x20c8ed, + 0x20cfe1, 0x20d1de, 0x20d5db, 0x20dae4, 0x20ddd1, 0x20dded, 0x20dfca, + 0x20e1e1, 0x20e6ca, 0x20e6d5, 0x20ede4, 0xc120da, 0xc120e6, 0xc3c8e6, + 0xc3cdc7, 0xc3cec8, 0xc3e3ed, 0xc4e6e4, 0xc6c90a, 0xc720c8, 0xc720e1, + 0xc720e6, 0xc720ed, 0xc72e0a, 0xc7c120, 0xc7c820, 0xc7c8e5, 0xc7cd20, + 0xc7ceca, 0xc7cfc6, 0xc7cfe1, 0xc7cfed, 0xc7d12e, 0xc7d1da, 0xc7d1ed, + 0xc7d320, 0xc7d3c8, 0xc7d6cd, 0xc7dacf, 0xc7dde1, 0xc7dfe5, 0xc7e1ca, + 0xc7e1cd, 0xc7e1ce, 0xc7e1d2, 0xc7e1d3, 0xc7e1d4, 0xc7e1d8, 0xc7e1dd, + 0xc7e1de, 0xc7e1e4, 0xc7e1e5, 0xc7e5ed, 0xc820dd, 0xc8c7cd, 0xc8c7cf, + 0xc8c7d1, 0xc8c92e, 0xc8ca0a, 0xc8cdcb, 0xc8d1e3, 0xc8d220, 0xc8e5c7, + 0xc8e5cf, 0xc8e6c7, 0xc8ede4, 0xc920c8, 0xc920d1, 0xca0a0a, 0xcac7d1, + 0xcac8c7, 0xcacd20, 0xcacddd, 0xcad3ca, 0xcad9d1, 0xcaddca, 0xcaedc7, + 0xcaedde, 0xcce6e4, + ], + }, + { + encoding: 'windows-1257', + language: 'et', + documents: 4, + trigrams: [ + 0x616420, 0x6a6120, 0x206a61, 0x206b6f, 0x616220, 0x766164, 0x207261, + 0x61616d, 0x616d61, 0x617475, 0x617661, 0x6d6174, 0x6e6520, 0x726161, + 0x746162, 0x0a0a4c, 0x206b69, 0x206d65, 0x2076e4, 0x2c206b, 0x642061, + 0x64206b, 0x64206f, 0x647573, 0x652074, 0x656420, 0x657369, 0x692e0a, + 0x696420, 0x696b75, 0x696d65, 0x696e6e, 0x69726a, 0x697461, 0x6b6972, + 0x6b6f67, 0x6b6f6e, 0x6b6f6f, 0x6b7564, 0x6c6173, 0x6d6573, 0x6f6775, + 0x736564, 0x73656c, 0x73692e, 0x74756b, 0x756420, 0x756b6f, 0x757573, + 0x0a0a54, 0x0a0a56, 0x0a4c69, 0x0a4c75, 0x0a5461, 0x0a56e4, 0x206165, + 0x206169, 0x20616a, 0x206176, 0x206275, 0x206573, 0x20696e, 0x206a75, + 0x206c65, 0x206c69, 0x206d75, 0x206dfc, 0x206e69, 0x206f6f, 0x206f74, + 0x207365, 0x207465, 0x207475, 0x2074f6, 0x20756b, 0x20e472, 0x20f570, + 0x20f575, 0x2c206a, 0x2c20f5, 0x2d206a, 0x446967, 0x486f6d, 0x4c6175, + 0x4c696e, 0x4c7567, 0x54616c, 0x566169, 0x56e469, 0x610a0a, 0x612061, + 0x612069, 0x61206b, 0x61206d, 0x612074, 0x612c20, 0x61616c, 0x616475, + 0x616567, 0x616876, 0x61696b, 0x616974, 0x616a61, 0x616b75, 0x616c6e, + 0x616c6f, 0x616c75, 0x616e65, 0x61726f, 0x61730a, 0x617365, 0x617369, + 0x617570, 0x622061, 0x62206b, 0x62206d, 0x622073, 0x6220f5, 0x62610a, + 0x62612c, 0x627573, 0x642065, 0x64206a, 0x64206d, 0x64206e, 0x642075, + 0x642076, 0x646920, + ], + }, + { + encoding: 'windows-1257', + language: 'lt', + documents: 4, + trigrams: [ + 0x616920, 0x6b6169, 0x206972, 0x207265, 0x612070, 0x616974, 0x646120, + 0x656e69, 0x696169, 0x696e69, 0x697220, 0x6c696f, 0x6e696e, 0x726961, + 0x732e0a, 0x757320, 0x206461, 0x207072, 0x617573, 0x626c69, 0x656461, + 0x692072, 0x696120, 0x69626c, 0x696573, 0x696f74, 0x6a6169, 0x6b696e, + 0x6d656e, 0x6eeb73, 0x6f7465, 0x732061, 0x732c20, 0x732e20, 0x74656b, + 0xeb7320, 0x206169, 0x206170, 0x206269, 0x206475, 0x206b6f, 0x206d65, + 0x206d6f, 0x206f20, 0x207061, 0x20736b, 0x20f076, 0x2c206d, 0x2c206f, + 0x536b61, 0x612064, 0x61206b, 0x61692e, 0x617069, 0x617320, 0x617369, + 0x626962, 0x657269, 0x657374, 0x677573, 0x67f820, 0x692074, 0x692e20, + 0x696e6b, 0x696eeb, 0x696f20, 0x69746d, 0x697573, 0x6b6f20, 0x6d6f6b, + 0x6d756f, 0x6e6461, 0x6e6961, 0x6e6b61, 0x6f20fe, 0x6f6275, 0x6f64e0, + 0x6f6a61, 0x6f6e6f, 0x6f7269, 0x6f7320, 0x707261, 0x72206b, 0x7261f0, + 0x72656e, 0x730a0a, 0x732064, 0x732069, 0x73206d, 0x736920, 0x736b61, + 0x73746f, 0x746173, 0x746d65, 0x746f6a, 0x746f72, 0x757267, 0x757269, + 0x75732c, 0x75732e, 0x766965, 0x796c69, 0xe02c20, 0xeb6a61, 0xf07669, + 0x0a0a4d, 0x0a0a4e, 0x0a0a52, 0x0a0a54, 0x0a4d69, 0x0a4e65, 0x0a5261, + 0x0a5475, 0x204269, 0x204469, 0x204b61, 0x205069, 0x20536b, 0x205669, + 0x206174, 0x206175, 0x206275, 0x206469, 0x206965, 0x206973, 0x206b65, + 0x206b6e, 0x206b75, + ], + }, + { + encoding: 'windows-1257', + language: 'lv', + documents: 4, + trigrams: [ + 0x20756e, 0x617320, 0x756e20, 0x207069, 0x61732e, 0x757320, 0x20697a, + 0x206c61, 0x61206b, 0x626c69, 0x646120, 0x692073, 0x692e20, 0x69626c, + 0x69656d, 0x696f74, 0x6a6920, 0x6b7374, 0x6c696f, 0x6c7573, 0x732c20, + 0x732e0a, 0x732e20, 0x206269, 0x206365, 0x206461, 0x206761, 0x206b6f, + 0x206d61, 0x206d65, 0x2070e2, 0x207265, 0x20736b, 0x207a69, 0x610a0a, + 0x616b73, 0x617262, 0x61756b, 0x626962, 0x627573, 0x6365ef, 0x646172, + 0x64752c, 0x656d2e, 0x676974, 0x677573, 0x692070, 0x692c20, 0x69656e, + 0x696769, 0x696c73, 0x697267, 0x6974e2, 0x697a76, 0x6a756d, 0x6b6920, + 0x6be272, 0x6c6173, 0x6c73e7, 0x6ce76e, 0x6d6172, 0x6d6f73, 0x6e2063, + 0x6e2073, 0x6e6965, 0x6eee63, 0x6f2069, 0x6f74e7, 0x706965, 0x70696c, + 0x70e272, 0x72616b, 0x726775, 0x726920, 0x727669, 0x732061, 0x73206c, + 0x73206d, 0x732070, 0x732075, 0x736920, 0x737420, 0x737461, 0x737475, + 0x73e774, 0x746120, 0x747573, 0x74e26a, 0x74e26c, 0x74e76b, 0x752c20, + 0x75732c, 0x757369, 0x76e76c, 0x7a76e7, 0xe26a69, 0xe76e69, 0xe774e2, + 0xee6361, 0x0a0a4e, 0x0a0a50, 0x0a0a52, 0x0a0a54, 0x0a4e65, 0x0a5069, + 0x0a5261, 0x0a5469, 0x204269, 0x204469, 0x204b61, 0x204c61, 0x205069, + 0x205669, 0x205a65, 0x206170, 0x206172, 0x206174, 0x206175, 0x206265, + 0x206369, 0x206465, 0x206469, 0x206475, 0x2064e2, 0x206772, 0x206a75, + 0x206b6c, 0x206b72, + ], + }, + { + encoding: 'windows-1258', + language: 'vi', + documents: 4, + trigrams: [ + 0x6e6720, 0x207468, 0x206368, 0x207068, 0x2076e0, 0x672074, 0x6e6820, + 0x76e020, 0x2064e2, 0x207469, 0x207472, 0x686ff2, 0x6e2062, 0x6e2074, + 0x7068f4, 0x7468fd, 0xf27420, 0x206261, 0x2062e1, 0x206769, 0x20686f, + 0x206c69, 0x206d61, 0x206e68, 0x207175, 0x2073e1, 0x207461, 0x2074e0, + 0x207669, 0x61d220, 0x6261d2, 0x62e16e, 0x636820, 0x672064, 0x6720f0, + 0x682070, 0x68e06e, 0x68f4ec, 0x68fd20, 0x68fdec, 0x69eaf2, 0x6f6e67, + 0x6ff263, 0x746875, 0x7669ea, 0xd22074, 0xd26e67, 0xe0206d, 0xe06e67, + 0xe06e68, 0xe16e20, 0xe2f274, 0xec0a0a, 0xec7420, 0xf26320, 0xf26920, + 0xf26e20, 0xf46e67, 0xf4ec0a, 0xfdf5cc, 0x0a0a4d, 0x0a0a4e, 0x0a0a54, + 0x0a0ad0, 0x0a4df4, 0x0a4ef4, 0x0a5468, 0x0ad0f4, 0x206275, 0x20636f, + 0x2063e0, 0x2063fd, 0x2068fd, 0x206b68, 0x206b69, 0x206b79, 0x206ce0, + 0x206d69, 0x206d6f, 0x206de2, 0x206df4, 0x206df5, 0x206e67, 0x206f6e, + 0x207261, 0x20726f, 0x2072e0, 0x207369, 0x2073f4, 0x2073fd, 0x2074e1, + 0x207865, 0x2079ea, 0x20f0e2, 0x20f0f4, 0x20f0fa, 0x2c2063, 0x2c206b, + 0x2c2070, 0x2c2071, 0x2c2074, 0x4275f4, 0x4368f5, 0x4df4f2, 0x4ef46e, + 0x5468e0, 0x5468fd, 0x58fdf5, 0x612068, 0x61206d, 0x612076, 0x616920, + 0x61d26e, 0x61d26f, 0x61d279, 0x61de20, 0x61de69, 0x61f269, 0x627579, + 0x632064, 0x632067, 0x632073, 0x632c20, 0x632e0a, 0x63686f, 0x636875, + 0x6368e2, 0x6368f5, + ], + }, + { + encoding: 'windows-874', + language: 'th', + documents: 4, + trigrams: [ + 0xe1c5d0, 0xb7d5e8, 0xc1d7cd, 0x20e1c5, 0xb5c3ec, 0xbbc3d0, 0xbcd9e9, + 0xc8d2ca, 0xca20e1, 0xcab5c3, 0xcad2c3, 0xcbe9cd, 0xd1a7ca, 0xd2cab5, + 0xd5c2b9, 0xd7cda7, 0xe0c1d7, 0xe9cda7, 0x0a0aaa, 0x0a0ab7, 0x0a0abc, + 0x0a0ae0, 0x0aaad2, 0x0ab7d5, 0x0abcd9, 0x0ae0c1, 0x20aad5, 0x20b5c3, + 0x20b5d7, 0x20bac3, 0x20bcc5, 0x20c3e9, 0x20e620, 0xa1a2e9, 0xa1a4d1, + 0xa1a8d1, 0xa1c9ec, 0xa1d2e1, 0xa1e0c3, 0xa2b9c1, 0xa2d2c2, 0xa2d5c2, + 0xa2e9cd, 0xa4b9c3, 0xa4d1a1, 0xa4d3cd, 0xa4d6a1, 0xa4d9e8, 0xa4e8cd, + 0xa4e9b9, 0xa70a0a, 0xa7a4e8, 0xa7b7d3, 0xa7ba0a, 0xa7cab4, 0xa7cac1, + 0xa7cad7, 0xa7d2b9, 0xa7d5c2, 0xa7e3b9, 0xa8b90a, 0xa8c3cb, 0xa8d1b4, + 0xa8d4b7, 0xaad1b4, 0xaad2c7, 0xaad5ca, 0xaae8c7, 0xaae9d2, 0xb1ec20, + 0xb3b1ec, 0xb3d2c3, 0xb420bc, 0xb4b7d3, 0xb4bbc3, 0xb4c2ca, 0xb4c7d1, + 0xb4d4a8, 0xb4e0a7, 0xb4e0a8, 0xb5c3c7, 0xb5c5d2, 0xb5d4c8, 0xb5d7e8, + 0xb5d9e1, 0xb6e2b4, 0xb7c2d2, 0xb7d1c5, 0xb7d3a4, 0xb7d3a7, 0xb7d5c1, + 0xb8c0d1, 0xb8d4ba, 0xb920c3, 0xb9a1d2, 0xb9a4d3, 0xb9a4e9, 0xb9b4d4, + 0xb9b7d5, 0xb9c1bb, 0xb9c3cd, 0xb9c5d2, 0xb9cbd2, 0xb9d1a1, 0xb9d1a7, + 0xb9d2a2, 0xb9e0c1, 0xb9e0c5, 0xb9e0ca, 0xb9e9d3, 0xba0a0a, 0xbac3c3, + 0xbacaa7, 0xbad2c2, 0xbbd1a7, 0xbbd4b4, 0xbcc5e4, 0xbcd6e9, 0xbed4b8, + 0xbed4be, 0xbfe0bb, 0xc0d1b3, 0xc1bbd1, 0xc1d8b4, 0xc1d9c5, 0xc1e0c5, + 0xc1e920, 0xc220e6, + ], + }, { encoding: 'x-mac-cyrillic', language: 'ru', diff --git a/corpus/generated/windows-1250/cs/test/community-garden.bin b/corpus/generated/windows-1250/cs/test/community-garden.bin new file mode 100644 index 0000000..61bfe36 --- /dev/null +++ b/corpus/generated/windows-1250/cs/test/community-garden.bin @@ -0,0 +1,3 @@ +Sousedsk zahrada + +Soused prom髶ili pr嫙dn dv鑴 v zelenou zahradu, d鮅i namalovaly barevn zna鋘y a rodiny zal憝aj rostliny. diff --git a/corpus/generated/windows-1250/cs/train/city-morning.bin b/corpus/generated/windows-1250/cs/train/city-morning.bin new file mode 100644 index 0000000..6efcbfe --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/city-morning.bin @@ -0,0 +1,3 @@ +R嫕o ve m鮀t + +M鮀to se pomalu probouz, kav嫫ny otev甏aj dve鷫 a lid 鋀kaj na prvn autobusy. diff --git a/corpus/generated/windows-1250/cs/train/library.bin b/corpus/generated/windows-1250/cs/train/library.bin new file mode 100644 index 0000000..2d9442e --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/library.bin @@ -0,0 +1,3 @@ +Tich knihovna + +ten暟i hledaj knihy o historii a v駤, zat璥co knihovnice rad 嫜籯. diff --git a/corpus/generated/windows-1250/cs/train/market.bin b/corpus/generated/windows-1250/cs/train/market.bin new file mode 100644 index 0000000..f274a31 --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/market.bin @@ -0,0 +1,3 @@ +Sobotn trh + +Farm暟i prod嫛aj 鋀rstv chl嶵, med, ovoce a vo瀡v byliny. diff --git a/corpus/generated/windows-1250/cs/train/workshop.bin b/corpus/generated/windows-1250/cs/train/workshop.bin new file mode 100644 index 0000000..78a9f82 --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/workshop.bin @@ -0,0 +1,3 @@ +Digit嫮n d璱na + +Mal tm p鷡pravuje pr驠odce muzeem, kontroluje k鏚 a pe jasn popisy. diff --git a/corpus/generated/windows-1250/cs/validation/river-trip.bin b/corpus/generated/windows-1250/cs/validation/river-trip.bin new file mode 100644 index 0000000..dc3d158 --- /dev/null +++ b/corpus/generated/windows-1250/cs/validation/river-trip.bin @@ -0,0 +1,3 @@ +Vlet k 鷫ce + +P鸅tel vzali mapu a tepl 醀j, proli lesem a odpo鴈vali u 鷫ky. diff --git a/corpus/generated/windows-1250/hu/test/community-garden.bin b/corpus/generated/windows-1250/hu/test/community-garden.bin new file mode 100644 index 0000000..251d1d3 --- /dev/null +++ b/corpus/generated/windows-1250/hu/test/community-garden.bin @@ -0,0 +1,3 @@ +K驆飉s嶲i kert + +A szomsz嶮ok z闤d kertt alak癃ott嫜 az res udvart, a gyerekek t墎l嫜at festettek, a csal墂ok pedig 霵t驆ik a n饘幯yeket. diff --git a/corpus/generated/windows-1250/hu/train/city-morning.bin b/corpus/generated/windows-1250/hu/train/city-morning.bin new file mode 100644 index 0000000..74c3adc --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/city-morning.bin @@ -0,0 +1,3 @@ +Reggel a v嫫osban + +A v嫫os lassan 嶵red, a k嫛憴鏦 kinyitnak, az emberek pedig buszra v嫫nak. diff --git a/corpus/generated/windows-1250/hu/train/library.bin b/corpus/generated/windows-1250/hu/train/library.bin new file mode 100644 index 0000000..4bbeea1 --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/library.bin @@ -0,0 +1,3 @@ +A csendes k霵yvt嫫 + +Az olvas鏦 t顤t幯elmi 廥 tudom嫕yos k霵yveket keresnek, a k霵yvt嫫os seg癃 a di嫜oknak. diff --git a/corpus/generated/windows-1250/hu/train/market.bin b/corpus/generated/windows-1250/hu/train/market.bin new file mode 100644 index 0000000..b95088b --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/market.bin @@ -0,0 +1,3 @@ +Szombati piac + +A gazd嫜 friss kenyeret, alm嫢, sajtot 廥 m憴et 嫫ulnak a t廨en. diff --git a/corpus/generated/windows-1250/hu/train/workshop.bin b/corpus/generated/windows-1250/hu/train/workshop.bin new file mode 100644 index 0000000..819901d --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/workshop.bin @@ -0,0 +1,3 @@ +Digit嫮is mhely + +Egy kis csapat mzeumi tmutat鏒 k廥z癃, k鏚ot ellen鰎iz 廥 vil墔os le甏嫳okat 甏. diff --git a/corpus/generated/windows-1250/hu/validation/river-trip.bin b/corpus/generated/windows-1250/hu/validation/river-trip.bin new file mode 100644 index 0000000..f807c95 --- /dev/null +++ b/corpus/generated/windows-1250/hu/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kir嫕dul嫳 a foly鏌oz + +A bar嫢ok t廨k廧et 廥 meleg te嫢 vittek, majd megpihentek a foly partj嫕. diff --git a/corpus/generated/windows-1250/pl/test/community-garden.bin b/corpus/generated/windows-1250/pl/test/community-garden.bin new file mode 100644 index 0000000..0e547a6 --- /dev/null +++ b/corpus/generated/windows-1250/pl/test/community-garden.bin @@ -0,0 +1,3 @@ +Ogr鏚 s零iedzki + +Mieszka鎍y zmienili pusty dziedziniec w zielony ogr鏚 z kwiatami, warzywami i m這dymi drzewami. Dzieci zrobi造 kolorowe tabliczki, a rodziny co tydzie podlewaj roliny. diff --git a/corpus/generated/windows-1250/pl/train/city-morning.bin b/corpus/generated/windows-1250/pl/train/city-morning.bin new file mode 100644 index 0000000..9ff76bb --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/city-morning.bin @@ -0,0 +1,3 @@ +Poranek w miecie + +Miasto budzi si powoli. Kawiarnie otwieraj drzwi, autobusy rozpoczynaj pierwsze kursy, a ludzie czytaj wiadomoci. Nad dachami robi si coraz janiej. diff --git a/corpus/generated/windows-1250/pl/train/library.bin b/corpus/generated/windows-1250/pl/train/library.bin new file mode 100644 index 0000000..a649424 --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/library.bin @@ -0,0 +1,3 @@ +Cicha biblioteka + +W osiedlowej bibliotece s造cha szelest kartek. Czytelnicy szukaj ksi嘀ek o historii, nauce i podr騜ach. Bibliotekarka przygotowuje wystaw dla uczni闚. diff --git a/corpus/generated/windows-1250/pl/train/market.bin b/corpus/generated/windows-1250/pl/train/market.bin new file mode 100644 index 0000000..1171993 --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/market.bin @@ -0,0 +1,3 @@ +Sobotni targ + +Targ wype軟ia rynek kolorami i zapachami. Rolnicy sprzedaj jab趾a, mi鏚, ser i wie篡 chleb. Klienci wybieraj warzywa i wymieniaj rodzinne przepisy. diff --git a/corpus/generated/windows-1250/pl/train/workshop.bin b/corpus/generated/windows-1250/pl/train/workshop.bin new file mode 100644 index 0000000..c1eb29a --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/workshop.bin @@ -0,0 +1,3 @@ +Cyfrowa pracownia + +Ma造 zesp馧 tworzy cyfrowy przewodnik po muzeum. Projektanci uk豉daj menu, programici sprawdzaj kod, a redaktorzy pisz jasne opisy dla goci. diff --git a/corpus/generated/windows-1250/pl/validation/river-trip.bin b/corpus/generated/windows-1250/pl/validation/river-trip.bin new file mode 100644 index 0000000..e817765 --- /dev/null +++ b/corpus/generated/windows-1250/pl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Spacer nad rzek + +W niedziel przyjaciele poszli nad spokojn rzek za miastem. Zabrali map, ciep章 herbat i aparat. Po d逝gim spacerze odpoczywali na brzegu. diff --git a/corpus/generated/windows-1250/ro/test/community-garden.bin b/corpus/generated/windows-1250/ro/test/community-garden.bin new file mode 100644 index 0000000..54f8405 --- /dev/null +++ b/corpus/generated/windows-1250/ro/test/community-garden.bin @@ -0,0 +1,3 @@ +Gr緂ina comunit裻ii + +Vecinii au transformat o curte goal 螽tr-o gr緂in cu flori, legume 槐 pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se 螽t滎nesc s綯t緆滱al pentru a uda plantele. diff --git a/corpus/generated/windows-1250/ro/train/city-morning.bin b/corpus/generated/windows-1250/ro/train/city-morning.bin new file mode 100644 index 0000000..08d7469 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/city-morning.bin @@ -0,0 +1,3 @@ +Dimineaa 螽 ora + +Ora滾l se treze演e 螽cet. Cafenelele deschid u槐le, autobuzele pornesc pe traseu, iar oamenii citesc 演irile. Cerul devine mai luminos deasupra cl緂irilor. diff --git a/corpus/generated/windows-1250/ro/train/library.bin b/corpus/generated/windows-1250/ro/train/library.bin new file mode 100644 index 0000000..1afc415 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/library.bin @@ -0,0 +1,3 @@ +Biblioteca lini演it + +姷 biblioteca de cartier se aud pagini 螽toarse 螽cet. Cititorii caut c綖i despre istorie, 演iin 槐 c緄綟orii. Bibliotecara preg綟e演e o expoziie pentru elevi. diff --git a/corpus/generated/windows-1250/ro/train/market.bin b/corpus/generated/windows-1250/ro/train/market.bin new file mode 100644 index 0000000..15a0338 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/market.bin @@ -0,0 +1,3 @@ +Piaa de s滵b綟 + +Piaa umple centrul cu mirosuri 槐 culori. Fermierii aduc mere, miere, br滱z 槐 p毃ne proasp綟. Cump綖綟orii aleg legume 槐 schimb reete de familie. diff --git a/corpus/generated/windows-1250/ro/train/workshop.bin b/corpus/generated/windows-1250/ro/train/workshop.bin new file mode 100644 index 0000000..132f010 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/workshop.bin @@ -0,0 +1,3 @@ +Atelier digital + +O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. diff --git a/corpus/generated/windows-1250/ro/validation/river-trip.bin b/corpus/generated/windows-1250/ro/validation/river-trip.bin new file mode 100644 index 0000000..c125be0 --- /dev/null +++ b/corpus/generated/windows-1250/ro/validation/river-trip.bin @@ -0,0 +1,3 @@ +Plimbare l滱g r漉 + +Duminic, prietenii au mers spre un r漉 lini演it din afara ora滾lui. Au luat o hart, ceai cald 槐 un aparat foto. Dup plimbare s-au odihnit pe mal. diff --git a/corpus/generated/windows-1251/ru/test/community-garden.bin b/corpus/generated/windows-1251/ru/test/community-garden.bin new file mode 100644 index 0000000..2f6119a --- /dev/null +++ b/corpus/generated/windows-1251/ru/test/community-garden.bin @@ -0,0 +1,3 @@ +挓鴇 黟 + +や蠈錒 碪殥蠈 瀔魤槶 瀁儋 嚦僦 鳱闉 瀁黟儰錒 攦殣, 谾錼薤 斁鋋噊 麧謥禕. 櫻蠂 鼥槶鳪 謤鳺 蠉摠儚膱, 勷鼫儰 儋蜦碭謶錒嬿 罻緛嚲 翴麧錌 瀁錒瘔譔 譇嚦樇. diff --git a/corpus/generated/windows-1251/ru/train/city-morning.bin b/corpus/generated/windows-1251/ru/train/city-morning.bin new file mode 100644 index 0000000..5d58e7b --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/city-morning.bin @@ -0,0 +1,3 @@ +郖豂 蜦豂麧 + +鏝豂 瀔闃櫡殣 瀁嚦樖樇膼. 侲 闃蠉膼瞃僛 瀁碲襙 櫡嚭馰麃, 摷鋋籦 闅膴瘔殣 鳱歑, 櫇謼 飶襜摷嬾 禖羻 縺 憵蹢賾. 坢儰 玁蠉 勷鍕樇 鍕嚧緛僠 瀇鳧 縺 膼禖 麧薤. diff --git a/corpus/generated/windows-1251/ru/train/library.bin b/corpus/generated/windows-1251/ru/train/library.bin new file mode 100644 index 0000000..3c62d20 --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/library.bin @@ -0,0 +1,3 @@ +冓濎臇 摜摠鴅蠈罻 + + 摜摠鴅蠈耪 蠂羻 鷿錼嚦 嚦譇膻籜. 郯蠉蠈錒 禖摜譇 膫魬 鍕 黓襜謶, 縺鵳 瀀蠈鷿嚦睯. 栦襝鵫膻膱 蜦襜 禖嚦飶膧, 闅瞂欑 縺 碭瀔闃 瀁斁蜬 齆鎀膻罻 縺澽 薃緱 憵蠈謶鳪. diff --git a/corpus/generated/windows-1251/ru/train/market.bin b/corpus/generated/windows-1251/ru/train/market.bin new file mode 100644 index 0000000..668f4f3 --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/market.bin @@ -0,0 +1,3 @@ +昢樇 謽僦罻 + +侲 瀇霘馯 瀔隰鍱鼏 闃樇 謽僦罻. 婍謽歑 瀔魤樄錒 摠鍧, 鼽, 嬾 鼀樉鴇 纆槦. 砐膧櫡蠈錒 嚫飶膻瘔 瀔鍱鵳譖, 摎鼫劌 瀔鍱飶攡擯 諘歞嬾瘔 鴈蠈謥劖 儋憵齶鳺 謥欈濿. diff --git a/corpus/generated/windows-1251/ru/train/workshop.bin b/corpus/generated/windows-1251/ru/train/workshop.bin new file mode 100644 index 0000000..53d315f --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/workshop.bin @@ -0,0 +1,3 @@ +冓搿欑 憵嚦歑齕 + +俜搿錪鸆 膰憵罽 勷諙鉊 錼膷豂臇 罻蠉鋋 檍谾. 殲諘澼歑 鍕嚧緛僠 懤薞, 瀔鍏譇擤黓譖 瀔鍒歑 膰, 謥魡膷闉 蜦襜 劖 闀黓鳧 儇 瀁鼫蠂蠈錼 譇賝 碭趜僔襜. diff --git a/corpus/generated/windows-1251/ru/validation/river-trip.bin b/corpus/generated/windows-1251/ru/validation/river-trip.bin new file mode 100644 index 0000000..afd48bf --- /dev/null +++ b/corpus/generated/windows-1251/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ +砐樄儆 謥耪 + + 碭齕謥鼫薤 僽鵰 闅瀔飶鳹黓 厴鍧鍣膼 謥耪 諘 蜦豂儋. 昲 瞀錒 罻貗, 蜦玁 欑 鐕襜黽櫡譇. 砐儴 儋鍺鍣 瀔鍏鵻膱 瀀蠈鷿嚦瞂臇鴀 闅噊矐錒 縺 摎謥蜰 儴齁鳪 瞂灚謺儓 濿儗. diff --git a/corpus/generated/windows-1252/da/test/community-garden.bin b/corpus/generated/windows-1252/da/test/community-garden.bin new file mode 100644 index 0000000..bbb5980 --- /dev/null +++ b/corpus/generated/windows-1252/da/test/community-garden.bin @@ -0,0 +1,3 @@ +F熞leshaven + +Naboerne gjorde en tom g緳d til en gr鷢 have, b鷨nene malede skilte, og familierne vander planterne hver uge. diff --git a/corpus/generated/windows-1252/da/train/city-morning.bin b/corpus/generated/windows-1252/da/train/city-morning.bin new file mode 100644 index 0000000..aafe557 --- /dev/null +++ b/corpus/generated/windows-1252/da/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen v嶓ner langsomt, caf嶪rne 嶙ner, og folk venter p de f鷨ste busser. diff --git a/corpus/generated/windows-1252/da/train/library.bin b/corpus/generated/windows-1252/da/train/library.bin new file mode 100644 index 0000000..dfcd24b --- /dev/null +++ b/corpus/generated/windows-1252/da/train/library.bin @@ -0,0 +1,3 @@ +Det stille bibliotek + +L犘ere finder b鷤er om historie og videnskab, mens bibliotekaren hj熞per eleverne. diff --git a/corpus/generated/windows-1252/da/train/market.bin b/corpus/generated/windows-1252/da/train/market.bin new file mode 100644 index 0000000..e156a58 --- /dev/null +++ b/corpus/generated/windows-1252/da/train/market.bin @@ -0,0 +1,3 @@ +L鷨dagsmarked + +B鷢der s熞ger frisk br鷣, 熲ler, ost og honning p den travle plads. diff --git a/corpus/generated/windows-1252/da/train/workshop.bin b/corpus/generated/windows-1252/da/train/workshop.bin new file mode 100644 index 0000000..04a0b0c --- /dev/null +++ b/corpus/generated/windows-1252/da/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt v熳ksted + +Et lille hold bygger en museumsguide, unders鷤er koden og skriver tydelige tekster. diff --git a/corpus/generated/windows-1252/da/validation/river-trip.bin b/corpus/generated/windows-1252/da/validation/river-trip.bin new file mode 100644 index 0000000..386c495 --- /dev/null +++ b/corpus/generated/windows-1252/da/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til floden + +Vennerne tog et kort og varm te med p en s鷢dag og hvilede bagefter ved floden. diff --git a/corpus/generated/windows-1252/de/test/community-garden.bin b/corpus/generated/windows-1252/de/test/community-garden.bin new file mode 100644 index 0000000..df4bbec --- /dev/null +++ b/corpus/generated/windows-1252/de/test/community-garden.bin @@ -0,0 +1,3 @@ +Der Gemeinschaftsgarten + +Die Nachbarn haben einen leeren Hof begrnt und dort Kr酳ter, Gemse und junge B酳me gepflanzt. Die Schler malten bunte Schilder, w鄣rend 鄟tere Bewohner regelm魠ig gie絽n und Ger酹e aufr酳men. diff --git a/corpus/generated/windows-1252/de/train/city-morning.bin b/corpus/generated/windows-1252/de/train/city-morning.bin new file mode 100644 index 0000000..3ccc201 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen in der Stadt + +Die Stadt erwacht langsam. B踄kereien 鐪fnen ihre Tren, Busse beginnen ihre Fahrt und viele Menschen lesen unterwegs die Nachrichten. 鈁er den D踄hern wird der Himmel allm鄣lich heller. diff --git a/corpus/generated/windows-1252/de/train/library.bin b/corpus/generated/windows-1252/de/train/library.bin new file mode 100644 index 0000000..992cca1 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/library.bin @@ -0,0 +1,3 @@ +Die ruhige Bibliothek + +In der kleinen Bibliothek rascheln leise die Seiten. Besucher suchen Bcher ber Geschichte, Technik und ferne L鄚der. Eine Mitarbeiterin bereitet eine Ausstellung fr Schler vor. diff --git a/corpus/generated/windows-1252/de/train/market.bin b/corpus/generated/windows-1252/de/train/market.bin new file mode 100644 index 0000000..db29233 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/market.bin @@ -0,0 +1,3 @@ +Markt am Samstag + +Auf dem Platz bieten Bauern 癥fel, K酲e, Honig und frisches Brot an. Die Kunden vergleichen Gemse, sprechen ber die Ernte und tauschen einfache Rezepte aus. diff --git a/corpus/generated/windows-1252/de/train/workshop.bin b/corpus/generated/windows-1252/de/train/workshop.bin new file mode 100644 index 0000000..17ea61c --- /dev/null +++ b/corpus/generated/windows-1252/de/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale Werkstatt + +Ein kleines Team entwickelt einen digitalen Museumsfhrer. Designer ordnen das Men, Entwickler prfen den Code und Redakteure schreiben verst鄚dliche Texte fr G酲te. diff --git a/corpus/generated/windows-1252/de/validation/river-trip.bin b/corpus/generated/windows-1252/de/validation/river-trip.bin new file mode 100644 index 0000000..09bc118 --- /dev/null +++ b/corpus/generated/windows-1252/de/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ausflug an den Fluss + +Am Sonntag wandern Freunde zu einem ruhigen Fluss. Sie nehmen eine Karte, hei絽n Tee und eine Kamera mit. Nach dem langen Weg sitzen sie am Ufer und h顤en den V鐷eln zu. diff --git a/corpus/generated/windows-1252/en/test/community-garden.bin b/corpus/generated/windows-1252/en/test/community-garden.bin new file mode 100644 index 0000000..549b374 --- /dev/null +++ b/corpus/generated/windows-1252/en/test/community-garden.bin @@ -0,0 +1,3 @@ +The community garden + +Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small caf table. diff --git a/corpus/generated/windows-1252/en/train/city-morning.bin b/corpus/generated/windows-1252/en/train/city-morning.bin new file mode 100644 index 0000000..f7199ae --- /dev/null +++ b/corpus/generated/windows-1252/en/train/city-morning.bin @@ -0,0 +1,3 @@ +Morning in the city + +The city wakes slowly; caf廥 open their doors and commuters read news beside a fa蓷de. diff --git a/corpus/generated/windows-1252/en/train/library.bin b/corpus/generated/windows-1252/en/train/library.bin new file mode 100644 index 0000000..20400f3 --- /dev/null +++ b/corpus/generated/windows-1252/en/train/library.bin @@ -0,0 +1,3 @@ +The quiet library + +Readers browse books about history, science, and na鴳e art while the librarian prepares a r廥um. diff --git a/corpus/generated/windows-1252/en/train/market.bin b/corpus/generated/windows-1252/en/train/market.bin new file mode 100644 index 0000000..c760513 --- /dev/null +++ b/corpus/generated/windows-1252/en/train/market.bin @@ -0,0 +1,3 @@ +Saturday market + +Farmers sell bread, apples, jalape隳s, and p漮 while neighbours exchange recipes. diff --git a/corpus/generated/windows-1252/en/train/workshop.bin b/corpus/generated/windows-1252/en/train/workshop.bin new file mode 100644 index 0000000..788536f --- /dev/null +++ b/corpus/generated/windows-1252/en/train/workshop.bin @@ -0,0 +1,3 @@ +Digital workshop + +A small team designs the museum guide and discusses its r犨e, code, and clear descriptions. diff --git a/corpus/generated/windows-1252/en/validation/river-trip.bin b/corpus/generated/windows-1252/en/validation/river-trip.bin new file mode 100644 index 0000000..b686334 --- /dev/null +++ b/corpus/generated/windows-1252/en/validation/river-trip.bin @@ -0,0 +1,3 @@ +Journey to the river + +Friends carried a map and a thermos, walked past the caf, and rested by the river. diff --git a/corpus/generated/windows-1252/es/test/community-garden.bin b/corpus/generated/windows-1252/es/test/community-garden.bin new file mode 100644 index 0000000..3c82a6f --- /dev/null +++ b/corpus/generated/windows-1252/es/test/community-garden.bin @@ -0,0 +1,3 @@ +El jard璯 comunitario + +Los vecinos convirtieron un patio vac甐 en un jard璯 lleno de 嫫boles, hierbas y verduras. Los ni隳s pintaron carteles de colores y cada s墎ado varias familias riegan las plantas. diff --git a/corpus/generated/windows-1252/es/train/city-morning.bin b/corpus/generated/windows-1252/es/train/city-morning.bin new file mode 100644 index 0000000..518bc97 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/city-morning.bin @@ -0,0 +1,3 @@ +La ma鎙na en la ciudad + +La ciudad despierta poco a poco. Las cafeter燰s abren sus puertas, los autobuses inician la ruta y los vecinos leen las noticias. El cielo se vuelve m嫳 claro sobre los tejados. diff --git a/corpus/generated/windows-1252/es/train/library.bin b/corpus/generated/windows-1252/es/train/library.bin new file mode 100644 index 0000000..4d6010f --- /dev/null +++ b/corpus/generated/windows-1252/es/train/library.bin @@ -0,0 +1,3 @@ +Una biblioteca tranquila + +En la biblioteca del barrio se oyen p墔inas y pasos suaves. Los lectores buscan libros de historia, ciencia y viajes. La bibliotecaria prepara una exposici鏮 para los alumnos. diff --git a/corpus/generated/windows-1252/es/train/market.bin b/corpus/generated/windows-1252/es/train/market.bin new file mode 100644 index 0000000..31af767 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/market.bin @@ -0,0 +1,3 @@ +El mercado del s墎ado + +El mercado llena la plaza de colores. Los agricultores venden manzanas, miel, queso y pan reci幯 hecho. Los clientes comparan las verduras y comparten recetas de sus familias. diff --git a/corpus/generated/windows-1252/es/train/workshop.bin b/corpus/generated/windows-1252/es/train/workshop.bin new file mode 100644 index 0000000..92a2f60 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/workshop.bin @@ -0,0 +1,3 @@ +Taller digital + +Un equipo peque隳 crea el cat嫮ogo digital del museo. Los dise鎙dores ordenan el men, los programadores revisan el c鏚igo y los editores escriben textos claros para el pblico. diff --git a/corpus/generated/windows-1252/es/validation/river-trip.bin b/corpus/generated/windows-1252/es/validation/river-trip.bin new file mode 100644 index 0000000..022d6f6 --- /dev/null +++ b/corpus/generated/windows-1252/es/validation/river-trip.bin @@ -0,0 +1,3 @@ +Paseo junto al r甐 + +El domingo unos amigos caminan hacia un r甐 tranquilo. Llevan un mapa, t caliente y una c嫥ara. Despu廥 del paseo descansan en la orilla y escuchan a los p奫aros. diff --git a/corpus/generated/windows-1252/fr/test/community-garden.bin b/corpus/generated/windows-1252/fr/test/community-garden.bin new file mode 100644 index 0000000..2dce62d --- /dev/null +++ b/corpus/generated/windows-1252/fr/test/community-garden.bin @@ -0,0 +1,3 @@ +Le jardin partag + +Les habitants ont transform une cour oubli嶪 en jardin fleuri. Les enfants ont pr廧ar des 彋iquettes color嶪s, tandis que les voisins se r徼nissent chaque semaine pour arroser les l嶲umes et les jeunes arbres. diff --git a/corpus/generated/windows-1252/fr/train/city-morning.bin b/corpus/generated/windows-1252/fr/train/city-morning.bin new file mode 100644 index 0000000..411550d --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/city-morning.bin @@ -0,0 +1,3 @@ +Le matin en ville + +La ville se r憝eille lentement. Les caf廥 ouvrent leurs portes, les autobus commencent leur trajet et les passants consultent les nouvelles. Chacun pr廧are sa journ嶪 sous un ciel encore p滎e. diff --git a/corpus/generated/windows-1252/fr/train/library.bin b/corpus/generated/windows-1252/fr/train/library.bin new file mode 100644 index 0000000..e06a9ef --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/library.bin @@ -0,0 +1,3 @@ +La biblioth鋂ue calme + +Dans la biblioth鋂ue du quartier, les pages tournent en silence. Les lecteurs cherchent des livres sur l'histoire, les sciences et les voyages. Une biblioth嶰aire pr廧are une exposition pour les 幨鋦es. diff --git a/corpus/generated/windows-1252/fr/train/market.bin b/corpus/generated/windows-1252/fr/train/market.bin new file mode 100644 index 0000000..9717faa --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/market.bin @@ -0,0 +1,3 @@ +Le march du samedi + +Le march remplit la place de couleurs et de parfums. Les producteurs proposent des pommes, du miel, du fromage et du pain frais. Les clients discutent des saisons et 嶰hangent des recettes familiales. diff --git a/corpus/generated/windows-1252/fr/train/workshop.bin b/corpus/generated/windows-1252/fr/train/workshop.bin new file mode 100644 index 0000000..4dea00e --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/workshop.bin @@ -0,0 +1,3 @@ +Un atelier num廨ique + +Une petite 廦uipe construit le catalogue num廨ique du mus嶪. Les graphistes organisent le menu, les d憝eloppeurs v廨ifient le code et les r嶮acteurs pr廧arent des textes clairs pour le public. diff --git a/corpus/generated/windows-1252/fr/validation/river-trip.bin b/corpus/generated/windows-1252/fr/validation/river-trip.bin new file mode 100644 index 0000000..48f3a71 --- /dev/null +++ b/corpus/generated/windows-1252/fr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Une promenade pr鋊 de la rivi鋨e + +Dimanche, des amis partent vers une rivi鋨e tranquille. Ils emportent une carte, du th chaud et un appareil photo. Apr鋊 une longue marche, ils se reposent sur la rive et 嶰outent les oiseaux. diff --git a/corpus/generated/windows-1252/it/test/community-garden.bin b/corpus/generated/windows-1252/it/test/community-garden.bin new file mode 100644 index 0000000..76a54ea --- /dev/null +++ b/corpus/generated/windows-1252/it/test/community-garden.bin @@ -0,0 +1,3 @@ +Il giardino comune + +I vicini hanno trasformato un cortile vuoto in un giardino della citt, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante. diff --git a/corpus/generated/windows-1252/it/train/city-morning.bin b/corpus/generated/windows-1252/it/train/city-morning.bin new file mode 100644 index 0000000..a8faca7 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/city-morning.bin @@ -0,0 +1,3 @@ +Mattina in citt + +La citt si sveglia lentamente, i caff aprono e la gente aspetta i primi autobus. diff --git a/corpus/generated/windows-1252/it/train/library.bin b/corpus/generated/windows-1252/it/train/library.bin new file mode 100644 index 0000000..e9b65ab --- /dev/null +++ b/corpus/generated/windows-1252/it/train/library.bin @@ -0,0 +1,3 @@ +La biblioteca tranquilla + +I lettori cercano pi libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti pi giovani. diff --git a/corpus/generated/windows-1252/it/train/market.bin b/corpus/generated/windows-1252/it/train/market.bin new file mode 100644 index 0000000..780f149 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/market.bin @@ -0,0 +1,3 @@ +Mercato del sabato + +Gli agricoltori vendono pane fresco, caff, mele, formaggio e miele nella piazza della citt. diff --git a/corpus/generated/windows-1252/it/train/workshop.bin b/corpus/generated/windows-1252/it/train/workshop.bin new file mode 100644 index 0000000..21734d6 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/workshop.bin @@ -0,0 +1,3 @@ +Laboratorio digitale + +Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perch siano utili a pi persone. diff --git a/corpus/generated/windows-1252/it/validation/river-trip.bin b/corpus/generated/windows-1252/it/validation/river-trip.bin new file mode 100644 index 0000000..0ff2d62 --- /dev/null +++ b/corpus/generated/windows-1252/it/validation/river-trip.bin @@ -0,0 +1,3 @@ +Gita al fiume + +Gli amici portano una mappa e t caldo, camminano nel bosco e riposano sulla riva. diff --git a/corpus/generated/windows-1252/nl/test/community-garden.bin b/corpus/generated/windows-1252/nl/test/community-garden.bin new file mode 100644 index 0000000..89526b9 --- /dev/null +++ b/corpus/generated/windows-1252/nl/test/community-garden.bin @@ -0,0 +1,3 @@ +De buurttuin + +Buren veranderden een lege binnenplaats naast een caf in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water. diff --git a/corpus/generated/windows-1252/nl/train/city-morning.bin b/corpus/generated/windows-1252/nl/train/city-morning.bin new file mode 100644 index 0000000..13f5ea7 --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/city-morning.bin @@ -0,0 +1,3 @@ +Ochtend in de stad + +De stad ontwaakt langzaam, caf廥 openen hun deuren en mensen wachten op de eerste bussen. diff --git a/corpus/generated/windows-1252/nl/train/library.bin b/corpus/generated/windows-1252/nl/train/library.bin new file mode 100644 index 0000000..cdf34ac --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/library.bin @@ -0,0 +1,3 @@ +De stille bibliotheek + +Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris ge鮦teresseerde leerlingen en 澭n leraar helpt. diff --git a/corpus/generated/windows-1252/nl/train/market.bin b/corpus/generated/windows-1252/nl/train/market.bin new file mode 100644 index 0000000..9c584bb --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/market.bin @@ -0,0 +1,3 @@ +Zaterdagmarkt + +Boeren verkopen vers brood, appels, kaas en honing naast twee caf廥 op het drukke plein. diff --git a/corpus/generated/windows-1252/nl/train/workshop.bin b/corpus/generated/windows-1252/nl/train/workshop.bin new file mode 100644 index 0000000..53ff667 --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale werkplaats + +Een klein team maakt een museumgids, controleert de code en schrijft duidelijke idee螚 over po褯ie en teksten. diff --git a/corpus/generated/windows-1252/nl/validation/river-trip.bin b/corpus/generated/windows-1252/nl/validation/river-trip.bin new file mode 100644 index 0000000..27db29e --- /dev/null +++ b/corpus/generated/windows-1252/nl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Reis naar de rivier + +Vrienden namen een kaart en warme thee mee en rustten na 澭n lange wandeling bij de rivier. diff --git a/corpus/generated/windows-1252/no/test/community-garden.bin b/corpus/generated/windows-1252/no/test/community-garden.bin new file mode 100644 index 0000000..b82814c --- /dev/null +++ b/corpus/generated/windows-1252/no/test/community-garden.bin @@ -0,0 +1,3 @@ +Felleshagen + +Naboene gjorde en tom g緳d til en gr鷢n hage, barna malte skilt og familiene vanner plantene hver uke. diff --git a/corpus/generated/windows-1252/no/train/city-morning.bin b/corpus/generated/windows-1252/no/train/city-morning.bin new file mode 100644 index 0000000..972847d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen v嶡ner langsomt, kafeene 幠ner d鷨ene og folk venter p de f鷨ste bussene n熳 bryggen og sj鷫n. diff --git a/corpus/generated/windows-1252/no/train/library.bin b/corpus/generated/windows-1252/no/train/library.bin new file mode 100644 index 0000000..f972c3d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/library.bin @@ -0,0 +1,3 @@ +Det stille biblioteket + +Lesere finner b鷦er om historie, 熳lige fortellinger og vitenskap mens bibliotekaren hjelper elevene. diff --git a/corpus/generated/windows-1252/no/train/market.bin b/corpus/generated/windows-1252/no/train/market.bin new file mode 100644 index 0000000..c020f8d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/market.bin @@ -0,0 +1,3 @@ +L鷨dagsmarked + +B鷢der selger ferskt br鷣, epler, ost og honning p det travle torget. diff --git a/corpus/generated/windows-1252/no/train/workshop.bin b/corpus/generated/windows-1252/no/train/workshop.bin new file mode 100644 index 0000000..954e228 --- /dev/null +++ b/corpus/generated/windows-1252/no/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt verksted + +Et lite lag lager en museumsguide, unders鷦er koden og skriver tydelige tekster. diff --git a/corpus/generated/windows-1252/no/validation/river-trip.bin b/corpus/generated/windows-1252/no/validation/river-trip.bin new file mode 100644 index 0000000..1489859 --- /dev/null +++ b/corpus/generated/windows-1252/no/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til elven + +Vennene tok med kart og varm te p s鷢dag og hvilte senere ved elven. diff --git a/corpus/generated/windows-1252/pt/test/community-garden.bin b/corpus/generated/windows-1252/pt/test/community-garden.bin new file mode 100644 index 0000000..65848b6 --- /dev/null +++ b/corpus/generated/windows-1252/pt/test/community-garden.bin @@ -0,0 +1,3 @@ +A horta comunit嫫ia + +Os vizinhos transformaram um p嫢io vazio em jardim, as crian蓷s pintaram placas e as fam璱ias regam as plantas toda semana. diff --git a/corpus/generated/windows-1252/pt/train/city-morning.bin b/corpus/generated/windows-1252/pt/train/city-morning.bin new file mode 100644 index 0000000..18b861f --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/city-morning.bin @@ -0,0 +1,3 @@ +Manh na cidade + +A cidade acorda devagar, os caf廥 abrem e as pessoas esperam os primeiros 獼ibus. diff --git a/corpus/generated/windows-1252/pt/train/library.bin b/corpus/generated/windows-1252/pt/train/library.bin new file mode 100644 index 0000000..50b8034 --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/library.bin @@ -0,0 +1,3 @@ +A biblioteca tranquila + +Os leitores procuram livros de hist鏎ia e ci瘽cia enquanto a bibliotec嫫ia ajuda os alunos. diff --git a/corpus/generated/windows-1252/pt/train/market.bin b/corpus/generated/windows-1252/pt/train/market.bin new file mode 100644 index 0000000..3810190 --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/market.bin @@ -0,0 +1,3 @@ +Mercado de s墎ado + +Os agricultores vendem p緌 fresco, ma誽s, queijo e mel na pra蓷 movimentada. diff --git a/corpus/generated/windows-1252/pt/train/workshop.bin b/corpus/generated/windows-1252/pt/train/workshop.bin new file mode 100644 index 0000000..ac78c4e --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/workshop.bin @@ -0,0 +1,3 @@ +Oficina digital + +Uma pequena equipe prepara o guia do museu, verifica o c鏚igo e escreve descri踥es claras. diff --git a/corpus/generated/windows-1252/pt/validation/river-trip.bin b/corpus/generated/windows-1252/pt/validation/river-trip.bin new file mode 100644 index 0000000..de2590a --- /dev/null +++ b/corpus/generated/windows-1252/pt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Viagem ao rio + +Os amigos levaram um mapa e ch quente, caminharam pela mata e descansaram junto ao rio. diff --git a/corpus/generated/windows-1252/sv/test/community-garden.bin b/corpus/generated/windows-1252/sv/test/community-garden.bin new file mode 100644 index 0000000..0acb233 --- /dev/null +++ b/corpus/generated/windows-1252/sv/test/community-garden.bin @@ -0,0 +1,3 @@ +Gemensam tr輐g緳d + +Grannarna gjorde en tom g緳d till en gr霵 tr輐g緳d, barnen m嶚ade skyltar och familjerna vattnar v鉶terna varje vecka. diff --git a/corpus/generated/windows-1252/sv/train/city-morning.bin b/corpus/generated/windows-1252/sv/train/city-morning.bin new file mode 100644 index 0000000..f0ba24b --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgon i staden + +Staden vaknar l幩gsamt, kaf嶪rna 鞿pnar och m鄚niskor v鄚tar p de f顤sta bussarna. diff --git a/corpus/generated/windows-1252/sv/train/library.bin b/corpus/generated/windows-1252/sv/train/library.bin new file mode 100644 index 0000000..a7319ac --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/library.bin @@ -0,0 +1,3 @@ +Det tysta biblioteket + +L酲are s闥er b鐼ker om historia och vetenskap medan bibliotekarien hj鄟per eleverna. diff --git a/corpus/generated/windows-1252/sv/train/market.bin b/corpus/generated/windows-1252/sv/train/market.bin new file mode 100644 index 0000000..0bd6cfb --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/market.bin @@ -0,0 +1,3 @@ +L顤dagsmarknad + +B霵der s鄟jer f酺skt br鐰, 鄡plen, ost och honung p det livliga torget. diff --git a/corpus/generated/windows-1252/sv/train/workshop.bin b/corpus/generated/windows-1252/sv/train/workshop.bin new file mode 100644 index 0000000..81db320 --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/workshop.bin @@ -0,0 +1,3 @@ +Digital verkstad + +Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter f顤 bes闥are. diff --git a/corpus/generated/windows-1252/sv/validation/river-trip.bin b/corpus/generated/windows-1252/sv/validation/river-trip.bin new file mode 100644 index 0000000..4704973 --- /dev/null +++ b/corpus/generated/windows-1252/sv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Utflykt till floden + +V鄚nerna tog med karta och varmt te och vilade sedan vid floden. diff --git a/corpus/generated/windows-1253/el/test/community-garden.bin b/corpus/generated/windows-1253/el/test/community-garden.bin new file mode 100644 index 0000000..2e8196c --- /dev/null +++ b/corpus/generated/windows-1253/el/test/community-garden.bin @@ -0,0 +1,3 @@ + 膬澼濻濋 縤豵 + +矨 蜲裎瀍殣 椻摿摿 擩 塤樥 斠錞 貘嫇澼 縤豵, 酃 謯毈橎 翦蜤媺澞摿 謮罅罃麧 罼 毉 毉膬蜙翴毇 豵轚緮罍 酃 皭轗. diff --git a/corpus/generated/windows-1253/el/train/city-morning.bin b/corpus/generated/windows-1253/el/train/city-morning.bin new file mode 100644 index 0000000..f58abba --- /dev/null +++ b/corpus/generated/windows-1253/el/train/city-morning.bin @@ -0,0 +1,3 @@ +哢 黀賝 蹞錣 + + 蹞錣 隰謺 摙蜡, 酃 罼巕 摿檽蜧罍 罼 毉 塥黓豵 謥齌徾臩罍 酃 錼獿鼫葐. diff --git a/corpus/generated/windows-1253/el/train/library.bin b/corpus/generated/windows-1253/el/train/library.bin new file mode 100644 index 0000000..6724218 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/library.bin @@ -0,0 +1,3 @@ +幛蘙 睾碲濈魶聬 + +矨 摿摦薞黀殣 鸂籦瀦 睾碲葐 澞鐐麌摥 罼 歑澞轘懞 罼 睾碲濈鳿縔齌濿 碨賥 鐐蘪 憼鳿轕. diff --git a/corpus/generated/windows-1253/el/train/market.bin b/corpus/generated/windows-1253/el/train/market.bin new file mode 100644 index 0000000..40995f7 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/market.bin @@ -0,0 +1,3 @@ +貤瞁敳橎鏻聬 摦瀎 + +矨 摦嬿醲 豵纆燽 矙懃, 徾錁, 瓕燽酃 罼 檁夒僸縔 黀賝 謻敳橀. diff --git a/corpus/generated/windows-1253/el/train/workshop.bin b/corpus/generated/windows-1253/el/train/workshop.bin new file mode 100644 index 0000000..4f44381 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/workshop.bin @@ -0,0 +1,3 @@ +崱灖摓 殥蝁黀碖濈 + +欥 擩膲 瀌塤 殦毉幪緪 檷誽 斀蘦橀瀦, 槶楯灚 鐐 蕘僸罼 罼 蜤媺樥 罼魨鵘 耪蜉樇. diff --git a/corpus/generated/windows-1253/el/validation/river-trip.bin b/corpus/generated/windows-1253/el/validation/river-trip.bin new file mode 100644 index 0000000..dcea70f --- /dev/null +++ b/corpus/generated/windows-1253/el/validation/river-trip.bin @@ -0,0 +1,3 @@ +髑儊瀌 黀 豵轗擩 + +矨 廲錝 謰鼁 攩嚪 罼 緪黀 鏵塕 罼 鍷膬蘘嫇鐋罼 魠謻 黀 豵轗擩. diff --git a/corpus/generated/windows-1254/tr/test/community-garden.bin b/corpus/generated/windows-1254/tr/test/community-garden.bin new file mode 100644 index 0000000..831e0ea --- /dev/null +++ b/corpus/generated/windows-1254/tr/test/community-garden.bin @@ -0,0 +1,3 @@ +Mahalle bah蔒si + +Komular bo avluyu yeil bir bah蔒ye 蔒virdi, 蔞cuklar renkli tabelalar yapt ve aileler bitkileri her hafta suluyor. diff --git a/corpus/generated/windows-1254/tr/train/city-morning.bin b/corpus/generated/windows-1254/tr/train/city-morning.bin new file mode 100644 index 0000000..e8458f6 --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/city-morning.bin @@ -0,0 +1,3 @@ +溳hirde sabah + +溳hir yava蓷 uyanr, kafeler a踦lr ve insanlar ilk otobsleri bekler. diff --git a/corpus/generated/windows-1254/tr/train/library.bin b/corpus/generated/windows-1254/tr/train/library.bin new file mode 100644 index 0000000..20c465c --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/library.bin @@ -0,0 +1,3 @@ +Sessiz ktphane + +Okurlar tarih ve bilim kitaplar ararken ktphaneci 瓘rencilere yardmc olur. diff --git a/corpus/generated/windows-1254/tr/train/market.bin b/corpus/generated/windows-1254/tr/train/market.bin new file mode 100644 index 0000000..0856260 --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/market.bin @@ -0,0 +1,3 @@ +Cumartesi pazar + +パft蓾ler kalabalk meydanda taze ekmek, elma, peynir ve bal satar. diff --git a/corpus/generated/windows-1254/tr/train/workshop.bin b/corpus/generated/windows-1254/tr/train/workshop.bin new file mode 100644 index 0000000..844ca1e --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/workshop.bin @@ -0,0 +1,3 @@ +Dijital at闤ye + +K踙k bir ekip mze rehberi hazrlar, kodu denetler ve a踦k a踦klamalar yazar. diff --git a/corpus/generated/windows-1254/tr/validation/river-trip.bin b/corpus/generated/windows-1254/tr/validation/river-trip.bin new file mode 100644 index 0000000..2568000 --- /dev/null +++ b/corpus/generated/windows-1254/tr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Nehir gezisi + +Arkadalar harita ve scak 蓷y alp ormanda yrd, sonra nehir kysnda dinlendi. diff --git a/corpus/generated/windows-1255/he/test/community-garden.bin b/corpus/generated/windows-1255/he/test/community-garden.bin new file mode 100644 index 0000000..28d54b2 --- /dev/null +++ b/corpus/generated/windows-1255/he/test/community-garden.bin @@ -0,0 +1,3 @@ +鳱潞 噂僸檡澬 + +嘵鋺澼 勱錼 踤 鼞灛 懧潞 澮毿, 僸懠澼 灖澮 擯澼 槼霘鐋漦 霘灚 僬 噈鍡澼 摠 摎. diff --git a/corpus/generated/windows-1255/he/train/city-morning.bin b/corpus/generated/windows-1255/he/train/city-morning.bin new file mode 100644 index 0000000..90ce4a0 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/city-morning.bin @@ -0,0 +1,3 @@ +摎蠲 摥澮 + +劋澮 霝蠈矘 憵, 朄 噂醵 贄賟 樈蹠澼 鍡錁 憵樘槦殥澼 噌僓歑澼. diff --git a/corpus/generated/windows-1255/he/train/library.bin b/corpus/generated/windows-1255/he/train/library.bin new file mode 100644 index 0000000..d4abd33 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/library.bin @@ -0,0 +1,3 @@ +儊鐖澭 嘵玁 + +灚鸆澼 鍡鐒澼 嚪鼞 僸鼨氂毈 槴蜚 槼嚪斸澬 蠈翬 檡斁殫澼 憼谾 霠氂漦. diff --git a/corpus/generated/windows-1255/he/train/market.bin b/corpus/generated/windows-1255/he/train/market.bin new file mode 100644 index 0000000..d4e1672 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/market.bin @@ -0,0 +1,3 @@ +毿 朄 + +踮憵澼 鍷鋿澼 懞 儑, 蝁, 鏻鷿 槻摋謱 摠澣 劋鍷鼥. diff --git a/corpus/generated/windows-1255/he/train/workshop.bin b/corpus/generated/windows-1255/he/train/workshop.bin new file mode 100644 index 0000000..9120483 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/workshop.bin @@ -0,0 +1,3 @@ +鼖謱 蜒睾鳻澬 + +欈漦 玁 鎀濈 鍏鼞 斁樉歖樖, 摎裱 僬 噂樍 槶漦 歖氂澼 暟氂澼. diff --git a/corpus/generated/windows-1255/he/validation/river-trip.bin b/corpus/generated/windows-1255/he/validation/river-trip.bin new file mode 100644 index 0000000..f2a8f51 --- /dev/null +++ b/corpus/generated/windows-1255/he/validation/river-trip.bin @@ -0,0 +1,3 @@ +鴇樦 鳭 僽噌 + +儃暟澼 檉谾 隮 漦 賝, 僶錼 摋譊 歑谾 擩 僽噌. diff --git a/corpus/generated/windows-1256/ar/test/community-garden.bin b/corpus/generated/windows-1256/ar/test/community-garden.bin new file mode 100644 index 0000000..613b5d4 --- /dev/null +++ b/corpus/generated/windows-1256/ar/test/community-garden.bin @@ -0,0 +1,3 @@ +迍縶 ш俀 + +眒 ш氝捚 ш砱歶 磈愨堄и 楜 蚡虭 瘐罾氶 窳骱 ш贈晸 慳楟в 蝁緛 磈忡 ш枃縩 筈 豇 ш靿вв. diff --git a/corpus/generated/windows-1256/ar/train/city-morning.bin b/corpus/generated/windows-1256/ar/train/city-morning.bin new file mode 100644 index 0000000..59e7a5d --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/city-morning.bin @@ -0,0 +1,3 @@ +桱е ш蓔罽 + +弝扱睟 ш蓔罽 樕碻 磉楟 ш蜛ь 蠍磈暷 罽蒯 罽忭 ш鞁 ш虰楩в 緱痶欄 ш襞и. diff --git a/corpus/generated/windows-1256/ar/train/library.bin b/corpus/generated/windows-1256/ar/train/library.bin new file mode 100644 index 0000000..394f9ea --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/library.bin @@ -0,0 +1,3 @@ +ш蝃岓 ш暷玅 + +篸豖 ш痶Щ 筀 萯 ш岆栔 磈摍撂 緱蚡硨 辭罽 ш蝃岓 ш嵎а 楜 ё扱и ш蓛д ш蜠к. diff --git a/corpus/generated/windows-1256/ar/train/market.bin b/corpus/generated/windows-1256/ar/train/market.bin new file mode 100644 index 0000000..0ef6121 --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/market.bin @@ -0,0 +1,3 @@ +趷 ш蚧 + +縩晑 ш蒰и筅 ш恛 磈摍貤 磈摴ц暯 緱岓ж ш祔и ш窳晸 磈愨虰砅 ш禢篽. diff --git a/corpus/generated/windows-1256/ar/train/workshop.bin b/corpus/generated/windows-1256/ar/train/workshop.bin new file mode 100644 index 0000000..b9579e6 --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/workshop.bin @@ -0,0 +1,3 @@ +歶埩 栻蝂 + +縸蝆 楪縶 殏縼 眃罅 摽蒘俍 緱捚枆 ш蒨栜杸 ш婃挸 緱萯 ш蒠旄緛 窳晸 磈笴. diff --git a/corpus/generated/windows-1256/ar/validation/river-trip.bin b/corpus/generated/windows-1256/ar/validation/river-trip.bin new file mode 100644 index 0000000..fee14e0 --- /dev/null +++ b/corpus/generated/windows-1256/ar/validation/river-trip.bin @@ -0,0 +1,3 @@ +敊慓 鑤 ш麧 + +俁 ш譏癹Щ 恮繀 窴⑤ 蚡挕 緷娹 虮 ш麧 呾 к忻筅 鑤 崼歶 ш蒑Щ. diff --git a/corpus/generated/windows-1257/et/test/community-garden.bin b/corpus/generated/windows-1257/et/test/community-garden.bin new file mode 100644 index 0000000..46ad8a9 --- /dev/null +++ b/corpus/generated/windows-1257/et/test/community-garden.bin @@ -0,0 +1,3 @@ +Kogukonna aed + +Naabrid muutsid thja hoovi roheliseks aiaks, lapsed maalisid sildid ja pered kastavad taimi igal n輐alal. diff --git a/corpus/generated/windows-1257/et/train/city-morning.bin b/corpus/generated/windows-1257/et/train/city-morning.bin new file mode 100644 index 0000000..f24f4c7 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/city-morning.bin @@ -0,0 +1,3 @@ +Hommik linnas + +Linn 酺kab aeglaselt, kohvikud avavad uksed ja inimesed ootavad esimesi busse. diff --git a/corpus/generated/windows-1257/et/train/library.bin b/corpus/generated/windows-1257/et/train/library.bin new file mode 100644 index 0000000..1791206 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/library.bin @@ -0,0 +1,3 @@ +Vaikne raamatukogu + +Lugejad otsivad ajaloo- ja teadusraamatuid ning raamatukoguhoidja aitab 鱀ilasi. diff --git a/corpus/generated/windows-1257/et/train/market.bin b/corpus/generated/windows-1257/et/train/market.bin new file mode 100644 index 0000000..f7f3563 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/market.bin @@ -0,0 +1,3 @@ +Laup輑vane turg + +Talunikud mvad v酺sket leiba, 鰔nu, juustu ja mett rahvarohkel v鄟jakul. diff --git a/corpus/generated/windows-1257/et/train/workshop.bin b/corpus/generated/windows-1257/et/train/workshop.bin new file mode 100644 index 0000000..a61fe78 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/workshop.bin @@ -0,0 +1,3 @@ +Digitaalne t礵tuba + +V鄜ke meeskond koostab muuseumijuhti, kontrollib koodi ja kirjutab selgeid kirjeldusi. diff --git a/corpus/generated/windows-1257/et/validation/river-trip.bin b/corpus/generated/windows-1257/et/validation/river-trip.bin new file mode 100644 index 0000000..9b2f7eb --- /dev/null +++ b/corpus/generated/windows-1257/et/validation/river-trip.bin @@ -0,0 +1,3 @@ +Retk j髊 鳵rde + +S騸rad v鰗sid kaardi ja sooja tee ning puhkasid p酺ast jalutusk鄜ku j髊 鳵res. diff --git a/corpus/generated/windows-1257/lt/test/community-garden.bin b/corpus/generated/windows-1257/lt/test/community-garden.bin new file mode 100644 index 0000000..5d1e77b --- /dev/null +++ b/corpus/generated/windows-1257/lt/test/community-garden.bin @@ -0,0 +1,3 @@ +Bendruomen褭 sodas + +Kaimynai tu謶i kiem pavert aliu sodu su g螔螜is, darov螜is ir jaunais mediais. Vaikai nupie謻 spalvingus enklus, o 簟imos kas savait laisto augalus. diff --git a/corpus/generated/windows-1257/lt/train/city-morning.bin b/corpus/generated/windows-1257/lt/train/city-morning.bin new file mode 100644 index 0000000..4ab6bae --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/city-morning.bin @@ -0,0 +1,3 @@ +Rytas mieste + +Miestas bunda pamau. Kavin褭 atveria duris, autobusai pradeda pirmuosius mar繢utus, o mon褭 skaito inias. Vir sen stog 繗ies螝a dangus. diff --git a/corpus/generated/windows-1257/lt/train/library.bin b/corpus/generated/windows-1257/lt/train/library.bin new file mode 100644 index 0000000..3731c52 --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/library.bin @@ -0,0 +1,3 @@ +Tyli biblioteka + +Rajono bibliotekoje tyliai 簨ugda puslapiai. Skaitytojai ie簥o knyg apie istorij, moksl ir keliones. Bibliotekinink rengia parod mokiniams. diff --git a/corpus/generated/windows-1257/lt/train/market.bin b/corpus/generated/windows-1257/lt/train/market.bin new file mode 100644 index 0000000..d2203ae --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/market.bin @@ -0,0 +1,3 @@ +胗繑adienio turgus + +Turgaus aik繑 pripildo spalvos ir kvapai. 菣ininkai silo obuoli, medaus, srio ir 繗ieios duonos. Pirk螝ai renkasi daroves ir dalijasi receptais. diff --git a/corpus/generated/windows-1257/lt/train/workshop.bin b/corpus/generated/windows-1257/lt/train/workshop.bin new file mode 100644 index 0000000..3a5f804 --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/workshop.bin @@ -0,0 +1,3 @@ +Skaitmenin褭 dirbtuv褭 + +Nedidel komanda kuria skaitmenin muziejaus vadov. Dizaineriai tvarko meniu, programuotojai tikrina kod, o redaktoriai ra繖 ai簥ius apra羳mus. diff --git a/corpus/generated/windows-1257/lt/validation/river-trip.bin b/corpus/generated/windows-1257/lt/validation/river-trip.bin new file mode 100644 index 0000000..77189e1 --- /dev/null +++ b/corpus/generated/windows-1257/lt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kelion prie up褭 + +Sekmadien draugai i謻jo prie ramios up褭 u miesto. Jie pasi螜 em螔ap, 簨ltos arbatos ir fotoaparat. Po ilgo ygio visi ils螝osi ant kranto. diff --git a/corpus/generated/windows-1257/lv/test/community-garden.bin b/corpus/generated/windows-1257/lv/test/community-garden.bin new file mode 100644 index 0000000..9caae38 --- /dev/null +++ b/corpus/generated/windows-1257/lv/test/community-garden.bin @@ -0,0 +1,3 @@ +Kopienas d漷zs + +Kaimi爌 p漷v蔦ta tuk繠 pagalmu par za鮯 d漷zu ar pu耬m, d漷ze爌em un jauniem kokiem. B蔦ni izgatavoja kr滻ainas z螸es, bet 駧menes katru ned趠u laista augus. diff --git a/corpus/generated/windows-1257/lv/train/city-morning.bin b/corpus/generated/windows-1257/lv/train/city-morning.bin new file mode 100644 index 0000000..c32767d --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/city-morning.bin @@ -0,0 +1,3 @@ +R褾s pils蓨 + +Pils蓨a mostas l蓽i. Kafejn蟃as atver durvis, autobusi s毾 pirmos mar繢utus, un cilv蔝i lasa zi瀡s. Virs m毄u jumtiem debesis k燹st arvien gai謼kas. diff --git a/corpus/generated/windows-1257/lv/train/library.bin b/corpus/generated/windows-1257/lv/train/library.bin new file mode 100644 index 0000000..c112840 --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/library.bin @@ -0,0 +1,3 @@ +Klus bibliot蔝a + +Rajona bibliot蔝 klusi 醀ukst gr滵atu lapas. Las褾毄i mekl darbus par v蓧turi, zin漮ni un ce鮡jumiem. Bibliotek漷e gatavo izst歍i skol蓽iem. diff --git a/corpus/generated/windows-1257/lv/train/market.bin b/corpus/generated/windows-1257/lv/train/market.bin new file mode 100644 index 0000000..5d940ef --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/market.bin @@ -0,0 +1,3 @@ +Sestdienas tirgus + +Tirgus laukumu piepilda kr滻as un smaras. Zemnieki pied潎 槆olus, medu, sieru un svaigu maizi. Pirc蔨i izv蔮as d漷ze璷s un p漷run 駧menes receptes. diff --git a/corpus/generated/windows-1257/lv/train/workshop.bin b/corpus/generated/windows-1257/lv/train/workshop.bin new file mode 100644 index 0000000..0fbfb56 --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/workshop.bin @@ -0,0 +1,3 @@ +Digit滎 darbn蟃a + +Neliela komanda veido muzeja digit滎o ce鴳edi. Dizaineri sak漷to izv蔮ni, programm蓨毄i p漷bauda kodu, bet redaktori raksta skaidrus aprakstus. diff --git a/corpus/generated/windows-1257/lv/validation/river-trip.bin b/corpus/generated/windows-1257/lv/validation/river-trip.bin new file mode 100644 index 0000000..5ba00ad --- /dev/null +++ b/corpus/generated/windows-1257/lv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Pastaiga gar upi + +Sv蓨dien draugi dev滻 uz klusu upi 漷pus pils蓨as. Vi爌 pa蠊ma karti, siltu t蔨u un fotoapar漮u. P蓳 gar滻 pastaigas visi atpt滻 krast. diff --git a/corpus/generated/windows-1258/vi/test/community-garden.bin b/corpus/generated/windows-1258/vi/test/community-garden.bin new file mode 100644 index 0000000..be14fff --- /dev/null +++ b/corpus/generated/windows-1258/vi/test/community-garden.bin @@ -0,0 +1,3 @@ +V懽n c鐍ng 贄帔g + +H跣g x鏔 bi膵n s滱 tr鐔ng th跣h khu v懽n xanh, tre em ve bi縓n m軿 v c塶 gia 簨帔h t纍i c漧 m轘i tu獑n. diff --git a/corpus/generated/windows-1258/vi/train/city-morning.bin b/corpus/generated/windows-1258/vi/train/city-morning.bin new file mode 100644 index 0000000..321ee45 --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/city-morning.bin @@ -0,0 +1,3 @@ +Bu譣i s嫕g trong th跣h ph鐔 + +Th跣h ph鐔 th駥 d碞y ch碞m ra溮, qu嫕 c ph m欀 c浾 v mo爌 ng懽i ch懽 chuy膵n xe buy鮅 謼徂 ti瘽. diff --git a/corpus/generated/windows-1258/vi/train/library.bin b/corpus/generated/windows-1258/vi/train/library.bin new file mode 100644 index 0000000..89d20aa --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/library.bin @@ -0,0 +1,3 @@ +Th vi膷n y瘽 ti煣h + +哧瀣 gia ti帗 s塶h li瀣h s v khoa ho瀣, co帔 thu th gip ho瀣 sinh cho犥 t跬 li膷u. diff --git a/corpus/generated/windows-1258/vi/train/market.bin b/corpus/generated/windows-1258/vi/train/market.bin new file mode 100644 index 0000000..deb93fb --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/market.bin @@ -0,0 +1,3 @@ +Ch蘪 th ba烇 + +N獼g d滱 b嫕 b嫕h mi, t嫪, ph mai v m碞t ong ta爌 qua烋g tr懽ng 贄ng 蹗c. diff --git a/corpus/generated/windows-1258/vi/train/workshop.bin b/corpus/generated/windows-1258/vi/train/workshop.bin new file mode 100644 index 0000000..6f4c26b --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/workshop.bin @@ -0,0 +1,3 @@ +X欀ng ky thu碞t s鐔 + +M鐍t nh鏔 nho l跩 h纍ng d瘑n ba缹 t跣g, ki縓m tra ma v vi膵t m ta ro r跣g. diff --git a/corpus/generated/windows-1258/vi/validation/river-trip.bin b/corpus/generated/windows-1258/vi/validation/river-trip.bin new file mode 100644 index 0000000..aca81b2 --- /dev/null +++ b/corpus/generated/windows-1258/vi/validation/river-trip.bin @@ -0,0 +1,3 @@ +Chuy膵n 簨 ra s獼g + +Ba犥 b mang ba烋 贄 v tr n鏮g, 簨 qua r帔g r覹i nghi b瘽 b懽 s獼g. diff --git a/corpus/generated/windows-874/th/test/community-garden.bin b/corpus/generated/windows-874/th/test/community-garden.bin new file mode 100644 index 0000000..8f1080c --- /dev/null +++ b/corpus/generated/windows-874/th/test/community-garden.bin @@ -0,0 +1,3 @@ +岆鼓媮疚 + +鉧郯芧粹珗鉬纓餔嘉珗я狶鉬蝜岆嘟淀Ｇ藉 鉥蝖 й暑橏轍瓚勻芮勻捚芾Ъ×嘔晶橔菟嫣鍾煥﹋捋棍咘 diff --git a/corpus/generated/windows-874/th/train/city-morning.bin b/corpus/generated/windows-874/th/train/city-morning.bin new file mode 100644 index 0000000..3325a1b --- /dev/null +++ b/corpus/generated/windows-874/th/train/city-morning.bin @@ -0,0 +1,3 @@ +鄋橏蓍鉡谹 + +鉡谹坐駖 菁頩 鏑珗÷彃鉬埭酵迮殕瓚樑擗嘔芤嗯朝弅蹴煬餔ш獺 diff --git a/corpus/generated/windows-874/th/train/library.bin b/corpus/generated/windows-874/th/train/library.bin new file mode 100644 index 0000000..8574c4f --- /dev/null +++ b/corpus/generated/windows-874/th/train/library.bin @@ -0,0 +1,3 @@ +呯竻岏奡鄑梒監宋 + +樑樿骳馱暽刳佴悝忑芼譜и菠疺腕憼瓚л溘疿疺腕 疑籀疰恁扥芾Ъ塾﹤譏繒鉣谹﹜樿還 diff --git a/corpus/generated/windows-874/th/train/market.bin b/corpus/generated/windows-874/th/train/market.bin new file mode 100644 index 0000000..e2a1717 --- /dev/null +++ b/corpus/generated/windows-874/th/train/market.bin @@ -0,0 +1,3 @@ +腋珒и嫩弅鏈 + +肫Л牶畟Ⅰ覬悝妠 敷靾 肪 慞邿橔樂擏蓍籤僩涷少﹞恁 diff --git a/corpus/generated/windows-874/th/train/workshop.bin b/corpus/generated/windows-874/th/train/workshop.bin new file mode 100644 index 0000000..c571435 --- /dev/null +++ b/corpus/generated/windows-874/th/train/workshop.bin @@ -0,0 +1,3 @@ +呯竻照忌僑唻埸捇 + +煬醜驗”挬照屯餑谹戰戰蛻捖桿 腕ヮ襠捗 慞冓Ｇ繒太芵埏畟煬隤挬鄖 diff --git a/corpus/generated/windows-874/th/validation/river-trip.bin b/corpus/generated/windows-874/th/validation/river-trip.bin new file mode 100644 index 0000000..0e376c8 --- /dev/null +++ b/corpus/generated/windows-874/th/validation/river-trip.bin @@ -0,0 +1,3 @@ +鈮涷藉譎鍍錘寥 + +鉧郯芧墓廒僩涷×漯疰樿嫣韶椒嫘劂骳慞橶憾□駖嘔埡徶頩橔 diff --git a/corpus/manifest.json b/corpus/manifest.json index 5121f91..776c6c4 100644 --- a/corpus/manifest.json +++ b/corpus/manifest.json @@ -1,5 +1,45 @@ { "encodings": [ + { + "name": "ISO-8859-1", + "iconv": "ISO-8859-1", + "languages": ["da", "de", "en", "es", "fr", "it", "nl", "no", "pt", "sv"] + }, + { + "name": "ISO-8859-2", + "iconv": "ISO-8859-2", + "languages": ["cs", "hu", "pl", "ro"], + "sourceReplacements": { "": "", "": "", "": "籠", "": "籟" } + }, + { "name": "ISO-8859-5", "iconv": "ISO-8859-5", "languages": ["ru"] }, + { "name": "ISO-8859-6", "iconv": "ISO-8859-6", "languages": ["ar"] }, + { "name": "ISO-8859-7", "iconv": "ISO-8859-7", "languages": ["el"] }, + { "name": "ISO-8859-8", "iconv": "ISO-8859-8", "languages": ["he"] }, + { "name": "ISO-8859-9", "iconv": "ISO-8859-9", "languages": ["tr"] }, + { + "name": "windows-1250", + "iconv": "WINDOWS-1250", + "languages": ["cs", "hu", "pl", "ro"], + "sourceReplacements": { "": "", "": "", "": "籠", "": "籟" } + }, + { "name": "windows-1251", "iconv": "WINDOWS-1251", "languages": ["ru"] }, + { + "name": "windows-1252", + "iconv": "WINDOWS-1252", + "languages": ["da", "de", "en", "es", "fr", "it", "nl", "no", "pt", "sv"] + }, + { "name": "windows-1253", "iconv": "WINDOWS-1253", "languages": ["el"] }, + { "name": "windows-1254", "iconv": "WINDOWS-1254", "languages": ["tr"] }, + { "name": "windows-1255", "iconv": "WINDOWS-1255", "languages": ["he"] }, + { "name": "windows-1256", "iconv": "WINDOWS-1256", "languages": ["ar"] }, + { + "name": "windows-1257", + "iconv": "WINDOWS-1257", + "languages": ["et", "lv", "lt"] + }, + { "name": "windows-1258", "iconv": "WINDOWS-1258", "languages": ["vi"] }, + { "name": "windows-874", "iconv": "WINDOWS-874", "languages": ["th"] }, + { "name": "KOI8-R", "iconv": "KOI8-R", "languages": ["ru"] }, { "name": "KOI8-U", "iconv": "KOI8-U", "languages": ["uk"] }, { "name": "IBM866", "iconv": "CP866", "languages": ["ru"] }, { "name": "IBM855", "iconv": "CP855", "languages": ["ru", "sr"] }, @@ -13,7 +53,44 @@ "iconv": "MAC-CYRILLIC", "languages": ["ru", "uk"] }, - { "name": "CP949", "iconv": "CP949", "languages": ["ko"] }, + { + "name": "Shift_JIS", + "iconv": "SHIFT_JIS", + "languages": ["ja"], + "family": "multibyte" + }, + { + "name": "EUC-JP", + "iconv": "EUC-JP", + "languages": ["ja"], + "family": "multibyte" + }, + { + "name": "Big5", + "iconv": "BIG5", + "languages": ["zh"], + "family": "multibyte" + }, + { + "name": "GB18030", + "iconv": "GB18030", + "languages": ["zh"], + "family": "multibyte" + }, + { + "name": "EUC-KR", + "iconv": "EUC-KR", + "languages": ["ko"], + "family": "multibyte", + "sourceReplacements": { "": "篣", "賰": "" } + }, + { + "name": "CP949", + "iconv": "CP949", + "languages": ["ko"], + "family": "multibyte", + "runtime": false + }, { "name": "ISO-8859-3", "iconv": "ISO-8859-3", "languages": ["mt"] }, { "name": "ISO-8859-4", "iconv": "ISO-8859-4", "languages": ["lv"] }, { "name": "ISO-8859-10", "iconv": "ISO-8859-10", "languages": ["is"] }, @@ -24,7 +101,12 @@ "iconv": "ISO-8859-15", "languages": ["fr", "de", "es"] }, - { "name": "ISO-8859-16", "iconv": "ISO-8859-16", "languages": ["ro"] }, + { + "name": "ISO-8859-16", + "iconv": "ISO-8859-16", + "languages": ["ro"], + "sourceSuffix": " Preul este 10 ." + }, { "name": "CP850", "iconv": "CP850", "languages": ["fr", "de", "es"] }, { "name": "CP852", "iconv": "CP852", "languages": ["pl"] } ] diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 31a0927..dde38c1 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -1,9 +1,11 @@ { "summary": { - "tests": 22, - "encodingCorrect": 22, - "languageCorrect": 22, - "topScoreTies": 0 + "tests": 66, + "encodingExact": 46, + "encodingEquivalent": 20, + "encodingCorrect": 66, + "languageCorrect": 66, + "topScoreTies": 37 }, "confusion": { "CP850": { @@ -18,33 +20,88 @@ "IBM866": { "IBM866": 1 }, + "ISO-8859-1": { + "ISO-8859-1": 10 + }, "ISO-8859-10": { "ISO-8859-10": 1 }, "ISO-8859-13": { - "ISO-8859-13": 1 + "windows-1257": 1 }, "ISO-8859-14": { "ISO-8859-14": 1 }, "ISO-8859-15": { - "ISO-8859-15": 3 + "ISO-8859-1": 3 }, "ISO-8859-16": { "ISO-8859-16": 1 }, + "ISO-8859-2": { + "ISO-8859-2": 4 + }, "ISO-8859-3": { "ISO-8859-3": 1 }, "ISO-8859-4": { "ISO-8859-4": 1 }, + "ISO-8859-5": { + "ISO-8859-5": 1 + }, + "ISO-8859-6": { + "ISO-8859-6": 1 + }, + "ISO-8859-7": { + "ISO-8859-7": 1 + }, + "ISO-8859-8": { + "ISO-8859-8": 1 + }, + "ISO-8859-9": { + "ISO-8859-9": 1 + }, + "KOI8-R": { + "KOI8-R": 1 + }, "KOI8-U": { "KOI8-U": 1 }, "macintosh": { "macintosh": 3 }, + "windows-1250": { + "ISO-8859-2": 3, + "windows-1250": 1 + }, + "windows-1251": { + "windows-1251": 1 + }, + "windows-1252": { + "ISO-8859-1": 10 + }, + "windows-1253": { + "ISO-8859-7": 1 + }, + "windows-1254": { + "ISO-8859-9": 1 + }, + "windows-1255": { + "ISO-8859-8": 1 + }, + "windows-1256": { + "windows-1256": 1 + }, + "windows-1257": { + "windows-1257": 3 + }, + "windows-1258": { + "windows-1258": 1 + }, + "windows-874": { + "windows-874": 1 + }, "x-mac-cyrillic": { "x-mac-cyrillic": 2 } @@ -57,30 +114,50 @@ }, "predicted": { "encoding": "CP850", - "confidence": 73, "language": "de", + "confidence": 73, "byteLogLikelihood": -3.169712077519, "hits": 53, "total": 216, "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "CP850", - "confidence": 73, "language": "de", + "confidence": 73, "byteLogLikelihood": -3.169712077519, "hits": 53, "total": 216, "hitRate": 0.24537037037037038 }, + { + "encoding": "windows-1252", + "language": "de", + "confidence": 72, + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, { "encoding": "ISO-8859-15", + "language": "de", + "confidence": 72, + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "ISO-8859-1", + "language": "de", "confidence": 72, - "language": "es", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -88,8 +165,8 @@ }, { "encoding": "macintosh", + "language": "de", "confidence": 70, - "language": "es", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -97,53 +174,53 @@ }, { "encoding": "ISO-8859-14", - "confidence": 22, "language": "cy", + "confidence": 22, "byteLogLikelihood": -4.912654885736, "hits": 16, "total": 215, "hitRate": 0.07441860465116279 }, { - "encoding": "ISO-8859-16", - "confidence": 13, + "encoding": "ISO-8859-2", "language": "ro", + "confidence": 13, "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 215, "hitRate": 0.046511627906976744 }, { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 13, + "byteLogLikelihood": -5.075173815234, + "hits": 10, "total": 215, - "hitRate": 0.037209302325581395 + "hitRate": 0.046511627906976744 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 10, "total": 215, - "hitRate": 0.018604651162790697 + "hitRate": 0.046511627906976744 }, { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 + "encoding": "windows-1257", + "language": "lv", + "confidence": 11, + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 215, + "hitRate": 0.037209302325581395 }, { "encoding": "ISO-8859-10", - "confidence": 4, "language": "is", + "confidence": 4, "byteLogLikelihood": -4.97295762689, "hits": 3, "total": 215, @@ -151,48 +228,12 @@ }, { "encoding": "ISO-8859-13", - "confidence": 2, "language": "lt", + "confidence": 2, "byteLogLikelihood": -5.00843944931, "hits": 2, "total": 215, "hitRate": 0.009302325581395349 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.695189876932, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.428281637736, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.621314697937, - "hits": 0, - "total": 216, - "hitRate": 0 } ] }, @@ -203,66 +244,77 @@ }, "predicted": { "encoding": "CP850", - "confidence": 61, "language": "es", + "confidence": 61, "byteLogLikelihood": -3.834657935767, "hits": 41, "total": 200, "hitRate": 0.205 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "CP850", - "confidence": 61, "language": "es", + "confidence": 61, "byteLogLikelihood": -3.834657935767, "hits": 41, "total": 200, "hitRate": 0.205 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-1", + "language": "es", "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "windows-1252", "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, "hitRate": 0.19597989949748743 }, { - "encoding": "ISO-8859-15", + "encoding": "macintosh", + "language": "es", "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "ISO-8859-15", "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, "hitRate": 0.19597989949748743 }, { - "encoding": "ISO-8859-4", - "confidence": 25, + "encoding": "windows-1257", "language": "lv", + "confidence": 25, "byteLogLikelihood": -5.19295685089, "hits": 17, "total": 199, "hitRate": 0.08542713567839195 }, - { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 13, - "total": 199, - "hitRate": 0.06532663316582915 - }, { "encoding": "ISO-8859-14", - "confidence": 16, "language": "cy", + "confidence": 16, "byteLogLikelihood": -4.912654885736, "hits": 11, "total": 199, @@ -270,44 +322,26 @@ }, { "encoding": "ISO-8859-16", - "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "confidence": 15, + "byteLogLikelihood": -4.952116173006, "hits": 10, "total": 199, "hitRate": 0.05025125628140704 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 199, - "hitRate": 0.01507537688442211 - }, - { - "encoding": "KOI8-U", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.351652470383, + "byteLogLikelihood": -4.61627103998, "hits": 0, "total": 199, "hitRate": 0 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", + "confidence": 0, "byteLogLikelihood": -3.374008967132, "hits": 0, "total": 200, @@ -315,26 +349,17 @@ }, { "encoding": "IBM855", - "confidence": 0, "language": "sr", + "confidence": 0, "byteLogLikelihood": -4.64715417232, "hits": 0, "total": 200, "hitRate": 0 }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 199, - "hitRate": 0 - }, { "encoding": "ISO-8859-3", - "confidence": 0, "language": "mt", + "confidence": 0, "byteLogLikelihood": -4.759371883621, "hits": 0, "total": 199, @@ -349,30 +374,50 @@ }, "predicted": { "encoding": "CP850", - "confidence": 77, "language": "fr", + "confidence": 77, "byteLogLikelihood": -2.06619631493, "hits": 59, "total": 228, "hitRate": 0.25877192982456143 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "CP850", - "confidence": 77, "language": "fr", + "confidence": 77, "byteLogLikelihood": -2.06619631493, "hits": 59, "total": 228, "hitRate": 0.25877192982456143 }, + { + "encoding": "windows-1252", + "language": "fr", + "confidence": 75, + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, { "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 75, + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "ISO-8859-1", + "language": "fr", "confidence": 75, - "language": "es", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -380,111 +425,75 @@ }, { "encoding": "macintosh", + "language": "fr", "confidence": 71, - "language": "es", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, "hitRate": 0.23684210526315788 }, { - "encoding": "ISO-8859-10", + "encoding": "ISO-8859-2", + "language": "ro", "confidence": 12, - "language": "is", - "byteLogLikelihood": -5.204006687077, + "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, "hitRate": 0.04017857142857143 }, { - "encoding": "ISO-8859-16", - "confidence": 12, + "encoding": "windows-1250", "language": "ro", + "confidence": 12, "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, "hitRate": 0.04017857142857143 }, { - "encoding": "ISO-8859-4", - "confidence": 9, + "encoding": "windows-1257", "language": "lv", + "confidence": 9, "byteLogLikelihood": -5.19295685089, "hits": 7, "total": 224, "hitRate": 0.03125 }, - { - "encoding": "ISO-8859-3", - "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 224, - "hitRate": 0.026785714285714284 - }, { "encoding": "CP852", - "confidence": 7, "language": "pl", + "confidence": 7, "byteLogLikelihood": -5.087596335232, "hits": 6, "total": 228, "hitRate": 0.02631578947368421 }, { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 6, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 6, + "byteLogLikelihood": -5.075173815234, + "hits": 5, "total": 224, - "hitRate": 0.017857142857142856 + "hitRate": 0.022321428571428572 }, { "encoding": "ISO-8859-14", - "confidence": 4, "language": "cy", + "confidence": 4, "byteLogLikelihood": -4.912654885736, "hits": 3, "total": 224, "hitRate": 0.013392857142857142 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.520621127559, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, - "hits": 0, - "total": 228, - "hitRate": 0 } ] }, @@ -495,142 +504,126 @@ }, "predicted": { "encoding": "CP852", - "confidence": 40, "language": "pl", + "confidence": 40, "byteLogLikelihood": -3.52910527172, "hits": 25, "total": 184, "hitRate": 0.1358695652173913 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "CP852", - "confidence": 40, "language": "pl", + "confidence": 40, "byteLogLikelihood": -3.52910527172, "hits": 25, "total": 184, "hitRate": 0.1358695652173913 }, { - "encoding": "macintosh", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 4, - "total": 182, - "hitRate": 0.02197802197802198 + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 37, + "byteLogLikelihood": -5.087596335232, + "hits": 23, + "total": 184, + "hitRate": 0.125 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "language": "et", "confidence": 6, - "language": "lt", - "byteLogLikelihood": -5.123963979403, + "byteLogLikelihood": -4.536305162191, "hits": 4, "total": 183, "hitRate": 0.02185792349726776 }, + { + "encoding": "macintosh", + "language": "fr", + "confidence": 6, + "byteLogLikelihood": -5.010635294096, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, { "encoding": "ISO-8859-15", + "language": "es", "confidence": 6, - "language": "de", "byteLogLikelihood": -4.948759890378, "hits": 4, "total": 183, "hitRate": 0.02185792349726776 }, - { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 183, - "hitRate": 0.01639344262295082 - }, - { - "encoding": "ISO-8859-4", - "confidence": 4, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 184, - "hitRate": 0.016304347826086956 - }, { "encoding": "CP850", + "language": "fr", "confidence": 4, - "language": "es", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 184, "hitRate": 0.016304347826086956 }, - { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 184, - "hitRate": 0.010869565217391304 - }, { "encoding": "ISO-8859-14", - "confidence": 3, "language": "cy", + "confidence": 3, "byteLogLikelihood": -4.912654885736, "hits": 2, "total": 184, "hitRate": 0.010869565217391304 }, { - "encoding": "ISO-8859-16", - "confidence": 3, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 2, + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 1, + "byteLogLikelihood": -5.075173815234, + "hits": 1, "total": 183, - "hitRate": 0.01092896174863388 + "hitRate": 0.00546448087431694 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.496246341796, - "hits": 0, + "encoding": "windows-1254", + "language": "tr", + "confidence": 1, + "byteLogLikelihood": -5.075173815234, + "hits": 1, "total": 183, - "hitRate": 0 + "hitRate": 0.00546448087431694 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.085936520567, - "hits": 0, - "total": 184, - "hitRate": 0 - }, - { - "encoding": "IBM855", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.780131236409, + "byteLogLikelihood": -5.085936520567, "hits": 0, "total": 184, "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.963380725619, - "hits": 0, - "total": 183, - "hitRate": 0 } ] }, @@ -641,141 +634,125 @@ }, "predicted": { "encoding": "IBM855", - "confidence": 43, "language": "ru", + "confidence": 43, "byteLogLikelihood": -3.313760777855, "hits": 26, "total": 179, "hitRate": 0.1452513966480447 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "IBM855", - "confidence": 43, "language": "ru", + "confidence": 43, "byteLogLikelihood": -3.313760777855, "hits": 26, "total": 179, "hitRate": 0.1452513966480447 }, - { - "encoding": "KOI8-U", - "confidence": 12, - "language": "uk", - "byteLogLikelihood": -5.728613539801, - "hits": 7, - "total": 171, - "hitRate": 0.04093567251461988 - }, - { - "encoding": "IBM866", - "confidence": 11, - "language": "ru", - "byteLogLikelihood": -5.08851401058, - "hits": 5, - "total": 130, - "hitRate": 0.038461538461538464 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 4, - "language": "ru", - "byteLogLikelihood": -5.827543836069, - "hits": 2, - "total": 137, - "hitRate": 0.014598540145985401 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.06395526639, - "hits": 1, - "total": 160, - "hitRate": 0.00625 - }, { "encoding": "ISO-8859-14", - "confidence": 1, "language": "cy", + "confidence": 1, "byteLogLikelihood": -4.908064507057, "hits": 1, "total": 174, "hitRate": 0.005747126436781609 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-1", + "language": "pt", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.934208734237, + "byteLogLikelihood": -4.789358524023, "hits": 0, - "total": 73, + "total": 139, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.011017819902, + "byteLogLikelihood": -4.901389021393, "hits": 0, - "total": 123, + "total": 165, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.171130116678, + "byteLogLikelihood": -4.901389021393, "hits": 0, - "total": 169, + "total": 141, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "windows-1252", + "language": "pt", "confidence": 0, + "byteLogLikelihood": -4.789358524023, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1257", "language": "lt", + "confidence": 0, "byteLogLikelihood": -4.913061957297, "hits": 0, "total": 149, "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 0, + "encoding": "macintosh", "language": "es", - "byteLogLikelihood": -4.805721913065, + "confidence": 0, + "byteLogLikelihood": -4.934208734237, "hits": 0, - "total": 160, + "total": 73, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-3", + "language": "mt", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.991915414162, + "byteLogLikelihood": -5.011017819902, "hits": 0, - "total": 162, + "total": 123, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 0, + "byteLogLikelihood": -4.913061957297, + "hits": 0, + "total": 149, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", "language": "es", - "byteLogLikelihood": -4.829101213886, + "confidence": 0, + "byteLogLikelihood": -4.805721913065, "hits": 0, - "total": 167, + "total": 160, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.06627927679, + "byteLogLikelihood": -4.829101213886, "hits": 0, - "total": 178, + "total": 167, "hitRate": 0 } ] @@ -787,21 +764,23 @@ }, "predicted": { "encoding": "IBM855", - "confidence": 28, "language": "sr", + "confidence": 28, "byteLogLikelihood": -3.307897947426, "hits": 16, "total": 166, "hitRate": 0.0963855421686747 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "IBM855", - "confidence": 28, "language": "sr", + "confidence": 28, "byteLogLikelihood": -3.307897947426, "hits": 16, "total": 166, @@ -809,71 +788,71 @@ }, { "encoding": "IBM866", - "confidence": 26, "language": "ru", + "confidence": 26, "byteLogLikelihood": -4.750368103744, "hits": 12, "total": 137, "hitRate": 0.08759124087591241 }, { - "encoding": "KOI8-U", - "confidence": 9, - "language": "uk", - "byteLogLikelihood": -5.956096731591, - "hits": 4, - "total": 130, - "hitRate": 0.03076923076923077 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 0, + "byteLogLikelihood": -4.764166439133, + "hits": 0, + "total": 104, + "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", - "confidence": 8, - "language": "ru", - "byteLogLikelihood": -5.930887805719, - "hits": 3, - "total": 112, - "hitRate": 0.026785714285714284 + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.902465386166, + "hits": 0, + "total": 128, + "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.933799879287, + "byteLogLikelihood": -4.902465386166, "hits": 0, - "total": 73, + "total": 110, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1252", + "language": "pt", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.97248951392, + "byteLogLikelihood": -4.764166439133, "hits": 0, - "total": 99, + "total": 110, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "byteLogLikelihood": -4.902859191011, "hits": 0, - "total": 131, + "total": 136, "hitRate": 0 }, { - "encoding": "ISO-8859-10", + "encoding": "macintosh", + "language": "es", "confidence": 0, - "language": "is", - "byteLogLikelihood": -5.076797848077, + "byteLogLikelihood": -4.933799879287, "hits": 0, - "total": 145, + "total": 73, "hitRate": 0 }, { "encoding": "ISO-8859-13", - "confidence": 0, "language": "lt", + "confidence": 0, "byteLogLikelihood": -4.940581981725, "hits": 0, "total": 132, @@ -881,8 +860,8 @@ }, { "encoding": "ISO-8859-14", - "confidence": 0, "language": "cy", + "confidence": 0, "byteLogLikelihood": -4.907668215372, "hits": 0, "total": 157, @@ -890,39 +869,21 @@ }, { "encoding": "ISO-8859-15", - "confidence": 0, "language": "es", + "confidence": 0, "byteLogLikelihood": -4.832419313648, "hits": 0, "total": 138, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.064961155115, - "hits": 0, - "total": 142, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", "byteLogLikelihood": -4.704589417548, "hits": 0, "total": 150, "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.064438955197, - "hits": 0, - "total": 165, - "hitRate": 0 } ] }, @@ -933,102 +894,95 @@ }, "predicted": { "encoding": "IBM866", - "confidence": 41, "language": "ru", + "confidence": 41, "byteLogLikelihood": -3.313760777855, "hits": 25, "total": 179, "hitRate": 0.13966480446927373 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "IBM866", - "confidence": 41, "language": "ru", + "confidence": 41, "byteLogLikelihood": -3.313760777855, "hits": 25, "total": 179, "hitRate": 0.13966480446927373 }, - { - "encoding": "x-mac-cyrillic", - "confidence": 15, - "language": "uk", - "byteLogLikelihood": -5.792699163742, - "hits": 7, - "total": 133, - "hitRate": 0.05263157894736842 - }, - { - "encoding": "KOI8-U", - "confidence": 11, - "language": "uk", - "byteLogLikelihood": -6.434389971368, - "hits": 4, - "total": 103, - "hitRate": 0.038834951456310676 - }, - { - "encoding": "IBM855", - "confidence": 8, - "language": "sr", - "byteLogLikelihood": -5.021646546946, - "hits": 5, - "total": 169, - "hitRate": 0.029585798816568046 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.103902669993, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 - }, { "encoding": "ISO-8859-14", - "confidence": 1, "language": "cy", + "confidence": 1, "byteLogLikelihood": -4.83002806951, "hits": 1, "total": 169, "hitRate": 0.005917159763313609 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-1", + "language": "it", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944169511699, + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 67, + "total": 81, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.096764716542, + "byteLogLikelihood": -4.941337938888, "hits": 0, - "total": 116, + "total": 145, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "windows-1250", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 117, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "it", "confidence": 0, + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 82, + "hitRate": 0 + }, + { + "encoding": "windows-1257", "language": "lv", - "byteLogLikelihood": -4.979786266462, + "confidence": 0, + "byteLogLikelihood": -4.928541579896, "hits": 0, - "total": 148, + "total": 117, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "macintosh", + "language": "es", "confidence": 0, + "byteLogLikelihood": -4.944169511699, + "hits": 0, + "total": 67, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", "language": "lt", + "confidence": 0, "byteLogLikelihood": -4.939479070188, "hits": 0, "total": 116, @@ -1036,26 +990,17 @@ }, { "encoding": "ISO-8859-15", - "confidence": 0, "language": "es", + "confidence": 0, "byteLogLikelihood": -4.848242748706, "hits": 0, "total": 111, "hitRate": 0 }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.06091613544, - "hits": 0, - "total": 144, - "hitRate": 0 - }, { "encoding": "CP850", - "confidence": 0, "language": "es", + "confidence": 0, "byteLogLikelihood": -4.677183836449, "hits": 0, "total": 130, @@ -1063,8 +1008,8 @@ }, { "encoding": "CP852", - "confidence": 0, "language": "pl", + "confidence": 0, "byteLogLikelihood": -4.699618505771, "hits": 0, "total": 166, @@ -1074,1898 +1019,7410 @@ }, { "expected": { - "encoding": "ISO-8859-10", - "language": "is" + "encoding": "ISO-8859-1", + "language": "da" }, "predicted": { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "ISO-8859-3", - "confidence": 12, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 + "encoding": "windows-1252", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "ISO-8859-13", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -3.989583472462, + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, "hits": 4, - "total": 171, - "hitRate": 0.023391812865497075 - }, - { - "encoding": "macintosh", - "confidence": 5, - "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 - }, - { - "encoding": "ISO-8859-4", - "confidence": 5, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "ISO-8859-15", - "confidence": 5, - "language": "es", - "byteLogLikelihood": -4.894502014278, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 + "encoding": "windows-1250", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "CP850", - "confidence": 5, - "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "encoding": "windows-1257", + "language": "et", + "confidence": 10, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 2, + "byteLogLikelihood": -4.658005606594, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -5.176149732574, + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 2, + "byteLogLikelihood": -4.145958228046, "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.026638569428, + "byteLogLikelihood": -4.689401143463, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.838861551509, + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.795369889152, + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 169, + "total": 120, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.733983711968, + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.657284871846, + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-13", - "language": "lt" + "encoding": "ISO-8859-1", + "language": "de" }, "predicted": { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 3, "candidates": [ { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "ISO-8859-4", - "confidence": 22, - "language": "lv", - "byteLogLikelihood": -4.610398759565, - "hits": 13, - "total": 173, - "hitRate": 0.07514450867052024 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "macintosh", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 168, - "hitRate": 0.041666666666666664 + "encoding": "windows-1252", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "ISO-8859-15", + "encoding": "ISO-8859-9", + "language": "tr", "confidence": 12, - "language": "fr", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 }, { - "encoding": "CP850", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, "hits": 7, - "total": 170, - "hitRate": 0.041176470588235294 + "total": 216, + "hitRate": 0.032407407407407406 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "en" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 29, + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + { + "encoding": "windows-1252", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 131, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 0, + "byteLogLikelihood": -4.510859506517, + "hits": 0, + "total": 132, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 10, + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "it" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1258", + "language": "vi", + "confidence": 16, + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 15, + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 15, + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 3, + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 63, + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "windows-1252", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 11, + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 151, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "no" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "windows-1252", + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 7, + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 7, + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 2, + "byteLogLikelihood": -4.210065739287, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 2, + "byteLogLikelihood": -4.319245023186, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1250", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.846220145127, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -3.503504267293, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "language": "sr", + "confidence": 0, + "byteLogLikelihood": -3.774998078834, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.766179593299, + "hits": 0, + "total": 143, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "sv" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1252", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 17, + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 4, + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 4, + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -3.287368473007, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-10", + "language": "is" + }, + "predicted": { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 38, + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 38, + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 + }, + { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 14, + "byteLogLikelihood": -4.633646206449, + "hits": 8, + "total": 171, + "hitRate": 0.04678362573099415 + }, + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 7, + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 7, + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 3, + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 3, + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.963723644165, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.249634019938, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.84046735452, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.963723644165, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.249634019938, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.733983711968, + "hits": 0, + "total": 171, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-13", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 22, + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 1, + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.531827362419, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.441328843196, + "hits": 0, + "total": 173, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-14", + "language": "cy" + }, + "predicted": { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 50, + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 50, + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 + }, + { + "encoding": "windows-1258", + "language": "vi", + "confidence": 14, + "byteLogLikelihood": -4.34192292436, + "hits": 9, + "total": 188, + "hitRate": 0.047872340425531915 + }, + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 6, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 189, + "hitRate": 0.021164021164021163 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 4, + "byteLogLikelihood": -4.009500374258, + "hits": 3, + "total": 189, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 1, + "byteLogLikelihood": -3.647025046455, + "hits": 1, + "total": 189, + "hitRate": 0.005291005291005291 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.254156765951, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.467895804539, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.254156765951, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.255815902698, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 0, + "byteLogLikelihood": -3.903274585551, + "hits": 0, + "total": 189, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 10, + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-16", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 53, + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 53, + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 44, + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 44, + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 8, + "byteLogLikelihood": -4.456762061842, + "hits": 6, + "total": 214, + "hitRate": 0.028037383177570093 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.499845100591, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.429071192897, + "hits": 0, + "total": 213, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.382241948417, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.429071192897, + "hits": 0, + "total": 213, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.333669615722, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.382241948417, + "hits": 0, + "total": 214, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 7, + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.249199684222, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.226561011787, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 0, + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 4, + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 2, + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 2, + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "pl" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "windows-1250", + "language": "pl", + "confidence": 37, + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 183, + "hitRate": 0.12568306010928962 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 6, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 6, + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 6, + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "CP850", + "language": "fr", + "confidence": 6, + "byteLogLikelihood": -5.010635294096, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 4, + "byteLogLikelihood": -4.794788079951, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 1, + "byteLogLikelihood": -4.476342508555, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 1, + "byteLogLikelihood": -4.71573455878, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 38, + "byteLogLikelihood": -2.700458868498, + "hits": 26, + "total": 202, + "hitRate": 0.12871287128712872 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-3", + "language": "mt" + }, + "predicted": { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 52, + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 52, + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 14, + "byteLogLikelihood": -4.81334289805, + "hits": 9, + "total": 181, + "hitRate": 0.049723756906077346 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 14, + "byteLogLikelihood": -4.81334289805, + "hits": 9, + "total": 181, + "hitRate": 0.049723756906077346 + }, + { + "encoding": "windows-1250", + "language": "pl", + "confidence": 13, + "byteLogLikelihood": -4.980958307454, + "hits": 8, + "total": 182, + "hitRate": 0.04395604395604396 + }, + { + "encoding": "windows-1258", + "language": "vi", + "confidence": 13, + "byteLogLikelihood": -4.506236351586, + "hits": 8, + "total": 181, + "hitRate": 0.04419889502762431 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 11, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 11, + "byteLogLikelihood": -4.945351018208, + "hits": 7, + "total": 183, + "hitRate": 0.03825136612021858 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 11, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 + }, + { + "encoding": "CP850", + "language": "es", + "confidence": 11, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 177, + "hitRate": 0.03954802259887006 + }, + { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 9, + "byteLogLikelihood": -3.834078315244, + "hits": 6, + "total": 183, + "hitRate": 0.03278688524590164 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 182, + "hitRate": 0.03296703296703297 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 6, + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-4", + "language": "lv" + }, + "predicted": { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 40, + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 40, + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 31, + "byteLogLikelihood": -5.111410123766, + "hits": 18, + "total": 169, + "hitRate": 0.10650887573964497 + }, + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 14, + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 14, + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 14, + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 10, + "byteLogLikelihood": -4.596975900219, + "hits": 6, + "total": 169, + "hitRate": 0.03550295857988166 + }, + { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 8, + "byteLogLikelihood": -4.763992911083, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "windows-1250", + "language": "pl", + "confidence": 8, + "byteLogLikelihood": -4.846752537455, + "hits": 5, + "total": 169, + "hitRate": 0.029585798816568046 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.704365179493, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -4.578946937177, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.65535318117, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 0, + "byteLogLikelihood": -4.616838329205, + "hits": 0, + "total": 169, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-5", + "language": "ru" + }, + "predicted": { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + { + "encoding": "IBM855", + "language": "ru", + "confidence": 9, + "byteLogLikelihood": -4.826799236728, + "hits": 5, + "total": 166, + "hitRate": 0.030120481927710843 + }, + { + "encoding": "KOI8-U", + "language": "uk", + "confidence": 1, + "byteLogLikelihood": -4.816522715875, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 0, + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 0, + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 0, + "byteLogLikelihood": -4.847671496604, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "language": "vi", + "confidence": 0, + "byteLogLikelihood": -4.755485363394, + "hits": 0, + "total": 166, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -4.810914106722, + "hits": 0, + "total": 84, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 0, + "byteLogLikelihood": -4.847671496604, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 0, + "byteLogLikelihood": -4.83002806951, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.848242748706, + "hits": 0, + "total": 177, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-6", + "language": "ar" + }, + "predicted": { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 10, + "byteLogLikelihood": -3.968382111451, + "hits": 4, + "total": 110, + "hitRate": 0.03636363636363636 + }, + { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.69584393439, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 0, + "byteLogLikelihood": -4.933475282212, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.69584393439, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 0, + "byteLogLikelihood": -4.933475282212, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 0, + "byteLogLikelihood": -4.72024182268, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -4.643135408712, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 55, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 0, + "byteLogLikelihood": -4.799641758471, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 0, + "byteLogLikelihood": -4.755317425962, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "CP850", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 98, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-7", + "language": "el" + }, + "predicted": { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 2, + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 2, + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 2, + "byteLogLikelihood": -4.203232324163, + "hits": 1, + "total": 131, + "hitRate": 0.007633587786259542 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 0, + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.288697198304, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 0, + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 0, + "byteLogLikelihood": -4.576407109739, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.603974037796, + "hits": 0, + "total": 134, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-8", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.199203901043, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 0, + "byteLogLikelihood": -4.639272141503, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.134202940863, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 0, + "byteLogLikelihood": -4.521941300627, + "hits": 0, + "total": 102, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-9", + "language": "tr" + }, + "predicted": { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "windows-1252", + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 11, + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 11, + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.22610845571, + "hits": 0, + "total": 134, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-R", + "language": "ru" + }, + "predicted": { + "encoding": "KOI8-R", + "language": "ru", + "confidence": 43, + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-R", + "language": "ru", + "confidence": 43, + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + { + "encoding": "KOI8-U", + "language": "uk", + "confidence": 28, + "byteLogLikelihood": -3.443215105753, + "hits": 17, + "total": 179, + "hitRate": 0.09497206703910614 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 12, + "byteLogLikelihood": -4.705552828452, + "hits": 7, + "total": 162, + "hitRate": 0.043209876543209874 + }, + { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 0, + "byteLogLikelihood": -4.900684399759, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.759099059367, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "nl", + "confidence": 0, + "byteLogLikelihood": -4.900684399759, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.796335743252, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 0, + "byteLogLikelihood": -4.91533239831, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 0, + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 0, + "byteLogLikelihood": -4.935090625757, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "CP850", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-U", + "language": "uk" + }, + "predicted": { + "encoding": "KOI8-U", + "language": "uk", + "confidence": 32, + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-U", + "language": "uk", + "confidence": 32, + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 + }, + { + "encoding": "KOI8-R", + "language": "ru", + "confidence": 17, + "byteLogLikelihood": -3.510012695801, + "hits": 10, + "total": 172, + "hitRate": 0.05813953488372093 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 11, + "byteLogLikelihood": -4.843281209678, + "hits": 6, + "total": 163, + "hitRate": 0.03680981595092025 + }, + { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 1, + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 1, + "byteLogLikelihood": -4.726706027838, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "windows-1252", + "language": "en", + "confidence": 1, + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 1, + "byteLogLikelihood": -4.693044688745, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 0, + "byteLogLikelihood": -4.922686041886, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 89, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 0, + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.944199711559, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "CP850", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "de" + }, + "predicted": { + "encoding": "macintosh", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "language": "de", + "confidence": 70, + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 74, + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 74, + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "CP850", + "language": "de", + "confidence": 69, + "byteLogLikelihood": -5.003946305945, + "hits": 50, + "total": 216, + "hitRate": 0.23148148148148148 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 22, + "byteLogLikelihood": -4.912654885736, + "hits": 16, + "total": 214, + "hitRate": 0.07476635514018691 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 214, + "hitRate": 0.0514018691588785 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 216, + "hitRate": 0.05092592592592592 + }, + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 14, + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 214, + "hitRate": 0.04672897196261682 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -5.075173815234, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 11, + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 214, + "hitRate": 0.037383177570093455 + }, + { + "encoding": "CP852", + "language": "pl", + "confidence": 5, + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "es" + }, + "predicted": { + "encoding": "macintosh", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "language": "es", + "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 25, + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 16, + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 199, + "hitRate": 0.05527638190954774 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 7, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 7, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "CP852", + "language": "pl", + "confidence": 7, + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "fr" + }, + "predicted": { + "encoding": "macintosh", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "language": "fr", + "confidence": 71, + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 75, + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 75, + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "CP850", + "language": "fr", + "confidence": 71, + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 12, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 7, + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 6, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 6, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 4, + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 224, + "hitRate": 0.013392857142857142 + }, + { + "encoding": "IBM855", + "language": "sr", + "confidence": 0, + "byteLogLikelihood": -3.576182148392, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 7, + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.249199684222, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.226561011787, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 0, + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 4, + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 2, + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 2, + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "pl" + }, + "predicted": { + "encoding": "windows-1250", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1250", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 37, + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 184, + "hitRate": 0.125 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 6, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 6, + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 4, + "byteLogLikelihood": -4.826691858304, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 4, + "byteLogLikelihood": -4.704684722593, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 3, + "byteLogLikelihood": -4.71573455878, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.848389618517, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.848389618517, + "hits": 0, + "total": 184, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 38, + "byteLogLikelihood": -2.700458868498, + "hits": 26, + "total": 202, + "hitRate": 0.12871287128712872 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1251", + "language": "ru" + }, + "predicted": { + "encoding": "windows-1251", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 34, + "byteLogLikelihood": -3.411211853364, + "hits": 20, + "total": 175, + "hitRate": 0.11428571428571428 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.724449359889, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 0, + "byteLogLikelihood": -4.61521723683, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 0, + "byteLogLikelihood": -4.66408545306, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 0, + "byteLogLikelihood": -4.760478683076, + "hits": 0, + "total": 179, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "da" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + { + "encoding": "windows-1252", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 10, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 2, + "byteLogLikelihood": -4.658005606594, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 2, + "byteLogLikelihood": -4.145958228046, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "en" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 29, + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + { + "encoding": "windows-1252", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1250", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 131, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 0, + "byteLogLikelihood": -4.510859506517, + "hits": 0, + "total": 132, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 10, + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "it" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1252", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1258", + "language": "vi", + "confidence": 16, + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 15, + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 15, + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 3, + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 63, + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 172, - "hitRate": 0.01744186046511628 + "encoding": "windows-1252", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.6272413408, - "hits": 2, - "total": 173, - "hitRate": 0.011560693641618497 + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 }, { - "encoding": "IBM855", - "confidence": 1, - "language": "sr", - "byteLogLikelihood": -5.488073838002, - "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 }, { "encoding": "ISO-8859-10", - "confidence": 1, "language": "is", - "byteLogLikelihood": -4.416382325607, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "confidence": 11, + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.659155714377, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.252539473952, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 173, + "total": 151, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.893109334109, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 171, + "total": 151, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "language": "th", "confidence": 0, + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", "language": "uk", - "byteLogLikelihood": -3.441328843196, + "confidence": 0, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 173, + "total": 151, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-14", - "language": "cy" + "encoding": "windows-1252", + "language": "no" }, "predicted": { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 + "encoding": "ISO-8859-1", + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 - }, - { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 - }, - { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "fr", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 189, - "hitRate": 0.037037037037037035 + "encoding": "ISO-8859-1", + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 + "encoding": "windows-1252", + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 7, + "byteLogLikelihood": -4.354336342997, "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 + "total": 114, + "hitRate": 0.02631578947368421 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -4.009500374258, + "encoding": "windows-1257", + "language": "et", + "confidence": 7, + "byteLogLikelihood": -4.934473933131, "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "CP852", - "confidence": 3, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 188, - "hitRate": 0.010638297872340425 + "total": 114, + "hitRate": 0.02631578947368421 }, { - "encoding": "ISO-8859-4", - "confidence": 1, - "language": "lv", - "byteLogLikelihood": -4.961907790704, + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 2, + "byteLogLikelihood": -4.210065739287, "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 + "total": 114, + "hitRate": 0.008771929824561403 }, { "encoding": "ISO-8859-13", - "confidence": 1, "language": "lt", - "byteLogLikelihood": -3.647025046455, + "confidence": 2, + "byteLogLikelihood": -4.319245023186, "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 + "total": 114, + "hitRate": 0.008771929824561403 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.821976831991, + "byteLogLikelihood": -4.154384602589, "hits": 0, - "total": 189, + "total": 114, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.350299845595, + "confidence": 0, + "byteLogLikelihood": -3.753215851617, "hits": 0, - "total": 188, + "total": 114, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.325879646284, + "byteLogLikelihood": -4.154384602589, "hits": 0, - "total": 188, + "total": 114, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.255815902698, + "byteLogLikelihood": -2.647504477241, "hits": 0, - "total": 189, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -3.981643419755, + "byteLogLikelihood": -3.753215851617, "hits": 0, - "total": 189, + "total": 114, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", - "language": "de" + "encoding": "windows-1252", + "language": "pt" }, "predicted": { - "encoding": "ISO-8859-15", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-15", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - { - "encoding": "macintosh", - "confidence": 72, - "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 52, - "total": 215, - "hitRate": 0.24186046511627907 - }, - { - "encoding": "CP850", - "confidence": 70, - "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 51, - "total": 216, - "hitRate": 0.2361111111111111 - }, - { - "encoding": "ISO-8859-14", - "confidence": 20, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 15, - "total": 216, - "hitRate": 0.06944444444444445 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 }, { - "encoding": "ISO-8859-16", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 + "encoding": "windows-1252", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 }, { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 }, { - "encoding": "CP852", - "confidence": 6, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "encoding": "windows-1250", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, "hits": 5, - "total": 216, - "hitRate": 0.023148148148148147 + "total": 143, + "hitRate": 0.03496503496503497 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 216, - "hitRate": 0.009259259259259259 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 1, - "total": 216, - "hitRate": 0.004629629629629629 + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.846220145127, + "hits": 0, + "total": 143, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.245943193261, + "byteLogLikelihood": -3.017269736716, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -3.50352863905, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1256", + "language": "ar", "confidence": 0, + "byteLogLikelihood": -3.503504267293, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM855", "language": "sr", - "byteLogLikelihood": -5.942998477092, + "confidence": 0, + "byteLogLikelihood": -3.774998078834, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", - "byteLogLikelihood": -3.629332579687, + "confidence": 0, + "byteLogLikelihood": -3.766179593299, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", - "language": "es" + "encoding": "windows-1252", + "language": "sv" }, "predicted": { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "macintosh", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 + "encoding": "windows-1252", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, { - "encoding": "ISO-8859-4", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.009854802779, - "hits": 16, - "total": 200, - "hitRate": 0.08 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 17, + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 }, { - "encoding": "ISO-8859-13", - "confidence": 19, + "encoding": "windows-1257", "language": "lt", - "byteLogLikelihood": -4.892914919217, - "hits": 13, - "total": 200, - "hitRate": 0.065 - }, - { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 200, - "hitRate": 0.055 + "confidence": 4, + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 4, + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 0, + "byteLogLikelihood": -3.287368473007, + "hits": 0, + "total": 136, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -3.846028832089, - "hits": 3, - "total": 200, - "hitRate": 0.015 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -3.066153183614, "hits": 0, - "total": 200, + "total": 136, "hitRate": 0 }, { - "encoding": "IBM866", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -4.977292083619, + "confidence": 0, + "byteLogLikelihood": -3.393743210995, "hits": 0, - "total": 200, + "total": 136, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.203740544852, + "byteLogLikelihood": -3.876181596107, "hits": 0, - "total": 200, + "total": 136, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, + "byteLogLikelihood": -3.066153183614, "hits": 0, - "total": 200, + "total": 136, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "byteLogLikelihood": -3.393743210995, "hits": 0, - "total": 200, + "total": 136, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", - "language": "fr" + "encoding": "windows-1253", + "language": "el" }, "predicted": { - "encoding": "ISO-8859-15", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-15", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "macintosh", - "confidence": 71, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "CP850", - "confidence": 71, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 + "encoding": "windows-1253", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, { - "encoding": "ISO-8859-10", - "confidence": 10, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 2, + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 }, { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "windows-1255", + "language": "he", + "confidence": 2, + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 2, + "byteLogLikelihood": -4.203232324163, + "hits": 1, + "total": 131, + "hitRate": 0.007633587786259542 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 0, + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 228, - "hitRate": 0.013157894736842105 + "encoding": "windows-1250", + "language": "hu", + "confidence": 0, + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "byteLogLikelihood": -4.288697198304, "hits": 0, - "total": 228, + "total": 134, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1252", + "language": "pt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, + "byteLogLikelihood": -4.59932343766, "hits": 0, - "total": 228, + "total": 134, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, + "byteLogLikelihood": -4.576407109739, "hits": 0, - "total": 228, + "total": 134, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, + "byteLogLikelihood": -4.603974037796, "hits": 0, - "total": 228, + "total": 134, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-16", - "language": "ro" + "encoding": "windows-1254", + "language": "tr" }, "predicted": { - "encoding": "ISO-8859-16", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-16", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 }, { - "encoding": "macintosh", - "confidence": 19, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 198, - "hitRate": 0.06565656565656566 + "encoding": "windows-1254", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 }, { - "encoding": "CP850", - "confidence": 15, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 }, { - "encoding": "ISO-8859-15", - "confidence": 14, - "language": "fr", - "byteLogLikelihood": -4.903997266318, - "hits": 10, - "total": 202, - "hitRate": 0.04950495049504951 + "encoding": "windows-1252", + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", + "language": "ro", "confidence": 13, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 9, - "total": 200, - "hitRate": 0.045 + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 }, { - "encoding": "CP852", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, "hits": 6, - "total": 201, - "hitRate": 0.029850746268656716 + "total": 134, + "hitRate": 0.04477611940298507 }, { - "encoding": "ISO-8859-4", - "confidence": 7, + "encoding": "windows-1257", "language": "lv", - "byteLogLikelihood": -4.936633427031, + "confidence": 11, + "byteLogLikelihood": -3.613497993373, "hits": 5, - "total": 202, - "hitRate": 0.024752475247524754 - }, - { - "encoding": "ISO-8859-10", - "confidence": 5, - "language": "is", - "byteLogLikelihood": -5.150687673188, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 + "total": 134, + "hitRate": 0.03731343283582089 }, { "encoding": "ISO-8859-13", - "confidence": 4, "language": "lt", - "byteLogLikelihood": -4.974278583322, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.539421788511, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "confidence": 11, + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.32130473174, - "hits": 0, - "total": 201, - "hitRate": 0 + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.208609169552, + "byteLogLikelihood": -4.216966174066, "hits": 0, - "total": 201, + "total": 134, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.608648280371, + "byteLogLikelihood": -4.216966174066, "hits": 0, - "total": 201, + "total": 134, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, + "byteLogLikelihood": -4.22610845571, "hits": 0, - "total": 202, + "total": 134, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-3", - "language": "mt" + "encoding": "windows-1255", + "language": "he" }, "predicted": { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 - }, - { - "encoding": "CP852", - "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 8, - "total": 177, - "hitRate": 0.04519774011299435 - }, - { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 }, { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -4.945351018208, - "hits": 7, - "total": 183, - "hitRate": 0.03825136612021858 + "encoding": "windows-1255", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 }, { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 }, { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 177, - "hitRate": 0.03954802259887006 + "encoding": "windows-1253", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 }, { - "encoding": "ISO-8859-16", - "confidence": 9, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 6, - "total": 182, - "hitRate": 0.03296703296703297 + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 6, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.986136327924, - "hits": 3, - "total": 182, - "hitRate": 0.016483516483516484 + "encoding": "windows-1250", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 183, - "hitRate": 0.01092896174863388 + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.199203901043, + "hits": 0, + "total": 102, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1252", + "language": "sv", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.383683979141, + "byteLogLikelihood": -4.395116706452, "hits": 0, - "total": 181, + "total": 102, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.404570694946, + "byteLogLikelihood": -4.639272141503, "hits": 0, - "total": 181, + "total": 102, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.943656762101, + "byteLogLikelihood": -4.134202940863, "hits": 0, - "total": 181, + "total": 102, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-15", + "language": "fr", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.572384450833, + "byteLogLikelihood": -4.521941300627, "hits": 0, - "total": 182, + "total": 102, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-4", - "language": "lv" + "encoding": "windows-1256", + "language": "ar" }, "predicted": { - "encoding": "ISO-8859-4", - "confidence": 40, - "language": "lv", - "byteLogLikelihood": -3.1328616083, + "encoding": "windows-1256", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, "hits": 23, - "total": 170, - "hitRate": 0.13529411764705881 + "total": 110, + "hitRate": 0.20909090909090908 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-4", - "confidence": 40, - "language": "lv", - "byteLogLikelihood": -3.1328616083, + "encoding": "windows-1256", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, "hits": 23, - "total": 170, - "hitRate": 0.13529411764705881 + "total": 110, + "hitRate": 0.20909090909090908 }, { - "encoding": "CP850", - "confidence": 16, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 9, - "total": 168, - "hitRate": 0.05357142857142857 - }, - { - "encoding": "macintosh", - "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 15, + "byteLogLikelihood": -3.973986161297, + "hits": 5, + "total": 99, + "hitRate": 0.050505050505050504 }, { - "encoding": "ISO-8859-15", - "confidence": 14, + "encoding": "ISO-8859-1", "language": "es", - "byteLogLikelihood": -4.721042183854, - "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 - }, - { - "encoding": "CP852", - "confidence": 12, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 + "confidence": 0, + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 10, - "language": "lt", - "byteLogLikelihood": -4.596975900219, - "hits": 6, - "total": 169, - "hitRate": 0.03550295857988166 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 }, { - "encoding": "ISO-8859-3", - "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 5, - "total": 170, - "hitRate": 0.029411764705882353 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 0, + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 7, - "language": "is", - "byteLogLikelihood": -5.074758182528, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 + "encoding": "windows-1250", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.572632236701, - "hits": 1, - "total": 169, - "hitRate": 0.005917159763313609 + "encoding": "windows-1252", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1254", + "language": "tr", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.609505685025, + "byteLogLikelihood": -4.906860727803, "hits": 0, - "total": 169, + "total": 109, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.267508614334, + "byteLogLikelihood": -4.856571347512, "hits": 0, - "total": 168, + "total": 110, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.182018138172, + "byteLogLikelihood": -4.736714293626, "hits": 0, - "total": 169, + "total": 90, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.65535318117, + "byteLogLikelihood": -4.796168589227, "hits": 0, - "total": 170, + "total": 111, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "byteLogLikelihood": -4.70358763711, "hits": 0, - "total": 170, + "total": 110, "hitRate": 0 } ] }, { "expected": { - "encoding": "KOI8-U", - "language": "uk" + "encoding": "windows-1257", + "language": "et" }, "predicted": { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 + "encoding": "windows-1257", + "language": "et", + "confidence": 40, + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 + "encoding": "windows-1257", + "language": "et", + "confidence": 40, + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 }, { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.128376292359, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -3.976561526566, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -3.976561526566, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.582632026409, + "byteLogLikelihood": -4.30822322351, "hits": 0, - "total": 22, + "total": 120, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.922515418323, + "byteLogLikelihood": -4.41969872812, "hits": 0, - "total": 94, + "total": 120, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -4.51563851542, "hits": 0, - "total": 89, + "total": 120, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.589678128375, + "byteLogLikelihood": -4.453415335268, "hits": 0, - "total": 117, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "byteLogLikelihood": -4.41969872812, "hits": 0, - "total": 175, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "byteLogLikelihood": -3.632538332325, "hits": 0, - "total": 176, + "total": 120, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1257", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 }, { "encoding": "ISO-8859-13", - "confidence": 0, "language": "lt", - "byteLogLikelihood": -5.110283442945, + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 22, + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 1, + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.472500251122, "hits": 0, - "total": 168, + "total": 173, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "byteLogLikelihood": -4.444220754634, "hits": 0, - "total": 177, + "total": 171, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944199711559, + "byteLogLikelihood": -3.531827362419, "hits": 0, - "total": 176, + "total": 173, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.148788659657, + "byteLogLikelihood": -4.472500251122, "hits": 0, - "total": 177, + "total": 173, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -4.444220754634, "hits": 0, - "total": 92, + "total": 171, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "byteLogLikelihood": -3.441328843196, "hits": 0, - "total": 94, + "total": 173, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "de" + "encoding": "windows-1257", + "language": "lv" }, "predicted": { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "windows-1257", + "language": "lv", + "confidence": 40, + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - { - "encoding": "ISO-8859-15", - "confidence": 74, - "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 53, - "total": 214, - "hitRate": 0.24766355140186916 + "encoding": "windows-1257", + "language": "lv", + "confidence": 40, + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 }, { - "encoding": "CP850", - "confidence": 69, + "encoding": "ISO-8859-1", "language": "fr", - "byteLogLikelihood": -5.003946305945, - "hits": 50, - "total": 216, - "hitRate": 0.23148148148148148 - }, - { - "encoding": "ISO-8859-14", - "confidence": 22, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 16, - "total": 214, - "hitRate": 0.07476635514018691 + "confidence": 14, + "byteLogLikelihood": -4.806768476285, + "hits": 8, + "total": 170, + "hitRate": 0.047058823529411764 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 11, - "total": 214, - "hitRate": 0.0514018691588785 + "encoding": "windows-1258", + "language": "vi", + "confidence": 7, + "byteLogLikelihood": -4.342387954294, + "hits": 4, + "total": 169, + "hitRate": 0.023668639053254437 }, { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 214, - "hitRate": 0.037383177570093455 + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -4.459882683974, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.703260469986, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -4.207917523838, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 214, - "hitRate": 0.014018691588785047 + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -3.543665829263, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 214, - "hitRate": 0.009345794392523364 + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.703260469986, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.534754518494, + "byteLogLikelihood": -4.207917523838, "hits": 0, - "total": 215, + "total": 170, "hitRate": 0 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.345510152227, + "confidence": 0, + "byteLogLikelihood": -4.459882683974, "hits": 0, - "total": 216, + "total": 170, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.337519079447, + "byteLogLikelihood": -3.416476215238, "hits": 0, - "total": 216, + "total": 170, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "byteLogLikelihood": -4.423374522988, "hits": 0, - "total": 216, + "total": 170, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "es" + "encoding": "windows-1258", + "language": "vi" }, "predicted": { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "windows-1258", + "language": "vi", + "confidence": 98, + "byteLogLikelihood": -3.323414763285, + "hits": 42, + "total": 124, + "hitRate": 0.3387096774193548 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "windows-1258", + "language": "vi", + "confidence": 98, + "byteLogLikelihood": -3.323414763285, + "hits": 42, + "total": 124, + "hitRate": 0.3387096774193548 }, { - "encoding": "ISO-8859-15", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 9, + "byteLogLikelihood": -4.721506246424, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 + "encoding": "windows-1252", + "language": "pt", + "confidence": 9, + "byteLogLikelihood": -4.721506246424, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { - "encoding": "ISO-8859-4", - "confidence": 25, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 17, - "total": 199, - "hitRate": 0.08542713567839195 + "encoding": "windows-1257", + "language": "lt", + "confidence": 9, + "byteLogLikelihood": -4.767575136152, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { "encoding": "ISO-8859-13", - "confidence": 19, "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 13, - "total": 199, - "hitRate": 0.06532663316582915 + "confidence": 9, + "byteLogLikelihood": -4.767575136152, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { "encoding": "ISO-8859-14", - "confidence": 16, "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 199, - "hitRate": 0.05527638190954774 - }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 + "confidence": 9, + "byteLogLikelihood": -4.643097648852, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { - "encoding": "CP852", + "encoding": "ISO-8859-9", + "language": "tr", "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, + "byteLogLikelihood": -4.76855606835, "hits": 3, - "total": 199, - "hitRate": 0.01507537688442211 + "total": 126, + "hitRate": 0.023809523809523808 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 199, - "hitRate": 0 + "encoding": "windows-1254", + "language": "tr", + "confidence": 7, + "byteLogLikelihood": -4.76855606835, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -4.428592570712, "hits": 0, - "total": 200, + "total": 125, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.624741392945, + "byteLogLikelihood": -4.428592570712, "hits": 0, - "total": 200, + "total": 125, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -4.782036156761, "hits": 0, - "total": 200, + "total": 125, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "byteLogLikelihood": -4.794574996468, "hits": 0, - "total": 199, + "total": 126, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "fr" + "encoding": "windows-874", + "language": "th" }, "predicted": { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "encoding": "windows-874", + "language": "th", + "confidence": 21, + "byteLogLikelihood": -3.827006747769, + "hits": 7, + "total": 100, + "hitRate": 0.07 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "ISO-8859-15", - "confidence": 75, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 56, - "total": 224, - "hitRate": 0.25 - }, - { - "encoding": "CP850", - "confidence": 71, - "language": "de", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "encoding": "windows-874", + "language": "th", + "confidence": 21, + "byteLogLikelihood": -3.827006747769, + "hits": 7, + "total": 100, + "hitRate": 0.07 }, { - "encoding": "ISO-8859-10", - "confidence": 12, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 0, + "byteLogLikelihood": -4.767489617961, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.903917269007, + "hits": 0, + "total": 103, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 224, - "hitRate": 0.03125 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 0, + "byteLogLikelihood": -4.985031025935, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-3", - "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 224, - "hitRate": 0.026785714285714284 + "encoding": "windows-1250", + "language": "pl", + "confidence": 0, + "byteLogLikelihood": -4.798656950405, + "hits": 0, + "total": 101, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "windows-1252", + "language": "it", + "confidence": 0, + "byteLogLikelihood": -4.767489617961, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 224, - "hitRate": 0.017857142857142856 + "encoding": "windows-1257", + "language": "et", + "confidence": 0, + "byteLogLikelihood": -4.916002186025, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 224, - "hitRate": 0.013392857142857142 + "encoding": "macintosh", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 65, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-4", + "language": "lv", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "byteLogLikelihood": -4.783724339715, "hits": 0, - "total": 224, + "total": 103, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, + "byteLogLikelihood": -4.912654885736, "hits": 0, - "total": 228, + "total": 103, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-15", + "language": "fr", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -3.576182148392, + "byteLogLikelihood": -4.801693311206, "hits": 0, - "total": 228, + "total": 101, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, + "byteLogLikelihood": -4.887413059939, "hits": 0, - "total": 228, + "total": 85, "hitRate": 0 } ] @@ -2977,102 +8434,113 @@ }, "predicted": { "encoding": "x-mac-cyrillic", - "confidence": 36, "language": "ru", + "confidence": 36, "byteLogLikelihood": -3.313760777855, "hits": 22, "total": 179, "hitRate": 0.12290502793296089 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "x-mac-cyrillic", - "confidence": 36, "language": "ru", + "confidence": 36, "byteLogLikelihood": -3.313760777855, "hits": 22, "total": 179, "hitRate": 0.12290502793296089 }, { - "encoding": "IBM855", - "confidence": 9, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.603105135042, - "hits": 5, - "total": 165, - "hitRate": 0.030303030303030304 + "confidence": 42, + "byteLogLikelihood": -3.411211853364, + "hits": 25, + "total": 177, + "hitRate": 0.14124293785310735 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 3, - "language": "uk", - "byteLogLikelihood": -6.334488595021, + "byteLogLikelihood": -3.941595102268, "hits": 2, - "total": 177, - "hitRate": 0.011299435028248588 + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 }, { "encoding": "ISO-8859-14", - "confidence": 1, "language": "cy", + "confidence": 1, "byteLogLikelihood": -4.724449359889, "hits": 1, "total": 177, "hitRate": 0.005649717514124294 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-1", + "language": "sv", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.05026366488, + "byteLogLikelihood": -4.665227236998, "hits": 0, - "total": 169, + "total": 177, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944169511699, + "byteLogLikelihood": -4.148248171081, "hits": 0, - "total": 151, + "total": 177, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1252", + "language": "sv", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.904224209153, + "byteLogLikelihood": -4.665227236998, "hits": 0, - "total": 175, + "total": 177, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.79659105312, + "byteLogLikelihood": -4.148248171081, "hits": 0, "total": 177, "hitRate": 0 }, { - "encoding": "ISO-8859-10", + "encoding": "windows-1257", + "language": "lv", "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.939249573759, + "byteLogLikelihood": -4.61521723683, "hits": 0, "total": 177, "hitRate": 0 }, { "encoding": "ISO-8859-13", - "confidence": 0, "language": "lt", + "confidence": 0, "byteLogLikelihood": -4.66408545306, "hits": 0, "total": 177, @@ -3080,39 +8548,12 @@ }, { "encoding": "ISO-8859-15", - "confidence": 0, "language": "fr", - "byteLogLikelihood": -4.760478683076, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.935887060165, + "byteLogLikelihood": -4.760478683076, "hits": 0, "total": 177, "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 137, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.083005956553, - "hits": 0, - "total": 156, - "hitRate": 0 } ] }, @@ -3123,144 +8564,427 @@ }, "predicted": { "encoding": "x-mac-cyrillic", - "confidence": 32, "language": "uk", + "confidence": 32, "byteLogLikelihood": -3.34369756932, "hits": 19, "total": 177, "hitRate": 0.10734463276836158 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "x-mac-cyrillic", - "confidence": 32, "language": "uk", + "confidence": 32, "byteLogLikelihood": -3.34369756932, "hits": 19, "total": 177, "hitRate": 0.10734463276836158 }, { - "encoding": "IBM855", - "confidence": 5, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.830774318921, - "hits": 3, - "total": 163, - "hitRate": 0.018404907975460124 - }, - { - "encoding": "KOI8-U", - "confidence": 1, - "language": "uk", - "byteLogLikelihood": -6.341901231006, - "hits": 1, - "total": 175, - "hitRate": 0.005714285714285714 + "confidence": 17, + "byteLogLikelihood": -3.606822645813, + "hits": 10, + "total": 176, + "hitRate": 0.056818181818181816 }, { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.100977296295, - "hits": 0, - "total": 167, - "hitRate": 0 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -4.062591421695, + "hits": 2, + "total": 165, + "hitRate": 0.012121212121212121 }, { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 139, - "hitRate": 0 + "encoding": "windows-1255", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -4.062591421695, + "hits": 2, + "total": 165, + "hitRate": 0.012121212121212121 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-1", + "language": "it", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.035218126431, + "byteLogLikelihood": -4.629500955096, "hits": 0, - "total": 164, + "total": 169, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.726279355538, + "byteLogLikelihood": -4.300043808736, "hits": 0, "total": 169, "hitRate": 0 }, { - "encoding": "ISO-8859-10", + "encoding": "windows-1252", + "language": "it", "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.876716873437, + "byteLogLikelihood": -4.629500955096, "hits": 0, - "total": 175, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.669220995205, + "byteLogLikelihood": -4.300043808736, "hits": 0, "total": 169, "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.725687554138, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", + "encoding": "windows-1257", + "language": "lv", "confidence": 0, - "language": "fr", - "byteLogLikelihood": -4.753004092615, + "byteLogLikelihood": -4.60874799529, "hits": 0, - "total": 175, + "total": 169, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-4", + "language": "lv", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.986714390957, + "byteLogLikelihood": -4.726279355538, "hits": 0, - "total": 175, + "total": 169, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -4.669220995205, "hits": 0, - "total": 137, + "total": 169, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "byteLogLikelihood": -4.725687554138, "hits": 0, - "total": 152, + "total": 175, "hitRate": 0 } ] } - ] + ], + "multibyte": { + "summary": { + "tests": 5, + "encodingCorrect": 5, + "languageCorrect": 5 + }, + "results": [ + { + "expected": { + "encoding": "Big5", + "language": "zh" + }, + "predicted": { + "encoding": "Big5", + "language": "zh", + "confidence": 0.31, + "hits": 16, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.31, + "hits": 16, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 27, + "invalidCharacters": 25 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 1, + "multibyteCharacters": 27, + "invalidCharacters": 25 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 13, + "invalidCharacters": 1 + } + ] + }, + { + "expected": { + "encoding": "EUC-JP", + "language": "ja" + }, + "predicted": { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.1, + "hits": 6, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.03, + "hits": 2, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 21, + "invalidCharacters": 1 + } + ] + }, + { + "expected": { + "encoding": "EUC-KR", + "language": "ko" + }, + "predicted": { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.59, + "hits": 41, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.59, + "hits": 41, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.03, + "hits": 2, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.01, + "hits": 1, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.01, + "hits": 1, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 12, + "invalidCharacters": 7 + } + ] + }, + { + "expected": { + "encoding": "GB18030", + "language": "zh" + }, + "predicted": { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.33, + "hits": 17, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.33, + "hits": 17, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.02, + "hits": 1, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 4, + "multibyteCharacters": 40, + "invalidCharacters": 7 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 1, + "multibyteCharacters": 40, + "invalidCharacters": 7 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 14, + "invalidCharacters": 0 + } + ] + }, + { + "expected": { + "encoding": "Shift_JIS", + "language": "ja" + }, + "predicted": { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 3, + "invalidCharacters": 48 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 3, + "invalidCharacters": 48 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + } + ] + } + ] + } } diff --git a/corpus/sources.json b/corpus/sources.json index 3b05948..83a82ae 100644 --- a/corpus/sources.json +++ b/corpus/sources.json @@ -505,6 +505,654 @@ "text": "W niedziel przyjaciele poszli nad spokojn rzek za miastem. Zabrali map, ciep herbat i aparat. Po dugim spacerze odpoczywali na brzegu." } ] + }, + { + "code": "ar", + "name": "Arabic", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "媯堥塈堶 塈堹堜", + "text": "堛堻堛婺 塈堹堜 堥堹堙 堛堛堶 塈塈 堧堥塈堥塈 堥塈 堛婺堭 塈塈堻 塈堶塈塈堛 堭堣 塈堧堮堥塈堭." + }, + { + "id": "library", + "split": "train", + "title": "塈堛堥堜 塈塈堹埵堜", + "text": "堥堶堳 塈堭塈堙 媢 堛堥 塈堛塈堭堮 塈媢 堻塈媢堹 堧 塈堛堥堜 塈媟塈堥 塈堮堛塈堭 塈堭塈堿媢 塈塈堻堥堜." + }, + { + "id": "market", + "split": "train", + "title": "堻 塈堻堥堛", + "text": "媢堭媔 塈堬塈堭媢 塈堮堥堬 塈媢堻 塈塈堜 堛堥塈堹 塈堬塈堭 塈媯塈堛 塈堧堶塈堹堳 塈堹堜." + }, + { + "id": "workshop", + "split": "train", + "title": "堭奡堜 堭堜", + "text": "媯 堭 媯媞堭 堹塈 堛堶 堭塈堿媢 塈堥堭堿 塈奡堭堜 堛堥 塈堶堭堭 媯塈 塈媔堶塈." + }, + { + "id": "river-trip", + "split": "validation", + "title": "堭堶堜 堨 塈堭", + "text": "堶 塈堧媯堹塈堙 堮堭媟堜 奡塈塈 堻塈堮塈 奡塈 堶堛 塈堭 堳 塈堻堛媢塈 堨 媟堭 塈堻塈堙." + } + ] + }, + { + "code": "cs", + "name": "Czech", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "R獺no ve mst", + "text": "Msto se pomalu probouz穩, kav獺rny otev穩raj穩 dvee a lid矇 ekaj穩 na prvn穩 autobusy." + }, + { + "id": "library", + "split": "train", + "title": "Tich獺 knihovna", + "text": "ten獺i hledaj穩 knihy o historii a vd, zat穩mco knihovnice rad穩 鱉獺k轡m." + }, + { + "id": "market", + "split": "train", + "title": "Sobotn穩 trh", + "text": "Farm獺i prod獺vaj穩 erstv羸 chl矇b, med, ovoce a voav矇 byliny." + }, + { + "id": "workshop", + "split": "train", + "title": "Digit獺ln穩 d穩lna", + "text": "Mal羸 t羸m pipravuje pr轡vodce muzeem, kontroluje k籀d a p穩禳e jasn矇 popisy." + }, + { + "id": "river-trip", + "split": "validation", + "title": "V羸let k ece", + "text": "P獺tel矇 vzali mapu a tepl羸 aj, pro禳li lesem a odpo穩vali u eky." + } + ] + }, + { + "code": "da", + "name": "Danish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgen i byen", + "text": "Byen v疇gner langsomt, caf矇erne 疇bner, og folk venter p疇 de f繪rste busser." + }, + { + "id": "library", + "split": "train", + "title": "Det stille bibliotek", + "text": "L疆sere finder b繪ger om historie og videnskab, mens bibliotekaren hj疆lper eleverne." + }, + { + "id": "market", + "split": "train", + "title": "L繪rdagsmarked", + "text": "B繪nder s疆lger frisk br繪d, 疆bler, ost og honning p疇 den travle plads." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitalt v疆rksted", + "text": "Et lille hold bygger en museumsguide, unders繪ger koden og skriver tydelige tekster." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Tur til floden", + "text": "Vennerne tog et kort og varm te med p疇 en s繪ndag og hvilede bagefter ved floden." + } + ] + }, + { + "code": "en", + "name": "English", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morning in the city", + "text": "The city wakes slowly; caf矇s open their doors and commuters read news beside a fa癟ade." + }, + { + "id": "library", + "split": "train", + "title": "The quiet library", + "text": "Readers browse books about history, science, and na簿ve art while the librarian prepares a r矇sum矇." + }, + { + "id": "market", + "split": "train", + "title": "Saturday market", + "text": "Farmers sell bread, apples, jalape簽os, and p璽t矇 while neighbours exchange recipes." + }, + { + "id": "workshop", + "split": "train", + "title": "Digital workshop", + "text": "A small team designs the museum guide and discusses its r繫le, code, and clear descriptions." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Journey to the river", + "text": "Friends carried a map and a thermos, walked past the caf矇, and rested by the river." + } + ] + }, + { + "code": "el", + "name": "Greek", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "巹 庢彖 弇庢", + "text": " 弇庢 徆彖峎 帢帠峎, 帢 庥帢峟 帢彖怷巹帠怷彖 庥帢庣 怷庣 峎彖庛怷庣 庰庣弮峟彖怷彖 帢 弇庰怷庰巹帢." + }, + { + "id": "library", + "split": "train", + "title": "庢 帣庣帣弇庣怷庛峸庥庢", + "text": "庣 帢彖帢帠彖庰 峎彖怷彖 帣庣帣弇巹帢 庣怷巹帢 庥帢庣 庰庣峸弮庢 庥帢庣 庢 帣庣帣弇庣怷庛庢庥峎庣怷 帣怷庢庛峎 怷 弮帢庛庢峟." + }, + { + "id": "market", + "split": "train", + "title": "峉帢帣帣帢庣峎庣庥庢 帢帠怷峎", + "text": "庣 帢帠庰 怷弇怷彖 弮巹, 弮峟弇庣, 怷帢 庥帢庣 弮帤庣庥峎 庢彖 弇帢庰巹帢." + }, + { + "id": "workshop", + "split": "train", + "title": "峔庢庣帢庥 庰帠帢峸庣怷", + "text": "庣帢 弮庣庥峸 怷弮峎帤帢 庰怷庣弮峎庤庰庣 怷帤庢帠 弮怷庰巹怷, 庰弇峟帠庰庣 怷彖 庥帤庣庥帢 庥帢庣 帠峎庰庣 庥帢庛帢峎 庥庰巹弮庰彖帢." + }, + { + "id": "river-trip", + "split": "validation", + "title": "庥帤怷弮峸 怷 怷峎弮庣", + "text": "庣 巹弇怷庣 峸帢彖 峎庢 庥帢庣 庤庰 峎庣 庥帢庣 徆庰庥怷峎庢庥帢彖 帤巹弇帢 怷 怷峎弮庣." + } + ] + }, + { + "code": "et", + "name": "Estonian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Hommik linnas", + "text": "Linn 瓣rkab aeglaselt, kohvikud avavad uksed ja inimesed ootavad esimesi busse." + }, + { + "id": "library", + "split": "train", + "title": "Vaikne raamatukogu", + "text": "Lugejad otsivad ajaloo- ja teadusraamatuid ning raamatukoguhoidja aitab 繭pilasi." + }, + { + "id": "market", + "split": "train", + "title": "Laup瓣evane turg", + "text": "Talunikud m羹羹vad v瓣rsket leiba, 繭unu, juustu ja mett rahvarohkel v瓣ljakul." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitaalne t繹繹tuba", + "text": "V瓣ike meeskond koostab muuseumijuhti, kontrollib koodi ja kirjutab selgeid kirjeldusi." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Retk j繭e 瓣瓣rde", + "text": "S繭brad v繭tsid kaardi ja sooja tee ning puhkasid p瓣rast jalutusk瓣iku j繭e 瓣瓣res." + } + ] + }, + { + "code": "he", + "name": "Hebrew", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "蚹蚳 蛁蚳", + "text": "蛁蚳 蛌蛁蚳蚳蛌 , 蛌 蚹蚺 蚺蛌 蚸 蛂 蚳蚸." + }, + { + "id": "library", + "split": "train", + "title": "蛂蚺蚳 蚸蚹", + "text": "蚹蚳 蚺蚸 蛂蚺蚳 蛂蚳 蛁 蛂蚺蚳蛌 蛁蚳蛌 蛌 蚳 蚹蚳蛌." + }, + { + "id": "market", + "split": "train", + "title": "蚸蚹 蚸蛌", + "text": "蚹 蚳 蚳, 蚸, 蚺蚳蛌 蚳 蛁蛂." + }, + { + "id": "workshop", + "split": "train", + "title": "蛂 蛌", + "text": "蛈蛌 蚹 蚳 , 蚹 蛌 蚹 蛌 蛌蚳 蚳蚳." + }, + { + "id": "river-trip", + "split": "validation", + "title": " 蚳", + "text": "蚳 蚹 蚺 蛌 , 蛁蚳 蚳." + } + ] + }, + { + "code": "hu", + "name": "Hungarian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Reggel a v獺rosban", + "text": "A v獺ros lassan 矇bred, a k獺v矇z籀k kinyitnak, az emberek pedig buszra v獺rnak." + }, + { + "id": "library", + "split": "train", + "title": "A csendes k繹nyvt獺r", + "text": "Az olvas籀k t繹rt矇nelmi 矇s tudom獺nyos k繹nyveket keresnek, a k繹nyvt獺ros seg穩t a di獺koknak." + }, + { + "id": "market", + "split": "train", + "title": "Szombati piac", + "text": "A gazd獺k friss kenyeret, alm獺t, sajtot 矇s m矇zet 獺rulnak a t矇ren." + }, + { + "id": "workshop", + "split": "train", + "title": "Digit獺lis m鑄hely", + "text": "Egy kis csapat m繳zeumi 繳tmutat籀t k矇sz穩t, k籀dot ellenriz 矇s vil獺gos le穩r獺sokat 穩r." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Kir獺ndul獺s a foly籀hoz", + "text": "A bar獺tok t矇rk矇pet 矇s meleg te獺t vittek, majd megpihentek a foly籀 partj獺n." + } + ] + }, + { + "code": "it", + "name": "Italian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Mattina in citt", + "text": "La citt si sveglia lentamente, i caff癡 aprono e la gente aspetta i primi autobus." + }, + { + "id": "library", + "split": "train", + "title": "La biblioteca tranquilla", + "text": "I lettori cercano pi羅 libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti pi羅 giovani." + }, + { + "id": "market", + "split": "train", + "title": "Mercato del sabato", + "text": "Gli agricoltori vendono pane fresco, caff癡, mele, formaggio e miele nella piazza della citt." + }, + { + "id": "workshop", + "split": "train", + "title": "Laboratorio digitale", + "text": "Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perch矇 siano utili a pi羅 persone." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Gita al fiume", + "text": "Gli amici portano una mappa e t癡 caldo, camminano nel bosco e riposano sulla riva." + } + ] + }, + { + "code": "nl", + "name": "Dutch", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Ochtend in de stad", + "text": "De stad ontwaakt langzaam, caf矇s openen hun deuren en mensen wachten op de eerste bussen." + }, + { + "id": "library", + "split": "train", + "title": "De stille bibliotheek", + "text": "Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris ge簿nteresseerde leerlingen en 矇矇n leraar helpt." + }, + { + "id": "market", + "split": "train", + "title": "Zaterdagmarkt", + "text": "Boeren verkopen vers brood, appels, kaas en honing naast twee caf矇s op het drukke plein." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitale werkplaats", + "text": "Een klein team maakt een museumgids, controleert de code en schrijft duidelijke idee禱n over po禱zie en teksten." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Reis naar de rivier", + "text": "Vrienden namen een kaart en warme thee mee en rustten na 矇矇n lange wandeling bij de rivier." + } + ] + }, + { + "code": "no", + "name": "Norwegian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgen i byen", + "text": "Byen v疇kner langsomt, kafeene 疇pner d繪rene og folk venter p疇 de f繪rste bussene n疆r bryggen og sj繪en." + }, + { + "id": "library", + "split": "train", + "title": "Det stille biblioteket", + "text": "Lesere finner b繪ker om historie, 疆rlige fortellinger og vitenskap mens bibliotekaren hjelper elevene." + }, + { + "id": "market", + "split": "train", + "title": "L繪rdagsmarked", + "text": "B繪nder selger ferskt br繪d, epler, ost og honning p疇 det travle torget." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitalt verksted", + "text": "Et lite lag lager en museumsguide, unders繪ker koden og skriver tydelige tekster." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Tur til elven", + "text": "Vennene tok med kart og varm te p疇 s繪ndag og hvilte senere ved elven." + } + ] + }, + { + "code": "pt", + "name": "Portuguese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Manh瓊 na cidade", + "text": "A cidade acorda devagar, os caf矇s abrem e as pessoas esperam os primeiros 繫nibus." + }, + { + "id": "library", + "split": "train", + "title": "A biblioteca tranquila", + "text": "Os leitores procuram livros de hist籀ria e ci礙ncia enquanto a bibliotec獺ria ajuda os alunos." + }, + { + "id": "market", + "split": "train", + "title": "Mercado de s獺bado", + "text": "Os agricultores vendem p瓊o fresco, ma癟瓊s, queijo e mel na pra癟a movimentada." + }, + { + "id": "workshop", + "split": "train", + "title": "Oficina digital", + "text": "Uma pequena equipe prepara o guia do museu, verifica o c籀digo e escreve descri癟繭es claras." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Viagem ao rio", + "text": "Os amigos levaram um mapa e ch獺 quente, caminharam pela mata e descansaram junto ao rio." + } + ] + }, + { + "code": "sv", + "name": "Swedish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgon i staden", + "text": "Staden vaknar l疇ngsamt, kaf矇erna 繹ppnar och m瓣nniskor v瓣ntar p疇 de f繹rsta bussarna." + }, + { + "id": "library", + "split": "train", + "title": "Det tysta biblioteket", + "text": "L瓣sare s繹ker b繹cker om historia och vetenskap medan bibliotekarien hj瓣lper eleverna." + }, + { + "id": "market", + "split": "train", + "title": "L繹rdagsmarknad", + "text": "B繹nder s瓣ljer f瓣rskt br繹d, 瓣pplen, ost och honung p疇 det livliga torget." + }, + { + "id": "workshop", + "split": "train", + "title": "Digital verkstad", + "text": "Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter f繹r bes繹kare." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Utflykt till floden", + "text": "V瓣nnerna tog med karta och varmt te och vilade sedan vid floden." + } + ] + }, + { + "code": "th", + "name": "Thai", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "鉆鉊鉆鉊耜鉊鉆鉊﹤虞鉊冢", + "text": "鉆鉊﹤虞鉊冢鉊鉆鉊冢腺 鉆 鉊鉊獅鉊 鉊鉆鉊耜鉊鉊耜鉊鉆鉊鉊毯鉊鉊鉊啤鉊嫩鉊丞萼鉊鉊嫩鉊鉊鉊鉊冢腦鉊鉆鉊鉊Ｒ葵鉊耜腦鉆鉊鉊菽鉊Ｒ葷鉆鉊鉊" + }, + { + "id": "library", + "split": "train", + "title": "鉊徇鉊冢鉊芹腹鉊詮鉆鉊鉊菽腺鉊鉊芹鉊", + "text": "鉊鉊嫩鉊冢鉊耜鉊鉆鉊鉊徇葡鉊徇鉊晤鉊芹虞鉊冢鉊鉊啤葷鉊晤鉊毯落鉊耜葵鉊鉊鉆鉆鉊丞萼鉊抉葩鉊鉊Ｒ葡鉊兒葡鉊芹鉊鉆 鉊鉊鉊鉊鉊耜腦鉊晤鉊拈鉊鉆鉊抉腺鉊鉊晤鉆鉊鉊菽腺鉊鉆鉊丞虞鉊冢鉊鉆鉊冢腹鉊嫩艇" + }, + { + "id": "market", + "split": "train", + "title": "鉊鉊丞葡鉊鉊抉萵鉊鉆鉊芹葡鉊鉆", + "text": "鉊鉊耜葷鉊鉊耜鉊耜腺鉊鉊鉊﹤鉊晤鉊芹 鉊鉊丞鉊﹤ 鉊鉊菽葵 鉆鉊丞萼鉊鉆鉊喪鉊嗣鉊鉆鉊鉊丞葡鉊鉊鉊菽鉊鉊嗣鉊鉊晤" + }, + { + "id": "workshop", + "split": "train", + "title": "鉊徇鉊冢鉊鉊喪鉊耜鉊鉊毯鉊毯鉊晤艇", + "text": "鉊鉊菽腹鉆鉊丞鉊鉊鉊晤鉊鉊喪鉊嫩鉊﹤虞鉊冢鉊毯鉊毯鉊鉊晤鉊鉆 鉊鉊鉊抉鉊鉊徇萵鉊 鉆鉊丞萼鉆鉊鉊菽腺鉊鉊鉊喪葉鉊鉊毯鉊耜腺鉊鉊菽鉊鉊晤鉆鉊鉊" + }, + { + "id": "river-trip", + "split": "validation", + "title": "鉆鉊鉊菽鉊Ｒ葷鉊鉊毯腹鉆鉊﹤鉊鉆鉊", + "text": "鉆鉊鉊獅鉊冢鉊鉊喪鉊鉊鉊鉊菽鉊鉊晤鉊鉊耜腦鉆鉊冢鉆鉊鉆鉊鉊毯鉆鉊鉊鉆鉊耜鉊丞鉊抉鉊晤鉊鉆鉊冢鉊鉊毯腹鉆鉊﹤鉊鉆鉊" + } + ] + }, + { + "code": "tr", + "name": "Turkish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "ehirde sabah", + "text": "ehir yava癟a uyan覺r, kafeler a癟覺l覺r ve insanlar ilk otob羹sleri bekler." + }, + { + "id": "library", + "split": "train", + "title": "Sessiz k羹t羹phane", + "text": "Okurlar tarih ve bilim kitaplar覺 ararken k羹t羹phaneci 繹rencilere yard覺mc覺 olur." + }, + { + "id": "market", + "split": "train", + "title": "Cumartesi pazar覺", + "text": "ift癟iler kalabal覺k meydanda taze ekmek, elma, peynir ve bal satar." + }, + { + "id": "workshop", + "split": "train", + "title": "Dijital at繹lye", + "text": "K羹癟羹k bir ekip m羹ze rehberi haz覺rlar, kodu denetler ve a癟覺k a癟覺klamalar yazar." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Nehir gezisi", + "text": "Arkadalar harita ve s覺cak 癟ay al覺p ormanda y羹r羹d羹, sonra nehir k覺y覺s覺nda dinlendi." + } + ] + }, + { + "code": "vi", + "name": "Vietnamese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Bu廙i s獺ng trong thnh ph廙", + "text": "Thnh ph廙 th廙妾 d廕軌 ch廕衫 r瓊i, qu獺n c ph礙 m廙 c廙苔 v m廙i ngが廙i ch廙 chuy廕積 xe bu羸t 廕吟 ti礙n." + }, + { + "id": "library", + "split": "train", + "title": "Thが vi廙n y礙n t藺nh", + "text": "廙c gi廕 t穫m s獺ch l廙ch s廙 v khoa h廙c, c簷n th廙 thが gi繳p h廙c sinh ch廙n ti li廙u." + }, + { + "id": "market", + "split": "train", + "title": "Ch廙 th廙 b廕ㄊ", + "text": "N繫ng d璽n b獺n b獺nh m穫, t獺o, ph繫 mai v m廕負 ong t廕【 qu廕τg trが廙ng 繫ng 繳c." + }, + { + "id": "workshop", + "split": "train", + "title": "Xが廙ng k廙 thu廕負 s廙", + "text": "M廙t nh籀m nh廙 lm hが廙ng d廕南 b廕υ tng, ki廙m tra m瓊 v vi廕篙 m繫 t廕 r繭 rng." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Chuy廕積 i ra s繫ng", + "text": "B廕》 b癡 mang b廕τ 廙 v tr n籀ng, i qua r廙南g r廙i ngh廙 b礙n b廙 s繫ng." + } + ] + }, + { + "code": "ja", + "name": "Japanese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "銵格", + "text": "株胯柴閬整擏怒舫扎鈭箝整怨嗅柴嫘芷韏啜憪" + }, + { + "id": "library", + "split": "train", + "title": "芸單賊尹", + "text": "啣桀單賊尹扼舀研柴潦詻喋隤剛舀韭脯蝘摮艾銵格研Ｕ豢詻臬郎株矽寧押隡" + }, + { + "id": "market", + "split": "train", + "title": "乓桀", + "text": "摨氬桀氬怒舀圈悅芷押潦艾柴喋銝艾嗚鞎瑯拙恥航噙摰嗚典迤蝭格怒扎西店" + }, + { + "id": "workshop", + "split": "train", + "title": "詻踴怠極", + "text": "撠芥潦拚尹格雿艾閮剛駁Ｕ氬箄喋潦蝣箄蝺券隤祆詻" + }, + { + "id": "river-trip", + "split": "validation", + "title": "撌詻桀銵", + "text": "鈭箝∼臬啣喋冽萱嗚行ㄝ甇押瑯甇押桀芸颲箝找踴芥憭嫘桅野桀ㄟ" + } + ] + }, + { + "code": "zh", + "name": "Chinese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "撣拇", + "text": "撣Ｘａ靘嚗頠蝡鈭銝剔鈭綽∪憭折嚗蝚砌剖砍望掃頠憪蝛輸摰銵" + }, + { + "id": "library", + "split": "train", + "title": "摰賊尹", + "text": "蝷曉賊尹鋆∪芣蝧餅貊脤喋霈撠暹風脯蝘摮詨銵寥Ｙ賂蝞∠∪鼠拙飛交曇" + }, + { + "id": "market", + "split": "train", + "title": "剔撣", + "text": "撱港撣湔箸遛圈悅祈瘞湔暻萄憿批恥颲脫閰Ｗ嗆嚗銝虫漱瘚摰嗅虜瘜" + }, + { + "id": "workshop", + "split": "train", + "title": "訾撌乩摰", + "text": "銝撠甇刻ˊ雿拚尹閮剛撣急渡ｇ潸瑼Ｘ亦撘嚗蝺刻摩箔撟湧翩閫啣神皜璆隤芣" + }, + { + "id": "river-trip", + "split": "validation", + "title": "瘝喲銵", + "text": "撣嗉啣梯嗥忽璅寞韏唬敺銋隞亙嚗隞典瘝喲隡荔賢曈仿陷" + } + ] } ] } diff --git a/corpus/test-sources.json b/corpus/test-sources.json index 9f75718..fb6a3c8 100644 --- a/corpus/test-sources.json +++ b/corpus/test-sources.json @@ -85,6 +85,114 @@ "id": "community-garden", "title": "Ogr籀d ssiedzki", "text": "Mieszkacy zmienili pusty dziedziniec w zielony ogr籀d z kwiatami, warzywami i modymi drzewami. Dzieci zrobiy kolorowe tabliczki, a rodziny co tydzie podlewaj roliny." + }, + { + "language": "ar", + "id": "community-garden", + "title": "堶堹堜 塈堶", + "text": "堬堭媢 塈堿堭塈 塈堬堭 塈堧奡堿塈堭 堻塈堶堜 堹堜 媯媢 塈堧媟塈 塈堛塈堛 堜 塈堛 塈堿媢 媢 堻 塈堥塈堛塈堛." + }, + { + "language": "cs", + "id": "community-garden", + "title": "Sousedsk獺 zahrada", + "text": "Soused矇 promnili pr獺zdn羸 dv轡r v zelenou zahradu, dti namalovaly barevn矇 znaky a rodiny zal矇vaj穩 rostliny." + }, + { + "language": "da", + "id": "community-garden", + "title": "F疆lleshaven", + "text": "Naboerne gjorde en tom g疇rd til en gr繪n have, b繪rnene malede skilte, og familierne vander planterne hver uge." + }, + { + "language": "en", + "id": "community-garden", + "title": "The community garden", + "text": "Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small caf矇 table." + }, + { + "language": "el", + "id": "community-garden", + "title": " 庥怷庣彖怷庣庥 庥峸怷", + "text": "庣 帠庰巹怷彖庰 峟庥帢彖帢彖 弮庣帢 峎帤庰庣帢 帢弇峸 峎庣彖怷 庥峸怷, 帢 帢庣帤庣峎 庤帠峎庣帢彖 庣彖帢庥巹帤庰 庥帢庣 怷庣 怷庣庥怷帠峟彖庰庣庰 怷巹庤怷彖 帢 峎." + }, + { + "language": "et", + "id": "community-garden", + "title": "Kogukonna aed", + "text": "Naabrid muutsid t羹hja hoovi roheliseks aiaks, lapsed maalisid sildid ja pered kastavad taimi igal n瓣dalal." + }, + { + "language": "he", + "id": "community-garden", + "title": " 蚹蛌蛌", + "text": "蚸 蚺 蛈蚳 蚳蚹 蚳蚹, 蛈蚳 蚸 蚸蚺蛌 蚸蚹蛌 蛌 蛈 蚸蛁." + }, + { + "language": "hu", + "id": "community-garden", + "title": "K繹z繹ss矇gi kert", + "text": "A szomsz矇dok z繹ld kertt矇 alak穩tott獺k az 羹res udvart, a gyerekek t獺bl獺kat festettek, a csal獺dok pedig 繹nt繹zik a n繹v矇nyeket." + }, + { + "language": "it", + "id": "community-garden", + "title": "Il giardino comune", + "text": "I vicini hanno trasformato un cortile vuoto in un giardino della citt, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante." + }, + { + "language": "nl", + "id": "community-garden", + "title": "De buurttuin", + "text": "Buren veranderden een lege binnenplaats naast een caf矇 in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water." + }, + { + "language": "no", + "id": "community-garden", + "title": "Felleshagen", + "text": "Naboene gjorde en tom g疇rd til en gr繪nn hage, barna malte skilt og familiene vanner plantene hver uke." + }, + { + "language": "pt", + "id": "community-garden", + "title": "A horta comunit獺ria", + "text": "Os vizinhos transformaram um p獺tio vazio em jardim, as crian癟as pintaram placas e as fam穩lias regam as plantas toda semana." + }, + { + "language": "sv", + "id": "community-garden", + "title": "Gemensam tr瓣dg疇rd", + "text": "Grannarna gjorde en tom g疇rd till en gr繹n tr瓣dg疇rd, barnen m疇lade skyltar och familjerna vattnar v瓣xterna varje vecka." + }, + { + "language": "th", + "id": "community-garden", + "title": "鉊芹葷鉊鉊鉊詮腹鉊鉊", + "text": "鉆鉊鉊獅鉊冢鉊鉆鉊耜鉆鉊鉊丞葭鉆鉊Ｒ鉊丞葡鉊鉊抉鉊耜鉆鉊鉆鉊鉊芹葷鉊鉊芹葭鉆鉊鉊菽腺鉊 鉆鉊鉆鉊 鉆 鉊抉葡鉊鉊鉆鉊耜腺鉆鉊丞萼鉊鉊鉊冢鉊鉊鉊晤葷鉊鉆鉊抉腺鉊鉊晤鉊鉊鉊鉆鉊喪鉆鉊鉆鉊﹤鉊鉊詮鉊芹萵鉊鉊鉊耜葦鉆" + }, + { + "language": "tr", + "id": "community-garden", + "title": "Mahalle bah癟esi", + "text": "Komular bo avluyu yeil bir bah癟eye 癟evirdi, 癟ocuklar renkli tabelalar yapt覺 ve aileler bitkileri her hafta suluyor." + }, + { + "language": "vi", + "id": "community-garden", + "title": "Vが廙n c廙ng 廙ng", + "text": "Hng x籀m bi廕積 s璽n tr廙ng thnh khu vが廙n xanh, tr廕 em v廕 bi廙n mu v c獺c gia 穫nh tが廙i c璽y m廙i tu廕吵." + }, + { + "language": "ja", + "id": "community-garden", + "title": "啣桀滬", + "text": "餈桐犖∼舐征艾摨准氬晞乓具璊摮押∼航脤悅芣准雿摰嗆舀望押急偌" + }, + { + "language": "zh", + "id": "community-garden", + "title": "蝷曉勗", + "text": "啣蝛粹Ｗ寥蝬脰勗嚗蝔桐擙株晞祈撠璅嫘摮拙思敶抵脫嚗摰嗆曹韏瑞策璊拇瘞氬" } ] } diff --git a/package.json b/package.json index e644490..1cb77cb 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,11 @@ "build": "rm -rf lib/* && tsc -p tsconfig.build.json", "corpus:build": "node scripts/build-corpus.mjs", "corpus:verify": "node scripts/verify-corpus.mjs", - "models:build": "node scripts/build-models.mjs", - "models:evaluate": "node scripts/evaluate-models.mjs", - "models:verify": "node scripts/verify-models.mjs", + "models:build": "tsx scripts/build-models.mjs", + "models:evaluate": "tsx scripts/evaluate-models.mjs", + "models:verify": "tsx scripts/verify-models.mjs", + "benchmark": "tsx scripts/benchmark.mjs", + "validate": "tsx scripts/validate.mjs", "format": "prettier --write \"**/*.{ts,json,md,mjs}\"", "format:check": "prettier --list-different \"**/*.{ts,json,md,mjs}\"", "test": "npm run test:typecheck && vitest run --coverage", diff --git a/scripts/benchmark.mjs b/scripts/benchmark.mjs new file mode 100644 index 0000000..d5bb06e --- /dev/null +++ b/scripts/benchmark.mjs @@ -0,0 +1,93 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../src/index.ts'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const generatedCorpus = join(root, 'corpus', 'generated'); +const corpusIndex = JSON.parse( + readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), +); +const manifest = JSON.parse( + readFileSync(join(root, 'corpus', 'manifest.json'), 'utf8'), +); +const familyByEncoding = new Map( + manifest.encodings.map((encoding) => [ + encoding.name, + encoding.family === 'multibyte' ? 'multibyte' : 'singlebyte', + ]), +); + +function option(name, fallback) { + const prefix = `--${name}=`; + const arg = process.argv.find((value) => value.startsWith(prefix)); + return arg ? arg.slice(prefix.length) : fallback; +} + +const iterations = Number(option('iterations', '100')); +const split = option('split', 'all'); +const family = option('family', 'all'); + +if (!Number.isInteger(iterations) || iterations < 1) { + throw new Error('--iterations must be a positive integer'); +} +if (!['all', 'singlebyte', 'multibyte'].includes(family)) { + throw new Error('--family must be all, singlebyte, or multibyte'); +} + +const rows = corpusIndex.filter( + (row) => + (split === 'all' || row.split === split) && + (family === 'all' || familyByEncoding.get(row.encoding) === family), +); +if (rows.length === 0) { + throw new Error(`No corpus files matched split: ${split}`); +} + +const inputs = rows.map((row) => ({ + ...row, + buffer: readFileSync(join(generatedCorpus, row.path)), +})); +const workloadHash = createHash('sha256') + .update(rows.map((row) => `${row.path}\0${row.sha256}\n`).join('')) + .digest('hex'); + +for (const input of inputs) analyse(input.buffer); + +const startedAt = process.hrtime.bigint(); +let matches = 0; +let bytes = 0; + +for (let iteration = 0; iteration < iterations; iteration += 1) { + for (const input of inputs) { + matches += analyse(input.buffer).length; + bytes += input.buffer.length; + } +} + +const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; +const calls = inputs.length * iterations; +const bytesPerSecond = bytes / (elapsedMs / 1000); + +console.log('Library corpus benchmark'); +console.log('========================'); +console.log(`Split: ${split}`); +console.log(`Family: ${family}`); +console.log(`Files: ${inputs.length}`); +console.log( + `Input bytes per iteration: ${inputs.reduce( + (total, input) => total + input.buffer.length, + 0, + )}`, +); +console.log(`Workload SHA-256: ${workloadHash}`); +console.log(`Iterations: ${iterations}`); +console.log(`Calls: ${calls}`); +console.log(`Bytes processed: ${bytes}`); +console.log(`Matches produced: ${matches}`); +console.log(`Elapsed: ${elapsedMs.toFixed(2)} ms`); +console.log(`Throughput: ${(calls / (elapsedMs / 1000)).toFixed(2)} files/sec`); +console.log( + `Byte throughput: ${(bytesPerSecond / 1024 / 1024).toFixed(2)} MiB/sec`, +); diff --git a/scripts/corpus-utils.mjs b/scripts/corpus-utils.mjs index 0d6b230..bdb185b 100644 --- a/scripts/corpus-utils.mjs +++ b/scripts/corpus-utils.mjs @@ -137,13 +137,22 @@ export function buildCorpus(destination) { let modelHighByteCount = 0; const modelHighByteValues = new Set(); for (const document of language.documents) { - const utf8 = Buffer.from( - `${document.title}\n\n${document.text}\n`, - 'utf8', - ); + let sourceText = `${document.title}\n\n${document.text}\n`; + for (const [from, to] of Object.entries( + encoding.sourceReplacements ?? {}, + )) { + sourceText = sourceText.replaceAll(from, to); + } + if (encoding.sourceSuffix) { + sourceText = `${sourceText.trimEnd()}${encoding.sourceSuffix}\n`; + } + const utf8 = Buffer.from(sourceText, 'utf8'); const encoded = iconv(utf8, 'UTF-8', encoding.iconv); const decoded = iconv(encoded, encoding.iconv, 'UTF-8'); - if (!decoded.equals(utf8)) { + if ( + decoded.toString('utf8').normalize('NFC') !== + utf8.toString('utf8').normalize('NFC') + ) { throw new Error( `${encoding.name}/${languageCode}/${document.id} did not round trip`, ); diff --git a/scripts/evaluate-models.mjs b/scripts/evaluate-models.mjs index f80c70d..2b557e0 100644 --- a/scripts/evaluate-models.mjs +++ b/scripts/evaluate-models.mjs @@ -4,7 +4,11 @@ const { report } = compileModels(); for (const result of report.results) { const expected = `${result.expected.encoding}/${result.expected.language}`; const predicted = `${result.predicted.encoding}/${result.predicted.language}`; - const marker = result.encodingCorrect ? 'PASS' : 'FAIL'; + const marker = result.encodingExact + ? 'PASS' + : result.encodingEquivalent + ? 'EQUIV' + : 'FAIL'; console.log( `${marker} ${expected.padEnd(24)} -> ${predicted.padEnd(24)} ${result.predicted.confidence}%`, ); @@ -12,6 +16,8 @@ for (const result of report.results) { console.log( `\nEncoding: ${report.summary.encodingCorrect}/${report.summary.tests}; ` + + `exact: ${report.summary.encodingExact}/${report.summary.tests}; ` + + `equivalent: ${report.summary.encodingEquivalent}; ` + `language: ${report.summary.languageCorrect}/${report.summary.tests}; ` + `top-score ties: ${report.summary.topScoreTies}`, ); diff --git a/scripts/model-utils.mjs b/scripts/model-utils.mjs index 7c66d93..604db30 100644 --- a/scripts/model-utils.mjs +++ b/scripts/model-utils.mjs @@ -2,6 +2,13 @@ import { spawnSync } from 'node:child_process'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + prepareSBCSModels, + scoreSBCS, + trigrams, +} from '../src/encoding/sbcs-scoring.ts'; +import { big5, euc_jp, euc_kr, gb_18030, sjis } from '../src/encoding/mbcs.ts'; +import { prepareMBCSModel, scoreMBCS } from '../src/encoding/mbcs-scoring.ts'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const corpus = join(root, 'corpus'); @@ -90,30 +97,6 @@ function byteMapFor(encoding) { return byteMap; } -function normalizedBytes(buffer, byteMap) { - const normalized = []; - let ignoreSpace = false; - for (const byte of buffer) { - const mapped = byteMap[byte]; - if (mapped !== 0) { - if (!(mapped === 0x20 && ignoreSpace)) normalized.push(mapped); - ignoreSpace = mapped === 0x20; - } - } - normalized.push(0x20); - return normalized; -} - -function trigrams(buffer, byteMap) { - const values = []; - let trigram = 0; - for (const byte of normalizedBytes(buffer, byteMap)) { - trigram = ((trigram << 8) + byte) & 0xffffff; - values.push(trigram); - } - return values; -} - function detectorNgrams(buffers, byteMap) { const counts = new Map(); for (const buffer of buffers) { @@ -160,47 +143,28 @@ function highByteDistribution(buffers) { }; } -function byteLogLikelihood(buffer, distribution) { - const counts = new Map(distribution.counts); - const denominator = distribution.total + 128; - let highBytes = 0; - let likelihood = 0; - for (const byte of buffer) { - if (byte >= 0x80) { - likelihood += Math.log(((counts.get(byte) ?? 0) + 1) / denominator); - highBytes += 1; - } - } - return highBytes === 0 - ? Number.NEGATIVE_INFINITY - : Number((likelihood / highBytes).toFixed(12)); -} - -function ngramScore(buffer, byteMap, modelNgrams) { - const known = new Set(modelNgrams); - const values = trigrams(buffer, byteMap); - const hits = values.filter((trigram) => known.has(trigram)).length; - const rawPercent = hits / values.length; - return { - confidence: rawPercent > 0.33 ? 98 : Math.floor(rawPercent * 300), - hits, - total: values.length, - hitRate: rawPercent, - }; +function decodedText(buffer, encodingName) { + const encoding = manifest.encodings.find( + (candidate) => candidate.name === encodingName, + ); + if (!encoding) throw new Error(`Missing manifest encoding: ${encodingName}`); + return iconv(buffer, encoding.iconv, 'UTF-8'); } -function statisticallyCompetitive(best, candidate) { - const variance = - (best.hitRate * (1 - best.hitRate)) / best.total + - (candidate.hitRate * (1 - candidate.hitRate)) / candidate.total; - const margin = 1.96 * Math.sqrt(variance); - return best.hitRate - candidate.hitRate <= margin; +function byteEquivalent(buffer, leftEncoding, rightEncoding) { + try { + return decodedText(buffer, leftEncoding).equals( + decodedText(buffer, rightEncoding), + ); + } catch { + return false; + } } function compileSingleByteModels() { const models = []; for (const encoding of manifest.encodings) { - if (encoding.name === 'CP949') continue; + if (encoding.family === 'multibyte') continue; const byteMap = byteMapFor(encoding.iconv); const languages = encoding.languages.map((language) => { @@ -228,68 +192,105 @@ function compileSingleByteModels() { return models; } +const multibyteRecognisers = new Map([ + ['Shift_JIS', new sjis()], + ['Big5', new big5()], + ['EUC-JP', new euc_jp()], + ['EUC-KR', new euc_kr()], + ['GB18030', new gb_18030()], +]); + +function detectorContext(buffer) { + const byteStats = Array(256).fill(0); + for (const byte of buffer) byteStats[byte] += 1; + return { + byteStats, + c1Bytes: buffer.some((byte) => byte >= 0x80 && byte <= 0x9f), + rawInput: buffer, + rawLen: buffer.length, + inputBytes: buffer, + inputLen: buffer.length, + }; +} + +function compileMultibyteModels() { + return manifest.encodings + .filter( + (encoding) => + encoding.family === 'multibyte' && encoding.runtime !== false, + ) + .map((encoding) => { + const recogniser = multibyteRecognisers.get(encoding.name); + if (!recogniser) { + throw new Error(`Missing multibyte recogniser: ${encoding.name}`); + } + if (encoding.languages.length !== 1) { + throw new Error( + `Expected one language for multibyte model: ${encoding.name}`, + ); + } + const language = encoding.languages[0]; + const frequencies = new Map(); + let total = 0; + for (const row of corpusIndex.filter( + (candidate) => + candidate.encoding === encoding.name && + candidate.language === language && + candidate.split === 'train', + )) { + const buffer = readFileSync(join(generatedCorpus, row.path)); + const statistics = recogniser.statistics(detectorContext(buffer)); + if (statistics.invalidCharacters !== 0) { + throw new Error( + `${encoding.name}/${language}/${row.document} has invalid multibyte characters`, + ); + } + for (const value of statistics.multibyteCharacters) { + frequencies.set(value, (frequencies.get(value) ?? 0) + 1); + total += 1; + } + } + if (total === 0) { + throw new Error(`Empty multibyte model: ${encoding.name}/${language}`); + } + return { + encoding: encoding.name, + language, + total, + commonCharacters: [...frequencies] + .sort( + ([left, leftCount], [right, rightCount]) => + rightCount - leftCount || left - right, + ) + .slice(0, 128) + .map(([value]) => value) + .sort((left, right) => left - right), + }; + }); +} + function evaluate(models) { + const preparedModels = prepareSBCSModels(models); + const modelNames = new Set(models.map((model) => model.encoding)); const tests = corpusIndex.filter( - (row) => row.split === 'test' && row.encoding !== 'CP949', + (row) => row.split === 'test' && modelNames.has(row.encoding), ); const results = tests.map((test) => { const buffer = readFileSync(join(generatedCorpus, test.path)); - const candidates = models.map((model) => { - const languageScores = model.languages.map((language) => ({ - language: language.language, - ...ngramScore(buffer, model.byteMap, language.ngrams), - byteLogLikelihood: byteLogLikelihood(buffer, language.highBytes), - })); - const confidenceScore = Math.max( - ...languageScores.map((score) => score.confidence), - ); - const confidenceLanguages = languageScores.filter( - (score) => score.confidence === confidenceScore, - ); - confidenceLanguages.sort( - (left, right) => right.byteLogLikelihood - left.byteLogLikelihood, - ); - const confidenceLanguage = confidenceLanguages[0]; - languageScores.sort( - (left, right) => - right.byteLogLikelihood - left.byteLogLikelihood || - right.confidence - left.confidence, - ); - return { - encoding: model.encoding, - confidence: confidenceScore, - language: languageScores[0].language, - byteLogLikelihood: confidenceLanguage.byteLogLikelihood, - hits: confidenceLanguage.hits, - total: confidenceLanguage.total, - hitRate: confidenceLanguage.hitRate, - }; - }); - candidates.sort((left, right) => right.confidence - left.confidence); - const strongest = candidates[0]; - const competitive = candidates.filter((candidate) => - statisticallyCompetitive(strongest, candidate), - ); - const competitiveEncodings = new Set( - competitive.map((candidate) => candidate.encoding), - ); - competitive.sort( - (left, right) => - right.byteLogLikelihood - left.byteLogLikelihood || - right.confidence - left.confidence, - ); - const remaining = candidates.filter( - (candidate) => !competitiveEncodings.has(candidate.encoding), - ); - candidates.splice(0, candidates.length, ...competitive, ...remaining); + const candidates = scoreSBCS(buffer, preparedModels); const predicted = candidates[0]; + const encodingExact = predicted.encoding === test.encoding; + const encodingEquivalent = + !encodingExact && + byteEquivalent(buffer, test.encoding, predicted.encoding); + const encodingCorrect = encodingExact || encodingEquivalent; return { expected: { encoding: test.encoding, language: test.language }, predicted, - encodingCorrect: predicted.encoding === test.encoding, - languageCorrect: - predicted.encoding === test.encoding && - predicted.language === test.language, + encodingExact, + encodingEquivalent, + encodingCorrect, + languageCorrect: encodingCorrect && predicted.language === test.language, tiedAtTop: candidates.filter( (candidate) => candidate.confidence === predicted.confidence, ).length, @@ -308,6 +309,9 @@ function evaluate(models) { return { summary: { tests: results.length, + encodingExact: results.filter((result) => result.encodingExact).length, + encodingEquivalent: results.filter((result) => result.encodingEquivalent) + .length, encodingCorrect: results.filter((result) => result.encodingCorrect) .length, languageCorrect: results.filter((result) => result.languageCorrect) @@ -319,6 +323,56 @@ function evaluate(models) { }; } +function evaluateMultibyte(models) { + const preparedModels = models.map((model) => ({ + model, + prepared: prepareMBCSModel(model), + recogniser: multibyteRecognisers.get(model.encoding), + })); + const tests = corpusIndex.filter( + (row) => + row.split === 'test' && + models.some((model) => model.encoding === row.encoding), + ); + const results = tests.map((test) => { + const buffer = readFileSync(join(generatedCorpus, test.path)); + const candidates = preparedModels + .map(({ model, prepared, recogniser }) => { + const score = scoreMBCS( + recogniser.statistics(detectorContext(buffer)), + prepared, + ); + return { + encoding: model.encoding, + language: model.language, + ...score, + }; + }) + .sort( + (left, right) => + right.confidence - left.confidence || + left.encoding.localeCompare(right.encoding), + ); + return { + expected: { encoding: test.encoding, language: test.language }, + predicted: candidates[0], + encodingCorrect: candidates[0].encoding === test.encoding, + languageCorrect: candidates[0].language === test.language, + candidates, + }; + }); + return { + summary: { + tests: results.length, + encodingCorrect: results.filter((result) => result.encodingCorrect) + .length, + languageCorrect: results.filter((result) => result.languageCorrect) + .length, + }, + results, + }; +} + function javascriptString(value) { return `'${value.replaceAll('\\', '\\\\').replaceAll("'", "\\'")}'`; } @@ -327,9 +381,10 @@ function hex(value, width) { return `0x${value.toString(16).padStart(width, '0')}`; } -function serializeModels(models) { +function serializeModels(models, multibyteModels) { const lines = [ '// Generated by npm run models:build. Do not edit manually.', + "import type { GeneratedMBCSModel } from './types';", '', 'export const generatedSBCSModels = [', ]; @@ -383,18 +438,47 @@ function serializeModels(models) { lines.push(' },'); } lines.push('] as const;', ''); + lines.push( + 'export const generatedMBCSModels: readonly GeneratedMBCSModel[] = [', + ); + for (const model of multibyteModels) { + lines.push(' {'); + lines.push(` encoding: ${javascriptString(model.encoding)},`); + lines.push(` language: ${javascriptString(model.language)},`); + lines.push(` total: ${model.total},`); + lines.push(' commonCharacters: ['); + for (let index = 0; index < model.commonCharacters.length; index += 6) { + lines.push( + ` ${model.commonCharacters + .slice(index, index + 6) + .map((value) => hex(value, 8)) + .join(', ')},`, + ); + } + lines.push(' ],'); + lines.push(' },'); + } + lines.push('];', ''); return lines.join('\n'); } export function compileModels() { const models = compileSingleByteModels(); - return { models, report: evaluate(models) }; + const multibyteModels = compileMultibyteModels(); + return { + models, + multibyteModels, + report: { + ...evaluate(models), + multibyte: evaluateMultibyte(multibyteModels), + }, + }; } export function buildModels(modelPath, evaluationPath) { - const { models, report } = compileModels(); + const { models, multibyteModels, report } = compileModels(); mkdirSync(dirname(modelPath), { recursive: true }); mkdirSync(dirname(evaluationPath), { recursive: true }); - writeFileSync(modelPath, serializeModels(models)); + writeFileSync(modelPath, serializeModels(models, multibyteModels)); writeFileSync(evaluationPath, `${JSON.stringify(report, null, 2)}\n`); } diff --git a/scripts/validate.mjs b/scripts/validate.mjs new file mode 100644 index 0000000..80c2aeb --- /dev/null +++ b/scripts/validate.mjs @@ -0,0 +1,155 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../src/index.ts'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const generatedCorpus = join(root, 'corpus', 'generated'); +const corpusIndex = JSON.parse( + readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), +); + +function option(name, fallback) { + const prefix = `--${name}=`; + const arg = process.argv.find((value) => value.startsWith(prefix)); + return arg ? arg.slice(prefix.length) : fallback; +} + +function percent(value, total) { + return total === 0 ? '0.00%' : `${((value / total) * 100).toFixed(2)}%`; +} + +function increment(object, key) { + object[key] = (object[key] ?? 0) + 1; +} + +const split = option('split', 'all'); +const rows = corpusIndex.filter( + (row) => split === 'all' || row.split === split, +); +if (rows.length === 0) { + throw new Error(`No corpus files matched split: ${split}`); +} + +const results = rows.map((row) => { + const buffer = readFileSync(join(generatedCorpus, row.path)); + const matches = analyse(buffer); + const predicted = matches[0] ?? null; + return { + expected: { + encoding: row.encoding, + language: row.language, + split: row.split, + path: row.path, + }, + predicted, + encodingCorrect: predicted?.name === row.encoding, + languageCorrect: + predicted?.name === row.encoding && predicted?.lang === row.language, + }; +}); + +const summary = { + files: results.length, + encodingCorrect: results.filter((result) => result.encodingCorrect).length, + languageCorrect: results.filter((result) => result.languageCorrect).length, + noPrediction: results.filter((result) => result.predicted === null).length, +}; + +const bySplit = {}; +const byExpectedEncoding = {}; +const topPredictions = {}; + +for (const result of results) { + const splitSummary = (bySplit[result.expected.split] ??= { + files: 0, + encodingCorrect: 0, + languageCorrect: 0, + }); + splitSummary.files += 1; + if (result.encodingCorrect) splitSummary.encodingCorrect += 1; + if (result.languageCorrect) splitSummary.languageCorrect += 1; + + const encodingSummary = (byExpectedEncoding[result.expected.encoding] ??= { + files: 0, + encodingCorrect: 0, + languageCorrect: 0, + predictions: {}, + }); + encodingSummary.files += 1; + if (result.encodingCorrect) encodingSummary.encodingCorrect += 1; + if (result.languageCorrect) encodingSummary.languageCorrect += 1; + increment(encodingSummary.predictions, result.predicted?.name ?? ''); + increment(topPredictions, result.predicted?.name ?? ''); +} + +console.log('Library corpus validation'); +console.log('========================='); +console.log(`Split: ${split}`); +console.log(`Files: ${summary.files}`); +console.log( + `Encoding correct: ${summary.encodingCorrect}/${summary.files} (${percent( + summary.encodingCorrect, + summary.files, + )})`, +); +console.log( + `Language correct: ${summary.languageCorrect}/${summary.files} (${percent( + summary.languageCorrect, + summary.files, + )})`, +); +console.log(`No prediction: ${summary.noPrediction}`); + +console.log('\nBy split'); +console.table( + Object.entries(bySplit).map(([name, item]) => ({ + split: name, + files: item.files, + encodingCorrect: item.encodingCorrect, + encodingAccuracy: percent(item.encodingCorrect, item.files), + languageCorrect: item.languageCorrect, + languageAccuracy: percent(item.languageCorrect, item.files), + })), +); + +console.log('\nBy expected encoding'); +console.table( + Object.entries(byExpectedEncoding) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([encoding, item]) => ({ + encoding, + files: item.files, + encodingCorrect: item.encodingCorrect, + encodingAccuracy: percent(item.encodingCorrect, item.files), + languageCorrect: item.languageCorrect, + mostCommonPrediction: Object.entries(item.predictions).sort( + ([leftName, leftCount], [rightName, rightCount]) => + rightCount - leftCount || leftName.localeCompare(rightName), + )[0][0], + })), +); + +console.log('\nTop predicted encodings'); +console.table( + Object.entries(topPredictions) + .sort( + ([leftName, leftCount], [rightName, rightCount]) => + rightCount - leftCount || leftName.localeCompare(rightName), + ) + .map(([encoding, count]) => ({ encoding, count })), +); + +const failures = results.filter((result) => !result.encodingCorrect); +console.log('\nFirst 20 encoding mismatches'); +console.table( + failures.slice(0, 20).map((result) => ({ + expected: result.expected.encoding, + language: result.expected.language, + split: result.expected.split, + predicted: result.predicted?.name ?? '', + predictedLanguage: result.predicted?.lang ?? '', + confidence: result.predicted?.confidence ?? '', + path: result.expected.path, + })), +); diff --git a/scripts/verify-models.mjs b/scripts/verify-models.mjs index 56442e5..a964fce 100644 --- a/scripts/verify-models.mjs +++ b/scripts/verify-models.mjs @@ -15,12 +15,24 @@ try { if (!readFileSync(rebuiltModel).equals(readFileSync(generatedModel))) { throw new Error('Generated detector models are out of date'); } + const evaluation = JSON.parse(readFileSync(rebuiltEvaluation, 'utf8')); if ( !readFileSync(rebuiltEvaluation).equals(readFileSync(generatedEvaluation)) ) { throw new Error('Generated model evaluation is out of date'); } - const evaluation = JSON.parse(readFileSync(rebuiltEvaluation, 'utf8')); + if ( + evaluation.multibyte.summary.encodingCorrect !== + evaluation.multibyte.summary.tests || + evaluation.multibyte.summary.languageCorrect !== + evaluation.multibyte.summary.tests + ) { + throw new Error( + `Multibyte model evaluation failed: ${evaluation.multibyte.summary.encodingCorrect}/` + + `${evaluation.multibyte.summary.tests} encodings, ${evaluation.multibyte.summary.languageCorrect}/` + + `${evaluation.multibyte.summary.tests} languages`, + ); + } if ( evaluation.summary.encodingCorrect !== evaluation.summary.tests || evaluation.summary.languageCorrect !== evaluation.summary.tests diff --git a/src/encoding/ascii.ts b/src/encoding/ascii.ts index 013c286..d65535e 100644 --- a/src/encoding/ascii.ts +++ b/src/encoding/ascii.ts @@ -16,6 +16,6 @@ export default class Ascii implements Recogniser { } } - return match(det, this, 100); + return match(det, this, 1); } } diff --git a/src/encoding/iso2022.test.ts b/src/encoding/iso2022.test.ts index 15b611b..c7d7b7e 100644 --- a/src/encoding/iso2022.test.ts +++ b/src/encoding/iso2022.test.ts @@ -11,7 +11,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-JP', () => { expect(analyse('iso2022jp')).toEqual({ - confidence: 100, + confidence: 1, lang: 'ja', name: 'ISO-2022-JP', }); @@ -19,7 +19,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-KR', () => { expect(analyse('iso2022kr')).toEqual({ - confidence: 100, + confidence: 1, lang: 'kr', name: 'ISO-2022-KR', }); @@ -27,7 +27,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-CN', () => { expect(analyse('iso2022cn')).toEqual({ - confidence: 100, + confidence: 1, lang: 'zh', name: 'ISO-2022-CN', }); diff --git a/src/encoding/iso2022.ts b/src/encoding/iso2022.ts index 6252639..5cc1d80 100644 --- a/src/encoding/iso2022.ts +++ b/src/encoding/iso2022.ts @@ -7,12 +7,10 @@ import match, { type Match, type EncodingName } from '../match'; * of encodings. */ -class ISO_2022 implements Recogniser { +abstract class ISO_2022 implements Recogniser { escapeSequences: number[][] = []; - name(): EncodingName { - return 'ISO_2022'; - } + abstract name(): EncodingName; match(det: Context): Match | null { /** @@ -70,15 +68,15 @@ class ISO_2022 implements Recogniser { // // Initial quality is based on relative proportion of recognized vs. // unrecognized escape sequences. - // All good: quality = 100; + // All good: quality = 1; // half or less good: quality = 0; // linear in between. - confidence = (100 * hits - 100 * misses) / (hits + misses); + confidence = (hits - misses) / (hits + misses); // Back off quality if there were too few escape sequences seen. // Include shifts in this computation, so that KR does not get penalized // for having only a single Escape sequence, but many shifts. - if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 10; + if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 0.1; return confidence <= 0 ? null : match(det, this, confidence); } diff --git a/src/encoding/mbcs-scoring.test.ts b/src/encoding/mbcs-scoring.test.ts new file mode 100644 index 0000000..79354e4 --- /dev/null +++ b/src/encoding/mbcs-scoring.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import type { GeneratedMBCSModel } from './models/types'; +import { + prepareMBCSModel, + scoreMBCS, + type MBCSStatistics, +} from './mbcs-scoring'; + +const model: GeneratedMBCSModel = { + encoding: 'EUC-JP', + language: 'ja', + total: 100, + commonCharacters: [0xa1a1, 0xa4a2], +}; +const prepared = prepareMBCSModel(model); + +function statistics( + multibyteCharacters: number[], + invalidCharacters = 0, +): MBCSStatistics { + return { + totalCharacters: multibyteCharacters.length + invalidCharacters, + multibyteCharacters, + invalidCharacters, + }; +} + +describe('generated MBCS scoring', () => { + it('scores model coverage rather than structural compatibility alone', () => { + expect( + scoreMBCS(statistics(Array(20).fill(0xa4a2)), prepared), + ).toMatchObject({ confidence: 1, hits: 20 }); + expect( + scoreMBCS(statistics(Array(20).fill(0xb0b0)), prepared), + ).toMatchObject({ confidence: 0, hits: 0 }); + }); + + it('reduces model confidence when invalid sequences are present', () => { + expect( + scoreMBCS(statistics(Array(20).fill(0xa4a2), 1), prepared).confidence, + ).toBe(0.5); + }); + + it('returns finite boundary scores for short and empty input', () => { + expect(scoreMBCS(statistics([]), prepared).confidence).toBe(0); + expect(scoreMBCS(statistics([0xa4a2]), prepared).confidence).toBe(0.1); + }); +}); diff --git a/src/encoding/mbcs-scoring.ts b/src/encoding/mbcs-scoring.ts new file mode 100644 index 0000000..b47b32a --- /dev/null +++ b/src/encoding/mbcs-scoring.ts @@ -0,0 +1,55 @@ +import type { GeneratedMBCSModel } from './models/types'; + +export interface MBCSStatistics { + readonly totalCharacters: number; + readonly multibyteCharacters: readonly number[]; + readonly invalidCharacters: number; +} + +export interface MBCSScore { + readonly confidence: number; + readonly hits: number; + readonly multibyteCharacters: number; + readonly invalidCharacters: number; +} + +export interface PreparedMBCSModel { + readonly model: GeneratedMBCSModel; + readonly commonCharacters: ReadonlySet; +} + +export function prepareMBCSModel(model: GeneratedMBCSModel): PreparedMBCSModel { + return { + model, + commonCharacters: new Set(model.commonCharacters), + }; +} + +export function scoreMBCS( + statistics: MBCSStatistics, + prepared: PreparedMBCSModel, +): MBCSScore { + const multibyteCharacters = statistics.multibyteCharacters.length; + const invalidCharacters = statistics.invalidCharacters; + const hits = statistics.multibyteCharacters.reduce( + (count, value) => count + (prepared.commonCharacters.has(value) ? 1 : 0), + 0, + ); + + let confidence = 0; + if (multibyteCharacters <= 10 && invalidCharacters === 0) { + confidence = + multibyteCharacters === 0 && statistics.totalCharacters < 10 ? 0 : 0.1; + } else if (multibyteCharacters >= 20 * invalidCharacters) { + const structuralValidity = + multibyteCharacters / (multibyteCharacters + invalidCharacters * 20); + confidence = (hits / multibyteCharacters) * structuralValidity; + } + + return { + confidence: Number(Math.min(confidence, 1).toFixed(2)), + hits, + multibyteCharacters, + invalidCharacters, + }; +} diff --git a/src/encoding/mbcs.test.ts b/src/encoding/mbcs.test.ts index 4a510bc..5a0b418 100644 --- a/src/encoding/mbcs.test.ts +++ b/src/encoding/mbcs.test.ts @@ -1,8 +1,11 @@ import * as chardet from '..'; +import fs from 'fs'; +import path from 'path'; import { describe, expect, it } from 'vitest'; describe('Multibyte Character Sets', () => { const base = __dirname + '/../test/data/encodings'; + const corpus = path.join(__dirname, '../../corpus/generated'); it('should return Shift_JIS', () => { expect(chardet.detectFileSync(base + '/shiftjis')).toBe('Shift_JIS'); @@ -23,4 +26,23 @@ describe('Multibyte Character Sets', () => { it('should return EUC-KR', () => { expect(chardet.detectFileSync(base + '/euc_kr')).toBe('EUC-KR'); }); + + it.each([ + ['Shift_JIS', 'ja'], + ['EUC-JP', 'ja'], + ['Big5', 'zh'], + ['GB18030', 'zh'], + ['EUC-KR', 'ko'], + ])('uses the generated %s model for held-out %s text', (encoding, lang) => { + const fixture = path.join( + corpus, + encoding, + lang, + 'test/community-garden.bin', + ); + expect(chardet.analyse(fs.readFileSync(fixture))[0]).toMatchObject({ + name: encoding, + lang, + }); + }); }); diff --git a/src/encoding/mbcs.ts b/src/encoding/mbcs.ts index 4eb2b97..0dff1c0 100644 --- a/src/encoding/mbcs.ts +++ b/src/encoding/mbcs.ts @@ -1,59 +1,26 @@ import type { Context, Recogniser } from '.'; -import match, { type Match, type EncodingName } from '../match'; +import match, { type EncodingName, type Match } from '../match'; +import { + prepareMBCSModel, + scoreMBCS, + type MBCSStatistics, +} from './mbcs-scoring'; +import { generatedMBCSModels } from './models/generated'; +import type { GeneratedMBCSModel } from './models/types'; + +const models = new Map( + (generatedMBCSModels as readonly GeneratedMBCSModel[]).map((model) => [ + model.encoding, + prepareMBCSModel(model), + ]), +); -/** - * Binary search implementation (recursive) - */ -function binarySearch(arr: number[], searchValue: number) { - const find = ( - arr: number[], - searchValue: number, - left: number, - right: number, - ): number => { - if (right < left) return -1; - - /* - int mid = mid = (left + right) / 2; - There is a bug in the above line; - Joshua Bloch suggests the following replacement: - */ - const mid = Math.floor((left + right) >>> 1); - if (searchValue > arr[mid]) return find(arr, searchValue, mid + 1, right); - - if (searchValue < arr[mid]) return find(arr, searchValue, left, mid - 1); - - return mid; - }; - - return find(arr, searchValue, 0, arr.length - 1); -} - -// 'Character' iterated character class. -// Recognizers for specific mbcs encodings make their 'characters' available -// by providing a nextChar() function that fills in an instance of iteratedChar -// with the next char from the input. -// The returned characters are not converted to Unicode, but remain as the raw -// bytes (concatenated into an int) from the codepage data. -// -// For Asian charsets, use the raw input rather than the input that has been -// stripped of markup. Detection only considers multi-byte chars, effectively -// stripping markup anyway, and double byte chars do occur in markup too. -// class IteratedChar { - charValue: number; // 1-4 bytes from the raw input data - index: number; - nextIndex: number; - error: boolean; - done: boolean; - - constructor() { - this.charValue = 0; // 1-4 bytes from the raw input data - this.index = 0; - this.nextIndex = 0; - this.error = false; - this.done = false; - } + charValue = 0; + index = 0; + nextIndex = 0; + error = false; + done = false; reset() { this.charValue = 0; @@ -68,135 +35,43 @@ class IteratedChar { this.done = true; return -1; } - const byteValue = det.rawInput[this.nextIndex++] & 0x00ff; - return byteValue; + return det.rawInput[this.nextIndex++] & 0xff; } } /** - * Asian double or multi-byte - charsets. - * Match is determined mostly by the input data adhering to the - * encoding scheme for the charset, and, optionally, - * frequency-of-occurrence of characters. + * Base recogniser for encodings whose structural validity is determined by + * parsing variable-width byte sequences. Character likelihood comes from the + * generated corpus model for the concrete encoding. */ - -class mbcs implements Recogniser { - commonChars: number[] = []; - - name(): EncodingName { - return 'mbcs'; - } - - /** - * Test the match of this charset with the input text data - * which is obtained via the CharsetDetector object. - * - * @param det The CharsetDetector, which contains the input text - * to be checked for being in this charset. - * @return Two values packed into one int (Damn java, anyhow) - * bits 0-7: the match confidence, ranging from 0-100 - * bits 8-15: The match reason, an enum-like value. - */ - match(det: Context): Match | null { - let doubleByteCharCount = 0, - commonCharCount = 0, - badCharCount = 0, - totalCharCount = 0, - confidence = 0; - +export abstract class mbcs implements Recogniser { + abstract name(): EncodingName; + abstract nextChar(iter: IteratedChar, det: Context): boolean; + + statistics(det: Context): MBCSStatistics { + let invalidCharacters = 0; + let totalCharacters = 0; + const multibyteCharacters: number[] = []; const iter = new IteratedChar(); - detectBlock: { - for (iter.reset(); this.nextChar(iter, det); ) { - totalCharCount++; - if (iter.error) { - badCharCount++; - } else { - const cv = iter.charValue & 0xffffffff; - - if (cv > 0xff) { - doubleByteCharCount++; - if (this.commonChars != null) { - // NOTE: This assumes that there are no 4-byte common chars. - if (binarySearch(this.commonChars, cv) >= 0) { - commonCharCount++; - } - } - } - } - if (badCharCount >= 2 && badCharCount * 5 >= doubleByteCharCount) { - // console.log('its here!') - // Bail out early if the byte data is not matching the encoding scheme. - break detectBlock; - } - } - - if (doubleByteCharCount <= 10 && badCharCount == 0) { - // Not many multi-byte chars. - if (doubleByteCharCount == 0 && totalCharCount < 10) { - // There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes. - // We don't have enough data to have any confidence. - // Statistical analysis of single byte non-ASCII characters would probably help here. - confidence = 0; - } else { - // ASCII or ISO file? It's probably not our encoding, - // but is not incompatible with our encoding, so don't give it a zero. - confidence = 10; - } - break detectBlock; - } - - // - // No match if there are too many characters that don't fit the encoding scheme. - // (should we have zero tolerance for these?) - // - if (doubleByteCharCount < 20 * badCharCount) { - confidence = 0; - break detectBlock; + for (iter.reset(); this.nextChar(iter, det); ) { + totalCharacters += 1; + if (iter.error) invalidCharacters += 1; + else if (iter.charValue > 0xff) { + multibyteCharacters.push(iter.charValue >>> 0); } - - if (this.commonChars == null) { - // We have no statistics on frequently occurring characters. - // Assess confidence purely on having a reasonable number of - // multi-byte characters (the more the better - confidence = 30 + doubleByteCharCount - 20 * badCharCount; - if (confidence > 100) { - confidence = 100; - } - } else { - // Frequency of occurrence statistics exist. - const maxVal = Math.log(doubleByteCharCount / 4); - const scaleFactor = 90.0 / maxVal; - confidence = Math.floor( - Math.log(commonCharCount + 1) * scaleFactor + 10, - ); - confidence = Math.min(confidence, 100); - } - } // end of detectBlock: - - return confidence == 0 ? null : match(det, this, confidence); + } + return { totalCharacters, multibyteCharacters, invalidCharacters }; } - /** - * Get the next character (however many bytes it is) from the input data - * Subclasses for specific charset encodings must implement this function - * to get characters according to the rules of their encoding scheme. - * - * This function is not a method of class iteratedChar only because - * that would require a lot of extra derived classes, which is awkward. - * @param it The iteratedChar 'struct' into which the returned char is placed. - * @param det The charset detector, which is needed to get at the input byte data - * being iterated over. - * @return True if a character was returned, false at end of input. - */ - nextChar(_iter: IteratedChar, _det: Context): boolean { - return true; + match(det: Context): Match | null { + const prepared = models.get(this.name()); + if (!prepared) return null; + const { confidence } = scoreMBCS(this.statistics(det), prepared); + return confidence === 0 ? null : match(det, this, confidence); } } -/** - * Shift_JIS charset recognizer. - */ export class sjis extends mbcs { name(): EncodingName { return 'Shift_JIS'; @@ -206,32 +81,17 @@ export class sjis extends mbcs { return 'ja'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. - commonChars = [ - 0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, - 0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, - 0x82b3, 0x82b5, 0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, - 0x82c8, 0x82c9, 0x82cc, 0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, - 0x82ea, 0x82f0, 0x82f1, 0x8341, 0x8343, 0x834e, 0x834f, 0x8358, 0x835e, - 0x8362, 0x8367, 0x8375, 0x8376, 0x8389, 0x838a, 0x838b, 0x838d, 0x8393, - 0x8e96, 0x93fa, 0x95aa, - ]; - nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - const firstByte = (iter.charValue = iter.nextByte(det)); if (firstByte < 0) return false; - - if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) + if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) { return true; + } const secondByte = iter.nextByte(det); if (secondByte < 0) return false; - iter.charValue = (firstByte << 8) | secondByte; if ( !( @@ -239,16 +99,12 @@ export class sjis extends mbcs { (secondByte >= 0x80 && secondByte <= 0xff) ) ) { - // Illegal second byte value. iter.error = true; } return true; } } -/** - * Big5 charset recognizer. - */ export class big5 extends mbcs { name(): EncodingName { return 'Big5'; @@ -257,112 +113,53 @@ export class big5 extends mbcs { language() { return 'zh'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. - commonChars = [ - 0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, - 0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, - 0xa477, 0xa4a3, 0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, - 0xa4fd, 0xa540, 0xa548, 0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, - 0xa662, 0xa668, 0xa670, 0xa6a8, 0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, - 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da, 0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, - 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, 0xaa6b, 0xaaba, 0xaabe, - 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, 0xaec9, 0xafe0, - 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, 0xb5a5, - 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, - 0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f, - ]; nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - const firstByte = (iter.charValue = iter.nextByte(det)); - if (firstByte < 0) return false; - - // single byte character. - if (firstByte <= 0x7f || firstByte == 0xff) return true; + if (firstByte <= 0x7f || firstByte === 0xff) return true; const secondByte = iter.nextByte(det); - if (secondByte < 0) return false; - - iter.charValue = (iter.charValue << 8) | secondByte; - - if (secondByte < 0x40 || secondByte == 0x7f || secondByte == 0xff) + iter.charValue = (firstByte << 8) | secondByte; + if (secondByte < 0x40 || secondByte === 0x7f || secondByte === 0xff) { iter.error = true; - + } return true; } } -/** - * EUC charset recognizers. One abstract class that provides the common function - * for getting the next character according to the EUC encoding scheme, - * and nested derived classes for EUC_KR, EUC_JP, EUC_CN. - * - * Get the next character value for EUC based encodings. - * Character 'value' is simply the raw bytes that make up the character - * packed into an int. - */ function eucNextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - let firstByte = 0; - let secondByte = 0; - let thirdByte = 0; - //int fourthByte = 0; - buildChar: { - firstByte = iter.charValue = iter.nextByte(det); - if (firstByte < 0) { - // Ran off the end of the input data - iter.done = true; - break buildChar; - } - if (firstByte <= 0x8d) { - // single byte char - break buildChar; - } - secondByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | secondByte; - if (firstByte >= 0xa1 && firstByte <= 0xfe) { - // Two byte Char - if (secondByte < 0xa1) { - iter.error = true; - } - break buildChar; - } - if (firstByte == 0x8e) { - // Code Set 2. - // In EUC-JP, total char size is 2 bytes, only one byte of actual char value. - // In EUC-TW, total char size is 4 bytes, three bytes contribute to char value. - // We don't know which we've got. - // Treat it like EUC-JP. If the data really was EUC-TW, the following two - // bytes will look like a well formed 2 byte char. - if (secondByte < 0xa1) { - iter.error = true; - } - break buildChar; - } - if (firstByte == 0x8f) { - // Code set 3. - // Three byte total char size, two bytes of actual char value. - thirdByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | thirdByte; - if (thirdByte < 0xa1) { - iter.error = true; - } - } + const firstByte = (iter.charValue = iter.nextByte(det)); + if (firstByte < 0) { + iter.done = true; + return false; + } + if (firstByte <= 0x8d) return true; + + const secondByte = iter.nextByte(det); + if (secondByte < 0) return false; + iter.charValue = (firstByte << 8) | secondByte; + if (firstByte >= 0xa1 && firstByte <= 0xfe) { + if (secondByte < 0xa1) iter.error = true; + return true; + } + if (firstByte === 0x8e) { + if (secondByte < 0xa1) iter.error = true; + return true; } - return iter.done == false; + if (firstByte === 0x8f) { + const thirdByte = iter.nextByte(det); + iter.charValue = (iter.charValue << 8) | thirdByte; + if (thirdByte < 0xa1) iter.error = true; + } + return !iter.done; } -/** - * The charset recognize for EUC-JP. A singleton instance of this class - * is created and kept by the public CharsetDetector class - */ export class euc_jp extends mbcs { name(): EncodingName { return 'EUC-JP'; @@ -372,31 +169,9 @@ export class euc_jp extends mbcs { return 'ja'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. - commonChars = [ - 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, - 0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, - 0xa4b1, 0xa4b3, 0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, - 0xa4c1, 0xa4c3, 0xa4c4, 0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, - 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de, 0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, - 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, 0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, - 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, 0xa5b0, 0xa5b3, 0xa5b5, - 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, 0xa5c8, 0xa5c9, - 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, 0xa5e5, - 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, - 0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, - 0xcdd1, - ]; - nextChar = eucNextChar; } -/** - * The charset recognize for EUC-KR. A singleton instance of this class - * is created and kept by the public CharsetDetector class - */ export class euc_kr extends mbcs { name(): EncodingName { return 'EUC-KR'; @@ -406,30 +181,9 @@ export class euc_kr extends mbcs { return 'ko'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. - commonChars = [ - 0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, - 0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, - 0xb4cf, 0xb4d9, 0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, - 0xb7af, 0xb7c2, 0xb7ce, 0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, - 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce, 0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, - 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, 0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, - 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, 0xbef8, 0xbefa, 0xbfa1, - 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, 0xc0af, 0xc0b8, - 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, 0xc0da, - 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, - 0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, - 0xc8ad, - ]; - nextChar = eucNextChar; } -/** - * GB-18030 recognizer. Uses simplified Chinese statistics. - */ export class gb_18030 extends mbcs { name(): EncodingName { return 'GB18030'; @@ -439,74 +193,39 @@ export class gb_18030 extends mbcs { return 'zh'; } - /* - * Get the next character value for EUC based encodings. - * Character 'value' is simply the raw bytes that make up the character - * packed into an int. - */ - nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - let firstByte = 0; - let secondByte = 0; - let thirdByte = 0; - let fourthByte = 0; - buildChar: { - firstByte = iter.charValue = iter.nextByte(det); - if (firstByte < 0) { - // Ran off the end of the input data - iter.done = true; - break buildChar; - } - if (firstByte <= 0x80) { - // single byte char - break buildChar; + const firstByte = (iter.charValue = iter.nextByte(det)); + if (firstByte < 0) { + iter.done = true; + return false; + } + if (firstByte <= 0x80) return true; + + const secondByte = iter.nextByte(det); + if (secondByte < 0) return false; + iter.charValue = (firstByte << 8) | secondByte; + if (firstByte >= 0x81 && firstByte <= 0xfe) { + if ( + (secondByte >= 0x40 && secondByte <= 0x7e) || + (secondByte >= 0x80 && secondByte <= 0xfe) + ) { + return true; } - secondByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | secondByte; - if (firstByte >= 0x81 && firstByte <= 0xfe) { - // Two byte Char - if ( - (secondByte >= 0x40 && secondByte <= 0x7e) || - (secondByte >= 80 && secondByte <= 0xfe) - ) { - break buildChar; - } - // Four byte char - if (secondByte >= 0x30 && secondByte <= 0x39) { - thirdByte = iter.nextByte(det); - if (thirdByte >= 0x81 && thirdByte <= 0xfe) { - fourthByte = iter.nextByte(det); - if (fourthByte >= 0x30 && fourthByte <= 0x39) { - iter.charValue = - (iter.charValue << 16) | (thirdByte << 8) | fourthByte; - break buildChar; - } + if (secondByte >= 0x30 && secondByte <= 0x39) { + const thirdByte = iter.nextByte(det); + if (thirdByte >= 0x81 && thirdByte <= 0xfe) { + const fourthByte = iter.nextByte(det); + if (fourthByte >= 0x30 && fourthByte <= 0x39) { + iter.charValue = + (iter.charValue << 16) | (thirdByte << 8) | fourthByte; + return true; } } - iter.error = true; - break buildChar; } + iter.error = true; } - return iter.done == false; + return !iter.done; } - - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. - commonChars = [ - 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, - 0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, - 0xb5bd, 0xb5c4, 0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, - 0xb7d6, 0xb7dd, 0xb8b4, 0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, - 0xb9fd, 0xbacd, 0xbba7, 0xbbd6, 0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, - 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, 0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, - 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, 0xc7f8, 0xc8ab, 0xc8cb, - 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, 0xcad0, 0xcad6, - 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, 0xcfb5, - 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, - 0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, - 0xd6d0, - ]; } diff --git a/src/encoding/models/generated.ts b/src/encoding/models/generated.ts index 0f95a79..5aad487 100644 --- a/src/encoding/models/generated.ts +++ b/src/encoding/models/generated.ts @@ -1,6 +1,1996 @@ // Generated by npm run models:build. Do not edit manually. +import type { GeneratedMBCSModel } from './types'; export const generatedSBCSModels = [ + { + encoding: 'ISO-8859-1', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'da', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206279, 0x2062f8, 0x206465, 0x2066f8, + 0x20686f, 0x206ce6, 0x206f67, 0x2070e5, 0x2073e6, 0x2076e5, 0x2076e6, + 0x20e562, 0x20e662, 0x6166e9, 0x626962, 0x626c69, 0x6272f8, 0x627965, + 0x62f867, 0x62f86e, 0x642062, 0x6420e6, 0x646520, 0x64656e, 0x646572, + 0x652062, 0x652066, 0x6520e5, 0x656420, 0x656e20, 0x656e73, 0x657220, + 0x65726e, 0x657420, 0x66e965, 0x66f872, 0x676572, 0x686ae6, 0x69626c, + 0x696465, 0x696c6c, 0x696f74, 0x6ae66c, 0x6b7374, 0x6c6520, 0x6c696f, + 0x6c6c65, 0x6ce673, 0x6e6465, 0x6e6520, 0x6e6572, 0x6f6720, 0x6f7465, + 0x70e520, 0x722020, 0x722065, 0x72206f, 0x737465, 0x74656b, 0xe52064, + 0xf86765, + ], + highBytes: { + total: 16, + counts: [ + [0xe5, 4], + [0xe6, 5], + [0xe9, 1], + [0xf8, 6], + ], + }, + }, + { + language: 'de', + ngrams: [ + 0x206175, 0x206265, 0x206269, 0x2062e4, 0x2062fc, 0x206465, 0x206469, + 0x2064e4, 0x206569, 0x2066fc, 0x2067e4, 0x206be4, 0x206ce4, 0x207465, + 0x2074fc, 0x20756e, 0x20e470, 0x20f666, 0x20fc62, 0x616368, 0x616d20, + 0x617573, 0x626572, 0x62e463, 0x62fc63, 0x636865, 0x636874, 0x6368fc, + 0x64656e, 0x646572, 0x646965, 0x64e463, 0x652073, 0x652074, 0x65696e, + 0x656dfc, 0x656e20, 0x656efc, 0x657220, 0x65726e, 0x66fc68, 0x66fc72, + 0x67e473, 0x68656e, 0x687265, 0x696368, 0x696520, 0x696e20, 0x696e65, + 0x6c6569, 0x6e2062, 0x6e2064, 0x6e20fc, 0x6e6420, 0x6e656e, 0x736368, + 0x736520, 0x737461, 0x742065, 0x746520, 0x756e64, 0xe46e64, 0xfc6265, + 0xfc7220, + ], + highBytes: { + total: 21, + counts: [ + [0xc4, 1], + [0xdc, 1], + [0xe4, 7], + [0xf6, 1], + [0xfc, 11], + ], + }, + }, + { + language: 'en', + ngrams: [ + 0x206120, 0x20616e, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206661, + 0x206c69, 0x206e65, 0x2070e2, 0x207265, 0x2072e9, 0x2072f4, 0x207468, + 0x207768, 0x616420, 0x616465, 0x6166e9, 0x616e64, 0x61e761, 0x61ef76, + 0x626f75, 0x627261, 0x636974, 0x642063, 0x64206e, 0x646520, 0x646573, + 0x652061, 0x652063, 0x656164, 0x657273, 0x657320, 0x657369, 0x657420, + 0x65f16f, 0x6661e7, 0x66e973, 0x686520, 0x68696c, 0x696272, 0x696465, + 0x696c65, 0x6c6520, 0x6de920, 0x6e61ef, 0x6e6420, 0x7065f1, 0x70e274, + 0x726561, 0x727320, 0x72e973, 0x72f46c, 0x732061, 0x746865, 0x74e920, + 0x756de9, 0xe274e9, 0xe76164, 0xe92020, 0xe92077, 0xe97320, 0xe97375, + 0xef7665, + ], + highBytes: { + total: 9, + counts: [ + [0xe2, 1], + [0xe7, 1], + [0xe9, 4], + [0xef, 1], + [0xf1, 1], + [0xf4, 1], + ], + }, + }, + { + language: 'es', + ngrams: [ + 0x206269, 0x206369, 0x20636c, 0x20636f, 0x2063f3, 0x206465, 0x206469, + 0x20656c, 0x206c61, 0x206c6f, 0x206d65, 0x206de1, 0x207061, 0x2070e1, + 0x2070fa, 0x207265, 0x207375, 0x2073e1, 0x207665, 0x207920, 0x612062, + 0x612065, 0x612070, 0x61646f, 0x616e20, 0x617261, 0x617320, 0x6174e1, + 0x61f161, 0x626c69, 0x6369e9, 0x6369f3, 0x63f364, 0x656c20, 0x656e20, + 0x656efa, 0x6572ed, 0x657320, 0x65f161, 0x65f16f, 0x69e96e, 0x69f36e, + 0x6c206d, 0x6c6120, 0x6c6f73, 0x6d61f1, 0x6de173, 0x6e206c, 0x6e6120, + 0x6efa20, 0x6f206c, 0x6f7265, 0x6f7320, 0x706172, 0x70e167, 0x70fa62, + 0x726573, 0x72ed61, 0x732061, 0x732063, 0x73206c, 0x7365f1, 0x73e162, + 0x746f72, + ], + highBytes: { + total: 13, + counts: [ + [0xe1, 4], + [0xe9, 1], + [0xed, 1], + [0xf1, 3], + [0xf3, 2], + [0xfa, 2], + ], + }, + }, + { + language: 'fr', + ngrams: [ + 0x20636f, 0x206465, 0x206475, 0x2064e9, 0x206574, 0x206c65, 0x207061, + 0x20706f, 0x207072, 0x2070e2, 0x2072e9, 0x20756e, 0x2076e9, 0x20e963, + 0x20e96c, 0x20e971, 0x6166e9, 0x626c69, 0x6368e9, 0x646573, 0x647520, + 0x64e976, 0x652063, 0x652064, 0x65206c, 0x65206d, 0x652070, 0x6520e9, + 0x656e63, 0x656e74, 0x657320, 0x657420, 0x657572, 0x66e973, 0x68e871, + 0x68e920, 0x6c6120, 0x6c6520, 0x6c6573, 0x6c6c65, 0x6de972, 0x6e7420, + 0x6f7572, 0x706172, 0x7072e9, 0x717565, 0x72206c, 0x726520, 0x727320, + 0x72e970, 0x732063, 0x73206c, 0x732070, 0x742064, 0x74206c, 0x7468e8, + 0x756520, 0x756de9, 0x757273, 0xe87175, 0xe96520, 0xe97061, 0xe97269, + 0xe97665, + ], + highBytes: { + total: 22, + counts: [ + [0xe2, 1], + [0xe8, 3], + [0xe9, 18], + ], + }, + }, + { + language: 'it', + ngrams: [ + 0x00006c, 0x00006d, 0x006c61, 0x206269, 0x206361, 0x206369, 0x20636f, + 0x206465, 0x206469, 0x206520, 0x206765, 0x20676c, 0x206920, 0x206c61, + 0x206c65, 0x206d65, 0x207065, 0x207069, 0x207072, 0x207363, 0x207369, + 0x207374, 0x612062, 0x612063, 0x612064, 0x612067, 0x612069, 0x612070, + 0x61746f, 0x6368e9, 0x636974, 0x64656c, 0x656e74, 0x657263, 0x6666e8, + 0x66e820, 0x676c69, 0x68e920, 0x692061, 0x692063, 0x692073, 0x696120, + 0x697474, 0x69f920, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6f20, + 0x6f7269, 0x7069f9, 0x726920, 0x746f72, 0x7474e0, 0x74e020, 0xe02020, + 0xe0206c, 0xe02073, 0xe82061, 0xe8206d, 0xe92073, 0xf92067, 0xf9206c, + 0xf92070, + ], + highBytes: { + total: 9, + counts: [ + [0xe0, 3], + [0xe8, 2], + [0xe9, 1], + [0xf9, 3], + ], + }, + }, + { + language: 'nl', + ngrams: [ + 0x000064, 0x206269, 0x20626f, 0x206361, 0x20636f, 0x206465, 0x206565, + 0x20656e, 0x206765, 0x206865, 0x206c65, 0x206f70, 0x206f76, 0x207374, + 0x207465, 0x207665, 0x207765, 0x20e9e9, 0x61616b, 0x616173, 0x616420, + 0x6166e9, 0x616b74, 0x616d20, 0x626962, 0x626c69, 0x626f65, 0x636166, + 0x636874, 0x646520, 0x652062, 0x652063, 0x652065, 0x652073, 0x65656e, + 0x656572, 0x6565eb, 0x65696e, 0x656b65, 0x656e20, 0x657273, 0x65eb6e, + 0x65ef6e, 0x66e973, 0x6765ef, 0x696e20, 0x6b7420, 0x6e2020, 0x6e206f, + 0x6e20e9, 0x6feb7a, 0x706feb, 0x732065, 0x736368, 0x742064, 0x74656e, + 0x746572, 0x766572, 0xe96e20, 0xe97320, 0xe9e96e, 0xeb6e20, 0xeb7a69, + 0xef6e74, + ], + highBytes: { + total: 7, + counts: [ + [0xe9, 4], + [0xeb, 2], + [0xef, 1], + ], + }, + }, + { + language: 'no', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206272, 0x206279, 0x2062f8, 0x206465, + 0x2064f8, 0x20666f, 0x2066f8, 0x206c61, 0x206ee6, 0x206f67, 0x2070e5, + 0x207665, 0x2076e5, 0x20e570, 0x20e672, 0x626962, 0x626c69, 0x6272f8, + 0x627965, 0x62f86b, 0x62f86e, 0x642065, 0x646520, 0x646572, 0x646574, + 0x64f872, 0x652062, 0x652066, 0x652074, 0x6520e5, 0x6520e6, 0x656420, + 0x656e20, 0x656e65, 0x656e73, 0x657220, 0x657273, 0x657420, 0x66f872, + 0x672073, 0x676520, 0x67656e, 0x676572, 0x69626c, 0x696765, 0x696e67, + 0x696f74, 0x6af865, 0x6cf872, 0x6e6520, 0x6e6572, 0x6ee672, 0x6f6720, + 0x70e520, 0x72206f, 0x7273f8, 0x72f864, 0x737465, 0x74656b, 0xe52064, + 0xf86b65, + ], + highBytes: { + total: 14, + counts: [ + [0xe5, 4], + [0xe6, 2], + [0xf8, 8], + ], + }, + }, + { + language: 'pt', + ngrams: [ + 0x00006d, 0x206120, 0x206269, 0x206369, 0x2063f3, 0x206465, 0x206520, + 0x206573, 0x206e61, 0x206f20, 0x206f73, 0x207065, 0x207072, 0x2070e3, + 0x2073e1, 0x207665, 0x20f46e, 0x612062, 0x612063, 0x612064, 0x612065, + 0x61206f, 0x612070, 0x616465, 0x61646f, 0x6166e9, 0x616d20, 0x617261, + 0x617320, 0x61e761, 0x61e7e3, 0x626962, 0x626c69, 0x636120, 0x6369ea, + 0x63e172, 0x63f364, 0x646120, 0x646520, 0x646f20, 0x652061, 0x6563e1, + 0x657320, 0x657363, 0x66e973, 0x68e320, 0x696120, 0x69e7f5, 0x69ea6e, + 0x6d61e7, 0x6e6120, 0x6e68e3, 0x6f7320, 0x70e36f, 0x7261e7, 0x726573, + 0x7269e7, 0x732020, 0x732061, 0x732070, 0x7320f4, 0x7374f3, 0x73e162, + 0x74f372, + ], + highBytes: { + total: 14, + counts: [ + [0xe1, 2], + [0xe3, 3], + [0xe7, 3], + [0xe9, 1], + [0xea, 1], + [0xf3, 2], + [0xf4, 1], + [0xf5, 1], + ], + }, + }, + { + language: 'sv', + ngrams: [ + 0x000064, 0x006cf6, 0x206269, 0x2062f6, 0x206465, 0x2066e4, 0x2066f6, + 0x206c69, 0x206ce4, 0x206ce5, 0x206de4, 0x206f63, 0x2070e5, 0x20736b, + 0x207374, 0x2073e4, 0x2073f6, 0x207479, 0x207665, 0x2076e4, 0x20e470, + 0x20f670, 0x612020, 0x612062, 0x612074, 0x6120f6, 0x616420, 0x616465, + 0x6166e9, 0x617220, 0x617265, 0x626962, 0x626c69, 0x6272f6, 0x62f663, + 0x62f66e, 0x636820, 0x6420e4, 0x646520, 0x64656e, 0x646574, 0x656e20, + 0x657220, 0x65726e, 0x6573f6, 0x657420, 0x66f672, 0x676120, 0x69626c, + 0x696761, 0x696f74, 0x6b6170, 0x6b6172, 0x6e6120, 0x6f6368, 0x70e520, + 0x726e61, 0x736b61, 0x737461, 0x73f66b, 0x74206c, 0x746164, 0x766572, + 0xe52064, + ], + highBytes: { + total: 20, + counts: [ + [0xe4, 7], + [0xe5, 3], + [0xe9, 1], + [0xf6, 9], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-2', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0x20, + 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0x20, 0xb1, 0x20, 0xb3, + 0x20, 0xb5, 0xb6, 0xb7, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'cs', + ngrams: [ + 0x000064, 0x000072, 0x000073, 0x000074, 0x006469, 0x0072e1, 0x00736f, + 0x007469, 0x206120, 0x206175, 0x206279, 0x206368, 0x206476, 0x2064ed, + 0x206661, 0x206869, 0x20686c, 0x206b6e, 0x206bf3, 0x206dec, 0x20706f, + 0x207072, 0x2070ed, 0x2070f8, 0x2074fd, 0x2076ec, 0x20bee1, 0x20e865, + 0x20e874, 0x612070, 0x612076, 0x6120e8, 0x6164ed, 0x616aed, 0x616cfd, + 0x6174ed, 0x636520, 0x652061, 0x65206d, 0x652070, 0x686f76, 0x69686f, + 0x6a6520, 0x6aed20, 0x6b6e69, 0x6d616c, 0x6dec73, 0x6e6120, 0x6e6968, + 0x6e7920, 0x6eed20, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, 0x737920, + 0x756a65, 0x792020, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xed206b, + 0xf86920, + ], + highBytes: { + total: 46, + counts: [ + [0xb9, 1], + [0xbe, 1], + [0xc8, 1], + [0xe1, 8], + [0xe8, 2], + [0xe9, 4], + [0xec, 5], + [0xed, 13], + [0xf2, 1], + [0xf3, 1], + [0xf8, 4], + [0xf9, 2], + [0xfd, 3], + ], + }, + }, + { + language: 'hu', + ngrams: [ + 0x000061, 0x000064, 0x000072, 0x000073, 0x006120, 0x006469, 0x007265, + 0x00737a, 0x206120, 0x20616c, 0x20617a, 0x206275, 0x206373, 0x206b65, + 0x206b69, 0x206be1, 0x206be9, 0x206bf3, 0x206bf6, 0x206de9, 0x206dfa, + 0x206dfb, 0x2074e9, 0x2074f6, 0x2076e1, 0x20e172, 0x20e962, 0x20e973, + 0x20ed72, 0x20fa74, 0x61206b, 0x612076, 0x616b20, 0x616e20, 0x617420, + 0x617a20, 0x646967, 0x656b20, 0x657265, 0x657420, 0x697320, 0x6b2020, + 0x6b2061, 0x6bf66e, 0x6d6920, 0x6e616b, 0x6e7976, 0x6f7320, 0x6f7420, + 0x726f73, 0x73206b, 0x73206c, 0x73206d, 0x742061, 0x74206b, 0x74e172, + 0x7674e1, 0x76e172, 0x797674, 0xe1726f, 0xe97320, 0xed7420, 0xf36b20, + 0xf66e79, + ], + highBytes: { + total: 39, + counts: [ + [0xe1, 14], + [0xe9, 9], + [0xed, 4], + [0xf3, 4], + [0xf5, 1], + [0xf6, 4], + [0xfa, 2], + [0xfb, 1], + ], + }, + }, + { + language: 'pl', + ngrams: [ + 0x000063, 0x206120, 0x206269, 0x20637a, 0x20646c, 0x206920, 0x206a61, + 0x206b61, 0x206b6f, 0x206d69, 0x20706f, 0x207072, 0x20726f, 0x2073b3, + 0x207779, 0x20b677, 0x6162b3, 0x616368, 0x616ab1, 0x616d69, 0x6177ea, + 0x61b379, 0x61b66e, 0x61e620, 0x626962, 0x626c69, 0x62b36b, 0x636861, + 0x636920, 0x637a79, 0x6472f3, 0x647a69, 0x656b20, 0x65b36e, 0x65b663, + 0x65bf79, 0x676fb6, 0x6861e6, 0x692073, 0x6920b6, 0x69626c, 0x696520, + 0x696572, 0x696f74, 0x69ea20, 0x6ab120, 0x6b6120, 0x6c696f, 0x6d6920, + 0x6e6961, 0x6f7261, 0x6f7465, 0x6fb663, 0x706973, 0x70727a, 0x727a65, + 0x727a79, 0x7369ea, 0x737920, 0x74656b, 0xb1206a, 0xb1206b, 0xb12077, + 0xb66369, + ], + highBytes: { + total: 34, + counts: [ + [0xb1, 11], + [0xb3, 6], + [0xb6, 6], + [0xbf, 3], + [0xe6, 1], + [0xea, 3], + [0xf3, 4], + ], + }, + }, + { + language: 'ro', + ngrams: [ + 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, + 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, + 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, + 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, + 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, + 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, + 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, + 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, + 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, + 0xfe6120, + ], + highBytes: { + total: 49, + counts: [ + [0xba, 12], + [0xce, 1], + [0xe2, 3], + [0xe3, 21], + [0xee, 4], + [0xfe, 8], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-5', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff, 0xd0, 0xd1, 0xd2, 0xd3, + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0x20, 0xfe, 0xff, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000e0, 0x00e0d0, 0x20d020, 0x20d1d8, 0x20d220, 0x20d2de, 0x20d2eb, + 0x20d3de, 0x20d820, 0x20dade, 0x20dcd0, 0x20ddd0, 0x20ded1, 0x20dee1, + 0x20dee2, 0x20dfdb, 0x20dfde, 0x20dfe0, 0x20e0d5, 0x20e1de, 0x20e7d8, + 0x20efe0, 0xd020d2, 0xd020dd, 0xd020df, 0xd0d5e2, 0xd0ddd8, 0xd0e0da, + 0xd0e1e2, 0xd0e2d5, 0xd0eee2, 0xd0ef20, 0xd1d8d1, 0xd1dbd8, 0xd1e1e3, + 0xd2d0ee, 0xd2d5e0, 0xd2efe2, 0xd3dee0, 0xd3dee2, 0xd4d0ee, 0xd4d820, + 0xd5dbd5, 0xd5dbd8, 0xd5dddd, 0xd820df, 0xdad020, 0xdbd820, 0xdcd0e0, + 0xddd020, 0xddebd5, 0xdee1d5, 0xdfe0de, 0xe0ded4, 0xe0eb20, 0xe220e1, + 0xe2d5db, 0xe2d5e0, 0xe2ded2, 0xe2eb20, 0xebd520, 0xeee220, 0xef20df, + 0xefe220, + ], + highBytes: { + total: 678, + counts: [ + [0xb2, 1], + [0xb3, 1], + [0xb4, 1], + [0xbb, 1], + [0xbd, 3], + [0xbe, 1], + [0xbf, 1], + [0xc0, 2], + [0xc1, 1], + [0xc3, 1], + [0xc4, 1], + [0xc7, 1], + [0xd0, 63], + [0xd1, 16], + [0xd2, 26], + [0xd3, 7], + [0xd4, 21], + [0xd5, 50], + [0xd6, 5], + [0xd7, 7], + [0xd8, 47], + [0xd9, 7], + [0xda, 21], + [0xdb, 19], + [0xdc, 16], + [0xdd, 37], + [0xde, 60], + [0xdf, 23], + [0xe0, 41], + [0xe1, 34], + [0xe2, 55], + [0xe3, 14], + [0xe5, 7], + [0xe6, 3], + [0xe7, 4], + [0xe8, 6], + [0xe9, 2], + [0xeb, 26], + [0xec, 3], + [0xed, 1], + [0xee, 13], + [0xef, 27], + [0xf1, 2], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-6', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'ar', + ngrams: [ + 0x0000c7, 0x0000d3, 0x0000d5, 0x0000e8, 0x00c7e4, 0x00d3e8, 0x00d5c8, + 0x00e8d1, 0x20c3c8, 0x20c3e5, 0x20c7ce, 0x20c7e4, 0x20c8e7, 0x20c8ea, + 0x20cad3, 0x20cfe4, 0x20d1e2, 0x20d5da, 0x20d9e6, 0x20e1d1, 0x20e1ea, + 0x20e3ca, 0x20e4e4, 0x20e8c7, 0x20e8ca, 0x20e8d5, 0x20e8ea, 0xc7ca20, + 0xc7ccd9, 0xc7d120, 0xc7e4c3, 0xc7e4d9, 0xc7e4e5, 0xc7e4e8, 0xc820c7, + 0xc8c920, 0xc92020, 0xc920c7, 0xc920e8, 0xc920ea, 0xca20e8, 0xcac820, + 0xcac8c9, 0xcb20c7, 0xccd920, 0xcd20c7, 0xcfeae6, 0xd120c7, 0xd1c7cc, + 0xd5e1c7, 0xd920c7, 0xe3cac8, 0xe4e5cf, 0xe4e5e3, 0xe5cfea, 0xe5e3ca, + 0xe620c7, 0xe6c7d3, 0xe6c920, 0xe8c7e4, 0xe8d5e1, 0xe8e620, 0xeac920, + 0xeae6c9, + ], + highBytes: { + total: 324, + counts: [ + [0xc1, 2], + [0xc3, 4], + [0xc4, 1], + [0xc6, 1], + [0xc7, 56], + [0xc8, 17], + [0xc9, 11], + [0xca, 16], + [0xcb, 2], + [0xcc, 3], + [0xcd, 8], + [0xce, 4], + [0xcf, 9], + [0xd1, 19], + [0xd2, 3], + [0xd3, 7], + [0xd4, 2], + [0xd5, 5], + [0xd6, 2], + [0xd7, 1], + [0xd8, 2], + [0xd9, 8], + [0xda, 1], + [0xe1, 9], + [0xe2, 7], + [0xe3, 5], + [0xe4, 36], + [0xe5, 18], + [0xe6, 12], + [0xe7, 5], + [0xe8, 24], + [0xea, 24], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-7', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xdc, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'el', + ngrams: [ + 0x0000de, 0x0000f0, 0x0000f3, 0x0000f8, 0x00def3, 0x00f0f1, 0x00f3e1, + 0x00f8e7, 0x20dced, 0x20e1e3, 0x20e1ed, 0x20e1f1, 0x20e2e9, 0x20e2ef, + 0x20e3f1, 0x20e5eb, 0x20e5f0, 0x20e5f1, 0x20e720, 0x20eae1, 0x20ece9, + 0x20efe9, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xdc20f4, 0xdfe120, + 0xe12020, 0xe120ea, 0xe1e920, 0xe2e9e2, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, + 0xe5e920, 0xe5f220, 0xe720e2, 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9e2eb, + 0xe9efe8, 0xe9f3f4, 0xeae1e9, 0xeae720, 0xebe720, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xeff5ed, 0xf0fceb, 0xf1dc20, 0xf1e9ef, + 0xf220ea, 0xf3f4de, 0xf3f4e7, 0xf4e120, 0xf4e5f2, 0xf4e7ed, 0xf5ed20, + 0xfcebe7, + ], + highBytes: { + total: 327, + counts: [ + [0xb9, 1], + [0xc7, 1], + [0xcc, 1], + [0xcf, 2], + [0xd0, 1], + [0xd3, 1], + [0xd8, 1], + [0xdc, 13], + [0xdd, 5], + [0xde, 4], + [0xdf, 9], + [0xe1, 31], + [0xe2, 9], + [0xe3, 9], + [0xe4, 4], + [0xe5, 16], + [0xe6, 1], + [0xe7, 14], + [0xe8, 6], + [0xe9, 31], + [0xea, 16], + [0xeb, 10], + [0xec, 11], + [0xed, 15], + [0xee, 1], + [0xef, 24], + [0xf0, 8], + [0xf1, 16], + [0xf2, 7], + [0xf3, 8], + [0xf4, 17], + [0xf5, 10], + [0xf6, 5], + [0xf7, 3], + [0xf8, 2], + [0xf9, 5], + [0xfc, 5], + [0xfd, 2], + [0xfe, 2], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-8', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'he', + ngrams: [ + 0x0000e1, 0x0000e4, 0x0000f1, 0x0000f9, 0x00e1e5, 0x00e4f1, 0x00f1e3, + 0x00f9e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, 0x20e1fa, + 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f7, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e7f7, 0x20e8f8, + 0x20ece0, 0x20ece1, 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, + 0x20eee7, 0x20eeeb, 0xe0e9ed, 0xe4f1f4, 0xe5f8e9, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed20, 0xe9f820, 0xe9fa20, 0xed2020, 0xed20ec, + 0xed20ee, 0xef20ee, 0xf0e420, 0xf1f4f8, 0xf2e9f8, 0xf4f8e9, 0xf7e5f8, + 0xf820e4, 0xf820ee, 0xf8e5fa, 0xf8e920, 0xf8e9ed, 0xf8fa20, 0xf9e9ed, + 0xfa20ec, + ], + highBytes: { + total: 249, + counts: [ + [0xe0, 9], + [0xe1, 12], + [0xe2, 2], + [0xe3, 8], + [0xe4, 16], + [0xe5, 30], + [0xe6, 2], + [0xe7, 6], + [0xe8, 7], + [0xe9, 33], + [0xea, 1], + [0xeb, 6], + [0xec, 9], + [0xed, 13], + [0xee, 11], + [0xef, 3], + [0xf0, 6], + [0xf1, 7], + [0xf2, 6], + [0xf4, 7], + [0xf6, 1], + [0xf7, 10], + [0xf8, 22], + [0xf9, 7], + [0xfa, 15], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-9', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'tr', + ngrams: [ + 0x000063, 0x000064, 0x000073, 0x0000fe, 0x006375, 0x00fe65, 0x2061e7, + 0x206269, 0x20656b, 0x206b61, 0x206bfc, 0x206dfc, 0x207361, 0x207461, + 0x207665, 0x207961, 0x20e769, 0x20f6f0, 0x20fe65, 0x616261, 0x616c20, + 0x616c61, 0x616cfd, 0x616e65, 0x616efd, 0x617220, 0x6172fd, 0x6174f6, + 0x617a61, 0x617afd, 0x61e7fd, 0x61fee7, 0x62616c, 0x652062, 0x656869, + 0x657220, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697220, 0x697461, + 0x6bfc74, 0x6c6172, 0x6c6572, 0x706861, 0x722020, 0x72206b, 0x722076, + 0x722079, 0x726920, 0x726c61, 0x72fd20, 0x746172, 0x74fc70, 0x766520, + 0x7a6172, 0x7a6520, 0xe7fd6b, 0xfc7068, 0xfc74fc, 0xfd6b20, 0xfd7220, + 0xfe6568, + ], + highBytes: { + total: 32, + counts: [ + [0xc7, 1], + [0xde, 2], + [0xe7, 6], + [0xf0, 1], + [0xf6, 2], + [0xfc, 8], + [0xfd, 11], + [0xfe, 1], + ], + }, + }, + ], + }, + { + encoding: 'windows-1250', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa1, 0x20, 0xb3, 0x20, 0xb9, 0x20, 0x20, + 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0x20, 0xb3, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0xb9, 0xba, 0x20, 0xbe, 0x20, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'cs', + ngrams: [ + 0x000064, 0x000072, 0x000073, 0x000074, 0x006469, 0x0072e1, 0x00736f, + 0x007469, 0x206120, 0x206175, 0x206279, 0x206368, 0x206476, 0x2064ed, + 0x206661, 0x206869, 0x20686c, 0x206b6e, 0x206bf3, 0x206dec, 0x20706f, + 0x207072, 0x2070ed, 0x2070f8, 0x2074fd, 0x2076ec, 0x209ee1, 0x20e865, + 0x20e874, 0x612070, 0x612076, 0x6120e8, 0x6164ed, 0x616aed, 0x616cfd, + 0x6174ed, 0x636520, 0x652061, 0x65206d, 0x652070, 0x686f76, 0x69686f, + 0x6a6520, 0x6aed20, 0x6b6e69, 0x6d616c, 0x6dec73, 0x6e6120, 0x6e6968, + 0x6e7920, 0x6eed20, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, 0x737920, + 0x756a65, 0x792020, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xed206b, + 0xf86920, + ], + highBytes: { + total: 46, + counts: [ + [0x9a, 1], + [0x9e, 1], + [0xc8, 1], + [0xe1, 8], + [0xe8, 2], + [0xe9, 4], + [0xec, 5], + [0xed, 13], + [0xf2, 1], + [0xf3, 1], + [0xf8, 4], + [0xf9, 2], + [0xfd, 3], + ], + }, + }, + { + language: 'hu', + ngrams: [ + 0x000061, 0x000064, 0x000072, 0x000073, 0x006120, 0x006469, 0x007265, + 0x00737a, 0x206120, 0x20616c, 0x20617a, 0x206275, 0x206373, 0x206b65, + 0x206b69, 0x206be1, 0x206be9, 0x206bf3, 0x206bf6, 0x206de9, 0x206dfa, + 0x206dfb, 0x2074e9, 0x2074f6, 0x2076e1, 0x20e172, 0x20e962, 0x20e973, + 0x20ed72, 0x20fa74, 0x61206b, 0x612076, 0x616b20, 0x616e20, 0x617420, + 0x617a20, 0x646967, 0x656b20, 0x657265, 0x657420, 0x697320, 0x6b2020, + 0x6b2061, 0x6bf66e, 0x6d6920, 0x6e616b, 0x6e7976, 0x6f7320, 0x6f7420, + 0x726f73, 0x73206b, 0x73206c, 0x73206d, 0x742061, 0x74206b, 0x74e172, + 0x7674e1, 0x76e172, 0x797674, 0xe1726f, 0xe97320, 0xed7420, 0xf36b20, + 0xf66e79, + ], + highBytes: { + total: 39, + counts: [ + [0xe1, 14], + [0xe9, 9], + [0xed, 4], + [0xf3, 4], + [0xf5, 1], + [0xf6, 4], + [0xfa, 2], + [0xfb, 1], + ], + }, + }, + { + language: 'pl', + ngrams: [ + 0x000063, 0x206120, 0x206269, 0x20637a, 0x20646c, 0x206920, 0x206a61, + 0x206b61, 0x206b6f, 0x206d69, 0x20706f, 0x207072, 0x20726f, 0x2073b3, + 0x207779, 0x209c77, 0x6162b3, 0x616368, 0x616ab9, 0x616d69, 0x6177ea, + 0x619c6e, 0x61b379, 0x61e620, 0x626962, 0x626c69, 0x62b36b, 0x636861, + 0x636920, 0x637a79, 0x6472f3, 0x647a69, 0x656b20, 0x659c63, 0x65b36e, + 0x65bf79, 0x676f9c, 0x6861e6, 0x692073, 0x69209c, 0x69626c, 0x696520, + 0x696572, 0x696f74, 0x69ea20, 0x6ab920, 0x6b6120, 0x6c696f, 0x6d6920, + 0x6e6961, 0x6f7261, 0x6f7465, 0x6f9c63, 0x706973, 0x70727a, 0x727a65, + 0x727a79, 0x7369ea, 0x737920, 0x74656b, 0x9c6369, 0xb9206a, 0xb9206b, + 0xb92077, + ], + highBytes: { + total: 34, + counts: [ + [0x9c, 6], + [0xb3, 6], + [0xb9, 11], + [0xbf, 3], + [0xe6, 1], + [0xea, 3], + [0xf3, 4], + ], + }, + }, + { + language: 'ro', + ngrams: [ + 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, + 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, + 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, + 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, + 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, + 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, + 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, + 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, + 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, + 0xfe6120, + ], + highBytes: { + total: 49, + counts: [ + [0xba, 12], + [0xce, 1], + [0xe2, 3], + [0xe3, 21], + [0xee, 4], + [0xfe, 8], + ], + }, + }, + ], + }, + { + encoding: 'windows-1251', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x90, 0x83, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa2, 0xa2, 0xbc, 0x20, 0xb4, 0x20, 0x20, + 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0xb3, 0xb3, + 0xb4, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0xbc, 0xbe, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000f0, 0x00f0e0, 0x20e020, 0x20e1e8, 0x20e220, 0x20e2ee, 0x20e2fb, + 0x20e3ee, 0x20e820, 0x20eaee, 0x20ece0, 0x20ede0, 0x20eee1, 0x20eef1, + 0x20eef2, 0x20efeb, 0x20efee, 0x20eff0, 0x20f0e5, 0x20f1ee, 0x20f7e8, + 0x20fff0, 0xe020e2, 0xe020ed, 0xe020ef, 0xe0e5f2, 0xe0ede8, 0xe0f0ea, + 0xe0f1f2, 0xe0f2e5, 0xe0fef2, 0xe0ff20, 0xe1e8e1, 0xe1ebe8, 0xe1f1f3, + 0xe2e0fe, 0xe2e5f0, 0xe2fff2, 0xe3eef0, 0xe3eef2, 0xe4e0fe, 0xe4e820, + 0xe5ebe5, 0xe5ebe8, 0xe5eded, 0xe820ef, 0xeae020, 0xebe820, 0xece0f0, + 0xede020, 0xedfbe5, 0xeef1e5, 0xeff0ee, 0xf0eee4, 0xf0fb20, 0xf220f1, + 0xf2e5eb, 0xf2e5f0, 0xf2eee2, 0xf2fb20, 0xfbe520, 0xfef220, 0xff20ef, + 0xfff220, + ], + highBytes: { + total: 678, + counts: [ + [0xb8, 2], + [0xc2, 1], + [0xc3, 1], + [0xc4, 1], + [0xcb, 1], + [0xcd, 3], + [0xce, 1], + [0xcf, 1], + [0xd0, 2], + [0xd1, 1], + [0xd3, 1], + [0xd4, 1], + [0xd7, 1], + [0xe0, 63], + [0xe1, 16], + [0xe2, 26], + [0xe3, 7], + [0xe4, 21], + [0xe5, 50], + [0xe6, 5], + [0xe7, 7], + [0xe8, 47], + [0xe9, 7], + [0xea, 21], + [0xeb, 19], + [0xec, 16], + [0xed, 37], + [0xee, 60], + [0xef, 23], + [0xf0, 41], + [0xf1, 34], + [0xf2, 55], + [0xf3, 14], + [0xf5, 7], + [0xf6, 3], + [0xf7, 4], + [0xf8, 6], + [0xf9, 2], + [0xfb, 26], + [0xfc, 3], + [0xfd, 1], + [0xfe, 13], + [0xff, 27], + ], + }, + }, + ], + }, + { + encoding: 'windows-1252', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x9a, 0x20, 0x9c, 0x20, 0x9e, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x9e, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'da', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206279, 0x2062f8, 0x206465, 0x2066f8, + 0x20686f, 0x206ce6, 0x206f67, 0x2070e5, 0x2073e6, 0x2076e5, 0x2076e6, + 0x20e562, 0x20e662, 0x6166e9, 0x626962, 0x626c69, 0x6272f8, 0x627965, + 0x62f867, 0x62f86e, 0x642062, 0x6420e6, 0x646520, 0x64656e, 0x646572, + 0x652062, 0x652066, 0x6520e5, 0x656420, 0x656e20, 0x656e73, 0x657220, + 0x65726e, 0x657420, 0x66e965, 0x66f872, 0x676572, 0x686ae6, 0x69626c, + 0x696465, 0x696c6c, 0x696f74, 0x6ae66c, 0x6b7374, 0x6c6520, 0x6c696f, + 0x6c6c65, 0x6ce673, 0x6e6465, 0x6e6520, 0x6e6572, 0x6f6720, 0x6f7465, + 0x70e520, 0x722020, 0x722065, 0x72206f, 0x737465, 0x74656b, 0xe52064, + 0xf86765, + ], + highBytes: { + total: 16, + counts: [ + [0xe5, 4], + [0xe6, 5], + [0xe9, 1], + [0xf8, 6], + ], + }, + }, + { + language: 'de', + ngrams: [ + 0x206175, 0x206265, 0x206269, 0x2062e4, 0x2062fc, 0x206465, 0x206469, + 0x2064e4, 0x206569, 0x2066fc, 0x2067e4, 0x206be4, 0x206ce4, 0x207465, + 0x2074fc, 0x20756e, 0x20e470, 0x20f666, 0x20fc62, 0x616368, 0x616d20, + 0x617573, 0x626572, 0x62e463, 0x62fc63, 0x636865, 0x636874, 0x6368fc, + 0x64656e, 0x646572, 0x646965, 0x64e463, 0x652073, 0x652074, 0x65696e, + 0x656dfc, 0x656e20, 0x656efc, 0x657220, 0x65726e, 0x66fc68, 0x66fc72, + 0x67e473, 0x68656e, 0x687265, 0x696368, 0x696520, 0x696e20, 0x696e65, + 0x6c6569, 0x6e2062, 0x6e2064, 0x6e20fc, 0x6e6420, 0x6e656e, 0x736368, + 0x736520, 0x737461, 0x742065, 0x746520, 0x756e64, 0xe46e64, 0xfc6265, + 0xfc7220, + ], + highBytes: { + total: 21, + counts: [ + [0xc4, 1], + [0xdc, 1], + [0xe4, 7], + [0xf6, 1], + [0xfc, 11], + ], + }, + }, + { + language: 'en', + ngrams: [ + 0x206120, 0x20616e, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206661, + 0x206c69, 0x206e65, 0x2070e2, 0x207265, 0x2072e9, 0x2072f4, 0x207468, + 0x207768, 0x616420, 0x616465, 0x6166e9, 0x616e64, 0x61e761, 0x61ef76, + 0x626f75, 0x627261, 0x636974, 0x642063, 0x64206e, 0x646520, 0x646573, + 0x652061, 0x652063, 0x656164, 0x657273, 0x657320, 0x657369, 0x657420, + 0x65f16f, 0x6661e7, 0x66e973, 0x686520, 0x68696c, 0x696272, 0x696465, + 0x696c65, 0x6c6520, 0x6de920, 0x6e61ef, 0x6e6420, 0x7065f1, 0x70e274, + 0x726561, 0x727320, 0x72e973, 0x72f46c, 0x732061, 0x746865, 0x74e920, + 0x756de9, 0xe274e9, 0xe76164, 0xe92020, 0xe92077, 0xe97320, 0xe97375, + 0xef7665, + ], + highBytes: { + total: 9, + counts: [ + [0xe2, 1], + [0xe7, 1], + [0xe9, 4], + [0xef, 1], + [0xf1, 1], + [0xf4, 1], + ], + }, + }, + { + language: 'es', + ngrams: [ + 0x206269, 0x206369, 0x20636c, 0x20636f, 0x2063f3, 0x206465, 0x206469, + 0x20656c, 0x206c61, 0x206c6f, 0x206d65, 0x206de1, 0x207061, 0x2070e1, + 0x2070fa, 0x207265, 0x207375, 0x2073e1, 0x207665, 0x207920, 0x612062, + 0x612065, 0x612070, 0x61646f, 0x616e20, 0x617261, 0x617320, 0x6174e1, + 0x61f161, 0x626c69, 0x6369e9, 0x6369f3, 0x63f364, 0x656c20, 0x656e20, + 0x656efa, 0x6572ed, 0x657320, 0x65f161, 0x65f16f, 0x69e96e, 0x69f36e, + 0x6c206d, 0x6c6120, 0x6c6f73, 0x6d61f1, 0x6de173, 0x6e206c, 0x6e6120, + 0x6efa20, 0x6f206c, 0x6f7265, 0x6f7320, 0x706172, 0x70e167, 0x70fa62, + 0x726573, 0x72ed61, 0x732061, 0x732063, 0x73206c, 0x7365f1, 0x73e162, + 0x746f72, + ], + highBytes: { + total: 13, + counts: [ + [0xe1, 4], + [0xe9, 1], + [0xed, 1], + [0xf1, 3], + [0xf3, 2], + [0xfa, 2], + ], + }, + }, + { + language: 'fr', + ngrams: [ + 0x20636f, 0x206465, 0x206475, 0x2064e9, 0x206574, 0x206c65, 0x207061, + 0x20706f, 0x207072, 0x2070e2, 0x2072e9, 0x20756e, 0x2076e9, 0x20e963, + 0x20e96c, 0x20e971, 0x6166e9, 0x626c69, 0x6368e9, 0x646573, 0x647520, + 0x64e976, 0x652063, 0x652064, 0x65206c, 0x65206d, 0x652070, 0x6520e9, + 0x656e63, 0x656e74, 0x657320, 0x657420, 0x657572, 0x66e973, 0x68e871, + 0x68e920, 0x6c6120, 0x6c6520, 0x6c6573, 0x6c6c65, 0x6de972, 0x6e7420, + 0x6f7572, 0x706172, 0x7072e9, 0x717565, 0x72206c, 0x726520, 0x727320, + 0x72e970, 0x732063, 0x73206c, 0x732070, 0x742064, 0x74206c, 0x7468e8, + 0x756520, 0x756de9, 0x757273, 0xe87175, 0xe96520, 0xe97061, 0xe97269, + 0xe97665, + ], + highBytes: { + total: 22, + counts: [ + [0xe2, 1], + [0xe8, 3], + [0xe9, 18], + ], + }, + }, + { + language: 'it', + ngrams: [ + 0x00006c, 0x00006d, 0x006c61, 0x206269, 0x206361, 0x206369, 0x20636f, + 0x206465, 0x206469, 0x206520, 0x206765, 0x20676c, 0x206920, 0x206c61, + 0x206c65, 0x206d65, 0x207065, 0x207069, 0x207072, 0x207363, 0x207369, + 0x207374, 0x612062, 0x612063, 0x612064, 0x612067, 0x612069, 0x612070, + 0x61746f, 0x6368e9, 0x636974, 0x64656c, 0x656e74, 0x657263, 0x6666e8, + 0x66e820, 0x676c69, 0x68e920, 0x692061, 0x692063, 0x692073, 0x696120, + 0x697474, 0x69f920, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6f20, + 0x6f7269, 0x7069f9, 0x726920, 0x746f72, 0x7474e0, 0x74e020, 0xe02020, + 0xe0206c, 0xe02073, 0xe82061, 0xe8206d, 0xe92073, 0xf92067, 0xf9206c, + 0xf92070, + ], + highBytes: { + total: 9, + counts: [ + [0xe0, 3], + [0xe8, 2], + [0xe9, 1], + [0xf9, 3], + ], + }, + }, + { + language: 'nl', + ngrams: [ + 0x000064, 0x206269, 0x20626f, 0x206361, 0x20636f, 0x206465, 0x206565, + 0x20656e, 0x206765, 0x206865, 0x206c65, 0x206f70, 0x206f76, 0x207374, + 0x207465, 0x207665, 0x207765, 0x20e9e9, 0x61616b, 0x616173, 0x616420, + 0x6166e9, 0x616b74, 0x616d20, 0x626962, 0x626c69, 0x626f65, 0x636166, + 0x636874, 0x646520, 0x652062, 0x652063, 0x652065, 0x652073, 0x65656e, + 0x656572, 0x6565eb, 0x65696e, 0x656b65, 0x656e20, 0x657273, 0x65eb6e, + 0x65ef6e, 0x66e973, 0x6765ef, 0x696e20, 0x6b7420, 0x6e2020, 0x6e206f, + 0x6e20e9, 0x6feb7a, 0x706feb, 0x732065, 0x736368, 0x742064, 0x74656e, + 0x746572, 0x766572, 0xe96e20, 0xe97320, 0xe9e96e, 0xeb6e20, 0xeb7a69, + 0xef6e74, + ], + highBytes: { + total: 7, + counts: [ + [0xe9, 4], + [0xeb, 2], + [0xef, 1], + ], + }, + }, + { + language: 'no', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206272, 0x206279, 0x2062f8, 0x206465, + 0x2064f8, 0x20666f, 0x2066f8, 0x206c61, 0x206ee6, 0x206f67, 0x2070e5, + 0x207665, 0x2076e5, 0x20e570, 0x20e672, 0x626962, 0x626c69, 0x6272f8, + 0x627965, 0x62f86b, 0x62f86e, 0x642065, 0x646520, 0x646572, 0x646574, + 0x64f872, 0x652062, 0x652066, 0x652074, 0x6520e5, 0x6520e6, 0x656420, + 0x656e20, 0x656e65, 0x656e73, 0x657220, 0x657273, 0x657420, 0x66f872, + 0x672073, 0x676520, 0x67656e, 0x676572, 0x69626c, 0x696765, 0x696e67, + 0x696f74, 0x6af865, 0x6cf872, 0x6e6520, 0x6e6572, 0x6ee672, 0x6f6720, + 0x70e520, 0x72206f, 0x7273f8, 0x72f864, 0x737465, 0x74656b, 0xe52064, + 0xf86b65, + ], + highBytes: { + total: 14, + counts: [ + [0xe5, 4], + [0xe6, 2], + [0xf8, 8], + ], + }, + }, + { + language: 'pt', + ngrams: [ + 0x00006d, 0x206120, 0x206269, 0x206369, 0x2063f3, 0x206465, 0x206520, + 0x206573, 0x206e61, 0x206f20, 0x206f73, 0x207065, 0x207072, 0x2070e3, + 0x2073e1, 0x207665, 0x20f46e, 0x612062, 0x612063, 0x612064, 0x612065, + 0x61206f, 0x612070, 0x616465, 0x61646f, 0x6166e9, 0x616d20, 0x617261, + 0x617320, 0x61e761, 0x61e7e3, 0x626962, 0x626c69, 0x636120, 0x6369ea, + 0x63e172, 0x63f364, 0x646120, 0x646520, 0x646f20, 0x652061, 0x6563e1, + 0x657320, 0x657363, 0x66e973, 0x68e320, 0x696120, 0x69e7f5, 0x69ea6e, + 0x6d61e7, 0x6e6120, 0x6e68e3, 0x6f7320, 0x70e36f, 0x7261e7, 0x726573, + 0x7269e7, 0x732020, 0x732061, 0x732070, 0x7320f4, 0x7374f3, 0x73e162, + 0x74f372, + ], + highBytes: { + total: 14, + counts: [ + [0xe1, 2], + [0xe3, 3], + [0xe7, 3], + [0xe9, 1], + [0xea, 1], + [0xf3, 2], + [0xf4, 1], + [0xf5, 1], + ], + }, + }, + { + language: 'sv', + ngrams: [ + 0x000064, 0x006cf6, 0x206269, 0x2062f6, 0x206465, 0x2066e4, 0x2066f6, + 0x206c69, 0x206ce4, 0x206ce5, 0x206de4, 0x206f63, 0x2070e5, 0x20736b, + 0x207374, 0x2073e4, 0x2073f6, 0x207479, 0x207665, 0x2076e4, 0x20e470, + 0x20f670, 0x612020, 0x612062, 0x612074, 0x6120f6, 0x616420, 0x616465, + 0x6166e9, 0x617220, 0x617265, 0x626962, 0x626c69, 0x6272f6, 0x62f663, + 0x62f66e, 0x636820, 0x6420e4, 0x646520, 0x64656e, 0x646574, 0x656e20, + 0x657220, 0x65726e, 0x6573f6, 0x657420, 0x66f672, 0x676120, 0x69626c, + 0x696761, 0x696f74, 0x6b6170, 0x6b6172, 0x6e6120, 0x6f6368, 0x70e520, + 0x726e61, 0x736b61, 0x737461, 0x73f66b, 0x74206c, 0x746164, 0x766572, + 0xe52064, + ], + highBytes: { + total: 20, + counts: [ + [0xe4, 7], + [0xe5, 3], + [0xe9, 1], + [0xf6, 9], + ], + }, + }, + ], + }, + { + encoding: 'windows-1253', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xdc, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'el', + ngrams: [ + 0x0000de, 0x0000f0, 0x0000f3, 0x0000f8, 0x00def3, 0x00f0f1, 0x00f3e1, + 0x00f8e7, 0x20dced, 0x20e1e3, 0x20e1ed, 0x20e1f1, 0x20e2e9, 0x20e2ef, + 0x20e3f1, 0x20e5eb, 0x20e5f0, 0x20e5f1, 0x20e720, 0x20eae1, 0x20ece9, + 0x20efe9, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xdc20f4, 0xdfe120, + 0xe12020, 0xe120ea, 0xe1e920, 0xe2e9e2, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, + 0xe5e920, 0xe5f220, 0xe720e2, 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9e2eb, + 0xe9efe8, 0xe9f3f4, 0xeae1e9, 0xeae720, 0xebe720, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xeff5ed, 0xf0fceb, 0xf1dc20, 0xf1e9ef, + 0xf220ea, 0xf3f4de, 0xf3f4e7, 0xf4e120, 0xf4e5f2, 0xf4e7ed, 0xf5ed20, + 0xfcebe7, + ], + highBytes: { + total: 327, + counts: [ + [0xb9, 1], + [0xc7, 1], + [0xcc, 1], + [0xcf, 2], + [0xd0, 1], + [0xd3, 1], + [0xd8, 1], + [0xdc, 13], + [0xdd, 5], + [0xde, 4], + [0xdf, 9], + [0xe1, 31], + [0xe2, 9], + [0xe3, 9], + [0xe4, 4], + [0xe5, 16], + [0xe6, 1], + [0xe7, 14], + [0xe8, 6], + [0xe9, 31], + [0xea, 16], + [0xeb, 10], + [0xec, 11], + [0xed, 15], + [0xee, 1], + [0xef, 24], + [0xf0, 8], + [0xf1, 16], + [0xf2, 7], + [0xf3, 8], + [0xf4, 17], + [0xf5, 10], + [0xf6, 5], + [0xf7, 3], + [0xf8, 2], + [0xf9, 5], + [0xfc, 5], + [0xfd, 2], + [0xfe, 2], + ], + }, + }, + ], + }, + { + encoding: 'windows-1254', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x9a, 0x20, 0x9c, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'tr', + ngrams: [ + 0x000063, 0x000064, 0x000073, 0x0000fe, 0x006375, 0x00fe65, 0x2061e7, + 0x206269, 0x20656b, 0x206b61, 0x206bfc, 0x206dfc, 0x207361, 0x207461, + 0x207665, 0x207961, 0x20e769, 0x20f6f0, 0x20fe65, 0x616261, 0x616c20, + 0x616c61, 0x616cfd, 0x616e65, 0x616efd, 0x617220, 0x6172fd, 0x6174f6, + 0x617a61, 0x617afd, 0x61e7fd, 0x61fee7, 0x62616c, 0x652062, 0x656869, + 0x657220, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697220, 0x697461, + 0x6bfc74, 0x6c6172, 0x6c6572, 0x706861, 0x722020, 0x72206b, 0x722076, + 0x722079, 0x726920, 0x726c61, 0x72fd20, 0x746172, 0x74fc70, 0x766520, + 0x7a6172, 0x7a6520, 0xe7fd6b, 0xfc7068, 0xfc74fc, 0xfd6b20, 0xfd7220, + 0xfe6568, + ], + highBytes: { + total: 32, + counts: [ + [0xc7, 1], + [0xde, 2], + [0xe7, 6], + [0xf0, 1], + [0xf6, 2], + [0xfc, 8], + [0xfd, 11], + [0xfe, 1], + ], + }, + }, + ], + }, + { + encoding: 'windows-1255', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xd4, 0xd5, 0xd6, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'he', + ngrams: [ + 0x0000e1, 0x0000e4, 0x0000f1, 0x0000f9, 0x00e1e5, 0x00e4f1, 0x00f1e3, + 0x00f9e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, 0x20e1fa, + 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f7, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e7f7, 0x20e8f8, + 0x20ece0, 0x20ece1, 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, + 0x20eee7, 0x20eeeb, 0xe0e9ed, 0xe4f1f4, 0xe5f8e9, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed20, 0xe9f820, 0xe9fa20, 0xed2020, 0xed20ec, + 0xed20ee, 0xef20ee, 0xf0e420, 0xf1f4f8, 0xf2e9f8, 0xf4f8e9, 0xf7e5f8, + 0xf820e4, 0xf820ee, 0xf8e5fa, 0xf8e920, 0xf8e9ed, 0xf8fa20, 0xf9e9ed, + 0xfa20ec, + ], + highBytes: { + total: 249, + counts: [ + [0xe0, 9], + [0xe1, 12], + [0xe2, 2], + [0xe3, 8], + [0xe4, 16], + [0xe5, 30], + [0xe6, 2], + [0xe7, 6], + [0xe8, 7], + [0xe9, 33], + [0xea, 1], + [0xeb, 6], + [0xec, 9], + [0xed, 13], + [0xee, 11], + [0xef, 3], + [0xf0, 6], + [0xf1, 7], + [0xf2, 6], + [0xf4, 7], + [0xf6, 1], + [0xf7, 10], + [0xf8, 22], + [0xf9, 7], + [0xfa, 15], + ], + }, + }, + ], + }, + { + encoding: 'windows-1256', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x81, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x8a, 0x20, 0x9c, 0x8d, 0x8e, 0x8f, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x98, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x20, 0x9f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0x20, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0x20, 0x20, 0x20, 0x20, 0xf4, 0x20, 0x20, 0x20, 0x20, 0xf9, 0x20, 0xfb, + 0xfc, 0x20, 0x20, 0xff, + ], + languages: [ + { + language: 'ar', + ngrams: [ + 0x0000c7, 0x0000d3, 0x0000d5, 0x0000e6, 0x00c7e1, 0x00d3e6, 0x00d5c8, + 0x00e6d1, 0x20c3c8, 0x20c3e3, 0x20c7ce, 0x20c7e1, 0x20c8e5, 0x20c8ed, + 0x20cad3, 0x20cfe1, 0x20d1de, 0x20d5db, 0x20dae4, 0x20ddd1, 0x20dded, + 0x20dfca, 0x20e1e1, 0x20e6c7, 0x20e6ca, 0x20e6d5, 0x20e6ed, 0xc7ca20, + 0xc7ccda, 0xc7d120, 0xc7e1c3, 0xc7e1da, 0xc7e1e3, 0xc7e1e6, 0xc820c7, + 0xc8c920, 0xc92020, 0xc920c7, 0xc920e6, 0xc920ed, 0xca20e6, 0xcac820, + 0xcac8c9, 0xcb20c7, 0xccda20, 0xcd20c7, 0xcfede4, 0xd120c7, 0xd1c7cc, + 0xd5ddc7, 0xda20c7, 0xdfcac8, 0xe1e3cf, 0xe1e3df, 0xe3cfed, 0xe3dfca, + 0xe420c7, 0xe4c7d3, 0xe4c920, 0xe6c7e1, 0xe6d5dd, 0xe6e420, 0xedc920, + 0xede4c9, + ], + highBytes: { + total: 324, + counts: [ + [0xc1, 2], + [0xc3, 4], + [0xc4, 1], + [0xc6, 1], + [0xc7, 56], + [0xc8, 17], + [0xc9, 11], + [0xca, 16], + [0xcb, 2], + [0xcc, 3], + [0xcd, 8], + [0xce, 4], + [0xcf, 9], + [0xd1, 19], + [0xd2, 3], + [0xd3, 7], + [0xd4, 2], + [0xd5, 5], + [0xd6, 2], + [0xd8, 1], + [0xd9, 2], + [0xda, 8], + [0xdb, 1], + [0xdd, 9], + [0xde, 7], + [0xdf, 5], + [0xe1, 36], + [0xe3, 18], + [0xe4, 12], + [0xe5, 5], + [0xe6, 24], + [0xed, 24], + ], + }, + }, + ], + }, + { + encoding: 'windows-1257', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x8e, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'et', + ngrams: [ + 0x206a61, 0x206b69, 0x206b6f, 0x206c69, 0x206d65, 0x206dfc, 0x207261, + 0x2074f6, 0x2076e4, 0x20e472, 0x20f570, 0x20f575, 0x6120f5, 0x61616d, + 0x616220, 0x616420, 0x616d61, 0x617475, 0x617661, 0x6220f5, 0x626120, + 0x642061, 0x64206b, 0x64206f, 0x647573, 0x652074, 0x656420, 0x657369, + 0x692020, 0x696420, 0x696b75, 0x696d65, 0x696e6e, 0x69726a, 0x697461, + 0x6a6120, 0x6b6972, 0x6b6f67, 0x6b6f6e, 0x6b6f6f, 0x6b7564, 0x6c6173, + 0x6d6174, 0x6dfcfc, 0x6e20e4, 0x6e6520, 0x70e465, 0x726161, 0x736920, + 0x746162, 0x74f6f6, 0x7570e4, 0x766164, 0x76e469, 0x76e46c, 0x76e472, + 0xe46576, 0xe4696b, 0xe46c6a, 0xe4726b, 0xe47273, 0xf57069, 0xf5756e, + 0xf67475, + ], + highBytes: { + total: 11, + counts: [ + [0xe4, 5], + [0xf5, 2], + [0xf6, 2], + [0xfc, 2], + ], + }, + }, + { + language: 'lv', + ngrams: [ + 0x0072ee, 0x206269, 0x206365, 0x206461, 0x206469, 0x2064e2, 0x206761, + 0x20697a, 0x206b6f, 0x206bef, 0x206c61, 0x206ce7, 0x206d61, 0x206d65, + 0x206de2, 0x207069, 0x2070e2, 0x207261, 0x207265, 0x20756e, 0x61206b, + 0x617320, 0x626962, 0x626c69, 0x6365ef, 0x646120, 0x656d20, 0x692070, + 0x692073, 0x69626c, 0x69656d, 0x696f74, 0x6974e2, 0x6a6920, 0x6b7374, + 0x6be272, 0x6c6173, 0x6c696f, 0x6c73e7, 0x6c7573, 0x6ce76e, 0x6e6920, + 0x6eee63, 0x6f74e7, 0x70696c, 0x70e272, 0x726920, 0x732020, 0x732061, + 0x73206c, 0x73206d, 0x732075, 0x73e774, 0x74e26a, 0x74e26c, 0x74e76b, + 0x756e20, 0x757320, 0x76e76c, 0x7a76e7, 0xe26a69, 0xe76e69, 0xe774e2, + 0xee6361, + ], + highBytes: { + total: 52, + counts: [ + [0xe2, 24], + [0xe7, 13], + [0xe8, 1], + [0xec, 1], + [0xee, 4], + [0xef, 3], + [0xf0, 2], + [0xf2, 2], + [0xfb, 1], + [0xfe, 1], + ], + }, + }, + { + language: 'lt', + ngrams: [ + 0x0000f0, 0x00f065, 0x206169, 0x206170, 0x206269, 0x206461, 0x206972, + 0x207072, 0x207265, 0x20736b, 0x2073fb, 0x20f069, 0x20f076, 0x20fb6b, + 0x20fe69, 0x20fe6d, 0x612070, 0x616920, 0x616974, 0x6169f0, 0x6172f0, + 0x6172fe, 0x617320, 0x617573, 0x61f06f, 0x61f079, 0x626962, 0x626c69, + 0x646120, 0x64e020, 0x656461, 0x656e69, 0x67f820, 0x692072, 0x696120, + 0x696169, 0x69626c, 0x696573, 0x696e69, 0x696eeb, 0x696f74, 0x697220, + 0x6a6169, 0x6b6169, 0x6b696e, 0x6c696f, 0x6d656e, 0x6e696e, 0x6eeb73, + 0x6f20fe, 0x6f64e0, 0x6f7320, 0x6f7465, 0x7261f0, 0x726961, 0x732020, + 0x732061, 0x736b61, 0x74656b, 0x757320, 0xe0206d, 0xeb6a61, 0xeb7320, + 0xf07669, + ], + highBytes: { + total: 40, + counts: [ + [0xd0, 1], + [0xdb, 1], + [0xe0, 5], + [0xe1, 1], + [0xe6, 1], + [0xeb, 8], + [0xf0, 11], + [0xf8, 4], + [0xfb, 2], + [0xfe, 6], + ], + }, + }, + ], + }, + { + encoding: 'windows-1258', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x20, 0x20, 0x9c, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x9c, 0x20, 0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0x20, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0x20, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0x20, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0x20, 0xff, + ], + languages: [ + { + language: 'vi', + ngrams: [ + 0x206261, 0x2062e1, 0x206320, 0x206368, 0x2064e2, 0x206769, 0x20686f, + 0x206920, 0x206c69, 0x206d20, 0x206d61, 0x206df4, 0x206e20, 0x206e67, + 0x206e68, 0x207068, 0x207175, 0x2073e1, 0x207420, 0x207461, 0x207468, + 0x207469, 0x207472, 0x2074e0, 0x207520, 0x207669, 0x2076e0, 0x207920, + 0x20f0f4, 0x612069, 0x612076, 0x626120, 0x62e16e, 0x636820, 0x6368f5, + 0x64e220, 0x672064, 0x67206b, 0x672074, 0x6720f0, 0x682070, 0x686f20, + 0x68e06e, 0x68f420, 0x68f520, 0x68fd20, 0x692063, 0x69206e, 0x692071, + 0x69ea20, 0x6df420, 0x6e2062, 0x6e2074, 0x6e6720, 0x6e6820, 0x6f2063, + 0x6f206e, 0x7068f4, 0x7468e0, 0x7468fd, 0x76e020, 0xf42074, 0xf5206e, + 0xfdf520, + ], + highBytes: { + total: 116, + counts: [ + [0xcc, 7], + [0xd0, 1], + [0xd2, 13], + [0xde, 6], + [0xe0, 11], + [0xe1, 6], + [0xe2, 7], + [0xea, 8], + [0xec, 9], + [0xf0, 3], + [0xf2, 15], + [0xf3, 1], + [0xf4, 10], + [0xf5, 7], + [0xfa, 2], + [0xfd, 10], + ], + }, + }, + ], + }, + { + encoding: 'windows-874', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0x20, 0xd2, 0xd3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'th', + ngrams: [ + 0x0000b5, 0x0000cb, 0x0000e0, 0x00b5c5, 0x00cb20, 0x00e0aa, 0x20a120, + 0x20a1a4, 0x20a1a8, 0x20a1c9, 0x20a1e0, 0x20a420, 0x20a4b9, 0x20a7ca, + 0x20a7e3, 0x20a820, 0x20aa20, 0x20aad2, 0x20b3b1, 0x20b4b7, 0x20b4bb, + 0x20b4e0, 0x20b520, 0x20b5c3, 0x20b720, 0x20b7c2, 0x20b8c0, 0x20b920, + 0x20b9cb, 0x20b9e0, 0x20bac3, 0x20bad2, 0x20bc20, 0x20bcc5, 0x20be20, + 0x20c120, 0x20c1e0, 0x20c2b9, 0x20c2ba, 0x20c2c7, 0x20c320, 0x20c520, + 0x20c7c2, 0x20c8d2, 0x20ca20, 0x20cda7, 0x20d2b9, 0x20e1c5, 0xa12020, + 0xa420a1, 0xb5c320, 0xbbc3d0, 0xc120cd, 0xc8d2ca, 0xca20e1, 0xcab5c3, + 0xcad2c3, 0xcb20cd, 0xd0c720, 0xd2c320, 0xd2cab5, 0xe0c120, 0xe0c520, + 0xe1c5d0, + ], + highBytes: { + total: 301, + counts: [ + [0xa1, 8], + [0xa2, 4], + [0xa4, 7], + [0xa7, 10], + [0xa8, 4], + [0xaa, 5], + [0xb1, 1], + [0xb3, 2], + [0xb4, 8], + [0xb5, 7], + [0xb6, 1], + [0xb7, 8], + [0xb8, 2], + [0xb9, 18], + [0xba, 4], + [0xbb, 4], + [0xbc, 4], + [0xbe, 2], + [0xbf, 1], + [0xc0, 1], + [0xc1, 8], + [0xc2, 10], + [0xc3, 16], + [0xc5, 11], + [0xc7, 7], + [0xc8, 2], + [0xc9, 1], + [0xca, 10], + [0xcb, 5], + [0xcd, 12], + [0xd0, 6], + [0xd1, 12], + [0xd2, 18], + [0xd3, 4], + [0xd4, 8], + [0xd5, 8], + [0xd6, 2], + [0xd7, 6], + [0xd8, 1], + [0xd9, 5], + [0xe0, 12], + [0xe1, 6], + [0xe2, 1], + [0xe3, 2], + [0xe4, 1], + [0xe6, 1], + [0xe7, 1], + [0xe8, 8], + [0xe9, 11], + [0xec, 5], + ], + }, + }, + ], + }, + { + encoding: 'KOI8-R', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, + 0xdc, 0xdd, 0xde, 0xdf, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000d2, 0x00d2c1, 0x20c120, 0x20c2c9, 0x20c7cf, 0x20c920, 0x20cbcf, + 0x20cdc1, 0x20cec1, 0x20cfc2, 0x20cfd3, 0x20cfd4, 0x20d0cc, 0x20d0cf, + 0x20d0d2, 0x20d1d2, 0x20d2c5, 0x20d3cf, 0x20d720, 0x20d7cf, 0x20d7d9, + 0x20dec9, 0xc0d420, 0xc120ce, 0xc120d0, 0xc120d7, 0xc1c0d4, 0xc1c5d4, + 0xc1cec9, 0xc1d120, 0xc1d2cb, 0xc1d3d4, 0xc1d4c5, 0xc2c9c2, 0xc2ccc9, + 0xc2d3d5, 0xc4c1c0, 0xc4c920, 0xc5ccc5, 0xc5ccc9, 0xc5cece, 0xc5d2c9, + 0xc5d2d9, 0xc5d3d4, 0xc7cfd2, 0xc7cfd4, 0xc920d0, 0xcbc120, 0xccc920, + 0xcdc1d2, 0xcec120, 0xced9c5, 0xcfd3c5, 0xd0d2cf, 0xd120d0, 0xd1d420, + 0xd2cfc4, 0xd2d920, 0xd420d3, 0xd4c5cc, 0xd4c5d2, 0xd4cfd7, 0xd4d920, + 0xd9c520, + ], + highBytes: { + total: 678, + counts: [ + [0xa3, 2], + [0xc0, 13], + [0xc1, 63], + [0xc2, 16], + [0xc3, 3], + [0xc4, 21], + [0xc5, 50], + [0xc7, 7], + [0xc8, 7], + [0xc9, 47], + [0xca, 7], + [0xcb, 21], + [0xcc, 19], + [0xcd, 16], + [0xce, 37], + [0xcf, 60], + [0xd0, 23], + [0xd1, 27], + [0xd2, 41], + [0xd3, 34], + [0xd4, 55], + [0xd5, 14], + [0xd6, 5], + [0xd7, 26], + [0xd8, 3], + [0xd9, 26], + [0xda, 7], + [0xdb, 6], + [0xdc, 1], + [0xdd, 2], + [0xde, 4], + [0xe4, 1], + [0xe6, 1], + [0xe7, 1], + [0xec, 1], + [0xee, 3], + [0xef, 1], + [0xf0, 1], + [0xf2, 2], + [0xf3, 1], + [0xf5, 1], + [0xf7, 1], + [0xfe, 1], + ], + }, + }, + ], + }, { encoding: 'KOI8-U', byteMap: [ @@ -1014,26 +3004,27 @@ export const generatedSBCSModels = [ { language: 'ro', ngrams: [ - 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, - 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, - 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, - 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, - 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, - 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, - 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, - 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, - 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, - 0xfe6120, + 0x206361, 0x206375, 0x2063e3, 0x206465, 0x206573, 0x206d69, 0x206f72, + 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, 0x616c20, 0x617ae3, + 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, 0x646520, 0x646573, + 0x652020, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, 0x656c65, + 0x657269, 0x657363, 0x657374, 0x65ba74, 0x65fe75, 0x692063, 0x6961fe, + 0x69626c, 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, + 0x6c2065, 0x6c6520, 0x6c696f, 0x6d62e3, 0x6f7269, 0x707265, 0x7261ba, + 0x726520, 0x7265fe, 0x726969, 0x737465, 0x746520, 0x746f72, 0x74e320, + 0x756c20, 0x7ae320, 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xfe6120, + 0xfe756c, ], highBytes: { - total: 49, + total: 57, counts: [ + [0xa4, 4], [0xba, 12], [0xce, 1], [0xe2, 3], [0xe3, 21], [0xee, 4], - [0xfe, 8], + [0xfe, 12], ], }, }, @@ -1199,3 +3190,151 @@ export const generatedSBCSModels = [ ], }, ] as const; + +export const generatedMBCSModels: readonly GeneratedMBCSModel[] = [ + { + encoding: 'Shift_JIS', + language: 'ja', + total: 238, + commonCharacters: [ + 0x00008141, 0x00008142, 0x0000815b, 0x000082a2, 0x000082a4, 0x000082a6, + 0x000082a9, 0x000082aa, 0x000082ab, 0x000082ad, 0x000082af, 0x000082b1, + 0x000082b3, 0x000082b5, 0x000082b7, 0x000082bd, 0x000082c1, 0x000082c2, + 0x000082c4, 0x000082c5, 0x000082c6, 0x000082c8, 0x000082c9, 0x000082cc, + 0x000082cd, 0x000082d4, 0x000082d7, 0x000082dc, 0x000082df, 0x000082e2, + 0x000082e4, 0x000082e8, 0x000082e9, 0x000082f0, 0x00008352, 0x00008357, + 0x00008358, 0x0000835e, 0x00008360, 0x00008366, 0x00008368, 0x0000836f, + 0x00008370, 0x00008379, 0x00008380, 0x0000838b, 0x00008393, 0x000088c4, + 0x000088e6, 0x00008977, 0x000089b9, 0x000089c6, 0x000089c8, 0x000089ca, + 0x000089e6, 0x00008a4a, 0x00008a58, 0x00008a6d, 0x00008a6f, 0x00008a77, + 0x00008ad9, 0x00008b47, 0x00008b69, 0x00008b71, 0x00008bce, 0x00008c76, + 0x00008d48, 0x00008d4c, 0x00008d73, 0x00008dc5, 0x00008dd8, 0x00008dec, + 0x00008e69, 0x00008e6a, 0x00008e6e, 0x00008e73, 0x00008ed2, 0x00008ee8, + 0x00008f57, 0x00008f89, 0x00008f91, 0x00008fac, 0x00008fc4, 0x00008fea, + 0x00009056, 0x0000906c, 0x0000907d, 0x000090ae, 0x000090b6, 0x000090c3, + 0x000090dd, 0x000090df, 0x000090e0, 0x0000914e, 0x00009196, 0x00009254, + 0x0000926e, 0x00009283, 0x000092a9, 0x000092b2, 0x000092ca, 0x00009358, + 0x00009360, 0x00009379, 0x000093c7, 0x000093e0, 0x000093fa, 0x00009446, + 0x0000945f, 0x00009483, 0x0000948e, 0x000094ad, 0x000094e0, 0x000095a8, + 0x000095aa, 0x000095b7, 0x000095c0, 0x000095d2, 0x00009649, 0x0000965b, + 0x0000967b, 0x000096a8, 0x000096be, 0x000096ca, 0x000096da, 0x000096ec, + 0x0000976a, 0x0000979d, + ], + }, + { + encoding: 'EUC-JP', + language: 'ja', + total: 238, + commonCharacters: [ + 0x0000a1a2, 0x0000a1a3, 0x0000a1bc, 0x0000a4a4, 0x0000a4a6, 0x0000a4a8, + 0x0000a4ab, 0x0000a4ac, 0x0000a4ad, 0x0000a4af, 0x0000a4b1, 0x0000a4b3, + 0x0000a4b5, 0x0000a4b7, 0x0000a4b9, 0x0000a4bf, 0x0000a4c3, 0x0000a4c4, + 0x0000a4c6, 0x0000a4c7, 0x0000a4c8, 0x0000a4ca, 0x0000a4cb, 0x0000a4ce, + 0x0000a4cf, 0x0000a4d6, 0x0000a4d9, 0x0000a4de, 0x0000a4e1, 0x0000a4e4, + 0x0000a4e6, 0x0000a4ea, 0x0000a4eb, 0x0000a4f2, 0x0000a5b3, 0x0000a5b8, + 0x0000a5b9, 0x0000a5bf, 0x0000a5c1, 0x0000a5c7, 0x0000a5c9, 0x0000a5d0, + 0x0000a5d1, 0x0000a5da, 0x0000a5e0, 0x0000a5eb, 0x0000a5f3, 0x0000b0c6, + 0x0000b0e8, 0x0000b1d8, 0x0000b2bb, 0x0000b2c8, 0x0000b2ca, 0x0000b2cc, + 0x0000b2e8, 0x0000b3ab, 0x0000b3b9, 0x0000b3ce, 0x0000b3d0, 0x0000b3d8, + 0x0000b4db, 0x0000b5a8, 0x0000b5ca, 0x0000b5d2, 0x0000b6d0, 0x0000b7d7, + 0x0000b9a9, 0x0000b9ad, 0x0000b9d4, 0x0000bac7, 0x0000bada, 0x0000baee, + 0x0000bbca, 0x0000bbcb, 0x0000bbcf, 0x0000bbd4, 0x0000bcd4, 0x0000bcea, + 0x0000bdb8, 0x0000bde9, 0x0000bdf1, 0x0000beae, 0x0000bec6, 0x0000beec, + 0x0000bfb7, 0x0000bfcd, 0x0000bfde, 0x0000c0b0, 0x0000c0b8, 0x0000c0c5, + 0x0000c0df, 0x0000c0e1, 0x0000c0e2, 0x0000c1af, 0x0000c1f6, 0x0000c3b5, + 0x0000c3cf, 0x0000c3e3, 0x0000c4ab, 0x0000c4b4, 0x0000c4cc, 0x0000c5b9, + 0x0000c5c1, 0x0000c5da, 0x0000c6c9, 0x0000c6e2, 0x0000c6fc, 0x0000c7a7, + 0x0000c7c0, 0x0000c7e3, 0x0000c7ee, 0x0000c8af, 0x0000c8e2, 0x0000caaa, + 0x0000caac, 0x0000cab9, 0x0000cac2, 0x0000cad4, 0x0000cbaa, 0x0000cbbc, + 0x0000cbdc, 0x0000ccaa, 0x0000ccc0, 0x0000cccc, 0x0000ccdc, 0x0000ccee, + 0x0000cdcb, 0x0000cdfd, + ], + }, + { + encoding: 'Big5', + language: 'zh', + total: 199, + commonCharacters: [ + 0x0000a141, 0x0000a142, 0x0000a143, 0x0000a440, 0x0000a446, 0x0000a448, + 0x0000a457, 0x0000a46a, 0x0000a470, 0x0000a475, 0x0000a4a3, 0x0000a4bb, + 0x0000a4bd, 0x0000a4e8, 0x0000a4f4, 0x0000a55d, 0x0000a575, 0x0000a576, + 0x0000a5ab, 0x0000a5b4, 0x0000a5bf, 0x0000a5c1, 0x0000a5cd, 0x0000a5e6, + 0x0000a640, 0x0000a650, 0x0000a656, 0x0000a662, 0x0000a677, 0x0000a67e, + 0x0000a6a1, 0x0000a6a8, 0x0000a6ac, 0x0000a6ad, 0x0000a6b3, 0x0000a6e6, + 0x0000a6ec, 0x0000a740, 0x0000a755, 0x0000a7e4, 0x0000a854, 0x0000a8ae, + 0x0000a8c3, 0x0000a8d3, 0x0000a940, 0x0000a94d, 0x0000a96c, 0x0000a9b1, + 0x0000a9fa, 0x0000aa47, 0x0000aa6b, 0x0000aaab, 0x0000aaba, 0x0000aac0, + 0x0000aacc, 0x0000aaf9, 0x0000ab6e, 0x0000abb0, 0x0000abc7, 0x0000abc8, + 0x0000abfc, 0x0000ac50, 0x0000ac64, 0x0000ac79, 0x0000acb0, 0x0000acec, + 0x0000acef, 0x0000ad70, 0x0000adb1, 0x0000adb5, 0x0000adb6, 0x0000add3, + 0x0000adfb, 0x0000ae61, 0x0000ae76, 0x0000aec6, 0x0000aec8, 0x0000aed1, + 0x0000af5a, 0x0000afb8, 0x0000b0b5, 0x0000b0cf, 0x0000b0d1, 0x0000b0d8, + 0x0000b0dd, 0x0000b160, 0x0000b1e1, 0x0000b24d, 0x0000b27a, 0x0000b2c4, + 0x0000b35d, 0x0000b3d5, 0x0000b3f5, 0x0000b44d, 0x0000b4c1, 0x0000b56f, + 0x0000b57b, 0x0000b5e6, 0x0000b5f3, 0x0000b67d, 0x0000b6a4, 0x0000b6b0, + 0x0000b773, 0x0000b7a1, 0x0000b8c1, 0x0000b8cc, 0x0000b8df, 0x0000b8ea, + 0x0000b941, 0x0000b944, 0x0000b94c, 0x0000b9ce, 0x0000b9cf, 0x0000ba43, + 0x0000baa1, 0x0000bade, 0x0000bb45, 0x0000bb65, 0x0000bb73, 0x0000bba1, + 0x0000bc67, 0x0000bc73, 0x0000bcb6, 0x0000bcc6, 0x0000bd73, 0x0000bec7, + 0x0000c052, 0x0000c05d, + ], + }, + { + encoding: 'GB18030', + language: 'zh', + total: 199, + commonCharacters: [ + 0x0000814b, 0x000081ed, 0x00008280, 0x0000855e, 0x000085a2, 0x00008654, + 0x00008696, 0x00008844, 0x00008846, 0x000088f6, 0x00008c57, 0x00008c91, + 0x00008ca4, 0x00008e9f, 0x00008ecd, 0x00008f56, 0x0000945b, 0x000094b5, + 0x000095f8, 0x0000997a, 0x00009a76, 0x00009d4d, 0x00009ee9, 0x0000a1a2, + 0x0000a1a3, 0x0000a3ac, 0x0000b06c, 0x0000b0b2, 0x0000b0e0, 0x0000b0fc, + 0x0000b2a9, 0x0000b2bb, 0x0000b2cb, 0x0000b2e9, 0x0000b3a3, 0x0000b3bf, + 0x0000b3c7, 0x0000b3c9, 0x0000b3cc, 0x0000b3fe, 0x0000b4a9, 0x0000b4f2, + 0x0000b4f3, 0x0000b5c0, 0x0000b5c4, 0x0000b5da, 0x0000b5ea, 0x0000b7a8, + 0x0000b7ad, 0x0000b7bd, 0x0000b7c8, 0x0000b7e4, 0x0000b9a4, 0x0000b9ab, + 0x0000b9b2, 0x0000b9dc, 0x0000b9fb, 0x0000bacd, 0x0000bcaf, 0x0000bcd2, + 0x0000bdbb, 0x0000bdd6, 0x0000be8e, 0x0000bedb, 0x0000bfa7, 0x0000bfc6, + 0x0000bfcd, 0x0000c0ed, 0x0000c1cb, 0x0000c1cf, 0x0000c1f7, 0x0000c1f9, + 0x0000c295, 0x0000c2c3, 0x0000c2fd, 0x0000c3db, 0x0000c3e6, 0x0000c3f1, + 0x0000c3f7, 0x0000c4cf, 0x0000c4ea, 0x0000c6da, 0x0000c6fb, 0x0000c7e5, + 0x0000c8cb, 0x0000c9cf, 0x0000c9e7, 0x0000c9fa, 0x0000cab7, 0x0000cabc, + 0x0000cabd, 0x0000cad0, 0x0000cad2, 0x0000cad5, 0x0000cadf, 0x0000cbae, + 0x0000cdac, 0x0000cebb, 0x0000ceef, 0x0000cff2, 0x0000d0a1, 0x0000d0c2, + 0x0000d0c7, 0x0000d0d0, 0x0000d0d1, 0x0000d165, 0x0000d175, 0x0000d2bb, + 0x0000d2f4, 0x0000d35e, 0x0000d38b, 0x0000d3d0, 0x0000d44f, 0x0000d483, + 0x0000d4da, 0x0000d4e7, 0x0000d566, 0x0000d5be, 0x0000d5d2, 0x0000d5df, + 0x0000d5fb, 0x0000d5fd, 0x0000d6b8, 0x0000d7f7, 0x0000dc87, 0x0000e95f, + 0x0000ec6f, 0x0000f05e, + ], + }, + { + encoding: 'EUC-KR', + language: 'ko', + total: 266, + commonCharacters: [ + 0x0000b0a1, 0x0000b0ab, 0x0000b0b3, 0x0000b0b4, 0x0000b0c7, 0x0000b0cb, + 0x0000b0d4, 0x0000b0e8, 0x0000b0ed, 0x0000b0fa, 0x0000b0fc, 0x0000b1a4, + 0x0000b1b8, 0x0000b1d7, 0x0000b1db, 0x0000b1e2, 0x0000b2dc, 0x0000b3aa, + 0x0000b3bb, 0x0000b3ca, 0x0000b3d1, 0x0000b3d7, 0x0000b3f3, 0x0000b3f5, + 0x0000b4a9, 0x0000b4ba, 0x0000b4c2, 0x0000b4d4, 0x0000b4d9, 0x0000b4eb, + 0x0000b5b5, 0x0000b5b6, 0x0000b5bf, 0x0000b5e5, 0x0000b5e7, 0x0000b5e9, + 0x0000b5f0, 0x0000b6f7, 0x0000b7a5, 0x0000b7ce, 0x0000b8a3, 0x0000b8a6, + 0x0000b8ae, 0x0000b8b0, 0x0000b8b8, 0x0000b8c2, 0x0000b8de, 0x0000b8e7, + 0x0000b8ed, 0x0000b9ae, 0x0000b9b0, 0x0000b9da, 0x0000b9df, 0x0000b9e0, + 0x0000b9e6, 0x0000b9f6, 0x0000b9f8, 0x0000b9fd, 0x0000bace, 0x0000baf1, + 0x0000bafb, 0x0000bba7, 0x0000bbe7, 0x0000bbfd, 0x0000bcad, 0x0000bcb3, + 0x0000bcd2, 0x0000bcd5, 0x0000bdac, 0x0000bdba, 0x0000bdc3, 0x0000bdc7, + 0x0000beb4, 0x0000bec6, 0x0000bec8, 0x0000bedf, 0x0000bef7, 0x0000bfa1, + 0x0000bfa9, 0x0000bfaa, 0x0000bfad, 0x0000bfcd, 0x0000bfe4, 0x0000bfeb, + 0x0000bfee, 0x0000c0a7, 0x0000c0ba, 0x0000c0bb, 0x0000c0c7, 0x0000c0cc, + 0x0000c0ce, 0x0000c0cf, 0x0000c0d0, 0x0000c0da, 0x0000c0db, 0x0000c0dc, + 0x0000c0e5, 0x0000c0fc, 0x0000c0fd, 0x0000c1a4, 0x0000c1b6, 0x0000c1d8, + 0x0000c1ee, 0x0000c1f6, 0x0000c1f8, 0x0000c1fd, 0x0000c3a3, 0x0000c3a4, + 0x0000c3a5, 0x0000c3b5, 0x0000c3b9, 0x0000c4a1, 0x0000c4a7, 0x0000c4ab, + 0x0000c4da, 0x0000c5d0, 0x0000c5e4, 0x0000c6c0, 0x0000c6e4, 0x0000c6ed, + 0x0000c7c1, 0x0000c7cf, 0x0000c7d0, 0x0000c7d1, 0x0000c7de, 0x0000c7e0, + 0x0000c8ad, 0x0000c8ae, + ], + }, +]; diff --git a/src/encoding/models/types.ts b/src/encoding/models/types.ts new file mode 100644 index 0000000..93b4fbd --- /dev/null +++ b/src/encoding/models/types.ts @@ -0,0 +1,25 @@ +import type { EncodingName } from '../../match'; + +export interface HighByteDistribution { + readonly total: number; + readonly counts: readonly (readonly [number, number])[]; +} + +export interface GeneratedSBCSLanguage { + readonly language: string; + readonly ngrams: readonly number[]; + readonly highBytes: HighByteDistribution; +} + +export interface GeneratedSBCSModel { + readonly encoding: EncodingName; + readonly byteMap: readonly number[]; + readonly languages: readonly GeneratedSBCSLanguage[]; +} + +export interface GeneratedMBCSModel { + readonly encoding: EncodingName; + readonly language: string; + readonly total: number; + readonly commonCharacters: readonly number[]; +} diff --git a/src/encoding/sbcs-scoring.test.ts b/src/encoding/sbcs-scoring.test.ts new file mode 100644 index 0000000..5f6aee1 --- /dev/null +++ b/src/encoding/sbcs-scoring.test.ts @@ -0,0 +1,81 @@ +import fs from 'fs'; +import path from 'path'; +import { describe, expect, it } from 'vitest'; +import type { Context } from '.'; +import { generatedSBCSModels } from './models/generated'; +import type { GeneratedSBCSModel } from './models/types'; +import { analyseGeneratedSBCS } from './sbcs'; +import { prepareSBCSModels, scoreSBCS, trigrams } from './sbcs-scoring'; + +const preparedModels = prepareSBCSModels( + generatedSBCSModels as readonly GeneratedSBCSModel[], +); + +function context(input: Uint8Array): Context { + const byteStats = Array(256).fill(0); + for (const byte of input) byteStats[byte] += 1; + return { + byteStats, + c1Bytes: input.some((byte) => byte >= 0x80 && byte <= 0x9f), + rawInput: input, + rawLen: input.length, + inputBytes: input, + inputLen: input.length, + }; +} + +describe('shared SBCS scoring', () => { + it('normalizes ignored bytes and repeated spaces before scoring trigrams', () => { + const byteMap = Array.from({ length: 256 }, (_, byte) => byte); + byteMap[0x27] = 0; + + expect( + trigrams(new Uint8Array([0x41, 0x20, 0x20, 0x27, 0x42]), byteMap), + ).toEqual([0x41, 0x4120, 0x412042, 0x204220]); + }); + + it.each([ + new Uint8Array(), + new Uint8Array([0x41]), + new Uint8Array([0xff]), + new Uint8Array([0xc3]), + new Uint8Array([0x00, 0xff, 0x80, 0x00]), + ])('returns deterministic finite confidence scores for %j', (input) => { + const first = scoreSBCS(input, preparedModels); + const second = scoreSBCS(input, preparedModels); + + expect(second).toEqual(first); + expect(first.length).toBeGreaterThan(0); + expect( + first.every( + ({ confidence }) => + Number.isFinite(confidence) && confidence >= 0 && confidence <= 100, + ), + ).toBe(true); + }); + + it('keeps the runtime wrapper aligned with the shared scorer on the corpus', () => { + const corpusRoot = path.join(__dirname, '../../corpus/generated'); + const corpusIndex = JSON.parse( + fs.readFileSync(path.join(corpusRoot, 'index.json'), 'utf8'), + ) as { encoding: string; path: string }[]; + const modelNames = new Set( + generatedSBCSModels.map(({ encoding }) => encoding as string), + ); + const rows = corpusIndex.filter(({ encoding }) => modelNames.has(encoding)); + + for (const row of rows) { + const input = fs.readFileSync(path.join(corpusRoot, row.path)); + const raw = scoreSBCS(input, preparedModels).find( + ({ confidence }) => confidence > 0, + ); + const runtime = analyseGeneratedSBCS(context(input))[0]; + + expect(runtime, row.path).toEqual({ + name: raw?.encoding, + lang: raw?.language, + confidence: raw ? raw.confidence / 100 : undefined, + }); + } + }); +}); diff --git a/src/encoding/sbcs-scoring.ts b/src/encoding/sbcs-scoring.ts new file mode 100644 index 0000000..d192928 --- /dev/null +++ b/src/encoding/sbcs-scoring.ts @@ -0,0 +1,304 @@ +import type { EncodingName } from '../match'; +import type { + GeneratedSBCSLanguage, + GeneratedSBCSModel, + HighByteDistribution, +} from './models/types'; + +const N_GRAM_MASK = 0xffffff; +// Byte distributions are cheap and sufficiently discriminative to exclude +// most encodings before the more expensive n-gram pass. The corpus needs 11 +// candidates to preserve full-result accuracy; keep one additional candidate +// as headroom for unseen inputs. +const MAX_SBCS_CANDIDATES = 12; + +export interface SBCSScore { + readonly confidence: number; + readonly hitRate: number; + readonly hits: number; + readonly total: number; +} + +export interface SBCSCandidate extends SBCSScore { + readonly encoding: EncodingName; + readonly language: string; + readonly byteLogLikelihood: number; +} + +interface RankedSBCSCandidate extends SBCSCandidate { + readonly modelIndex: number; +} + +interface PreparedSBCSLanguage { + readonly model: GeneratedSBCSLanguage; + readonly highByteLogProbabilities: Float64Array; +} + +export interface PreparedSBCSModel { + readonly model: GeneratedSBCSModel; + readonly modelIndex: number; + readonly languages: readonly PreparedSBCSLanguage[]; + readonly ngramLanguageIndexes: ReadonlyMap; +} + +type HighByteCounts = readonly (readonly [number, number])[]; + +const equivalentEncodingFamilies = [ + ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], + ['ISO-8859-2', 'windows-1250'], + ['ISO-8859-7', 'windows-1253'], + ['ISO-8859-8', 'windows-1255'], + ['ISO-8859-9', 'windows-1254'], + ['ISO-8859-13', 'windows-1257'], +] as const; + +function scanTrigrams( + input: Uint8Array, + byteMap: readonly number[], + record: (value: number) => void, +): number { + let ignoreSpace = false; + let ngram = 0; + let total = 0; + + const append = (mapped: number) => { + ngram = ((ngram << 8) + mapped) & N_GRAM_MASK; + total += 1; + record(ngram); + }; + + for (const byte of input) { + const mapped = byteMap[byte]; + if (mapped !== 0) { + if (!(mapped === 0x20 && ignoreSpace)) append(mapped); + ignoreSpace = mapped === 0x20; + } + } + + append(0x20); + return total; +} + +export function trigrams( + input: Uint8Array, + byteMap: readonly number[], +): number[] { + const values: number[] = []; + scanTrigrams(input, byteMap, (value) => values.push(value)); + return values; +} + +export function statisticallyCompetitive( + best: SBCSScore, + candidate: SBCSScore, +): boolean { + const variance = + (best.hitRate * (1 - best.hitRate)) / best.total + + (candidate.hitRate * (1 - candidate.hitRate)) / candidate.total; + return best.hitRate - candidate.hitRate <= 1.96 * Math.sqrt(variance); +} + +function highByteCounts(input: Uint8Array): HighByteCounts { + const counts = new Uint32Array(128); + for (const byte of input) { + if (byte >= 0x80) counts[byte - 0x80] += 1; + } + + const result: [number, number][] = []; + counts.forEach((count, index) => { + if (count > 0) result.push([index + 0x80, count]); + }); + return result; +} + +function highByteLogProbabilities(distribution: HighByteDistribution) { + const counts = new Map(distribution.counts); + const denominator = distribution.total + 128; + return Float64Array.from({ length: 128 }, (_, index) => + Math.log(((counts.get(index + 0x80) ?? 0) + 1) / denominator), + ); +} + +function ngramLanguageIndexes(model: GeneratedSBCSModel) { + const indexes = new Map(); + model.languages.forEach((language, languageIndex) => { + for (const ngram of new Set(language.ngrams)) { + const languageIndexes = indexes.get(ngram) ?? []; + languageIndexes.push(languageIndex); + indexes.set(ngram, languageIndexes); + } + }); + return indexes; +} + +export function prepareSBCSModels( + models: readonly GeneratedSBCSModel[], +): readonly PreparedSBCSModel[] { + return models.map((model, modelIndex) => ({ + model, + modelIndex, + languages: model.languages.map((language) => ({ + model: language, + highByteLogProbabilities: highByteLogProbabilities(language.highBytes), + })), + ngramLanguageIndexes: ngramLanguageIndexes(model), + })); +} + +function byteLogLikelihood( + counts: HighByteCounts, + logProbabilities: Float64Array, +) { + let highBytes = 0; + let likelihood = 0; + + for (const [byte, count] of counts) { + likelihood += logProbabilities[byte - 0x80] * count; + highBytes += count; + } + + return highBytes === 0 + ? Number.NEGATIVE_INFINITY + : Number((likelihood / highBytes).toFixed(12)); +} + +function modelByteLogLikelihood( + counts: HighByteCounts, + prepared: PreparedSBCSModel, +) { + let best = Number.NEGATIVE_INFINITY; + for (const language of prepared.languages) { + best = Math.max( + best, + byteLogLikelihood(counts, language.highByteLogProbabilities), + ); + } + return best; +} + +function candidate( + input: Uint8Array, + counts: HighByteCounts, + prepared: PreparedSBCSModel, + hits: Uint32Array, +): RankedSBCSCandidate { + hits.fill(0, 0, prepared.languages.length); + const total = scanTrigrams(input, prepared.model.byteMap, (value) => { + for (const languageIndex of prepared.ngramLanguageIndexes.get(value) ?? + []) { + hits[languageIndex] += 1; + } + }); + + let bestIndex = 0; + let bestConfidence = -1; + let bestByteLogLikelihood = Number.NEGATIVE_INFINITY; + let bestHitRate = 0; + + for (let index = 0; index < prepared.languages.length; index += 1) { + const hitRate = hits[index] / total; + const confidence = hitRate > 0.33 ? 98 : Math.floor(hitRate * 300); + const likelihood = byteLogLikelihood( + counts, + prepared.languages[index].highByteLogProbabilities, + ); + if ( + confidence > bestConfidence || + (confidence === bestConfidence && likelihood > bestByteLogLikelihood) + ) { + bestIndex = index; + bestConfidence = confidence; + bestByteLogLikelihood = likelihood; + bestHitRate = hitRate; + } + } + + return { + encoding: prepared.model.encoding, + language: prepared.languages[bestIndex].model.language, + confidence: bestConfidence, + byteLogLikelihood: bestByteLogLikelihood, + hits: hits[bestIndex], + total, + hitRate: bestHitRate, + modelIndex: prepared.modelIndex, + }; +} + +function family(name: EncodingName) { + return equivalentEncodingFamilies.find((value) => + value.includes(name as never), + ); +} + +function familyPreference( + left: SBCSCandidate, + right: SBCSCandidate, + c1Bytes: boolean, +) { + const leftFamily = family(left.encoding); + if (!leftFamily || leftFamily !== family(right.encoding)) return 0; + const preferred = + leftFamily[0] === 'ISO-8859-13' || c1Bytes + ? [...leftFamily].reverse() + : [...leftFamily]; + return ( + preferred.indexOf(left.encoding as never) - + preferred.indexOf(right.encoding as never) + ); +} + +export function scoreSBCS( + input: Uint8Array, + preparedModels: readonly PreparedSBCSModel[], + c1Bytes = input.some((byte) => byte >= 0x80 && byte <= 0x9f), +): SBCSCandidate[] { + if (preparedModels.length === 0) return []; + + const counts = highByteCounts(input); + const hits = new Uint32Array( + Math.max(...preparedModels.map((model) => model.languages.length)), + ); + const models = + counts.length === 0 + ? preparedModels + : preparedModels + .map((model) => ({ + model, + likelihood: modelByteLogLikelihood(counts, model), + })) + .sort( + (left, right) => + right.likelihood - left.likelihood || + left.model.modelIndex - right.model.modelIndex, + ) + .slice(0, MAX_SBCS_CANDIDATES) + .map(({ model }) => model); + const candidates = models.map((model) => + candidate(input, counts, model, hits), + ); + candidates.sort( + (left, right) => + right.confidence - left.confidence || left.modelIndex - right.modelIndex, + ); + + const strongest = candidates[0]; + const competitive = candidates.filter((value) => + statisticallyCompetitive(strongest, value), + ); + const competitiveNames = new Set(competitive.map((value) => value.encoding)); + competitive.sort( + (left, right) => + familyPreference(left, right, c1Bytes) || + right.byteLogLikelihood - left.byteLogLikelihood || + right.confidence - left.confidence || + left.modelIndex - right.modelIndex, + ); + const remaining = candidates.filter( + (value) => !competitiveNames.has(value.encoding), + ); + + return [...competitive, ...remaining].map( + ({ modelIndex: _modelIndex, ...value }) => value, + ); +} diff --git a/src/encoding/sbcs.test.ts b/src/encoding/sbcs.test.ts index bcae0fd..3beb615 100644 --- a/src/encoding/sbcs.test.ts +++ b/src/encoding/sbcs.test.ts @@ -1,10 +1,11 @@ import * as chardet from '..'; import fs from 'fs'; import path from 'path'; -import { describe, expect, it, test } from 'vitest'; +import { describe, expect, it } from 'vitest'; describe('Singlebyte Character Sets', () => { const base = path.join(__dirname, '/../test/data/encodings'); + const corpus = path.join(__dirname, '../../corpus/generated'); const detect = (filename: string) => { return chardet.detectFileSync(path.join(base, filename)); @@ -14,16 +15,42 @@ describe('Singlebyte Character Sets', () => { return chardet.analyse(fs.readFileSync(path.join(base, filename)))[0]; }; + const corpusFixture = ( + encoding: string, + language: string, + filename = 'validation/river-trip.bin', + ) => path.join(corpus, encoding, language, filename); + it('should return ISO-8859-1 (English)', () => { expect(detect('iso88591_en')).toBe('ISO-8859-1'); }); + it('should retain a weak ISO-8859-1 candidate after byte prefiltering', () => { + const fixture = path.join( + __dirname, + '../../corpus/generated/ISO-8859-1/es/validation/river-trip.bin', + ); + expect(chardet.detectFileSync(fixture)).toBe('ISO-8859-1'); + }); + it('should return ISO-8859-2 (Czech)', () => { expect(detect('iso88592_cs')).toBe('ISO-8859-2'); }); - test.todo('should return ISO-8859-3'); - test.todo('should return ISO-8859-4'); + it.each([ + ['ISO-8859-3', 'mt'], + ['ISO-8859-4', 'lv'], + ['ISO-8859-10', 'is'], + ['ISO-8859-14', 'cy'], + ['ISO-8859-16', 'ro'], + ])('should return %s (%s)', (encoding, language) => { + const fixture = corpusFixture(encoding, language); + expect(chardet.detectFileSync(fixture)).toBe(encoding); + expect(chardet.analyse(fs.readFileSync(fixture))[0]).toMatchObject({ + name: encoding, + lang: language, + }); + }); it('should return ISO-8859-5 (Russian)', () => { expect(detect('iso88595_ru')).toBe('ISO-8859-5'); @@ -45,18 +72,43 @@ describe('Singlebyte Character Sets', () => { expect(detect('iso88599_tr')).toBe('ISO-8859-9'); }); - test.todo('should return ISO-8859-10'); - test.todo('should return ISO-8859-11'); - // iso-8859-12 is abandoned - test.todo('should return ISO-8859-13'); - test.todo('should return ISO-8859-14'); - test.todo('should return ISO-8859-15'); - test.todo('should return ISO-8859-16'); - - it('should return windows-874 for Thai text', () => { + it('should return windows-874 for ISO-8859-11-compatible Thai text', () => { expect(detect('windows_874')).toBe('windows-874'); }); + it.each(['validation/river-trip.bin', 'test/community-garden.bin'])( + 'should prefer windows-874 to weak multibyte matches in %s', + (filename) => { + const matches = chardet.analyse( + fs.readFileSync(corpusFixture('windows-874', 'th', filename)), + ); + const windows874 = matches.find((match) => match.name === 'windows-874'); + const eucJp = matches.find((match) => match.name === 'EUC-JP'); + + expect(matches[0]).toMatchObject({ name: 'windows-874', lang: 'th' }); + expect(eucJp?.confidence).toBeLessThanOrEqual( + windows874?.confidence ?? 0, + ); + }, + ); + + // iso-8859-12 is abandoned + it.each([ + ['ISO-8859-13', 'lt'], + ['ISO-8859-15', 'fr'], + ])( + 'should return a %s candidate for byte-compatible text', + (encoding, language) => { + const fixture = corpusFixture(encoding, language); + expect(chardet.analyse(fs.readFileSync(fixture))).toContainEqual( + expect.objectContaining({ + name: encoding, + lang: language, + }), + ); + }, + ); + it('should return windows-1250 (Czech)', () => { expect(detect('windows_1250')).toBe('windows-1250'); }); diff --git a/src/encoding/sbcs.ts b/src/encoding/sbcs.ts index 279531f..bc45362 100644 --- a/src/encoding/sbcs.ts +++ b/src/encoding/sbcs.ts @@ -1,1093 +1,23 @@ -import type { Context, Recogniser } from '.'; -import match, { type EncodingName, type Match } from '../match'; - -/** - * This class recognizes single-byte encodings. Because the encoding scheme is so - * simple, language statistics are used to do the matching. - */ - -const N_GRAM_MASK = 0xffffff; - -class NGramParser { - byteIndex: number = 0; - ngram: number = 0; - - ngramCount: number = 0; - hitCount: number = 0; - - ngramList: number[]; - byteMap: number[]; - - // TODO: is it safe to set it like this? - spaceChar: number = 0x20; - - constructor(theNgramList: number[], theByteMap: number[]) { - this.ngramList = theNgramList; - this.byteMap = theByteMap; - } - - /* - * Binary search for value in table, which must have exactly 64 entries. - */ - search(table: number[], value: number) { - let index = 0; - - if (table[index + 32] <= value) index += 32; - if (table[index + 16] <= value) index += 16; - if (table[index + 8] <= value) index += 8; - if (table[index + 4] <= value) index += 4; - if (table[index + 2] <= value) index += 2; - if (table[index + 1] <= value) index += 1; - if (table[index] > value) index -= 1; - - if (index < 0 || table[index] != value) return -1; - - return index; - } - - lookup(thisNgram: number) { - this.ngramCount += 1; - if (this.search(this.ngramList, thisNgram) >= 0) { - this.hitCount += 1; - } - } - - addByte(b: number) { - this.ngram = ((this.ngram << 8) + (b & 0xff)) & N_GRAM_MASK; - this.lookup(this.ngram); - } - - nextByte(det: Context) { - if (this.byteIndex >= det.inputLen) return -1; - - return det.inputBytes[this.byteIndex++] & 0xff; - } - - parse(det: Context, spaceCh: number) { - let b, - ignoreSpace = false; - this.spaceChar = spaceCh; - - while ((b = this.nextByte(det)) >= 0) { - const mb = this.byteMap[b]; - - // TODO: 0x20 might not be a space in all character sets... - if (mb != 0) { - if (!(mb == this.spaceChar && ignoreSpace)) { - this.addByte(mb); - } - - ignoreSpace = mb == this.spaceChar; - } - } - - // TODO: Is this OK? The buffer could have ended in the middle of a word... - this.addByte(this.spaceChar); - - const rawPercent = this.hitCount / this.ngramCount; - - // TODO - This is a bit of a hack to take care of a case - // were we were getting a confidence of 135... - if (rawPercent > 0.33) return 98; - - return Math.floor(rawPercent * 300.0); - } +import type { Match } from '../match'; +import type { Context } from '.'; +import { generatedSBCSModels } from './models/generated'; +import type { GeneratedSBCSModel } from './models/types'; +import { prepareSBCSModels, scoreSBCS } from './sbcs-scoring'; + +const preparedSBCSModels = prepareSBCSModels( + generatedSBCSModels as readonly GeneratedSBCSModel[], +); + +export function analyseGeneratedSBCS(context: Context): Match[] { + let confidenceCeiling = 1; + return scoreSBCS(context.inputBytes, preparedSBCSModels, context.c1Bytes) + .filter((value) => value.confidence > 0) + .map(({ encoding, language, confidence }) => { + confidenceCeiling = Math.min(confidenceCeiling, confidence / 100); + return { + name: encoding, + lang: language, + confidence: confidenceCeiling, + }; + }); } - -class NGramsPlusLang { - fLang: string; - fNGrams: number[]; - - constructor(la: string, ng: number[]) { - this.fLang = la; - this.fNGrams = ng; - } -} - -const isFlatNgrams = (val: NGramsPlusLang[] | number[]): val is number[] => - Array.isArray(val) && isFinite(val[0] as number); - -class sbcs implements Recogniser { - spaceChar = 0x20; - - private nGramLang?: string = undefined; - - ngrams(): NGramsPlusLang[] | number[] { - return []; - } - - byteMap(): number[] { - return []; - } - - name(_input: Context): EncodingName { - return 'sbcs'; - } - - language(): string | undefined { - return this.nGramLang; - } - - match(det: Context): Match | null { - // This feels a bit dirty. Simpler alternative would be - // splitting classes ISO_8859_1 etc into language-specific ones - // with hardcoded languages like ISO_8859_9. - this.nGramLang = undefined; - - const ngrams = this.ngrams(); - - if (isFlatNgrams(ngrams)) { - const parser = new NGramParser(ngrams, this.byteMap()); - const confidence = parser.parse(det, this.spaceChar); - return confidence <= 0 ? null : match(det, this, confidence); - } - - let bestConfidence = -1; - - for (let i = ngrams.length - 1; i >= 0; i--) { - const ngl = ngrams[i]; - - const parser = new NGramParser(ngl.fNGrams, this.byteMap()); - const confidence = parser.parse(det, this.spaceChar); - if (confidence > bestConfidence) { - bestConfidence = confidence; - this.nGramLang = ngl.fLang; - } - } - - return bestConfidence <= 0 ? null : match(det, this, bestConfidence); - } -} - -export class ISO_8859_1 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - new NGramsPlusLang( - 'da', - [ - 0x206166, 0x206174, 0x206465, 0x20656e, 0x206572, 0x20666f, 0x206861, - 0x206920, 0x206d65, 0x206f67, 0x2070e5, 0x207369, 0x207374, 0x207469, - 0x207669, 0x616620, 0x616e20, 0x616e64, 0x617220, 0x617420, 0x646520, - 0x64656e, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656e20, - 0x656e64, 0x657220, 0x657265, 0x657320, 0x657420, 0x666f72, 0x676520, - 0x67656e, 0x676572, 0x696765, 0x696c20, 0x696e67, 0x6b6520, 0x6b6b65, - 0x6c6572, 0x6c6967, 0x6c6c65, 0x6d6564, 0x6e6465, 0x6e6520, 0x6e6720, - 0x6e6765, 0x6f6720, 0x6f6d20, 0x6f7220, 0x70e520, 0x722064, 0x722065, - 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696c, - 0x766572, - ], - ), - new NGramsPlusLang( - 'de', - [ - 0x20616e, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, - 0x206765, 0x206861, 0x20696e, 0x206d69, 0x207363, 0x207365, 0x20756e, - 0x207665, 0x20766f, 0x207765, 0x207a75, 0x626572, 0x636820, 0x636865, - 0x636874, 0x646173, 0x64656e, 0x646572, 0x646965, 0x652064, 0x652073, - 0x65696e, 0x656974, 0x656e20, 0x657220, 0x657320, 0x67656e, 0x68656e, - 0x687420, 0x696368, 0x696520, 0x696e20, 0x696e65, 0x697420, 0x6c6963, - 0x6c6c65, 0x6e2061, 0x6e2064, 0x6e2073, 0x6e6420, 0x6e6465, 0x6e6520, - 0x6e6720, 0x6e6765, 0x6e7465, 0x722064, 0x726465, 0x726569, 0x736368, - 0x737465, 0x742064, 0x746520, 0x74656e, 0x746572, 0x756e64, 0x756e67, - 0x766572, - ], - ), - new NGramsPlusLang( - 'en', - [ - 0x206120, 0x20616e, 0x206265, 0x20636f, 0x20666f, 0x206861, 0x206865, - 0x20696e, 0x206d61, 0x206f66, 0x207072, 0x207265, 0x207361, 0x207374, - 0x207468, 0x20746f, 0x207768, 0x616964, 0x616c20, 0x616e20, 0x616e64, - 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, - 0x652073, 0x652074, 0x656420, 0x656e74, 0x657220, 0x657320, 0x666f72, - 0x686174, 0x686520, 0x686572, 0x696420, 0x696e20, 0x696e67, 0x696f6e, - 0x697320, 0x6e2061, 0x6e2074, 0x6e6420, 0x6e6720, 0x6e7420, 0x6f6620, - 0x6f6e20, 0x6f7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, - 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696f, 0x746f20, - 0x747320, - ], - ), - new NGramsPlusLang( - 'es', - [ - 0x206120, 0x206361, 0x20636f, 0x206465, 0x20656c, 0x20656e, 0x206573, - 0x20696e, 0x206c61, 0x206c6f, 0x207061, 0x20706f, 0x207072, 0x207175, - 0x207265, 0x207365, 0x20756e, 0x207920, 0x612063, 0x612064, 0x612065, - 0x61206c, 0x612070, 0x616369, 0x61646f, 0x616c20, 0x617220, 0x617320, - 0x6369f3, 0x636f6e, 0x646520, 0x64656c, 0x646f20, 0x652064, 0x652065, - 0x65206c, 0x656c20, 0x656e20, 0x656e74, 0x657320, 0x657374, 0x69656e, - 0x69f36e, 0x6c6120, 0x6c6f73, 0x6e2065, 0x6e7465, 0x6f2064, 0x6f2065, - 0x6f6e20, 0x6f7220, 0x6f7320, 0x706172, 0x717565, 0x726120, 0x726573, - 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746f20, 0x756520, - 0xf36e20, - ], - ), - new NGramsPlusLang( - 'fr', - [ - 0x206175, 0x20636f, 0x206461, 0x206465, 0x206475, 0x20656e, 0x206574, - 0x206c61, 0x206c65, 0x207061, 0x20706f, 0x207072, 0x207175, 0x207365, - 0x20736f, 0x20756e, 0x20e020, 0x616e74, 0x617469, 0x636520, 0x636f6e, - 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, - 0x65206c, 0x652070, 0x652073, 0x656e20, 0x656e74, 0x657220, 0x657320, - 0x657420, 0x657572, 0x696f6e, 0x697320, 0x697420, 0x6c6120, 0x6c6520, - 0x6c6573, 0x6d656e, 0x6e2064, 0x6e6520, 0x6e7320, 0x6e7420, 0x6f6e20, - 0x6f6e74, 0x6f7572, 0x717565, 0x72206c, 0x726520, 0x732061, 0x732064, - 0x732065, 0x73206c, 0x732070, 0x742064, 0x746520, 0x74696f, 0x756520, - 0x757220, - ], - ), - new NGramsPlusLang( - 'it', - [ - 0x20616c, 0x206368, 0x20636f, 0x206465, 0x206469, 0x206520, 0x20696c, - 0x20696e, 0x206c61, 0x207065, 0x207072, 0x20756e, 0x612063, 0x612064, - 0x612070, 0x612073, 0x61746f, 0x636865, 0x636f6e, 0x64656c, 0x646920, - 0x652061, 0x652063, 0x652064, 0x652069, 0x65206c, 0x652070, 0x652073, - 0x656c20, 0x656c6c, 0x656e74, 0x657220, 0x686520, 0x692061, 0x692063, - 0x692064, 0x692073, 0x696120, 0x696c20, 0x696e20, 0x696f6e, 0x6c6120, - 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6520, 0x6e6920, 0x6e6f20, 0x6e7465, - 0x6f2061, 0x6f2064, 0x6f2069, 0x6f2073, 0x6f6e20, 0x6f6e65, 0x706572, - 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746f20, - 0x7a696f, - ], - ), - new NGramsPlusLang( - 'nl', - [ - 0x20616c, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656e, - 0x206765, 0x206865, 0x20696e, 0x206d61, 0x206d65, 0x206f70, 0x207465, - 0x207661, 0x207665, 0x20766f, 0x207765, 0x207a69, 0x61616e, 0x616172, - 0x616e20, 0x616e64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656e, - 0x646572, 0x652062, 0x652076, 0x65656e, 0x656572, 0x656e20, 0x657220, - 0x657273, 0x657420, 0x67656e, 0x686574, 0x696520, 0x696e20, 0x696e67, - 0x697320, 0x6e2062, 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206f, 0x6e2076, - 0x6e6465, 0x6e6720, 0x6f6e64, 0x6f6f72, 0x6f7020, 0x6f7220, 0x736368, - 0x737465, 0x742064, 0x746520, 0x74656e, 0x746572, 0x76616e, 0x766572, - 0x766f6f, - ], - ), - new NGramsPlusLang( - 'no', - [ - 0x206174, 0x206176, 0x206465, 0x20656e, 0x206572, 0x20666f, 0x206861, - 0x206920, 0x206d65, 0x206f67, 0x2070e5, 0x207365, 0x20736b, 0x20736f, - 0x207374, 0x207469, 0x207669, 0x20e520, 0x616e64, 0x617220, 0x617420, - 0x646520, 0x64656e, 0x646574, 0x652073, 0x656420, 0x656e20, 0x656e65, - 0x657220, 0x657265, 0x657420, 0x657474, 0x666f72, 0x67656e, 0x696b6b, - 0x696c20, 0x696e67, 0x6b6520, 0x6b6b65, 0x6c6520, 0x6c6c65, 0x6d6564, - 0x6d656e, 0x6e2073, 0x6e6520, 0x6e6720, 0x6e6765, 0x6e6e65, 0x6f6720, - 0x6f6d20, 0x6f7220, 0x70e520, 0x722073, 0x726520, 0x736f6d, 0x737465, - 0x742073, 0x746520, 0x74656e, 0x746572, 0x74696c, 0x747420, 0x747465, - 0x766572, - ], - ), - new NGramsPlusLang( - 'pt', - [ - 0x206120, 0x20636f, 0x206461, 0x206465, 0x20646f, 0x206520, 0x206573, - 0x206d61, 0x206e6f, 0x206f20, 0x207061, 0x20706f, 0x207072, 0x207175, - 0x207265, 0x207365, 0x20756d, 0x612061, 0x612063, 0x612064, 0x612070, - 0x616465, 0x61646f, 0x616c20, 0x617220, 0x617261, 0x617320, 0x636f6d, - 0x636f6e, 0x646120, 0x646520, 0x646f20, 0x646f73, 0x652061, 0x652064, - 0x656d20, 0x656e74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6d656e, - 0x6e7465, 0x6e746f, 0x6f2061, 0x6f2063, 0x6f2064, 0x6f2065, 0x6f2070, - 0x6f7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, - 0x732065, 0x732070, 0x737461, 0x746520, 0x746f20, 0x756520, 0xe36f20, - 0xe7e36f, - ], - ), - new NGramsPlusLang( - 'sv', - [ - 0x206174, 0x206176, 0x206465, 0x20656e, 0x2066f6, 0x206861, 0x206920, - 0x20696e, 0x206b6f, 0x206d65, 0x206f63, 0x2070e5, 0x20736b, 0x20736f, - 0x207374, 0x207469, 0x207661, 0x207669, 0x20e472, 0x616465, 0x616e20, - 0x616e64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656e, 0x646572, - 0x646574, 0x656420, 0x656e20, 0x657220, 0x657420, 0x66f672, 0x67656e, - 0x696c6c, 0x696e67, 0x6b6120, 0x6c6c20, 0x6d6564, 0x6e2073, 0x6e6120, - 0x6e6465, 0x6e6720, 0x6e6765, 0x6e696e, 0x6f6368, 0x6f6d20, 0x6f6e20, - 0x70e520, 0x722061, 0x722073, 0x726120, 0x736b61, 0x736f6d, 0x742073, - 0x746120, 0x746520, 0x746572, 0x74696c, 0x747420, 0x766172, 0xe47220, - 0xf67220, - ], - ), - ]; - } - - name(input: Context): EncodingName { - return input && input.c1Bytes ? 'windows-1252' : 'ISO-8859-1'; - } -} - -export class ISO_8859_2 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0x20, - 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0x20, 0xb1, 0x20, 0xb3, - 0x20, 0xb5, 0xb6, 0xb7, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0x20, - ]; - } - - ngrams() { - return [ - new NGramsPlusLang( - 'cs', - [ - 0x206120, 0x206279, 0x20646f, 0x206a65, 0x206e61, 0x206e65, 0x206f20, - 0x206f64, 0x20706f, 0x207072, 0x2070f8, 0x20726f, 0x207365, 0x20736f, - 0x207374, 0x20746f, 0x207620, 0x207679, 0x207a61, 0x612070, 0x636520, - 0x636820, 0x652070, 0x652073, 0x652076, 0x656d20, 0x656eed, 0x686f20, - 0x686f64, 0x697374, 0x6a6520, 0x6b7465, 0x6c6520, 0x6c6920, 0x6e6120, - 0x6ee920, 0x6eec20, 0x6eed20, 0x6f2070, 0x6f646e, 0x6f6a69, 0x6f7374, - 0x6f7520, 0x6f7661, 0x706f64, 0x706f6a, 0x70726f, 0x70f865, 0x736520, - 0x736f75, 0x737461, 0x737469, 0x73746e, 0x746572, 0x746eed, 0x746f20, - 0x752070, 0xbe6520, 0xe16eed, 0xe9686f, 0xed2070, 0xed2073, 0xed6d20, - 0xf86564, - ], - ), - new NGramsPlusLang( - 'hu', - [ - 0x206120, 0x20617a, 0x206265, 0x206567, 0x20656c, 0x206665, 0x206861, - 0x20686f, 0x206973, 0x206b65, 0x206b69, 0x206bf6, 0x206c65, 0x206d61, - 0x206d65, 0x206d69, 0x206e65, 0x20737a, 0x207465, 0x20e973, 0x612061, - 0x61206b, 0x61206d, 0x612073, 0x616b20, 0x616e20, 0x617a20, 0x62616e, - 0x62656e, 0x656779, 0x656b20, 0x656c20, 0x656c65, 0x656d20, 0x656e20, - 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686f67, 0x696e74, - 0x697320, 0x6b2061, 0x6bf67a, 0x6d6567, 0x6d696e, 0x6e2061, 0x6e616b, - 0x6e656b, 0x6e656d, 0x6e7420, 0x6f6779, 0x732061, 0x737a65, 0x737a74, - 0x737ae1, 0x73e967, 0x742061, 0x747420, 0x74e173, 0x7a6572, 0xe16e20, - 0xe97320, - ], - ), - new NGramsPlusLang( - 'pl', - [ - 0x20637a, 0x20646f, 0x206920, 0x206a65, 0x206b6f, 0x206d61, 0x206d69, - 0x206e61, 0x206e69, 0x206f64, 0x20706f, 0x207072, 0x207369, 0x207720, - 0x207769, 0x207779, 0x207a20, 0x207a61, 0x612070, 0x612077, 0x616e69, - 0x636820, 0x637a65, 0x637a79, 0x646f20, 0x647a69, 0x652070, 0x652073, - 0x652077, 0x65207a, 0x65676f, 0x656a20, 0x656d20, 0x656e69, 0x676f20, - 0x696120, 0x696520, 0x69656a, 0x6b6120, 0x6b6920, 0x6b6965, 0x6d6965, - 0x6e6120, 0x6e6961, 0x6e6965, 0x6f2070, 0x6f7761, 0x6f7769, 0x706f6c, - 0x707261, 0x70726f, 0x70727a, 0x727a65, 0x727a79, 0x7369ea, 0x736b69, - 0x737461, 0x776965, 0x796368, 0x796d20, 0x7a6520, 0x7a6965, 0x7a7920, - 0xf37720, - ], - ), - new NGramsPlusLang( - 'ro', - [ - 0x206120, 0x206163, 0x206361, 0x206365, 0x20636f, 0x206375, 0x206465, - 0x206469, 0x206c61, 0x206d61, 0x207065, 0x207072, 0x207365, 0x2073e3, - 0x20756e, 0x20ba69, 0x20ee6e, 0x612063, 0x612064, 0x617265, 0x617420, - 0x617465, 0x617520, 0x636172, 0x636f6e, 0x637520, 0x63e320, 0x646520, - 0x652061, 0x652063, 0x652064, 0x652070, 0x652073, 0x656120, 0x656920, - 0x656c65, 0x656e74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, - 0x696520, 0x696920, 0x696e20, 0x6c6120, 0x6c6520, 0x6c6f72, 0x6c7569, - 0x6e6520, 0x6e7472, 0x6f7220, 0x70656e, 0x726520, 0x726561, 0x727520, - 0x73e320, 0x746520, 0x747275, 0x74e320, 0x756920, 0x756c20, 0xba6920, - 0xee6e20, - ], - ), - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1250' : 'ISO-8859-2'; - } -} - -export class ISO_8859_5 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0x20, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - 0x20d220, 0x20d2de, 0x20d4de, 0x20d7d0, 0x20d820, 0x20dad0, 0x20dade, - 0x20ddd0, 0x20ddd5, 0x20ded1, 0x20dfde, 0x20dfe0, 0x20e0d0, 0x20e1de, - 0x20e1e2, 0x20e2de, 0x20e7e2, 0x20ede2, 0xd0ddd8, 0xd0e2ec, 0xd3de20, - 0xd5dbec, 0xd5ddd8, 0xd5e1e2, 0xd5e220, 0xd820df, 0xd8d520, 0xd8d820, - 0xd8ef20, 0xdbd5dd, 0xdbd820, 0xdbecdd, 0xddd020, 0xddd520, 0xddd8d5, - 0xddd8ef, 0xddde20, 0xddded2, 0xde20d2, 0xde20df, 0xde20e1, 0xded220, - 0xded2d0, 0xded3de, 0xded920, 0xdedbec, 0xdedc20, 0xdee1e2, 0xdfdedb, - 0xdfe0d5, 0xdfe0d8, 0xdfe0de, 0xe0d0d2, 0xe0d5d4, 0xe1e2d0, 0xe1e2d2, - 0xe1e2d8, 0xe1ef20, 0xe2d5db, 0xe2de20, 0xe2dee0, 0xe2ec20, 0xe7e2de, - 0xebe520, - ]; - } - - name(): EncodingName { - return 'ISO-8859-5'; - } - - language() { - return 'ru'; - } -} - -export class ISO_8859_6 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, - ]; - } - - ngrams() { - return [ - 0x20c7e4, 0x20c7e6, 0x20c8c7, 0x20d9e4, 0x20e1ea, 0x20e4e4, 0x20e5e6, - 0x20e8c7, 0xc720c7, 0xc7c120, 0xc7ca20, 0xc7d120, 0xc7e420, 0xc7e4c3, - 0xc7e4c7, 0xc7e4c8, 0xc7e4ca, 0xc7e4cc, 0xc7e4cd, 0xc7e4cf, 0xc7e4d3, - 0xc7e4d9, 0xc7e4e2, 0xc7e4e5, 0xc7e4e8, 0xc7e4ea, 0xc7e520, 0xc7e620, - 0xc7e6ca, 0xc820c7, 0xc920c7, 0xc920e1, 0xc920e4, 0xc920e5, 0xc920e8, - 0xca20c7, 0xcf20c7, 0xcfc920, 0xd120c7, 0xd1c920, 0xd320c7, 0xd920c7, - 0xd9e4e9, 0xe1ea20, 0xe420c7, 0xe4c920, 0xe4e920, 0xe4ea20, 0xe520c7, - 0xe5c720, 0xe5c920, 0xe5e620, 0xe620c7, 0xe720c7, 0xe7c720, 0xe8c7e4, - 0xe8e620, 0xe920c7, 0xea20c7, 0xea20e5, 0xea20e8, 0xeac920, 0xead120, - 0xeae620, - ]; - } - - name(): EncodingName { - return 'ISO-8859-6'; - } - - language() { - return 'ar'; - } -} - -export class ISO_8859_7 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0xa1, 0xa2, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xdc, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, - 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0x20, - ]; - } - - ngrams() { - return [ - 0x20e1ed, 0x20e1f0, 0x20e3e9, 0x20e4e9, 0x20e5f0, 0x20e720, 0x20eae1, - 0x20ece5, 0x20ede1, 0x20ef20, 0x20f0e1, 0x20f0ef, 0x20f0f1, 0x20f3f4, - 0x20f3f5, 0x20f4e7, 0x20f4ef, 0xdfe120, 0xe120e1, 0xe120f4, 0xe1e920, - 0xe1ed20, 0xe1f0fc, 0xe1f220, 0xe3e9e1, 0xe5e920, 0xe5f220, 0xe720f4, - 0xe7ed20, 0xe7f220, 0xe920f4, 0xe9e120, 0xe9eade, 0xe9f220, 0xeae1e9, - 0xeae1f4, 0xece520, 0xed20e1, 0xed20e5, 0xed20f0, 0xede120, 0xeff220, - 0xeff520, 0xf0eff5, 0xf0f1ef, 0xf0fc20, 0xf220e1, 0xf220e5, 0xf220ea, - 0xf220f0, 0xf220f4, 0xf3e520, 0xf3e720, 0xf3f4ef, 0xf4e120, 0xf4e1e9, - 0xf4e7ed, 0xf4e7f2, 0xf4e9ea, 0xf4ef20, 0xf4eff5, 0xf4f9ed, 0xf9ed20, - 0xfeed20, - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1253' : 'ISO-8859-7'; - } - - language() { - return 'el'; - } -} - -export class ISO_8859_8 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, - 0x20, 0x20, 0x20, 0x20, - ]; - } - - ngrams() { - return [ - new NGramsPlusLang( - 'he', - [ - 0x20e0e5, 0x20e0e7, 0x20e0e9, 0x20e0fa, 0x20e1e9, 0x20e1ee, 0x20e4e0, - 0x20e4e5, 0x20e4e9, 0x20e4ee, 0x20e4f2, 0x20e4f9, 0x20e4fa, 0x20ece0, - 0x20ece4, 0x20eee0, 0x20f2ec, 0x20f9ec, 0xe0fa20, 0xe420e0, 0xe420e1, - 0xe420e4, 0xe420ec, 0xe420ee, 0xe420f9, 0xe4e5e0, 0xe5e020, 0xe5ed20, - 0xe5ef20, 0xe5f820, 0xe5fa20, 0xe920e4, 0xe9e420, 0xe9e5fa, 0xe9e9ed, - 0xe9ed20, 0xe9ef20, 0xe9f820, 0xe9fa20, 0xec20e0, 0xec20e4, 0xece020, - 0xece420, 0xed20e0, 0xed20e1, 0xed20e4, 0xed20ec, 0xed20ee, 0xed20f9, - 0xeee420, 0xef20e4, 0xf0e420, 0xf0e920, 0xf0e9ed, 0xf2ec20, 0xf820e4, - 0xf8e9ed, 0xf9ec20, 0xfa20e0, 0xfa20e1, 0xfa20e4, 0xfa20ec, 0xfa20ee, - 0xfa20f9, - ], - ), - new NGramsPlusLang( - 'he', - [ - 0x20e0e5, 0x20e0ec, 0x20e4e9, 0x20e4ec, 0x20e4ee, 0x20e4f0, 0x20e9f0, - 0x20ecf2, 0x20ecf9, 0x20ede5, 0x20ede9, 0x20efe5, 0x20efe9, 0x20f8e5, - 0x20f8e9, 0x20fae0, 0x20fae5, 0x20fae9, 0xe020e4, 0xe020ec, 0xe020ed, - 0xe020fa, 0xe0e420, 0xe0e5e4, 0xe0ec20, 0xe0ee20, 0xe120e4, 0xe120ed, - 0xe120fa, 0xe420e4, 0xe420e9, 0xe420ec, 0xe420ed, 0xe420ef, 0xe420f8, - 0xe420fa, 0xe4ec20, 0xe5e020, 0xe5e420, 0xe7e020, 0xe9e020, 0xe9e120, - 0xe9e420, 0xec20e4, 0xec20ed, 0xec20fa, 0xecf220, 0xecf920, 0xede9e9, - 0xede9f0, 0xede9f8, 0xee20e4, 0xee20ed, 0xee20fa, 0xeee120, 0xeee420, - 0xf2e420, 0xf920e4, 0xf920ed, 0xf920fa, 0xf9e420, 0xfae020, 0xfae420, - 0xfae5e9, - ], - ), - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1255' : 'ISO-8859-8'; - } - - language() { - return 'he'; - } -} - -export class ISO_8859_9 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x69, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, - 0x20696c, 0x206b61, 0x206b6f, 0x206d61, 0x206f6c, 0x207361, 0x207461, - 0x207665, 0x207961, 0x612062, 0x616b20, 0x616c61, 0x616d61, 0x616e20, - 0x616efd, 0x617220, 0x617261, 0x6172fd, 0x6173fd, 0x617961, 0x626972, - 0x646120, 0x646520, 0x646920, 0x652062, 0x65206b, 0x656469, 0x656e20, - 0x657220, 0x657269, 0x657369, 0x696c65, 0x696e20, 0x696e69, 0x697220, - 0x6c616e, 0x6c6172, 0x6c6520, 0x6c6572, 0x6e2061, 0x6e2062, 0x6e206b, - 0x6e6461, 0x6e6465, 0x6e6520, 0x6e6920, 0x6e696e, 0x6efd20, 0x72696e, - 0x72fd6e, 0x766520, 0x796120, 0x796f72, 0xfd6e20, 0xfd6e64, 0xfd6efd, - 0xfdf0fd, - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1254' : 'ISO-8859-9'; - } - - language() { - return 'tr'; - } -} - -export class windows_1251 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x90, 0x83, 0x20, 0x83, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, - 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, - 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa2, 0xa2, 0xbc, 0x20, 0xb4, 0x20, 0x20, - 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0xb3, 0xb3, - 0xb4, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0xbc, 0xbe, 0xbe, 0xbf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - 0x20e220, 0x20e2ee, 0x20e4ee, 0x20e7e0, 0x20e820, 0x20eae0, 0x20eaee, - 0x20ede0, 0x20ede5, 0x20eee1, 0x20efee, 0x20eff0, 0x20f0e0, 0x20f1ee, - 0x20f1f2, 0x20f2ee, 0x20f7f2, 0x20fdf2, 0xe0ede8, 0xe0f2fc, 0xe3ee20, - 0xe5ebfc, 0xe5ede8, 0xe5f1f2, 0xe5f220, 0xe820ef, 0xe8e520, 0xe8e820, - 0xe8ff20, 0xebe5ed, 0xebe820, 0xebfced, 0xede020, 0xede520, 0xede8e5, - 0xede8ff, 0xedee20, 0xedeee2, 0xee20e2, 0xee20ef, 0xee20f1, 0xeee220, - 0xeee2e0, 0xeee3ee, 0xeee920, 0xeeebfc, 0xeeec20, 0xeef1f2, 0xefeeeb, - 0xeff0e5, 0xeff0e8, 0xeff0ee, 0xf0e0e2, 0xf0e5e4, 0xf1f2e0, 0xf1f2e2, - 0xf1f2e8, 0xf1ff20, 0xf2e5eb, 0xf2ee20, 0xf2eef0, 0xf2fc20, 0xf7f2ee, - 0xfbf520, - ]; - } - - name(): EncodingName { - return 'windows-1251'; - } - - language() { - return 'ru'; - } -} - -export class windows_1256 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x81, 0x20, 0x83, - 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x8a, 0x20, 0x9c, 0x8d, 0x8e, 0x8f, - 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x98, 0x20, 0x9a, 0x20, - 0x9c, 0x20, 0x20, 0x9f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0x20, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0x20, 0x20, 0x20, 0x20, 0xf4, 0x20, 0x20, 0x20, 0x20, 0xf9, 0x20, 0xfb, - 0xfc, 0x20, 0x20, 0xff, - ]; - } - - ngrams() { - return [ - 0x20c7e1, 0x20c7e4, 0x20c8c7, 0x20dae1, 0x20dded, 0x20e1e1, 0x20e3e4, - 0x20e6c7, 0xc720c7, 0xc7c120, 0xc7ca20, 0xc7d120, 0xc7e120, 0xc7e1c3, - 0xc7e1c7, 0xc7e1c8, 0xc7e1ca, 0xc7e1cc, 0xc7e1cd, 0xc7e1cf, 0xc7e1d3, - 0xc7e1da, 0xc7e1de, 0xc7e1e3, 0xc7e1e6, 0xc7e1ed, 0xc7e320, 0xc7e420, - 0xc7e4ca, 0xc820c7, 0xc920c7, 0xc920dd, 0xc920e1, 0xc920e3, 0xc920e6, - 0xca20c7, 0xcf20c7, 0xcfc920, 0xd120c7, 0xd1c920, 0xd320c7, 0xda20c7, - 0xdae1ec, 0xdded20, 0xe120c7, 0xe1c920, 0xe1ec20, 0xe1ed20, 0xe320c7, - 0xe3c720, 0xe3c920, 0xe3e420, 0xe420c7, 0xe520c7, 0xe5c720, 0xe6c7e1, - 0xe6e420, 0xec20c7, 0xed20c7, 0xed20e3, 0xed20e6, 0xedc920, 0xedd120, - 0xede420, - ]; - } - - name(): EncodingName { - return 'windows-1256'; - } - - language() { - return 'ar'; - } -} - -export class windows_1257 extends sbcs { - match(det: Context): Match | null { - return det.inputBytes.some((byte) => byte >= 0x80) - ? super.match(det) - : null; - } - - byteMap() { - const byteMap = new Array(256).fill(0x20); - byteMap[0x27] = 0; - - for (let i = 0x41; i <= 0x5a; i++) byteMap[i] = i + 0x20; - for (let i = 0x61; i <= 0x7a; i++) byteMap[i] = i; - - byteMap[0x8e] = 0x8e; - byteMap[0xa8] = byteMap[0xb8] = 0xb8; - byteMap[0xaa] = byteMap[0xba] = 0xba; - byteMap[0xaf] = byteMap[0xbf] = 0xbf; - byteMap[0xb5] = 0xb5; - - for (let i = 0xc0; i <= 0xd6; i++) byteMap[i] = i + 0x20; - for (let i = 0xd8; i <= 0xde; i++) byteMap[i] = i + 0x20; - byteMap[0xdf] = 0xdf; - for (let i = 0xe0; i <= 0xf6; i++) byteMap[i] = i; - for (let i = 0xf8; i <= 0xfe; i++) byteMap[i] = i; - - return byteMap; - } - - ngrams() { - // Generated from original Estonian, Latvian, and Lithuanian prose samples. - return [ - new NGramsPlusLang( - 'et', - [ - 0x20616a, 0x206172, 0x206565, 0x206967, 0x206a61, 0x206b61, 0x206b65, - 0x206b69, 0x206b6f, 0x206b75, 0x206c61, 0x206e69, 0x206f6e, 0x207269, - 0x207365, 0x207465, 0x612061, 0x61206a, 0x61206b, 0x61206c, 0x616220, - 0x616420, 0x616b73, 0x617375, 0x617461, 0x617661, 0x642074, 0x646120, - 0x647573, 0x65206b, 0x652073, 0x65656c, 0x656573, 0x656c20, 0x656c65, - 0x656c74, 0x657320, 0x657374, 0x69206b, 0x696420, 0x69726a, 0x697320, - 0x6a6120, 0x6b6173, 0x6b6565, 0x6b6972, 0x6b6f6f, 0x6b7365, 0x6b7374, - 0x6c696b, 0x6c7420, 0x732061, 0x73206b, 0x736520, 0x73656c, 0x737465, - 0x737469, 0x737574, 0x74616b, 0x746174, 0x746920, 0x757320, 0x757365, - 0x757461, - ], - ), - new NGramsPlusLang( - 'lv', - [ - 0x206172, 0x206369, 0x206461, 0x206972, 0x20697a, 0x206c61, 0x206de2, - 0x207061, 0x207072, 0x207261, 0x207361, 0x20736b, 0x207465, 0x20756e, - 0x207661, 0x612069, 0x612075, 0x612076, 0x616269, 0x616920, 0x616b73, - 0x616c6f, 0x617072, 0x617275, 0x617320, 0x617476, 0x626965, 0x627520, - 0x63696c, 0x636974, 0x646120, 0x646172, 0x647520, 0x647a20, 0x656472, - 0x65696b, 0x656b73, 0x656e61, 0x657320, 0x65f075, 0x692069, 0x696564, - 0x6965f0, 0x696ae2, 0x696c76, 0x6b7374, 0x6c6174, 0x6c6f64, 0x6e6120, - 0x6f6461, 0x72616b, 0x72756e, 0x732073, 0x732075, 0x732076, 0x737475, - 0x7374ee, 0x747520, 0x747669, 0x752076, 0x756e20, 0x757320, 0x76616c, - 0xe27320, - ], - ), - new NGramsPlusLang( - 'lt', - [ - 0x206169, 0x206265, 0x206972, 0x206b61, 0x206c69, 0x206d6f, 0x207261, - 0x207361, 0x20736b, 0x207375, 0x207461, 0x207465, 0x207475, 0x207661, - 0x207669, 0x612074, 0x6120f0, 0x616920, 0x616974, 0x6169f0, 0x616c62, - 0x616d61, 0x616e74, 0x617262, 0x617274, 0x617520, 0x61756a, 0x61f079, - 0x626120, 0x626569, 0x62656e, 0x656e69, 0x657475, 0x692061, 0x692069, - 0x69206b, 0x69206d, 0x696169, 0x69656e, 0x696574, 0x696e67, 0x697220, - 0x697320, 0x69f06b, 0x6a616d, 0x6a6520, 0x6b6169, 0x6b616c, 0x6c6261, - 0x6c6965, 0x6d6973, 0x6d6f6b, 0x6eeb20, 0x6f6a65, 0x6f6b79, 0x6f7320, - 0x7261f0, 0x732069, 0x73206b, 0x732076, 0x736b61, 0x746920, 0x747576, - 0x796b6c, - ], - ), - ]; - } - - name(): EncodingName { - return 'windows-1257'; - } -} - -export class windows_1258 extends sbcs { - match(det: Context): Match | null { - return det.inputBytes.some((byte) => byte >= 0x80) - ? super.match(det) - : null; - } - - byteMap() { - const byteMap = new Array(256).fill(0x20); - byteMap[0x27] = 0; - - for (let i = 0x41; i <= 0x5a; i++) byteMap[i] = i + 0x20; - for (let i = 0x61; i <= 0x7a; i++) byteMap[i] = i; - - byteMap[0x83] = 0x83; - byteMap[0x88] = 0x88; - byteMap[0x8c] = byteMap[0x9c] = 0x9c; - byteMap[0x9f] = 0xff; - byteMap[0xaa] = 0xaa; - byteMap[0xb5] = 0xb5; - byteMap[0xba] = 0xba; - - for (let i = 0xc0; i <= 0xcb; i++) byteMap[i] = i + 0x20; - byteMap[0xcc] = 0xcc; - for (let i = 0xcd; i <= 0xd1; i++) byteMap[i] = i + 0x20; - byteMap[0xd2] = 0xd2; - for (let i = 0xd3; i <= 0xd6; i++) byteMap[i] = i + 0x20; - for (let i = 0xd8; i <= 0xdd; i++) byteMap[i] = i + 0x20; - byteMap[0xde] = 0xde; - byteMap[0xdf] = 0xdf; - for (let i = 0xe0; i <= 0xf6; i++) byteMap[i] = i; - for (let i = 0xf8; i <= 0xfd; i++) byteMap[i] = i; - byteMap[0xff] = 0xff; - - return byteMap; - } - - ngrams() { - // Generated from an original Vietnamese prose sample. - return [ - 0x206368, 0x2063e1, 0x206769, 0x20686f, 0x206e67, 0x207068, 0x207468, - 0x207469, 0x207472, 0x207669, 0x2076e0, 0x61f274, 0x632063, 0x632076, - 0x636820, 0x6368ed, 0x63e163, 0x672068, 0x67206e, 0x672074, 0x672076, - 0x67f46e, 0x67fdde, 0x67fdf5, 0x682074, 0x686ff2, 0x68ed6e, 0x69206e, - 0x692074, 0x69ead2, 0x69eaec, 0x69eaf2, 0x6e2063, 0x6e206e, 0x6e2074, - 0x6e6720, 0x6e67f4, 0x6e67fd, 0x6e6820, 0x6f6e67, 0x6ff263, 0x742073, - 0x7468fd, 0x7469ea, 0x74726f, 0x752074, 0x7669ea, 0x76e020, 0xcc6920, - 0xcc6e67, 0xde2068, 0xe16368, 0xeaec6e, 0xeaf274, 0xec6e67, 0xec7420, - 0xed6e68, 0xf26320, 0xf26920, 0xf26e20, 0xf27420, 0xf46e20, 0xf5cc69, - 0xfdf5cc, - ]; - } - - name(): EncodingName { - return 'windows-1258'; - } - - language() { - return 'vi'; - } -} - -export class windows_874 extends sbcs { - byteMap() { - const byteMap = new Array(256).fill(0x20); - byteMap[0x27] = 0; - - for (let i = 0x41; i <= 0x5a; i++) byteMap[i] = i + 0x20; - for (let i = 0x61; i <= 0x7a; i++) byteMap[i] = i; - for (let i = 0xa1; i <= 0xda; i++) byteMap[i] = i; - for (let i = 0xdf; i <= 0xfb; i++) byteMap[i] = i; - - return byteMap; - } - - ngrams() { - return [ - 0x20a1d2, 0x20a2e8, 0x20c0d2, 0x20c7c3, 0x20e1c5, 0xa1d1b9, 0xa1d2c3, - 0xa2cda7, 0xa2d5c2, 0xa4c7d2, 0xa8d3c7, 0xaad5c7, 0xaae9c0, 0xaae9e3, - 0xb5bbc3, 0xb7c2e0, 0xb7c2e3, 0xb7d5e8, 0xb920c0, 0xb9a1d2, 0xb9c0d2, - 0xbbc3d0, 0xbbe7b9, 0xc0d2c9, 0xc3c120, 0xc3c3b3, 0xc3c3c1, 0xc3d0a8, - 0xc3d0e2, 0xc3e0a2, 0xc7c3c3, 0xc7d1b9, 0xc7d2c1, 0xc7d4b5, 0xc9d2b7, - 0xc9d2e4, 0xcad7e8, 0xcda4c7, 0xd0a8d3, 0xd0e2c2, 0xd1b920, 0xd2a7a1, - 0xd2c1cb, 0xd2c2e4, 0xd2c320, 0xd2c3e0, 0xd2c9d2, 0xd2e4b7, 0xd3c7d1, - 0xd4b5bb, 0xd5c2b9, 0xd5c7d4, 0xd7e8cd, 0xe0a2d5, 0xe0bbe7, 0xe1c5d0, - 0xe2c2a4, 0xe3aae9, 0xe3b9a1, 0xe4b7c2, 0xe8d2a7, 0xe8e3aa, 0xe9c0d2, - 0xe9e3b9, - ]; - } - - name(): EncodingName { - return 'windows-874'; - } - - language() { - return 'th'; - } -} - -export class KOI8_R extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, - ]; - } - - ngrams() { - return [ - 0x20c4cf, 0x20c920, 0x20cbc1, 0x20cbcf, 0x20cec1, 0x20cec5, 0x20cfc2, - 0x20d0cf, 0x20d0d2, 0x20d2c1, 0x20d3cf, 0x20d3d4, 0x20d4cf, 0x20d720, - 0x20d7cf, 0x20dac1, 0x20dcd4, 0x20ded4, 0xc1cec9, 0xc1d4d8, 0xc5ccd8, - 0xc5cec9, 0xc5d3d4, 0xc5d420, 0xc7cf20, 0xc920d0, 0xc9c520, 0xc9c920, - 0xc9d120, 0xccc5ce, 0xccc920, 0xccd8ce, 0xcec120, 0xcec520, 0xcec9c5, - 0xcec9d1, 0xcecf20, 0xcecfd7, 0xcf20d0, 0xcf20d3, 0xcf20d7, 0xcfc7cf, - 0xcfca20, 0xcfccd8, 0xcfcd20, 0xcfd3d4, 0xcfd720, 0xcfd7c1, 0xd0cfcc, - 0xd0d2c5, 0xd0d2c9, 0xd0d2cf, 0xd2c1d7, 0xd2c5c4, 0xd3d120, 0xd3d4c1, - 0xd3d4c9, 0xd3d4d7, 0xd4c5cc, 0xd4cf20, 0xd4cfd2, 0xd4d820, 0xd9c820, - 0xded4cf, - ]; - } - - name(): EncodingName { - return 'KOI8-R'; - } - - language() { - return 'ru'; - } -} - -/* -module.exports.ISO_8859_7 = function() { - this.byteMap = function() { - return [ - - ]; - }; - - this.ngrams = function() { - return [ - - ]; - }; - - this.name = function(det) { - if (typeof det == 'undefined') - return 'ISO-8859-7'; - return det.c1Bytes ? 'windows-1253' : 'ISO-8859-7'; - }; - - language() { - return 'el'; - }; -}; -util.inherits(module.exports.ISO_8859_7, sbcs); -*/ diff --git a/src/encoding/unicode.ts b/src/encoding/unicode.ts index 7ca1a47..098b200 100644 --- a/src/encoding/unicode.ts +++ b/src/encoding/unicode.ts @@ -18,7 +18,7 @@ export class UTF_16BE implements Recogniser { (input[0] & 0xff) == 0xfe && (input[1] & 0xff) == 0xff ) { - return match(det, this, 100); // confidence = 100 + return match(det, this, 1); } // TODO: Do some statistics to check for unsigned UTF-16BE @@ -44,7 +44,7 @@ export class UTF_16LE implements Recogniser { // It is probably UTF-32 LE, not UTF-16 return null; } - return match(det, this, 100); // confidence = 100 + return match(det, this, 1); } // TODO: Do some statistics to check for unsigned UTF-16LE @@ -94,19 +94,18 @@ class UTF_32 implements Recogniser, WithGetChar { // Cook up some sort of confidence score, based on presence of a BOM // and the existence of valid and/or invalid multi-byte sequences. if (hasBOM && numInvalid == 0) { - confidence = 100; + confidence = 1; } else if (hasBOM && numValid > numInvalid * 10) { - confidence = 80; + confidence = 0.8; } else if (numValid > 3 && numInvalid == 0) { - confidence = 100; + confidence = 1; } else if (numValid > 0 && numInvalid == 0) { - confidence = 80; + confidence = 0.8; } else if (numValid > numInvalid * 10) { // Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance. - confidence = 25; + confidence = 0.25; } - // return confidence == 0 ? null : new CharsetMatch(det, this, confidence); return confidence == 0 ? null : match(det, this, confidence); } } diff --git a/src/encoding/utf8.ts b/src/encoding/utf8.ts index 6e5c4d5..6c8c087 100644 --- a/src/encoding/utf8.ts +++ b/src/encoding/utf8.ts @@ -60,16 +60,16 @@ export default class Utf8 implements Recogniser { // Cook up some sort of confidence score, based on presence of a BOM // and the existence of valid and/or invalid multi-byte sequences. confidence = 0; - if (hasBOM && numInvalid == 0) confidence = 100; - else if (hasBOM && numValid > numInvalid * 10) confidence = 80; - else if (numValid > 3 && numInvalid == 0) confidence = 100; - else if (numValid > 0 && numInvalid == 0) confidence = 80; + if (hasBOM && numInvalid == 0) confidence = 1; + else if (hasBOM && numValid > numInvalid * 10) confidence = 0.8; + else if (numValid > 3 && numInvalid == 0) confidence = 1; + else if (numValid > 0 && numInvalid == 0) confidence = 0.8; else if (numValid == 0 && numInvalid == 0) // Plain ASCII. - confidence = 10; + confidence = 0.1; else if (numValid > numInvalid * 10) // Probably corrupt utf-8 data. Valid sequences aren't likely by chance. - confidence = 25; + confidence = 0.25; else return null; return match(det, this, confidence); diff --git a/src/index.test.ts b/src/index.test.ts index 6111043..392e730 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -5,18 +5,18 @@ import { describe, expect, it } from 'vitest'; describe('chardet', () => { const path = __dirname + '/test/data/encodings/utf8'; - const expectedEncodingsFromPath = [ - { confidence: 100, name: 'UTF-8', lang: undefined }, - { confidence: 32, name: 'windows-1252', lang: 'fr' }, - { confidence: 19, name: 'KOI8-R', lang: 'ru' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, // Mandarin - { confidence: 10, name: 'windows-1253', lang: 'el' }, // Greek - { confidence: 6, name: 'windows-1250', lang: 'pl' }, - { confidence: 4, name: 'windows-1254', lang: 'tr' }, - { confidence: 3, name: 'windows-1257', lang: 'et' }, - { confidence: 2, name: 'windows-1251', lang: 'ru' }, - { confidence: 1, name: 'windows-1258', lang: 'vi' }, + const legacyExpectedEncodingsFromPath = [ + { confidence: 1, name: 'UTF-8', lang: undefined }, + { confidence: 0.32, name: 'windows-1252', lang: 'fr' }, + { confidence: 0.19, name: 'KOI8-R', lang: 'ru' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, // Mandarin + { confidence: 0.1, name: 'windows-1253', lang: 'el' }, // Greek + { confidence: 0.06, name: 'windows-1250', lang: 'pl' }, + { confidence: 0.04, name: 'windows-1254', lang: 'tr' }, + { confidence: 0.03, name: 'windows-1257', lang: 'et' }, + { confidence: 0.02, name: 'windows-1251', lang: 'ru' }, + { confidence: 0.01, name: 'windows-1258', lang: 'vi' }, { confidence: 0, name: 'ASCII', lang: undefined }, ]; @@ -94,7 +94,16 @@ describe('chardet', () => { describe('#analyse', () => { it('should return a list of encodings, sorted by confidence level in descending order', () => { const matches = chardet.analyse(fs.readFileSync(path)); - expect(matches).toEqual(expectedEncodingsFromPath); + expect(matches[0]).toEqual(legacyExpectedEncodingsFromPath[0]); + expect(matches.length).toBeGreaterThan( + legacyExpectedEncodingsFromPath.length, + ); + expect( + matches.every( + (match, index) => + index === 0 || matches[index - 1].confidence >= match.confidence, + ), + ).toBe(true); }); }); }); diff --git a/src/index.ts b/src/index.ts index bd9402c..cf1ab7c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import Ascii from './encoding/ascii'; import Utf8 from './encoding/utf8'; import * as unicode from './encoding/unicode'; import * as mbcs from './encoding/mbcs'; -import * as sbcs from './encoding/sbcs'; +import { analyseGeneratedSBCS } from './encoding/sbcs'; import * as iso2022 from './encoding/iso2022'; import { isByteArray } from './utils'; @@ -32,19 +32,6 @@ const recognisers: Recogniser[] = [ new iso2022.ISO_2022_JP(), new iso2022.ISO_2022_KR(), new iso2022.ISO_2022_CN(), - new sbcs.ISO_8859_1(), - new sbcs.ISO_8859_2(), - new sbcs.ISO_8859_5(), - new sbcs.ISO_8859_6(), - new sbcs.ISO_8859_7(), - new sbcs.ISO_8859_8(), - new sbcs.ISO_8859_9(), - new sbcs.windows_1251(), - new sbcs.windows_1256(), - new sbcs.windows_1257(), - new sbcs.windows_1258(), - new sbcs.windows_874(), - new sbcs.KOI8_R(), new Ascii(), ]; @@ -84,16 +71,27 @@ export const analyse = (buffer: Uint8Array): AnalyseResult => { inputLen: buffer.length, }; - const matches = recognisers - .map((rec) => { - return rec.match(context); - }) - .filter((match) => { - return !!match; - }) - .sort((a, b) => { - return b!.confidence - a!.confidence; - }); + const generatedMatches = analyseGeneratedSBCS(context); + const generatedRank = new Map( + generatedMatches.map((value, index) => [value.name, index]), + ); + const matches = [ + ...recognisers + .map((rec) => { + return rec.match(context); + }) + .filter((match) => { + return !!match; + }), + ...generatedMatches, + ].sort((a, b) => { + const leftRank = generatedRank.get(a!.name); + const rightRank = generatedRank.get(b!.name); + if (leftRank !== undefined && rightRank !== undefined) { + return leftRank - rightRank; + } + return b!.confidence - a!.confidence; + }); return matches as Match[]; }; diff --git a/src/match.ts b/src/match.ts index a9bc9a2..1e60883 100644 --- a/src/match.ts +++ b/src/match.ts @@ -6,22 +6,32 @@ export type EncodingName = | 'EUC-JP' | 'EUC-KR' | 'GB18030' - | 'ISO_2022' // TODO: Use hyphen | 'ISO-2022-CN' | 'ISO-2022-JP' | 'ISO-2022-KR' | 'ISO-8859-1' | 'ISO-8859-2' + | 'ISO-8859-3' + | 'ISO-8859-4' | 'ISO-8859-5' | 'ISO-8859-6' | 'ISO-8859-7' | 'ISO-8859-8' | 'ISO-8859-9' - | 'ISO-8859-9' + | 'ISO-8859-10' + | 'ISO-8859-13' + | 'ISO-8859-14' + | 'ISO-8859-15' + | 'ISO-8859-16' + | 'IBM855' + | 'IBM866' + | 'CP850' + | 'CP852' | 'KOI8-R' - | 'mbcs' - | 'sbcs' - | 'Shift_JIS' // TODO: Use hyphen + | 'KOI8-U' + | 'macintosh' + | 'x-mac-cyrillic' + | 'Shift_JIS' | 'UTF-16BE' | 'UTF-16LE' | 'UTF-32' @@ -33,7 +43,6 @@ export type EncodingName = | 'windows-1252' | 'windows-1253' | 'windows-1254' - | 'windows-1254' | 'windows-1255' | 'windows-1256' | 'windows-1257'