From 8f2fd923e96e8d6eb5a29a66881f52687d08739e Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Fri, 26 Jun 2026 12:14:46 +1000 Subject: [PATCH 1/6] feat!: normalize encoding names and confidence scores (#128) * feat!: normalize encoding names and confidence scores BREAKING CHANGE: Match confidence values now use a 0..1 range instead of 0..100. BREAKING CHANGE: EncodingName now only includes public encoding names returned by concrete recognizers. Internal recognizer names such as ISO_2022, mbcs, and sbcs are no longer part of the public type. * fix: update test-build confidence values to match normalized 0-1 scale --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- .github/workflows/test-build.js | 18 +++++++------- .github/workflows/test-build.ts | 18 +++++++------- README.md | 4 +-- src/encoding/ascii.ts | 2 +- src/encoding/iso2022.test.ts | 6 ++--- src/encoding/iso2022.ts | 12 ++++----- src/encoding/mbcs.ts | 44 +++++++++------------------------ src/encoding/sbcs.ts | 42 +++---------------------------- src/encoding/unicode.ts | 15 ++++++----- src/encoding/utf8.ts | 12 ++++----- src/index.test.ts | 22 ++++++++--------- src/match.ts | 7 +----- 12 files changed, 69 insertions(+), 133 deletions(-) diff --git a/.github/workflows/test-build.js b/.github/workflows/test-build.js index febbf05..85dde71 100644 --- a/.github/workflows/test-build.js +++ b/.github/workflows/test-build.js @@ -8,15 +8,15 @@ assert(typeof chardet.detectFile, 'function'); assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ - { confidence: 100, name: 'ASCII', lang: undefined }, - { confidence: 98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 98, name: 'ISO-8859-2', lang: 'hu' }, - { confidence: 10, name: 'UTF-8', lang: undefined }, - { confidence: 10, name: 'Shift_JIS', lang: 'ja' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'EUC-JP', lang: 'ja' }, - { confidence: 10, name: 'EUC-KR', lang: 'ko' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, + { confidence: 1, name: 'ASCII', lang: undefined }, + { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' }, + { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' }, + { confidence: 0.1, name: 'UTF-8', lang: undefined }, + { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'EUC-JP', lang: 'ja' }, + { confidence: 0.1, name: 'EUC-KR', lang: 'ko' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, ]); console.log(' > test-build.js OK'); diff --git a/.github/workflows/test-build.ts b/.github/workflows/test-build.ts index ae04c30..cdc31d8 100644 --- a/.github/workflows/test-build.ts +++ b/.github/workflows/test-build.ts @@ -11,15 +11,15 @@ const main = async () => { assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ - { confidence: 100, name: 'ASCII', lang: undefined }, - { confidence: 98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 98, name: 'ISO-8859-2', lang: 'hu' }, - { confidence: 10, name: 'UTF-8', lang: undefined }, - { confidence: 10, name: 'Shift_JIS', lang: 'ja' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'EUC-JP', lang: 'ja' }, - { confidence: 10, name: 'EUC-KR', lang: 'ko' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, + { confidence: 1, name: 'ASCII', lang: undefined }, + { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' }, + { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' }, + { confidence: 0.1, name: 'UTF-8', lang: undefined }, + { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'EUC-JP', lang: 'ja' }, + { confidence: 0.1, name: 'EUC-KR', lang: 'ko' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, ]); }; diff --git a/README.md b/README.md index 6c10081..2a797b8 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ Returned value is an array of objects sorted by confidence value in descending o ```javascript [ - { confidence: 90, name: 'UTF-8' }, - { confidence: 20, name: 'windows-1252', lang: 'fr' }, + { confidence: 0.9, name: 'UTF-8' }, + { confidence: 0.2, name: 'windows-1252', lang: 'fr' }, ]; ``` diff --git a/src/encoding/ascii.ts b/src/encoding/ascii.ts index 013c286..d65535e 100644 --- a/src/encoding/ascii.ts +++ b/src/encoding/ascii.ts @@ -16,6 +16,6 @@ export default class Ascii implements Recogniser { } } - return match(det, this, 100); + return match(det, this, 1); } } diff --git a/src/encoding/iso2022.test.ts b/src/encoding/iso2022.test.ts index 15b611b..c7d7b7e 100644 --- a/src/encoding/iso2022.test.ts +++ b/src/encoding/iso2022.test.ts @@ -11,7 +11,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-JP', () => { expect(analyse('iso2022jp')).toEqual({ - confidence: 100, + confidence: 1, lang: 'ja', name: 'ISO-2022-JP', }); @@ -19,7 +19,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-KR', () => { expect(analyse('iso2022kr')).toEqual({ - confidence: 100, + confidence: 1, lang: 'kr', name: 'ISO-2022-KR', }); @@ -27,7 +27,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-CN', () => { expect(analyse('iso2022cn')).toEqual({ - confidence: 100, + confidence: 1, lang: 'zh', name: 'ISO-2022-CN', }); diff --git a/src/encoding/iso2022.ts b/src/encoding/iso2022.ts index 6252639..5cc1d80 100644 --- a/src/encoding/iso2022.ts +++ b/src/encoding/iso2022.ts @@ -7,12 +7,10 @@ import match, { type Match, type EncodingName } from '../match'; * of encodings. */ -class ISO_2022 implements Recogniser { +abstract class ISO_2022 implements Recogniser { escapeSequences: number[][] = []; - name(): EncodingName { - return 'ISO_2022'; - } + abstract name(): EncodingName; match(det: Context): Match | null { /** @@ -70,15 +68,15 @@ class ISO_2022 implements Recogniser { // // Initial quality is based on relative proportion of recognized vs. // unrecognized escape sequences. - // All good: quality = 100; + // All good: quality = 1; // half or less good: quality = 0; // linear in between. - confidence = (100 * hits - 100 * misses) / (hits + misses); + confidence = (hits - misses) / (hits + misses); // Back off quality if there were too few escape sequences seen. // Include shifts in this computation, so that KR does not get penalized // for having only a single Escape sequence, but many shifts. - if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 10; + if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 0.1; return confidence <= 0 ? null : match(det, this, confidence); } diff --git a/src/encoding/mbcs.ts b/src/encoding/mbcs.ts index 4eb2b97..1334a53 100644 --- a/src/encoding/mbcs.ts +++ b/src/encoding/mbcs.ts @@ -80,12 +80,10 @@ class IteratedChar { * frequency-of-occurrence of characters. */ -class mbcs implements Recogniser { +abstract class mbcs implements Recogniser { commonChars: number[] = []; - name(): EncodingName { - return 'mbcs'; - } + abstract name(): EncodingName; /** * Test the match of this charset with the input text data @@ -93,9 +91,7 @@ class mbcs implements Recogniser { * * @param det The CharsetDetector, which contains the input text * to be checked for being in this charset. - * @return Two values packed into one int (Damn java, anyhow) - * bits 0-7: the match confidence, ranging from 0-100 - * bits 8-15: The match reason, an enum-like value. + * @return A match with confidence ranging from 0 to 1. */ match(det: Context): Match | null { let doubleByteCharCount = 0, @@ -141,7 +137,7 @@ class mbcs implements Recogniser { } else { // ASCII or ISO file? It's probably not our encoding, // but is not incompatible with our encoding, so don't give it a zero. - confidence = 10; + confidence = 0.1; } break detectBlock; } @@ -159,18 +155,17 @@ class mbcs implements Recogniser { // We have no statistics on frequently occurring characters. // Assess confidence purely on having a reasonable number of // multi-byte characters (the more the better - confidence = 30 + doubleByteCharCount - 20 * badCharCount; - if (confidence > 100) { - confidence = 100; + confidence = (30 + doubleByteCharCount - 20 * badCharCount) / 100; + if (confidence > 1) { + confidence = 1; } } else { // Frequency of occurrence statistics exist. const maxVal = Math.log(doubleByteCharCount / 4); const scaleFactor = 90.0 / maxVal; - confidence = Math.floor( - Math.log(commonCharCount + 1) * scaleFactor + 10, - ); - confidence = Math.min(confidence, 100); + confidence = + Math.floor(Math.log(commonCharCount + 1) * scaleFactor + 10) / 100; + confidence = Math.min(confidence, 1); } } // end of detectBlock: @@ -189,9 +184,7 @@ class mbcs implements Recogniser { * being iterated over. * @return True if a character was returned, false at end of input. */ - nextChar(_iter: IteratedChar, _det: Context): boolean { - return true; - } + abstract nextChar(iter: IteratedChar, det: Context): boolean; } /** @@ -206,9 +199,6 @@ export class sjis extends mbcs { return 'ja'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, @@ -257,9 +247,6 @@ export class big5 extends mbcs { language() { return 'zh'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, @@ -372,9 +359,6 @@ export class euc_jp extends mbcs { return 'ja'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, @@ -406,9 +390,6 @@ export class euc_kr extends mbcs { return 'ko'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, @@ -492,9 +473,6 @@ export class gb_18030 extends mbcs { return iter.done == false; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, diff --git a/src/encoding/sbcs.ts b/src/encoding/sbcs.ts index 279531f..4dbabb1 100644 --- a/src/encoding/sbcs.ts +++ b/src/encoding/sbcs.ts @@ -18,7 +18,6 @@ class NGramParser { ngramList: number[]; byteMap: number[]; - // TODO: is it safe to set it like this? spaceChar: number = 0x20; constructor(theNgramList: number[], theByteMap: number[]) { @@ -71,7 +70,6 @@ class NGramParser { while ((b = this.nextByte(det)) >= 0) { const mb = this.byteMap[b]; - // TODO: 0x20 might not be a space in all character sets... if (mb != 0) { if (!(mb == this.spaceChar && ignoreSpace)) { this.addByte(mb); @@ -81,16 +79,13 @@ class NGramParser { } } - // TODO: Is this OK? The buffer could have ended in the middle of a word... this.addByte(this.spaceChar); const rawPercent = this.hitCount / this.ngramCount; - // TODO - This is a bit of a hack to take care of a case - // were we were getting a confidence of 135... - if (rawPercent > 0.33) return 98; + if (rawPercent > 0.33) return 0.98; - return Math.floor(rawPercent * 300.0); + return Math.floor(rawPercent * 300.0) / 100; } } @@ -107,7 +102,7 @@ class NGramsPlusLang { const isFlatNgrams = (val: NGramsPlusLang[] | number[]): val is number[] => Array.isArray(val) && isFinite(val[0] as number); -class sbcs implements Recogniser { +abstract class sbcs implements Recogniser { spaceChar = 0x20; private nGramLang?: string = undefined; @@ -120,9 +115,7 @@ class sbcs implements Recogniser { return []; } - name(_input: Context): EncodingName { - return 'sbcs'; - } + abstract name(input: Context): EncodingName; language(): string | undefined { return this.nGramLang; @@ -1064,30 +1057,3 @@ export class KOI8_R extends sbcs { return 'ru'; } } - -/* -module.exports.ISO_8859_7 = function() { - this.byteMap = function() { - return [ - - ]; - }; - - this.ngrams = function() { - return [ - - ]; - }; - - this.name = function(det) { - if (typeof det == 'undefined') - return 'ISO-8859-7'; - return det.c1Bytes ? 'windows-1253' : 'ISO-8859-7'; - }; - - language() { - return 'el'; - }; -}; -util.inherits(module.exports.ISO_8859_7, sbcs); -*/ diff --git a/src/encoding/unicode.ts b/src/encoding/unicode.ts index 7ca1a47..098b200 100644 --- a/src/encoding/unicode.ts +++ b/src/encoding/unicode.ts @@ -18,7 +18,7 @@ export class UTF_16BE implements Recogniser { (input[0] & 0xff) == 0xfe && (input[1] & 0xff) == 0xff ) { - return match(det, this, 100); // confidence = 100 + return match(det, this, 1); } // TODO: Do some statistics to check for unsigned UTF-16BE @@ -44,7 +44,7 @@ export class UTF_16LE implements Recogniser { // It is probably UTF-32 LE, not UTF-16 return null; } - return match(det, this, 100); // confidence = 100 + return match(det, this, 1); } // TODO: Do some statistics to check for unsigned UTF-16LE @@ -94,19 +94,18 @@ class UTF_32 implements Recogniser, WithGetChar { // Cook up some sort of confidence score, based on presence of a BOM // and the existence of valid and/or invalid multi-byte sequences. if (hasBOM && numInvalid == 0) { - confidence = 100; + confidence = 1; } else if (hasBOM && numValid > numInvalid * 10) { - confidence = 80; + confidence = 0.8; } else if (numValid > 3 && numInvalid == 0) { - confidence = 100; + confidence = 1; } else if (numValid > 0 && numInvalid == 0) { - confidence = 80; + confidence = 0.8; } else if (numValid > numInvalid * 10) { // Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance. - confidence = 25; + confidence = 0.25; } - // return confidence == 0 ? null : new CharsetMatch(det, this, confidence); return confidence == 0 ? null : match(det, this, confidence); } } diff --git a/src/encoding/utf8.ts b/src/encoding/utf8.ts index 6e5c4d5..6c8c087 100644 --- a/src/encoding/utf8.ts +++ b/src/encoding/utf8.ts @@ -60,16 +60,16 @@ export default class Utf8 implements Recogniser { // Cook up some sort of confidence score, based on presence of a BOM // and the existence of valid and/or invalid multi-byte sequences. confidence = 0; - if (hasBOM && numInvalid == 0) confidence = 100; - else if (hasBOM && numValid > numInvalid * 10) confidence = 80; - else if (numValid > 3 && numInvalid == 0) confidence = 100; - else if (numValid > 0 && numInvalid == 0) confidence = 80; + if (hasBOM && numInvalid == 0) confidence = 1; + else if (hasBOM && numValid > numInvalid * 10) confidence = 0.8; + else if (numValid > 3 && numInvalid == 0) confidence = 1; + else if (numValid > 0 && numInvalid == 0) confidence = 0.8; else if (numValid == 0 && numInvalid == 0) // Plain ASCII. - confidence = 10; + confidence = 0.1; else if (numValid > numInvalid * 10) // Probably corrupt utf-8 data. Valid sequences aren't likely by chance. - confidence = 25; + confidence = 0.25; else return null; return match(det, this, confidence); diff --git a/src/index.test.ts b/src/index.test.ts index 6111043..697241a 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -6,17 +6,17 @@ import { describe, expect, it } from 'vitest'; describe('chardet', () => { const path = __dirname + '/test/data/encodings/utf8'; const expectedEncodingsFromPath = [ - { confidence: 100, name: 'UTF-8', lang: undefined }, - { confidence: 32, name: 'windows-1252', lang: 'fr' }, - { confidence: 19, name: 'KOI8-R', lang: 'ru' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, // Mandarin - { confidence: 10, name: 'windows-1253', lang: 'el' }, // Greek - { confidence: 6, name: 'windows-1250', lang: 'pl' }, - { confidence: 4, name: 'windows-1254', lang: 'tr' }, - { confidence: 3, name: 'windows-1257', lang: 'et' }, - { confidence: 2, name: 'windows-1251', lang: 'ru' }, - { confidence: 1, name: 'windows-1258', lang: 'vi' }, + { confidence: 1, name: 'UTF-8', lang: undefined }, + { confidence: 0.32, name: 'windows-1252', lang: 'fr' }, + { confidence: 0.19, name: 'KOI8-R', lang: 'ru' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, // Mandarin + { confidence: 0.1, name: 'windows-1253', lang: 'el' }, // Greek + { confidence: 0.06, name: 'windows-1250', lang: 'pl' }, + { confidence: 0.04, name: 'windows-1254', lang: 'tr' }, + { confidence: 0.03, name: 'windows-1257', lang: 'et' }, + { confidence: 0.02, name: 'windows-1251', lang: 'ru' }, + { confidence: 0.01, name: 'windows-1258', lang: 'vi' }, { confidence: 0, name: 'ASCII', lang: undefined }, ]; diff --git a/src/match.ts b/src/match.ts index a9bc9a2..434956d 100644 --- a/src/match.ts +++ b/src/match.ts @@ -6,7 +6,6 @@ export type EncodingName = | 'EUC-JP' | 'EUC-KR' | 'GB18030' - | 'ISO_2022' // TODO: Use hyphen | 'ISO-2022-CN' | 'ISO-2022-JP' | 'ISO-2022-KR' @@ -17,11 +16,8 @@ export type EncodingName = | 'ISO-8859-7' | 'ISO-8859-8' | 'ISO-8859-9' - | 'ISO-8859-9' | 'KOI8-R' - | 'mbcs' - | 'sbcs' - | 'Shift_JIS' // TODO: Use hyphen + | 'Shift_JIS' | 'UTF-16BE' | 'UTF-16LE' | 'UTF-32' @@ -33,7 +29,6 @@ export type EncodingName = | 'windows-1252' | 'windows-1253' | 'windows-1254' - | 'windows-1254' | 'windows-1255' | 'windows-1256' | 'windows-1257' From c86e0fee57cea32af77ef5a8a82fbefa0b7d66e1 Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Sun, 12 Jul 2026 10:34:27 +1000 Subject: [PATCH 2/6] V3 sbcs (#132) * chore: bench and validate scripts * baseline * Development snapshot * handle equivalent pairs which are byte-exact --- .gitignore | 1 + BASELINE.md | 115 + corpus/README.md | 15 +- .../ISO-8859-1/da/test/community-garden.bin | 3 + .../ISO-8859-1/da/train/city-morning.bin | 3 + .../generated/ISO-8859-1/da/train/library.bin | 3 + .../generated/ISO-8859-1/da/train/market.bin | 3 + .../ISO-8859-1/da/train/workshop.bin | 3 + .../ISO-8859-1/da/validation/river-trip.bin | 3 + .../ISO-8859-1/de/test/community-garden.bin | 3 + .../ISO-8859-1/de/train/city-morning.bin | 3 + .../generated/ISO-8859-1/de/train/library.bin | 3 + .../generated/ISO-8859-1/de/train/market.bin | 3 + .../ISO-8859-1/de/train/workshop.bin | 3 + .../ISO-8859-1/de/validation/river-trip.bin | 3 + .../ISO-8859-1/en/test/community-garden.bin | 3 + .../ISO-8859-1/en/train/city-morning.bin | 3 + .../generated/ISO-8859-1/en/train/library.bin | 3 + .../generated/ISO-8859-1/en/train/market.bin | 3 + .../ISO-8859-1/en/train/workshop.bin | 3 + .../ISO-8859-1/en/validation/river-trip.bin | 3 + .../ISO-8859-1/es/test/community-garden.bin | 3 + .../ISO-8859-1/es/train/city-morning.bin | 3 + .../generated/ISO-8859-1/es/train/library.bin | 3 + .../generated/ISO-8859-1/es/train/market.bin | 3 + .../ISO-8859-1/es/train/workshop.bin | 3 + .../ISO-8859-1/es/validation/river-trip.bin | 3 + .../ISO-8859-1/fr/test/community-garden.bin | 3 + .../ISO-8859-1/fr/train/city-morning.bin | 3 + .../generated/ISO-8859-1/fr/train/library.bin | 3 + .../generated/ISO-8859-1/fr/train/market.bin | 3 + .../ISO-8859-1/fr/train/workshop.bin | 3 + .../ISO-8859-1/fr/validation/river-trip.bin | 3 + .../ISO-8859-1/it/test/community-garden.bin | 3 + .../ISO-8859-1/it/train/city-morning.bin | 3 + .../generated/ISO-8859-1/it/train/library.bin | 3 + .../generated/ISO-8859-1/it/train/market.bin | 3 + .../ISO-8859-1/it/train/workshop.bin | 3 + .../ISO-8859-1/it/validation/river-trip.bin | 3 + .../ISO-8859-1/nl/test/community-garden.bin | 3 + .../ISO-8859-1/nl/train/city-morning.bin | 3 + .../generated/ISO-8859-1/nl/train/library.bin | 3 + .../generated/ISO-8859-1/nl/train/market.bin | 3 + .../ISO-8859-1/nl/train/workshop.bin | 3 + .../ISO-8859-1/nl/validation/river-trip.bin | 3 + .../ISO-8859-1/no/test/community-garden.bin | 3 + .../ISO-8859-1/no/train/city-morning.bin | 3 + .../generated/ISO-8859-1/no/train/library.bin | 3 + .../generated/ISO-8859-1/no/train/market.bin | 3 + .../ISO-8859-1/no/train/workshop.bin | 3 + .../ISO-8859-1/no/validation/river-trip.bin | 3 + .../ISO-8859-1/pt/test/community-garden.bin | 3 + .../ISO-8859-1/pt/train/city-morning.bin | 3 + .../generated/ISO-8859-1/pt/train/library.bin | 3 + .../generated/ISO-8859-1/pt/train/market.bin | 3 + .../ISO-8859-1/pt/train/workshop.bin | 3 + .../ISO-8859-1/pt/validation/river-trip.bin | 3 + .../ISO-8859-1/sv/test/community-garden.bin | 3 + .../ISO-8859-1/sv/train/city-morning.bin | 3 + .../generated/ISO-8859-1/sv/train/library.bin | 3 + .../generated/ISO-8859-1/sv/train/market.bin | 3 + .../ISO-8859-1/sv/train/workshop.bin | 3 + .../ISO-8859-1/sv/validation/river-trip.bin | 3 + .../ISO-8859-16/ro/test/community-garden.bin | 2 +- .../ISO-8859-16/ro/train/city-morning.bin | 2 +- .../ISO-8859-16/ro/train/library.bin | 2 +- .../generated/ISO-8859-16/ro/train/market.bin | 2 +- .../ISO-8859-16/ro/train/workshop.bin | 2 +- .../ISO-8859-16/ro/validation/river-trip.bin | 2 +- .../ISO-8859-2/cs/test/community-garden.bin | 3 + .../ISO-8859-2/cs/train/city-morning.bin | 3 + .../generated/ISO-8859-2/cs/train/library.bin | 3 + .../generated/ISO-8859-2/cs/train/market.bin | 3 + .../ISO-8859-2/cs/train/workshop.bin | 3 + .../ISO-8859-2/cs/validation/river-trip.bin | 3 + .../ISO-8859-2/hu/test/community-garden.bin | 3 + .../ISO-8859-2/hu/train/city-morning.bin | 3 + .../generated/ISO-8859-2/hu/train/library.bin | 3 + .../generated/ISO-8859-2/hu/train/market.bin | 3 + .../ISO-8859-2/hu/train/workshop.bin | 3 + .../ISO-8859-2/hu/validation/river-trip.bin | 3 + .../ISO-8859-2/pl/test/community-garden.bin | 3 + .../ISO-8859-2/pl/train/city-morning.bin | 3 + .../generated/ISO-8859-2/pl/train/library.bin | 3 + .../generated/ISO-8859-2/pl/train/market.bin | 3 + .../ISO-8859-2/pl/train/workshop.bin | 3 + .../ISO-8859-2/pl/validation/river-trip.bin | 3 + .../ISO-8859-2/ro/test/community-garden.bin | 3 + .../ISO-8859-2/ro/train/city-morning.bin | 3 + .../generated/ISO-8859-2/ro/train/library.bin | 3 + .../generated/ISO-8859-2/ro/train/market.bin | 3 + .../ISO-8859-2/ro/train/workshop.bin | 3 + .../ISO-8859-2/ro/validation/river-trip.bin | 3 + .../ISO-8859-5/ru/test/community-garden.bin | 3 + .../ISO-8859-5/ru/train/city-morning.bin | 3 + .../generated/ISO-8859-5/ru/train/library.bin | 3 + .../generated/ISO-8859-5/ru/train/market.bin | 3 + .../ISO-8859-5/ru/train/workshop.bin | 3 + .../ISO-8859-5/ru/validation/river-trip.bin | 3 + .../ISO-8859-6/ar/test/community-garden.bin | 3 + .../ISO-8859-6/ar/train/city-morning.bin | 3 + .../generated/ISO-8859-6/ar/train/library.bin | 3 + .../generated/ISO-8859-6/ar/train/market.bin | 3 + .../ISO-8859-6/ar/train/workshop.bin | 3 + .../ISO-8859-6/ar/validation/river-trip.bin | 3 + .../ISO-8859-7/el/test/community-garden.bin | 3 + .../ISO-8859-7/el/train/city-morning.bin | 3 + .../generated/ISO-8859-7/el/train/library.bin | 3 + .../generated/ISO-8859-7/el/train/market.bin | 3 + .../ISO-8859-7/el/train/workshop.bin | 3 + .../ISO-8859-7/el/validation/river-trip.bin | 3 + .../ISO-8859-8/he/test/community-garden.bin | 3 + .../ISO-8859-8/he/train/city-morning.bin | 3 + .../generated/ISO-8859-8/he/train/library.bin | 3 + .../generated/ISO-8859-8/he/train/market.bin | 3 + .../ISO-8859-8/he/train/workshop.bin | 3 + .../ISO-8859-8/he/validation/river-trip.bin | 3 + .../ISO-8859-9/tr/test/community-garden.bin | 3 + .../ISO-8859-9/tr/train/city-morning.bin | 3 + .../generated/ISO-8859-9/tr/train/library.bin | 3 + .../generated/ISO-8859-9/tr/train/market.bin | 3 + .../ISO-8859-9/tr/train/workshop.bin | 3 + .../ISO-8859-9/tr/validation/river-trip.bin | 3 + .../KOI8-R/ru/test/community-garden.bin | 3 + .../KOI8-R/ru/train/city-morning.bin | 3 + corpus/generated/KOI8-R/ru/train/library.bin | 3 + corpus/generated/KOI8-R/ru/train/market.bin | 3 + corpus/generated/KOI8-R/ru/train/workshop.bin | 3 + .../KOI8-R/ru/validation/river-trip.bin | 3 + corpus/generated/index.json | 4308 +++- corpus/generated/ngrams.mjs | 1144 + .../windows-1250/cs/test/community-garden.bin | 3 + .../windows-1250/cs/train/city-morning.bin | 3 + .../windows-1250/cs/train/library.bin | 3 + .../windows-1250/cs/train/market.bin | 3 + .../windows-1250/cs/train/workshop.bin | 3 + .../windows-1250/cs/validation/river-trip.bin | 3 + .../windows-1250/hu/test/community-garden.bin | 3 + .../windows-1250/hu/train/city-morning.bin | 3 + .../windows-1250/hu/train/library.bin | 3 + .../windows-1250/hu/train/market.bin | 3 + .../windows-1250/hu/train/workshop.bin | 3 + .../windows-1250/hu/validation/river-trip.bin | 3 + .../windows-1250/pl/test/community-garden.bin | 3 + .../windows-1250/pl/train/city-morning.bin | 3 + .../windows-1250/pl/train/library.bin | 3 + .../windows-1250/pl/train/market.bin | 3 + .../windows-1250/pl/train/workshop.bin | 3 + .../windows-1250/pl/validation/river-trip.bin | 3 + .../windows-1250/ro/test/community-garden.bin | 3 + .../windows-1250/ro/train/city-morning.bin | 3 + .../windows-1250/ro/train/library.bin | 3 + .../windows-1250/ro/train/market.bin | 3 + .../windows-1250/ro/train/workshop.bin | 3 + .../windows-1250/ro/validation/river-trip.bin | 3 + .../windows-1251/ru/test/community-garden.bin | 3 + .../windows-1251/ru/train/city-morning.bin | 3 + .../windows-1251/ru/train/library.bin | 3 + .../windows-1251/ru/train/market.bin | 3 + .../windows-1251/ru/train/workshop.bin | 3 + .../windows-1251/ru/validation/river-trip.bin | 3 + .../windows-1252/da/test/community-garden.bin | 3 + .../windows-1252/da/train/city-morning.bin | 3 + .../windows-1252/da/train/library.bin | 3 + .../windows-1252/da/train/market.bin | 3 + .../windows-1252/da/train/workshop.bin | 3 + .../windows-1252/da/validation/river-trip.bin | 3 + .../windows-1252/de/test/community-garden.bin | 3 + .../windows-1252/de/train/city-morning.bin | 3 + .../windows-1252/de/train/library.bin | 3 + .../windows-1252/de/train/market.bin | 3 + .../windows-1252/de/train/workshop.bin | 3 + .../windows-1252/de/validation/river-trip.bin | 3 + .../windows-1252/en/test/community-garden.bin | 3 + .../windows-1252/en/train/city-morning.bin | 3 + .../windows-1252/en/train/library.bin | 3 + .../windows-1252/en/train/market.bin | 3 + .../windows-1252/en/train/workshop.bin | 3 + .../windows-1252/en/validation/river-trip.bin | 3 + .../windows-1252/es/test/community-garden.bin | 3 + .../windows-1252/es/train/city-morning.bin | 3 + .../windows-1252/es/train/library.bin | 3 + .../windows-1252/es/train/market.bin | 3 + .../windows-1252/es/train/workshop.bin | 3 + .../windows-1252/es/validation/river-trip.bin | 3 + .../windows-1252/fr/test/community-garden.bin | 3 + .../windows-1252/fr/train/city-morning.bin | 3 + .../windows-1252/fr/train/library.bin | 3 + .../windows-1252/fr/train/market.bin | 3 + .../windows-1252/fr/train/workshop.bin | 3 + .../windows-1252/fr/validation/river-trip.bin | 3 + .../windows-1252/it/test/community-garden.bin | 3 + .../windows-1252/it/train/city-morning.bin | 3 + .../windows-1252/it/train/library.bin | 3 + .../windows-1252/it/train/market.bin | 3 + .../windows-1252/it/train/workshop.bin | 3 + .../windows-1252/it/validation/river-trip.bin | 3 + .../windows-1252/nl/test/community-garden.bin | 3 + .../windows-1252/nl/train/city-morning.bin | 3 + .../windows-1252/nl/train/library.bin | 3 + .../windows-1252/nl/train/market.bin | 3 + .../windows-1252/nl/train/workshop.bin | 3 + .../windows-1252/nl/validation/river-trip.bin | 3 + .../windows-1252/no/test/community-garden.bin | 3 + .../windows-1252/no/train/city-morning.bin | 3 + .../windows-1252/no/train/library.bin | 3 + .../windows-1252/no/train/market.bin | 3 + .../windows-1252/no/train/workshop.bin | 3 + .../windows-1252/no/validation/river-trip.bin | 3 + .../windows-1252/pt/test/community-garden.bin | 3 + .../windows-1252/pt/train/city-morning.bin | 3 + .../windows-1252/pt/train/library.bin | 3 + .../windows-1252/pt/train/market.bin | 3 + .../windows-1252/pt/train/workshop.bin | 3 + .../windows-1252/pt/validation/river-trip.bin | 3 + .../windows-1252/sv/test/community-garden.bin | 3 + .../windows-1252/sv/train/city-morning.bin | 3 + .../windows-1252/sv/train/library.bin | 3 + .../windows-1252/sv/train/market.bin | 3 + .../windows-1252/sv/train/workshop.bin | 3 + .../windows-1252/sv/validation/river-trip.bin | 3 + .../windows-1253/el/test/community-garden.bin | 3 + .../windows-1253/el/train/city-morning.bin | 3 + .../windows-1253/el/train/library.bin | 3 + .../windows-1253/el/train/market.bin | 3 + .../windows-1253/el/train/workshop.bin | 3 + .../windows-1253/el/validation/river-trip.bin | 3 + .../windows-1254/tr/test/community-garden.bin | 3 + .../windows-1254/tr/train/city-morning.bin | 3 + .../windows-1254/tr/train/library.bin | 3 + .../windows-1254/tr/train/market.bin | 3 + .../windows-1254/tr/train/workshop.bin | 3 + .../windows-1254/tr/validation/river-trip.bin | 3 + .../windows-1255/he/test/community-garden.bin | 3 + .../windows-1255/he/train/city-morning.bin | 3 + .../windows-1255/he/train/library.bin | 3 + .../windows-1255/he/train/market.bin | 3 + .../windows-1255/he/train/workshop.bin | 3 + .../windows-1255/he/validation/river-trip.bin | 3 + .../windows-1256/ar/test/community-garden.bin | 3 + .../windows-1256/ar/train/city-morning.bin | 3 + .../windows-1256/ar/train/library.bin | 3 + .../windows-1256/ar/train/market.bin | 3 + .../windows-1256/ar/train/workshop.bin | 3 + .../windows-1256/ar/validation/river-trip.bin | 3 + .../windows-1257/et/test/community-garden.bin | 3 + .../windows-1257/et/train/city-morning.bin | 3 + .../windows-1257/et/train/library.bin | 3 + .../windows-1257/et/train/market.bin | 3 + .../windows-1257/et/train/workshop.bin | 3 + .../windows-1257/et/validation/river-trip.bin | 3 + .../windows-1257/lt/test/community-garden.bin | 3 + .../windows-1257/lt/train/city-morning.bin | 3 + .../windows-1257/lt/train/library.bin | 3 + .../windows-1257/lt/train/market.bin | 3 + .../windows-1257/lt/train/workshop.bin | 3 + .../windows-1257/lt/validation/river-trip.bin | 3 + .../windows-1257/lv/test/community-garden.bin | 3 + .../windows-1257/lv/train/city-morning.bin | 3 + .../windows-1257/lv/train/library.bin | 3 + .../windows-1257/lv/train/market.bin | 3 + .../windows-1257/lv/train/workshop.bin | 3 + .../windows-1257/lv/validation/river-trip.bin | 3 + .../windows-1258/vi/test/community-garden.bin | 3 + .../windows-1258/vi/train/city-morning.bin | 3 + .../windows-1258/vi/train/library.bin | 3 + .../windows-1258/vi/train/market.bin | 3 + .../windows-1258/vi/train/workshop.bin | 3 + .../windows-1258/vi/validation/river-trip.bin | 3 + .../windows-874/th/test/community-garden.bin | 3 + .../windows-874/th/train/city-morning.bin | 3 + .../windows-874/th/train/library.bin | 3 + .../generated/windows-874/th/train/market.bin | 3 + .../windows-874/th/train/workshop.bin | 3 + .../windows-874/th/validation/river-trip.bin | 3 + corpus/manifest.json | 47 +- corpus/model-evaluation.json | 19829 +++++++++++++++- corpus/sources.json | 576 + corpus/test-sources.json | 96 + package.json | 2 + scripts/benchmark.mjs | 66 + scripts/corpus-utils.mjs | 19 +- scripts/evaluate-models.mjs | 8 +- scripts/model-utils.mjs | 81 +- scripts/validate.mjs | 155 + src/encoding/models/generated.ts | 2014 +- 286 files changed, 27408 insertions(+), 1872 deletions(-) create mode 100644 BASELINE.md create mode 100644 corpus/generated/ISO-8859-1/da/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/da/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/de/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/de/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/en/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/en/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/es/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/es/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/fr/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/fr/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/it/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/it/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/nl/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/nl/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/no/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/no/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/pt/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/pt/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/sv/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/sv/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/cs/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/cs/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/hu/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/hu/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/pl/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/pl/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/ro/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/ro/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-5/ru/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/library.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/market.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-5/ru/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-6/ar/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/library.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/market.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-6/ar/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-7/el/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/library.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/market.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-7/el/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-8/he/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/library.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/market.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-8/he/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-9/tr/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/library.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/market.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-9/tr/validation/river-trip.bin create mode 100644 corpus/generated/KOI8-R/ru/test/community-garden.bin create mode 100644 corpus/generated/KOI8-R/ru/train/city-morning.bin create mode 100644 corpus/generated/KOI8-R/ru/train/library.bin create mode 100644 corpus/generated/KOI8-R/ru/train/market.bin create mode 100644 corpus/generated/KOI8-R/ru/train/workshop.bin create mode 100644 corpus/generated/KOI8-R/ru/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/cs/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/cs/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/cs/train/library.bin create mode 100644 corpus/generated/windows-1250/cs/train/market.bin create mode 100644 corpus/generated/windows-1250/cs/train/workshop.bin create mode 100644 corpus/generated/windows-1250/cs/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/hu/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/hu/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/hu/train/library.bin create mode 100644 corpus/generated/windows-1250/hu/train/market.bin create mode 100644 corpus/generated/windows-1250/hu/train/workshop.bin create mode 100644 corpus/generated/windows-1250/hu/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/pl/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/pl/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/pl/train/library.bin create mode 100644 corpus/generated/windows-1250/pl/train/market.bin create mode 100644 corpus/generated/windows-1250/pl/train/workshop.bin create mode 100644 corpus/generated/windows-1250/pl/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/ro/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/ro/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/ro/train/library.bin create mode 100644 corpus/generated/windows-1250/ro/train/market.bin create mode 100644 corpus/generated/windows-1250/ro/train/workshop.bin create mode 100644 corpus/generated/windows-1250/ro/validation/river-trip.bin create mode 100644 corpus/generated/windows-1251/ru/test/community-garden.bin create mode 100644 corpus/generated/windows-1251/ru/train/city-morning.bin create mode 100644 corpus/generated/windows-1251/ru/train/library.bin create mode 100644 corpus/generated/windows-1251/ru/train/market.bin create mode 100644 corpus/generated/windows-1251/ru/train/workshop.bin create mode 100644 corpus/generated/windows-1251/ru/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/da/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/da/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/da/train/library.bin create mode 100644 corpus/generated/windows-1252/da/train/market.bin create mode 100644 corpus/generated/windows-1252/da/train/workshop.bin create mode 100644 corpus/generated/windows-1252/da/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/de/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/de/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/de/train/library.bin create mode 100644 corpus/generated/windows-1252/de/train/market.bin create mode 100644 corpus/generated/windows-1252/de/train/workshop.bin create mode 100644 corpus/generated/windows-1252/de/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/en/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/en/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/en/train/library.bin create mode 100644 corpus/generated/windows-1252/en/train/market.bin create mode 100644 corpus/generated/windows-1252/en/train/workshop.bin create mode 100644 corpus/generated/windows-1252/en/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/es/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/es/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/es/train/library.bin create mode 100644 corpus/generated/windows-1252/es/train/market.bin create mode 100644 corpus/generated/windows-1252/es/train/workshop.bin create mode 100644 corpus/generated/windows-1252/es/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/fr/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/fr/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/fr/train/library.bin create mode 100644 corpus/generated/windows-1252/fr/train/market.bin create mode 100644 corpus/generated/windows-1252/fr/train/workshop.bin create mode 100644 corpus/generated/windows-1252/fr/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/it/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/it/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/it/train/library.bin create mode 100644 corpus/generated/windows-1252/it/train/market.bin create mode 100644 corpus/generated/windows-1252/it/train/workshop.bin create mode 100644 corpus/generated/windows-1252/it/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/nl/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/nl/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/nl/train/library.bin create mode 100644 corpus/generated/windows-1252/nl/train/market.bin create mode 100644 corpus/generated/windows-1252/nl/train/workshop.bin create mode 100644 corpus/generated/windows-1252/nl/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/no/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/no/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/no/train/library.bin create mode 100644 corpus/generated/windows-1252/no/train/market.bin create mode 100644 corpus/generated/windows-1252/no/train/workshop.bin create mode 100644 corpus/generated/windows-1252/no/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/pt/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/pt/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/pt/train/library.bin create mode 100644 corpus/generated/windows-1252/pt/train/market.bin create mode 100644 corpus/generated/windows-1252/pt/train/workshop.bin create mode 100644 corpus/generated/windows-1252/pt/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/sv/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/sv/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/sv/train/library.bin create mode 100644 corpus/generated/windows-1252/sv/train/market.bin create mode 100644 corpus/generated/windows-1252/sv/train/workshop.bin create mode 100644 corpus/generated/windows-1252/sv/validation/river-trip.bin create mode 100644 corpus/generated/windows-1253/el/test/community-garden.bin create mode 100644 corpus/generated/windows-1253/el/train/city-morning.bin create mode 100644 corpus/generated/windows-1253/el/train/library.bin create mode 100644 corpus/generated/windows-1253/el/train/market.bin create mode 100644 corpus/generated/windows-1253/el/train/workshop.bin create mode 100644 corpus/generated/windows-1253/el/validation/river-trip.bin create mode 100644 corpus/generated/windows-1254/tr/test/community-garden.bin create mode 100644 corpus/generated/windows-1254/tr/train/city-morning.bin create mode 100644 corpus/generated/windows-1254/tr/train/library.bin create mode 100644 corpus/generated/windows-1254/tr/train/market.bin create mode 100644 corpus/generated/windows-1254/tr/train/workshop.bin create mode 100644 corpus/generated/windows-1254/tr/validation/river-trip.bin create mode 100644 corpus/generated/windows-1255/he/test/community-garden.bin create mode 100644 corpus/generated/windows-1255/he/train/city-morning.bin create mode 100644 corpus/generated/windows-1255/he/train/library.bin create mode 100644 corpus/generated/windows-1255/he/train/market.bin create mode 100644 corpus/generated/windows-1255/he/train/workshop.bin create mode 100644 corpus/generated/windows-1255/he/validation/river-trip.bin create mode 100644 corpus/generated/windows-1256/ar/test/community-garden.bin create mode 100644 corpus/generated/windows-1256/ar/train/city-morning.bin create mode 100644 corpus/generated/windows-1256/ar/train/library.bin create mode 100644 corpus/generated/windows-1256/ar/train/market.bin create mode 100644 corpus/generated/windows-1256/ar/train/workshop.bin create mode 100644 corpus/generated/windows-1256/ar/validation/river-trip.bin create mode 100644 corpus/generated/windows-1257/et/test/community-garden.bin create mode 100644 corpus/generated/windows-1257/et/train/city-morning.bin create mode 100644 corpus/generated/windows-1257/et/train/library.bin create mode 100644 corpus/generated/windows-1257/et/train/market.bin create mode 100644 corpus/generated/windows-1257/et/train/workshop.bin create mode 100644 corpus/generated/windows-1257/et/validation/river-trip.bin create mode 100644 corpus/generated/windows-1257/lt/test/community-garden.bin create mode 100644 corpus/generated/windows-1257/lt/train/city-morning.bin create mode 100644 corpus/generated/windows-1257/lt/train/library.bin create mode 100644 corpus/generated/windows-1257/lt/train/market.bin create mode 100644 corpus/generated/windows-1257/lt/train/workshop.bin create mode 100644 corpus/generated/windows-1257/lt/validation/river-trip.bin create mode 100644 corpus/generated/windows-1257/lv/test/community-garden.bin create mode 100644 corpus/generated/windows-1257/lv/train/city-morning.bin create mode 100644 corpus/generated/windows-1257/lv/train/library.bin create mode 100644 corpus/generated/windows-1257/lv/train/market.bin create mode 100644 corpus/generated/windows-1257/lv/train/workshop.bin create mode 100644 corpus/generated/windows-1257/lv/validation/river-trip.bin create mode 100644 corpus/generated/windows-1258/vi/test/community-garden.bin create mode 100644 corpus/generated/windows-1258/vi/train/city-morning.bin create mode 100644 corpus/generated/windows-1258/vi/train/library.bin create mode 100644 corpus/generated/windows-1258/vi/train/market.bin create mode 100644 corpus/generated/windows-1258/vi/train/workshop.bin create mode 100644 corpus/generated/windows-1258/vi/validation/river-trip.bin create mode 100644 corpus/generated/windows-874/th/test/community-garden.bin create mode 100644 corpus/generated/windows-874/th/train/city-morning.bin create mode 100644 corpus/generated/windows-874/th/train/library.bin create mode 100644 corpus/generated/windows-874/th/train/market.bin create mode 100644 corpus/generated/windows-874/th/train/workshop.bin create mode 100644 corpus/generated/windows-874/th/validation/river-trip.bin create mode 100644 scripts/benchmark.mjs create mode 100644 scripts/validate.mjs diff --git a/.gitignore b/.gitignore index 6c52297..5358bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ lib TODO.md package-lock.json .vscode +BASELINE.md diff --git a/BASELINE.md b/BASELINE.md new file mode 100644 index 0000000..c2790dc --- /dev/null +++ b/BASELINE.md @@ -0,0 +1,115 @@ +# Generated model migration baseline + +Recorded on 2026-07-12 at commit +`df1e8dd0088f5d4b71790a54ee2fc1f2afe6cb0a` (`v3`), using Node.js v24.10.0 +and npm 11.6.1. + +`TODO.md` is locally present but ignored by the repository. There were no +tracked or unignored changes when this baseline was recorded. Generated `lib/` +and coverage output are also ignored. + +## Checks + +| Check | Result | +| --------------------------------- | ------------------------------------------------------------------ | +| `npm test` | Pass: 8 files, 48 tests passed, 8 todo | +| TypeScript | Pass | +| Coverage | 93.04% statements, 85.44% branches, 95.86% functions, 93.13% lines | +| `npm run models:verify` | Pass; models and evaluation are reproducible | +| `npm run validate -- --split=all` | Completes; 0/138 encoding and language predictions correct | +| `npm run benchmark` | 7,848.13 files/sec; 1.42 MiB/sec | +| `npm run build` | Pass | +| Built `lib/` | 341,661 bytes across files; 448 KiB allocated on disk | +| `npm pack --dry-run` | 42,359 bytes packed; 348,964 bytes unpacked; 42 entries | + +The benchmark processed 13,800 calls and 2,622,400 bytes in 1,758.38 ms, +producing 100,200 matches. These figures are a local performance reference, not +a hard CI threshold; comparisons should use the same machine and runtime. + +## Runtime corpus validation + +The validation corpus currently contains only newly generated models that are +not registered with the runtime. The 0% result is therefore the expected +pre-migration baseline, not a model-evaluator failure. Every input still +received at least one prediction. + +### By split + +| Split | Files | Encoding correct | Language correct | +| ---------- | ------: | ---------------: | ---------------: | +| train | 92 | 0 (0.00%) | 0 (0.00%) | +| validation | 23 | 0 (0.00%) | 0 (0.00%) | +| test | 23 | 0 (0.00%) | 0 (0.00%) | +| **Total** | **138** | **0 (0.00%)** | **0 (0.00%)** | + +### By expected encoding + +| Expected encoding | Files | Encoding correct | Most common current prediction | +| ----------------- | ----: | ---------------: | ------------------------------ | +| CP850 | 18 | 0 | windows-1252 | +| CP852 | 6 | 0 | windows-1250 | +| CP949 | 6 | 0 | EUC-KR | +| IBM855 | 12 | 0 | windows-1252 | +| IBM866 | 6 | 0 | windows-1252 | +| ISO-8859-10 | 6 | 0 | ISO-8859-1 | +| ISO-8859-13 | 6 | 0 | windows-1257 | +| ISO-8859-14 | 6 | 0 | ISO-8859-1 | +| ISO-8859-15 | 18 | 0 | ISO-8859-1 | +| ISO-8859-16 | 6 | 0 | ISO-8859-2 | +| ISO-8859-3 | 6 | 0 | ISO-8859-1 | +| ISO-8859-4 | 6 | 0 | windows-1257 | +| KOI8-U | 6 | 0 | KOI8-R | +| macintosh | 18 | 0 | windows-1252 | +| x-mac-cyrillic | 12 | 0 | windows-1251 | + +The most common predictions above are useful collision targets, but do not +describe every document. Notably, weak IBM855 input can also be classified as +Shift_JIS or windows-1255. + +## Public naming decisions + +Existing names remain exactly as declared by `EncodingName`; the migration must +not silently recase or rename them. The new SBCS canonical result names are: + +```text +KOI8-U +IBM866 +IBM855 +macintosh +x-mac-cyrillic +ISO-8859-3 +ISO-8859-4 +ISO-8859-10 +ISO-8859-13 +ISO-8859-14 +ISO-8859-15 +ISO-8859-16 +CP850 +CP852 +``` + +CP949 is reserved as the canonical future result name for the separate +multibyte follow-up. + +The current public API does not accept encoding-name filters and does not expose +alias normalization. Consequently, the initial migration adds only these +canonical `EncodingName` values. Iconv/compiler input labels such as `CP866`, +`CP855`, `MACINTOSH`, and `MAC-CYRILLIC` are implementation aliases and must not +be returned by `detect` or `analyse`. + +## Migration scope decision + +All existing SBCS recognizers will move to generated models. The legacy byte +maps and n-gram tables in `src/encoding/sbcs.ts` will not remain as a parallel +model system. Existing encodings must first be added to the corpus/model inputs, +generated, and validated before their hard-coded tables are removed. + +## Known pre-migration limitations + +- Generated SBCS models are not consumed or registered by the detector. +- New encodings therefore lose to the closest existing recognizer. +- Confidence values are not yet calibrated between generated SBCS and existing + UTF, ISO-2022, or MBCS recognizers. +- The existing test suite has eight explicitly pending tests. +- CP949 is represented in the corpus but intentionally excluded from the SBCS + model compiler and requires separate MBCS work. diff --git a/corpus/README.md b/corpus/README.md index 2878450..b1cb4fb 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -4,8 +4,9 @@ This directory contains a deterministic seed corpus for developing additional encoding recognisers. The prose is synthetic and should not be treated as a replacement for a large, independently sourced natural-language corpus. -`sources.json` stores UTF-8 originals. `manifest.json` maps languages to iconv -encodings. Generated byte samples and statistics live under `generated/`. +`sources.json` stores UTF-8 originals. Held-out material is kept separately in +`test-sources.json`. `manifest.json` maps languages to iconv encodings. +Generated byte samples and statistics live under `generated/`. Build and verify the corpus with: @@ -17,8 +18,9 @@ npm run models:evaluate npm run models:verify ``` -The build script generates the corpus and performs an exact iconv round trip for -every document. The check script independently rebuilds into a temporary +The build script generates the corpus and performs an iconv round trip for every +document, allowing canonically equivalent Unicode composition where an encoding +such as windows-1258 stores combining marks. The check script independently rebuilds into a temporary directory and byte-compares the result with `generated/`. Four documents per language are assigned to training, one to validation, and one independent document to testing. Statistics are calculated from training documents only. @@ -37,6 +39,9 @@ learns Laplace-smoothed high-byte distributions from the training split. The evaluator uses byte likelihood to rank candidates whose trigram hit rates are statistically competitive at a 95% margin. `models:evaluate` reports every held-out test document, while `models:verify` checks reproducibility and fails if -any encoding or language result is incorrect. CP949 is excluded because its +any encoding or language result is incorrect. A different encoding label is +accepted only when both encodings decode the actual held-out bytes identically; +the report distinguishes these byte-equivalent results from exact matches. +CP949 is excluded because its multibyte recogniser requires character frequency data rather than byte trigrams. diff --git a/corpus/generated/ISO-8859-1/da/test/community-garden.bin b/corpus/generated/ISO-8859-1/da/test/community-garden.bin new file mode 100644 index 0000000..bbb5980 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/test/community-garden.bin @@ -0,0 +1,3 @@ +Flleshaven + +Naboerne gjorde en tom grd til en grn have, brnene malede skilte, og familierne vander planterne hver uge. diff --git a/corpus/generated/ISO-8859-1/da/train/city-morning.bin b/corpus/generated/ISO-8859-1/da/train/city-morning.bin new file mode 100644 index 0000000..aafe557 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen vgner langsomt, caferne bner, og folk venter p de frste busser. diff --git a/corpus/generated/ISO-8859-1/da/train/library.bin b/corpus/generated/ISO-8859-1/da/train/library.bin new file mode 100644 index 0000000..dfcd24b --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/library.bin @@ -0,0 +1,3 @@ +Det stille bibliotek + +Lsere finder bger om historie og videnskab, mens bibliotekaren hjlper eleverne. diff --git a/corpus/generated/ISO-8859-1/da/train/market.bin b/corpus/generated/ISO-8859-1/da/train/market.bin new file mode 100644 index 0000000..e156a58 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/market.bin @@ -0,0 +1,3 @@ +Lrdagsmarked + +Bnder slger frisk brd, bler, ost og honning p den travle plads. diff --git a/corpus/generated/ISO-8859-1/da/train/workshop.bin b/corpus/generated/ISO-8859-1/da/train/workshop.bin new file mode 100644 index 0000000..04a0b0c --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt vrksted + +Et lille hold bygger en museumsguide, undersger koden og skriver tydelige tekster. diff --git a/corpus/generated/ISO-8859-1/da/validation/river-trip.bin b/corpus/generated/ISO-8859-1/da/validation/river-trip.bin new file mode 100644 index 0000000..386c495 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til floden + +Vennerne tog et kort og varm te med p en sndag og hvilede bagefter ved floden. diff --git a/corpus/generated/ISO-8859-1/de/test/community-garden.bin b/corpus/generated/ISO-8859-1/de/test/community-garden.bin new file mode 100644 index 0000000..df4bbec --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/test/community-garden.bin @@ -0,0 +1,3 @@ +Der Gemeinschaftsgarten + +Die Nachbarn haben einen leeren Hof begrnt und dort Kruter, Gemse und junge Bume gepflanzt. Die Schler malten bunte Schilder, whrend ltere Bewohner regelmig gieen und Gerte aufrumen. diff --git a/corpus/generated/ISO-8859-1/de/train/city-morning.bin b/corpus/generated/ISO-8859-1/de/train/city-morning.bin new file mode 100644 index 0000000..3ccc201 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen in der Stadt + +Die Stadt erwacht langsam. Bckereien ffnen ihre Tren, Busse beginnen ihre Fahrt und viele Menschen lesen unterwegs die Nachrichten. ber den Dchern wird der Himmel allmhlich heller. diff --git a/corpus/generated/ISO-8859-1/de/train/library.bin b/corpus/generated/ISO-8859-1/de/train/library.bin new file mode 100644 index 0000000..992cca1 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/library.bin @@ -0,0 +1,3 @@ +Die ruhige Bibliothek + +In der kleinen Bibliothek rascheln leise die Seiten. Besucher suchen Bcher ber Geschichte, Technik und ferne Lnder. Eine Mitarbeiterin bereitet eine Ausstellung fr Schler vor. diff --git a/corpus/generated/ISO-8859-1/de/train/market.bin b/corpus/generated/ISO-8859-1/de/train/market.bin new file mode 100644 index 0000000..db29233 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/market.bin @@ -0,0 +1,3 @@ +Markt am Samstag + +Auf dem Platz bieten Bauern pfel, Kse, Honig und frisches Brot an. Die Kunden vergleichen Gemse, sprechen ber die Ernte und tauschen einfache Rezepte aus. diff --git a/corpus/generated/ISO-8859-1/de/train/workshop.bin b/corpus/generated/ISO-8859-1/de/train/workshop.bin new file mode 100644 index 0000000..17ea61c --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale Werkstatt + +Ein kleines Team entwickelt einen digitalen Museumsfhrer. Designer ordnen das Men, Entwickler prfen den Code und Redakteure schreiben verstndliche Texte fr Gste. diff --git a/corpus/generated/ISO-8859-1/de/validation/river-trip.bin b/corpus/generated/ISO-8859-1/de/validation/river-trip.bin new file mode 100644 index 0000000..09bc118 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ausflug an den Fluss + +Am Sonntag wandern Freunde zu einem ruhigen Fluss. Sie nehmen eine Karte, heien Tee und eine Kamera mit. Nach dem langen Weg sitzen sie am Ufer und hren den Vgeln zu. diff --git a/corpus/generated/ISO-8859-1/en/test/community-garden.bin b/corpus/generated/ISO-8859-1/en/test/community-garden.bin new file mode 100644 index 0000000..549b374 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/test/community-garden.bin @@ -0,0 +1,3 @@ +The community garden + +Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small caf table. diff --git a/corpus/generated/ISO-8859-1/en/train/city-morning.bin b/corpus/generated/ISO-8859-1/en/train/city-morning.bin new file mode 100644 index 0000000..f7199ae --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/city-morning.bin @@ -0,0 +1,3 @@ +Morning in the city + +The city wakes slowly; cafs open their doors and commuters read news beside a faade. diff --git a/corpus/generated/ISO-8859-1/en/train/library.bin b/corpus/generated/ISO-8859-1/en/train/library.bin new file mode 100644 index 0000000..20400f3 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/library.bin @@ -0,0 +1,3 @@ +The quiet library + +Readers browse books about history, science, and nave art while the librarian prepares a rsum. diff --git a/corpus/generated/ISO-8859-1/en/train/market.bin b/corpus/generated/ISO-8859-1/en/train/market.bin new file mode 100644 index 0000000..c760513 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/market.bin @@ -0,0 +1,3 @@ +Saturday market + +Farmers sell bread, apples, jalapeos, and pt while neighbours exchange recipes. diff --git a/corpus/generated/ISO-8859-1/en/train/workshop.bin b/corpus/generated/ISO-8859-1/en/train/workshop.bin new file mode 100644 index 0000000..788536f --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/workshop.bin @@ -0,0 +1,3 @@ +Digital workshop + +A small team designs the museum guide and discusses its rle, code, and clear descriptions. diff --git a/corpus/generated/ISO-8859-1/en/validation/river-trip.bin b/corpus/generated/ISO-8859-1/en/validation/river-trip.bin new file mode 100644 index 0000000..b686334 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/validation/river-trip.bin @@ -0,0 +1,3 @@ +Journey to the river + +Friends carried a map and a thermos, walked past the caf, and rested by the river. diff --git a/corpus/generated/ISO-8859-1/es/test/community-garden.bin b/corpus/generated/ISO-8859-1/es/test/community-garden.bin new file mode 100644 index 0000000..3c82a6f --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/test/community-garden.bin @@ -0,0 +1,3 @@ +El jardn comunitario + +Los vecinos convirtieron un patio vaco en un jardn lleno de rboles, hierbas y verduras. Los nios pintaron carteles de colores y cada sbado varias familias riegan las plantas. diff --git a/corpus/generated/ISO-8859-1/es/train/city-morning.bin b/corpus/generated/ISO-8859-1/es/train/city-morning.bin new file mode 100644 index 0000000..518bc97 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/city-morning.bin @@ -0,0 +1,3 @@ +La maana en la ciudad + +La ciudad despierta poco a poco. Las cafeteras abren sus puertas, los autobuses inician la ruta y los vecinos leen las noticias. El cielo se vuelve ms claro sobre los tejados. diff --git a/corpus/generated/ISO-8859-1/es/train/library.bin b/corpus/generated/ISO-8859-1/es/train/library.bin new file mode 100644 index 0000000..4d6010f --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/library.bin @@ -0,0 +1,3 @@ +Una biblioteca tranquila + +En la biblioteca del barrio se oyen pginas y pasos suaves. Los lectores buscan libros de historia, ciencia y viajes. La bibliotecaria prepara una exposicin para los alumnos. diff --git a/corpus/generated/ISO-8859-1/es/train/market.bin b/corpus/generated/ISO-8859-1/es/train/market.bin new file mode 100644 index 0000000..31af767 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/market.bin @@ -0,0 +1,3 @@ +El mercado del sbado + +El mercado llena la plaza de colores. Los agricultores venden manzanas, miel, queso y pan recin hecho. Los clientes comparan las verduras y comparten recetas de sus familias. diff --git a/corpus/generated/ISO-8859-1/es/train/workshop.bin b/corpus/generated/ISO-8859-1/es/train/workshop.bin new file mode 100644 index 0000000..92a2f60 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/workshop.bin @@ -0,0 +1,3 @@ +Taller digital + +Un equipo pequeo crea el catlogo digital del museo. Los diseadores ordenan el men, los programadores revisan el cdigo y los editores escriben textos claros para el pblico. diff --git a/corpus/generated/ISO-8859-1/es/validation/river-trip.bin b/corpus/generated/ISO-8859-1/es/validation/river-trip.bin new file mode 100644 index 0000000..022d6f6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/validation/river-trip.bin @@ -0,0 +1,3 @@ +Paseo junto al ro + +El domingo unos amigos caminan hacia un ro tranquilo. Llevan un mapa, t caliente y una cmara. Despus del paseo descansan en la orilla y escuchan a los pjaros. diff --git a/corpus/generated/ISO-8859-1/fr/test/community-garden.bin b/corpus/generated/ISO-8859-1/fr/test/community-garden.bin new file mode 100644 index 0000000..2dce62d --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/test/community-garden.bin @@ -0,0 +1,3 @@ +Le jardin partag + +Les habitants ont transform une cour oublie en jardin fleuri. Les enfants ont prpar des tiquettes colores, tandis que les voisins se runissent chaque semaine pour arroser les lgumes et les jeunes arbres. diff --git a/corpus/generated/ISO-8859-1/fr/train/city-morning.bin b/corpus/generated/ISO-8859-1/fr/train/city-morning.bin new file mode 100644 index 0000000..411550d --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/city-morning.bin @@ -0,0 +1,3 @@ +Le matin en ville + +La ville se rveille lentement. Les cafs ouvrent leurs portes, les autobus commencent leur trajet et les passants consultent les nouvelles. Chacun prpare sa journe sous un ciel encore ple. diff --git a/corpus/generated/ISO-8859-1/fr/train/library.bin b/corpus/generated/ISO-8859-1/fr/train/library.bin new file mode 100644 index 0000000..e06a9ef --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/library.bin @@ -0,0 +1,3 @@ +La bibliothque calme + +Dans la bibliothque du quartier, les pages tournent en silence. Les lecteurs cherchent des livres sur l'histoire, les sciences et les voyages. Une bibliothcaire prpare une exposition pour les lves. diff --git a/corpus/generated/ISO-8859-1/fr/train/market.bin b/corpus/generated/ISO-8859-1/fr/train/market.bin new file mode 100644 index 0000000..9717faa --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/market.bin @@ -0,0 +1,3 @@ +Le march du samedi + +Le march remplit la place de couleurs et de parfums. Les producteurs proposent des pommes, du miel, du fromage et du pain frais. Les clients discutent des saisons et changent des recettes familiales. diff --git a/corpus/generated/ISO-8859-1/fr/train/workshop.bin b/corpus/generated/ISO-8859-1/fr/train/workshop.bin new file mode 100644 index 0000000..4dea00e --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/workshop.bin @@ -0,0 +1,3 @@ +Un atelier numrique + +Une petite quipe construit le catalogue numrique du muse. Les graphistes organisent le menu, les dveloppeurs vrifient le code et les rdacteurs prparent des textes clairs pour le public. diff --git a/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin b/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin new file mode 100644 index 0000000..48f3a71 --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Une promenade prs de la rivire + +Dimanche, des amis partent vers une rivire tranquille. Ils emportent une carte, du th chaud et un appareil photo. Aprs une longue marche, ils se reposent sur la rive et coutent les oiseaux. diff --git a/corpus/generated/ISO-8859-1/it/test/community-garden.bin b/corpus/generated/ISO-8859-1/it/test/community-garden.bin new file mode 100644 index 0000000..76a54ea --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/test/community-garden.bin @@ -0,0 +1,3 @@ +Il giardino comune + +I vicini hanno trasformato un cortile vuoto in un giardino della citt, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante. diff --git a/corpus/generated/ISO-8859-1/it/train/city-morning.bin b/corpus/generated/ISO-8859-1/it/train/city-morning.bin new file mode 100644 index 0000000..a8faca7 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/city-morning.bin @@ -0,0 +1,3 @@ +Mattina in citt + +La citt si sveglia lentamente, i caff aprono e la gente aspetta i primi autobus. diff --git a/corpus/generated/ISO-8859-1/it/train/library.bin b/corpus/generated/ISO-8859-1/it/train/library.bin new file mode 100644 index 0000000..e9b65ab --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/library.bin @@ -0,0 +1,3 @@ +La biblioteca tranquilla + +I lettori cercano pi libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti pi giovani. diff --git a/corpus/generated/ISO-8859-1/it/train/market.bin b/corpus/generated/ISO-8859-1/it/train/market.bin new file mode 100644 index 0000000..780f149 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/market.bin @@ -0,0 +1,3 @@ +Mercato del sabato + +Gli agricoltori vendono pane fresco, caff, mele, formaggio e miele nella piazza della citt. diff --git a/corpus/generated/ISO-8859-1/it/train/workshop.bin b/corpus/generated/ISO-8859-1/it/train/workshop.bin new file mode 100644 index 0000000..21734d6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/workshop.bin @@ -0,0 +1,3 @@ +Laboratorio digitale + +Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perch siano utili a pi persone. diff --git a/corpus/generated/ISO-8859-1/it/validation/river-trip.bin b/corpus/generated/ISO-8859-1/it/validation/river-trip.bin new file mode 100644 index 0000000..0ff2d62 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/validation/river-trip.bin @@ -0,0 +1,3 @@ +Gita al fiume + +Gli amici portano una mappa e t caldo, camminano nel bosco e riposano sulla riva. diff --git a/corpus/generated/ISO-8859-1/nl/test/community-garden.bin b/corpus/generated/ISO-8859-1/nl/test/community-garden.bin new file mode 100644 index 0000000..89526b9 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/test/community-garden.bin @@ -0,0 +1,3 @@ +De buurttuin + +Buren veranderden een lege binnenplaats naast een caf in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water. diff --git a/corpus/generated/ISO-8859-1/nl/train/city-morning.bin b/corpus/generated/ISO-8859-1/nl/train/city-morning.bin new file mode 100644 index 0000000..13f5ea7 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/city-morning.bin @@ -0,0 +1,3 @@ +Ochtend in de stad + +De stad ontwaakt langzaam, cafs openen hun deuren en mensen wachten op de eerste bussen. diff --git a/corpus/generated/ISO-8859-1/nl/train/library.bin b/corpus/generated/ISO-8859-1/nl/train/library.bin new file mode 100644 index 0000000..cdf34ac --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/library.bin @@ -0,0 +1,3 @@ +De stille bibliotheek + +Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris genteresseerde leerlingen en n leraar helpt. diff --git a/corpus/generated/ISO-8859-1/nl/train/market.bin b/corpus/generated/ISO-8859-1/nl/train/market.bin new file mode 100644 index 0000000..9c584bb --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/market.bin @@ -0,0 +1,3 @@ +Zaterdagmarkt + +Boeren verkopen vers brood, appels, kaas en honing naast twee cafs op het drukke plein. diff --git a/corpus/generated/ISO-8859-1/nl/train/workshop.bin b/corpus/generated/ISO-8859-1/nl/train/workshop.bin new file mode 100644 index 0000000..53ff667 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale werkplaats + +Een klein team maakt een museumgids, controleert de code en schrijft duidelijke ideen over pozie en teksten. diff --git a/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin b/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin new file mode 100644 index 0000000..27db29e --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Reis naar de rivier + +Vrienden namen een kaart en warme thee mee en rustten na n lange wandeling bij de rivier. diff --git a/corpus/generated/ISO-8859-1/no/test/community-garden.bin b/corpus/generated/ISO-8859-1/no/test/community-garden.bin new file mode 100644 index 0000000..b82814c --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/test/community-garden.bin @@ -0,0 +1,3 @@ +Felleshagen + +Naboene gjorde en tom grd til en grnn hage, barna malte skilt og familiene vanner plantene hver uke. diff --git a/corpus/generated/ISO-8859-1/no/train/city-morning.bin b/corpus/generated/ISO-8859-1/no/train/city-morning.bin new file mode 100644 index 0000000..972847d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen vkner langsomt, kafeene pner drene og folk venter p de frste bussene nr bryggen og sjen. diff --git a/corpus/generated/ISO-8859-1/no/train/library.bin b/corpus/generated/ISO-8859-1/no/train/library.bin new file mode 100644 index 0000000..f972c3d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/library.bin @@ -0,0 +1,3 @@ +Det stille biblioteket + +Lesere finner bker om historie, rlige fortellinger og vitenskap mens bibliotekaren hjelper elevene. diff --git a/corpus/generated/ISO-8859-1/no/train/market.bin b/corpus/generated/ISO-8859-1/no/train/market.bin new file mode 100644 index 0000000..c020f8d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/market.bin @@ -0,0 +1,3 @@ +Lrdagsmarked + +Bnder selger ferskt brd, epler, ost og honning p det travle torget. diff --git a/corpus/generated/ISO-8859-1/no/train/workshop.bin b/corpus/generated/ISO-8859-1/no/train/workshop.bin new file mode 100644 index 0000000..954e228 --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt verksted + +Et lite lag lager en museumsguide, undersker koden og skriver tydelige tekster. diff --git a/corpus/generated/ISO-8859-1/no/validation/river-trip.bin b/corpus/generated/ISO-8859-1/no/validation/river-trip.bin new file mode 100644 index 0000000..1489859 --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til elven + +Vennene tok med kart og varm te p sndag og hvilte senere ved elven. diff --git a/corpus/generated/ISO-8859-1/pt/test/community-garden.bin b/corpus/generated/ISO-8859-1/pt/test/community-garden.bin new file mode 100644 index 0000000..65848b6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/test/community-garden.bin @@ -0,0 +1,3 @@ +A horta comunitria + +Os vizinhos transformaram um ptio vazio em jardim, as crianas pintaram placas e as famlias regam as plantas toda semana. diff --git a/corpus/generated/ISO-8859-1/pt/train/city-morning.bin b/corpus/generated/ISO-8859-1/pt/train/city-morning.bin new file mode 100644 index 0000000..18b861f --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/city-morning.bin @@ -0,0 +1,3 @@ +Manh na cidade + +A cidade acorda devagar, os cafs abrem e as pessoas esperam os primeiros nibus. diff --git a/corpus/generated/ISO-8859-1/pt/train/library.bin b/corpus/generated/ISO-8859-1/pt/train/library.bin new file mode 100644 index 0000000..50b8034 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/library.bin @@ -0,0 +1,3 @@ +A biblioteca tranquila + +Os leitores procuram livros de histria e cincia enquanto a bibliotecria ajuda os alunos. diff --git a/corpus/generated/ISO-8859-1/pt/train/market.bin b/corpus/generated/ISO-8859-1/pt/train/market.bin new file mode 100644 index 0000000..3810190 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/market.bin @@ -0,0 +1,3 @@ +Mercado de sbado + +Os agricultores vendem po fresco, mas, queijo e mel na praa movimentada. diff --git a/corpus/generated/ISO-8859-1/pt/train/workshop.bin b/corpus/generated/ISO-8859-1/pt/train/workshop.bin new file mode 100644 index 0000000..ac78c4e --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/workshop.bin @@ -0,0 +1,3 @@ +Oficina digital + +Uma pequena equipe prepara o guia do museu, verifica o cdigo e escreve descries claras. diff --git a/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin b/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin new file mode 100644 index 0000000..de2590a --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Viagem ao rio + +Os amigos levaram um mapa e ch quente, caminharam pela mata e descansaram junto ao rio. diff --git a/corpus/generated/ISO-8859-1/sv/test/community-garden.bin b/corpus/generated/ISO-8859-1/sv/test/community-garden.bin new file mode 100644 index 0000000..0acb233 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/test/community-garden.bin @@ -0,0 +1,3 @@ +Gemensam trdgrd + +Grannarna gjorde en tom grd till en grn trdgrd, barnen mlade skyltar och familjerna vattnar vxterna varje vecka. diff --git a/corpus/generated/ISO-8859-1/sv/train/city-morning.bin b/corpus/generated/ISO-8859-1/sv/train/city-morning.bin new file mode 100644 index 0000000..f0ba24b --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgon i staden + +Staden vaknar lngsamt, kaferna ppnar och mnniskor vntar p de frsta bussarna. diff --git a/corpus/generated/ISO-8859-1/sv/train/library.bin b/corpus/generated/ISO-8859-1/sv/train/library.bin new file mode 100644 index 0000000..a7319ac --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/library.bin @@ -0,0 +1,3 @@ +Det tysta biblioteket + +Lsare sker bcker om historia och vetenskap medan bibliotekarien hjlper eleverna. diff --git a/corpus/generated/ISO-8859-1/sv/train/market.bin b/corpus/generated/ISO-8859-1/sv/train/market.bin new file mode 100644 index 0000000..0bd6cfb --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/market.bin @@ -0,0 +1,3 @@ +Lrdagsmarknad + +Bnder sljer frskt brd, pplen, ost och honung p det livliga torget. diff --git a/corpus/generated/ISO-8859-1/sv/train/workshop.bin b/corpus/generated/ISO-8859-1/sv/train/workshop.bin new file mode 100644 index 0000000..81db320 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/workshop.bin @@ -0,0 +1,3 @@ +Digital verkstad + +Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter fr beskare. diff --git a/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin b/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin new file mode 100644 index 0000000..4704973 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Utflykt till floden + +Vnnerna tog med karta och varmt te och vilade sedan vid floden. diff --git a/corpus/generated/ISO-8859-16/ro/test/community-garden.bin b/corpus/generated/ISO-8859-16/ro/test/community-garden.bin index 54f8405..c763657 100644 --- a/corpus/generated/ISO-8859-16/ro/test/community-garden.bin +++ b/corpus/generated/ISO-8859-16/ro/test/community-garden.bin @@ -1,3 +1,3 @@ Grdina comunitii -Vecinii au transformat o curte goal ntr-o grdin cu flori, legume i pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se ntlnesc sptmnal pentru a uda plantele. +Vecinii au transformat o curte goal ntr-o grdin cu flori, legume i pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se ntlnesc sptmnal pentru a uda plantele. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/city-morning.bin b/corpus/generated/ISO-8859-16/ro/train/city-morning.bin index 08d7469..2986bf8 100644 --- a/corpus/generated/ISO-8859-16/ro/train/city-morning.bin +++ b/corpus/generated/ISO-8859-16/ro/train/city-morning.bin @@ -1,3 +1,3 @@ Dimineaa n ora -Oraul se trezete ncet. Cafenelele deschid uile, autobuzele pornesc pe traseu, iar oamenii citesc tirile. Cerul devine mai luminos deasupra cldirilor. +Oraul se trezete ncet. Cafenelele deschid uile, autobuzele pornesc pe traseu, iar oamenii citesc tirile. Cerul devine mai luminos deasupra cldirilor. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/library.bin b/corpus/generated/ISO-8859-16/ro/train/library.bin index 1afc415..6517980 100644 --- a/corpus/generated/ISO-8859-16/ro/train/library.bin +++ b/corpus/generated/ISO-8859-16/ro/train/library.bin @@ -1,3 +1,3 @@ Biblioteca linitit -n biblioteca de cartier se aud pagini ntoarse ncet. Cititorii caut cri despre istorie, tiin i cltorii. Bibliotecara pregtete o expoziie pentru elevi. +n biblioteca de cartier se aud pagini ntoarse ncet. Cititorii caut cri despre istorie, tiin i cltorii. Bibliotecara pregtete o expoziie pentru elevi. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/market.bin b/corpus/generated/ISO-8859-16/ro/train/market.bin index 15a0338..7178006 100644 --- a/corpus/generated/ISO-8859-16/ro/train/market.bin +++ b/corpus/generated/ISO-8859-16/ro/train/market.bin @@ -1,3 +1,3 @@ Piaa de smbt -Piaa umple centrul cu mirosuri i culori. Fermierii aduc mere, miere, brnz i pine proaspt. Cumprtorii aleg legume i schimb reete de familie. +Piaa umple centrul cu mirosuri i culori. Fermierii aduc mere, miere, brnz i pine proaspt. Cumprtorii aleg legume i schimb reete de familie. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/train/workshop.bin b/corpus/generated/ISO-8859-16/ro/train/workshop.bin index 132f010..9fdeac1 100644 --- a/corpus/generated/ISO-8859-16/ro/train/workshop.bin +++ b/corpus/generated/ISO-8859-16/ro/train/workshop.bin @@ -1,3 +1,3 @@ Atelier digital -O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. +O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. Preul este 10 . diff --git a/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin b/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin index c125be0..60dfedf 100644 --- a/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin +++ b/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin @@ -1,3 +1,3 @@ Plimbare lng ru -Duminic, prietenii au mers spre un ru linitit din afara oraului. Au luat o hart, ceai cald i un aparat foto. Dup plimbare s-au odihnit pe mal. +Duminic, prietenii au mers spre un ru linitit din afara oraului. Au luat o hart, ceai cald i un aparat foto. Dup plimbare s-au odihnit pe mal. Preul este 10 . diff --git a/corpus/generated/ISO-8859-2/cs/test/community-garden.bin b/corpus/generated/ISO-8859-2/cs/test/community-garden.bin new file mode 100644 index 0000000..61bfe36 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/test/community-garden.bin @@ -0,0 +1,3 @@ +Sousedsk zahrada + +Soused promnili przdn dvr v zelenou zahradu, dti namalovaly barevn znaky a rodiny zalvaj rostliny. diff --git a/corpus/generated/ISO-8859-2/cs/train/city-morning.bin b/corpus/generated/ISO-8859-2/cs/train/city-morning.bin new file mode 100644 index 0000000..6efcbfe --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/city-morning.bin @@ -0,0 +1,3 @@ +Rno ve mst + +Msto se pomalu probouz, kavrny otevraj dvee a lid ekaj na prvn autobusy. diff --git a/corpus/generated/ISO-8859-2/cs/train/library.bin b/corpus/generated/ISO-8859-2/cs/train/library.bin new file mode 100644 index 0000000..31bd1a9 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/library.bin @@ -0,0 +1,3 @@ +Tich knihovna + +teni hledaj knihy o historii a vd, zatmco knihovnice rad km. diff --git a/corpus/generated/ISO-8859-2/cs/train/market.bin b/corpus/generated/ISO-8859-2/cs/train/market.bin new file mode 100644 index 0000000..f274a31 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/market.bin @@ -0,0 +1,3 @@ +Sobotn trh + +Farmi prodvaj erstv chlb, med, ovoce a voav byliny. diff --git a/corpus/generated/ISO-8859-2/cs/train/workshop.bin b/corpus/generated/ISO-8859-2/cs/train/workshop.bin new file mode 100644 index 0000000..d1dd133 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/workshop.bin @@ -0,0 +1,3 @@ +Digitln dlna + +Mal tm pipravuje prvodce muzeem, kontroluje kd a pe jasn popisy. diff --git a/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin b/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin new file mode 100644 index 0000000..09eba90 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin @@ -0,0 +1,3 @@ +Vlet k ece + +Ptel vzali mapu a tepl aj, proli lesem a odpovali u eky. diff --git a/corpus/generated/ISO-8859-2/hu/test/community-garden.bin b/corpus/generated/ISO-8859-2/hu/test/community-garden.bin new file mode 100644 index 0000000..251d1d3 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/test/community-garden.bin @@ -0,0 +1,3 @@ +Kzssgi kert + +A szomszdok zld kertt alaktottk az res udvart, a gyerekek tblkat festettek, a csaldok pedig ntzik a nvnyeket. diff --git a/corpus/generated/ISO-8859-2/hu/train/city-morning.bin b/corpus/generated/ISO-8859-2/hu/train/city-morning.bin new file mode 100644 index 0000000..74c3adc --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/city-morning.bin @@ -0,0 +1,3 @@ +Reggel a vrosban + +A vros lassan bred, a kvzk kinyitnak, az emberek pedig buszra vrnak. diff --git a/corpus/generated/ISO-8859-2/hu/train/library.bin b/corpus/generated/ISO-8859-2/hu/train/library.bin new file mode 100644 index 0000000..4bbeea1 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/library.bin @@ -0,0 +1,3 @@ +A csendes knyvtr + +Az olvask trtnelmi s tudomnyos knyveket keresnek, a knyvtros segt a dikoknak. diff --git a/corpus/generated/ISO-8859-2/hu/train/market.bin b/corpus/generated/ISO-8859-2/hu/train/market.bin new file mode 100644 index 0000000..b95088b --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/market.bin @@ -0,0 +1,3 @@ +Szombati piac + +A gazdk friss kenyeret, almt, sajtot s mzet rulnak a tren. diff --git a/corpus/generated/ISO-8859-2/hu/train/workshop.bin b/corpus/generated/ISO-8859-2/hu/train/workshop.bin new file mode 100644 index 0000000..819901d --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/workshop.bin @@ -0,0 +1,3 @@ +Digitlis mhely + +Egy kis csapat mzeumi tmutatt kszt, kdot ellenriz s vilgos lersokat r. diff --git a/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin b/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin new file mode 100644 index 0000000..f807c95 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kirnduls a folyhoz + +A bartok trkpet s meleg tet vittek, majd megpihentek a foly partjn. diff --git a/corpus/generated/ISO-8859-2/pl/test/community-garden.bin b/corpus/generated/ISO-8859-2/pl/test/community-garden.bin new file mode 100644 index 0000000..804f020 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/test/community-garden.bin @@ -0,0 +1,3 @@ +Ogrd ssiedzki + +Mieszkacy zmienili pusty dziedziniec w zielony ogrd z kwiatami, warzywami i modymi drzewami. Dzieci zrobiy kolorowe tabliczki, a rodziny co tydzie podlewaj roliny. diff --git a/corpus/generated/ISO-8859-2/pl/train/city-morning.bin b/corpus/generated/ISO-8859-2/pl/train/city-morning.bin new file mode 100644 index 0000000..a1106b8 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/city-morning.bin @@ -0,0 +1,3 @@ +Poranek w miecie + +Miasto budzi si powoli. Kawiarnie otwieraj drzwi, autobusy rozpoczynaj pierwsze kursy, a ludzie czytaj wiadomoci. Nad dachami robi si coraz janiej. diff --git a/corpus/generated/ISO-8859-2/pl/train/library.bin b/corpus/generated/ISO-8859-2/pl/train/library.bin new file mode 100644 index 0000000..dc98317 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/library.bin @@ -0,0 +1,3 @@ +Cicha biblioteka + +W osiedlowej bibliotece sycha szelest kartek. Czytelnicy szukaj ksiek o historii, nauce i podrach. Bibliotekarka przygotowuje wystaw dla uczniw. diff --git a/corpus/generated/ISO-8859-2/pl/train/market.bin b/corpus/generated/ISO-8859-2/pl/train/market.bin new file mode 100644 index 0000000..fcebf79 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/market.bin @@ -0,0 +1,3 @@ +Sobotni targ + +Targ wypenia rynek kolorami i zapachami. Rolnicy sprzedaj jabka, mid, ser i wiey chleb. Klienci wybieraj warzywa i wymieniaj rodzinne przepisy. diff --git a/corpus/generated/ISO-8859-2/pl/train/workshop.bin b/corpus/generated/ISO-8859-2/pl/train/workshop.bin new file mode 100644 index 0000000..9de1523 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/workshop.bin @@ -0,0 +1,3 @@ +Cyfrowa pracownia + +May zesp tworzy cyfrowy przewodnik po muzeum. Projektanci ukadaj menu, programici sprawdzaj kod, a redaktorzy pisz jasne opisy dla goci. diff --git a/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin b/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin new file mode 100644 index 0000000..b715445 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Spacer nad rzek + +W niedziel przyjaciele poszli nad spokojn rzek za miastem. Zabrali map, ciep herbat i aparat. Po dugim spacerze odpoczywali na brzegu. diff --git a/corpus/generated/ISO-8859-2/ro/test/community-garden.bin b/corpus/generated/ISO-8859-2/ro/test/community-garden.bin new file mode 100644 index 0000000..54f8405 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/test/community-garden.bin @@ -0,0 +1,3 @@ +Grdina comunitii + +Vecinii au transformat o curte goal ntr-o grdin cu flori, legume i pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se ntlnesc sptmnal pentru a uda plantele. diff --git a/corpus/generated/ISO-8859-2/ro/train/city-morning.bin b/corpus/generated/ISO-8859-2/ro/train/city-morning.bin new file mode 100644 index 0000000..08d7469 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/city-morning.bin @@ -0,0 +1,3 @@ +Dimineaa n ora + +Oraul se trezete ncet. Cafenelele deschid uile, autobuzele pornesc pe traseu, iar oamenii citesc tirile. Cerul devine mai luminos deasupra cldirilor. diff --git a/corpus/generated/ISO-8859-2/ro/train/library.bin b/corpus/generated/ISO-8859-2/ro/train/library.bin new file mode 100644 index 0000000..1afc415 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/library.bin @@ -0,0 +1,3 @@ +Biblioteca linitit + +n biblioteca de cartier se aud pagini ntoarse ncet. Cititorii caut cri despre istorie, tiin i cltorii. Bibliotecara pregtete o expoziie pentru elevi. diff --git a/corpus/generated/ISO-8859-2/ro/train/market.bin b/corpus/generated/ISO-8859-2/ro/train/market.bin new file mode 100644 index 0000000..15a0338 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/market.bin @@ -0,0 +1,3 @@ +Piaa de smbt + +Piaa umple centrul cu mirosuri i culori. Fermierii aduc mere, miere, brnz i pine proaspt. Cumprtorii aleg legume i schimb reete de familie. diff --git a/corpus/generated/ISO-8859-2/ro/train/workshop.bin b/corpus/generated/ISO-8859-2/ro/train/workshop.bin new file mode 100644 index 0000000..132f010 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/workshop.bin @@ -0,0 +1,3 @@ +Atelier digital + +O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. diff --git a/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin b/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin new file mode 100644 index 0000000..c125be0 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin @@ -0,0 +1,3 @@ +Plimbare lng ru + +Duminic, prietenii au mers spre un ru linitit din afara oraului. Au luat o hart, ceai cald i un aparat foto. Dup plimbare s-au odihnit pe mal. diff --git a/corpus/generated/ISO-8859-5/ru/test/community-garden.bin b/corpus/generated/ISO-8859-5/ru/test/community-garden.bin new file mode 100644 index 0000000..1ed1849 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . , . diff --git a/corpus/generated/ISO-8859-5/ru/train/city-morning.bin b/corpus/generated/ISO-8859-5/ru/train/city-morning.bin new file mode 100644 index 0000000..e329ef5 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/city-morning.bin @@ -0,0 +1,3 @@ + + + . , , . . diff --git a/corpus/generated/ISO-8859-5/ru/train/library.bin b/corpus/generated/ISO-8859-5/ru/train/library.bin new file mode 100644 index 0000000..a05a46f --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/library.bin @@ -0,0 +1,3 @@ + + + . , . , . diff --git a/corpus/generated/ISO-8859-5/ru/train/market.bin b/corpus/generated/ISO-8859-5/ru/train/market.bin new file mode 100644 index 0000000..3ff323b --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/market.bin @@ -0,0 +1,3 @@ + + + . , , . , . diff --git a/corpus/generated/ISO-8859-5/ru/train/workshop.bin b/corpus/generated/ISO-8859-5/ru/train/workshop.bin new file mode 100644 index 0000000..54ff5e6 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/workshop.bin @@ -0,0 +1,3 @@ + + + . , , . diff --git a/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin b/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin new file mode 100644 index 0000000..1c97742 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . , . . diff --git a/corpus/generated/ISO-8859-6/ar/test/community-garden.bin b/corpus/generated/ISO-8859-6/ar/test/community-garden.bin new file mode 100644 index 0000000..d2a9d0c --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/test/community-garden.bin @@ -0,0 +1,3 @@ + + + ɬ . diff --git a/corpus/generated/ISO-8859-6/ar/train/city-morning.bin b/corpus/generated/ISO-8859-6/ar/train/city-morning.bin new file mode 100644 index 0000000..50c1ce8 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/city-morning.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-6/ar/train/library.bin b/corpus/generated/ISO-8859-6/ar/train/library.bin new file mode 100644 index 0000000..ddd9db9 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/library.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-6/ar/train/market.bin b/corpus/generated/ISO-8859-6/ar/train/market.bin new file mode 100644 index 0000000..67ef9f6 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/market.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-6/ar/train/workshop.bin b/corpus/generated/ISO-8859-6/ar/train/workshop.bin new file mode 100644 index 0000000..27001c3 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/workshop.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin b/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin new file mode 100644 index 0000000..2d416cb --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-7/el/test/community-garden.bin b/corpus/generated/ISO-8859-7/el/test/community-garden.bin new file mode 100644 index 0000000..2e8196c --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-7/el/train/city-morning.bin b/corpus/generated/ISO-8859-7/el/train/city-morning.bin new file mode 100644 index 0000000..f58abba --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/city-morning.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-7/el/train/library.bin b/corpus/generated/ISO-8859-7/el/train/library.bin new file mode 100644 index 0000000..6724218 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/library.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-7/el/train/market.bin b/corpus/generated/ISO-8859-7/el/train/market.bin new file mode 100644 index 0000000..40995f7 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/market.bin @@ -0,0 +1,3 @@ + + + , , . diff --git a/corpus/generated/ISO-8859-7/el/train/workshop.bin b/corpus/generated/ISO-8859-7/el/train/workshop.bin new file mode 100644 index 0000000..4f44381 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/workshop.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-7/el/validation/river-trip.bin b/corpus/generated/ISO-8859-7/el/validation/river-trip.bin new file mode 100644 index 0000000..dcea70f --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-8/he/test/community-garden.bin b/corpus/generated/ISO-8859-8/he/test/community-garden.bin new file mode 100644 index 0000000..28d54b2 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-8/he/train/city-morning.bin b/corpus/generated/ISO-8859-8/he/train/city-morning.bin new file mode 100644 index 0000000..90ce4a0 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/city-morning.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-8/he/train/library.bin b/corpus/generated/ISO-8859-8/he/train/library.bin new file mode 100644 index 0000000..d4abd33 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/library.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/ISO-8859-8/he/train/market.bin b/corpus/generated/ISO-8859-8/he/train/market.bin new file mode 100644 index 0000000..d4e1672 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/market.bin @@ -0,0 +1,3 @@ + + + , , . diff --git a/corpus/generated/ISO-8859-8/he/train/workshop.bin b/corpus/generated/ISO-8859-8/he/train/workshop.bin new file mode 100644 index 0000000..9120483 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/workshop.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-8/he/validation/river-trip.bin b/corpus/generated/ISO-8859-8/he/validation/river-trip.bin new file mode 100644 index 0000000..f2a8f51 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/ISO-8859-9/tr/test/community-garden.bin b/corpus/generated/ISO-8859-9/tr/test/community-garden.bin new file mode 100644 index 0000000..831e0ea --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/test/community-garden.bin @@ -0,0 +1,3 @@ +Mahalle bahesi + +Komular bo avluyu yeil bir baheye evirdi, ocuklar renkli tabelalar yapt ve aileler bitkileri her hafta suluyor. diff --git a/corpus/generated/ISO-8859-9/tr/train/city-morning.bin b/corpus/generated/ISO-8859-9/tr/train/city-morning.bin new file mode 100644 index 0000000..e8458f6 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/city-morning.bin @@ -0,0 +1,3 @@ +ehirde sabah + +ehir yavaa uyanr, kafeler alr ve insanlar ilk otobsleri bekler. diff --git a/corpus/generated/ISO-8859-9/tr/train/library.bin b/corpus/generated/ISO-8859-9/tr/train/library.bin new file mode 100644 index 0000000..20c465c --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/library.bin @@ -0,0 +1,3 @@ +Sessiz ktphane + +Okurlar tarih ve bilim kitaplar ararken ktphaneci rencilere yardmc olur. diff --git a/corpus/generated/ISO-8859-9/tr/train/market.bin b/corpus/generated/ISO-8859-9/tr/train/market.bin new file mode 100644 index 0000000..0856260 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/market.bin @@ -0,0 +1,3 @@ +Cumartesi pazar + +iftiler kalabalk meydanda taze ekmek, elma, peynir ve bal satar. diff --git a/corpus/generated/ISO-8859-9/tr/train/workshop.bin b/corpus/generated/ISO-8859-9/tr/train/workshop.bin new file mode 100644 index 0000000..844ca1e --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/workshop.bin @@ -0,0 +1,3 @@ +Dijital atlye + +Kk bir ekip mze rehberi hazrlar, kodu denetler ve ak aklamalar yazar. diff --git a/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin b/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin new file mode 100644 index 0000000..2568000 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Nehir gezisi + +Arkadalar harita ve scak ay alp ormanda yrd, sonra nehir kysnda dinlendi. diff --git a/corpus/generated/KOI8-R/ru/test/community-garden.bin b/corpus/generated/KOI8-R/ru/test/community-garden.bin new file mode 100644 index 0000000..17de462 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . , . diff --git a/corpus/generated/KOI8-R/ru/train/city-morning.bin b/corpus/generated/KOI8-R/ru/train/city-morning.bin new file mode 100644 index 0000000..2bf62de --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/city-morning.bin @@ -0,0 +1,3 @@ + + + . , , . . diff --git a/corpus/generated/KOI8-R/ru/train/library.bin b/corpus/generated/KOI8-R/ru/train/library.bin new file mode 100644 index 0000000..5e2b1e4 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/library.bin @@ -0,0 +1,3 @@ + + + . , . , . diff --git a/corpus/generated/KOI8-R/ru/train/market.bin b/corpus/generated/KOI8-R/ru/train/market.bin new file mode 100644 index 0000000..689aa9b --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/market.bin @@ -0,0 +1,3 @@ + + + . , ͣ, . , . diff --git a/corpus/generated/KOI8-R/ru/train/workshop.bin b/corpus/generated/KOI8-R/ru/train/workshop.bin new file mode 100644 index 0000000..9133544 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/workshop.bin @@ -0,0 +1,3 @@ + + + . , , . diff --git a/corpus/generated/KOI8-R/ru/validation/river-trip.bin b/corpus/generated/KOI8-R/ru/validation/river-trip.bin new file mode 100644 index 0000000..cb9b8fa --- /dev/null +++ b/corpus/generated/KOI8-R/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . , . . diff --git a/corpus/generated/index.json b/corpus/generated/index.json index cc7e572..136b1b1 100644 --- a/corpus/generated/index.json +++ b/corpus/generated/index.json @@ -576,940 +576,4108 @@ "sha256": "7f39f7fb36cbabea98997b9c390d71e0742ef2fad5c7deca432ee6444e164c49" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "test", - "document": "community-garden", - "path": "ISO-8859-10/is/test/community-garden.bin", - "bytes": 174, - "highBytes": 19, - "uniqueHighBytes": 6, - "sha256": "11bbdafb4917a3935f98940af20e169c4a7eef5642d4a159297b44cd8b565d78" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "city-morning", - "path": "ISO-8859-10/is/train/city-morning.bin", - "bytes": 161, - "highBytes": 11, - "uniqueHighBytes": 8, - "sha256": "3c8a3358a8195de52877343b34b5a71606732b3df110594862cebeb9c76aa3cf" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "library", - "path": "ISO-8859-10/is/train/library.bin", - "bytes": 166, - "highBytes": 18, - "uniqueHighBytes": 7, - "sha256": "a78d39ae9c97ed64273f11792d1e5a353080658d28ab6a85cc248f28b91d18a0" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "market", - "path": "ISO-8859-10/is/train/market.bin", - "bytes": 168, - "highBytes": 10, - "uniqueHighBytes": 5, - "sha256": "c15c77b0cb233941ca1b038ddd269d4d866e4b5b48d61989220169b6cd851c9b" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "workshop", - "path": "ISO-8859-10/is/train/workshop.bin", - "bytes": 167, - "highBytes": 15, - "uniqueHighBytes": 6, - "sha256": "1bdd4e4aa76ecbd47f82533ec096fe3ee5c637debdefad8bc10251e8f3867af0" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "validation", - "document": "river-trip", - "path": "ISO-8859-10/is/validation/river-trip.bin", - "bytes": 168, - "highBytes": 17, - "uniqueHighBytes": 9, - "sha256": "ce28fe8d7fe2ee466f03495c48f00a6da6fa7d3df530c7052873c444e1dd6f59" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "test", "document": "community-garden", - "path": "ISO-8859-13/lt/test/community-garden.bin", - "bytes": 177, - "highBytes": 17, - "uniqueHighBytes": 6, - "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + "path": "ISO-8859-1/da/test/community-garden.bin", + "bytes": 123, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "d742c49869d5f97ff0dc9bed58a7e819cf5a9f3cde419ff785f2201591fa7991" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "city-morning", - "path": "ISO-8859-13/lt/train/city-morning.bin", - "bytes": 156, - "highBytes": 11, - "uniqueHighBytes": 4, - "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + "path": "ISO-8859-1/da/train/city-morning.bin", + "bytes": 89, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "75320357a51c6716244970d7c802f42af40a6a65fa1602e4f38e239e16ccd887" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "library", - "path": "ISO-8859-13/lt/train/library.bin", - "bytes": 163, - "highBytes": 8, - "uniqueHighBytes": 5, - "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", - "split": "train", - "document": "market", - "path": "ISO-8859-13/lt/train/market.bin", - "bytes": 171, - "highBytes": 12, - "uniqueHighBytes": 8, - "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", - "split": "train", - "document": "workshop", - "path": "ISO-8859-13/lt/train/workshop.bin", - "bytes": 170, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", - "split": "validation", - "document": "river-trip", - "path": "ISO-8859-13/lt/validation/river-trip.bin", - "bytes": 165, - "highBytes": 16, - "uniqueHighBytes": 5, - "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "test", - "document": "community-garden", - "path": "ISO-8859-14/cy/test/community-garden.bin", - "bytes": 194, + "path": "ISO-8859-1/da/train/library.bin", + "bytes": 105, "highBytes": 3, - "uniqueHighBytes": 3, - "sha256": "11475514e6d6789bb154d27be8a381507f5c136b1ec88219820a77d3f11a352a" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "city-morning", - "path": "ISO-8859-14/cy/train/city-morning.bin", - "bytes": 195, - "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "5310db3607ef3fcadf53799acdd8836356af51cf840f1f391c4c31acdb3188ce" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "library", - "path": "ISO-8859-14/cy/train/library.bin", - "bytes": 194, - "highBytes": 1, - "uniqueHighBytes": 1, - "sha256": "c8445d5b31a86bb8a0f295792afe5d4632e9c30182460cb0bc590c6fa6156afb" + "sha256": "4b0f38943f7eb950af8b04b16ac6ce581c48f70e933f66bd03725842d9e1bd58" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "market", - "path": "ISO-8859-14/cy/train/market.bin", - "bytes": 183, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "c2b02e1a6e22d7f21483955ebf4d00e4b0500c3ee91dcb67e67cb07f015932e3" + "path": "ISO-8859-1/da/train/market.bin", + "bytes": 84, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "61d9285e5b354f3cffb320f3bb62af7d4ce86fe34ba0e5067d5bc26aa9cd010c" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "workshop", - "path": "ISO-8859-14/cy/train/workshop.bin", - "bytes": 173, + "path": "ISO-8859-1/da/train/workshop.bin", + "bytes": 103, "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "2fe7199c4f96f4c5941e8f6b95aa44ae6f8d85863552550993b3b019acdeee6f" + "sha256": "aca6cded29cf164fc9445de75292a45f007c971ea9407d7ef0d794413a7cd975" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "validation", "document": "river-trip", - "path": "ISO-8859-14/cy/validation/river-trip.bin", - "bytes": 161, + "path": "ISO-8859-1/da/validation/river-trip.bin", + "bytes": 97, "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "318d863b3f7efa4a716e75ba4cd138fe9b9c337fb6f88eafe7ee1c671629e054" + "sha256": "0edb6dd9323283e842dea2112213cf7b86608b257a36cfd318b8c71b3c01c8e0" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/de/test/community-garden.bin", + "path": "ISO-8859-1/de/test/community-garden.bin", "bytes": 220, "highBytes": 12, "uniqueHighBytes": 3, "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/de/train/city-morning.bin", + "path": "ISO-8859-1/de/train/city-morning.bin", "bytes": 208, "highBytes": 6, "uniqueHighBytes": 4, "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "library", - "path": "ISO-8859-15/de/train/library.bin", + "path": "ISO-8859-1/de/train/library.bin", "bytes": 204, "highBytes": 5, "uniqueHighBytes": 2, "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "market", - "path": "ISO-8859-15/de/train/market.bin", + "path": "ISO-8859-1/de/train/market.bin", "bytes": 177, "highBytes": 4, "uniqueHighBytes": 3, "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "workshop", - "path": "ISO-8859-15/de/train/workshop.bin", + "path": "ISO-8859-1/de/train/workshop.bin", "bytes": 188, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/de/validation/river-trip.bin", + "path": "ISO-8859-1/de/validation/river-trip.bin", "bytes": 192, "highBytes": 3, "uniqueHighBytes": 2, "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-1/en/test/community-garden.bin", + "bytes": 136, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8574d38336d96e1a48fd42b012b3924bc36ee57550bc58a28ed6d5c17d514c1d" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-1/en/train/city-morning.bin", + "bytes": 108, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "ff22d70d940981c1ea141576fb34c74d841dbf2e2140d80ee55f51dd63bd9ef0" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "library", + "path": "ISO-8859-1/en/train/library.bin", + "bytes": 117, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "12ea1dc5452af3ca116753189e184ef0a10645eee17f33211dc5f7d182318f38" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "market", + "path": "ISO-8859-1/en/train/market.bin", + "bytes": 100, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "cbaa356df57d8557b3d2c3ca4d6ec5845077db27583e102aebdaedcacf9f2aee" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "workshop", + "path": "ISO-8859-1/en/train/workshop.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "157d773566f7cd87f7bafb87780ba98dcc090adbdaaea2c03483e7a3b6a93a70" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-1/en/validation/river-trip.bin", + "bytes": 106, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c4e405fd6466194b2ddc91f07f8bcb624c904531d6d96f37e35d04847c6abbb9" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/es/test/community-garden.bin", + "path": "ISO-8859-1/es/test/community-garden.bin", "bytes": 203, "highBytes": 6, "uniqueHighBytes": 3, "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/es/train/city-morning.bin", + "path": "ISO-8859-1/es/train/city-morning.bin", "bytes": 202, "highBytes": 3, "uniqueHighBytes": 3, "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "library", - "path": "ISO-8859-15/es/train/library.bin", + "path": "ISO-8859-1/es/train/library.bin", "bytes": 202, "highBytes": 2, "uniqueHighBytes": 2, "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "market", - "path": "ISO-8859-15/es/train/market.bin", + "path": "ISO-8859-1/es/train/market.bin", "bytes": 199, "highBytes": 2, "uniqueHighBytes": 2, "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "workshop", - "path": "ISO-8859-15/es/train/workshop.bin", + "path": "ISO-8859-1/es/train/workshop.bin", "bytes": 194, "highBytes": 6, "uniqueHighBytes": 4, "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/es/validation/river-trip.bin", + "path": "ISO-8859-1/es/validation/river-trip.bin", "bytes": 184, "highBytes": 6, "uniqueHighBytes": 3, "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/fr/test/community-garden.bin", + "path": "ISO-8859-1/fr/test/community-garden.bin", "bytes": 231, "highBytes": 9, "uniqueHighBytes": 1, "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/fr/train/city-morning.bin", + "path": "ISO-8859-1/fr/train/city-morning.bin", "bytes": 212, "highBytes": 5, "uniqueHighBytes": 2, "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "library", - "path": "ISO-8859-15/fr/train/library.bin", + "path": "ISO-8859-1/fr/train/library.bin", "bytes": 226, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "market", - "path": "ISO-8859-15/fr/train/market.bin", + "path": "ISO-8859-1/fr/train/market.bin", "bytes": 223, "highBytes": 3, "uniqueHighBytes": 1, "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "workshop", - "path": "ISO-8859-15/fr/train/workshop.bin", + "path": "ISO-8859-1/fr/train/workshop.bin", "bytes": 215, "highBytes": 8, "uniqueHighBytes": 1, "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/fr/validation/river-trip.bin", + "path": "ISO-8859-1/fr/validation/river-trip.bin", "bytes": 228, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "test", "document": "community-garden", - "path": "ISO-8859-16/ro/test/community-garden.bin", - "bytes": 206, - "highBytes": 13, - "uniqueHighBytes": 5, - "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + "path": "ISO-8859-1/it/test/community-garden.bin", + "bytes": 161, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "53bb7b070795cd69fba5bcc79a4958b2ac5bb23677ebdac99c80623f96d501dd" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "city-morning", - "path": "ISO-8859-16/ro/train/city-morning.bin", - "bytes": 175, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + "path": "ISO-8859-1/it/train/city-morning.bin", + "bytes": 101, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "7c59ad9c3c9ac0b84c02c4829bffabdc83817989f08836d9ee44afd8a392e43d" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "library", - "path": "ISO-8859-16/ro/train/library.bin", - "bytes": 187, - "highBytes": 17, - "uniqueHighBytes": 5, - "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + "path": "ISO-8859-1/it/train/library.bin", + "bytes": 141, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "bb28d91b3f10f0208886078bf0fafbc85c7e03148069ddf5a9762809f28f8981" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "market", - "path": "ISO-8859-16/ro/train/market.bin", - "bytes": 171, - "highBytes": 17, - "uniqueHighBytes": 4, - "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + "path": "ISO-8859-1/it/train/market.bin", + "bytes": 114, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "40ce81cfc5ec4c5b655c2fb9ba519022a77da2c319950c868bc16c7f69278b11" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "workshop", - "path": "ISO-8859-16/ro/train/workshop.bin", - "bytes": 170, - "highBytes": 6, + "path": "ISO-8859-1/it/train/workshop.bin", + "bytes": 152, + "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + "sha256": "5b5704c6191b4179ee6966a2302452eaaf99c348b34e26bc210879c77def38bc" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "validation", "document": "river-trip", - "path": "ISO-8859-16/ro/validation/river-trip.bin", - "bytes": 170, - "highBytes": 10, - "uniqueHighBytes": 3, - "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + "path": "ISO-8859-1/it/validation/river-trip.bin", + "bytes": 98, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8fdde80f1933a9dfb82cc0181c6770e9947ccf5bfd745b8251ee02f54e6a19a7" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "test", "document": "community-garden", - "path": "ISO-8859-3/mt/test/community-garden.bin", - "bytes": 186, - "highBytes": 13, - "uniqueHighBytes": 4, - "sha256": "d6874f4f630507a619f841cad55a2046be0ee930fb9d450e20bcf0e9525759d5" + "path": "ISO-8859-1/nl/test/community-garden.bin", + "bytes": 153, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "26b1ccb49c443abe7580037a8a836bf99e9d5a5135b1751b77a73e3f61c1a9bf" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "city-morning", - "path": "ISO-8859-3/mt/train/city-morning.bin", - "bytes": 171, - "highBytes": 7, - "uniqueHighBytes": 3, - "sha256": "e40f845da038bcae4f7c6edb026ca823788410f8895cd7ae2765ca4afde49bfb" + "path": "ISO-8859-1/nl/train/city-morning.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "201fbe0db619c5d0d4ccecd723c27c68e0807437531553e8c17ba84cfc4335b2" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "library", - "path": "ISO-8859-3/mt/train/library.bin", - "bytes": 181, - "highBytes": 6, + "path": "ISO-8859-1/nl/train/library.bin", + "bytes": 150, + "highBytes": 3, "uniqueHighBytes": 2, - "sha256": "b16045c76dede00ff236e27cd647ea45e25a98d4cf04530c9baf947dbb1a9cb2" + "sha256": "5d9cdae67a5a7cf6cb319750006ea3cf6afaf379b11c35aa241ef63b203cb17b" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "market", - "path": "ISO-8859-3/mt/train/market.bin", - "bytes": 172, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "be0fbd5745a025d769bb128a55d8c80bb1397590edde4d79d28db1310589af8e" + "path": "ISO-8859-1/nl/train/market.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "15466fb33051e25a441363879415fe1c6ec949d753105ad851a75b2054d8c0da" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "workshop", - "path": "ISO-8859-3/mt/train/workshop.bin", - "bytes": 164, - "highBytes": 15, - "uniqueHighBytes": 5, - "sha256": "433e386a7ccaeaeef6e9fbd1daf1f586cc61788cd13973523c2d679026964420" + "path": "ISO-8859-1/nl/train/workshop.bin", + "bytes": 132, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "4fa51f6d8f9543c9df0acef5a2711a5d805fd18286c1164d786bd8ac715c183b" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "validation", "document": "river-trip", - "path": "ISO-8859-3/mt/validation/river-trip.bin", - "bytes": 176, - "highBytes": 9, - "uniqueHighBytes": 2, - "sha256": "9c73d965fbad70cf830b5a3db04e12c4f0b3bd50ead049f7d6703b5c19cf4eb1" + "path": "ISO-8859-1/nl/validation/river-trip.bin", + "bytes": 113, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "d9448db92c3fe1270814d7798759eca44290f94fd76c0ff9012dc2401b426253" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "test", "document": "community-garden", - "path": "ISO-8859-4/lv/test/community-garden.bin", - "bytes": 174, - "highBytes": 17, - "uniqueHighBytes": 8, - "sha256": "c0382f3b40081b8a5d30ca7d26699a2f9e9eb2e453edac9beb97b4a4a9c3051b" + "path": "ISO-8859-1/no/test/community-garden.bin", + "bytes": 116, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b86618ac6687d3769cca67ec9cb49a0d4d1dea45476592a316ae6eef40d576db" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "city-morning", - "path": "ISO-8859-4/lv/train/city-morning.bin", - "bytes": 163, - "highBytes": 15, - "uniqueHighBytes": 7, - "sha256": "d16cbb8016fdba887337226f08a30e96f2752b657dacbabec3a00b36d5538f60" + "path": "ISO-8859-1/no/train/city-morning.bin", + "bytes": 116, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "445f262b0f2ea6f724271b8486b0cfc861cf4e081e4617db895bc36a33dea2ee" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "library", - "path": "ISO-8859-4/lv/train/library.bin", - "bytes": 161, - "highBytes": 15, - "uniqueHighBytes": 5, - "sha256": "c6757b519e310d300ab70b574b32056a85d7bdf140afd265d5e44101cca573e0" + "path": "ISO-8859-1/no/train/library.bin", + "bytes": 126, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "96c146e3d2b86817371ba6ca5d07361d31519cb0d16fbbecfaf7bba36f0ff2eb" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "market", - "path": "ISO-8859-4/lv/train/market.bin", - "bytes": 170, - "highBytes": 12, - "uniqueHighBytes": 5, - "sha256": "33999e249def18a8469c731d110dbdb9614103b3dce69d8005bc6993b878fc88" + "path": "ISO-8859-1/no/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "a0279dbf28e14a6d9841789f50751ec9cdb09c2743f4ac1c61e72f310e916732" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "workshop", - "path": "ISO-8859-4/lv/train/workshop.bin", - "bytes": 163, - "highBytes": 10, - "uniqueHighBytes": 4, - "sha256": "1429abf14fb2e4144c64dd59657aec2615da7795c92f4e6f5e5f5534ee4be9a0" + "path": "ISO-8859-1/no/train/workshop.bin", + "bytes": 100, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "9c70c92214b0f0ac9000a131773f1ed1756d630037293a759982edcb621e056a" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "validation", "document": "river-trip", - "path": "ISO-8859-4/lv/validation/river-trip.bin", - "bytes": 156, - "highBytes": 14, - "uniqueHighBytes": 4, - "sha256": "282b08484ad284a0a069c2d8113e09bc80de8163fee60568042c538b0541fc3f" + "path": "ISO-8859-1/no/validation/river-trip.bin", + "bytes": 85, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "931a303e54cbfe3ccddf45aa0dd3956be07e4cf16bf1fe5ee0874bd4ad2bab9a" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "test", "document": "community-garden", - "path": "KOI8-U/uk/test/community-garden.bin", - "bytes": 181, - "highBytes": 152, - "uniqueHighBytes": 29, - "sha256": "25d71bdf3678270bf3a539771f8e429faaef3157c5649c0f2edc4ec06e0c80fa" + "path": "ISO-8859-1/pt/test/community-garden.bin", + "bytes": 145, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "a8e56da45e10d94745a21dd36a2721410e1b0d04b5bfee15b2c451a27c8ba62d" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "city-morning", - "path": "KOI8-U/uk/train/city-morning.bin", - "bytes": 205, - "highBytes": 168, - "uniqueHighBytes": 31, - "sha256": "853b422f64ef06d7a64528db49d4eca0f11240852392e51cb2700b1b8b9fd122" + "path": "ISO-8859-1/pt/train/city-morning.bin", + "bytes": 99, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "c9a43be7ac9a1af0be31146446c7cea5142ba97545c304e9fdfef166c0dfabe1" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "library", - "path": "KOI8-U/uk/train/library.bin", - "bytes": 182, - "highBytes": 151, - "uniqueHighBytes": 32, - "sha256": "42455a09e6cafcd694b38198b1f172e44e63774cde6c9e760181334974536fe5" + "path": "ISO-8859-1/pt/train/library.bin", + "bytes": 116, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "634c4f0d24e7f1a8bbf9b1fd68d0fe414376d2efeff0fbfc7637d921709f2a62" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "market", - "path": "KOI8-U/uk/train/market.bin", - "bytes": 199, - "highBytes": 165, - "uniqueHighBytes": 33, - "sha256": "0d8b40da8ad30cc4d6d849f2439f472db8092d622e428747aa55e1fa1797800b" + "path": "ISO-8859-1/pt/train/market.bin", + "bytes": 96, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "79f397f7b2ac2cca51091f006462be7777befb52d36c54237a8cfb0cb20cd924" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "workshop", - "path": "KOI8-U/uk/train/workshop.bin", - "bytes": 189, - "highBytes": 161, - "uniqueHighBytes": 30, - "sha256": "b4b92fa64c4b9252511ed0240a7e1cb565a0b1158f07b50fefb93fab1bd5f0f7" + "path": "ISO-8859-1/pt/train/workshop.bin", + "bytes": 108, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "fbb52707258ca57000887397d2bd8220907fc178ed36b929c62e72bf211848e1" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "validation", "document": "river-trip", - "path": "KOI8-U/uk/validation/river-trip.bin", - "bytes": 195, - "highBytes": 160, - "uniqueHighBytes": 31, - "sha256": "1b015aa68443a92670b3e73e37fa96d70d1e2dea1a9ed736bebfcf077bde9dfe" + "path": "ISO-8859-1/pt/validation/river-trip.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "db583dead964fc7fe8a257ebd69f9dd7388c89384bac132955a76b02150c753f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "test", "document": "community-garden", - "path": "macintosh/de/test/community-garden.bin", - "bytes": 220, - "highBytes": 12, + "path": "ISO-8859-1/sv/test/community-garden.bin", + "bytes": 138, + "highBytes": 8, "uniqueHighBytes": 3, - "sha256": "9a35ac96816b79920d3076818a106719d95169d017b89832ff735652fdbc45d5" + "sha256": "7229bf29e0adc2b37cdb6663bb696e500e13000633665f0f226915f5f74c4853" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "city-morning", - "path": "macintosh/de/train/city-morning.bin", - "bytes": 208, - "highBytes": 6, + "path": "ISO-8859-1/sv/train/city-morning.bin", + "bytes": 101, + "highBytes": 7, "uniqueHighBytes": 4, - "sha256": "d826a59c9fe0ea77efe5554f688a96a25ba31edd3b502177f88b229237bfafff" + "sha256": "5ac24bf0fe3bfb1d64aa066ff78e3af67bcf0b4a6f8d142fefeff0b8ad4ae72c" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "library", - "path": "macintosh/de/train/library.bin", - "bytes": 204, - "highBytes": 5, + "path": "ISO-8859-1/sv/train/library.bin", + "bytes": 108, + "highBytes": 4, "uniqueHighBytes": 2, - "sha256": "29829a1ac13b7f7ad2ecd8abd7d0f2454b3beb4ede31af4ab1a8901cb73103b9" + "sha256": "fc05c9ad957334217feaa333f6cfe7cd206b250fbae9ea51a19d9c69f72158e7" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "market", - "path": "macintosh/de/train/market.bin", - "bytes": 177, - "highBytes": 4, + "path": "ISO-8859-1/sv/train/market.bin", + "bytes": 89, + "highBytes": 7, "uniqueHighBytes": 3, - "sha256": "f135969c32b723540e69570512179a361f12aca33782130567cafc8b8cb40974" + "sha256": "9f4f10ac20699e493cdd99b8c1ee6caa29d4a9359962aab15ac23bb773f1d26f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "workshop", - "path": "macintosh/de/train/workshop.bin", - "bytes": 188, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "a77aa45de75ac7b572174e1c99aa8aab3d7a3cb0278b2f68932d1c7fe4598218" + "path": "ISO-8859-1/sv/train/workshop.bin", + "bytes": 110, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "989f41c9b83ba4e3475a6902c33340fa7b07bc965776a09e9db165321bdcb3cb" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "validation", "document": "river-trip", - "path": "macintosh/de/validation/river-trip.bin", - "bytes": 192, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "b2edf27b09f5aeaf742baed4ebc17e0a890da6a9f0aa52a40adbcb16104e5538" + "path": "ISO-8859-1/sv/validation/river-trip.bin", + "bytes": 86, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "84cc4d3ab5abaa2d2575d868f0056375043c7e17d6c34ecd4112fca4632bc321" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "test", "document": "community-garden", - "path": "macintosh/es/test/community-garden.bin", - "bytes": 203, - "highBytes": 6, - "uniqueHighBytes": 3, - "sha256": "b675b0218b3db70d7c6f6f43cd7eb00fe7b45713f6e23e3fbac03a176e95935a" + "path": "ISO-8859-10/is/test/community-garden.bin", + "bytes": 174, + "highBytes": 19, + "uniqueHighBytes": 6, + "sha256": "11bbdafb4917a3935f98940af20e169c4a7eef5642d4a159297b44cd8b565d78" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "city-morning", - "path": "macintosh/es/train/city-morning.bin", - "bytes": 202, - "highBytes": 3, - "uniqueHighBytes": 3, - "sha256": "15f13077700d48818a5a5050c47683e1c14260cc22ce956abd29bd07eddf10eb" + "path": "ISO-8859-10/is/train/city-morning.bin", + "bytes": 161, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "3c8a3358a8195de52877343b34b5a71606732b3df110594862cebeb9c76aa3cf" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "library", - "path": "macintosh/es/train/library.bin", - "bytes": 202, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "a489cbb2f2f276564faf85fd6f86795a1ec5a1e5b1cfeda35874e2302e9d3d1b" + "path": "ISO-8859-10/is/train/library.bin", + "bytes": 166, + "highBytes": 18, + "uniqueHighBytes": 7, + "sha256": "a78d39ae9c97ed64273f11792d1e5a353080658d28ab6a85cc248f28b91d18a0" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "market", - "path": "macintosh/es/train/market.bin", - "bytes": 199, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "80770be98c0f1e40ee59e9de08615620aa8a0af1ff73ef8b6a3cc483b14a3a4e" + "path": "ISO-8859-10/is/train/market.bin", + "bytes": 168, + "highBytes": 10, + "uniqueHighBytes": 5, + "sha256": "c15c77b0cb233941ca1b038ddd269d4d866e4b5b48d61989220169b6cd851c9b" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "workshop", - "path": "macintosh/es/train/workshop.bin", - "bytes": 194, - "highBytes": 6, - "uniqueHighBytes": 4, - "sha256": "5b25ecfaa0df015d5c4c1fb52fce66af137588ac8677acd50c468b65b5112018" + "path": "ISO-8859-10/is/train/workshop.bin", + "bytes": 167, + "highBytes": 15, + "uniqueHighBytes": 6, + "sha256": "1bdd4e4aa76ecbd47f82533ec096fe3ee5c637debdefad8bc10251e8f3867af0" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "validation", "document": "river-trip", - "path": "macintosh/es/validation/river-trip.bin", - "bytes": 184, - "highBytes": 6, - "uniqueHighBytes": 3, - "sha256": "d68dda4671b8158cbff67ab7b476d2b819ac45ad00719d4dbe48f22ba1862baa" + "path": "ISO-8859-10/is/validation/river-trip.bin", + "bytes": 168, + "highBytes": 17, + "uniqueHighBytes": 9, + "sha256": "ce28fe8d7fe2ee466f03495c48f00a6da6fa7d3df530c7052873c444e1dd6f59" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "test", "document": "community-garden", - "path": "macintosh/fr/test/community-garden.bin", - "bytes": 231, - "highBytes": 9, - "uniqueHighBytes": 1, - "sha256": "50363a67b8df69b6fbdb3b7a5a25c22c40cdac6ccca71d3bfafeff490e122c58" + "path": "ISO-8859-13/lt/test/community-garden.bin", + "bytes": 177, + "highBytes": 17, + "uniqueHighBytes": 6, + "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "city-morning", - "path": "macintosh/fr/train/city-morning.bin", - "bytes": 212, - "highBytes": 5, - "uniqueHighBytes": 2, - "sha256": "5f8dbd5bcfe437962ef5943c707e1695edce736abace97d5b29ef68945c42415" + "path": "ISO-8859-13/lt/train/city-morning.bin", + "bytes": 156, + "highBytes": 11, + "uniqueHighBytes": 4, + "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "library", - "path": "macintosh/fr/train/library.bin", - "bytes": 226, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "66ecc14482e9af6c7d2c813d11462760e1a70b98a6d6c0c966821cd03f07eee3" + "path": "ISO-8859-13/lt/train/library.bin", + "bytes": 163, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "market", - "path": "macintosh/fr/train/market.bin", - "bytes": 223, - "highBytes": 3, - "uniqueHighBytes": 1, - "sha256": "5e81aa00b93ef4299cde155e0d215906411a1b0566a676ea307403ef658274d5" + "path": "ISO-8859-13/lt/train/market.bin", + "bytes": 171, + "highBytes": 12, + "uniqueHighBytes": 8, + "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "workshop", - "path": "macintosh/fr/train/workshop.bin", - "bytes": 215, - "highBytes": 8, - "uniqueHighBytes": 1, - "sha256": "ee23ebed42809949b51365120200f135cb051471430331b7c6ccf8282841075a" + "path": "ISO-8859-13/lt/train/workshop.bin", + "bytes": 170, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "validation", "document": "river-trip", - "path": "macintosh/fr/validation/river-trip.bin", - "bytes": 228, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" + "path": "ISO-8859-13/lt/validation/river-trip.bin", + "bytes": 165, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-14/cy/test/community-garden.bin", + "bytes": 194, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "11475514e6d6789bb154d27be8a381507f5c136b1ec88219820a77d3f11a352a" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-14/cy/train/city-morning.bin", + "bytes": 195, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "5310db3607ef3fcadf53799acdd8836356af51cf840f1f391c4c31acdb3188ce" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "library", + "path": "ISO-8859-14/cy/train/library.bin", + "bytes": 194, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c8445d5b31a86bb8a0f295792afe5d4632e9c30182460cb0bc590c6fa6156afb" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "market", + "path": "ISO-8859-14/cy/train/market.bin", + "bytes": 183, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "c2b02e1a6e22d7f21483955ebf4d00e4b0500c3ee91dcb67e67cb07f015932e3" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "workshop", + "path": "ISO-8859-14/cy/train/workshop.bin", + "bytes": 173, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "2fe7199c4f96f4c5941e8f6b95aa44ae6f8d85863552550993b3b019acdeee6f" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-14/cy/validation/river-trip.bin", + "bytes": 161, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "318d863b3f7efa4a716e75ba4cd138fe9b9c337fb6f88eafe7ee1c671629e054" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "library", + "path": "ISO-8859-15/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "market", + "path": "ISO-8859-15/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "library", + "path": "ISO-8859-15/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "market", + "path": "ISO-8859-15/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "library", + "path": "ISO-8859-15/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "market", + "path": "ISO-8859-15/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-16/ro/test/community-garden.bin", + "bytes": 224, + "highBytes": 15, + "uniqueHighBytes": 6, + "sha256": "cbb06341ae7b36086fa1cc8e9ce8df8a66cf37ac24a4558200e71f037eaf6bc5" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-16/ro/train/city-morning.bin", + "bytes": 193, + "highBytes": 11, + "uniqueHighBytes": 5, + "sha256": "47d8c1bf653e17a058fd3c8a3141eaa08652db1c8e27a46be5b177a8770b1006" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "library", + "path": "ISO-8859-16/ro/train/library.bin", + "bytes": 205, + "highBytes": 19, + "uniqueHighBytes": 6, + "sha256": "fbd051c8abf29897208b0f3199e4504b8d40677a58edea5d541b37bfea98b459" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "market", + "path": "ISO-8859-16/ro/train/market.bin", + "bytes": 189, + "highBytes": 19, + "uniqueHighBytes": 5, + "sha256": "8584a99429d049f70c73ea6c50208b23faef446f1c50b9833b7e31e2ef3adbc5" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "ISO-8859-16/ro/train/workshop.bin", + "bytes": 188, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "d3dbc5ecf59a9705bc6dd1d7ed97222304426d439ee87084d3729d221a00145c" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-16/ro/validation/river-trip.bin", + "bytes": 188, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "d130ff9bdb9431000be39f0d1f6cd5d7a0aa805e8bd4eca9723c71761b110020" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/cs/test/community-garden.bin", + "bytes": 128, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "1eba2d29ae0f73a2f12f43469971a65f81c58779530b1b753285cb2d8c29200d" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/cs/train/city-morning.bin", + "bytes": 98, + "highBytes": 13, + "uniqueHighBytes": 6, + "sha256": "a0f0dfa252459d39f725fee0b0a13818681efd1c28e854e4eaf32509e03f5a55" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "library", + "path": "ISO-8859-2/cs/train/library.bin", + "bytes": 88, + "highBytes": 12, + "uniqueHighBytes": 7, + "sha256": "cd1af77dd042d83bb4106f82c6a213eb32babb8c8be59d4b3b07ece358a651a4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "market", + "path": "ISO-8859-2/cs/train/market.bin", + "bytes": 74, + "highBytes": 10, + "uniqueHighBytes": 7, + "sha256": "083f02c5caabf56bf78ab98fc83cabb38dfc79337e33d3a0f694de80c2403e0e" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/cs/train/workshop.bin", + "bytes": 90, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "05d2ab8033b5d71b2c863a14f4c54ca237af31f71158a2370622417e783788cf" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/cs/validation/river-trip.bin", + "bytes": 80, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "3eba80db3fa617e843e8165ca7dd7ee1332d362e9024b85ffa3f9dcedf4f4651" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/hu/test/community-garden.bin", + "bytes": 139, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "d55e643af71274694d74cff7dbf12589bfd13a2b86f2ddcfb53476578a3132b8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/hu/train/city-morning.bin", + "bytes": 94, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "97945a679eae0390b0b3c65608b9cffe4cd8ba6196b8af4be0cec9272bf25ae4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "library", + "path": "ISO-8859-2/hu/train/library.bin", + "bytes": 108, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "bb34de899f6968d695ac39ed18a9f72077d309da97d9721d4e6551562d644427" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "market", + "path": "ISO-8859-2/hu/train/market.bin", + "bytes": 80, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a6a508b4927acd6751a091826a4cc4945330021699819d1c956c8ea6b440a56b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/hu/train/workshop.bin", + "bytes": 101, + "highBytes": 14, + "uniqueHighBytes": 7, + "sha256": "2a294d4fef2aeeee691b74de2293bdf7a7bae7854b3021607fc7df7d17a1a3a4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/hu/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "ce35a0b3ccaa4b07727c0b9e925c958ad050a5d774d91c3f25962c3b7291ffab" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/pl/test/community-garden.bin", + "bytes": 188, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "f9e88a60cbcc42a798c28f8e9b56ee86c8f657f6c8bf8487b8ce2c61c1f02a8b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/pl/train/city-morning.bin", + "bytes": 174, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "7aa1f6ba90080099f3a458933c75a4d32aa5d1f7687cbbc2900c9da738361402" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "library", + "path": "ISO-8859-2/pl/train/library.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 6, + "sha256": "4533cb3ea7bd91bf69478c029c0d4e71755df177c96a18d5363289a80aa08df8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "market", + "path": "ISO-8859-2/pl/train/market.bin", + "bytes": 166, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "14e939af68d4b46232b52c9527d0ec87ab4a5bfb2d178fb4f7a58f8fdcd45f6a" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/pl/train/workshop.bin", + "bytes": 165, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "f749532f6c64c2ff857f54f2e4ef7a0b42ac1dfb6a60bfeeb518fdc75d966c6a" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/pl/validation/river-trip.bin", + "bytes": 161, + "highBytes": 9, + "uniqueHighBytes": 3, + "sha256": "9b6685ebb982c76027a1f70a1fb36ad3913d4400c76dae5f54b460f61704e2d7" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "library", + "path": "ISO-8859-2/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "market", + "path": "ISO-8859-2/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-3/mt/test/community-garden.bin", + "bytes": 186, + "highBytes": 13, + "uniqueHighBytes": 4, + "sha256": "d6874f4f630507a619f841cad55a2046be0ee930fb9d450e20bcf0e9525759d5" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-3/mt/train/city-morning.bin", + "bytes": 171, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "e40f845da038bcae4f7c6edb026ca823788410f8895cd7ae2765ca4afde49bfb" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "library", + "path": "ISO-8859-3/mt/train/library.bin", + "bytes": 181, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "b16045c76dede00ff236e27cd647ea45e25a98d4cf04530c9baf947dbb1a9cb2" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "market", + "path": "ISO-8859-3/mt/train/market.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "be0fbd5745a025d769bb128a55d8c80bb1397590edde4d79d28db1310589af8e" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "workshop", + "path": "ISO-8859-3/mt/train/workshop.bin", + "bytes": 164, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "433e386a7ccaeaeef6e9fbd1daf1f586cc61788cd13973523c2d679026964420" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-3/mt/validation/river-trip.bin", + "bytes": 176, + "highBytes": 9, + "uniqueHighBytes": 2, + "sha256": "9c73d965fbad70cf830b5a3db04e12c4f0b3bd50ead049f7d6703b5c19cf4eb1" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-4/lv/test/community-garden.bin", + "bytes": 174, + "highBytes": 17, + "uniqueHighBytes": 8, + "sha256": "c0382f3b40081b8a5d30ca7d26699a2f9e9eb2e453edac9beb97b4a4a9c3051b" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-4/lv/train/city-morning.bin", + "bytes": 163, + "highBytes": 15, + "uniqueHighBytes": 7, + "sha256": "d16cbb8016fdba887337226f08a30e96f2752b657dacbabec3a00b36d5538f60" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "library", + "path": "ISO-8859-4/lv/train/library.bin", + "bytes": 161, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "c6757b519e310d300ab70b574b32056a85d7bdf140afd265d5e44101cca573e0" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "market", + "path": "ISO-8859-4/lv/train/market.bin", + "bytes": 170, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "33999e249def18a8469c731d110dbdb9614103b3dce69d8005bc6993b878fc88" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "workshop", + "path": "ISO-8859-4/lv/train/workshop.bin", + "bytes": 163, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "1429abf14fb2e4144c64dd59657aec2615da7795c92f4e6f5e5f5534ee4be9a0" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-4/lv/validation/river-trip.bin", + "bytes": 156, + "highBytes": 14, + "uniqueHighBytes": 4, + "sha256": "282b08484ad284a0a069c2d8113e09bc80de8163fee60568042c538b0541fc3f" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-5/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "1bb1e574a38f791fa40a6e3742960a95e6576dc9016efa256a7a9d7e869d3fb4" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-5/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "aeb268c03cb92a112177352ea91fa6f13691d0bbd28729397e25ca9e36a63b30" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "library", + "path": "ISO-8859-5/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "61e10db337d7b99d3290cc4d0e83412621abf39f97bfa5f28a9e1249117d74f0" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "market", + "path": "ISO-8859-5/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "8d252284135141f156b8483f0eaea527549221996fdd2034b53337d8fe37782a" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "ISO-8859-5/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "7f22de320a26b19ac6054b59dbe79cc7e8e82857d0529c227c948ca33ef309b4" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-5/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "9bbfd60e8c09ca445265bae5c320ca4565c3800acb17f3fe18af74f0d4b53f66" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-6/ar/test/community-garden.bin", + "bytes": 112, + "highBytes": 92, + "uniqueHighBytes": 25, + "sha256": "e49c5c7a78e9e802de85f4e3b075486c68fa32625c0682b0bc19267616009b42" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-6/ar/train/city-morning.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 22, + "sha256": "e5a89c48b2108cf6a4e864621eb2a7894e83566bceeeac7ea4458904751cb4d4" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "library", + "path": "ISO-8859-6/ar/train/library.bin", + "bytes": 107, + "highBytes": 89, + "uniqueHighBytes": 25, + "sha256": "f91cf3a719aa42bb5d2c16831a0e01a37fbf699e3d394b0b417b43a03494c26f" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "market", + "path": "ISO-8859-6/ar/train/market.bin", + "bytes": 89, + "highBytes": 75, + "uniqueHighBytes": 24, + "sha256": "08be46278b902f4cf10072e2506c7d2d29cb075c4bc087fb632cdd29553233bc" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "workshop", + "path": "ISO-8859-6/ar/train/workshop.bin", + "bytes": 91, + "highBytes": 75, + "uniqueHighBytes": 21, + "sha256": "dccb5be7cce959474487e4c395c238144116f43076dc650e4e3be5a56a38dc02" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-6/ar/validation/river-trip.bin", + "bytes": 91, + "highBytes": 73, + "uniqueHighBytes": 24, + "sha256": "b9afd70647f0be078fd19fa1040fddfe01cd90170a944ffeaf4e529112b45684" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-7/el/test/community-garden.bin", + "bytes": 136, + "highBytes": 112, + "uniqueHighBytes": 25, + "sha256": "8f7531c39aa8efa5f4e38e8eb948f713d69cbe72badea15391e5c62870179370" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-7/el/train/city-morning.bin", + "bytes": 93, + "highBytes": 74, + "uniqueHighBytes": 25, + "sha256": "80e97194d91d40aeae9020cd96d22ac6ad01e3c39020377e32bdf63c2b9622e0" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "library", + "path": "ISO-8859-7/el/train/library.bin", + "bytes": 111, + "highBytes": 94, + "uniqueHighBytes": 27, + "sha256": "eb0448b06be644629054c0f94e0c6dccb7458d80a6c477eda1ab321f378c51bb" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "market", + "path": "ISO-8859-7/el/train/market.bin", + "bytes": 85, + "highBytes": 69, + "uniqueHighBytes": 28, + "sha256": "ee94d165652f333d57c5be996825a8723cfa76aba286f1f22a0b0d40312eece9" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "workshop", + "path": "ISO-8859-7/el/train/workshop.bin", + "bytes": 108, + "highBytes": 90, + "uniqueHighBytes": 27, + "sha256": "2466d051c231cbb22d9fa5a962bb180a88dbe805ad1b609467bd06afe5de3b02" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-7/el/validation/river-trip.bin", + "bytes": 92, + "highBytes": 75, + "uniqueHighBytes": 25, + "sha256": "4dc6ccfd555071ba0df444d4f2fd5b0eb96cf44e389dc872279e8ac1096b359e" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-8/he/test/community-garden.bin", + "bytes": 104, + "highBytes": 84, + "uniqueHighBytes": 21, + "sha256": "6044aa4ee9d7f7517ca965d0a83029c5569a83e8efad7b9f39974fa95fd61298" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-8/he/train/city-morning.bin", + "bytes": 79, + "highBytes": 64, + "uniqueHighBytes": 19, + "sha256": "5c0eddaa74015d9d135cd9c9d80b117cd82a3aa3effe6298deaca2c06ac1305b" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "library", + "path": "ISO-8859-8/he/train/library.bin", + "bytes": 86, + "highBytes": 72, + "uniqueHighBytes": 20, + "sha256": "b9b2605035a9017f18f9de847e03e623581ffed30ea1623fe332efcac420637b" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "market", + "path": "ISO-8859-8/he/train/market.bin", + "bytes": 64, + "highBytes": 49, + "uniqueHighBytes": 21, + "sha256": "ed2975da284ad08b0261f536f69344981747bdd3b37d0ae6a4ec0a36ed1d57ca" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "workshop", + "path": "ISO-8859-8/he/train/workshop.bin", + "bytes": 80, + "highBytes": 64, + "uniqueHighBytes": 21, + "sha256": "763e4f5501fa3308c2862f48f5173d29d5fd1ccf730664a9bc4c40a714dfcbc6" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-8/he/validation/river-trip.bin", + "bytes": 63, + "highBytes": 47, + "uniqueHighBytes": 18, + "sha256": "5319026034eb165464faad8f2de9c2e6c97f01c92e71d2fc3441310417d70b28" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-9/tr/test/community-garden.bin", + "bytes": 136, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "a2224b89fee790662cf9c3cac7200da0f8c5842b4ce5d26512d76e522c6db437" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-9/tr/train/city-morning.bin", + "bytes": 87, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "a1231eabb26c3d7d770c885972062ee78365f871d1f62be5977332949e8b1a80" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "library", + "path": "ISO-8859-9/tr/train/library.bin", + "bytes": 98, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "432b50510bf2a5ab8b7b51aaf608341bea95afbf43edff4fa4d404f4a2d87a70" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "market", + "path": "ISO-8859-9/tr/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "25354256b7d0e3f936eb224f6343e5d3d73f78a504d5b2d3c260a2015a62e356" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "workshop", + "path": "ISO-8859-9/tr/train/workshop.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "0bc87de2045407c5c4db2589ab89cb9faf40cf5556d17caf0bfa916a1b4a0b03" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-9/tr/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "b94d965d212deb815bd73eabf712bb673f632c88c0a0c9f772202109e4d85e8d" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "KOI8-R/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "7be1a487f774fe73d3dd7a96f9c443d194edf77b1316e89483096c6e0c815e6c" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "KOI8-R/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "42e7be1a39fd943782012cc3c291fa4cf0d922c51b300ec7b5a3d7f3cfc5cc51" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "library", + "path": "KOI8-R/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "9ec8dd417b12bf60f2141ea177caf67a444bbd9b53cb031596df8fe60ea398c6" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "market", + "path": "KOI8-R/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "6096b9d90fa7ab1e9ec13727c3e8d92d5c57841066b4c795b1609cee03dc3110" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "KOI8-R/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "23a3947cc61c1418df7b89c74fe7eb940ef773ab0d04723cdd3c6857cc09a448" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "KOI8-R/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "893c13722e2ba6f81610cbebf9abb72940b3ef07a639613b2ca87cd993c925d8" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "test", + "document": "community-garden", + "path": "KOI8-U/uk/test/community-garden.bin", + "bytes": 181, + "highBytes": 152, + "uniqueHighBytes": 29, + "sha256": "25d71bdf3678270bf3a539771f8e429faaef3157c5649c0f2edc4ec06e0c80fa" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "city-morning", + "path": "KOI8-U/uk/train/city-morning.bin", + "bytes": 205, + "highBytes": 168, + "uniqueHighBytes": 31, + "sha256": "853b422f64ef06d7a64528db49d4eca0f11240852392e51cb2700b1b8b9fd122" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "library", + "path": "KOI8-U/uk/train/library.bin", + "bytes": 182, + "highBytes": 151, + "uniqueHighBytes": 32, + "sha256": "42455a09e6cafcd694b38198b1f172e44e63774cde6c9e760181334974536fe5" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "market", + "path": "KOI8-U/uk/train/market.bin", + "bytes": 199, + "highBytes": 165, + "uniqueHighBytes": 33, + "sha256": "0d8b40da8ad30cc4d6d849f2439f472db8092d622e428747aa55e1fa1797800b" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "workshop", + "path": "KOI8-U/uk/train/workshop.bin", + "bytes": 189, + "highBytes": 161, + "uniqueHighBytes": 30, + "sha256": "b4b92fa64c4b9252511ed0240a7e1cb565a0b1158f07b50fefb93fab1bd5f0f7" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "validation", + "document": "river-trip", + "path": "KOI8-U/uk/validation/river-trip.bin", + "bytes": 195, + "highBytes": 160, + "uniqueHighBytes": 31, + "sha256": "1b015aa68443a92670b3e73e37fa96d70d1e2dea1a9ed736bebfcf077bde9dfe" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "macintosh/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "9a35ac96816b79920d3076818a106719d95169d017b89832ff735652fdbc45d5" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "macintosh/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "d826a59c9fe0ea77efe5554f688a96a25ba31edd3b502177f88b229237bfafff" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "library", + "path": "macintosh/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "29829a1ac13b7f7ad2ecd8abd7d0f2454b3beb4ede31af4ab1a8901cb73103b9" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "market", + "path": "macintosh/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "f135969c32b723540e69570512179a361f12aca33782130567cafc8b8cb40974" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "workshop", + "path": "macintosh/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a77aa45de75ac7b572174e1c99aa8aab3d7a3cb0278b2f68932d1c7fe4598218" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "macintosh/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "b2edf27b09f5aeaf742baed4ebc17e0a890da6a9f0aa52a40adbcb16104e5538" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "macintosh/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "b675b0218b3db70d7c6f6f43cd7eb00fe7b45713f6e23e3fbac03a176e95935a" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "macintosh/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "15f13077700d48818a5a5050c47683e1c14260cc22ce956abd29bd07eddf10eb" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "library", + "path": "macintosh/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "a489cbb2f2f276564faf85fd6f86795a1ec5a1e5b1cfeda35874e2302e9d3d1b" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "market", + "path": "macintosh/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "80770be98c0f1e40ee59e9de08615620aa8a0af1ff73ef8b6a3cc483b14a3a4e" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "workshop", + "path": "macintosh/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "5b25ecfaa0df015d5c4c1fb52fce66af137588ac8677acd50c468b65b5112018" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "macintosh/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "d68dda4671b8158cbff67ab7b476d2b819ac45ad00719d4dbe48f22ba1862baa" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "macintosh/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "50363a67b8df69b6fbdb3b7a5a25c22c40cdac6ccca71d3bfafeff490e122c58" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "macintosh/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "5f8dbd5bcfe437962ef5943c707e1695edce736abace97d5b29ef68945c42415" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "library", + "path": "macintosh/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "66ecc14482e9af6c7d2c813d11462760e1a70b98a6d6c0c966821cd03f07eee3" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "market", + "path": "macintosh/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "5e81aa00b93ef4299cde155e0d215906411a1b0566a676ea307403ef658274d5" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "macintosh/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "ee23ebed42809949b51365120200f135cb051471430331b7c6ccf8282841075a" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "macintosh/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "test", + "document": "community-garden", + "path": "windows-1250/cs/test/community-garden.bin", + "bytes": 128, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "1eba2d29ae0f73a2f12f43469971a65f81c58779530b1b753285cb2d8c29200d" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "city-morning", + "path": "windows-1250/cs/train/city-morning.bin", + "bytes": 98, + "highBytes": 13, + "uniqueHighBytes": 6, + "sha256": "a0f0dfa252459d39f725fee0b0a13818681efd1c28e854e4eaf32509e03f5a55" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "library", + "path": "windows-1250/cs/train/library.bin", + "bytes": 88, + "highBytes": 12, + "uniqueHighBytes": 7, + "sha256": "1469e1d9d57e6f147a56fb9caf1f5b0eab61dedd7c5482c19199de6853ade2ef" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "market", + "path": "windows-1250/cs/train/market.bin", + "bytes": 74, + "highBytes": 10, + "uniqueHighBytes": 7, + "sha256": "083f02c5caabf56bf78ab98fc83cabb38dfc79337e33d3a0f694de80c2403e0e" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "workshop", + "path": "windows-1250/cs/train/workshop.bin", + "bytes": 90, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "98ad6215f5c5e7ebaa4324bd133c34bd89e63abc0fcffaac9bfa73ecd3175f17" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/cs/validation/river-trip.bin", + "bytes": 80, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "7934c52e77661f3ba94150b262a68510c2da7c9a6d493fe1d72c2ea7658a97b5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "test", + "document": "community-garden", + "path": "windows-1250/hu/test/community-garden.bin", + "bytes": 139, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "d55e643af71274694d74cff7dbf12589bfd13a2b86f2ddcfb53476578a3132b8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "city-morning", + "path": "windows-1250/hu/train/city-morning.bin", + "bytes": 94, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "97945a679eae0390b0b3c65608b9cffe4cd8ba6196b8af4be0cec9272bf25ae4" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "library", + "path": "windows-1250/hu/train/library.bin", + "bytes": 108, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "bb34de899f6968d695ac39ed18a9f72077d309da97d9721d4e6551562d644427" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "market", + "path": "windows-1250/hu/train/market.bin", + "bytes": 80, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a6a508b4927acd6751a091826a4cc4945330021699819d1c956c8ea6b440a56b" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "workshop", + "path": "windows-1250/hu/train/workshop.bin", + "bytes": 101, + "highBytes": 14, + "uniqueHighBytes": 7, + "sha256": "2a294d4fef2aeeee691b74de2293bdf7a7bae7854b3021607fc7df7d17a1a3a4" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/hu/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "ce35a0b3ccaa4b07727c0b9e925c958ad050a5d774d91c3f25962c3b7291ffab" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "test", + "document": "community-garden", + "path": "windows-1250/pl/test/community-garden.bin", + "bytes": 188, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "bfb9b6bd673324ae75dba20653fe290162c78cc26d13be88fa4fd57ee3a7b101" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "city-morning", + "path": "windows-1250/pl/train/city-morning.bin", + "bytes": 174, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "809b42296a766532c1dedb56aaf287db6538397f3f221c8164a2c627871187d5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "library", + "path": "windows-1250/pl/train/library.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 6, + "sha256": "d9f13cd5121d5f68350e48835968eee29e76da5406a7e6de2b3ed5481c7f51f0" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "market", + "path": "windows-1250/pl/train/market.bin", + "bytes": 166, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "bda4333ebf0ad7c3f37cf924b23be19ab53ddc3117f6031d8075c846a76299b5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "workshop", + "path": "windows-1250/pl/train/workshop.bin", + "bytes": 165, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "3344ae802a8cccebcb4561ccc3320991aa1c3c789bcf6172b7427d9bbf7fa6a0" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/pl/validation/river-trip.bin", + "bytes": 161, + "highBytes": 9, + "uniqueHighBytes": 3, + "sha256": "aac1576aea489691f339c2cce058bb908effa44befab53e4f04a82698896fc54" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "windows-1250/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "windows-1250/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "library", + "path": "windows-1250/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "market", + "path": "windows-1250/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "windows-1250/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "windows-1251/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "b02d0279ed09a186f1a89564cc9e00cf02fd68b0db74a1be7529ee85c8481256" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "windows-1251/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "74c61cbdff726b61e50b34c19a5777c7c13fef80f82ff10677136da50064ada8" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "library", + "path": "windows-1251/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "52bcdfd21b84cf735a555d8a7669e5dc36c12509459df325f3eb3ce5b5891377" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "market", + "path": "windows-1251/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "ca18ba887b386031b0a5dc1fc1731c94314ad760c063947f18513da039200773" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "windows-1251/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "cc7dfbf50476621ae34c5f6de6eae7ba3e13fc650f507a17fc6c07e77bf45ee8" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "windows-1251/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "78219367c687aac640cc10f65cb672bcc274a0418acb42669d050d81c849c329" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "test", + "document": "community-garden", + "path": "windows-1252/da/test/community-garden.bin", + "bytes": 123, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "d742c49869d5f97ff0dc9bed58a7e819cf5a9f3cde419ff785f2201591fa7991" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "city-morning", + "path": "windows-1252/da/train/city-morning.bin", + "bytes": 89, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "75320357a51c6716244970d7c802f42af40a6a65fa1602e4f38e239e16ccd887" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "library", + "path": "windows-1252/da/train/library.bin", + "bytes": 105, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "4b0f38943f7eb950af8b04b16ac6ce581c48f70e933f66bd03725842d9e1bd58" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "market", + "path": "windows-1252/da/train/market.bin", + "bytes": 84, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "61d9285e5b354f3cffb320f3bb62af7d4ce86fe34ba0e5067d5bc26aa9cd010c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "workshop", + "path": "windows-1252/da/train/workshop.bin", + "bytes": 103, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "aca6cded29cf164fc9445de75292a45f007c971ea9407d7ef0d794413a7cd975" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/da/validation/river-trip.bin", + "bytes": 97, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "0edb6dd9323283e842dea2112213cf7b86608b257a36cfd318b8c71b3c01c8e0" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "windows-1252/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "windows-1252/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "library", + "path": "windows-1252/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "market", + "path": "windows-1252/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "workshop", + "path": "windows-1252/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "test", + "document": "community-garden", + "path": "windows-1252/en/test/community-garden.bin", + "bytes": 136, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8574d38336d96e1a48fd42b012b3924bc36ee57550bc58a28ed6d5c17d514c1d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "city-morning", + "path": "windows-1252/en/train/city-morning.bin", + "bytes": 108, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "ff22d70d940981c1ea141576fb34c74d841dbf2e2140d80ee55f51dd63bd9ef0" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "library", + "path": "windows-1252/en/train/library.bin", + "bytes": 117, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "12ea1dc5452af3ca116753189e184ef0a10645eee17f33211dc5f7d182318f38" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "market", + "path": "windows-1252/en/train/market.bin", + "bytes": 100, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "cbaa356df57d8557b3d2c3ca4d6ec5845077db27583e102aebdaedcacf9f2aee" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "workshop", + "path": "windows-1252/en/train/workshop.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "157d773566f7cd87f7bafb87780ba98dcc090adbdaaea2c03483e7a3b6a93a70" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/en/validation/river-trip.bin", + "bytes": 106, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c4e405fd6466194b2ddc91f07f8bcb624c904531d6d96f37e35d04847c6abbb9" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "windows-1252/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "windows-1252/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "library", + "path": "windows-1252/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "market", + "path": "windows-1252/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "workshop", + "path": "windows-1252/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "windows-1252/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "windows-1252/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "library", + "path": "windows-1252/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "market", + "path": "windows-1252/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "windows-1252/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "test", + "document": "community-garden", + "path": "windows-1252/it/test/community-garden.bin", + "bytes": 161, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "53bb7b070795cd69fba5bcc79a4958b2ac5bb23677ebdac99c80623f96d501dd" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "city-morning", + "path": "windows-1252/it/train/city-morning.bin", + "bytes": 101, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "7c59ad9c3c9ac0b84c02c4829bffabdc83817989f08836d9ee44afd8a392e43d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "library", + "path": "windows-1252/it/train/library.bin", + "bytes": 141, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "bb28d91b3f10f0208886078bf0fafbc85c7e03148069ddf5a9762809f28f8981" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "market", + "path": "windows-1252/it/train/market.bin", + "bytes": 114, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "40ce81cfc5ec4c5b655c2fb9ba519022a77da2c319950c868bc16c7f69278b11" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "workshop", + "path": "windows-1252/it/train/workshop.bin", + "bytes": 152, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "5b5704c6191b4179ee6966a2302452eaaf99c348b34e26bc210879c77def38bc" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/it/validation/river-trip.bin", + "bytes": 98, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8fdde80f1933a9dfb82cc0181c6770e9947ccf5bfd745b8251ee02f54e6a19a7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "test", + "document": "community-garden", + "path": "windows-1252/nl/test/community-garden.bin", + "bytes": 153, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "26b1ccb49c443abe7580037a8a836bf99e9d5a5135b1751b77a73e3f61c1a9bf" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "city-morning", + "path": "windows-1252/nl/train/city-morning.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "201fbe0db619c5d0d4ccecd723c27c68e0807437531553e8c17ba84cfc4335b2" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "library", + "path": "windows-1252/nl/train/library.bin", + "bytes": 150, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "5d9cdae67a5a7cf6cb319750006ea3cf6afaf379b11c35aa241ef63b203cb17b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "market", + "path": "windows-1252/nl/train/market.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "15466fb33051e25a441363879415fe1c6ec949d753105ad851a75b2054d8c0da" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "workshop", + "path": "windows-1252/nl/train/workshop.bin", + "bytes": 132, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "4fa51f6d8f9543c9df0acef5a2711a5d805fd18286c1164d786bd8ac715c183b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/nl/validation/river-trip.bin", + "bytes": 113, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "d9448db92c3fe1270814d7798759eca44290f94fd76c0ff9012dc2401b426253" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "test", + "document": "community-garden", + "path": "windows-1252/no/test/community-garden.bin", + "bytes": 116, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b86618ac6687d3769cca67ec9cb49a0d4d1dea45476592a316ae6eef40d576db" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "city-morning", + "path": "windows-1252/no/train/city-morning.bin", + "bytes": 116, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "445f262b0f2ea6f724271b8486b0cfc861cf4e081e4617db895bc36a33dea2ee" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "library", + "path": "windows-1252/no/train/library.bin", + "bytes": 126, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "96c146e3d2b86817371ba6ca5d07361d31519cb0d16fbbecfaf7bba36f0ff2eb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "market", + "path": "windows-1252/no/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "a0279dbf28e14a6d9841789f50751ec9cdb09c2743f4ac1c61e72f310e916732" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "workshop", + "path": "windows-1252/no/train/workshop.bin", + "bytes": 100, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "9c70c92214b0f0ac9000a131773f1ed1756d630037293a759982edcb621e056a" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/no/validation/river-trip.bin", + "bytes": 85, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "931a303e54cbfe3ccddf45aa0dd3956be07e4cf16bf1fe5ee0874bd4ad2bab9a" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "test", + "document": "community-garden", + "path": "windows-1252/pt/test/community-garden.bin", + "bytes": 145, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "a8e56da45e10d94745a21dd36a2721410e1b0d04b5bfee15b2c451a27c8ba62d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "city-morning", + "path": "windows-1252/pt/train/city-morning.bin", + "bytes": 99, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "c9a43be7ac9a1af0be31146446c7cea5142ba97545c304e9fdfef166c0dfabe1" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "library", + "path": "windows-1252/pt/train/library.bin", + "bytes": 116, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "634c4f0d24e7f1a8bbf9b1fd68d0fe414376d2efeff0fbfc7637d921709f2a62" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "market", + "path": "windows-1252/pt/train/market.bin", + "bytes": 96, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "79f397f7b2ac2cca51091f006462be7777befb52d36c54237a8cfb0cb20cd924" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "workshop", + "path": "windows-1252/pt/train/workshop.bin", + "bytes": 108, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "fbb52707258ca57000887397d2bd8220907fc178ed36b929c62e72bf211848e1" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/pt/validation/river-trip.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "db583dead964fc7fe8a257ebd69f9dd7388c89384bac132955a76b02150c753f" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "test", + "document": "community-garden", + "path": "windows-1252/sv/test/community-garden.bin", + "bytes": 138, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "7229bf29e0adc2b37cdb6663bb696e500e13000633665f0f226915f5f74c4853" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "city-morning", + "path": "windows-1252/sv/train/city-morning.bin", + "bytes": 101, + "highBytes": 7, + "uniqueHighBytes": 4, + "sha256": "5ac24bf0fe3bfb1d64aa066ff78e3af67bcf0b4a6f8d142fefeff0b8ad4ae72c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "library", + "path": "windows-1252/sv/train/library.bin", + "bytes": 108, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "fc05c9ad957334217feaa333f6cfe7cd206b250fbae9ea51a19d9c69f72158e7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "market", + "path": "windows-1252/sv/train/market.bin", + "bytes": 89, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "9f4f10ac20699e493cdd99b8c1ee6caa29d4a9359962aab15ac23bb773f1d26f" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "workshop", + "path": "windows-1252/sv/train/workshop.bin", + "bytes": 110, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "989f41c9b83ba4e3475a6902c33340fa7b07bc965776a09e9db165321bdcb3cb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/sv/validation/river-trip.bin", + "bytes": 86, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "84cc4d3ab5abaa2d2575d868f0056375043c7e17d6c34ecd4112fca4632bc321" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "test", + "document": "community-garden", + "path": "windows-1253/el/test/community-garden.bin", + "bytes": 136, + "highBytes": 112, + "uniqueHighBytes": 25, + "sha256": "8f7531c39aa8efa5f4e38e8eb948f713d69cbe72badea15391e5c62870179370" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "city-morning", + "path": "windows-1253/el/train/city-morning.bin", + "bytes": 93, + "highBytes": 74, + "uniqueHighBytes": 25, + "sha256": "80e97194d91d40aeae9020cd96d22ac6ad01e3c39020377e32bdf63c2b9622e0" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "library", + "path": "windows-1253/el/train/library.bin", + "bytes": 111, + "highBytes": 94, + "uniqueHighBytes": 27, + "sha256": "eb0448b06be644629054c0f94e0c6dccb7458d80a6c477eda1ab321f378c51bb" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "market", + "path": "windows-1253/el/train/market.bin", + "bytes": 85, + "highBytes": 69, + "uniqueHighBytes": 28, + "sha256": "ee94d165652f333d57c5be996825a8723cfa76aba286f1f22a0b0d40312eece9" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "workshop", + "path": "windows-1253/el/train/workshop.bin", + "bytes": 108, + "highBytes": 90, + "uniqueHighBytes": 27, + "sha256": "2466d051c231cbb22d9fa5a962bb180a88dbe805ad1b609467bd06afe5de3b02" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "validation", + "document": "river-trip", + "path": "windows-1253/el/validation/river-trip.bin", + "bytes": 92, + "highBytes": 75, + "uniqueHighBytes": 25, + "sha256": "4dc6ccfd555071ba0df444d4f2fd5b0eb96cf44e389dc872279e8ac1096b359e" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "test", + "document": "community-garden", + "path": "windows-1254/tr/test/community-garden.bin", + "bytes": 136, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "a2224b89fee790662cf9c3cac7200da0f8c5842b4ce5d26512d76e522c6db437" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "city-morning", + "path": "windows-1254/tr/train/city-morning.bin", + "bytes": 87, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "a1231eabb26c3d7d770c885972062ee78365f871d1f62be5977332949e8b1a80" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "library", + "path": "windows-1254/tr/train/library.bin", + "bytes": 98, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "432b50510bf2a5ab8b7b51aaf608341bea95afbf43edff4fa4d404f4a2d87a70" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "market", + "path": "windows-1254/tr/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "25354256b7d0e3f936eb224f6343e5d3d73f78a504d5b2d3c260a2015a62e356" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "workshop", + "path": "windows-1254/tr/train/workshop.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "0bc87de2045407c5c4db2589ab89cb9faf40cf5556d17caf0bfa916a1b4a0b03" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "validation", + "document": "river-trip", + "path": "windows-1254/tr/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "b94d965d212deb815bd73eabf712bb673f632c88c0a0c9f772202109e4d85e8d" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "test", + "document": "community-garden", + "path": "windows-1255/he/test/community-garden.bin", + "bytes": 104, + "highBytes": 84, + "uniqueHighBytes": 21, + "sha256": "6044aa4ee9d7f7517ca965d0a83029c5569a83e8efad7b9f39974fa95fd61298" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "city-morning", + "path": "windows-1255/he/train/city-morning.bin", + "bytes": 79, + "highBytes": 64, + "uniqueHighBytes": 19, + "sha256": "5c0eddaa74015d9d135cd9c9d80b117cd82a3aa3effe6298deaca2c06ac1305b" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "library", + "path": "windows-1255/he/train/library.bin", + "bytes": 86, + "highBytes": 72, + "uniqueHighBytes": 20, + "sha256": "b9b2605035a9017f18f9de847e03e623581ffed30ea1623fe332efcac420637b" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "market", + "path": "windows-1255/he/train/market.bin", + "bytes": 64, + "highBytes": 49, + "uniqueHighBytes": 21, + "sha256": "ed2975da284ad08b0261f536f69344981747bdd3b37d0ae6a4ec0a36ed1d57ca" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "workshop", + "path": "windows-1255/he/train/workshop.bin", + "bytes": 80, + "highBytes": 64, + "uniqueHighBytes": 21, + "sha256": "763e4f5501fa3308c2862f48f5173d29d5fd1ccf730664a9bc4c40a714dfcbc6" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "validation", + "document": "river-trip", + "path": "windows-1255/he/validation/river-trip.bin", + "bytes": 63, + "highBytes": 47, + "uniqueHighBytes": 18, + "sha256": "5319026034eb165464faad8f2de9c2e6c97f01c92e71d2fc3441310417d70b28" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "test", + "document": "community-garden", + "path": "windows-1256/ar/test/community-garden.bin", + "bytes": 112, + "highBytes": 92, + "uniqueHighBytes": 25, + "sha256": "a36cd96bbd0a9597105cb4292371e4c2998aabd844ba3c9b6cd7a0d85dfe23e8" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "city-morning", + "path": "windows-1256/ar/train/city-morning.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 22, + "sha256": "fd5a2fe9debabe95e8ca2e20fc6a95db0d4f7ed301bfb285a49f3c4e84e7409c" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "library", + "path": "windows-1256/ar/train/library.bin", + "bytes": 107, + "highBytes": 89, + "uniqueHighBytes": 25, + "sha256": "7d22becffc494a5a0cca03a06f66167f909ceb67d914125e0668bb5db170ec9e" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "market", + "path": "windows-1256/ar/train/market.bin", + "bytes": 89, + "highBytes": 75, + "uniqueHighBytes": 24, + "sha256": "153ade153ba47b62ef5a64e3b1215b54f3cd01607fd44708a288eb78889faf82" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "workshop", + "path": "windows-1256/ar/train/workshop.bin", + "bytes": 91, + "highBytes": 75, + "uniqueHighBytes": 21, + "sha256": "3aa1c63ed62f9fd08039dcc50fa8705a239a1b133a96fdb81c7340d5d4cd9a79" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "validation", + "document": "river-trip", + "path": "windows-1256/ar/validation/river-trip.bin", + "bytes": 91, + "highBytes": 73, + "uniqueHighBytes": 24, + "sha256": "dd2d2c110d7753296192722e41de23e67c3350eea9217a1f5935829742bd2c69" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "test", + "document": "community-garden", + "path": "windows-1257/et/test/community-garden.bin", + "bytes": 122, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b09cf914e826b9230579833e56b483b71e75865e26b0cfdd921b9f386dafb448" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "city-morning", + "path": "windows-1257/et/train/city-morning.bin", + "bytes": 94, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "68d9d20af9ad1104bdfa39df3ce31e24acc520754c942351d037bc9378a9c5bc" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "library", + "path": "windows-1257/et/train/library.bin", + "bytes": 101, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "60b30c22898450c95e27d36443d3984c00987bbd4933d796c92a6e3ca3a38d5c" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "market", + "path": "windows-1257/et/train/market.bin", + "bytes": 92, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "fd5fc0e9d16f48c3a3f750929689afe78e5da298455fce90960e402427a64d9d" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "workshop", + "path": "windows-1257/et/train/workshop.bin", + "bytes": 107, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "731b2b0230faadc87c3a981fc8ae34ec858d99970a1f73dbf0aefdcb4b58f3d0" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/et/validation/river-trip.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 2, + "sha256": "81df0820697d38b422b3a2e8954782c17d2e93d39be7111c48e0d6f7f9b4c07e" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "test", + "document": "community-garden", + "path": "windows-1257/lt/test/community-garden.bin", + "bytes": 177, + "highBytes": 17, + "uniqueHighBytes": 6, + "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "city-morning", + "path": "windows-1257/lt/train/city-morning.bin", + "bytes": 156, + "highBytes": 11, + "uniqueHighBytes": 4, + "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "library", + "path": "windows-1257/lt/train/library.bin", + "bytes": 163, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "market", + "path": "windows-1257/lt/train/market.bin", + "bytes": 171, + "highBytes": 12, + "uniqueHighBytes": 8, + "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "workshop", + "path": "windows-1257/lt/train/workshop.bin", + "bytes": 170, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/lt/validation/river-trip.bin", + "bytes": 165, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "test", + "document": "community-garden", + "path": "windows-1257/lv/test/community-garden.bin", + "bytes": 174, + "highBytes": 17, + "uniqueHighBytes": 8, + "sha256": "a42bdb992cc4b4c21108bc9a5850cb2ddff8d6b568c17a7505a69efdd8a815ba" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "city-morning", + "path": "windows-1257/lv/train/city-morning.bin", + "bytes": 163, + "highBytes": 15, + "uniqueHighBytes": 7, + "sha256": "bbd09e02d22432ac8e2295607379bdd17cf7ec9ea40d8f86c44d7bcce144c221" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "library", + "path": "windows-1257/lv/train/library.bin", + "bytes": 161, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "05ef86295a393e71f052f9932d4c2492f44fae597764429f188ab3ddcd0be5bd" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "market", + "path": "windows-1257/lv/train/market.bin", + "bytes": 170, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "1fbdc5105ca4e06c0c87edfe63a807d1543944a170f8e41f3bd50c0523753ba4" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "workshop", + "path": "windows-1257/lv/train/workshop.bin", + "bytes": 163, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "14a636582f0cdc34bcb262c63e52f58d31a81b04c91504839032ec88764f82df" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/lv/validation/river-trip.bin", + "bytes": 156, + "highBytes": 14, + "uniqueHighBytes": 4, + "sha256": "58b262baaa8f97950fb68db83d3a55a43ad3f140bc74010a9bb2a57fb1b86cc4" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "test", + "document": "community-garden", + "path": "windows-1258/vi/test/community-garden.bin", + "bytes": 128, + "highBytes": 36, + "uniqueHighBytes": 14, + "sha256": "9e50ec1534c1b7fc362908307bc6890705ecf794c0bb1e159917ec107fd77875" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "city-morning", + "path": "windows-1258/vi/train/city-morning.bin", + "bytes": 132, + "highBytes": 37, + "uniqueHighBytes": 13, + "sha256": "c2568706a878d17ba6f98ffd6881f24da76d012ee67d8469aed7556fadcb92bb" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "library", + "path": "windows-1258/vi/train/library.bin", + "bytes": 111, + "highBytes": 25, + "uniqueHighBytes": 11, + "sha256": "f319f96fe6fcbca8b0d120b30065f27299f9a356733c406bca99fa1e0d8a4d60" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "market", + "path": "windows-1258/vi/train/market.bin", + "bytes": 94, + "highBytes": 24, + "uniqueHighBytes": 12, + "sha256": "6d5fad1cf1c98c236cbe21b8ce770cf07998b21eb7a0cb84fc2723f432388a18" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "workshop", + "path": "windows-1258/vi/train/workshop.bin", + "bytes": 105, + "highBytes": 30, + "uniqueHighBytes": 11, + "sha256": "f4c388a881bcb3bde3a1c02e1b371de703b0d2c75425be1206428cb828c9d7a1" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "validation", + "document": "river-trip", + "path": "windows-1258/vi/validation/river-trip.bin", + "bytes": 93, + "highBytes": 23, + "uniqueHighBytes": 12, + "sha256": "5379d3a9a9d3a72910233391382978a455ef5c80afb3cd31931bfe08a112fcd8" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "test", + "document": "community-garden", + "path": "windows-874/th/test/community-garden.bin", + "bytes": 103, + "highBytes": 97, + "uniqueHighBytes": 35, + "sha256": "77d8715caa7248f93f1e2a9518680d6026d3688a1711b6fe5e65722fc030771a" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "city-morning", + "path": "windows-874/th/train/city-morning.bin", + "bytes": 75, + "highBytes": 69, + "uniqueHighBytes": 32, + "sha256": "69a85d6da187e6944204dc80897bd3ecfe407193e3bb7964acc21b964bc0f58e" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "library", + "path": "windows-874/th/train/library.bin", + "bytes": 99, + "highBytes": 95, + "uniqueHighBytes": 36, + "sha256": "5e22a173b7c6cc9a867f0b13cecf42da7fae43ad4a15e0affade46786443bcd6" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "market", + "path": "windows-874/th/train/market.bin", + "bytes": 66, + "highBytes": 60, + "uniqueHighBytes": 30, + "sha256": "9299c007d348b83f096f7ceca669b27be5bc82f8bf2dd5f006604fb8302cc2ce" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "workshop", + "path": "windows-874/th/train/workshop.bin", + "bytes": 82, + "highBytes": 77, + "uniqueHighBytes": 38, + "sha256": "6ece49322ff6115b207f13799d415787aeaa84d251b8a8b7f5e318d7acca9530" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "validation", + "document": "river-trip", + "path": "windows-874/th/validation/river-trip.bin", + "bytes": 72, + "highBytes": 69, + "uniqueHighBytes": 27, + "sha256": "257c08098491bde6fa4830723d074c608082f28073699548f1242c9e0565366e" }, { "encoding": "x-mac-cyrillic", diff --git a/corpus/generated/ngrams.mjs b/corpus/generated/ngrams.mjs index 8fb8a41..5b27b94 100644 --- a/corpus/generated/ngrams.mjs +++ b/corpus/generated/ngrams.mjs @@ -207,6 +207,266 @@ export default [ 0x0a83ae, 0x0a8da0, ], }, + { + encoding: 'ISO-8859-1', + language: 'da', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x676572, 0x6f6720, 0x64656e, 0x646572, + 0x6c6520, 0x6e6465, 0x737465, 0x74656b, 0x0a0a42, 0x206269, 0x206279, + 0x206465, 0x20686f, 0x2070e5, 0x2c206f, 0x626962, 0x626c69, 0x640a0a, + 0x652062, 0x652066, 0x65640a, 0x656e73, 0x65722c, 0x65722e, 0x65726e, + 0x69626c, 0x696465, 0x696c6c, 0x696f74, 0x6b7374, 0x6c696f, 0x6c6c65, + 0x6e6572, 0x6f7465, 0x70e520, 0x722065, 0x722c20, 0x722e0a, 0x726e65, + 0x736572, 0x746572, 0x766572, 0x79656e, 0xe52064, 0xf86765, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4ce6, 0x206272, 0x206275, + 0x2062f8, 0x206361, 0x20656c, 0x20656e, 0x206669, 0x20666f, 0x206672, + 0x2066f8, 0x206869, 0x20686a, 0x206920, 0x206b6f, 0x206c61, 0x206c69, + 0x206d65, 0x206d75, 0x206f6d, 0x206f73, 0x20706c, 0x20736b, 0x207374, + 0x2073e6, 0x207465, 0x207472, 0x207479, 0x20756e, 0x207665, 0x207669, + 0x2076e5, 0x2076e6, 0x20e562, 0x20e662, 0x2c2063, 0x2c206d, 0x2c2075, + 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, 0x457420, 0x4ce673, + 0x4cf872, 0x4d6f72, 0x61622c, 0x616473, 0x6166e9, 0x616773, 0x616c74, + 0x616e67, 0x617265, 0x61726b, 0x61766c, 0x622c20, 0x626c65, 0x626e65, + 0x6272f8, 0x627573, 0x627965, 0x627967, 0x62f867, 0x636166, 0x642062, + 0x642c20, 0x646167, 0x646520, 0x64652c, 0x64656c, 0x64732e, 0x652068, + 0x65206f, 0x652070, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'de', + documents: 4, + trigrams: [ + 0x656e20, 0x636865, 0x657220, 0x206465, 0x20756e, 0x696520, 0x736368, + 0x756e64, 0x65696e, 0x68656e, 0x696368, 0x696e65, 0x6e2064, 0x6e6420, + 0x6e656e, 0x206469, 0x626572, 0x646572, 0x687265, 0x6c6569, 0x205465, + 0x206569, 0x446965, 0x616368, 0x636874, 0x64656e, 0x646965, 0x652c20, + 0x656974, 0x65722e, 0x65726e, 0x686572, 0x696e20, 0x697461, 0x697465, + 0x6b6c65, 0x6c6572, 0x6e2042, 0x6e2069, 0x6e2e20, 0x6e6520, 0x726520, + 0x726569, 0x742065, 0x746520, 0x74656e, 0x204269, 0x204765, 0x204d65, + 0x205265, 0x205374, 0x206265, 0x2066fc, 0x206968, 0x206b6c, 0x206c65, + 0x207665, 0x20fc62, 0x2e2042, 0x2e2044, 0x426962, 0x45696e, 0x4d656e, + 0x537461, 0x616474, 0x616c65, 0x616d20, 0x617573, 0x626c69, 0x636872, + 0x636b65, 0x642066, 0x65204d, 0x652053, 0x652054, 0x652075, 0x656368, + 0x656c6c, 0x656e2e, 0x657265, 0x657277, 0x657320, 0x66fc72, 0x676974, + 0x686520, 0x68656b, 0x68656c, 0x687465, 0x69626c, 0x69636b, 0x696769, + 0x696872, 0x696f74, 0x6c6520, 0x6c6963, 0x6c696f, 0x6e206c, 0x6e2076, + 0x6e6465, 0x6e7465, 0x6e7477, 0x6f7468, 0x722047, 0x722053, 0x722064, + 0x722e0a, 0x722e20, 0x726e20, 0x736520, 0x73652c, 0x737461, 0x737465, + 0x737563, 0x740a0a, 0x742061, 0x746164, 0x74616c, 0x746572, 0x746865, + 0x747769, 0x756368, 0x757373, 0x766572, 0x776963, 0xe46e64, 0xfc6265, + 0xfc7220, 0x0a0a41, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'en', + documents: 4, + trigrams: [ + 0x20616e, 0x616e64, 0x686520, 0x6e6420, 0x727320, 0x207468, 0x2c2061, + 0x746865, 0x652061, 0x652c20, 0x656164, 0x657273, 0x657320, 0x732061, + 0x206120, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206c69, 0x206e65, + 0x207265, 0x207768, 0x546865, 0x616465, 0x626f75, 0x627261, 0x636974, + 0x642063, 0x64206e, 0x646520, 0x646573, 0x652063, 0x657369, 0x68696c, + 0x696272, 0x696465, 0x696c65, 0x697479, 0x6c6520, 0x6c6962, 0x6c6c20, + 0x6e2074, 0x722064, 0x726172, 0x726561, 0x732062, 0x732072, 0x732073, + 0x732c20, 0x732e0a, 0x776869, 0x790a0a, 0x0a0a41, 0x0a0a46, 0x0a0a52, + 0x0a0a54, 0x0a4120, 0x0a4661, 0x0a5265, 0x0a5468, 0x206162, 0x206170, + 0x206172, 0x206265, 0x20626f, 0x206361, 0x20636c, 0x206469, 0x20646f, + 0x206578, 0x206661, 0x206775, 0x206869, 0x20696e, 0x206974, 0x206a61, + 0x206d61, 0x206d75, 0x206e61, 0x206f70, 0x207072, 0x2070e2, 0x207175, + 0x2072e9, 0x2072f4, 0x207363, 0x207365, 0x20736c, 0x20736d, 0x207465, + 0x207761, 0x20776f, 0x2c2063, 0x2c206a, 0x2c2073, 0x3b2063, 0x412073, + 0x446967, 0x466172, 0x4d6f72, 0x526561, 0x536174, 0x612066, 0x612072, + 0x61626f, 0x616420, 0x61642c, 0x6166e9, 0x616b65, 0x616c20, 0x616c61, + 0x616c6c, 0x616d20, 0x616e20, 0x616e67, 0x617065, 0x617070, 0x617220, + 0x617265, 0x617269, 0x61726b, 0x61726d, 0x617274, 0x617279, 0x617475, + 0x617920, 0x61e761, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'es', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x617320, 0x656c20, 0x656e20, 0x657320, 0x206c61, + 0x206c6f, 0x207920, 0x2e204c, 0x61646f, 0x616e20, 0x6c6f73, 0x6e206c, + 0x6f7265, 0x726573, 0x706172, 0x732063, 0x204c6f, 0x206369, 0x20656c, + 0x207061, 0x4c6f73, 0x612065, 0x612070, 0x617261, 0x626c69, 0x6c206d, + 0x6c6120, 0x6e6120, 0x732061, 0x732e20, 0x746f72, 0x206269, 0x20636c, + 0x20636f, 0x206469, 0x206d65, 0x207265, 0x207375, 0x207665, 0x456c20, + 0x4c6120, 0x612062, 0x626962, 0x636961, 0x646520, 0x64656c, 0x646967, + 0x656361, 0x65732e, 0x69626c, 0x696369, 0x696f74, 0x6c2063, 0x6c696f, + 0x6e2065, 0x6f2073, 0x6f2e20, 0x6f7465, 0x726120, 0x732064, 0x732070, + 0x732076, 0x732e0a, 0x746563, 0x0a0a45, 0x204c61, 0x206361, 0x206c65, + 0x206d61, 0x20706f, 0x207072, 0x207365, 0x207465, 0x2c206c, 0x612063, + 0x612064, 0x61206c, 0x612079, 0x616e61, 0x61726f, 0x61732c, 0x61732e, + 0x627265, 0x627573, 0x636120, 0x636164, 0x636965, 0x636975, 0x636c61, + 0x636f2e, 0x636f6d, 0x646164, 0x64656e, 0x646f20, 0x646f72, 0x656369, + 0x656e61, 0x657175, 0x657263, 0x657274, 0x676974, 0x676f20, 0x696120, + 0x696173, 0x69656c, 0x69656e, 0x696769, 0x697461, 0x697564, 0x6c6172, + 0x6c6173, 0x6c6c65, 0x6d6572, 0x6d7061, 0x6e2070, 0x6e2072, 0x6e6173, + 0x6e6f73, 0x6f2064, 0x6f2079, 0x6f636f, 0x6f6d70, 0x6f732e, 0x706f63, + 0x717565, 0x717569, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'fr', + documents: 4, + trigrams: [ + 0x657320, 0x206c65, 0x656e74, 0x6c6573, 0x6e7420, 0x74206c, 0x732070, + 0x206465, 0x657420, 0x657572, 0x727320, 0x742064, 0x206475, 0x206574, + 0x647520, 0x6c6520, 0x732063, 0x757273, 0x204c65, 0x20636f, 0x207072, + 0x2e204c, 0x4c6573, 0x646573, 0x652063, 0x652070, 0x207061, 0x20706f, + 0x2c206c, 0x626c69, 0x65206d, 0x656e63, 0x65732e, 0x6c6c65, 0x6f7572, + 0x706172, 0x717565, 0x726520, 0x732e20, 0x746573, 0x756520, 0x757220, + 0x206269, 0x206361, 0x20656e, 0x206d61, 0x207361, 0x4c6520, 0x616765, + 0x617265, 0x626962, 0x637465, 0x646520, 0x650a0a, 0x652064, 0x652065, + 0x652073, 0x69626c, 0x69656e, 0x696c6c, 0x696f74, 0x6c6575, 0x6c696f, + 0x6d656e, 0x6e6365, 0x6e6520, 0x6f6e73, 0x6f7468, 0x7072e9, 0x72206c, + 0x726368, 0x72e970, 0x732065, 0x73206c, 0x732073, 0x746575, 0xe97061, + 0xe97269, 0x0a0a4c, 0x20636c, 0x206672, 0x206c61, 0x206e75, 0x207265, + 0x2072e9, 0x20756e, 0x207669, 0x2c2064, 0x4c6120, 0x556e65, 0x612062, + 0x616972, 0x616973, 0x617263, 0x636865, 0x6368e9, 0x636965, 0x636f6e, + 0x652e20, 0x656e20, 0x65732c, 0x676573, 0x686973, 0x68e871, 0x68e920, + 0x69656c, 0x696572, 0x696e20, 0x697175, 0x697265, 0x697374, 0x697420, + 0x6c6120, 0x6c656e, 0x6c6965, 0x6d6172, 0x6d6d65, 0x6de972, 0x6e2070, + 0x6e7320, 0x6e7473, 0x6e756d, 0x6f6d6d, 0x6f7576, 0x706f73, 0x706f75, + 0x70726f, 0x72656e, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'it', + documents: 4, + trigrams: [ + 0x6c6120, 0x207069, 0x656e74, 0x206465, 0x206520, 0x612070, 0x6c6c61, + 0x6e6f20, 0x6f7269, 0x746f72, 0x206369, 0x206c61, 0x612067, 0x612069, + 0x61746f, 0x636974, 0x64656c, 0x657263, 0x692061, 0x692063, 0x692073, + 0x696120, 0x697474, 0x69f920, 0x6c6920, 0x7069f9, 0x726920, 0x7474e0, + 0x206269, 0x206361, 0x20636f, 0x206469, 0x206765, 0x206920, 0x206c65, + 0x206d65, 0x207065, 0x207072, 0x207363, 0x207369, 0x207374, 0x2c2063, + 0x4c6120, 0x612062, 0x612063, 0x612064, 0x61206c, 0x616666, 0x616e6f, + 0x626962, 0x626c69, 0x636166, 0x636f6c, 0x637269, 0x65206c, 0x652073, + 0x652c20, 0x656361, 0x656c20, 0x656c65, 0x656c6c, 0x657363, 0x657474, + 0x6666e8, 0x67696f, 0x676c69, 0x692070, 0x69626c, 0x696f20, 0x696f74, + 0x6c696f, 0x6d656e, 0x6e6120, 0x6e7465, 0x6e7472, 0x6f2064, 0x6f2065, + 0x6f2070, 0x6f2c20, 0x6f6e6f, 0x6f7465, 0x706572, 0x726120, 0x726361, + 0x726520, 0x726961, 0x736372, 0x746120, 0x746563, 0x7a6120, 0x0a0a47, + 0x0a0a49, 0x0a0a4c, 0x0a0a55, 0x0a476c, 0x0a4920, 0x0a4c61, 0x0a556e, + 0x206120, 0x206167, 0x206169, 0x206170, 0x206173, 0x206175, 0x206365, + 0x206368, 0x20666f, 0x206672, 0x206769, 0x20676c, 0x206775, 0x20696c, + 0x20696e, 0x206c69, 0x206d69, 0x206d75, 0x206e65, 0x207061, 0x207361, + 0x207371, 0x207376, 0x207472, 0x207574, 0x207665, 0x2c2066, 0x2c2067, + 0x2c2069, 0x2c206d, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'nl', + documents: 4, + trigrams: [ + 0x656e20, 0x20656e, 0x646520, 0x206465, 0x656572, 0x74656e, 0x766572, + 0x206f70, 0x207374, 0x207465, 0x652062, 0x652065, 0x652073, 0x657273, + 0x6e206f, 0x6e2e0a, 0x736368, 0x742064, 0x746572, 0x206269, 0x206361, + 0x20636f, 0x206565, 0x206765, 0x206865, 0x206c65, 0x206f76, 0x207665, + 0x207765, 0x2c2063, 0x446520, 0x61616b, 0x616173, 0x6166e9, 0x616b74, + 0x626962, 0x626c69, 0x636166, 0x636874, 0x652063, 0x65696e, 0x656b65, + 0x656e2e, 0x656e73, 0x657220, 0x657264, 0x657265, 0x65726b, 0x66e973, + 0x687465, 0x69626c, 0x696465, 0x696e20, 0x696e67, 0x696f74, 0x697320, + 0x6b6520, 0x6b656e, 0x6b7420, 0x6c6520, 0x6c6565, 0x6c6569, 0x6c696f, + 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206d, 0x6e2074, 0x6e2076, 0x6e2077, + 0x6f656b, 0x6f6e74, 0x6f7020, 0x6f7065, 0x6f7468, 0x6f7665, 0x70656e, + 0x72656e, 0x727320, 0x732065, 0x73206f, 0x732c20, 0x73656e, 0x737365, + 0x737461, 0x737465, 0x746164, 0x746865, 0xe97320, 0x0a0a42, 0x0a0a44, + 0x0a0a45, 0x0a0a4c, 0x0a426f, 0x0a4465, 0x0a4565, 0x0a4c65, 0x206170, + 0x20626f, 0x206272, 0x206275, 0x206472, 0x206475, 0x20686f, 0x206875, + 0x206964, 0x20696e, 0x206b61, 0x206b6c, 0x206c61, 0x206d61, 0x206d65, + 0x206d75, 0x206e61, 0x206f6e, 0x20706c, 0x20706f, 0x207363, 0x207477, + 0x207761, 0x207a6f, 0x20e9e9, 0x2c2061, 0x2c206b, 0x426f65, 0x446967, + 0x45656e, 0x4c657a, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'no', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x6f6720, 0x656e65, 0x206c61, 0x652066, + 0x676572, 0x6e6520, 0x6e6572, 0x737465, 0x74656b, 0x0a0a42, 0x206269, + 0x206272, 0x206465, 0x20666f, 0x2070e5, 0x207665, 0x626962, 0x626c69, + 0x640a0a, 0x646572, 0x652062, 0x652074, 0x652c20, 0x65640a, 0x656e73, + 0x657273, 0x657420, 0x672073, 0x676520, 0x67656e, 0x69626c, 0x696765, + 0x696e67, 0x696f74, 0x697465, 0x6b6572, 0x6b7374, 0x6c6167, 0x6c6520, + 0x6c6967, 0x6c696f, 0x6e206f, 0x6e6465, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x72206f, 0x72656e, 0x726765, 0x746520, 0x746572, + 0x746f72, 0x76656e, 0x766572, 0x79656e, 0xe52064, 0xf86b65, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4c65, 0x206275, 0x206279, + 0x2062f8, 0x2064f8, 0x20656c, 0x20656e, 0x206570, 0x206665, 0x206669, + 0x2066f8, 0x206869, 0x20686a, 0x20686f, 0x206920, 0x206b61, 0x206b6f, + 0x206c69, 0x206d65, 0x206d75, 0x206ee6, 0x206f6d, 0x206f73, 0x207365, + 0x20736a, 0x20736b, 0x207374, 0x207465, 0x20746f, 0x207472, 0x207479, + 0x20756e, 0x207669, 0x2076e5, 0x20e570, 0x20e672, 0x2c2065, 0x2c206b, + 0x2c206f, 0x2c2075, 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, + 0x457420, 0x4c6573, 0x4cf872, 0x4d6f72, 0x616665, 0x616720, 0x616765, + 0x616773, 0x616c74, 0x616e67, 0x617020, 0x617265, 0x61726b, 0x61766c, + 0x627279, 0x6272f8, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'pt', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x206520, 0x207072, 0x696120, 0x6e6120, 0x206369, + 0x206f73, 0x612064, 0x612065, 0x61206f, 0x646520, 0x657320, 0x657363, + 0x726573, 0x732061, 0x732070, 0x732e0a, 0x0a0a4f, 0x0a4f73, 0x206269, + 0x206573, 0x206e61, 0x206f20, 0x207065, 0x207665, 0x4f7320, 0x612070, + 0x616465, 0x61646f, 0x616d20, 0x617261, 0x617320, 0x626962, 0x626c69, + 0x636120, 0x636964, 0x646120, 0x646164, 0x646967, 0x646f20, 0x652061, + 0x656d20, 0x657175, 0x666963, 0x69626c, 0x696461, 0x696d65, 0x696f74, + 0x6c696f, 0x6e7175, 0x6f2065, 0x6f7265, 0x6f7465, 0x717565, 0x717569, + 0x72616d, 0x726961, 0x726f73, 0x732063, 0x736372, 0x746563, 0x746f72, + 0x0a0a41, 0x0a0a55, 0x0a4120, 0x0a556d, 0x206120, 0x206162, 0x206163, + 0x206167, 0x20616a, 0x20616c, 0x206173, 0x206361, 0x20636c, 0x2063f3, + 0x206469, 0x20646f, 0x20656e, 0x206571, 0x206672, 0x206775, 0x206869, + 0x206c65, 0x206c69, 0x206d61, 0x206d65, 0x206d6f, 0x206d75, 0x2070e3, + 0x207175, 0x2073e1, 0x207472, 0x20f46e, 0x2c206d, 0x2c206f, 0x2c2071, + 0x2c2076, 0x412062, 0x412063, 0x4d616e, 0x4d6572, 0x4f6669, 0x556d61, + 0x610a0a, 0x612061, 0x612062, 0x612063, 0x61206d, 0x612074, 0x612e0a, + 0x616272, 0x61636f, 0x616461, 0x6166e9, 0x616761, 0x616772, 0x616a75, + 0x616c0a, 0x616c75, 0x616e68, 0x616e71, 0x616e74, 0x61722c, 0x61732e, + 0x61e761, 0x61e7e3, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'sv', + documents: 4, + trigrams: [ + 0x657220, 0x617220, 0x206f63, 0x636820, 0x656e20, 0x6f6368, 0x737461, + 0x64656e, 0x657420, 0x6b6172, 0x726e61, 0x736b61, 0x74206c, 0x746164, + 0x766572, 0x206269, 0x206465, 0x2066f6, 0x206c69, 0x2070e5, 0x20736b, + 0x207479, 0x207665, 0x612062, 0x612074, 0x612e0a, 0x61640a, 0x616465, + 0x617265, 0x626962, 0x626c69, 0x640a0a, 0x65726e, 0x66f672, 0x676120, + 0x69626c, 0x696761, 0x696f74, 0x6b6170, 0x6b6572, 0x6b6e61, 0x6c6967, + 0x6c696f, 0x6e612e, 0x6e6172, 0x6e736b, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x722066, 0x72206f, 0x736172, 0x73f66b, 0x746120, + 0x74656b, 0x746f72, 0xe52064, 0x0a0a42, 0x0a0a45, 0x0a0a4c, 0x0a0a53, + 0x0a42f6, 0x0a4574, 0x0a4ce4, 0x0a5374, 0x206265, 0x206272, 0x206275, + 0x2062f6, 0x20656c, 0x20656e, 0x2066e4, 0x206772, 0x206869, 0x20686a, + 0x20686f, 0x206920, 0x206b61, 0x206b6f, 0x206c61, 0x206ce5, 0x206d65, + 0x206d75, 0x206de4, 0x206f6d, 0x206f73, 0x207374, 0x2073e4, 0x2073f6, + 0x207465, 0x20746f, 0x207661, 0x2076e4, 0x20e470, 0x20f670, 0x2c2067, + 0x2c206b, 0x2c206f, 0x2c20e4, 0x42f66e, 0x446574, 0x446967, 0x457474, + 0x4ce473, 0x4cf672, 0x4d6f72, 0x537461, 0x61206f, 0x6120f6, 0x6166e9, + 0x616720, 0x616773, 0x616b6e, 0x616c20, 0x616d74, 0x616e20, 0x616e73, + 0x617020, 0x617061, 0x617269, 0x61726b, 0x61726e, 0x626573, 0x6272f6, + 0x627573, 0x62f663, + ], + }, { encoding: 'ISO-8859-10', language: 'is', @@ -367,6 +627,110 @@ export default [ encoding: 'ISO-8859-16', language: 'ro', documents: 4, + trigrams: [ + 0x696920, 0x756c20, 0x206465, 0x6f7269, 0x726969, 0x746520, 0x746f72, + 0x692063, 0x7265fe, 0x203130, 0x205072, 0x206573, 0x20a42e, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x2e2050, 0x3020a4, 0x313020, 0x507265, 0x652031, + 0x652c20, 0x656c65, 0x657374, 0x65fe75, 0x692e20, 0x696572, 0x6c2065, + 0x737465, 0xa42e0a, 0xba6920, 0xfe756c, 0x206d69, 0x207072, 0x207265, + 0x61fe61, 0x626c69, 0x636869, 0x646520, 0x652070, 0x652e20, 0x656361, + 0x657269, 0x657363, 0x69626c, 0x696e65, 0x696f74, 0x6c6520, 0x6c696f, + 0x6f7465, 0x736520, 0x746563, 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, + 0x20616c, 0x206175, 0x206361, 0x20636c, 0x206375, 0x2063e3, 0x206469, + 0x206578, 0x206961, 0x206d65, 0x206f72, 0x207065, 0x207363, 0x207365, + 0x207472, 0x20ba74, 0x2c2069, 0x426962, 0x506961, 0x612064, 0x616c20, + 0x617220, 0x617574, 0x617ae3, 0x636120, 0x636172, 0x636574, 0x63e320, + 0x646573, 0x646967, 0x652063, 0x652064, 0x652074, 0x6520ee, 0x65617a, + 0x656e69, 0x656e74, 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, + 0x65ba74, 0x676974, 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, + 0x6963e3, 0x696769, 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, + 0x6c2064, 0x6c6567, 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, + 0x6d696e, 0x6e6365, 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, + 0x7261ba, 0x72652c, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'cs', + documents: 4, + trigrams: [ + 0x206120, 0x207072, 0x616aed, 0x6aed20, 0x206b6e, 0x636520, 0x6b6e69, + 0x6e6968, 0x6eed20, 0x792e0a, 0x0a0a4d, 0x20706f, 0x20e865, 0x2c206b, + 0x610a0a, 0x612070, 0x612076, 0x652061, 0x65206d, 0x652070, 0x686f76, + 0x69686f, 0x6a6520, 0x6e610a, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, + 0x73792e, 0x756a65, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xf86920, + 0x0a0a46, 0x0a0ac8, 0x0a4661, 0x0a4d61, 0x0a4dec, 0x0ac874, 0x206175, + 0x206279, 0x206368, 0x206476, 0x2064ed, 0x206869, 0x20686c, 0x206a61, + 0x206b61, 0x206b6f, 0x206bf3, 0x206c69, 0x206d65, 0x206d75, 0x206dec, + 0x206e61, 0x206f20, 0x206f74, 0x206f76, 0x2070ed, 0x2070f8, 0x207261, + 0x207365, 0x207472, 0x2074fd, 0x207665, 0x20766f, 0x2076ec, 0x207a61, + 0x20bee1, 0x2c206d, 0x2c206f, 0x2c207a, 0x446967, 0x466172, 0x4d616c, + 0x4dec73, 0x52e16e, 0x536f62, 0x546963, 0x61206c, 0x6164ed, 0x616c75, + 0x616cfd, 0x61726d, 0x61736e, 0x6174ed, 0x617574, 0x617675, 0x6176e1, + 0x6176e9, 0x622c20, 0x626f74, 0x626f75, 0x627573, 0x62796c, 0x63686c, + 0x6368e1, 0x636f20, 0x642061, 0x642c20, 0x64616a, 0x646365, 0x647665, + 0x64e176, 0x64e920, 0x64ec2c, 0x64ed20, 0x64ed6c, 0x65206a, 0x65206b, + 0x652072, 0x65642c, 0x656461, 0x65656d, 0x656b61, 0x656d2c, 0x656ee1, + 0x657273, 0x6576ed, 0x65f865, 0x676974, 0x680a0a, 0x686973, 0x686c65, + 0x686ce9, 0x687920, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'hu', + documents: 4, + trigrams: [ + 0x206120, 0x2c2061, 0x6e616b, 0x6f7320, 0x0a0a41, 0x206bf6, 0x2076e1, + 0x20e973, 0x657265, 0x6bf66e, 0x6e7976, 0x726f73, 0x73206b, 0x742c20, + 0x76e172, 0xe1726f, 0xe97320, 0xf66e79, 0x0a4120, 0x206373, 0x206b65, + 0x206b69, 0x61206b, 0x612076, 0x616b2e, 0x617420, 0x657420, 0x697320, + 0x6b2c20, 0x6b2e0a, 0x6d6920, 0x6f7420, 0x73206c, 0x73206d, 0x74206b, + 0x74e172, 0x7674e1, 0x797674, 0xf36b20, 0x0a0a45, 0x0a417a, 0x0a4567, + 0x20616c, 0x20617a, 0x206275, 0x206469, 0x20656c, 0x20656d, 0x206672, + 0x206761, 0x206be1, 0x206be9, 0x206bf3, 0x206c61, 0x206c65, 0x206de9, + 0x206dfa, 0x206dfb, 0x206f6c, 0x207065, 0x207069, 0x207361, 0x207365, + 0x207475, 0x2074e9, 0x2074f6, 0x207669, 0x20e172, 0x20e962, 0x20ed72, + 0x20fa74, 0x2c206b, 0x2c2073, 0x412063, 0x412067, 0x412076, 0x417a20, + 0x446967, 0x456779, 0x526567, 0x537a6f, 0x612064, 0x612074, 0x61630a, + 0x616a74, 0x616b20, 0x616b2c, 0x616c6d, 0x616e0a, 0x616e20, 0x617061, + 0x617373, 0x6173f3, 0x617469, 0x6174f3, 0x617a20, 0x617a64, 0x62616e, + 0x626174, 0x626572, 0x627265, 0x627573, 0x630a0a, 0x637361, 0x637365, + 0x642c20, 0x646573, 0x646967, 0x6469e1, 0x646f6d, 0x646f74, 0x64e16b, + 0x65642c, 0x656469, 0x656767, 0x6567ed, 0x656b20, 0x656b2c, 0x656b65, + 0x656c20, 0x656c6c, 0x656c6d, 0x656c79, 0x656d62, 0x656e2e, 0x656e64, + 0x656e79, 0x656ef5, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'pl', + documents: 4, + trigrams: [ + 0x616ab1, 0x6ab120, 0x207072, 0x206920, 0x207779, 0x616d69, 0x636861, + 0x70727a, 0x727a79, 0xb66369, 0x206a61, 0x20706f, 0x20726f, 0x2c2061, + 0x616368, 0x626c69, 0x636920, 0x647a69, 0x656b20, 0x692073, 0x692e20, + 0x69626c, 0x696572, 0x696f74, 0x6c696f, 0x6e6961, 0x6f7261, 0x6f7465, + 0x706973, 0x727a65, 0x74656b, 0x0a0a4d, 0x206120, 0x206269, 0x20646c, + 0x206b6f, 0x206d69, 0x207069, 0x207369, 0x207370, 0x20737a, 0x2e204b, + 0x610a0a, 0x612070, 0x612072, 0x617267, 0x626962, 0x636520, 0x63692e, + 0x637920, 0x637a79, 0x642c20, 0x64616a, 0x646c61, 0x65206f, 0x656461, + 0x656b61, 0x657261, 0x66726f, 0x68616d, 0x692077, 0x692c20, 0x696379, + 0x696520, 0x69656e, 0x697379, 0x69ea20, 0x6b6172, 0x6c6120, 0x6c6e69, + 0x6d6920, 0x6d6965, 0x6e6369, 0x6e6520, 0x6e656b, 0x6e6963, 0x6e6965, + 0x6f727a, 0x6fb663, 0x707261, 0x72616a, 0x72616d, 0x726f77, 0x7369ea, + 0x737072, 0x73746f, 0x737920, 0x737a65, 0x746f72, 0x75647a, 0x776120, + 0x776961, 0x776965, 0x792063, 0x792070, 0x792073, 0x796672, 0x7a7920, + 0x7a7974, 0xb1206a, 0xb1206b, 0xb12077, 0x0a0a54, 0x0a0a57, 0x0a4d61, + 0x0a4d69, 0x0a5461, 0x0a5720, 0x204269, 0x20437a, 0x204b61, 0x204b6c, + 0x204e61, 0x205072, 0x20526f, 0x206175, 0x206275, 0x206368, 0x20636f, + 0x206379, 0x20637a, 0x206461, 0x206472, 0x20676f, 0x206869, 0x206b61, + 0x206b73, 0x206b75, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'ro', + documents: 4, trigrams: [ 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, @@ -441,6 +805,162 @@ export default [ 0x206b6c, 0x206b72, ], }, + { + encoding: 'ISO-8859-5', + language: 'ru', + documents: 4, + trigrams: [ + 0xeee220, 0xd0eee2, 0x20dfe0, 0xdfe0de, 0x20d820, 0x20ddd0, 0x20dfde, + 0xd0ef20, 0xe0ded4, 0xebd520, 0xefe220, 0x20d2eb, 0x20d3de, 0x20dcd0, + 0x20ded1, 0xd0e2d5, 0xd5dddd, 0xd820df, 0xd82c20, 0xdbd820, 0xdcd0e0, + 0xddd020, 0xddebd5, 0xe0eb20, 0xe220e1, 0xe2d5db, 0xe2d5e0, 0xe2ded2, + 0xef20df, 0x0a0abd, 0x20d020, 0x20d1d8, 0x20d2de, 0x20dade, 0x20dee1, + 0x20dee2, 0x20dfdb, 0x20e0d5, 0x20e1de, 0x20efe0, 0x2c20d0, 0x2c20d1, + 0xbdd020, 0xd00a0a, 0xd020df, 0xd0d5e2, 0xd0ddd8, 0xd0e0da, 0xd0e1e2, + 0xd1d8d1, 0xd1dbd8, 0xd1e1e3, 0xd2d0ee, 0xd2d5e0, 0xd2efe2, 0xd3dee2, + 0xd42c20, 0xd4d0ee, 0xd4d820, 0xd5dbd5, 0xd5dbd8, 0xd5e0d8, 0xd5e0eb, + 0xd5e1e2, 0xd6d4d0, 0xd820de, 0xd820e1, 0xd8d1db, 0xd8dee2, 0xd8e1e2, + 0xd8e2d0, 0xd8ef20, 0xdad00a, 0xdad520, 0xdbd8de, 0xddd0ef, 0xddd8da, + 0xddd8ef, 0xddddef, 0xddded2, 0xddefef, 0xded1e1, 0xded2ef, 0xdedbec, + 0xdedcd0, 0xdedddd, 0xdee0de, 0xdee1d5, 0xdee1e2, 0xdee1eb, 0xdee2d5, + 0xdee2de, 0xdfd8e1, 0xdfdee1, 0xe0dad0, 0xe0dcd0, 0xe0dee1, 0xe1d5dd, + 0xe1ddeb, 0xe1e2d0, 0xe1e2d5, 0xe1e2de, 0xe1e3d6, 0xe1eb20, 0xe1ef20, + 0xe220da, 0xe220dd, 0xe220df, 0xe2d5da, 0xe2dee0, 0xe2e0de, 0xe2e1ef, + 0xe3d6d4, 0xe5ded4, 0xeb20df, 0xeb2c20, 0xeb2e0a, 0xebd2d0, 0xebd920, + 0xef20ef, 0xefe0dc, 0xefeee2, 0xefef20, 0x0a0ab2, 0x0a0ab3, 0x0ab220, + 0x0ab3de, 0x0abdd0, + ], + }, + { + encoding: 'ISO-8859-6', + language: 'ar', + documents: 4, + trigrams: [ + 0x20c7e4, 0xc7e4e5, 0x20e8c7, 0x20e8ea, 0xe3cac8, 0xe620c7, 0xe8c7e4, + 0xe8e620, 0x0a0aea, 0xc90a0a, 0xd120c7, 0xc7ca20, 0xc7ccd9, 0xc7d120, + 0xc7e4c3, 0xc7e4d9, 0xc7e4e8, 0xc820c7, 0xc8c920, 0xc920c7, 0xc920e8, + 0xc92e0a, 0xca20e8, 0xcac820, 0xcac8c9, 0xcb20c7, 0xccd920, 0xcd20c7, + 0xcfeae6, 0xd1c7cc, 0xd5e1c7, 0xd920c7, 0xe4e5cf, 0xe4e5e3, 0xe5cfea, + 0xe5e3ca, 0xe6c7d3, 0xe8d5e1, 0xeae6c9, 0x0a0aca, 0x0acad3, 0x0aeac8, + 0x0aead5, 0x0aead9, 0x20c3c8, 0x20c3e5, 0x20c7ce, 0x20c8e7, 0x20c8ea, + 0x20cfe4, 0x20d1e2, 0x20d5da, 0x20d9e6, 0x20e1d1, 0x20e1ea, 0x20e3ca, + 0x20e4e4, 0x20e8ca, 0x20e8d5, 0x20eae6, 0xc120d9, 0xc120e8, 0xc3c8e8, + 0xc3cdc7, 0xc3cec8, 0xc3e5ea, 0xc4e8e6, 0xc6c90a, 0xc720c8, 0xc720e4, + 0xc720e8, 0xc720ea, 0xc72e0a, 0xc7c120, 0xc7c820, 0xc7c8e7, 0xc7cd20, + 0xc7ceca, 0xc7cfc6, 0xc7cfe4, 0xc7cfea, 0xc7d12e, 0xc7d1d9, 0xc7d1ea, + 0xc7d320, 0xc7d3c8, 0xc7d6cd, 0xc7d9cf, 0xc7e1e4, 0xc7e3e7, 0xc7e4ca, + 0xc7e4cd, 0xc7e4ce, 0xc7e4d2, 0xc7e4d3, 0xc7e4d4, 0xc7e4d7, 0xc7e4e1, + 0xc7e4e2, 0xc7e4e6, 0xc7e4e7, 0xc7e7ea, 0xc820e1, 0xc8c7cd, 0xc8c7cf, + 0xc8c7d1, 0xc8c92e, 0xc8ca0a, 0xc8cdcb, 0xc8d1e5, 0xc8d220, 0xc8e7c7, + 0xc8e7cf, 0xc8e8c7, 0xc8eae6, 0xc920c8, 0xc920d1, 0xca0a0a, 0xcac7d1, + 0xcac8c7, 0xcacd20, 0xcacde1, 0xcad3ca, 0xcad8d1, 0xcae1ca, 0xcaeac7, + 0xcaeae2, 0xcce8e6, + ], + }, + { + encoding: 'ISO-8859-7', + language: 'el', + documents: 4, + trigrams: [ + 0x20eae1, 0xe1e920, 0xeae1e9, 0x20e2e9, 0xe120ea, 0xe12e0a, 0xe2e9e2, + 0xe5e920, 0xe9e2eb, 0xeff5ed, 0xf4e120, 0xf5ed20, 0x0a0acf, 0x0acfe9, + 0x20e1e3, 0x20e1ed, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xcfe920, + 0xdfe12e, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, 0xe5f220, 0xe70a0a, 0xe720e2, + 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9efe8, 0xe9f3f4, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xf0fceb, 0xf1e9ef, 0xf220ea, 0xf3f4de, + 0xf3f4e7, 0xf4e5f2, 0xf4e7ed, 0xfcebe7, 0x0a0ac7, 0x0a0acc, 0x0ac720, + 0x0acce9, 0x20dced, 0x20e1f1, 0x20e2ef, 0x20e3f1, 0x20e5eb, 0x20e5f0, + 0x20e5f1, 0x20e5f4, 0x20e720, 0x20e9f3, 0x20eae5, 0x20eafe, 0x20ebe5, + 0x20ecdd, 0x20ece1, 0x20ece9, 0x20ecef, 0x20ecf5, 0x20eef5, 0x20efe4, + 0x20efe9, 0x20efec, 0x20f0e5, 0x20f0eb, 0x20f0ef, 0x20f6f1, 0x20f8dc, + 0x20f8f9, 0x2c20e5, 0x2c20ec, 0x2c20f4, 0x2c20f6, 0xb9f3f5, 0xc720f0, + 0xcce9e1, 0xd0f1f9, 0xd3e1e2, 0xd8e7f6, 0xdc0a0a, 0xdc20e1, 0xdc20ea, + 0xdc20f3, 0xdc20f4, 0xdc2c20, 0xdce4e1, 0xdce6e5, 0xdcede8, 0xdcf1e9, + 0xdcf4e9, 0xdcf6e5, 0xdcf7ed, 0xdd20e1, 0xdde3f7, 0xddebe9, 0xddedef, + 0xddf22e, 0xde20ef, 0xdeeae7, 0xdeece7, 0xdef1e9, 0xdf20f3, 0xdf2c20, + 0xdfe120, 0xdfe1f2, 0xdfe3ef, 0xdfece5, 0xdfeff5, 0xe120e5, 0xe120e9, + 0xe120eb, 0xe120ec, 0xe1e2e2, 0xe1e3ed, 0xe1e3ef, 0xe1e3f1, 0xe1e8e1, + 0xe1e8e7, 0xe1eafc, + ], + }, + { + encoding: 'ISO-8859-8', + language: 'he', + documents: 4, + trigrams: [ + 0xe9ed20, 0xe5f8e9, 0xed20ec, 0xed20ee, 0xf1f4f8, 0xf8e9ed, 0x20e4f7, + 0x20ece0, 0x20eee7, 0x2c20e1, 0xe0e9ed, 0xe4f1f4, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed2e, 0xed2e0a, 0xef20ee, 0xf0e420, 0xf2e9f8, + 0xf4f8e9, 0xf7e5f8, 0xf820ee, 0xf8e5fa, 0xf8fa20, 0xf9e9ed, 0xfa0a0a, + 0xfa20ec, 0x0a0ae4, 0x0a0ae7, 0x0a0af6, 0x0a0af7, 0x0ae4f2, 0x0ae7f7, + 0x0af6e5, 0x0af7e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, + 0x20e1fa, 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e8f8, 0x20ece1, + 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, 0x20eeeb, 0x20eef7, + 0x20eefa, 0x20f0f4, 0x20f1f4, 0x20f2e5, 0x20f4e9, 0x20f7e8, 0x20f9e1, + 0x20fae9, 0x2c20e3, 0x2c20f4, 0xe0e5e8, 0xe0e5ef, 0xe0e5f8, 0xe0e82c, + 0xe0f0f9, 0xe0f9e5, 0xe0fa20, 0xe120fa, 0xe1e5e3, 0xe1e5f1, 0xe1e5f7, + 0xe1e7e5, 0xe1e9f0, 0xe1ebe9, 0xe1f2e9, 0xe1f8e5, 0xe1f92c, 0xe1fa0a, + 0xe1fae9, 0xe2e1e9, 0xe2e9e8, 0xe320e5, 0xe3e1f9, 0xe3e9e2, 0xe3e9ed, + 0xe3f0e4, 0xe3f220, 0xe3f720, 0xe3f8e9, 0xe40a0a, 0xe420e1, 0xe420e3, + 0xe420e4, 0xe420e5, 0xe420f0, 0xe42e0a, 0xe4e9f1, 0xe4f2e9, 0xe4f2ee, + 0xe4f7e5, 0xe4f7f4, 0xe4f8e0, 0xe4f9f7, 0xe5e0f0, 0xe5e1e5, 0xe5e2e1, + 0xe5e320, 0xe5e3f7, 0xe5e4f1, 0xe5e5fa, 0xe5e6e9, 0xe5e6f8, 0xe5e8e5, + 0xe5ebe5, 0xe5ebf8, + ], + }, + { + encoding: 'ISO-8859-9', + language: 'tr', + documents: 4, + trigrams: [ + 0x6c6572, 0x6c6172, 0x207665, 0x722e0a, 0x766520, 0x2061e7, 0x207961, + 0x617220, 0x61e7fd, 0x657220, 0x697220, 0x722076, 0x206269, 0x20656b, + 0x206b61, 0x206bfc, 0x207361, 0x207461, 0x2c206b, 0x616261, 0x616c20, + 0x616c61, 0x616e65, 0x61722e, 0x6172fd, 0x617a61, 0x62616c, 0x650a0a, + 0x652062, 0x656869, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697461, + 0x6bfc74, 0x706861, 0x722079, 0x722c20, 0x726920, 0x726c61, 0x746172, + 0x74fc70, 0x7a6172, 0x7a6520, 0xde6568, 0xe7fd6b, 0xfc7068, 0xfc74fc, + 0xfd6b20, 0x0a0a4b, 0x0a0a4f, 0x0a0ac7, 0x0a0ade, 0x0a4bfc, 0x0a4f6b, + 0x0ac769, 0x0ade65, 0x206172, 0x206174, 0x206261, 0x206265, 0x206465, + 0x20656c, 0x206861, 0x20696c, 0x20696e, 0x206b69, 0x206b6f, 0x206d65, + 0x206dfc, 0x206f6c, 0x206f74, 0x207061, 0x207065, 0x207265, 0x207579, + 0x20f6f0, 0x2c2065, 0x2c2070, 0x43756d, 0x44696a, 0x4bfce7, 0x4f6b75, + 0x536573, 0x612074, 0x612075, 0x612c20, 0x616665, 0x61680a, 0x616cfd, + 0x616d61, 0x616e64, 0x616e6c, 0x616efd, 0x61706c, 0x61722c, 0x617261, + 0x617264, 0x617269, 0x61726b, 0x617274, 0x617461, 0x6174f6, 0x617661, + 0x617a65, 0x617afd, 0x61fee7, 0x626168, 0x62656b, 0x626572, 0x62696c, + 0x626972, 0x62fc73, 0x636920, 0x63696c, 0x63fd20, 0x646120, 0x64616e, + 0x646520, 0x64656e, 0x647520, 0x64fd6d, 0x652061, 0x652065, 0x652069, + 0x652072, 0x652073, + ], + }, + { + encoding: 'KOI8-R', + language: 'ru', + documents: 4, + trigrams: [ + 0xc0d420, 0xc1c0d4, 0x20d0d2, 0xd0d2cf, 0x20c920, 0x20cec1, 0x20d0cf, + 0xc1d120, 0xd1d420, 0xd2cfc4, 0xd9c520, 0x20c7cf, 0x20cdc1, 0x20cfc2, + 0x20d7d9, 0xc1d4c5, 0xc5cece, 0xc920d0, 0xc92c20, 0xccc920, 0xcdc1d2, + 0xcec120, 0xced9c5, 0xd120d0, 0xd2d920, 0xd420d3, 0xd4c5cc, 0xd4c5d2, + 0xd4cfd7, 0x0a0aee, 0x20c120, 0x20c2c9, 0x20cbcf, 0x20cfd3, 0x20cfd4, + 0x20d0cc, 0x20d1d2, 0x20d2c5, 0x20d3cf, 0x20d7cf, 0x2c20c1, 0x2c20c2, + 0xc10a0a, 0xc120d0, 0xc1c5d4, 0xc1cec9, 0xc1d2cb, 0xc1d3d4, 0xc2c9c2, + 0xc2ccc9, 0xc2d3d5, 0xc42c20, 0xc4c1c0, 0xc4c920, 0xc5ccc5, 0xc5ccc9, + 0xc5d2c9, 0xc5d2d9, 0xc5d3d4, 0xc7cfd4, 0xc8cfc4, 0xc920cf, 0xc920d3, + 0xc9c2cc, 0xc9cfd4, 0xc9d120, 0xc9d3d4, 0xc9d4c1, 0xcbc10a, 0xcbc520, + 0xccc9cf, 0xcec1d1, 0xcec9cb, 0xcec9d1, 0xceced1, 0xcecfd7, 0xced1d1, + 0xcfc2d3, 0xcfccd8, 0xcfcdc1, 0xcfcece, 0xcfd2cf, 0xcfd3c5, 0xcfd3d4, + 0xcfd3d9, 0xcfd4c5, 0xcfd4cf, 0xcfd7d1, 0xd0c9d3, 0xd0cfd3, 0xd120d1, + 0xd1c0d4, 0xd1d120, 0xd1d2cd, 0xd2cbc1, 0xd2cdc1, 0xd2cfd3, 0xd3c5ce, + 0xd3ced9, 0xd3d120, 0xd3d4c1, 0xd3d4c5, 0xd3d4cf, 0xd3d5d6, 0xd3d920, + 0xd420cb, 0xd420ce, 0xd420d0, 0xd4c5cb, 0xd4cfd2, 0xd4d2cf, 0xd4d3d1, + 0xd5d6c4, 0xd6c4c1, 0xd7c1c0, 0xd7c5d2, 0xd7d1d4, 0xd920d0, 0xd92c20, + 0xd92e0a, 0xd9ca20, 0xd9d7c1, 0xeec120, 0x0a0ae7, 0x0a0af7, 0x0ae7cf, + 0x0aeec1, 0x0aeec5, + ], + }, { encoding: 'KOI8-U', language: 'uk', @@ -545,6 +1065,630 @@ export default [ 0x70726f, 0x72656e, ], }, + { + encoding: 'windows-1250', + language: 'cs', + documents: 4, + trigrams: [ + 0x206120, 0x207072, 0x616aed, 0x6aed20, 0x206b6e, 0x636520, 0x6b6e69, + 0x6e6968, 0x6eed20, 0x792e0a, 0x0a0a4d, 0x20706f, 0x20e865, 0x2c206b, + 0x610a0a, 0x612070, 0x612076, 0x652061, 0x65206d, 0x652070, 0x686f76, + 0x69686f, 0x6a6520, 0x6e610a, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, + 0x73792e, 0x756a65, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xf86920, + 0x0a0a46, 0x0a0ac8, 0x0a4661, 0x0a4d61, 0x0a4dec, 0x0ac874, 0x206175, + 0x206279, 0x206368, 0x206476, 0x2064ed, 0x206869, 0x20686c, 0x206a61, + 0x206b61, 0x206b6f, 0x206bf3, 0x206c69, 0x206d65, 0x206d75, 0x206dec, + 0x206e61, 0x206f20, 0x206f74, 0x206f76, 0x2070ed, 0x2070f8, 0x207261, + 0x207365, 0x207472, 0x2074fd, 0x207665, 0x20766f, 0x2076ec, 0x207a61, + 0x209ee1, 0x2c206d, 0x2c206f, 0x2c207a, 0x446967, 0x466172, 0x4d616c, + 0x4dec73, 0x52e16e, 0x536f62, 0x546963, 0x61206c, 0x6164ed, 0x616c75, + 0x616cfd, 0x61726d, 0x61736e, 0x6174ed, 0x617574, 0x617675, 0x6176e1, + 0x6176e9, 0x622c20, 0x626f74, 0x626f75, 0x627573, 0x62796c, 0x63686c, + 0x6368e1, 0x636f20, 0x642061, 0x642c20, 0x64616a, 0x646365, 0x647665, + 0x64e176, 0x64e920, 0x64ec2c, 0x64ed20, 0x64ed6c, 0x65206a, 0x65206b, + 0x652072, 0x65642c, 0x656461, 0x65656d, 0x656b61, 0x656d2c, 0x656ee1, + 0x657273, 0x6576ed, 0x65f865, 0x676974, 0x680a0a, 0x686973, 0x686c65, + 0x686ce9, 0x687920, + ], + }, + { + encoding: 'windows-1250', + language: 'hu', + documents: 4, + trigrams: [ + 0x206120, 0x2c2061, 0x6e616b, 0x6f7320, 0x0a0a41, 0x206bf6, 0x2076e1, + 0x20e973, 0x657265, 0x6bf66e, 0x6e7976, 0x726f73, 0x73206b, 0x742c20, + 0x76e172, 0xe1726f, 0xe97320, 0xf66e79, 0x0a4120, 0x206373, 0x206b65, + 0x206b69, 0x61206b, 0x612076, 0x616b2e, 0x617420, 0x657420, 0x697320, + 0x6b2c20, 0x6b2e0a, 0x6d6920, 0x6f7420, 0x73206c, 0x73206d, 0x74206b, + 0x74e172, 0x7674e1, 0x797674, 0xf36b20, 0x0a0a45, 0x0a417a, 0x0a4567, + 0x20616c, 0x20617a, 0x206275, 0x206469, 0x20656c, 0x20656d, 0x206672, + 0x206761, 0x206be1, 0x206be9, 0x206bf3, 0x206c61, 0x206c65, 0x206de9, + 0x206dfa, 0x206dfb, 0x206f6c, 0x207065, 0x207069, 0x207361, 0x207365, + 0x207475, 0x2074e9, 0x2074f6, 0x207669, 0x20e172, 0x20e962, 0x20ed72, + 0x20fa74, 0x2c206b, 0x2c2073, 0x412063, 0x412067, 0x412076, 0x417a20, + 0x446967, 0x456779, 0x526567, 0x537a6f, 0x612064, 0x612074, 0x61630a, + 0x616a74, 0x616b20, 0x616b2c, 0x616c6d, 0x616e0a, 0x616e20, 0x617061, + 0x617373, 0x6173f3, 0x617469, 0x6174f3, 0x617a20, 0x617a64, 0x62616e, + 0x626174, 0x626572, 0x627265, 0x627573, 0x630a0a, 0x637361, 0x637365, + 0x642c20, 0x646573, 0x646967, 0x6469e1, 0x646f6d, 0x646f74, 0x64e16b, + 0x65642c, 0x656469, 0x656767, 0x6567ed, 0x656b20, 0x656b2c, 0x656b65, + 0x656c20, 0x656c6c, 0x656c6d, 0x656c79, 0x656d62, 0x656e2e, 0x656e64, + 0x656e79, 0x656ef5, + ], + }, + { + encoding: 'windows-1250', + language: 'pl', + documents: 4, + trigrams: [ + 0x616ab9, 0x6ab920, 0x207072, 0x206920, 0x207779, 0x616d69, 0x636861, + 0x70727a, 0x727a79, 0x9c6369, 0x206a61, 0x20706f, 0x20726f, 0x2c2061, + 0x616368, 0x626c69, 0x636920, 0x647a69, 0x656b20, 0x692073, 0x692e20, + 0x69626c, 0x696572, 0x696f74, 0x6c696f, 0x6e6961, 0x6f7261, 0x6f7465, + 0x706973, 0x727a65, 0x74656b, 0x0a0a4d, 0x206120, 0x206269, 0x20646c, + 0x206b6f, 0x206d69, 0x207069, 0x207369, 0x207370, 0x20737a, 0x2e204b, + 0x610a0a, 0x612070, 0x612072, 0x617267, 0x626962, 0x636520, 0x63692e, + 0x637920, 0x637a79, 0x642c20, 0x64616a, 0x646c61, 0x65206f, 0x656461, + 0x656b61, 0x657261, 0x66726f, 0x68616d, 0x692077, 0x692c20, 0x696379, + 0x696520, 0x69656e, 0x697379, 0x69ea20, 0x6b6172, 0x6c6120, 0x6c6e69, + 0x6d6920, 0x6d6965, 0x6e6369, 0x6e6520, 0x6e656b, 0x6e6963, 0x6e6965, + 0x6f727a, 0x6f9c63, 0x707261, 0x72616a, 0x72616d, 0x726f77, 0x7369ea, + 0x737072, 0x73746f, 0x737920, 0x737a65, 0x746f72, 0x75647a, 0x776120, + 0x776961, 0x776965, 0x792063, 0x792070, 0x792073, 0x796672, 0x7a7920, + 0x7a7974, 0xb9206a, 0xb9206b, 0xb92077, 0x0a0a54, 0x0a0a57, 0x0a4d61, + 0x0a4d69, 0x0a5461, 0x0a5720, 0x204269, 0x20437a, 0x204b61, 0x204b6c, + 0x204e61, 0x205072, 0x20526f, 0x206175, 0x206275, 0x206368, 0x20636f, + 0x206379, 0x20637a, 0x206461, 0x206472, 0x20676f, 0x206869, 0x206b61, + 0x206b73, 0x206b75, + ], + }, + { + encoding: 'windows-1250', + language: 'ro', + documents: 4, + trigrams: [ + 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, + 0x206d69, 0x207072, 0x207265, 0x61fe61, 0x626c69, 0x636869, 0x646520, + 0x652070, 0x656361, 0x657269, 0x657363, 0x692e20, 0x69626c, 0x696e65, + 0x696f74, 0x6c6520, 0x6c696f, 0x6f7465, 0x736520, 0x746520, 0x746563, + 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, 0x20616c, 0x206175, 0x206361, + 0x20636c, 0x206375, 0x2063e3, 0x206469, 0x206578, 0x206961, 0x206d65, + 0x206f72, 0x207065, 0x207363, 0x207365, 0x207472, 0x20ba74, 0x2c2069, + 0x426962, 0x506961, 0x612064, 0x616c20, 0x617220, 0x617574, 0x617ae3, + 0x636120, 0x636172, 0x636574, 0x63e320, 0x646573, 0x646967, 0x652063, + 0x652064, 0x652074, 0x6520ee, 0x652e0a, 0x65617a, 0x656e69, 0x656e74, + 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, 0x65ba74, 0x676974, + 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, 0x6963e3, 0x696769, + 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, 0x6c2064, 0x6c6567, + 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, 0x6d696e, 0x6e6365, + 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, 0x7261ba, 0x72652c, + 0x72696c, 0x72756c, 0x736320, 0x736368, 0x737572, 0x742e20, 0x74616c, + 0x746974, 0x747275, 0x74e30a, 0x752065, 0x756d70, 0x757a65, 0x7a6561, + 0xba7465, 0xe30a0a, + ], + }, + { + encoding: 'windows-1251', + language: 'ru', + documents: 4, + trigrams: [ + 0xfef220, 0xe0fef2, 0x20eff0, 0xeff0ee, 0x20e820, 0x20ede0, 0x20efee, + 0xe0ff20, 0xf0eee4, 0xfbe520, 0xfff220, 0x20e2fb, 0x20e3ee, 0x20ece0, + 0x20eee1, 0xe0f2e5, 0xe5eded, 0xe820ef, 0xe82c20, 0xebe820, 0xece0f0, + 0xede020, 0xedfbe5, 0xf0fb20, 0xf220f1, 0xf2e5eb, 0xf2e5f0, 0xf2eee2, + 0xff20ef, 0x0a0acd, 0x20e020, 0x20e1e8, 0x20e2ee, 0x20eaee, 0x20eef1, + 0x20eef2, 0x20efeb, 0x20f0e5, 0x20f1ee, 0x20fff0, 0x2c20e0, 0x2c20e1, + 0xcde020, 0xe00a0a, 0xe020ef, 0xe0e5f2, 0xe0ede8, 0xe0f0ea, 0xe0f1f2, + 0xe1e8e1, 0xe1ebe8, 0xe1f1f3, 0xe2e0fe, 0xe2e5f0, 0xe2fff2, 0xe3eef2, + 0xe42c20, 0xe4e0fe, 0xe4e820, 0xe5ebe5, 0xe5ebe8, 0xe5f0e8, 0xe5f0fb, + 0xe5f1f2, 0xe6e4e0, 0xe820ee, 0xe820f1, 0xe8e1eb, 0xe8eef2, 0xe8f1f2, + 0xe8f2e0, 0xe8ff20, 0xeae00a, 0xeae520, 0xebe8ee, 0xede0ff, 0xede8ea, + 0xede8ff, 0xededff, 0xedeee2, 0xedffff, 0xeee1f1, 0xeee2ff, 0xeeebfc, + 0xeeece0, 0xeeeded, 0xeef0ee, 0xeef1e5, 0xeef1f2, 0xeef1fb, 0xeef2e5, + 0xeef2ee, 0xefe8f1, 0xefeef1, 0xf0eae0, 0xf0ece0, 0xf0eef1, 0xf1e5ed, + 0xf1edfb, 0xf1f2e0, 0xf1f2e5, 0xf1f2ee, 0xf1f3e6, 0xf1fb20, 0xf1ff20, + 0xf220ea, 0xf220ed, 0xf220ef, 0xf2e5ea, 0xf2eef0, 0xf2f0ee, 0xf2f1ff, + 0xf3e6e4, 0xf5eee4, 0xfb20ef, 0xfb2c20, 0xfb2e0a, 0xfbe2e0, 0xfbe920, + 0xff20ff, 0xfff0ec, 0xfffef2, 0xffff20, 0x0a0ac2, 0x0a0ac3, 0x0ac220, + 0x0ac3ee, 0x0acde0, + ], + }, + { + encoding: 'windows-1252', + language: 'da', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x676572, 0x6f6720, 0x64656e, 0x646572, + 0x6c6520, 0x6e6465, 0x737465, 0x74656b, 0x0a0a42, 0x206269, 0x206279, + 0x206465, 0x20686f, 0x2070e5, 0x2c206f, 0x626962, 0x626c69, 0x640a0a, + 0x652062, 0x652066, 0x65640a, 0x656e73, 0x65722c, 0x65722e, 0x65726e, + 0x69626c, 0x696465, 0x696c6c, 0x696f74, 0x6b7374, 0x6c696f, 0x6c6c65, + 0x6e6572, 0x6f7465, 0x70e520, 0x722065, 0x722c20, 0x722e0a, 0x726e65, + 0x736572, 0x746572, 0x766572, 0x79656e, 0xe52064, 0xf86765, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4ce6, 0x206272, 0x206275, + 0x2062f8, 0x206361, 0x20656c, 0x20656e, 0x206669, 0x20666f, 0x206672, + 0x2066f8, 0x206869, 0x20686a, 0x206920, 0x206b6f, 0x206c61, 0x206c69, + 0x206d65, 0x206d75, 0x206f6d, 0x206f73, 0x20706c, 0x20736b, 0x207374, + 0x2073e6, 0x207465, 0x207472, 0x207479, 0x20756e, 0x207665, 0x207669, + 0x2076e5, 0x2076e6, 0x20e562, 0x20e662, 0x2c2063, 0x2c206d, 0x2c2075, + 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, 0x457420, 0x4ce673, + 0x4cf872, 0x4d6f72, 0x61622c, 0x616473, 0x6166e9, 0x616773, 0x616c74, + 0x616e67, 0x617265, 0x61726b, 0x61766c, 0x622c20, 0x626c65, 0x626e65, + 0x6272f8, 0x627573, 0x627965, 0x627967, 0x62f867, 0x636166, 0x642062, + 0x642c20, 0x646167, 0x646520, 0x64652c, 0x64656c, 0x64732e, 0x652068, + 0x65206f, 0x652070, + ], + }, + { + encoding: 'windows-1252', + language: 'de', + documents: 4, + trigrams: [ + 0x656e20, 0x636865, 0x657220, 0x206465, 0x20756e, 0x696520, 0x736368, + 0x756e64, 0x65696e, 0x68656e, 0x696368, 0x696e65, 0x6e2064, 0x6e6420, + 0x6e656e, 0x206469, 0x626572, 0x646572, 0x687265, 0x6c6569, 0x205465, + 0x206569, 0x446965, 0x616368, 0x636874, 0x64656e, 0x646965, 0x652c20, + 0x656974, 0x65722e, 0x65726e, 0x686572, 0x696e20, 0x697461, 0x697465, + 0x6b6c65, 0x6c6572, 0x6e2042, 0x6e2069, 0x6e2e20, 0x6e6520, 0x726520, + 0x726569, 0x742065, 0x746520, 0x74656e, 0x204269, 0x204765, 0x204d65, + 0x205265, 0x205374, 0x206265, 0x2066fc, 0x206968, 0x206b6c, 0x206c65, + 0x207665, 0x20fc62, 0x2e2042, 0x2e2044, 0x426962, 0x45696e, 0x4d656e, + 0x537461, 0x616474, 0x616c65, 0x616d20, 0x617573, 0x626c69, 0x636872, + 0x636b65, 0x642066, 0x65204d, 0x652053, 0x652054, 0x652075, 0x656368, + 0x656c6c, 0x656e2e, 0x657265, 0x657277, 0x657320, 0x66fc72, 0x676974, + 0x686520, 0x68656b, 0x68656c, 0x687465, 0x69626c, 0x69636b, 0x696769, + 0x696872, 0x696f74, 0x6c6520, 0x6c6963, 0x6c696f, 0x6e206c, 0x6e2076, + 0x6e6465, 0x6e7465, 0x6e7477, 0x6f7468, 0x722047, 0x722053, 0x722064, + 0x722e0a, 0x722e20, 0x726e20, 0x736520, 0x73652c, 0x737461, 0x737465, + 0x737563, 0x740a0a, 0x742061, 0x746164, 0x74616c, 0x746572, 0x746865, + 0x747769, 0x756368, 0x757373, 0x766572, 0x776963, 0xe46e64, 0xfc6265, + 0xfc7220, 0x0a0a41, + ], + }, + { + encoding: 'windows-1252', + language: 'en', + documents: 4, + trigrams: [ + 0x20616e, 0x616e64, 0x686520, 0x6e6420, 0x727320, 0x207468, 0x2c2061, + 0x746865, 0x652061, 0x652c20, 0x656164, 0x657273, 0x657320, 0x732061, + 0x206120, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206c69, 0x206e65, + 0x207265, 0x207768, 0x546865, 0x616465, 0x626f75, 0x627261, 0x636974, + 0x642063, 0x64206e, 0x646520, 0x646573, 0x652063, 0x657369, 0x68696c, + 0x696272, 0x696465, 0x696c65, 0x697479, 0x6c6520, 0x6c6962, 0x6c6c20, + 0x6e2074, 0x722064, 0x726172, 0x726561, 0x732062, 0x732072, 0x732073, + 0x732c20, 0x732e0a, 0x776869, 0x790a0a, 0x0a0a41, 0x0a0a46, 0x0a0a52, + 0x0a0a54, 0x0a4120, 0x0a4661, 0x0a5265, 0x0a5468, 0x206162, 0x206170, + 0x206172, 0x206265, 0x20626f, 0x206361, 0x20636c, 0x206469, 0x20646f, + 0x206578, 0x206661, 0x206775, 0x206869, 0x20696e, 0x206974, 0x206a61, + 0x206d61, 0x206d75, 0x206e61, 0x206f70, 0x207072, 0x2070e2, 0x207175, + 0x2072e9, 0x2072f4, 0x207363, 0x207365, 0x20736c, 0x20736d, 0x207465, + 0x207761, 0x20776f, 0x2c2063, 0x2c206a, 0x2c2073, 0x3b2063, 0x412073, + 0x446967, 0x466172, 0x4d6f72, 0x526561, 0x536174, 0x612066, 0x612072, + 0x61626f, 0x616420, 0x61642c, 0x6166e9, 0x616b65, 0x616c20, 0x616c61, + 0x616c6c, 0x616d20, 0x616e20, 0x616e67, 0x617065, 0x617070, 0x617220, + 0x617265, 0x617269, 0x61726b, 0x61726d, 0x617274, 0x617279, 0x617475, + 0x617920, 0x61e761, + ], + }, + { + encoding: 'windows-1252', + language: 'es', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x617320, 0x656c20, 0x656e20, 0x657320, 0x206c61, + 0x206c6f, 0x207920, 0x2e204c, 0x61646f, 0x616e20, 0x6c6f73, 0x6e206c, + 0x6f7265, 0x726573, 0x706172, 0x732063, 0x204c6f, 0x206369, 0x20656c, + 0x207061, 0x4c6f73, 0x612065, 0x612070, 0x617261, 0x626c69, 0x6c206d, + 0x6c6120, 0x6e6120, 0x732061, 0x732e20, 0x746f72, 0x206269, 0x20636c, + 0x20636f, 0x206469, 0x206d65, 0x207265, 0x207375, 0x207665, 0x456c20, + 0x4c6120, 0x612062, 0x626962, 0x636961, 0x646520, 0x64656c, 0x646967, + 0x656361, 0x65732e, 0x69626c, 0x696369, 0x696f74, 0x6c2063, 0x6c696f, + 0x6e2065, 0x6f2073, 0x6f2e20, 0x6f7465, 0x726120, 0x732064, 0x732070, + 0x732076, 0x732e0a, 0x746563, 0x0a0a45, 0x204c61, 0x206361, 0x206c65, + 0x206d61, 0x20706f, 0x207072, 0x207365, 0x207465, 0x2c206c, 0x612063, + 0x612064, 0x61206c, 0x612079, 0x616e61, 0x61726f, 0x61732c, 0x61732e, + 0x627265, 0x627573, 0x636120, 0x636164, 0x636965, 0x636975, 0x636c61, + 0x636f2e, 0x636f6d, 0x646164, 0x64656e, 0x646f20, 0x646f72, 0x656369, + 0x656e61, 0x657175, 0x657263, 0x657274, 0x676974, 0x676f20, 0x696120, + 0x696173, 0x69656c, 0x69656e, 0x696769, 0x697461, 0x697564, 0x6c6172, + 0x6c6173, 0x6c6c65, 0x6d6572, 0x6d7061, 0x6e2070, 0x6e2072, 0x6e6173, + 0x6e6f73, 0x6f2064, 0x6f2079, 0x6f636f, 0x6f6d70, 0x6f732e, 0x706f63, + 0x717565, 0x717569, + ], + }, + { + encoding: 'windows-1252', + language: 'fr', + documents: 4, + trigrams: [ + 0x657320, 0x206c65, 0x656e74, 0x6c6573, 0x6e7420, 0x74206c, 0x732070, + 0x206465, 0x657420, 0x657572, 0x727320, 0x742064, 0x206475, 0x206574, + 0x647520, 0x6c6520, 0x732063, 0x757273, 0x204c65, 0x20636f, 0x207072, + 0x2e204c, 0x4c6573, 0x646573, 0x652063, 0x652070, 0x207061, 0x20706f, + 0x2c206c, 0x626c69, 0x65206d, 0x656e63, 0x65732e, 0x6c6c65, 0x6f7572, + 0x706172, 0x717565, 0x726520, 0x732e20, 0x746573, 0x756520, 0x757220, + 0x206269, 0x206361, 0x20656e, 0x206d61, 0x207361, 0x4c6520, 0x616765, + 0x617265, 0x626962, 0x637465, 0x646520, 0x650a0a, 0x652064, 0x652065, + 0x652073, 0x69626c, 0x69656e, 0x696c6c, 0x696f74, 0x6c6575, 0x6c696f, + 0x6d656e, 0x6e6365, 0x6e6520, 0x6f6e73, 0x6f7468, 0x7072e9, 0x72206c, + 0x726368, 0x72e970, 0x732065, 0x73206c, 0x732073, 0x746575, 0xe97061, + 0xe97269, 0x0a0a4c, 0x20636c, 0x206672, 0x206c61, 0x206e75, 0x207265, + 0x2072e9, 0x20756e, 0x207669, 0x2c2064, 0x4c6120, 0x556e65, 0x612062, + 0x616972, 0x616973, 0x617263, 0x636865, 0x6368e9, 0x636965, 0x636f6e, + 0x652e20, 0x656e20, 0x65732c, 0x676573, 0x686973, 0x68e871, 0x68e920, + 0x69656c, 0x696572, 0x696e20, 0x697175, 0x697265, 0x697374, 0x697420, + 0x6c6120, 0x6c656e, 0x6c6965, 0x6d6172, 0x6d6d65, 0x6de972, 0x6e2070, + 0x6e7320, 0x6e7473, 0x6e756d, 0x6f6d6d, 0x6f7576, 0x706f73, 0x706f75, + 0x70726f, 0x72656e, + ], + }, + { + encoding: 'windows-1252', + language: 'it', + documents: 4, + trigrams: [ + 0x6c6120, 0x207069, 0x656e74, 0x206465, 0x206520, 0x612070, 0x6c6c61, + 0x6e6f20, 0x6f7269, 0x746f72, 0x206369, 0x206c61, 0x612067, 0x612069, + 0x61746f, 0x636974, 0x64656c, 0x657263, 0x692061, 0x692063, 0x692073, + 0x696120, 0x697474, 0x69f920, 0x6c6920, 0x7069f9, 0x726920, 0x7474e0, + 0x206269, 0x206361, 0x20636f, 0x206469, 0x206765, 0x206920, 0x206c65, + 0x206d65, 0x207065, 0x207072, 0x207363, 0x207369, 0x207374, 0x2c2063, + 0x4c6120, 0x612062, 0x612063, 0x612064, 0x61206c, 0x616666, 0x616e6f, + 0x626962, 0x626c69, 0x636166, 0x636f6c, 0x637269, 0x65206c, 0x652073, + 0x652c20, 0x656361, 0x656c20, 0x656c65, 0x656c6c, 0x657363, 0x657474, + 0x6666e8, 0x67696f, 0x676c69, 0x692070, 0x69626c, 0x696f20, 0x696f74, + 0x6c696f, 0x6d656e, 0x6e6120, 0x6e7465, 0x6e7472, 0x6f2064, 0x6f2065, + 0x6f2070, 0x6f2c20, 0x6f6e6f, 0x6f7465, 0x706572, 0x726120, 0x726361, + 0x726520, 0x726961, 0x736372, 0x746120, 0x746563, 0x7a6120, 0x0a0a47, + 0x0a0a49, 0x0a0a4c, 0x0a0a55, 0x0a476c, 0x0a4920, 0x0a4c61, 0x0a556e, + 0x206120, 0x206167, 0x206169, 0x206170, 0x206173, 0x206175, 0x206365, + 0x206368, 0x20666f, 0x206672, 0x206769, 0x20676c, 0x206775, 0x20696c, + 0x20696e, 0x206c69, 0x206d69, 0x206d75, 0x206e65, 0x207061, 0x207361, + 0x207371, 0x207376, 0x207472, 0x207574, 0x207665, 0x2c2066, 0x2c2067, + 0x2c2069, 0x2c206d, + ], + }, + { + encoding: 'windows-1252', + language: 'nl', + documents: 4, + trigrams: [ + 0x656e20, 0x20656e, 0x646520, 0x206465, 0x656572, 0x74656e, 0x766572, + 0x206f70, 0x207374, 0x207465, 0x652062, 0x652065, 0x652073, 0x657273, + 0x6e206f, 0x6e2e0a, 0x736368, 0x742064, 0x746572, 0x206269, 0x206361, + 0x20636f, 0x206565, 0x206765, 0x206865, 0x206c65, 0x206f76, 0x207665, + 0x207765, 0x2c2063, 0x446520, 0x61616b, 0x616173, 0x6166e9, 0x616b74, + 0x626962, 0x626c69, 0x636166, 0x636874, 0x652063, 0x65696e, 0x656b65, + 0x656e2e, 0x656e73, 0x657220, 0x657264, 0x657265, 0x65726b, 0x66e973, + 0x687465, 0x69626c, 0x696465, 0x696e20, 0x696e67, 0x696f74, 0x697320, + 0x6b6520, 0x6b656e, 0x6b7420, 0x6c6520, 0x6c6565, 0x6c6569, 0x6c696f, + 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206d, 0x6e2074, 0x6e2076, 0x6e2077, + 0x6f656b, 0x6f6e74, 0x6f7020, 0x6f7065, 0x6f7468, 0x6f7665, 0x70656e, + 0x72656e, 0x727320, 0x732065, 0x73206f, 0x732c20, 0x73656e, 0x737365, + 0x737461, 0x737465, 0x746164, 0x746865, 0xe97320, 0x0a0a42, 0x0a0a44, + 0x0a0a45, 0x0a0a4c, 0x0a426f, 0x0a4465, 0x0a4565, 0x0a4c65, 0x206170, + 0x20626f, 0x206272, 0x206275, 0x206472, 0x206475, 0x20686f, 0x206875, + 0x206964, 0x20696e, 0x206b61, 0x206b6c, 0x206c61, 0x206d61, 0x206d65, + 0x206d75, 0x206e61, 0x206f6e, 0x20706c, 0x20706f, 0x207363, 0x207477, + 0x207761, 0x207a6f, 0x20e9e9, 0x2c2061, 0x2c206b, 0x426f65, 0x446967, + 0x45656e, 0x4c657a, + ], + }, + { + encoding: 'windows-1252', + language: 'no', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x6f6720, 0x656e65, 0x206c61, 0x652066, + 0x676572, 0x6e6520, 0x6e6572, 0x737465, 0x74656b, 0x0a0a42, 0x206269, + 0x206272, 0x206465, 0x20666f, 0x2070e5, 0x207665, 0x626962, 0x626c69, + 0x640a0a, 0x646572, 0x652062, 0x652074, 0x652c20, 0x65640a, 0x656e73, + 0x657273, 0x657420, 0x672073, 0x676520, 0x67656e, 0x69626c, 0x696765, + 0x696e67, 0x696f74, 0x697465, 0x6b6572, 0x6b7374, 0x6c6167, 0x6c6520, + 0x6c6967, 0x6c696f, 0x6e206f, 0x6e6465, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x72206f, 0x72656e, 0x726765, 0x746520, 0x746572, + 0x746f72, 0x76656e, 0x766572, 0x79656e, 0xe52064, 0xf86b65, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4c65, 0x206275, 0x206279, + 0x2062f8, 0x2064f8, 0x20656c, 0x20656e, 0x206570, 0x206665, 0x206669, + 0x2066f8, 0x206869, 0x20686a, 0x20686f, 0x206920, 0x206b61, 0x206b6f, + 0x206c69, 0x206d65, 0x206d75, 0x206ee6, 0x206f6d, 0x206f73, 0x207365, + 0x20736a, 0x20736b, 0x207374, 0x207465, 0x20746f, 0x207472, 0x207479, + 0x20756e, 0x207669, 0x2076e5, 0x20e570, 0x20e672, 0x2c2065, 0x2c206b, + 0x2c206f, 0x2c2075, 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, + 0x457420, 0x4c6573, 0x4cf872, 0x4d6f72, 0x616665, 0x616720, 0x616765, + 0x616773, 0x616c74, 0x616e67, 0x617020, 0x617265, 0x61726b, 0x61766c, + 0x627279, 0x6272f8, + ], + }, + { + encoding: 'windows-1252', + language: 'pt', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x206520, 0x207072, 0x696120, 0x6e6120, 0x206369, + 0x206f73, 0x612064, 0x612065, 0x61206f, 0x646520, 0x657320, 0x657363, + 0x726573, 0x732061, 0x732070, 0x732e0a, 0x0a0a4f, 0x0a4f73, 0x206269, + 0x206573, 0x206e61, 0x206f20, 0x207065, 0x207665, 0x4f7320, 0x612070, + 0x616465, 0x61646f, 0x616d20, 0x617261, 0x617320, 0x626962, 0x626c69, + 0x636120, 0x636964, 0x646120, 0x646164, 0x646967, 0x646f20, 0x652061, + 0x656d20, 0x657175, 0x666963, 0x69626c, 0x696461, 0x696d65, 0x696f74, + 0x6c696f, 0x6e7175, 0x6f2065, 0x6f7265, 0x6f7465, 0x717565, 0x717569, + 0x72616d, 0x726961, 0x726f73, 0x732063, 0x736372, 0x746563, 0x746f72, + 0x0a0a41, 0x0a0a55, 0x0a4120, 0x0a556d, 0x206120, 0x206162, 0x206163, + 0x206167, 0x20616a, 0x20616c, 0x206173, 0x206361, 0x20636c, 0x2063f3, + 0x206469, 0x20646f, 0x20656e, 0x206571, 0x206672, 0x206775, 0x206869, + 0x206c65, 0x206c69, 0x206d61, 0x206d65, 0x206d6f, 0x206d75, 0x2070e3, + 0x207175, 0x2073e1, 0x207472, 0x20f46e, 0x2c206d, 0x2c206f, 0x2c2071, + 0x2c2076, 0x412062, 0x412063, 0x4d616e, 0x4d6572, 0x4f6669, 0x556d61, + 0x610a0a, 0x612061, 0x612062, 0x612063, 0x61206d, 0x612074, 0x612e0a, + 0x616272, 0x61636f, 0x616461, 0x6166e9, 0x616761, 0x616772, 0x616a75, + 0x616c0a, 0x616c75, 0x616e68, 0x616e71, 0x616e74, 0x61722c, 0x61732e, + 0x61e761, 0x61e7e3, + ], + }, + { + encoding: 'windows-1252', + language: 'sv', + documents: 4, + trigrams: [ + 0x657220, 0x617220, 0x206f63, 0x636820, 0x656e20, 0x6f6368, 0x737461, + 0x64656e, 0x657420, 0x6b6172, 0x726e61, 0x736b61, 0x74206c, 0x746164, + 0x766572, 0x206269, 0x206465, 0x2066f6, 0x206c69, 0x2070e5, 0x20736b, + 0x207479, 0x207665, 0x612062, 0x612074, 0x612e0a, 0x61640a, 0x616465, + 0x617265, 0x626962, 0x626c69, 0x640a0a, 0x65726e, 0x66f672, 0x676120, + 0x69626c, 0x696761, 0x696f74, 0x6b6170, 0x6b6572, 0x6b6e61, 0x6c6967, + 0x6c696f, 0x6e612e, 0x6e6172, 0x6e736b, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x722066, 0x72206f, 0x736172, 0x73f66b, 0x746120, + 0x74656b, 0x746f72, 0xe52064, 0x0a0a42, 0x0a0a45, 0x0a0a4c, 0x0a0a53, + 0x0a42f6, 0x0a4574, 0x0a4ce4, 0x0a5374, 0x206265, 0x206272, 0x206275, + 0x2062f6, 0x20656c, 0x20656e, 0x2066e4, 0x206772, 0x206869, 0x20686a, + 0x20686f, 0x206920, 0x206b61, 0x206b6f, 0x206c61, 0x206ce5, 0x206d65, + 0x206d75, 0x206de4, 0x206f6d, 0x206f73, 0x207374, 0x2073e4, 0x2073f6, + 0x207465, 0x20746f, 0x207661, 0x2076e4, 0x20e470, 0x20f670, 0x2c2067, + 0x2c206b, 0x2c206f, 0x2c20e4, 0x42f66e, 0x446574, 0x446967, 0x457474, + 0x4ce473, 0x4cf672, 0x4d6f72, 0x537461, 0x61206f, 0x6120f6, 0x6166e9, + 0x616720, 0x616773, 0x616b6e, 0x616c20, 0x616d74, 0x616e20, 0x616e73, + 0x617020, 0x617061, 0x617269, 0x61726b, 0x61726e, 0x626573, 0x6272f6, + 0x627573, 0x62f663, + ], + }, + { + encoding: 'windows-1253', + language: 'el', + documents: 4, + trigrams: [ + 0x20eae1, 0xe1e920, 0xeae1e9, 0x20e2e9, 0xe120ea, 0xe12e0a, 0xe2e9e2, + 0xe5e920, 0xe9e2eb, 0xeff5ed, 0xf4e120, 0xf5ed20, 0x0a0acf, 0x0acfe9, + 0x20e1e3, 0x20e1ed, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xcfe920, + 0xdfe12e, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, 0xe5f220, 0xe70a0a, 0xe720e2, + 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9efe8, 0xe9f3f4, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xf0fceb, 0xf1e9ef, 0xf220ea, 0xf3f4de, + 0xf3f4e7, 0xf4e5f2, 0xf4e7ed, 0xfcebe7, 0x0a0ac7, 0x0a0acc, 0x0ac720, + 0x0acce9, 0x20dced, 0x20e1f1, 0x20e2ef, 0x20e3f1, 0x20e5eb, 0x20e5f0, + 0x20e5f1, 0x20e5f4, 0x20e720, 0x20e9f3, 0x20eae5, 0x20eafe, 0x20ebe5, + 0x20ecdd, 0x20ece1, 0x20ece9, 0x20ecef, 0x20ecf5, 0x20eef5, 0x20efe4, + 0x20efe9, 0x20efec, 0x20f0e5, 0x20f0eb, 0x20f0ef, 0x20f6f1, 0x20f8dc, + 0x20f8f9, 0x2c20e5, 0x2c20ec, 0x2c20f4, 0x2c20f6, 0xb9f3f5, 0xc720f0, + 0xcce9e1, 0xd0f1f9, 0xd3e1e2, 0xd8e7f6, 0xdc0a0a, 0xdc20e1, 0xdc20ea, + 0xdc20f3, 0xdc20f4, 0xdc2c20, 0xdce4e1, 0xdce6e5, 0xdcede8, 0xdcf1e9, + 0xdcf4e9, 0xdcf6e5, 0xdcf7ed, 0xdd20e1, 0xdde3f7, 0xddebe9, 0xddedef, + 0xddf22e, 0xde20ef, 0xdeeae7, 0xdeece7, 0xdef1e9, 0xdf20f3, 0xdf2c20, + 0xdfe120, 0xdfe1f2, 0xdfe3ef, 0xdfece5, 0xdfeff5, 0xe120e5, 0xe120e9, + 0xe120eb, 0xe120ec, 0xe1e2e2, 0xe1e3ed, 0xe1e3ef, 0xe1e3f1, 0xe1e8e1, + 0xe1e8e7, 0xe1eafc, + ], + }, + { + encoding: 'windows-1254', + language: 'tr', + documents: 4, + trigrams: [ + 0x6c6572, 0x6c6172, 0x207665, 0x722e0a, 0x766520, 0x2061e7, 0x207961, + 0x617220, 0x61e7fd, 0x657220, 0x697220, 0x722076, 0x206269, 0x20656b, + 0x206b61, 0x206bfc, 0x207361, 0x207461, 0x2c206b, 0x616261, 0x616c20, + 0x616c61, 0x616e65, 0x61722e, 0x6172fd, 0x617a61, 0x62616c, 0x650a0a, + 0x652062, 0x656869, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697461, + 0x6bfc74, 0x706861, 0x722079, 0x722c20, 0x726920, 0x726c61, 0x746172, + 0x74fc70, 0x7a6172, 0x7a6520, 0xde6568, 0xe7fd6b, 0xfc7068, 0xfc74fc, + 0xfd6b20, 0x0a0a4b, 0x0a0a4f, 0x0a0ac7, 0x0a0ade, 0x0a4bfc, 0x0a4f6b, + 0x0ac769, 0x0ade65, 0x206172, 0x206174, 0x206261, 0x206265, 0x206465, + 0x20656c, 0x206861, 0x20696c, 0x20696e, 0x206b69, 0x206b6f, 0x206d65, + 0x206dfc, 0x206f6c, 0x206f74, 0x207061, 0x207065, 0x207265, 0x207579, + 0x20f6f0, 0x2c2065, 0x2c2070, 0x43756d, 0x44696a, 0x4bfce7, 0x4f6b75, + 0x536573, 0x612074, 0x612075, 0x612c20, 0x616665, 0x61680a, 0x616cfd, + 0x616d61, 0x616e64, 0x616e6c, 0x616efd, 0x61706c, 0x61722c, 0x617261, + 0x617264, 0x617269, 0x61726b, 0x617274, 0x617461, 0x6174f6, 0x617661, + 0x617a65, 0x617afd, 0x61fee7, 0x626168, 0x62656b, 0x626572, 0x62696c, + 0x626972, 0x62fc73, 0x636920, 0x63696c, 0x63fd20, 0x646120, 0x64616e, + 0x646520, 0x64656e, 0x647520, 0x64fd6d, 0x652061, 0x652065, 0x652069, + 0x652072, 0x652073, + ], + }, + { + encoding: 'windows-1255', + language: 'he', + documents: 4, + trigrams: [ + 0xe9ed20, 0xe5f8e9, 0xed20ec, 0xed20ee, 0xf1f4f8, 0xf8e9ed, 0x20e4f7, + 0x20ece0, 0x20eee7, 0x2c20e1, 0xe0e9ed, 0xe4f1f4, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed2e, 0xed2e0a, 0xef20ee, 0xf0e420, 0xf2e9f8, + 0xf4f8e9, 0xf7e5f8, 0xf820ee, 0xf8e5fa, 0xf8fa20, 0xf9e9ed, 0xfa0a0a, + 0xfa20ec, 0x0a0ae4, 0x0a0ae7, 0x0a0af6, 0x0a0af7, 0x0ae4f2, 0x0ae7f7, + 0x0af6e5, 0x0af7e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, + 0x20e1fa, 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e8f8, 0x20ece1, + 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, 0x20eeeb, 0x20eef7, + 0x20eefa, 0x20f0f4, 0x20f1f4, 0x20f2e5, 0x20f4e9, 0x20f7e8, 0x20f9e1, + 0x20fae9, 0x2c20e3, 0x2c20f4, 0xe0e5e8, 0xe0e5ef, 0xe0e5f8, 0xe0e82c, + 0xe0f0f9, 0xe0f9e5, 0xe0fa20, 0xe120fa, 0xe1e5e3, 0xe1e5f1, 0xe1e5f7, + 0xe1e7e5, 0xe1e9f0, 0xe1ebe9, 0xe1f2e9, 0xe1f8e5, 0xe1f92c, 0xe1fa0a, + 0xe1fae9, 0xe2e1e9, 0xe2e9e8, 0xe320e5, 0xe3e1f9, 0xe3e9e2, 0xe3e9ed, + 0xe3f0e4, 0xe3f220, 0xe3f720, 0xe3f8e9, 0xe40a0a, 0xe420e1, 0xe420e3, + 0xe420e4, 0xe420e5, 0xe420f0, 0xe42e0a, 0xe4e9f1, 0xe4f2e9, 0xe4f2ee, + 0xe4f7e5, 0xe4f7f4, 0xe4f8e0, 0xe4f9f7, 0xe5e0f0, 0xe5e1e5, 0xe5e2e1, + 0xe5e320, 0xe5e3f7, 0xe5e4f1, 0xe5e5fa, 0xe5e6e9, 0xe5e6f8, 0xe5e8e5, + 0xe5ebe5, 0xe5ebf8, + ], + }, + { + encoding: 'windows-1256', + language: 'ar', + documents: 4, + trigrams: [ + 0x20c7e1, 0xc7e1e3, 0x20e6c7, 0x20e6ed, 0xdfcac8, 0xe420c7, 0xe6c7e1, + 0xe6e420, 0x0a0aed, 0xc90a0a, 0xd120c7, 0xc7ca20, 0xc7ccda, 0xc7d120, + 0xc7e1c3, 0xc7e1da, 0xc7e1e6, 0xc820c7, 0xc8c920, 0xc920c7, 0xc920e6, + 0xc92e0a, 0xca20e6, 0xcac820, 0xcac8c9, 0xcb20c7, 0xccda20, 0xcd20c7, + 0xcfede4, 0xd1c7cc, 0xd5ddc7, 0xda20c7, 0xe1e3cf, 0xe1e3df, 0xe3cfed, + 0xe3dfca, 0xe4c7d3, 0xe6d5dd, 0xede4c9, 0x0a0aca, 0x0acad3, 0x0aedc8, + 0x0aedd5, 0x0aedda, 0x20c3c8, 0x20c3e3, 0x20c7ce, 0x20c8e5, 0x20c8ed, + 0x20cfe1, 0x20d1de, 0x20d5db, 0x20dae4, 0x20ddd1, 0x20dded, 0x20dfca, + 0x20e1e1, 0x20e6ca, 0x20e6d5, 0x20ede4, 0xc120da, 0xc120e6, 0xc3c8e6, + 0xc3cdc7, 0xc3cec8, 0xc3e3ed, 0xc4e6e4, 0xc6c90a, 0xc720c8, 0xc720e1, + 0xc720e6, 0xc720ed, 0xc72e0a, 0xc7c120, 0xc7c820, 0xc7c8e5, 0xc7cd20, + 0xc7ceca, 0xc7cfc6, 0xc7cfe1, 0xc7cfed, 0xc7d12e, 0xc7d1da, 0xc7d1ed, + 0xc7d320, 0xc7d3c8, 0xc7d6cd, 0xc7dacf, 0xc7dde1, 0xc7dfe5, 0xc7e1ca, + 0xc7e1cd, 0xc7e1ce, 0xc7e1d2, 0xc7e1d3, 0xc7e1d4, 0xc7e1d8, 0xc7e1dd, + 0xc7e1de, 0xc7e1e4, 0xc7e1e5, 0xc7e5ed, 0xc820dd, 0xc8c7cd, 0xc8c7cf, + 0xc8c7d1, 0xc8c92e, 0xc8ca0a, 0xc8cdcb, 0xc8d1e3, 0xc8d220, 0xc8e5c7, + 0xc8e5cf, 0xc8e6c7, 0xc8ede4, 0xc920c8, 0xc920d1, 0xca0a0a, 0xcac7d1, + 0xcac8c7, 0xcacd20, 0xcacddd, 0xcad3ca, 0xcad9d1, 0xcaddca, 0xcaedc7, + 0xcaedde, 0xcce6e4, + ], + }, + { + encoding: 'windows-1257', + language: 'et', + documents: 4, + trigrams: [ + 0x616420, 0x6a6120, 0x206a61, 0x206b6f, 0x616220, 0x766164, 0x207261, + 0x61616d, 0x616d61, 0x617475, 0x617661, 0x6d6174, 0x6e6520, 0x726161, + 0x746162, 0x0a0a4c, 0x206b69, 0x206d65, 0x2076e4, 0x2c206b, 0x642061, + 0x64206b, 0x64206f, 0x647573, 0x652074, 0x656420, 0x657369, 0x692e0a, + 0x696420, 0x696b75, 0x696d65, 0x696e6e, 0x69726a, 0x697461, 0x6b6972, + 0x6b6f67, 0x6b6f6e, 0x6b6f6f, 0x6b7564, 0x6c6173, 0x6d6573, 0x6f6775, + 0x736564, 0x73656c, 0x73692e, 0x74756b, 0x756420, 0x756b6f, 0x757573, + 0x0a0a54, 0x0a0a56, 0x0a4c69, 0x0a4c75, 0x0a5461, 0x0a56e4, 0x206165, + 0x206169, 0x20616a, 0x206176, 0x206275, 0x206573, 0x20696e, 0x206a75, + 0x206c65, 0x206c69, 0x206d75, 0x206dfc, 0x206e69, 0x206f6f, 0x206f74, + 0x207365, 0x207465, 0x207475, 0x2074f6, 0x20756b, 0x20e472, 0x20f570, + 0x20f575, 0x2c206a, 0x2c20f5, 0x2d206a, 0x446967, 0x486f6d, 0x4c6175, + 0x4c696e, 0x4c7567, 0x54616c, 0x566169, 0x56e469, 0x610a0a, 0x612061, + 0x612069, 0x61206b, 0x61206d, 0x612074, 0x612c20, 0x61616c, 0x616475, + 0x616567, 0x616876, 0x61696b, 0x616974, 0x616a61, 0x616b75, 0x616c6e, + 0x616c6f, 0x616c75, 0x616e65, 0x61726f, 0x61730a, 0x617365, 0x617369, + 0x617570, 0x622061, 0x62206b, 0x62206d, 0x622073, 0x6220f5, 0x62610a, + 0x62612c, 0x627573, 0x642065, 0x64206a, 0x64206d, 0x64206e, 0x642075, + 0x642076, 0x646920, + ], + }, + { + encoding: 'windows-1257', + language: 'lt', + documents: 4, + trigrams: [ + 0x616920, 0x6b6169, 0x206972, 0x207265, 0x612070, 0x616974, 0x646120, + 0x656e69, 0x696169, 0x696e69, 0x697220, 0x6c696f, 0x6e696e, 0x726961, + 0x732e0a, 0x757320, 0x206461, 0x207072, 0x617573, 0x626c69, 0x656461, + 0x692072, 0x696120, 0x69626c, 0x696573, 0x696f74, 0x6a6169, 0x6b696e, + 0x6d656e, 0x6eeb73, 0x6f7465, 0x732061, 0x732c20, 0x732e20, 0x74656b, + 0xeb7320, 0x206169, 0x206170, 0x206269, 0x206475, 0x206b6f, 0x206d65, + 0x206d6f, 0x206f20, 0x207061, 0x20736b, 0x20f076, 0x2c206d, 0x2c206f, + 0x536b61, 0x612064, 0x61206b, 0x61692e, 0x617069, 0x617320, 0x617369, + 0x626962, 0x657269, 0x657374, 0x677573, 0x67f820, 0x692074, 0x692e20, + 0x696e6b, 0x696eeb, 0x696f20, 0x69746d, 0x697573, 0x6b6f20, 0x6d6f6b, + 0x6d756f, 0x6e6461, 0x6e6961, 0x6e6b61, 0x6f20fe, 0x6f6275, 0x6f64e0, + 0x6f6a61, 0x6f6e6f, 0x6f7269, 0x6f7320, 0x707261, 0x72206b, 0x7261f0, + 0x72656e, 0x730a0a, 0x732064, 0x732069, 0x73206d, 0x736920, 0x736b61, + 0x73746f, 0x746173, 0x746d65, 0x746f6a, 0x746f72, 0x757267, 0x757269, + 0x75732c, 0x75732e, 0x766965, 0x796c69, 0xe02c20, 0xeb6a61, 0xf07669, + 0x0a0a4d, 0x0a0a4e, 0x0a0a52, 0x0a0a54, 0x0a4d69, 0x0a4e65, 0x0a5261, + 0x0a5475, 0x204269, 0x204469, 0x204b61, 0x205069, 0x20536b, 0x205669, + 0x206174, 0x206175, 0x206275, 0x206469, 0x206965, 0x206973, 0x206b65, + 0x206b6e, 0x206b75, + ], + }, + { + encoding: 'windows-1257', + language: 'lv', + documents: 4, + trigrams: [ + 0x20756e, 0x617320, 0x756e20, 0x207069, 0x61732e, 0x757320, 0x20697a, + 0x206c61, 0x61206b, 0x626c69, 0x646120, 0x692073, 0x692e20, 0x69626c, + 0x69656d, 0x696f74, 0x6a6920, 0x6b7374, 0x6c696f, 0x6c7573, 0x732c20, + 0x732e0a, 0x732e20, 0x206269, 0x206365, 0x206461, 0x206761, 0x206b6f, + 0x206d61, 0x206d65, 0x2070e2, 0x207265, 0x20736b, 0x207a69, 0x610a0a, + 0x616b73, 0x617262, 0x61756b, 0x626962, 0x627573, 0x6365ef, 0x646172, + 0x64752c, 0x656d2e, 0x676974, 0x677573, 0x692070, 0x692c20, 0x69656e, + 0x696769, 0x696c73, 0x697267, 0x6974e2, 0x697a76, 0x6a756d, 0x6b6920, + 0x6be272, 0x6c6173, 0x6c73e7, 0x6ce76e, 0x6d6172, 0x6d6f73, 0x6e2063, + 0x6e2073, 0x6e6965, 0x6eee63, 0x6f2069, 0x6f74e7, 0x706965, 0x70696c, + 0x70e272, 0x72616b, 0x726775, 0x726920, 0x727669, 0x732061, 0x73206c, + 0x73206d, 0x732070, 0x732075, 0x736920, 0x737420, 0x737461, 0x737475, + 0x73e774, 0x746120, 0x747573, 0x74e26a, 0x74e26c, 0x74e76b, 0x752c20, + 0x75732c, 0x757369, 0x76e76c, 0x7a76e7, 0xe26a69, 0xe76e69, 0xe774e2, + 0xee6361, 0x0a0a4e, 0x0a0a50, 0x0a0a52, 0x0a0a54, 0x0a4e65, 0x0a5069, + 0x0a5261, 0x0a5469, 0x204269, 0x204469, 0x204b61, 0x204c61, 0x205069, + 0x205669, 0x205a65, 0x206170, 0x206172, 0x206174, 0x206175, 0x206265, + 0x206369, 0x206465, 0x206469, 0x206475, 0x2064e2, 0x206772, 0x206a75, + 0x206b6c, 0x206b72, + ], + }, + { + encoding: 'windows-1258', + language: 'vi', + documents: 4, + trigrams: [ + 0x6e6720, 0x207468, 0x206368, 0x207068, 0x2076e0, 0x672074, 0x6e6820, + 0x76e020, 0x2064e2, 0x207469, 0x207472, 0x686ff2, 0x6e2062, 0x6e2074, + 0x7068f4, 0x7468fd, 0xf27420, 0x206261, 0x2062e1, 0x206769, 0x20686f, + 0x206c69, 0x206d61, 0x206e68, 0x207175, 0x2073e1, 0x207461, 0x2074e0, + 0x207669, 0x61d220, 0x6261d2, 0x62e16e, 0x636820, 0x672064, 0x6720f0, + 0x682070, 0x68e06e, 0x68f4ec, 0x68fd20, 0x68fdec, 0x69eaf2, 0x6f6e67, + 0x6ff263, 0x746875, 0x7669ea, 0xd22074, 0xd26e67, 0xe0206d, 0xe06e67, + 0xe06e68, 0xe16e20, 0xe2f274, 0xec0a0a, 0xec7420, 0xf26320, 0xf26920, + 0xf26e20, 0xf46e67, 0xf4ec0a, 0xfdf5cc, 0x0a0a4d, 0x0a0a4e, 0x0a0a54, + 0x0a0ad0, 0x0a4df4, 0x0a4ef4, 0x0a5468, 0x0ad0f4, 0x206275, 0x20636f, + 0x2063e0, 0x2063fd, 0x2068fd, 0x206b68, 0x206b69, 0x206b79, 0x206ce0, + 0x206d69, 0x206d6f, 0x206de2, 0x206df4, 0x206df5, 0x206e67, 0x206f6e, + 0x207261, 0x20726f, 0x2072e0, 0x207369, 0x2073f4, 0x2073fd, 0x2074e1, + 0x207865, 0x2079ea, 0x20f0e2, 0x20f0f4, 0x20f0fa, 0x2c2063, 0x2c206b, + 0x2c2070, 0x2c2071, 0x2c2074, 0x4275f4, 0x4368f5, 0x4df4f2, 0x4ef46e, + 0x5468e0, 0x5468fd, 0x58fdf5, 0x612068, 0x61206d, 0x612076, 0x616920, + 0x61d26e, 0x61d26f, 0x61d279, 0x61de20, 0x61de69, 0x61f269, 0x627579, + 0x632064, 0x632067, 0x632073, 0x632c20, 0x632e0a, 0x63686f, 0x636875, + 0x6368e2, 0x6368f5, + ], + }, + { + encoding: 'windows-874', + language: 'th', + documents: 4, + trigrams: [ + 0xe1c5d0, 0xb7d5e8, 0xc1d7cd, 0x20e1c5, 0xb5c3ec, 0xbbc3d0, 0xbcd9e9, + 0xc8d2ca, 0xca20e1, 0xcab5c3, 0xcad2c3, 0xcbe9cd, 0xd1a7ca, 0xd2cab5, + 0xd5c2b9, 0xd7cda7, 0xe0c1d7, 0xe9cda7, 0x0a0aaa, 0x0a0ab7, 0x0a0abc, + 0x0a0ae0, 0x0aaad2, 0x0ab7d5, 0x0abcd9, 0x0ae0c1, 0x20aad5, 0x20b5c3, + 0x20b5d7, 0x20bac3, 0x20bcc5, 0x20c3e9, 0x20e620, 0xa1a2e9, 0xa1a4d1, + 0xa1a8d1, 0xa1c9ec, 0xa1d2e1, 0xa1e0c3, 0xa2b9c1, 0xa2d2c2, 0xa2d5c2, + 0xa2e9cd, 0xa4b9c3, 0xa4d1a1, 0xa4d3cd, 0xa4d6a1, 0xa4d9e8, 0xa4e8cd, + 0xa4e9b9, 0xa70a0a, 0xa7a4e8, 0xa7b7d3, 0xa7ba0a, 0xa7cab4, 0xa7cac1, + 0xa7cad7, 0xa7d2b9, 0xa7d5c2, 0xa7e3b9, 0xa8b90a, 0xa8c3cb, 0xa8d1b4, + 0xa8d4b7, 0xaad1b4, 0xaad2c7, 0xaad5ca, 0xaae8c7, 0xaae9d2, 0xb1ec20, + 0xb3b1ec, 0xb3d2c3, 0xb420bc, 0xb4b7d3, 0xb4bbc3, 0xb4c2ca, 0xb4c7d1, + 0xb4d4a8, 0xb4e0a7, 0xb4e0a8, 0xb5c3c7, 0xb5c5d2, 0xb5d4c8, 0xb5d7e8, + 0xb5d9e1, 0xb6e2b4, 0xb7c2d2, 0xb7d1c5, 0xb7d3a4, 0xb7d3a7, 0xb7d5c1, + 0xb8c0d1, 0xb8d4ba, 0xb920c3, 0xb9a1d2, 0xb9a4d3, 0xb9a4e9, 0xb9b4d4, + 0xb9b7d5, 0xb9c1bb, 0xb9c3cd, 0xb9c5d2, 0xb9cbd2, 0xb9d1a1, 0xb9d1a7, + 0xb9d2a2, 0xb9e0c1, 0xb9e0c5, 0xb9e0ca, 0xb9e9d3, 0xba0a0a, 0xbac3c3, + 0xbacaa7, 0xbad2c2, 0xbbd1a7, 0xbbd4b4, 0xbcc5e4, 0xbcd6e9, 0xbed4b8, + 0xbed4be, 0xbfe0bb, 0xc0d1b3, 0xc1bbd1, 0xc1d8b4, 0xc1d9c5, 0xc1e0c5, + 0xc1e920, 0xc220e6, + ], + }, { encoding: 'x-mac-cyrillic', language: 'ru', diff --git a/corpus/generated/windows-1250/cs/test/community-garden.bin b/corpus/generated/windows-1250/cs/test/community-garden.bin new file mode 100644 index 0000000..61bfe36 --- /dev/null +++ b/corpus/generated/windows-1250/cs/test/community-garden.bin @@ -0,0 +1,3 @@ +Sousedsk zahrada + +Soused promnili przdn dvr v zelenou zahradu, dti namalovaly barevn znaky a rodiny zalvaj rostliny. diff --git a/corpus/generated/windows-1250/cs/train/city-morning.bin b/corpus/generated/windows-1250/cs/train/city-morning.bin new file mode 100644 index 0000000..6efcbfe --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/city-morning.bin @@ -0,0 +1,3 @@ +Rno ve mst + +Msto se pomalu probouz, kavrny otevraj dvee a lid ekaj na prvn autobusy. diff --git a/corpus/generated/windows-1250/cs/train/library.bin b/corpus/generated/windows-1250/cs/train/library.bin new file mode 100644 index 0000000..2d9442e --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/library.bin @@ -0,0 +1,3 @@ +Tich knihovna + +teni hledaj knihy o historii a vd, zatmco knihovnice rad km. diff --git a/corpus/generated/windows-1250/cs/train/market.bin b/corpus/generated/windows-1250/cs/train/market.bin new file mode 100644 index 0000000..f274a31 --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/market.bin @@ -0,0 +1,3 @@ +Sobotn trh + +Farmi prodvaj erstv chlb, med, ovoce a voav byliny. diff --git a/corpus/generated/windows-1250/cs/train/workshop.bin b/corpus/generated/windows-1250/cs/train/workshop.bin new file mode 100644 index 0000000..78a9f82 --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/workshop.bin @@ -0,0 +1,3 @@ +Digitln dlna + +Mal tm pipravuje prvodce muzeem, kontroluje kd a pe jasn popisy. diff --git a/corpus/generated/windows-1250/cs/validation/river-trip.bin b/corpus/generated/windows-1250/cs/validation/river-trip.bin new file mode 100644 index 0000000..dc3d158 --- /dev/null +++ b/corpus/generated/windows-1250/cs/validation/river-trip.bin @@ -0,0 +1,3 @@ +Vlet k ece + +Ptel vzali mapu a tepl aj, proli lesem a odpovali u eky. diff --git a/corpus/generated/windows-1250/hu/test/community-garden.bin b/corpus/generated/windows-1250/hu/test/community-garden.bin new file mode 100644 index 0000000..251d1d3 --- /dev/null +++ b/corpus/generated/windows-1250/hu/test/community-garden.bin @@ -0,0 +1,3 @@ +Kzssgi kert + +A szomszdok zld kertt alaktottk az res udvart, a gyerekek tblkat festettek, a csaldok pedig ntzik a nvnyeket. diff --git a/corpus/generated/windows-1250/hu/train/city-morning.bin b/corpus/generated/windows-1250/hu/train/city-morning.bin new file mode 100644 index 0000000..74c3adc --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/city-morning.bin @@ -0,0 +1,3 @@ +Reggel a vrosban + +A vros lassan bred, a kvzk kinyitnak, az emberek pedig buszra vrnak. diff --git a/corpus/generated/windows-1250/hu/train/library.bin b/corpus/generated/windows-1250/hu/train/library.bin new file mode 100644 index 0000000..4bbeea1 --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/library.bin @@ -0,0 +1,3 @@ +A csendes knyvtr + +Az olvask trtnelmi s tudomnyos knyveket keresnek, a knyvtros segt a dikoknak. diff --git a/corpus/generated/windows-1250/hu/train/market.bin b/corpus/generated/windows-1250/hu/train/market.bin new file mode 100644 index 0000000..b95088b --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/market.bin @@ -0,0 +1,3 @@ +Szombati piac + +A gazdk friss kenyeret, almt, sajtot s mzet rulnak a tren. diff --git a/corpus/generated/windows-1250/hu/train/workshop.bin b/corpus/generated/windows-1250/hu/train/workshop.bin new file mode 100644 index 0000000..819901d --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/workshop.bin @@ -0,0 +1,3 @@ +Digitlis mhely + +Egy kis csapat mzeumi tmutatt kszt, kdot ellenriz s vilgos lersokat r. diff --git a/corpus/generated/windows-1250/hu/validation/river-trip.bin b/corpus/generated/windows-1250/hu/validation/river-trip.bin new file mode 100644 index 0000000..f807c95 --- /dev/null +++ b/corpus/generated/windows-1250/hu/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kirnduls a folyhoz + +A bartok trkpet s meleg tet vittek, majd megpihentek a foly partjn. diff --git a/corpus/generated/windows-1250/pl/test/community-garden.bin b/corpus/generated/windows-1250/pl/test/community-garden.bin new file mode 100644 index 0000000..0e547a6 --- /dev/null +++ b/corpus/generated/windows-1250/pl/test/community-garden.bin @@ -0,0 +1,3 @@ +Ogrd ssiedzki + +Mieszkacy zmienili pusty dziedziniec w zielony ogrd z kwiatami, warzywami i modymi drzewami. Dzieci zrobiy kolorowe tabliczki, a rodziny co tydzie podlewaj roliny. diff --git a/corpus/generated/windows-1250/pl/train/city-morning.bin b/corpus/generated/windows-1250/pl/train/city-morning.bin new file mode 100644 index 0000000..9ff76bb --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/city-morning.bin @@ -0,0 +1,3 @@ +Poranek w miecie + +Miasto budzi si powoli. Kawiarnie otwieraj drzwi, autobusy rozpoczynaj pierwsze kursy, a ludzie czytaj wiadomoci. Nad dachami robi si coraz janiej. diff --git a/corpus/generated/windows-1250/pl/train/library.bin b/corpus/generated/windows-1250/pl/train/library.bin new file mode 100644 index 0000000..a649424 --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/library.bin @@ -0,0 +1,3 @@ +Cicha biblioteka + +W osiedlowej bibliotece sycha szelest kartek. Czytelnicy szukaj ksiek o historii, nauce i podrach. Bibliotekarka przygotowuje wystaw dla uczniw. diff --git a/corpus/generated/windows-1250/pl/train/market.bin b/corpus/generated/windows-1250/pl/train/market.bin new file mode 100644 index 0000000..1171993 --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/market.bin @@ -0,0 +1,3 @@ +Sobotni targ + +Targ wypenia rynek kolorami i zapachami. Rolnicy sprzedaj jabka, mid, ser i wiey chleb. Klienci wybieraj warzywa i wymieniaj rodzinne przepisy. diff --git a/corpus/generated/windows-1250/pl/train/workshop.bin b/corpus/generated/windows-1250/pl/train/workshop.bin new file mode 100644 index 0000000..c1eb29a --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/workshop.bin @@ -0,0 +1,3 @@ +Cyfrowa pracownia + +May zesp tworzy cyfrowy przewodnik po muzeum. Projektanci ukadaj menu, programici sprawdzaj kod, a redaktorzy pisz jasne opisy dla goci. diff --git a/corpus/generated/windows-1250/pl/validation/river-trip.bin b/corpus/generated/windows-1250/pl/validation/river-trip.bin new file mode 100644 index 0000000..e817765 --- /dev/null +++ b/corpus/generated/windows-1250/pl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Spacer nad rzek + +W niedziel przyjaciele poszli nad spokojn rzek za miastem. Zabrali map, ciep herbat i aparat. Po dugim spacerze odpoczywali na brzegu. diff --git a/corpus/generated/windows-1250/ro/test/community-garden.bin b/corpus/generated/windows-1250/ro/test/community-garden.bin new file mode 100644 index 0000000..54f8405 --- /dev/null +++ b/corpus/generated/windows-1250/ro/test/community-garden.bin @@ -0,0 +1,3 @@ +Grdina comunitii + +Vecinii au transformat o curte goal ntr-o grdin cu flori, legume i pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se ntlnesc sptmnal pentru a uda plantele. diff --git a/corpus/generated/windows-1250/ro/train/city-morning.bin b/corpus/generated/windows-1250/ro/train/city-morning.bin new file mode 100644 index 0000000..08d7469 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/city-morning.bin @@ -0,0 +1,3 @@ +Dimineaa n ora + +Oraul se trezete ncet. Cafenelele deschid uile, autobuzele pornesc pe traseu, iar oamenii citesc tirile. Cerul devine mai luminos deasupra cldirilor. diff --git a/corpus/generated/windows-1250/ro/train/library.bin b/corpus/generated/windows-1250/ro/train/library.bin new file mode 100644 index 0000000..1afc415 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/library.bin @@ -0,0 +1,3 @@ +Biblioteca linitit + +n biblioteca de cartier se aud pagini ntoarse ncet. Cititorii caut cri despre istorie, tiin i cltorii. Bibliotecara pregtete o expoziie pentru elevi. diff --git a/corpus/generated/windows-1250/ro/train/market.bin b/corpus/generated/windows-1250/ro/train/market.bin new file mode 100644 index 0000000..15a0338 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/market.bin @@ -0,0 +1,3 @@ +Piaa de smbt + +Piaa umple centrul cu mirosuri i culori. Fermierii aduc mere, miere, brnz i pine proaspt. Cumprtorii aleg legume i schimb reete de familie. diff --git a/corpus/generated/windows-1250/ro/train/workshop.bin b/corpus/generated/windows-1250/ro/train/workshop.bin new file mode 100644 index 0000000..132f010 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/workshop.bin @@ -0,0 +1,3 @@ +Atelier digital + +O echip mic realizeaz ghidul digital al muzeului. Designerii organizeaz meniul, programatorii verific sursa, iar redactorii scriu explicaii clare. diff --git a/corpus/generated/windows-1250/ro/validation/river-trip.bin b/corpus/generated/windows-1250/ro/validation/river-trip.bin new file mode 100644 index 0000000..c125be0 --- /dev/null +++ b/corpus/generated/windows-1250/ro/validation/river-trip.bin @@ -0,0 +1,3 @@ +Plimbare lng ru + +Duminic, prietenii au mers spre un ru linitit din afara oraului. Au luat o hart, ceai cald i un aparat foto. Dup plimbare s-au odihnit pe mal. diff --git a/corpus/generated/windows-1251/ru/test/community-garden.bin b/corpus/generated/windows-1251/ru/test/community-garden.bin new file mode 100644 index 0000000..2f6119a --- /dev/null +++ b/corpus/generated/windows-1251/ru/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . , . diff --git a/corpus/generated/windows-1251/ru/train/city-morning.bin b/corpus/generated/windows-1251/ru/train/city-morning.bin new file mode 100644 index 0000000..5d58e7b --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/city-morning.bin @@ -0,0 +1,3 @@ + + + . , , . . diff --git a/corpus/generated/windows-1251/ru/train/library.bin b/corpus/generated/windows-1251/ru/train/library.bin new file mode 100644 index 0000000..3c62d20 --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/library.bin @@ -0,0 +1,3 @@ + + + . , . , . diff --git a/corpus/generated/windows-1251/ru/train/market.bin b/corpus/generated/windows-1251/ru/train/market.bin new file mode 100644 index 0000000..668f4f3 --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/market.bin @@ -0,0 +1,3 @@ + + + . , , . , . diff --git a/corpus/generated/windows-1251/ru/train/workshop.bin b/corpus/generated/windows-1251/ru/train/workshop.bin new file mode 100644 index 0000000..53d315f --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/workshop.bin @@ -0,0 +1,3 @@ + + + . , , . diff --git a/corpus/generated/windows-1251/ru/validation/river-trip.bin b/corpus/generated/windows-1251/ru/validation/river-trip.bin new file mode 100644 index 0000000..afd48bf --- /dev/null +++ b/corpus/generated/windows-1251/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . , . . diff --git a/corpus/generated/windows-1252/da/test/community-garden.bin b/corpus/generated/windows-1252/da/test/community-garden.bin new file mode 100644 index 0000000..bbb5980 --- /dev/null +++ b/corpus/generated/windows-1252/da/test/community-garden.bin @@ -0,0 +1,3 @@ +Flleshaven + +Naboerne gjorde en tom grd til en grn have, brnene malede skilte, og familierne vander planterne hver uge. diff --git a/corpus/generated/windows-1252/da/train/city-morning.bin b/corpus/generated/windows-1252/da/train/city-morning.bin new file mode 100644 index 0000000..aafe557 --- /dev/null +++ b/corpus/generated/windows-1252/da/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen vgner langsomt, caferne bner, og folk venter p de frste busser. diff --git a/corpus/generated/windows-1252/da/train/library.bin b/corpus/generated/windows-1252/da/train/library.bin new file mode 100644 index 0000000..dfcd24b --- /dev/null +++ b/corpus/generated/windows-1252/da/train/library.bin @@ -0,0 +1,3 @@ +Det stille bibliotek + +Lsere finder bger om historie og videnskab, mens bibliotekaren hjlper eleverne. diff --git a/corpus/generated/windows-1252/da/train/market.bin b/corpus/generated/windows-1252/da/train/market.bin new file mode 100644 index 0000000..e156a58 --- /dev/null +++ b/corpus/generated/windows-1252/da/train/market.bin @@ -0,0 +1,3 @@ +Lrdagsmarked + +Bnder slger frisk brd, bler, ost og honning p den travle plads. diff --git a/corpus/generated/windows-1252/da/train/workshop.bin b/corpus/generated/windows-1252/da/train/workshop.bin new file mode 100644 index 0000000..04a0b0c --- /dev/null +++ b/corpus/generated/windows-1252/da/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt vrksted + +Et lille hold bygger en museumsguide, undersger koden og skriver tydelige tekster. diff --git a/corpus/generated/windows-1252/da/validation/river-trip.bin b/corpus/generated/windows-1252/da/validation/river-trip.bin new file mode 100644 index 0000000..386c495 --- /dev/null +++ b/corpus/generated/windows-1252/da/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til floden + +Vennerne tog et kort og varm te med p en sndag og hvilede bagefter ved floden. diff --git a/corpus/generated/windows-1252/de/test/community-garden.bin b/corpus/generated/windows-1252/de/test/community-garden.bin new file mode 100644 index 0000000..df4bbec --- /dev/null +++ b/corpus/generated/windows-1252/de/test/community-garden.bin @@ -0,0 +1,3 @@ +Der Gemeinschaftsgarten + +Die Nachbarn haben einen leeren Hof begrnt und dort Kruter, Gemse und junge Bume gepflanzt. Die Schler malten bunte Schilder, whrend ltere Bewohner regelmig gieen und Gerte aufrumen. diff --git a/corpus/generated/windows-1252/de/train/city-morning.bin b/corpus/generated/windows-1252/de/train/city-morning.bin new file mode 100644 index 0000000..3ccc201 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen in der Stadt + +Die Stadt erwacht langsam. Bckereien ffnen ihre Tren, Busse beginnen ihre Fahrt und viele Menschen lesen unterwegs die Nachrichten. ber den Dchern wird der Himmel allmhlich heller. diff --git a/corpus/generated/windows-1252/de/train/library.bin b/corpus/generated/windows-1252/de/train/library.bin new file mode 100644 index 0000000..992cca1 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/library.bin @@ -0,0 +1,3 @@ +Die ruhige Bibliothek + +In der kleinen Bibliothek rascheln leise die Seiten. Besucher suchen Bcher ber Geschichte, Technik und ferne Lnder. Eine Mitarbeiterin bereitet eine Ausstellung fr Schler vor. diff --git a/corpus/generated/windows-1252/de/train/market.bin b/corpus/generated/windows-1252/de/train/market.bin new file mode 100644 index 0000000..db29233 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/market.bin @@ -0,0 +1,3 @@ +Markt am Samstag + +Auf dem Platz bieten Bauern pfel, Kse, Honig und frisches Brot an. Die Kunden vergleichen Gemse, sprechen ber die Ernte und tauschen einfache Rezepte aus. diff --git a/corpus/generated/windows-1252/de/train/workshop.bin b/corpus/generated/windows-1252/de/train/workshop.bin new file mode 100644 index 0000000..17ea61c --- /dev/null +++ b/corpus/generated/windows-1252/de/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale Werkstatt + +Ein kleines Team entwickelt einen digitalen Museumsfhrer. Designer ordnen das Men, Entwickler prfen den Code und Redakteure schreiben verstndliche Texte fr Gste. diff --git a/corpus/generated/windows-1252/de/validation/river-trip.bin b/corpus/generated/windows-1252/de/validation/river-trip.bin new file mode 100644 index 0000000..09bc118 --- /dev/null +++ b/corpus/generated/windows-1252/de/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ausflug an den Fluss + +Am Sonntag wandern Freunde zu einem ruhigen Fluss. Sie nehmen eine Karte, heien Tee und eine Kamera mit. Nach dem langen Weg sitzen sie am Ufer und hren den Vgeln zu. diff --git a/corpus/generated/windows-1252/en/test/community-garden.bin b/corpus/generated/windows-1252/en/test/community-garden.bin new file mode 100644 index 0000000..549b374 --- /dev/null +++ b/corpus/generated/windows-1252/en/test/community-garden.bin @@ -0,0 +1,3 @@ +The community garden + +Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small caf table. diff --git a/corpus/generated/windows-1252/en/train/city-morning.bin b/corpus/generated/windows-1252/en/train/city-morning.bin new file mode 100644 index 0000000..f7199ae --- /dev/null +++ b/corpus/generated/windows-1252/en/train/city-morning.bin @@ -0,0 +1,3 @@ +Morning in the city + +The city wakes slowly; cafs open their doors and commuters read news beside a faade. diff --git a/corpus/generated/windows-1252/en/train/library.bin b/corpus/generated/windows-1252/en/train/library.bin new file mode 100644 index 0000000..20400f3 --- /dev/null +++ b/corpus/generated/windows-1252/en/train/library.bin @@ -0,0 +1,3 @@ +The quiet library + +Readers browse books about history, science, and nave art while the librarian prepares a rsum. diff --git a/corpus/generated/windows-1252/en/train/market.bin b/corpus/generated/windows-1252/en/train/market.bin new file mode 100644 index 0000000..c760513 --- /dev/null +++ b/corpus/generated/windows-1252/en/train/market.bin @@ -0,0 +1,3 @@ +Saturday market + +Farmers sell bread, apples, jalapeos, and pt while neighbours exchange recipes. diff --git a/corpus/generated/windows-1252/en/train/workshop.bin b/corpus/generated/windows-1252/en/train/workshop.bin new file mode 100644 index 0000000..788536f --- /dev/null +++ b/corpus/generated/windows-1252/en/train/workshop.bin @@ -0,0 +1,3 @@ +Digital workshop + +A small team designs the museum guide and discusses its rle, code, and clear descriptions. diff --git a/corpus/generated/windows-1252/en/validation/river-trip.bin b/corpus/generated/windows-1252/en/validation/river-trip.bin new file mode 100644 index 0000000..b686334 --- /dev/null +++ b/corpus/generated/windows-1252/en/validation/river-trip.bin @@ -0,0 +1,3 @@ +Journey to the river + +Friends carried a map and a thermos, walked past the caf, and rested by the river. diff --git a/corpus/generated/windows-1252/es/test/community-garden.bin b/corpus/generated/windows-1252/es/test/community-garden.bin new file mode 100644 index 0000000..3c82a6f --- /dev/null +++ b/corpus/generated/windows-1252/es/test/community-garden.bin @@ -0,0 +1,3 @@ +El jardn comunitario + +Los vecinos convirtieron un patio vaco en un jardn lleno de rboles, hierbas y verduras. Los nios pintaron carteles de colores y cada sbado varias familias riegan las plantas. diff --git a/corpus/generated/windows-1252/es/train/city-morning.bin b/corpus/generated/windows-1252/es/train/city-morning.bin new file mode 100644 index 0000000..518bc97 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/city-morning.bin @@ -0,0 +1,3 @@ +La maana en la ciudad + +La ciudad despierta poco a poco. Las cafeteras abren sus puertas, los autobuses inician la ruta y los vecinos leen las noticias. El cielo se vuelve ms claro sobre los tejados. diff --git a/corpus/generated/windows-1252/es/train/library.bin b/corpus/generated/windows-1252/es/train/library.bin new file mode 100644 index 0000000..4d6010f --- /dev/null +++ b/corpus/generated/windows-1252/es/train/library.bin @@ -0,0 +1,3 @@ +Una biblioteca tranquila + +En la biblioteca del barrio se oyen pginas y pasos suaves. Los lectores buscan libros de historia, ciencia y viajes. La bibliotecaria prepara una exposicin para los alumnos. diff --git a/corpus/generated/windows-1252/es/train/market.bin b/corpus/generated/windows-1252/es/train/market.bin new file mode 100644 index 0000000..31af767 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/market.bin @@ -0,0 +1,3 @@ +El mercado del sbado + +El mercado llena la plaza de colores. Los agricultores venden manzanas, miel, queso y pan recin hecho. Los clientes comparan las verduras y comparten recetas de sus familias. diff --git a/corpus/generated/windows-1252/es/train/workshop.bin b/corpus/generated/windows-1252/es/train/workshop.bin new file mode 100644 index 0000000..92a2f60 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/workshop.bin @@ -0,0 +1,3 @@ +Taller digital + +Un equipo pequeo crea el catlogo digital del museo. Los diseadores ordenan el men, los programadores revisan el cdigo y los editores escriben textos claros para el pblico. diff --git a/corpus/generated/windows-1252/es/validation/river-trip.bin b/corpus/generated/windows-1252/es/validation/river-trip.bin new file mode 100644 index 0000000..022d6f6 --- /dev/null +++ b/corpus/generated/windows-1252/es/validation/river-trip.bin @@ -0,0 +1,3 @@ +Paseo junto al ro + +El domingo unos amigos caminan hacia un ro tranquilo. Llevan un mapa, t caliente y una cmara. Despus del paseo descansan en la orilla y escuchan a los pjaros. diff --git a/corpus/generated/windows-1252/fr/test/community-garden.bin b/corpus/generated/windows-1252/fr/test/community-garden.bin new file mode 100644 index 0000000..2dce62d --- /dev/null +++ b/corpus/generated/windows-1252/fr/test/community-garden.bin @@ -0,0 +1,3 @@ +Le jardin partag + +Les habitants ont transform une cour oublie en jardin fleuri. Les enfants ont prpar des tiquettes colores, tandis que les voisins se runissent chaque semaine pour arroser les lgumes et les jeunes arbres. diff --git a/corpus/generated/windows-1252/fr/train/city-morning.bin b/corpus/generated/windows-1252/fr/train/city-morning.bin new file mode 100644 index 0000000..411550d --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/city-morning.bin @@ -0,0 +1,3 @@ +Le matin en ville + +La ville se rveille lentement. Les cafs ouvrent leurs portes, les autobus commencent leur trajet et les passants consultent les nouvelles. Chacun prpare sa journe sous un ciel encore ple. diff --git a/corpus/generated/windows-1252/fr/train/library.bin b/corpus/generated/windows-1252/fr/train/library.bin new file mode 100644 index 0000000..e06a9ef --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/library.bin @@ -0,0 +1,3 @@ +La bibliothque calme + +Dans la bibliothque du quartier, les pages tournent en silence. Les lecteurs cherchent des livres sur l'histoire, les sciences et les voyages. Une bibliothcaire prpare une exposition pour les lves. diff --git a/corpus/generated/windows-1252/fr/train/market.bin b/corpus/generated/windows-1252/fr/train/market.bin new file mode 100644 index 0000000..9717faa --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/market.bin @@ -0,0 +1,3 @@ +Le march du samedi + +Le march remplit la place de couleurs et de parfums. Les producteurs proposent des pommes, du miel, du fromage et du pain frais. Les clients discutent des saisons et changent des recettes familiales. diff --git a/corpus/generated/windows-1252/fr/train/workshop.bin b/corpus/generated/windows-1252/fr/train/workshop.bin new file mode 100644 index 0000000..4dea00e --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/workshop.bin @@ -0,0 +1,3 @@ +Un atelier numrique + +Une petite quipe construit le catalogue numrique du muse. Les graphistes organisent le menu, les dveloppeurs vrifient le code et les rdacteurs prparent des textes clairs pour le public. diff --git a/corpus/generated/windows-1252/fr/validation/river-trip.bin b/corpus/generated/windows-1252/fr/validation/river-trip.bin new file mode 100644 index 0000000..48f3a71 --- /dev/null +++ b/corpus/generated/windows-1252/fr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Une promenade prs de la rivire + +Dimanche, des amis partent vers une rivire tranquille. Ils emportent une carte, du th chaud et un appareil photo. Aprs une longue marche, ils se reposent sur la rive et coutent les oiseaux. diff --git a/corpus/generated/windows-1252/it/test/community-garden.bin b/corpus/generated/windows-1252/it/test/community-garden.bin new file mode 100644 index 0000000..76a54ea --- /dev/null +++ b/corpus/generated/windows-1252/it/test/community-garden.bin @@ -0,0 +1,3 @@ +Il giardino comune + +I vicini hanno trasformato un cortile vuoto in un giardino della citt, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante. diff --git a/corpus/generated/windows-1252/it/train/city-morning.bin b/corpus/generated/windows-1252/it/train/city-morning.bin new file mode 100644 index 0000000..a8faca7 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/city-morning.bin @@ -0,0 +1,3 @@ +Mattina in citt + +La citt si sveglia lentamente, i caff aprono e la gente aspetta i primi autobus. diff --git a/corpus/generated/windows-1252/it/train/library.bin b/corpus/generated/windows-1252/it/train/library.bin new file mode 100644 index 0000000..e9b65ab --- /dev/null +++ b/corpus/generated/windows-1252/it/train/library.bin @@ -0,0 +1,3 @@ +La biblioteca tranquilla + +I lettori cercano pi libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti pi giovani. diff --git a/corpus/generated/windows-1252/it/train/market.bin b/corpus/generated/windows-1252/it/train/market.bin new file mode 100644 index 0000000..780f149 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/market.bin @@ -0,0 +1,3 @@ +Mercato del sabato + +Gli agricoltori vendono pane fresco, caff, mele, formaggio e miele nella piazza della citt. diff --git a/corpus/generated/windows-1252/it/train/workshop.bin b/corpus/generated/windows-1252/it/train/workshop.bin new file mode 100644 index 0000000..21734d6 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/workshop.bin @@ -0,0 +1,3 @@ +Laboratorio digitale + +Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perch siano utili a pi persone. diff --git a/corpus/generated/windows-1252/it/validation/river-trip.bin b/corpus/generated/windows-1252/it/validation/river-trip.bin new file mode 100644 index 0000000..0ff2d62 --- /dev/null +++ b/corpus/generated/windows-1252/it/validation/river-trip.bin @@ -0,0 +1,3 @@ +Gita al fiume + +Gli amici portano una mappa e t caldo, camminano nel bosco e riposano sulla riva. diff --git a/corpus/generated/windows-1252/nl/test/community-garden.bin b/corpus/generated/windows-1252/nl/test/community-garden.bin new file mode 100644 index 0000000..89526b9 --- /dev/null +++ b/corpus/generated/windows-1252/nl/test/community-garden.bin @@ -0,0 +1,3 @@ +De buurttuin + +Buren veranderden een lege binnenplaats naast een caf in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water. diff --git a/corpus/generated/windows-1252/nl/train/city-morning.bin b/corpus/generated/windows-1252/nl/train/city-morning.bin new file mode 100644 index 0000000..13f5ea7 --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/city-morning.bin @@ -0,0 +1,3 @@ +Ochtend in de stad + +De stad ontwaakt langzaam, cafs openen hun deuren en mensen wachten op de eerste bussen. diff --git a/corpus/generated/windows-1252/nl/train/library.bin b/corpus/generated/windows-1252/nl/train/library.bin new file mode 100644 index 0000000..cdf34ac --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/library.bin @@ -0,0 +1,3 @@ +De stille bibliotheek + +Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris genteresseerde leerlingen en n leraar helpt. diff --git a/corpus/generated/windows-1252/nl/train/market.bin b/corpus/generated/windows-1252/nl/train/market.bin new file mode 100644 index 0000000..9c584bb --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/market.bin @@ -0,0 +1,3 @@ +Zaterdagmarkt + +Boeren verkopen vers brood, appels, kaas en honing naast twee cafs op het drukke plein. diff --git a/corpus/generated/windows-1252/nl/train/workshop.bin b/corpus/generated/windows-1252/nl/train/workshop.bin new file mode 100644 index 0000000..53ff667 --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale werkplaats + +Een klein team maakt een museumgids, controleert de code en schrijft duidelijke ideen over pozie en teksten. diff --git a/corpus/generated/windows-1252/nl/validation/river-trip.bin b/corpus/generated/windows-1252/nl/validation/river-trip.bin new file mode 100644 index 0000000..27db29e --- /dev/null +++ b/corpus/generated/windows-1252/nl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Reis naar de rivier + +Vrienden namen een kaart en warme thee mee en rustten na n lange wandeling bij de rivier. diff --git a/corpus/generated/windows-1252/no/test/community-garden.bin b/corpus/generated/windows-1252/no/test/community-garden.bin new file mode 100644 index 0000000..b82814c --- /dev/null +++ b/corpus/generated/windows-1252/no/test/community-garden.bin @@ -0,0 +1,3 @@ +Felleshagen + +Naboene gjorde en tom grd til en grnn hage, barna malte skilt og familiene vanner plantene hver uke. diff --git a/corpus/generated/windows-1252/no/train/city-morning.bin b/corpus/generated/windows-1252/no/train/city-morning.bin new file mode 100644 index 0000000..972847d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen vkner langsomt, kafeene pner drene og folk venter p de frste bussene nr bryggen og sjen. diff --git a/corpus/generated/windows-1252/no/train/library.bin b/corpus/generated/windows-1252/no/train/library.bin new file mode 100644 index 0000000..f972c3d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/library.bin @@ -0,0 +1,3 @@ +Det stille biblioteket + +Lesere finner bker om historie, rlige fortellinger og vitenskap mens bibliotekaren hjelper elevene. diff --git a/corpus/generated/windows-1252/no/train/market.bin b/corpus/generated/windows-1252/no/train/market.bin new file mode 100644 index 0000000..c020f8d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/market.bin @@ -0,0 +1,3 @@ +Lrdagsmarked + +Bnder selger ferskt brd, epler, ost og honning p det travle torget. diff --git a/corpus/generated/windows-1252/no/train/workshop.bin b/corpus/generated/windows-1252/no/train/workshop.bin new file mode 100644 index 0000000..954e228 --- /dev/null +++ b/corpus/generated/windows-1252/no/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt verksted + +Et lite lag lager en museumsguide, undersker koden og skriver tydelige tekster. diff --git a/corpus/generated/windows-1252/no/validation/river-trip.bin b/corpus/generated/windows-1252/no/validation/river-trip.bin new file mode 100644 index 0000000..1489859 --- /dev/null +++ b/corpus/generated/windows-1252/no/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til elven + +Vennene tok med kart og varm te p sndag og hvilte senere ved elven. diff --git a/corpus/generated/windows-1252/pt/test/community-garden.bin b/corpus/generated/windows-1252/pt/test/community-garden.bin new file mode 100644 index 0000000..65848b6 --- /dev/null +++ b/corpus/generated/windows-1252/pt/test/community-garden.bin @@ -0,0 +1,3 @@ +A horta comunitria + +Os vizinhos transformaram um ptio vazio em jardim, as crianas pintaram placas e as famlias regam as plantas toda semana. diff --git a/corpus/generated/windows-1252/pt/train/city-morning.bin b/corpus/generated/windows-1252/pt/train/city-morning.bin new file mode 100644 index 0000000..18b861f --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/city-morning.bin @@ -0,0 +1,3 @@ +Manh na cidade + +A cidade acorda devagar, os cafs abrem e as pessoas esperam os primeiros nibus. diff --git a/corpus/generated/windows-1252/pt/train/library.bin b/corpus/generated/windows-1252/pt/train/library.bin new file mode 100644 index 0000000..50b8034 --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/library.bin @@ -0,0 +1,3 @@ +A biblioteca tranquila + +Os leitores procuram livros de histria e cincia enquanto a bibliotecria ajuda os alunos. diff --git a/corpus/generated/windows-1252/pt/train/market.bin b/corpus/generated/windows-1252/pt/train/market.bin new file mode 100644 index 0000000..3810190 --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/market.bin @@ -0,0 +1,3 @@ +Mercado de sbado + +Os agricultores vendem po fresco, mas, queijo e mel na praa movimentada. diff --git a/corpus/generated/windows-1252/pt/train/workshop.bin b/corpus/generated/windows-1252/pt/train/workshop.bin new file mode 100644 index 0000000..ac78c4e --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/workshop.bin @@ -0,0 +1,3 @@ +Oficina digital + +Uma pequena equipe prepara o guia do museu, verifica o cdigo e escreve descries claras. diff --git a/corpus/generated/windows-1252/pt/validation/river-trip.bin b/corpus/generated/windows-1252/pt/validation/river-trip.bin new file mode 100644 index 0000000..de2590a --- /dev/null +++ b/corpus/generated/windows-1252/pt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Viagem ao rio + +Os amigos levaram um mapa e ch quente, caminharam pela mata e descansaram junto ao rio. diff --git a/corpus/generated/windows-1252/sv/test/community-garden.bin b/corpus/generated/windows-1252/sv/test/community-garden.bin new file mode 100644 index 0000000..0acb233 --- /dev/null +++ b/corpus/generated/windows-1252/sv/test/community-garden.bin @@ -0,0 +1,3 @@ +Gemensam trdgrd + +Grannarna gjorde en tom grd till en grn trdgrd, barnen mlade skyltar och familjerna vattnar vxterna varje vecka. diff --git a/corpus/generated/windows-1252/sv/train/city-morning.bin b/corpus/generated/windows-1252/sv/train/city-morning.bin new file mode 100644 index 0000000..f0ba24b --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgon i staden + +Staden vaknar lngsamt, kaferna ppnar och mnniskor vntar p de frsta bussarna. diff --git a/corpus/generated/windows-1252/sv/train/library.bin b/corpus/generated/windows-1252/sv/train/library.bin new file mode 100644 index 0000000..a7319ac --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/library.bin @@ -0,0 +1,3 @@ +Det tysta biblioteket + +Lsare sker bcker om historia och vetenskap medan bibliotekarien hjlper eleverna. diff --git a/corpus/generated/windows-1252/sv/train/market.bin b/corpus/generated/windows-1252/sv/train/market.bin new file mode 100644 index 0000000..0bd6cfb --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/market.bin @@ -0,0 +1,3 @@ +Lrdagsmarknad + +Bnder sljer frskt brd, pplen, ost och honung p det livliga torget. diff --git a/corpus/generated/windows-1252/sv/train/workshop.bin b/corpus/generated/windows-1252/sv/train/workshop.bin new file mode 100644 index 0000000..81db320 --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/workshop.bin @@ -0,0 +1,3 @@ +Digital verkstad + +Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter fr beskare. diff --git a/corpus/generated/windows-1252/sv/validation/river-trip.bin b/corpus/generated/windows-1252/sv/validation/river-trip.bin new file mode 100644 index 0000000..4704973 --- /dev/null +++ b/corpus/generated/windows-1252/sv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Utflykt till floden + +Vnnerna tog med karta och varmt te och vilade sedan vid floden. diff --git a/corpus/generated/windows-1253/el/test/community-garden.bin b/corpus/generated/windows-1253/el/test/community-garden.bin new file mode 100644 index 0000000..2e8196c --- /dev/null +++ b/corpus/generated/windows-1253/el/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1253/el/train/city-morning.bin b/corpus/generated/windows-1253/el/train/city-morning.bin new file mode 100644 index 0000000..f58abba --- /dev/null +++ b/corpus/generated/windows-1253/el/train/city-morning.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1253/el/train/library.bin b/corpus/generated/windows-1253/el/train/library.bin new file mode 100644 index 0000000..6724218 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/library.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1253/el/train/market.bin b/corpus/generated/windows-1253/el/train/market.bin new file mode 100644 index 0000000..40995f7 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/market.bin @@ -0,0 +1,3 @@ + + + , , . diff --git a/corpus/generated/windows-1253/el/train/workshop.bin b/corpus/generated/windows-1253/el/train/workshop.bin new file mode 100644 index 0000000..4f44381 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/workshop.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1253/el/validation/river-trip.bin b/corpus/generated/windows-1253/el/validation/river-trip.bin new file mode 100644 index 0000000..dcea70f --- /dev/null +++ b/corpus/generated/windows-1253/el/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1254/tr/test/community-garden.bin b/corpus/generated/windows-1254/tr/test/community-garden.bin new file mode 100644 index 0000000..831e0ea --- /dev/null +++ b/corpus/generated/windows-1254/tr/test/community-garden.bin @@ -0,0 +1,3 @@ +Mahalle bahesi + +Komular bo avluyu yeil bir baheye evirdi, ocuklar renkli tabelalar yapt ve aileler bitkileri her hafta suluyor. diff --git a/corpus/generated/windows-1254/tr/train/city-morning.bin b/corpus/generated/windows-1254/tr/train/city-morning.bin new file mode 100644 index 0000000..e8458f6 --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/city-morning.bin @@ -0,0 +1,3 @@ +ehirde sabah + +ehir yavaa uyanr, kafeler alr ve insanlar ilk otobsleri bekler. diff --git a/corpus/generated/windows-1254/tr/train/library.bin b/corpus/generated/windows-1254/tr/train/library.bin new file mode 100644 index 0000000..20c465c --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/library.bin @@ -0,0 +1,3 @@ +Sessiz ktphane + +Okurlar tarih ve bilim kitaplar ararken ktphaneci rencilere yardmc olur. diff --git a/corpus/generated/windows-1254/tr/train/market.bin b/corpus/generated/windows-1254/tr/train/market.bin new file mode 100644 index 0000000..0856260 --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/market.bin @@ -0,0 +1,3 @@ +Cumartesi pazar + +iftiler kalabalk meydanda taze ekmek, elma, peynir ve bal satar. diff --git a/corpus/generated/windows-1254/tr/train/workshop.bin b/corpus/generated/windows-1254/tr/train/workshop.bin new file mode 100644 index 0000000..844ca1e --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/workshop.bin @@ -0,0 +1,3 @@ +Dijital atlye + +Kk bir ekip mze rehberi hazrlar, kodu denetler ve ak aklamalar yazar. diff --git a/corpus/generated/windows-1254/tr/validation/river-trip.bin b/corpus/generated/windows-1254/tr/validation/river-trip.bin new file mode 100644 index 0000000..2568000 --- /dev/null +++ b/corpus/generated/windows-1254/tr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Nehir gezisi + +Arkadalar harita ve scak ay alp ormanda yrd, sonra nehir kysnda dinlendi. diff --git a/corpus/generated/windows-1255/he/test/community-garden.bin b/corpus/generated/windows-1255/he/test/community-garden.bin new file mode 100644 index 0000000..28d54b2 --- /dev/null +++ b/corpus/generated/windows-1255/he/test/community-garden.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1255/he/train/city-morning.bin b/corpus/generated/windows-1255/he/train/city-morning.bin new file mode 100644 index 0000000..90ce4a0 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/city-morning.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1255/he/train/library.bin b/corpus/generated/windows-1255/he/train/library.bin new file mode 100644 index 0000000..d4abd33 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/library.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1255/he/train/market.bin b/corpus/generated/windows-1255/he/train/market.bin new file mode 100644 index 0000000..d4e1672 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/market.bin @@ -0,0 +1,3 @@ + + + , , . diff --git a/corpus/generated/windows-1255/he/train/workshop.bin b/corpus/generated/windows-1255/he/train/workshop.bin new file mode 100644 index 0000000..9120483 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/workshop.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1255/he/validation/river-trip.bin b/corpus/generated/windows-1255/he/validation/river-trip.bin new file mode 100644 index 0000000..f2a8f51 --- /dev/null +++ b/corpus/generated/windows-1255/he/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + , . diff --git a/corpus/generated/windows-1256/ar/test/community-garden.bin b/corpus/generated/windows-1256/ar/test/community-garden.bin new file mode 100644 index 0000000..613b5d4 --- /dev/null +++ b/corpus/generated/windows-1256/ar/test/community-garden.bin @@ -0,0 +1,3 @@ + + + ɡ . diff --git a/corpus/generated/windows-1256/ar/train/city-morning.bin b/corpus/generated/windows-1256/ar/train/city-morning.bin new file mode 100644 index 0000000..59e7a5d --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/city-morning.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1256/ar/train/library.bin b/corpus/generated/windows-1256/ar/train/library.bin new file mode 100644 index 0000000..394f9ea --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/library.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1256/ar/train/market.bin b/corpus/generated/windows-1256/ar/train/market.bin new file mode 100644 index 0000000..0ef6121 --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/market.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1256/ar/train/workshop.bin b/corpus/generated/windows-1256/ar/train/workshop.bin new file mode 100644 index 0000000..b9579e6 --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/workshop.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1256/ar/validation/river-trip.bin b/corpus/generated/windows-1256/ar/validation/river-trip.bin new file mode 100644 index 0000000..fee14e0 --- /dev/null +++ b/corpus/generated/windows-1256/ar/validation/river-trip.bin @@ -0,0 +1,3 @@ + + + . diff --git a/corpus/generated/windows-1257/et/test/community-garden.bin b/corpus/generated/windows-1257/et/test/community-garden.bin new file mode 100644 index 0000000..46ad8a9 --- /dev/null +++ b/corpus/generated/windows-1257/et/test/community-garden.bin @@ -0,0 +1,3 @@ +Kogukonna aed + +Naabrid muutsid thja hoovi roheliseks aiaks, lapsed maalisid sildid ja pered kastavad taimi igal ndalal. diff --git a/corpus/generated/windows-1257/et/train/city-morning.bin b/corpus/generated/windows-1257/et/train/city-morning.bin new file mode 100644 index 0000000..f24f4c7 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/city-morning.bin @@ -0,0 +1,3 @@ +Hommik linnas + +Linn rkab aeglaselt, kohvikud avavad uksed ja inimesed ootavad esimesi busse. diff --git a/corpus/generated/windows-1257/et/train/library.bin b/corpus/generated/windows-1257/et/train/library.bin new file mode 100644 index 0000000..1791206 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/library.bin @@ -0,0 +1,3 @@ +Vaikne raamatukogu + +Lugejad otsivad ajaloo- ja teadusraamatuid ning raamatukoguhoidja aitab pilasi. diff --git a/corpus/generated/windows-1257/et/train/market.bin b/corpus/generated/windows-1257/et/train/market.bin new file mode 100644 index 0000000..f7f3563 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/market.bin @@ -0,0 +1,3 @@ +Laupevane turg + +Talunikud mvad vrsket leiba, unu, juustu ja mett rahvarohkel vljakul. diff --git a/corpus/generated/windows-1257/et/train/workshop.bin b/corpus/generated/windows-1257/et/train/workshop.bin new file mode 100644 index 0000000..a61fe78 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/workshop.bin @@ -0,0 +1,3 @@ +Digitaalne ttuba + +Vike meeskond koostab muuseumijuhti, kontrollib koodi ja kirjutab selgeid kirjeldusi. diff --git a/corpus/generated/windows-1257/et/validation/river-trip.bin b/corpus/generated/windows-1257/et/validation/river-trip.bin new file mode 100644 index 0000000..9b2f7eb --- /dev/null +++ b/corpus/generated/windows-1257/et/validation/river-trip.bin @@ -0,0 +1,3 @@ +Retk je rde + +Sbrad vtsid kaardi ja sooja tee ning puhkasid prast jalutuskiku je res. diff --git a/corpus/generated/windows-1257/lt/test/community-garden.bin b/corpus/generated/windows-1257/lt/test/community-garden.bin new file mode 100644 index 0000000..5d1e77b --- /dev/null +++ b/corpus/generated/windows-1257/lt/test/community-garden.bin @@ -0,0 +1,3 @@ +Bendruomens sodas + +Kaimynai tui kiem pavert aliu sodu su glmis, darovmis ir jaunais mediais. Vaikai nupie spalvingus enklus, o eimos kas savait laisto augalus. diff --git a/corpus/generated/windows-1257/lt/train/city-morning.bin b/corpus/generated/windows-1257/lt/train/city-morning.bin new file mode 100644 index 0000000..4ab6bae --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/city-morning.bin @@ -0,0 +1,3 @@ +Rytas mieste + +Miestas bunda pamau. Kavins atveria duris, autobusai pradeda pirmuosius marrutus, o mons skaito inias. Vir sen stog viesja dangus. diff --git a/corpus/generated/windows-1257/lt/train/library.bin b/corpus/generated/windows-1257/lt/train/library.bin new file mode 100644 index 0000000..3731c52 --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/library.bin @@ -0,0 +1,3 @@ +Tyli biblioteka + +Rajono bibliotekoje tyliai iugda puslapiai. Skaitytojai ieko knyg apie istorij, moksl ir keliones. Bibliotekinink rengia parod mokiniams. diff --git a/corpus/generated/windows-1257/lt/train/market.bin b/corpus/generated/windows-1257/lt/train/market.bin new file mode 100644 index 0000000..d2203ae --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/market.bin @@ -0,0 +1,3 @@ +etadienio turgus + +Turgaus aikt pripildo spalvos ir kvapai. kininkai silo obuoli, medaus, srio ir vieios duonos. Pirkjai renkasi daroves ir dalijasi receptais. diff --git a/corpus/generated/windows-1257/lt/train/workshop.bin b/corpus/generated/windows-1257/lt/train/workshop.bin new file mode 100644 index 0000000..3a5f804 --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/workshop.bin @@ -0,0 +1,3 @@ +Skaitmenins dirbtuvs + +Nedidel komanda kuria skaitmenin muziejaus vadov. Dizaineriai tvarko meniu, programuotojai tikrina kod, o redaktoriai rao aikius apraymus. diff --git a/corpus/generated/windows-1257/lt/validation/river-trip.bin b/corpus/generated/windows-1257/lt/validation/river-trip.bin new file mode 100644 index 0000000..77189e1 --- /dev/null +++ b/corpus/generated/windows-1257/lt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kelion prie ups + +Sekmadien draugai ijo prie ramios ups u miesto. Jie pasim emlap, iltos arbatos ir fotoaparat. Po ilgo ygio visi ilsjosi ant kranto. diff --git a/corpus/generated/windows-1257/lv/test/community-garden.bin b/corpus/generated/windows-1257/lv/test/community-garden.bin new file mode 100644 index 0000000..9caae38 --- /dev/null +++ b/corpus/generated/windows-1257/lv/test/community-garden.bin @@ -0,0 +1,3 @@ +Kopienas drzs + +Kaimii prvrta tuku pagalmu par zau drzu ar pum, drzeiem un jauniem kokiem. Brni izgatavoja krsainas zmes, bet imenes katru nedu laista augus. diff --git a/corpus/generated/windows-1257/lv/train/city-morning.bin b/corpus/generated/windows-1257/lv/train/city-morning.bin new file mode 100644 index 0000000..c32767d --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/city-morning.bin @@ -0,0 +1,3 @@ +Rts pilst + +Pilsta mostas lni. Kafejncas atver durvis, autobusi sk pirmos marrutus, un cilvki lasa zias. Virs mju jumtiem debesis kst arvien gaikas. diff --git a/corpus/generated/windows-1257/lv/train/library.bin b/corpus/generated/windows-1257/lv/train/library.bin new file mode 100644 index 0000000..c112840 --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/library.bin @@ -0,0 +1,3 @@ +Klus bibliotka + +Rajona bibliotk klusi aukst grmatu lapas. Lastji mekl darbus par vsturi, zintni un ceojumiem. Bibliotekre gatavo izstdi skolniem. diff --git a/corpus/generated/windows-1257/lv/train/market.bin b/corpus/generated/windows-1257/lv/train/market.bin new file mode 100644 index 0000000..5d940ef --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/market.bin @@ -0,0 +1,3 @@ +Sestdienas tirgus + +Tirgus laukumu piepilda krsas un smaras. Zemnieki piedv bolus, medu, sieru un svaigu maizi. Pircji izvlas drzeus un prrun imenes receptes. diff --git a/corpus/generated/windows-1257/lv/train/workshop.bin b/corpus/generated/windows-1257/lv/train/workshop.bin new file mode 100644 index 0000000..0fbfb56 --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/workshop.bin @@ -0,0 +1,3 @@ +Digitl darbnca + +Neliela komanda veido muzeja digitlo cevedi. Dizaineri sakrto izvlni, programmtji prbauda kodu, bet redaktori raksta skaidrus aprakstus. diff --git a/corpus/generated/windows-1257/lv/validation/river-trip.bin b/corpus/generated/windows-1257/lv/validation/river-trip.bin new file mode 100644 index 0000000..5ba00ad --- /dev/null +++ b/corpus/generated/windows-1257/lv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Pastaiga gar upi + +Svtdien draugi devs uz klusu upi rpus pilstas. Vii pama karti, siltu tju un fotoapartu. Pc gars pastaigas visi atpts krast. diff --git a/corpus/generated/windows-1258/vi/test/community-garden.bin b/corpus/generated/windows-1258/vi/test/community-garden.bin new file mode 100644 index 0000000..be14fff --- /dev/null +++ b/corpus/generated/windows-1258/vi/test/community-garden.bin @@ -0,0 +1,3 @@ +Vn cng ng + +Hng xm bin sn trng thnh khu vn xanh, tre em ve bin mu v cc gia inh ti cy mi tun. diff --git a/corpus/generated/windows-1258/vi/train/city-morning.bin b/corpus/generated/windows-1258/vi/train/city-morning.bin new file mode 100644 index 0000000..321ee45 --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/city-morning.bin @@ -0,0 +1,3 @@ +Bui sng trong thnh ph + +Thnh ph thc dy chm rai, qun c ph m ca v moi ngi ch chuyn xe buyt u tin. diff --git a/corpus/generated/windows-1258/vi/train/library.bin b/corpus/generated/windows-1258/vi/train/library.bin new file mode 100644 index 0000000..89d20aa --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/library.bin @@ -0,0 +1,3 @@ +Th vin yn tinh + +c gia tim sch lich s v khoa hoc, con thu th gip hoc sinh chon ti liu. diff --git a/corpus/generated/windows-1258/vi/train/market.bin b/corpus/generated/windows-1258/vi/train/market.bin new file mode 100644 index 0000000..deb93fb --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/market.bin @@ -0,0 +1,3 @@ +Ch th bay + +Nng dn bn bnh mi, to, ph mai v mt ong tai quang trng ng c. diff --git a/corpus/generated/windows-1258/vi/train/workshop.bin b/corpus/generated/windows-1258/vi/train/workshop.bin new file mode 100644 index 0000000..6f4c26b --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/workshop.bin @@ -0,0 +1,3 @@ +Xng ky thut s + +Mt nhm nho lm hng dn bao tng, kim tra ma v vit m ta ro rng. diff --git a/corpus/generated/windows-1258/vi/validation/river-trip.bin b/corpus/generated/windows-1258/vi/validation/river-trip.bin new file mode 100644 index 0000000..aca81b2 --- /dev/null +++ b/corpus/generated/windows-1258/vi/validation/river-trip.bin @@ -0,0 +1,3 @@ +Chuyn i ra sng + +Ban b mang ban v tr nng, i qua rng ri nghi bn b sng. diff --git a/corpus/generated/windows-874/th/test/community-garden.bin b/corpus/generated/windows-874/th/test/community-garden.bin new file mode 100644 index 0000000..8f1080c --- /dev/null +++ b/corpus/generated/windows-874/th/test/community-garden.bin @@ -0,0 +1,3 @@ +ǹ + +͹ҹ¹ҹҧǹ ҴФͺǪ¡ѹôӵءѻ diff --git a/corpus/generated/windows-874/th/train/city-morning.bin b/corpus/generated/windows-874/th/train/city-morning.bin new file mode 100644 index 0000000..3325a1b --- /dev/null +++ b/corpus/generated/windows-874/th/train/city-morning.bin @@ -0,0 +1,3 @@ +ͧ + +ͧ ҹԴем餹öá diff --git a/corpus/generated/windows-874/th/train/library.bin b/corpus/generated/windows-874/th/train/library.bin new file mode 100644 index 0000000..8574c4f --- /dev/null +++ b/corpus/generated/windows-874/th/train/library.bin @@ -0,0 +1,3 @@ +ͧشºʧ + +ҹ˹ѧͻѵʵԷʵ óѡ¹ѡ¹͡ diff --git a/corpus/generated/windows-874/th/train/market.bin b/corpus/generated/windows-874/th/train/market.bin new file mode 100644 index 0000000..e2a1717 --- /dev/null +++ b/corpus/generated/windows-874/th/train/market.bin @@ -0,0 +1,3 @@ +Ҵѹ + +ǹҢ¢ѧʴ йӼҹ֡ѡ diff --git a/corpus/generated/windows-874/th/train/workshop.bin b/corpus/generated/windows-874/th/train/workshop.bin new file mode 100644 index 0000000..c571435 --- /dev/null +++ b/corpus/generated/windows-874/th/train/workshop.bin @@ -0,0 +1,3 @@ +ͧӧҹԨԷ + +硨ѴӤ;ԾԸѳ Ǩ ¹͸Ժ·Ѵਹ diff --git a/corpus/generated/windows-874/th/validation/river-trip.bin b/corpus/generated/windows-874/th/validation/river-trip.bin new file mode 100644 index 0000000..0e376c8 --- /dev/null +++ b/corpus/generated/windows-874/th/validation/river-trip.bin @@ -0,0 +1,3 @@ + + +͹ἹѺ͹Թ㹻Ǿѡ͹ diff --git a/corpus/manifest.json b/corpus/manifest.json index 5121f91..c747e1e 100644 --- a/corpus/manifest.json +++ b/corpus/manifest.json @@ -1,5 +1,45 @@ { "encodings": [ + { + "name": "ISO-8859-1", + "iconv": "ISO-8859-1", + "languages": ["da", "de", "en", "es", "fr", "it", "nl", "no", "pt", "sv"] + }, + { + "name": "ISO-8859-2", + "iconv": "ISO-8859-2", + "languages": ["cs", "hu", "pl", "ro"], + "sourceReplacements": { "Ș": "Ş", "ș": "ş", "Ț": "Ţ", "ț": "ţ" } + }, + { "name": "ISO-8859-5", "iconv": "ISO-8859-5", "languages": ["ru"] }, + { "name": "ISO-8859-6", "iconv": "ISO-8859-6", "languages": ["ar"] }, + { "name": "ISO-8859-7", "iconv": "ISO-8859-7", "languages": ["el"] }, + { "name": "ISO-8859-8", "iconv": "ISO-8859-8", "languages": ["he"] }, + { "name": "ISO-8859-9", "iconv": "ISO-8859-9", "languages": ["tr"] }, + { + "name": "windows-1250", + "iconv": "WINDOWS-1250", + "languages": ["cs", "hu", "pl", "ro"], + "sourceReplacements": { "Ș": "Ş", "ș": "ş", "Ț": "Ţ", "ț": "ţ" } + }, + { "name": "windows-1251", "iconv": "WINDOWS-1251", "languages": ["ru"] }, + { + "name": "windows-1252", + "iconv": "WINDOWS-1252", + "languages": ["da", "de", "en", "es", "fr", "it", "nl", "no", "pt", "sv"] + }, + { "name": "windows-1253", "iconv": "WINDOWS-1253", "languages": ["el"] }, + { "name": "windows-1254", "iconv": "WINDOWS-1254", "languages": ["tr"] }, + { "name": "windows-1255", "iconv": "WINDOWS-1255", "languages": ["he"] }, + { "name": "windows-1256", "iconv": "WINDOWS-1256", "languages": ["ar"] }, + { + "name": "windows-1257", + "iconv": "WINDOWS-1257", + "languages": ["et", "lv", "lt"] + }, + { "name": "windows-1258", "iconv": "WINDOWS-1258", "languages": ["vi"] }, + { "name": "windows-874", "iconv": "WINDOWS-874", "languages": ["th"] }, + { "name": "KOI8-R", "iconv": "KOI8-R", "languages": ["ru"] }, { "name": "KOI8-U", "iconv": "KOI8-U", "languages": ["uk"] }, { "name": "IBM866", "iconv": "CP866", "languages": ["ru"] }, { "name": "IBM855", "iconv": "CP855", "languages": ["ru", "sr"] }, @@ -24,7 +64,12 @@ "iconv": "ISO-8859-15", "languages": ["fr", "de", "es"] }, - { "name": "ISO-8859-16", "iconv": "ISO-8859-16", "languages": ["ro"] }, + { + "name": "ISO-8859-16", + "iconv": "ISO-8859-16", + "languages": ["ro"], + "sourceSuffix": " Prețul este 10 €." + }, { "name": "CP850", "iconv": "CP850", "languages": ["fr", "de", "es"] }, { "name": "CP852", "iconv": "CP852", "languages": ["pl"] } ] diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 31a0927..013725c 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -1,9 +1,11 @@ { "summary": { - "tests": 22, - "encodingCorrect": 22, - "languageCorrect": 22, - "topScoreTies": 0 + "tests": 66, + "encodingExact": 44, + "encodingEquivalent": 22, + "encodingCorrect": 66, + "languageCorrect": 66, + "topScoreTies": 34 }, "confusion": { "CP850": { @@ -18,33 +20,90 @@ "IBM866": { "IBM866": 1 }, + "ISO-8859-1": { + "ISO-8859-1": 8, + "ISO-8859-15": 2 + }, "ISO-8859-10": { "ISO-8859-10": 1 }, "ISO-8859-13": { - "ISO-8859-13": 1 + "windows-1257": 1 }, "ISO-8859-14": { "ISO-8859-14": 1 }, "ISO-8859-15": { - "ISO-8859-15": 3 + "ISO-8859-1": 3 }, "ISO-8859-16": { "ISO-8859-16": 1 }, + "ISO-8859-2": { + "ISO-8859-2": 4 + }, "ISO-8859-3": { "ISO-8859-3": 1 }, "ISO-8859-4": { "ISO-8859-4": 1 }, + "ISO-8859-5": { + "ISO-8859-5": 1 + }, + "ISO-8859-6": { + "ISO-8859-6": 1 + }, + "ISO-8859-7": { + "ISO-8859-7": 1 + }, + "ISO-8859-8": { + "ISO-8859-8": 1 + }, + "ISO-8859-9": { + "ISO-8859-9": 1 + }, + "KOI8-R": { + "KOI8-R": 1 + }, "KOI8-U": { "KOI8-U": 1 }, "macintosh": { "macintosh": 3 }, + "windows-1250": { + "ISO-8859-2": 3, + "windows-1250": 1 + }, + "windows-1251": { + "windows-1251": 1 + }, + "windows-1252": { + "ISO-8859-1": 8, + "ISO-8859-15": 2 + }, + "windows-1253": { + "ISO-8859-7": 1 + }, + "windows-1254": { + "ISO-8859-9": 1 + }, + "windows-1255": { + "ISO-8859-8": 1 + }, + "windows-1256": { + "windows-1256": 1 + }, + "windows-1257": { + "windows-1257": 3 + }, + "windows-1258": { + "windows-1258": 1 + }, + "windows-874": { + "windows-874": 1 + }, "x-mac-cyrillic": { "x-mac-cyrillic": 2 } @@ -64,6 +123,8 @@ "total": 216, "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -77,10 +138,28 @@ "total": 216, "hitRate": 0.24537037037037038 }, + { + "encoding": "ISO-8859-1", + "confidence": 72, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "windows-1252", + "confidence": 72, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, { "encoding": "ISO-8859-15", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -89,7 +168,7 @@ { "encoding": "macintosh", "confidence": 70, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -105,7 +184,25 @@ "hitRate": 0.07441860465116279 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, + { + "encoding": "ISO-8859-9", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, + { + "encoding": "windows-1250", "confidence": 13, "language": "ro", "byteLogLikelihood": -5.176149732574, @@ -113,6 +210,24 @@ "total": 215, "hitRate": 0.046511627906976744 }, + { + "encoding": "windows-1254", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 215, + "hitRate": 0.037209302325581395 + }, { "encoding": "ISO-8859-4", "confidence": 11, @@ -122,6 +237,24 @@ "total": 215, "hitRate": 0.037209302325581395 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 215, + "hitRate": 0.037209302325581395 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.172849867117, + "hits": 5, + "total": 215, + "hitRate": 0.023255813953488372 + }, { "encoding": "ISO-8859-3", "confidence": 5, @@ -158,6 +291,96 @@ "total": 215, "hitRate": 0.009302325581395349 }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.099525732258, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.729917997667, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.542674768484, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.504753627871, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.219881518497, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.542674768484, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.504753627871, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.511862527725, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.737138560752, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 215, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -210,6 +433,8 @@ "total": 200, "hitRate": 0.205 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -223,6 +448,24 @@ "total": 200, "hitRate": 0.205 }, + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, { "encoding": "macintosh", "confidence": 58, @@ -241,6 +484,15 @@ "total": 199, "hitRate": 0.19597989949748743 }, + { + "encoding": "windows-1257", + "confidence": 25, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, { "encoding": "ISO-8859-4", "confidence": 25, @@ -269,7 +521,16 @@ "hitRate": 0.05527638190954774 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "windows-1250", "confidence": 15, "language": "ro", "byteLogLikelihood": -5.176149732574, @@ -277,6 +538,42 @@ "total": 199, "hitRate": 0.05025125628140704 }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -4.952116173006, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, { "encoding": "CP852", "confidence": 7, @@ -296,83 +593,193 @@ "hitRate": 0.01507537688442211 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-5", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.351652470383, + "language": "ru", + "byteLogLikelihood": -6.692083742507, "hits": 0, "total": 199, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.374008967132, + "language": "ar", + "byteLogLikelihood": -6.113682179832, "hits": 0, - "total": 200, + "total": 199, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.64715417232, + "language": "el", + "byteLogLikelihood": -6.120297418951, "hits": 0, - "total": 200, + "total": 199, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "he", + "byteLogLikelihood": -5.932245187448, "hits": 0, "total": 199, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1251", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.759371883621, + "language": "ru", + "byteLogLikelihood": -6.692083742507, "hits": 0, "total": 199, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "CP850", - "language": "fr" - }, - "predicted": { - "encoding": "CP850", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ + }, { - "encoding": "CP850", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.61627103998, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.351652470383, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.374008967132, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.64715417232, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.759371883621, + "hits": 0, + "total": 199, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "CP850", + "language": "fr" + }, + "predicted": { + "encoding": "CP850", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "CP850", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-1", + "confidence": 75, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "windows-1252", + "confidence": 75, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 }, { "encoding": "ISO-8859-15", "confidence": 75, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -381,12 +788,30 @@ { "encoding": "macintosh", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, "hitRate": 0.23684210526315788 }, + { + "encoding": "ISO-8859-2", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "windows-1250", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, { "encoding": "ISO-8859-10", "confidence": 12, @@ -400,11 +825,20 @@ "encoding": "ISO-8859-16", "confidence": 12, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 9, "total": 224, "hitRate": 0.04017857142857143 }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 224, + "hitRate": 0.03125 + }, { "encoding": "ISO-8859-4", "confidence": 9, @@ -432,6 +866,33 @@ "total": 228, "hitRate": 0.02631578947368421 }, + { + "encoding": "ISO-8859-9", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1254", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, { "encoding": "ISO-8859-13", "confidence": 5, @@ -450,6 +911,96 @@ "total": 224, "hitRate": 0.013392857142857142 }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -502,9 +1053,11 @@ "total": 184, "hitRate": 0.1358695652173913 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { "encoding": "CP852", @@ -516,32 +1069,86 @@ "hitRate": 0.1358695652173913 }, { - "encoding": "macintosh", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 4, - "total": 182, - "hitRate": 0.02197802197802198 + "encoding": "windows-1250", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 }, { - "encoding": "ISO-8859-13", - "confidence": 6, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, + "encoding": "ISO-8859-2", + "confidence": 37, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 23, + "total": 184, + "hitRate": 0.125 + }, + { + "encoding": "ISO-8859-1", + "confidence": 8, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 5, "total": 183, - "hitRate": 0.02185792349726776 + "hitRate": 0.0273224043715847 }, { - "encoding": "ISO-8859-15", - "confidence": 6, - "language": "de", + "encoding": "windows-1252", + "confidence": 8, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.536305162191, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "ISO-8859-13", + "confidence": 6, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "confidence": 6, + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 4, "total": 183, "hitRate": 0.02185792349726776 }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 183, + "hitRate": 0.01639344262295082 + }, { "encoding": "ISO-8859-3", "confidence": 4, @@ -563,7 +1170,7 @@ { "encoding": "CP850", "confidence": 4, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 184, @@ -591,11 +1198,119 @@ "encoding": "ISO-8859-16", "confidence": 3, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 2, "total": 183, "hitRate": 0.01092896174863388 }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.311255977022, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.762644549521, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.302642222102, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.005185419538, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.762644549521, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.302642222102, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.54369343373, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.549771342707, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.538051035716, + "hits": 0, + "total": 183, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -648,6 +1363,8 @@ "total": 179, "hitRate": 0.1452513966480447 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -679,15 +1396,60 @@ "total": 130, "hitRate": 0.038461538461538464 }, + { + "encoding": "windows-874", + "confidence": 10, + "language": "th", + "byteLogLikelihood": -5.027512765832, + "hits": 5, + "total": 146, + "hitRate": 0.03424657534246575 + }, + { + "encoding": "KOI8-R", + "confidence": 10, + "language": "ru", + "byteLogLikelihood": -6.090200292669, + "hits": 5, + "total": 140, + "hitRate": 0.03571428571428571 + }, + { + "encoding": "ISO-8859-7", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.19500629466, + "hits": 3, + "total": 138, + "hitRate": 0.021739130434782608 + }, + { + "encoding": "windows-1253", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.19500629466, + "hits": 3, + "total": 139, + "hitRate": 0.02158273381294964 + }, { "encoding": "x-mac-cyrillic", "confidence": 4, - "language": "ru", + "language": "uk", "byteLogLikelihood": -5.827543836069, "hits": 2, "total": 137, "hitRate": 0.014598540145985401 }, + { + "encoding": "ISO-8859-5", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.217274879676, + "hits": 1, + "total": 175, + "hitRate": 0.005714285714285714 + }, { "encoding": "ISO-8859-10", "confidence": 1, @@ -706,6 +1468,123 @@ "total": 174, "hitRate": 0.005747126436781609 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.789358524023, + "hits": 0, + "total": 139, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.901389021393, + "hits": 0, + "total": 165, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.523443710193, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.21454182173, + "hits": 0, + "total": 84, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.033184680772, + "hits": 0, + "total": 139, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.901389021393, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.690578567068, + "hits": 0, + "total": 156, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.789358524023, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.033184680772, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.21454182173, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.354655838496, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.913061957297, + "hits": 0, + "total": 149, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.237936257135, + "hits": 0, + "total": 134, + "hitRate": 0 + }, { "encoding": "macintosh", "confidence": 0, @@ -755,7 +1634,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.991915414162, + "byteLogLikelihood": -5.025462977445, "hits": 0, "total": 162, "hitRate": 0 @@ -794,6 +1673,8 @@ "total": 166, "hitRate": 0.0963855421686747 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -816,6 +1697,24 @@ "total": 137, "hitRate": 0.08759124087591241 }, + { + "encoding": "windows-874", + "confidence": 35, + "language": "th", + "byteLogLikelihood": -5.092901405888, + "hits": 15, + "total": 128, + "hitRate": 0.1171875 + }, + { + "encoding": "KOI8-R", + "confidence": 11, + "language": "ru", + "byteLogLikelihood": -6.207100144751, + "hits": 4, + "total": 104, + "hitRate": 0.038461538461538464 + }, { "encoding": "KOI8-U", "confidence": 9, @@ -828,52 +1727,196 @@ { "encoding": "x-mac-cyrillic", "confidence": 8, - "language": "ru", + "language": "uk", "byteLogLikelihood": -5.930887805719, "hits": 3, "total": 112, "hitRate": 0.026785714285714284 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-8", + "confidence": 4, + "language": "he", + "byteLogLikelihood": -5.427107080966, + "hits": 1, + "total": 64, + "hitRate": 0.015625 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -5.427107080966, + "hits": 1, + "total": 80, + "hitRate": 0.0125 + }, + { + "encoding": "ISO-8859-1", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.933799879287, + "language": "pt", + "byteLogLikelihood": -4.764166439133, "hits": 0, - "total": 73, + "total": 104, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.97248951392, + "language": "hu", + "byteLogLikelihood": -4.902465386166, "hits": 0, - "total": 99, + "total": 128, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "ISO-8859-5", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "language": "ru", + "byteLogLikelihood": -5.640443286036, "hits": 0, - "total": 131, + "total": 157, "hitRate": 0 }, { - "encoding": "ISO-8859-10", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "is", - "byteLogLikelihood": -5.076797848077, + "language": "ar", + "byteLogLikelihood": -5.592489718341, "hits": 0, - "total": 145, + "total": 84, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "lt", + "language": "el", + "byteLogLikelihood": -5.3747848425, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.028189084099, + "hits": 0, + "total": 104, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.902465386166, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.910594216272, + "hits": 0, + "total": 144, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.764166439133, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.3747848425, + "hits": 0, + "total": 107, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.028189084099, + "hits": 0, + "total": 104, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.564612100894, + "hits": 0, + "total": 108, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.902859191011, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.23021689626, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.933799879287, + "hits": 0, + "total": 73, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.97248951392, + "hits": 0, + "total": 99, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 131, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.076797848077, + "hits": 0, + "total": 145, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", "byteLogLikelihood": -4.940581981725, "hits": 0, "total": 132, @@ -901,7 +1944,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.064961155115, + "byteLogLikelihood": -5.086009867584, "hits": 0, "total": 142, "hitRate": 0 @@ -940,6 +1983,8 @@ "total": 179, "hitRate": 0.13966480446927373 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -953,10 +1998,19 @@ "total": 179, "hitRate": 0.13966480446927373 }, + { + "encoding": "KOI8-R", + "confidence": 23, + "language": "ru", + "byteLogLikelihood": -6.620542862927, + "hits": 6, + "total": 77, + "hitRate": 0.07792207792207792 + }, { "encoding": "x-mac-cyrillic", "confidence": 15, - "language": "uk", + "language": "ru", "byteLogLikelihood": -5.792699163742, "hits": 7, "total": 133, @@ -971,6 +2025,15 @@ "total": 103, "hitRate": 0.038834951456310676 }, + { + "encoding": "windows-874", + "confidence": 9, + "language": "th", + "byteLogLikelihood": -5.160775609519, + "hits": 5, + "total": 165, + "hitRate": 0.030303030303030304 + }, { "encoding": "IBM855", "confidence": 8, @@ -980,6 +2043,42 @@ "total": 169, "hitRate": 0.029585798816568046 }, + { + "encoding": "windows-1251", + "confidence": 7, + "language": "ru", + "byteLogLikelihood": -5.801879921101, + "hits": 4, + "total": 153, + "hitRate": 0.026143790849673203 + }, + { + "encoding": "windows-1253", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.523099191635, + "hits": 2, + "total": 89, + "hitRate": 0.02247191011235955 + }, + { + "encoding": "ISO-8859-5", + "confidence": 3, + "language": "ru", + "byteLogLikelihood": -5.798412758856, + "hits": 2, + "total": 174, + "hitRate": 0.011494252873563218 + }, + { + "encoding": "ISO-8859-7", + "confidence": 3, + "language": "el", + "byteLogLikelihood": -5.523099191635, + "hits": 1, + "total": 81, + "hitRate": 0.012345679012345678 + }, { "encoding": "ISO-8859-10", "confidence": 1, @@ -998,6 +2097,114 @@ "total": 169, "hitRate": 0.005917159763313609 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 81, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 145, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.82555458734, + "hits": 0, + "total": 57, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 75, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.062286993055, + "hits": 0, + "total": 81, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 117, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 82, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.062286993055, + "hits": 0, + "total": 81, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 75, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.857645252562, + "hits": 0, + "total": 82, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.928541579896, + "hits": 0, + "total": 117, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.078371422345, + "hits": 0, + "total": 76, + "hitRate": 0 + }, { "encoding": "macintosh", "confidence": 0, @@ -1047,7 +2254,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.06091613544, + "byteLogLikelihood": -4.987878406509, "hits": 0, "total": 144, "hitRate": 0 @@ -1074,460 +2281,352 @@ }, { "expected": { - "encoding": "ISO-8859-10", - "language": "is" + "encoding": "ISO-8859-1", + "language": "da" }, "predicted": { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 - }, - { - "encoding": "ISO-8859-3", - "confidence": 12, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "ISO-8859-13", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -3.989583472462, - "hits": 4, - "total": 171, - "hitRate": 0.023391812865497075 + "encoding": "windows-1252", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { "encoding": "macintosh", - "confidence": 5, - "language": "es", + "confidence": 30, + "language": "de", "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 - }, - { - "encoding": "ISO-8859-4", - "confidence": 5, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 + "hits": 12, + "total": 120, + "hitRate": 0.1 }, { "encoding": "ISO-8859-15", - "confidence": 5, - "language": "es", - "byteLogLikelihood": -4.894502014278, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 + "confidence": 30, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 }, { "encoding": "CP850", - "confidence": 5, - "language": "es", + "confidence": 30, + "language": "de", "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "ISO-8859-14", + "confidence": 15, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1257", + "confidence": 10, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -4.717529149813, + "hits": 3, + "total": 120, + "hitRate": 0.025 }, { "encoding": "CP852", "confidence": 5, "language": "pl", "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "hits": 2, + "total": 120, + "hitRate": 0.016666666666666666 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -5.176149732574, + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.658005606594, "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.026638569428, - "hits": 0, - "total": 171, - "hitRate": 0 + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.145958228046, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.838861551509, + "byteLogLikelihood": -5.825649766807, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.795369889152, + "language": "ar", + "byteLogLikelihood": -4.736335095675, "hits": 0, - "total": 169, + "total": 120, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.733983711968, + "language": "el", + "byteLogLikelihood": -4.689401143463, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.657284871846, + "language": "he", + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-13", - "language": "lt" - }, - "predicted": { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 - }, - { - "encoding": "ISO-8859-4", - "confidence": 22, - "language": "lv", - "byteLogLikelihood": -4.610398759565, - "hits": 13, - "total": 173, - "hitRate": 0.07514450867052024 - }, - { - "encoding": "macintosh", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 168, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-15", - "confidence": 12, - "language": "fr", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 - }, - { - "encoding": "CP850", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 170, - "hitRate": 0.041176470588235294 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 172, - "hitRate": 0.01744186046511628 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.6272413408, - "hits": 2, - "total": 173, - "hitRate": 0.011560693641618497 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "IBM855", - "confidence": 1, - "language": "sr", - "byteLogLikelihood": -5.488073838002, - "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.861023356308, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -4.416382325607, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.888170123788, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.659155714377, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 120, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.252539473952, + "byteLogLikelihood": -6.476992253447, "hits": 0, - "total": 173, + "total": 120, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.893109334109, + "byteLogLikelihood": -5.825649766807, "hits": 0, - "total": 171, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.685745819823, + "hits": 0, + "total": 120, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.441328843196, + "language": "ru", + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 173, + "total": 120, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-14", - "language": "cy" + "encoding": "ISO-8859-1", + "language": "de" }, "predicted": { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 3, "candidates": [ { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 - }, - { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 - }, - { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "fr", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 189, - "hitRate": 0.037037037037037035 - }, - { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 - }, - { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -4.009500374258, - "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "CP852", - "confidence": 3, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 188, - "hitRate": 0.010638297872340425 - }, - { - "encoding": "ISO-8859-4", - "confidence": 1, - "language": "lv", - "byteLogLikelihood": -4.961907790704, - "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 - }, - { - "encoding": "ISO-8859-13", - "confidence": 1, - "language": "lt", - "byteLogLikelihood": -3.647025046455, - "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.821976831991, - "hits": 0, - "total": 189, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.350299845595, - "hits": 0, - "total": 188, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.325879646284, - "hits": 0, - "total": 188, - "hitRate": 0 + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.255815902698, - "hits": 0, - "total": 189, - "hitRate": 0 + "encoding": "windows-1252", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -3.981643419755, - "hits": 0, - "total": 189, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-15", - "language": "de" - }, - "predicted": { - "encoding": "ISO-8859-15", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ { "encoding": "ISO-8859-15", "confidence": 73, @@ -1540,7 +2639,7 @@ { "encoding": "macintosh", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -1549,7 +2648,7 @@ { "encoding": "CP850", "confidence": 70, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -1565,7 +2664,16 @@ "hitRate": 0.06944444444444445 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "windows-1250", "confidence": 13, "language": "ro", "byteLogLikelihood": -5.176149732574, @@ -1573,6 +2681,42 @@ "total": 216, "hitRate": 0.046296296296296294 }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 216, + "hitRate": 0.037037037037037035 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, { "encoding": "ISO-8859-4", "confidence": 9, @@ -1591,6 +2735,15 @@ "total": 216, "hitRate": 0.023148148148148147 }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, { "encoding": "ISO-8859-3", "confidence": 5, @@ -1619,634 +2772,16260 @@ "hitRate": 0.004629629629629629 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-5", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.245943193261, + "language": "ru", + "byteLogLikelihood": -6.162408104115, "hits": 0, "total": 216, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "language": "ar", + "byteLogLikelihood": -4.007313397456, "hits": 0, "total": 216, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.942998477092, + "language": "el", + "byteLogLikelihood": -4.349754587225, "hits": 0, "total": 216, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.629332579687, + "language": "he", + "byteLogLikelihood": -4.279537403415, "hits": 0, "total": 216, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-15", - "language": "es" - }, - "predicted": { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 }, { - "encoding": "macintosh", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.542402054434, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 - }, - { - "encoding": "ISO-8859-4", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.009854802779, - "hits": 16, - "total": 200, - "hitRate": 0.08 - }, - { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -4.892914919217, - "hits": 13, - "total": 200, - "hitRate": 0.065 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 200, - "hitRate": 0.055 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.657121063601, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -3.846028832089, - "hits": 3, - "total": 200, - "hitRate": 0.015 + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.28774788718, + "hits": 0, + "total": 216, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -6.245943193261, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.977292083619, + "byteLogLikelihood": -6.692083742507, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.203740544852, + "language": "sr", + "byteLogLikelihood": -5.942998477092, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "uk", + "byteLogLikelihood": -3.629332579687, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", - "language": "fr" + "encoding": "ISO-8859-1", + "language": "en" }, "predicted": { "encoding": "ISO-8859-15", - "confidence": 77, - "language": "fr", + "confidence": 29, + "language": "en", "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 }, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { "encoding": "ISO-8859-15", - "confidence": 77, + "confidence": 29, "language": "fr", "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "en", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "en", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, { "encoding": "macintosh", - "confidence": 71, - "language": "es", + "confidence": 27, + "language": "fr", "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 }, { "encoding": "CP850", - "confidence": 71, - "language": "es", + "confidence": 27, + "language": "fr", "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1257", "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 }, { - "encoding": "ISO-8859-10", - "confidence": 10, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 + "encoding": "ISO-8859-14", + "confidence": 11, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "cs", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1250", + "confidence": 9, + "language": "cs", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1258", + "confidence": 9, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "ISO-8859-16", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { "encoding": "ISO-8859-4", - "confidence": 7, + "confidence": 6, "language": "lv", "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "hits": 3, + "total": 132, + "hitRate": 0.022727272727272728 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 }, { "encoding": "ISO-8859-3", - "confidence": 5, + "confidence": 4, "language": "mt", "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 }, { "encoding": "ISO-8859-13", - "confidence": 5, + "confidence": 2, "language": "lt", "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 + "hits": 1, + "total": 132, + "hitRate": 0.007575757575757576 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 228, - "hitRate": 0.013157894736842105 + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "ar", + "byteLogLikelihood": -6.113682179832, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, + "language": "el", + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, + "language": "he", + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1251", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, + "language": "ru", + "byteLogLikelihood": -4.612642200827, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-16", - "language": "ro" - }, - "predicted": { - "encoding": "ISO-8859-16", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-16", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - { - "encoding": "macintosh", - "confidence": 19, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 198, - "hitRate": 0.06565656565656566 - }, - { - "encoding": "CP850", - "confidence": 15, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "ISO-8859-15", - "confidence": 14, - "language": "fr", - "byteLogLikelihood": -4.903997266318, - "hits": 10, - "total": 202, - "hitRate": 0.04950495049504951 - }, - { - "encoding": "ISO-8859-3", - "confidence": 13, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 9, - "total": 200, - "hitRate": 0.045 }, { - "encoding": "CP852", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 201, - "hitRate": 0.029850746268656716 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -4.936633427031, - "hits": 5, - "total": 202, - "hitRate": 0.024752475247524754 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 5, - "language": "is", - "byteLogLikelihood": -5.150687673188, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.974278583322, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 131, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.539421788511, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 132, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.32130473174, + "byteLogLikelihood": -6.650279048587, "hits": 0, - "total": 201, + "total": 132, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.208609169552, + "byteLogLikelihood": -5.593471453839, "hits": 0, - "total": 201, + "total": 132, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.608648280371, + "language": "sr", + "byteLogLikelihood": -4.728861658331, "hits": 0, - "total": 201, + "total": 132, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, + "language": "uk", + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 202, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 0, + "total": 132, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-3", - "language": "mt" + "encoding": "ISO-8859-1", + "language": "es" }, "predicted": { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 3, "candidates": [ { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { - "encoding": "CP852", - "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 8, - "total": 177, - "hitRate": 0.04519774011299435 + "encoding": "windows-1252", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { "encoding": "macintosh", - "confidence": 11, + "confidence": 58, "language": "es", "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 16, + "total": 200, + "hitRate": 0.08 }, { "encoding": "ISO-8859-4", - "confidence": 11, + "confidence": 24, "language": "lv", - "byteLogLikelihood": -4.945351018208, - "hits": 7, - "total": 183, - "hitRate": 0.03825136612021858 + "byteLogLikelihood": -5.009854802779, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -4.892914919217, + "hits": 13, + "total": 200, + "hitRate": 0.065 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 200, + "hitRate": 0.055 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -4.848531508941, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -3.846028832089, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.346153815501, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.412820202576, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.203740544852, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "macintosh", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 228, + "hitRate": 0.013157894736842105 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "it" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "macintosh", + "confidence": 24, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 158, + "hitRate": 0.08227848101265822 + }, + { + "encoding": "ISO-8859-15", + "confidence": 24, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "CP850", + "confidence": 24, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "windows-1258", + "confidence": 16, + "language": "vi", + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "ISO-8859-16", + "confidence": 16, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1257", + "confidence": 15, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-4", + "confidence": 15, + "language": "lv", + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-14", + "confidence": 9, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 159, + "hitRate": 0.031446540880503145 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 159, + "hitRate": 0.025157232704402517 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -6.520621127559, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "nl", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "nl", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "windows-1252", + "confidence": 83, + "language": "nl", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "macintosh", + "confidence": 63, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-15", + "confidence": 63, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "CP850", + "confidence": 63, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-14", + "confidence": 33, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 17, + "total": 151, + "hitRate": 0.11258278145695365 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-9", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1254", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-10", + "confidence": 11, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 151, + "hitRate": 0.033112582781456956 + }, + { + "encoding": "windows-1258", + "confidence": 3, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-3", + "confidence": 1, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-4", + "confidence": 1, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "CP852", + "confidence": 1, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 151, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "no" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "macintosh", + "confidence": 18, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-14", + "confidence": 18, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-15", + "confidence": 18, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "CP850", + "confidence": 18, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-10", + "confidence": 13, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 5, + "total": 114, + "hitRate": 0.043859649122807015 + }, + { + "encoding": "windows-1258", + "confidence": 10, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-2", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1250", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.319245023186, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.641462690249, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.679407897139, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "pt", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "es", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "pt", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "pt", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1257", + "confidence": 29, + "language": "lt", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "ISO-8859-13", + "confidence": 29, + "language": "lt", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "macintosh", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "CP850", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "ISO-8859-4", + "confidence": 23, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 11, + "total": 143, + "hitRate": 0.07692307692307693 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "ISO-8859-14", + "confidence": 10, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -4.524213150766, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-10", + "confidence": 6, + "language": "is", + "byteLogLikelihood": -4.06291963921, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.514449766028, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.846220145127, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.503504267293, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.91521504926, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.476992253447, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -3.774998078834, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.766179593299, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 143, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "sv" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1258", + "confidence": 19, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "ISO-8859-10", + "confidence": 19, + "language": "is", + "byteLogLikelihood": -4.980036753423, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "macintosh", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP850", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "ISO-8859-9", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "windows-1254", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "ISO-8859-16", + "confidence": 6, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 136, + "hitRate": 0.022058823529411766 + }, + { + "encoding": "windows-1257", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.287368473007, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.25594643617, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.801526726218, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.432153549797, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.390348855877, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.419477704429, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-10", + "language": "is" + }, + "predicted": { + "encoding": "ISO-8859-10", + "confidence": 38, + "language": "is", + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-10", + "confidence": 38, + "language": "is", + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "sv", + "byteLogLikelihood": -4.633646206449, + "hits": 8, + "total": 171, + "hitRate": 0.04678362573099415 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "sv", + "byteLogLikelihood": -4.633646206449, + "hits": 8, + "total": 171, + "hitRate": 0.04678362573099415 + }, + { + "encoding": "ISO-8859-3", + "confidence": 12, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 7, + "total": 169, + "hitRate": 0.04142011834319527 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.642697796053, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "ISO-8859-13", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "macintosh", + "confidence": 5, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "ISO-8859-4", + "confidence": 5, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 3, + "total": 171, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-15", + "confidence": 5, + "language": "de", + "byteLogLikelihood": -4.894502014278, + "hits": 3, + "total": 171, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "CP850", + "confidence": 5, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "ISO-8859-2", + "confidence": 3, + "language": "cs", + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 + }, + { + "encoding": "windows-1250", + "confidence": 3, + "language": "cs", + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 + }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -4.646338703133, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -4.646338703133, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.838861551509, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.345124108708, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.963723644165, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.249634019938, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.84046735452, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.963723644165, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.249634019938, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.696464660765, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.644763656357, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.290788006393, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.026638569428, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.838861551509, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.795369889152, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.733983711968, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.657284871846, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 0, + "total": 171, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-13", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-13", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-4", + "confidence": 22, + "language": "lv", + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "ISO-8859-1", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "macintosh", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 168, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "CP850", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 170, + "hitRate": 0.041176470588235294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "windows-1254", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 172, + "hitRate": 0.01744186046511628 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.6272413408, + "hits": 2, + "total": 173, + "hitRate": 0.011560693641618497 + }, + { + "encoding": "windows-1258", + "confidence": 1, + "language": "vi", + "byteLogLikelihood": -4.960186085121, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "IBM855", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.488073838002, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.893109334109, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.773457169107, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.531827362419, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.924336543075, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.58967630288, + "hits": 0, + "total": 166, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.365896834008, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.252539473952, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.893109334109, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.441328843196, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.616838329205, + "hits": 0, + "total": 173, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-14", + "language": "cy" + }, + "predicted": { + "encoding": "ISO-8859-14", + "confidence": 50, + "language": "cy", + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-14", + "confidence": 50, + "language": "cy", + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "pt", + "byteLogLikelihood": -4.955827057601, + "hits": 9, + "total": 189, + "hitRate": 0.047619047619047616 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "pt", + "byteLogLikelihood": -4.955827057601, + "hits": 9, + "total": 189, + "hitRate": 0.047619047619047616 + }, + { + "encoding": "windows-1258", + "confidence": 14, + "language": "vi", + "byteLogLikelihood": -4.34192292436, + "hits": 9, + "total": 188, + "hitRate": 0.047872340425531915 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 188, + "hitRate": 0.03723404255319149 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 189, + "hitRate": 0.037037037037037035 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 188, + "hitRate": 0.03723404255319149 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "cs", + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1250", + "confidence": 9, + "language": "cs", + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 189, + "hitRate": 0.021164021164021163 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 189, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -4.009500374258, + "hits": 3, + "total": 189, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-9", + "confidence": 3, + "language": "tr", + "byteLogLikelihood": -4.613075694861, + "hits": 2, + "total": 189, + "hitRate": 0.010582010582010581 + }, + { + "encoding": "windows-1254", + "confidence": 3, + "language": "tr", + "byteLogLikelihood": -4.613075694861, + "hits": 2, + "total": 189, + "hitRate": 0.010582010582010581 + }, + { + "encoding": "CP852", + "confidence": 3, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 188, + "hitRate": 0.010638297872340425 + }, + { + "encoding": "ISO-8859-4", + "confidence": 1, + "language": "lv", + "byteLogLikelihood": -4.961907790704, + "hits": 1, + "total": 189, + "hitRate": 0.005291005291005291 + }, + { + "encoding": "ISO-8859-13", + "confidence": 1, + "language": "lt", + "byteLogLikelihood": -3.647025046455, + "hits": 1, + "total": 189, + "hitRate": 0.005291005291005291 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.350299845595, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.420534999272, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.254156765951, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.917404374874, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.467895804539, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.254156765951, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.917404374874, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.830407858741, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.229985622133, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -5.821976831991, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.350299845595, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.325879646284, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.255815902698, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -3.903274585551, + "hits": 0, + "total": 189, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "macintosh", + "confidence": 72, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "CP850", + "confidence": 70, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "ISO-8859-14", + "confidence": 20, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 15, + "total": 216, + "hitRate": 0.06944444444444445 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 216, + "hitRate": 0.037037037037037035 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 216, + "hitRate": 0.023148148148148147 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 216, + "hitRate": 0.009259259259259259 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 1, + "total": 216, + "hitRate": 0.004629629629629629 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.162408104115, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.542402054434, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.657121063601, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.28774788718, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.245943193261, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.942998477092, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "macintosh", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-4", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.009854802779, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -4.892914919217, + "hits": 13, + "total": 200, + "hitRate": 0.065 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 200, + "hitRate": 0.055 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -4.848531508941, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -3.846028832089, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.346153815501, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.412820202576, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.203740544852, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "macintosh", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 228, + "hitRate": 0.013157894736842105 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-16", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-16", + "confidence": 53, + "language": "ro", + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-16", + "confidence": 53, + "language": "ro", + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 + }, + { + "encoding": "ISO-8859-2", + "confidence": 44, + "language": "ro", + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 + }, + { + "encoding": "windows-1250", + "confidence": 44, + "language": "ro", + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 + }, + { + "encoding": "ISO-8859-1", + "confidence": 26, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 + }, + { + "encoding": "windows-1252", + "confidence": 26, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 + }, + { + "encoding": "macintosh", + "confidence": 22, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 16, + "total": 210, + "hitRate": 0.0761904761904762 + }, + { + "encoding": "CP850", + "confidence": 18, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 213, + "hitRate": 0.06103286384976526 + }, + { + "encoding": "ISO-8859-15", + "confidence": 16, + "language": "fr", + "byteLogLikelihood": -4.918215670022, + "hits": 12, + "total": 214, + "hitRate": 0.056074766355140186 + }, + { + "encoding": "ISO-8859-3", + "confidence": 12, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 9, + "total": 212, + "hitRate": 0.04245283018867924 + }, + { + "encoding": "ISO-8859-9", + "confidence": 9, + "language": "tr", + "byteLogLikelihood": -4.982754191159, + "hits": 7, + "total": 214, + "hitRate": 0.03271028037383177 + }, + { + "encoding": "windows-1254", + "confidence": 9, + "language": "tr", + "byteLogLikelihood": -4.982754191159, + "hits": 7, + "total": 214, + "hitRate": 0.03271028037383177 + }, + { + "encoding": "CP852", + "confidence": 9, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 215, + "hitRate": 0.03255813953488372 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.456762061842, + "hits": 6, + "total": 214, + "hitRate": 0.028037383177570093 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -4.924600071508, + "hits": 5, + "total": 214, + "hitRate": 0.02336448598130841 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.219909353069, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 + }, + { + "encoding": "ISO-8859-10", + "confidence": 5, + "language": "is", + "byteLogLikelihood": -5.111587063002, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -4.864509292863, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.542976389437, + "hits": 3, + "total": 216, + "hitRate": 0.013888888888888888 + }, + { + "encoding": "windows-874", + "confidence": 1, + "language": "th", + "byteLogLikelihood": -5.210426263201, + "hits": 1, + "total": 214, + "hitRate": 0.004672897196261682 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.539739112612, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.000268888635, + "hits": 0, + "total": 211, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.499845100591, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.429071192897, + "hits": 0, + "total": 213, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.382241948417, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.499845100591, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.429071192897, + "hits": 0, + "total": 213, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.739610656221, + "hits": 0, + "total": 213, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.414824870283, + "hits": 0, + "total": 213, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.199507530667, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.333669615722, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.794020051248, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.382241948417, + "hits": 0, + "total": 214, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "windows-1250", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "confidence": 11, + "language": "pt", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "confidence": 11, + "language": "pt", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "CP852", + "confidence": 11, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.506724065575, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "macintosh", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 124, + "hitRate": 0.024193548387096774 + }, + { + "encoding": "ISO-8859-13", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-15", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-16", + "confidence": 7, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "CP850", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 125, + "hitRate": 0.024 + }, + { + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -5.129943470839, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.402405269754, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.249199684222, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.164526575273, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.504431122267, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.566056982405, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.524252288486, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.108905030169, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.226561011787, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP852", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 135, + "hitRate": 0.05185185185185185 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 135, + "hitRate": 0.022222222222222223 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 134, + "hitRate": 0.022388059701492536 + }, + { + "encoding": "ISO-8859-15", + "confidence": 4, + "language": "fr", + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1257", + "confidence": 2, + "language": "et", + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "confidence": 2, + "language": "is", + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.950677184263, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.538035906584, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.6300671031, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.009772962617, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.010690688029, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.953752719217, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.606957349802, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.100966544791, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "pl" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "windows-1250", + "confidence": 37, + "language": "pl", + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 183, + "hitRate": 0.12568306010928962 + }, + { + "encoding": "CP852", + "confidence": 41, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 25, + "total": 182, + "hitRate": 0.13736263736263737 + }, + { + "encoding": "ISO-8859-1", + "confidence": 8, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "confidence": 8, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1258", + "confidence": 8, + "language": "vi", + "byteLogLikelihood": -5.343135518502, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-13", + "confidence": 6, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -4.794788079951, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-16", + "confidence": 3, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 183, + "hitRate": 0.01092896174863388 + }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "ISO-8859-3", + "confidence": 1, + "language": "mt", + "byteLogLikelihood": -4.476342508555, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -4.71573455878, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-14", + "confidence": 1, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.293914971567, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.002422325308, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.300217461931, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.002422325308, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.586271794593, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.538051035716, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.496246341796, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.447947678358, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.497577429688, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.300217461931, + "hits": 0, + "total": 183, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "confidence": 38, + "language": "ro", + "byteLogLikelihood": -2.700458868498, + "hits": 26, + "total": 202, + "hitRate": 0.12871287128712872 + }, + { + "encoding": "ISO-8859-1", + "confidence": 26, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "confidence": 26, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "macintosh", + "confidence": 19, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 198, + "hitRate": 0.06565656565656566 + }, + { + "encoding": "CP850", + "confidence": 15, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.903997266318, + "hits": 10, + "total": 202, + "hitRate": 0.04950495049504951 + }, + { + "encoding": "ISO-8859-3", + "confidence": 13, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 9, + "total": 200, + "hitRate": 0.045 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.396820338954, + "hits": 6, + "total": 202, + "hitRate": 0.0297029702970297 + }, + { + "encoding": "CP852", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 201, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -4.936633427031, + "hits": 5, + "total": 202, + "hitRate": 0.024752475247524754 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.177254141958, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-10", + "confidence": 5, + "language": "is", + "byteLogLikelihood": -5.150687673188, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.974278583322, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.539421788511, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.828974536143, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.528215037204, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.239455511679, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.32130473174, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.608648280371, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-3", + "language": "mt" + }, + "predicted": { + "encoding": "ISO-8859-3", + "confidence": 52, + "language": "mt", + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-3", + "confidence": 52, + "language": "mt", + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 + }, + { + "encoding": "ISO-8859-9", + "confidence": 16, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 181, + "hitRate": 0.055248618784530384 + }, + { + "encoding": "windows-1254", + "confidence": 16, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 181, + "hitRate": 0.055248618784530384 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "it", + "byteLogLikelihood": -4.81334289805, + "hits": 9, + "total": 181, + "hitRate": 0.049723756906077346 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "it", + "byteLogLikelihood": -4.81334289805, + "hits": 9, + "total": 181, + "hitRate": 0.049723756906077346 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -4.980958307454, + "hits": 8, + "total": 182, + "hitRate": 0.04395604395604396 + }, + { + "encoding": "windows-1258", + "confidence": 13, + "language": "vi", + "byteLogLikelihood": -4.506236351586, + "hits": 8, + "total": 181, + "hitRate": 0.04419889502762431 + }, + { + "encoding": "CP852", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 8, + "total": 177, + "hitRate": 0.04519774011299435 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -4.945351018208, + "hits": 7, + "total": 183, + "hitRate": 0.03825136612021858 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 177, + "hitRate": 0.03954802259887006 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "pl", + "byteLogLikelihood": -3.834078315244, + "hits": 6, + "total": 183, + "hitRate": 0.03278688524590164 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 182, + "hitRate": 0.03296703296703297 + }, + { + "encoding": "ISO-8859-14", + "confidence": 6, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.986136327924, + "hits": 3, + "total": 182, + "hitRate": 0.016483516483516484 + }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 182, + "hitRate": 0.016483516483516484 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 2, + "total": 183, + "hitRate": 0.01092896174863388 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.351251681057, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.198030006336, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.755123257218, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.572384450833, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.198030006336, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.755123257218, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.490920024976, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.383683979141, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.404570694946, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.943656762101, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.572384450833, + "hits": 0, + "total": 182, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-4", + "language": "lv" + }, + "predicted": { + "encoding": "ISO-8859-4", + "confidence": 40, + "language": "lv", + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-4", + "confidence": 40, + "language": "lv", + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + { + "encoding": "windows-1257", + "confidence": 31, + "language": "lv", + "byteLogLikelihood": -5.111410123766, + "hits": 18, + "total": 169, + "hitRate": 0.10650887573964497 + }, + { + "encoding": "CP850", + "confidence": 16, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 9, + "total": 168, + "hitRate": 0.05357142857142857 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "macintosh", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "CP852", + "confidence": 12, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 169, + "hitRate": 0.04142011834319527 + }, + { + "encoding": "ISO-8859-13", + "confidence": 10, + "language": "lt", + "byteLogLikelihood": -4.596975900219, + "hits": 6, + "total": 169, + "hitRate": 0.03550295857988166 + }, + { + "encoding": "ISO-8859-2", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -4.763992911083, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "windows-1250", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -4.846752537455, + "hits": 5, + "total": 169, + "hitRate": 0.029585798816568046 + }, + { + "encoding": "ISO-8859-3", + "confidence": 8, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 169, + "hitRate": 0.023668639053254437 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 169, + "hitRate": 0.023668639053254437 + }, + { + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -5.074758182528, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -4.725539964734, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.226735250772, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.427610697135, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.928827369245, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.704365179493, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.427610697135, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.928827369245, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.578946937177, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.610537015382, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.609505685025, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.267508614334, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.182018138172, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.65535318117, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.616838329205, + "hits": 0, + "total": 169, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-5", + "language": "ru" + }, + "predicted": { + "encoding": "ISO-8859-5", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-5", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + { + "encoding": "IBM866", + "confidence": 20, + "language": "ru", + "byteLogLikelihood": -5.798412758856, + "hits": 5, + "total": 75, + "hitRate": 0.06666666666666667 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 12, + "language": "uk", + "byteLogLikelihood": -5.66890403251, + "hits": 6, + "total": 147, + "hitRate": 0.04081632653061224 + }, + { + "encoding": "IBM855", + "confidence": 9, + "language": "ru", + "byteLogLikelihood": -4.826799236728, + "hits": 5, + "total": 166, + "hitRate": 0.030120481927710843 + }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -4.986561094704, + "hits": 1, + "total": 176, + "hitRate": 0.005681818181818182 + }, + { + "encoding": "KOI8-U", + "confidence": 1, + "language": "uk", + "byteLogLikelihood": -4.816522715875, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.219340428659, + "hits": 0, + "total": 152, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.141186484933, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 75, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.982255634278, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.667544038277, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.141186484933, + "hits": 0, + "total": 174, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.982255634278, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 113, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.847671496604, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.755485363394, + "hits": 0, + "total": 166, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.810914106722, + "hits": 0, + "total": 84, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977706457443, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 98, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.966015130424, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.103902669993, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.847671496604, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.83002806951, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.848242748706, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.105122227945, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 162, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.078415577874, + "hits": 0, + "total": 166, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-6", + "language": "ar" + }, + "predicted": { + "encoding": "ISO-8859-6", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-6", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + { + "encoding": "windows-1256", + "confidence": 10, + "language": "ar", + "byteLogLikelihood": -3.968382111451, + "hits": 4, + "total": 110, + "hitRate": 0.03636363636363636 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 10, + "language": "ru", + "byteLogLikelihood": -5.359123221706, + "hits": 3, + "total": 82, + "hitRate": 0.036585365853658534 + }, + { + "encoding": "ISO-8859-8", + "confidence": 8, + "language": "he", + "byteLogLikelihood": -5.049228744152, + "hits": 2, + "total": 70, + "hitRate": 0.02857142857142857 + }, + { + "encoding": "windows-1255", + "confidence": 8, + "language": "he", + "byteLogLikelihood": -5.049228744152, + "hits": 2, + "total": 72, + "hitRate": 0.027777777777777776 + }, + { + "encoding": "ISO-8859-7", + "confidence": 5, + "language": "el", + "byteLogLikelihood": -4.984009642853, + "hits": 2, + "total": 109, + "hitRate": 0.01834862385321101 + }, + { + "encoding": "windows-1253", + "confidence": 5, + "language": "el", + "byteLogLikelihood": -4.984009642853, + "hits": 2, + "total": 109, + "hitRate": 0.01834862385321101 + }, + { + "encoding": "KOI8-R", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.040654549299, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "KOI8-U", + "confidence": 2, + "language": "uk", + "byteLogLikelihood": -5.174167309306, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.69584393439, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -4.951980582514, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.534213772279, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.933475282212, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -4.951980582514, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.223507468988, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.69584393439, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.933475282212, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.72024182268, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029794115635, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.643135408712, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.068419445019, + "hits": 0, + "total": 72, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.811360688713, + "hits": 0, + "total": 100, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 55, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.047518534687, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.155285808468, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.051594021628, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.063690311528, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.799641758471, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "de", + "byteLogLikelihood": -4.755317425962, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.160082157204, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 98, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 100, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-7", + "language": "el" + }, + "predicted": { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "windows-1253", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "KOI8-U", + "confidence": 8, + "language": "uk", + "byteLogLikelihood": -6.254780178237, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "IBM855", + "confidence": 7, + "language": "ru", + "byteLogLikelihood": -5.358823187498, + "hits": 3, + "total": 118, + "hitRate": 0.025423728813559324 + }, + { + "encoding": "KOI8-R", + "confidence": 4, + "language": "ru", + "byteLogLikelihood": -6.302671202821, + "hits": 2, + "total": 134, + "hitRate": 0.014925373134328358 + }, + { + "encoding": "ISO-8859-6", + "confidence": 3, + "language": "ar", + "byteLogLikelihood": -5.268243094623, + "hits": 1, + "total": 81, + "hitRate": 0.012345679012345678 + }, + { + "encoding": "ISO-8859-8", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "windows-1255", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 2, + "language": "uk", + "byteLogLikelihood": -4.203232324163, + "hits": 1, + "total": 131, + "hitRate": 0.007633587786259542 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.165809265202, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288697198304, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.013410783916, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.827626146169, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.681848366371, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.483358374511, + "hits": 0, + "total": 52, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.55248803416, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 116, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.945729021813, + "hits": 0, + "total": 129, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.015268979336, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.576407109739, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.890771320266, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.788878603493, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.603974037796, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.137560045078, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 96, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-8", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "windows-1255", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "ISO-8859-7", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "IBM855", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.394495827364, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.124491051534, + "hits": 0, + "total": 73, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.199203901043, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.32364067176, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.639272141503, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029681805165, + "hits": 0, + "total": 99, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.31385343816, + "hits": 0, + "total": 48, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.40669720385, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.393692981541, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 72, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.134202940863, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.956632184798, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.146385148445, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.745792460039, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.842271132243, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.813633859942, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "fr", + "byteLogLikelihood": -4.521941300627, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.093070766591, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 85, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 89, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-9", + "language": "tr" + }, + "predicted": { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "ISO-8859-3", + "confidence": 22, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 10, + "total": 134, + "hitRate": 0.07462686567164178 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "da", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "da", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-10", + "confidence": 17, + "language": "is", + "byteLogLikelihood": -4.720106560713, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -4.93302665818, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-13", + "confidence": 11, + "language": "lt", + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.652724693026, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.197431316193, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.576073075491, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.714883328648, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.085579959517, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.742947096814, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.22610845571, + "hits": 0, + "total": 134, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-R", + "language": "ru" + }, + "predicted": { + "encoding": "KOI8-R", + "confidence": 43, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-R", + "confidence": 43, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + { + "encoding": "KOI8-U", + "confidence": 28, + "language": "uk", + "byteLogLikelihood": -3.443215105753, + "hits": 17, + "total": 179, + "hitRate": 0.09497206703910614 + }, + { + "encoding": "windows-874", + "confidence": 12, + "language": "th", + "byteLogLikelihood": -4.705552828452, + "hits": 7, + "total": 162, + "hitRate": 0.043209876543209874 + }, + { + "encoding": "ISO-8859-7", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.840779048091, + "hits": 4, + "total": 177, + "hitRate": 0.022598870056497175 + }, + { + "encoding": "windows-1253", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.840779048091, + "hits": 4, + "total": 177, + "hitRate": 0.022598870056497175 + }, + { + "encoding": "ISO-8859-5", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.543217452567, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.150827322214, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "nl", + "byteLogLikelihood": -4.900684399759, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.759099059366, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.899711049132, + "hits": 0, + "total": 7, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.056032280413, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.313908864785, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "nl", + "byteLogLikelihood": -4.900684399759, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.056032280413, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.899711049132, + "hits": 0, + "total": 27, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.796335743252, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.91533239831, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.174619785116, + "hits": 0, + "total": 174, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.670016162969, + "hits": 0, + "total": 7, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.983474020054, + "hits": 0, + "total": 96, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.595875032476, + "hits": 0, + "total": 116, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.182298321669, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.105602464686, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "de", + "byteLogLikelihood": -4.935090625757, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.206584689041, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 94, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-U", + "language": "uk" + }, + "predicted": { + "encoding": "KOI8-U", + "confidence": 32, + "language": "uk", + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-U", + "confidence": 32, + "language": "uk", + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 + }, + { + "encoding": "KOI8-R", + "confidence": 17, + "language": "ru", + "byteLogLikelihood": -3.510012695801, + "hits": 10, + "total": 172, + "hitRate": 0.05813953488372093 + }, + { + "encoding": "windows-874", + "confidence": 11, + "language": "th", + "byteLogLikelihood": -4.843281209678, + "hits": 6, + "total": 163, + "hitRate": 0.03680981595092025 + }, + { + "encoding": "ISO-8859-7", + "confidence": 8, + "language": "el", + "byteLogLikelihood": -5.876007286158, + "hits": 5, + "total": 168, + "hitRate": 0.02976190476190476 + }, + { + "encoding": "windows-1253", + "confidence": 8, + "language": "el", + "byteLogLikelihood": -5.876007286158, + "hits": 5, + "total": 168, + "hitRate": 0.02976190476190476 + }, + { + "encoding": "ISO-8859-1", + "confidence": 1, + "language": "en", + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "ISO-8859-5", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.626926512473, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 + }, + { + "encoding": "ISO-8859-6", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -4.726706027838, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "windows-1252", + "confidence": 1, + "language": "en", + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -4.693044688745, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.128376292359, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.883441366246, + "hits": 0, + "total": 7, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.055144062354, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.295907791508, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.055144062354, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.883441366246, + "hits": 0, + "total": 27, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.922686041886, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.213223244884, + "hits": 0, + "total": 164, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.582632026409, + "hits": 0, + "total": 22, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.922515418323, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 89, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.589678128375, + "hits": 0, + "total": 117, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.110283442945, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.944199711559, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.192994752161, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 94, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "de" + }, + "predicted": { + "encoding": "macintosh", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-1", + "confidence": 74, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "ISO-8859-15", + "confidence": 74, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "windows-1252", + "confidence": 70, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "CP850", + "confidence": 69, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 50, + "total": 216, + "hitRate": 0.23148148148148148 + }, + { + "encoding": "ISO-8859-14", + "confidence": 22, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 16, + "total": 214, + "hitRate": 0.07476635514018691 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 214, + "hitRate": 0.0514018691588785 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 216, + "hitRate": 0.05092592592592592 + }, + { + "encoding": "ISO-8859-9", + "confidence": 14, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 214, + "hitRate": 0.04672897196261682 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-16", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 214, + "hitRate": 0.04205607476635514 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 214, + "hitRate": 0.037383177570093455 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 214, + "hitRate": 0.037383177570093455 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 3, + "total": 214, + "hitRate": 0.014018691588785047 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 214, + "hitRate": 0.009345794392523364 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.661807706795, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.534754518494, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.345510152227, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -6.337519079447, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "es" + }, + "predicted": { + "encoding": "macintosh", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "ISO-8859-15", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 25, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, + { + "encoding": "ISO-8859-4", + "confidence": 25, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 13, + "total": 199, + "hitRate": 0.06532663316582915 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 199, + "hitRate": 0.05527638190954774 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 3, + "total": 199, + "hitRate": 0.01507537688442211 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.624741392945, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 199, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "fr" + }, + "predicted": { + "encoding": "macintosh", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-1", + "confidence": 75, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "ISO-8859-15", + "confidence": 75, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "windows-1252", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "ISO-8859-10", + "confidence": 12, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "ISO-8859-16", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 224, + "hitRate": 0.03125 + }, + { + "encoding": "ISO-8859-3", + "confidence": 8, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 6, + "total": 224, + "hitRate": 0.026785714285714284 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1254", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 224, + "hitRate": 0.017857142857142856 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 224, + "hitRate": 0.013392857142857142 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.998936561947, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -3.576182148392, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.998936561947, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "windows-1250", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "confidence": 11, + "language": "pt", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "confidence": 11, + "language": "pt", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "CP852", + "confidence": 11, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.506724065575, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "macintosh", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 124, + "hitRate": 0.024193548387096774 + }, + { + "encoding": "ISO-8859-13", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-15", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-16", + "confidence": 7, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "CP850", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 125, + "hitRate": 0.024 + }, + { + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -5.129943470839, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.402405269754, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.249199684222, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.164526575273, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.504431122267, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.566056982405, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.524252288486, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.108905030169, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.226561011787, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP852", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 135, + "hitRate": 0.05185185185185185 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 135, + "hitRate": 0.022222222222222223 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 134, + "hitRate": 0.022388059701492536 + }, + { + "encoding": "ISO-8859-15", + "confidence": 4, + "language": "fr", + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1257", + "confidence": 2, + "language": "et", + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "confidence": 2, + "language": "is", + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.950677184263, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.538035906584, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.6300671031, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.009772962617, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.010690688029, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.953752719217, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.606957349802, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.100966544791, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "pl" + }, + "predicted": { + "encoding": "windows-1250", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1250", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "ISO-8859-2", + "confidence": 37, + "language": "pl", + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 184, + "hitRate": 0.125 + }, + { + "encoding": "CP852", + "confidence": 41, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 25, + "total": 182, + "hitRate": 0.13736263736263737 + }, + { + "encoding": "ISO-8859-1", + "confidence": 8, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "confidence": 8, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.343135518502, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-13", + "confidence": 6, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "macintosh", + "confidence": 4, + "language": "es", + "byteLogLikelihood": -4.826691858304, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -4.704684722593, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-3", + "confidence": 3, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -4.71573455878, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-16", + "confidence": 3, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "ISO-8859-14", + "confidence": 1, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.293914971567, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.848389618517, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.300217461931, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.848389618517, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.163001081631, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.538051035716, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.496246341796, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.447947678358, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.204348837508, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.955485669412, + "hits": 0, + "total": 184, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "confidence": 38, + "language": "ro", + "byteLogLikelihood": -2.700458868498, + "hits": 26, + "total": 202, + "hitRate": 0.12871287128712872 + }, + { + "encoding": "ISO-8859-1", + "confidence": 26, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "confidence": 26, + "language": "it", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "macintosh", + "confidence": 19, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 198, + "hitRate": 0.06565656565656566 + }, + { + "encoding": "CP850", + "confidence": 15, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.903997266318, + "hits": 10, + "total": 202, + "hitRate": 0.04950495049504951 + }, + { + "encoding": "ISO-8859-3", + "confidence": 13, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 9, + "total": 200, + "hitRate": 0.045 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.396820338954, + "hits": 6, + "total": 202, + "hitRate": 0.0297029702970297 + }, + { + "encoding": "CP852", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 201, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -4.936633427031, + "hits": 5, + "total": 202, + "hitRate": 0.024752475247524754 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.177254141958, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-10", + "confidence": 5, + "language": "is", + "byteLogLikelihood": -5.150687673188, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.974278583322, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.539421788511, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.828974536143, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.528215037204, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.239455511679, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.32130473174, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.608648280371, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1251", + "language": "ru" + }, + "predicted": { + "encoding": "windows-1251", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1251", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 34, + "language": "ru", + "byteLogLikelihood": -3.411211853364, + "hits": 20, + "total": 175, + "hitRate": 0.11428571428571428 + }, + { + "encoding": "IBM855", + "confidence": 9, + "language": "sr", + "byteLogLikelihood": -5.604243216214, + "hits": 5, + "total": 164, + "hitRate": 0.03048780487804878 + }, + { + "encoding": "KOI8-U", + "confidence": 5, + "language": "uk", + "byteLogLikelihood": -6.290472763796, + "hits": 3, + "total": 179, + "hitRate": 0.01675977653631285 + }, + { + "encoding": "ISO-8859-8", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -5.552676456081, + "hits": 1, + "total": 165, + "hitRate": 0.006060606060606061 + }, + { + "encoding": "KOI8-R", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -6.346028593552, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-14", + "confidence": 1, + "language": "cy", + "byteLogLikelihood": -4.724449359889, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.054854043559, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.827625181516, + "hits": 0, + "total": 129, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.61521723683, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.899487478125, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.446015416921, + "hits": 0, + "total": 88, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.059444422239, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.904224209153, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.79659105312, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.939249573759, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.66408545306, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "fr", + "byteLogLikelihood": -4.760478683076, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.970632247101, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 155, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "da" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + { + "encoding": "windows-1252", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + { + "encoding": "macintosh", + "confidence": 30, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "ISO-8859-15", + "confidence": 30, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "CP850", + "confidence": 30, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "ISO-8859-14", + "confidence": 15, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1257", + "confidence": 10, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -4.717529149813, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 120, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.658005606594, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.145958228046, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.825649766807, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.736335095675, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.861023356308, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.888170123788, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.476992253447, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.825649766807, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.685745819823, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "macintosh", + "confidence": 72, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "CP850", + "confidence": 70, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "ISO-8859-14", + "confidence": 20, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 15, + "total": 216, + "hitRate": 0.06944444444444445 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 216, + "hitRate": 0.037037037037037035 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 216, + "hitRate": 0.023148148148148147 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 216, + "hitRate": 0.009259259259259259 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 1, + "total": 216, + "hitRate": 0.004629629629629629 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.162408104115, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.542402054434, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.657121063601, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.28774788718, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.245943193261, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.942998477092, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "en" + }, + "predicted": { + "encoding": "ISO-8859-15", + "confidence": 29, + "language": "en", + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-15", + "confidence": 29, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "en", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "en", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "macintosh", + "confidence": 27, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 + }, + { + "encoding": "CP850", + "confidence": 27, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-14", + "confidence": 11, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "cs", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1250", + "confidence": 9, + "language": "cs", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1258", + "confidence": 9, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "ISO-8859-16", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "ISO-8859-4", + "confidence": 6, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 3, + "total": 132, + "hitRate": 0.022727272727272728 + }, + { + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 1, + "total": 132, + "hitRate": 0.007575757575757576 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 131, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 0, + "total": 132, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "macintosh", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-4", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.009854802779, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -4.892914919217, + "hits": 13, + "total": 200, + "hitRate": 0.065 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 200, + "hitRate": 0.055 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -4.848531508941, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -3.846028832089, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.346153815501, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.412820202576, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.203740544852, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "macintosh", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 228, + "hitRate": 0.013157894736842105 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "it" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "macintosh", + "confidence": 24, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 158, + "hitRate": 0.08227848101265822 + }, + { + "encoding": "ISO-8859-15", + "confidence": 24, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "CP850", + "confidence": 24, + "language": "fr", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "windows-1258", + "confidence": 16, + "language": "vi", + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "ISO-8859-16", + "confidence": 16, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1257", + "confidence": 15, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-4", + "confidence": 15, + "language": "lv", + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-14", + "confidence": 9, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 159, + "hitRate": 0.031446540880503145 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 159, + "hitRate": 0.025157232704402517 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -6.520621127559, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "nl", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "nl", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "windows-1252", + "confidence": 83, + "language": "nl", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "macintosh", + "confidence": 63, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-15", + "confidence": 63, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "CP850", + "confidence": 63, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-14", + "confidence": 33, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 17, + "total": 151, + "hitRate": 0.11258278145695365 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-9", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1254", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-10", + "confidence": 11, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 151, + "hitRate": 0.033112582781456956 + }, + { + "encoding": "windows-1258", + "confidence": 3, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-3", + "confidence": 1, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-4", + "confidence": 1, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "CP852", + "confidence": 1, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 151, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "no" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "macintosh", + "confidence": 18, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-14", + "confidence": 18, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-15", + "confidence": 18, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "CP850", + "confidence": 18, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-10", + "confidence": 13, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 5, + "total": 114, + "hitRate": 0.043859649122807015 + }, + { + "encoding": "windows-1258", + "confidence": 10, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-2", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1250", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.319245023186, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.641462690249, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.679407897139, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "pt", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "es", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "pt", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "pt", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1257", + "confidence": 29, + "language": "lt", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "ISO-8859-13", + "confidence": 29, + "language": "lt", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "macintosh", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "CP850", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "ISO-8859-4", + "confidence": 23, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 11, + "total": 143, + "hitRate": 0.07692307692307693 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "ISO-8859-14", + "confidence": 10, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -4.524213150766, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-10", + "confidence": 6, + "language": "is", + "byteLogLikelihood": -4.06291963921, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.514449766028, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.846220145127, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.503504267293, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.91521504926, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.476992253447, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -3.774998078834, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.766179593299, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 143, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "sv" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1258", + "confidence": 19, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "ISO-8859-10", + "confidence": 19, + "language": "is", + "byteLogLikelihood": -4.980036753423, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "macintosh", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP850", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "ISO-8859-9", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "windows-1254", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "ISO-8859-16", + "confidence": 6, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 136, + "hitRate": 0.022058823529411766 + }, + { + "encoding": "windows-1257", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.287368473007, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.25594643617, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.801526726218, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.432153549797, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.390348855877, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.419477704429, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1253", + "language": "el" + }, + "predicted": { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "windows-1253", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "KOI8-U", + "confidence": 8, + "language": "uk", + "byteLogLikelihood": -6.254780178237, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "IBM855", + "confidence": 7, + "language": "ru", + "byteLogLikelihood": -5.358823187498, + "hits": 3, + "total": 118, + "hitRate": 0.025423728813559324 + }, + { + "encoding": "KOI8-R", + "confidence": 4, + "language": "ru", + "byteLogLikelihood": -6.302671202821, + "hits": 2, + "total": 134, + "hitRate": 0.014925373134328358 + }, + { + "encoding": "ISO-8859-6", + "confidence": 3, + "language": "ar", + "byteLogLikelihood": -5.268243094623, + "hits": 1, + "total": 81, + "hitRate": 0.012345679012345678 + }, + { + "encoding": "ISO-8859-8", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "windows-1255", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 2, + "language": "uk", + "byteLogLikelihood": -4.203232324163, + "hits": 1, + "total": 131, + "hitRate": 0.007633587786259542 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.165809265202, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288697198304, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.013410783916, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.827626146169, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.681848366371, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.483358374511, + "hits": 0, + "total": 52, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.55248803416, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 116, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.945729021813, + "hits": 0, + "total": 129, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.015268979336, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.576407109739, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.890771320266, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.788878603493, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.603974037796, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.137560045078, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 96, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1254", + "language": "tr" + }, + "predicted": { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "ISO-8859-3", + "confidence": 22, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 10, + "total": 134, + "hitRate": 0.07462686567164178 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "da", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "da", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-10", + "confidence": 17, + "language": "is", + "byteLogLikelihood": -4.720106560713, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -4.93302665818, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-13", + "confidence": 11, + "language": "lt", + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.652724693026, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.197431316193, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.576073075491, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.714883328648, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.085579959517, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.742947096814, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.22610845571, + "hits": 0, + "total": 134, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1255", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "windows-1255", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "ISO-8859-7", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "IBM855", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.394495827364, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.124491051534, + "hits": 0, + "total": 73, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.199203901043, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.32364067176, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.639272141503, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029681805165, + "hits": 0, + "total": 99, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.31385343816, + "hits": 0, + "total": 48, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.40669720385, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.393692981541, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 72, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.134202940863, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.956632184798, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.146385148445, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.745792460039, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.842271132243, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.813633859942, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "fr", + "byteLogLikelihood": -4.521941300627, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.093070766591, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 85, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 89, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1256", + "language": "ar" + }, + "predicted": { + "encoding": "windows-1256", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1256", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + { + "encoding": "IBM855", + "confidence": 21, + "language": "sr", + "byteLogLikelihood": -5.640676681359, + "hits": 7, + "total": 100, + "hitRate": 0.07 + }, + { + "encoding": "ISO-8859-6", + "confidence": 15, + "language": "ar", + "byteLogLikelihood": -3.973986161297, + "hits": 5, + "total": 99, + "hitRate": 0.050505050505050504 + }, + { + "encoding": "ISO-8859-8", + "confidence": 5, + "language": "he", + "byteLogLikelihood": -5.107875566922, + "hits": 1, + "total": 58, + "hitRate": 0.017241379310344827 + }, + { + "encoding": "windows-1255", + "confidence": 5, + "language": "he", + "byteLogLikelihood": -5.107875566922, + "hits": 1, + "total": 60, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-5", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.047579876205, + "hits": 1, + "total": 111, + "hitRate": 0.009009009009009009 + }, + { + "encoding": "KOI8-R", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.041761137138, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "KOI8-U", + "confidence": 2, + "language": "uk", + "byteLogLikelihood": -4.982175743167, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.916907952775, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.575363278011, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.916907952775, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.856571347512, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029693479499, + "hits": 0, + "total": 106, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.736714293626, + "hits": 0, + "total": 90, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.982133409941, + "hits": 0, + "total": 60, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 46, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.655679070563, + "hits": 0, + "total": 81, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.078935619012, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.796168589227, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.003416643654, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.119560962469, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.941225681894, + "hits": 0, + "total": 97, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 100, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1257", + "language": "et" + }, + "predicted": { + "encoding": "windows-1257", + "confidence": 40, + "language": "et", + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1257", + "confidence": 40, + "language": "et", + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 + }, + { + "encoding": "ISO-8859-1", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -3.976561526566, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1252", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -3.976561526566, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "macintosh", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-3", + "confidence": 12, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "CP850", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-4", + "confidence": 10, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-13", + "confidence": 10, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "CP852", + "confidence": 10, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-16", + "confidence": 5, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 120, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-14", + "confidence": 2, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.30822322351, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.41969872812, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.51563851542, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.453415335268, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.41969872812, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.51563851542, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.831207501101, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.714883328648, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.345510152227, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.827473946999, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.632538332325, + "hits": 0, + "total": 120, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1257", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-13", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-4", + "confidence": 22, + "language": "lv", + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "ISO-8859-1", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "macintosh", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 168, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "CP850", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 170, + "hitRate": 0.041176470588235294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "windows-1254", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 172, + "hitRate": 0.01744186046511628 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.6272413408, + "hits": 2, + "total": 173, + "hitRate": 0.011560693641618497 + }, + { + "encoding": "windows-1258", + "confidence": 1, + "language": "vi", + "byteLogLikelihood": -4.960186085121, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "IBM855", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.488073838002, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.893109334109, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.773457169107, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 177, - "hitRate": 0.03954802259887006 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.531827362419, + "hits": 0, + "total": 173, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 9, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 6, - "total": 182, - "hitRate": 0.03296703296703297 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 6, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.986136327924, - "hits": 3, - "total": 182, - "hitRate": 0.016483516483516484 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.924336543075, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 183, - "hitRate": 0.01092896174863388 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.58967630288, + "hits": 0, + "total": 166, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.365896834008, + "hits": 0, + "total": 173, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.383683979141, + "byteLogLikelihood": -6.252539473952, "hits": 0, - "total": 181, + "total": 173, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.404570694946, + "byteLogLikelihood": -4.893109334109, "hits": 0, - "total": 181, + "total": 171, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.943656762101, + "language": "uk", + "byteLogLikelihood": -3.441328843196, "hits": 0, - "total": 181, + "total": 173, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-16", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.572384450833, + "language": "ro", + "byteLogLikelihood": -4.616838329205, "hits": 0, - "total": 182, + "total": 173, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", "language": "lv" }, "predicted": { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", "confidence": 40, "language": "lv", "byteLogLikelihood": -3.1328616083, @@ -2254,12 +19033,14 @@ "total": 170, "hitRate": 0.13529411764705881 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", "confidence": 40, "language": "lv", "byteLogLikelihood": -3.1328616083, @@ -2267,14 +19048,41 @@ "total": 170, "hitRate": 0.13529411764705881 }, + { + "encoding": "ISO-8859-4", + "confidence": 31, + "language": "lv", + "byteLogLikelihood": -5.003611214133, + "hits": 18, + "total": 170, + "hitRate": 0.10588235294117647 + }, { "encoding": "CP850", - "confidence": 16, + "confidence": 15, "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 9, - "total": 168, - "hitRate": 0.05357142857142857 + "total": 170, + "hitRate": 0.052941176470588235 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.806768476285, + "hits": 8, + "total": 170, + "hitRate": 0.047058823529411764 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.806768476285, + "hits": 8, + "total": 170, + "hitRate": 0.047058823529411764 }, { "encoding": "macintosh", @@ -2282,17 +19090,17 @@ "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 + "total": 170, + "hitRate": 0.047058823529411764 }, { "encoding": "ISO-8859-15", "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.721042183854, + "language": "fr", + "byteLogLikelihood": -4.806768476285, "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 + "total": 170, + "hitRate": 0.047058823529411764 }, { "encoding": "CP852", @@ -2300,672 +19108,843 @@ "language": "pl", "byteLogLikelihood": -5.087596335232, "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 + "total": 170, + "hitRate": 0.041176470588235294 + }, + { + "encoding": "ISO-8859-3", + "confidence": 10, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 6, + "total": 170, + "hitRate": 0.03529411764705882 }, { "encoding": "ISO-8859-13", "confidence": 10, "language": "lt", - "byteLogLikelihood": -4.596975900219, + "byteLogLikelihood": -4.977793000004, "hits": 6, - "total": 169, - "hitRate": 0.03550295857988166 + "total": 170, + "hitRate": 0.03529411764705882 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "pl", + "byteLogLikelihood": -5.087596335232, "hits": 5, "total": 170, "hitRate": 0.029411764705882353 }, { - "encoding": "ISO-8859-10", + "encoding": "windows-1250", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "ISO-8859-9", "confidence": 7, - "language": "is", - "byteLogLikelihood": -5.074758182528, + "language": "tr", + "byteLogLikelihood": -4.576539240129, "hits": 4, "total": 170, "hitRate": 0.023529411764705882 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.572632236701, - "hits": 1, - "total": 169, - "hitRate": 0.005917159763313609 + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.576539240129, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.609505685025, - "hits": 0, + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.342387954294, + "hits": 4, "total": 169, - "hitRate": 0 + "hitRate": 0.023668639053254437 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -4.928587203128, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.267508614334, + "byteLogLikelihood": -4.459882683974, "hits": 0, - "total": 168, + "total": 170, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.182018138172, + "language": "ar", + "byteLogLikelihood": -5.080491263049, "hits": 0, "total": 169, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.65535318117, + "language": "el", + "byteLogLikelihood": -3.703260469986, "hits": 0, "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "language": "he", + "byteLogLikelihood": -4.207917523838, "hits": 0, "total": 170, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "KOI8-U", - "language": "uk" - }, - "predicted": { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.128376292359, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 }, { - "encoding": "IBM866", + "encoding": "windows-1251", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.582632026409, + "byteLogLikelihood": -3.543665829263, "hits": 0, - "total": 22, + "total": 170, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.922515418323, + "language": "el", + "byteLogLikelihood": -3.703260469986, "hits": 0, - "total": 94, + "total": 170, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "windows-1255", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "language": "he", + "byteLogLikelihood": -4.207917523838, "hits": 0, - "total": 89, + "total": 170, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1256", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.589678128375, + "language": "ar", + "byteLogLikelihood": -5.924336543075, "hits": 0, - "total": 117, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-874", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "th", + "byteLogLikelihood": -5.58909903103, "hits": 0, - "total": 175, + "total": 167, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "KOI8-R", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "language": "ru", + "byteLogLikelihood": -6.155101602334, "hits": 0, - "total": 176, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "KOI8-U", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.110283442945, + "language": "uk", + "byteLogLikelihood": -6.147141858115, "hits": 0, - "total": 168, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "IBM866", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "language": "ru", + "byteLogLikelihood": -4.459882683974, "hits": 0, - "total": 177, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "IBM855", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944199711559, + "language": "ru", + "byteLogLikelihood": -5.609454107834, "hits": 0, - "total": 176, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.148788659657, + "language": "uk", + "byteLogLikelihood": -3.416476215238, "hits": 0, - "total": 177, + "total": 170, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "ISO-8859-14", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "language": "cy", + "byteLogLikelihood": -4.423374522988, "hits": 0, - "total": 92, + "total": 170, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "ISO-8859-16", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "language": "ro", + "byteLogLikelihood": -4.717949371076, "hits": 0, - "total": 94, + "total": 170, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "de" + "encoding": "windows-1258", + "language": "vi" }, "predicted": { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "windows-1258", + "confidence": 98, + "language": "vi", + "byteLogLikelihood": -3.323414763285, + "hits": 42, + "total": 124, + "hitRate": 0.3387096774193548 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "windows-1258", + "confidence": 98, + "language": "vi", + "byteLogLikelihood": -3.323414763285, + "hits": 42, + "total": 124, + "hitRate": 0.3387096774193548 }, { - "encoding": "ISO-8859-15", - "confidence": 74, + "encoding": "ISO-8859-1", + "confidence": 9, + "language": "pt", + "byteLogLikelihood": -4.721506246424, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1252", + "confidence": 9, + "language": "pt", + "byteLogLikelihood": -4.721506246424, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lt", + "byteLogLikelihood": -4.767575136152, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "macintosh", + "confidence": 9, "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 53, - "total": 214, - "hitRate": 0.24766355140186916 + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 122, + "hitRate": 0.03278688524590164 }, { - "encoding": "CP850", - "confidence": 69, - "language": "fr", - "byteLogLikelihood": -5.003946305945, - "hits": 50, - "total": 216, - "hitRate": 0.23148148148148148 + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -4.83530398146, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "ISO-8859-13", + "confidence": 9, + "language": "lt", + "byteLogLikelihood": -4.767575136152, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { "encoding": "ISO-8859-14", - "confidence": 22, + "confidence": 9, "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 16, - "total": 214, - "hitRate": 0.07476635514018691 + "byteLogLikelihood": -4.643097648852, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { - "encoding": "ISO-8859-16", - "confidence": 15, + "encoding": "ISO-8859-2", + "confidence": 7, "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 11, - "total": 214, - "hitRate": 0.0514018691588785 + "byteLogLikelihood": -5.060625202481, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 }, { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 214, - "hitRate": 0.037383177570093455 + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.76855606835, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1250", + "confidence": 7, + "language": "ro", + "byteLogLikelihood": -5.060625202481, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.76855606835, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 }, { "encoding": "ISO-8859-3", - "confidence": 5, + "confidence": 7, "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 + "byteLogLikelihood": -4.906120867834, + "hits": 3, + "total": 124, + "hitRate": 0.024193548387096774 + }, + { + "encoding": "ISO-8859-15", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -4.873536273681, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "CP850", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 116, + "hitRate": 0.02586206896551724 }, { "encoding": "CP852", - "confidence": 5, + "confidence": 7, "language": "pl", "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, "hits": 3, - "total": 214, - "hitRate": 0.014018691588785047 + "total": 119, + "hitRate": 0.025210084033613446 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-16", "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 214, - "hitRate": 0.009345794392523364 + "language": "ro", + "byteLogLikelihood": -5.104831294985, + "hits": 1, + "total": 126, + "hitRate": 0.007936507936507936 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.315573016103, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.428051016065, + "hits": 0, + "total": 115, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.428592570712, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.890004274964, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.843070172798, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.428592570712, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.890004274964, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.628297137503, + "hits": 0, + "total": 121, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.351753988745, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.775144482981, + "hits": 0, + "total": 126, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.534754518494, + "byteLogLikelihood": -5.565906355903, "hits": 0, - "total": 215, + "total": 126, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.345510152227, + "byteLogLikelihood": -5.727056945557, "hits": 0, - "total": 216, + "total": 124, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.337519079447, + "language": "ru", + "byteLogLikelihood": -5.876501639082, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.782036156761, "hits": 0, - "total": 216, + "total": 125, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-10", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "is", + "byteLogLikelihood": -4.794574996468, "hits": 0, - "total": 216, + "total": 126, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "es" + "encoding": "windows-874", + "language": "th" }, "predicted": { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "windows-874", + "confidence": 21, + "language": "th", + "byteLogLikelihood": -3.827006747769, + "hits": 7, + "total": 100, + "hitRate": 0.07 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "windows-874", + "confidence": 21, + "language": "th", + "byteLogLikelihood": -3.827006747769, + "hits": 7, + "total": 100, + "hitRate": 0.07 }, { - "encoding": "ISO-8859-15", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 + "encoding": "ISO-8859-6", + "confidence": 6, + "language": "ar", + "byteLogLikelihood": -5.033884846794, + "hits": 2, + "total": 93, + "hitRate": 0.021505376344086023 }, { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 + "encoding": "ISO-8859-8", + "confidence": 6, + "language": "he", + "byteLogLikelihood": -5.415196995324, + "hits": 1, + "total": 43, + "hitRate": 0.023255813953488372 }, { - "encoding": "ISO-8859-4", - "confidence": 25, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 17, - "total": 199, - "hitRate": 0.08542713567839195 + "encoding": "windows-1255", + "confidence": 6, + "language": "he", + "byteLogLikelihood": -5.415196995324, + "hits": 1, + "total": 47, + "hitRate": 0.02127659574468085 }, { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 13, - "total": 199, - "hitRate": 0.06532663316582915 + "encoding": "ISO-8859-7", + "confidence": 12, + "language": "el", + "byteLogLikelihood": -5.563277974308, + "hits": 4, + "total": 97, + "hitRate": 0.041237113402061855 }, { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 199, - "hitRate": 0.05527638190954774 + "encoding": "windows-1253", + "confidence": 12, + "language": "el", + "byteLogLikelihood": -5.563277974308, + "hits": 4, + "total": 97, + "hitRate": 0.041237113402061855 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 + "encoding": "windows-1251", + "confidence": 5, + "language": "ru", + "byteLogLikelihood": -5.926448142289, + "hits": 2, + "total": 102, + "hitRate": 0.0196078431372549 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 + "encoding": "x-mac-cyrillic", + "confidence": 11, + "language": "ru", + "byteLogLikelihood": -6.059253444951, + "hits": 3, + "total": 79, + "hitRate": 0.0379746835443038 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 199, - "hitRate": 0.01507537688442211 + "encoding": "KOI8-R", + "confidence": 3, + "language": "ru", + "byteLogLikelihood": -5.413211053323, + "hits": 1, + "total": 94, + "hitRate": 0.010638297872340425 }, { - "encoding": "KOI8-U", + "encoding": "IBM855", + "confidence": 3, + "language": "sr", + "byteLogLikelihood": -5.781484174666, + "hits": 1, + "total": 88, + "hitRate": 0.011363636363636364 + }, + { + "encoding": "ISO-8859-1", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "it", + "byteLogLikelihood": -4.767489617961, "hits": 0, - "total": 199, + "total": 97, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.903917269007, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -5.535400651308, "hits": 0, - "total": 200, + "total": 103, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-9", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.624741392945, + "language": "tr", + "byteLogLikelihood": -4.985031025935, "hits": 0, - "total": 200, + "total": 97, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1250", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "language": "pl", + "byteLogLikelihood": -4.798656950405, "hits": 0, - "total": 200, + "total": 101, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1252", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "it", + "byteLogLikelihood": -4.767489617961, "hits": 0, - "total": 199, + "total": 97, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "macintosh", - "language": "fr" - }, - "predicted": { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ + }, { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.985031025935, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 75, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 56, - "total": 224, - "hitRate": 0.25 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.209625144761, + "hits": 0, + "total": 95, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 71, - "language": "de", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.916002186025, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 12, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.154894826847, + "hits": 0, + "total": 91, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -5.334497573029, + "hits": 0, + "total": 98, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 224, - "hitRate": 0.03125 + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.961753913164, + "hits": 0, + "total": 58, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 65, + "hitRate": 0 }, { "encoding": "ISO-8859-3", - "confidence": 8, + "confidence": 0, "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 224, - "hitRate": 0.026785714285714284 + "byteLogLikelihood": -5.084507932234, + "hits": 0, + "total": 98, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.783724339715, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.119633132705, + "hits": 0, + "total": 103, + "hitRate": 0 }, { "encoding": "ISO-8859-13", - "confidence": 5, + "confidence": 0, "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 224, - "hitRate": 0.017857142857142856 + "byteLogLikelihood": -5.010167702209, + "hits": 0, + "total": 97, + "hitRate": 0 }, { "encoding": "ISO-8859-14", - "confidence": 4, + "confidence": 0, "language": "cy", "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 224, - "hitRate": 0.013392857142857142 + "hits": 0, + "total": 103, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-15", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "fr", + "byteLogLikelihood": -4.801693311206, "hits": 0, - "total": 224, + "total": 101, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-16", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, + "language": "ro", + "byteLogLikelihood": -5.134285984462, "hits": 0, - "total": 228, + "total": 103, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "CP850", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -3.576182148392, + "language": "es", + "byteLogLikelihood": -4.887413059939, "hits": 0, - "total": 228, + "total": 85, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "CP852", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, + "language": "pl", + "byteLogLikelihood": -5.056712497361, "hits": 0, - "total": 228, + "total": 85, "hitRate": 0 } ] @@ -2984,6 +19963,8 @@ "total": 179, "hitRate": 0.12290502793296089 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -2997,15 +19978,42 @@ "total": 179, "hitRate": 0.12290502793296089 }, + { + "encoding": "windows-1251", + "confidence": 42, + "language": "ru", + "byteLogLikelihood": -3.411211853364, + "hits": 25, + "total": 177, + "hitRate": 0.14124293785310735 + }, { "encoding": "IBM855", "confidence": 9, - "language": "ru", + "language": "sr", "byteLogLikelihood": -5.603105135042, "hits": 5, "total": 165, "hitRate": 0.030303030303030304 }, + { + "encoding": "ISO-8859-8", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, { "encoding": "KOI8-U", "confidence": 3, @@ -3015,6 +20023,24 @@ "total": 177, "hitRate": 0.011299435028248588 }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -5.525051915661, + "hits": 1, + "total": 163, + "hitRate": 0.006134969325153374 + }, + { + "encoding": "KOI8-R", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -6.390589047944, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 + }, { "encoding": "ISO-8859-14", "confidence": 1, @@ -3024,6 +20050,123 @@ "total": 177, "hitRate": 0.005649717514124294 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.975257565806, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.847464468096, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.148248171081, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.148248171081, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.61521723683, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.899487478125, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.446015416921, + "hits": 0, + "total": 85, + "hitRate": 0 + }, { "encoding": "IBM866", "confidence": 0, @@ -3091,7 +20234,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.935887060165, + "byteLogLikelihood": -4.97522262578, "hits": 0, "total": 177, "hitRate": 0 @@ -3130,6 +20273,8 @@ "total": 177, "hitRate": 0.10734463276836158 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -3143,15 +20288,60 @@ "total": 177, "hitRate": 0.10734463276836158 }, + { + "encoding": "windows-1251", + "confidence": 17, + "language": "ru", + "byteLogLikelihood": -3.606822645813, + "hits": 10, + "total": 176, + "hitRate": 0.056818181818181816 + }, { "encoding": "IBM855", "confidence": 5, - "language": "ru", + "language": "sr", "byteLogLikelihood": -5.830774318921, "hits": 3, "total": 163, "hitRate": 0.018404907975460124 }, + { + "encoding": "ISO-8859-8", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -4.062591421695, + "hits": 2, + "total": 165, + "hitRate": 0.012121212121212121 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -4.062591421695, + "hits": 2, + "total": 165, + "hitRate": 0.012121212121212121 + }, + { + "encoding": "KOI8-R", + "confidence": 3, + "language": "ru", + "byteLogLikelihood": -6.397907632883, + "hits": 2, + "total": 169, + "hitRate": 0.011834319526627219 + }, + { + "encoding": "ISO-8859-6", + "confidence": 2, + "language": "ar", + "byteLogLikelihood": -4.990380473945, + "hits": 1, + "total": 119, + "hitRate": 0.008403361344537815 + }, { "encoding": "KOI8-U", "confidence": 1, @@ -3161,6 +20351,123 @@ "total": 175, "hitRate": 0.005714285714285714 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.629500955096, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.739775351774, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.989983227843, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.300043808736, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.951345660734, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.739775351774, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.629500955096, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.300043808736, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.951345660734, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.607550779971, + "hits": 0, + "total": 155, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.60874799529, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.777030248466, + "hits": 0, + "total": 164, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.197321748228, + "hits": 0, + "total": 83, + "hitRate": 0 + }, { "encoding": "IBM866", "confidence": 0, @@ -3237,7 +20544,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.986714390957, + "byteLogLikelihood": -5.030920483462, "hits": 0, "total": 175, "hitRate": 0 diff --git a/corpus/sources.json b/corpus/sources.json index 3b05948..e65eab5 100644 --- a/corpus/sources.json +++ b/corpus/sources.json @@ -505,6 +505,582 @@ "text": "W niedzielę przyjaciele poszli nad spokojną rzekę za miastem. Zabrali mapę, ciepłą herbatę i aparat. Po długim spacerze odpoczywali na brzegu." } ] + }, + { + "code": "ar", + "name": "Arabic", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "صباح المدينة", + "text": "تستيقظ المدينة بهدوء وتفتح المقاهي أبوابها بينما ينتظر الناس الحافلات ويقرؤون الأخبار." + }, + { + "id": "library", + "split": "train", + "title": "المكتبة الهادئة", + "text": "يبحث القراء عن كتب التاريخ والعلوم ويساعد أمين المكتبة الطلاب في اختيار المراجع المناسبة." + }, + { + "id": "market", + "split": "train", + "title": "سوق السبت", + "text": "يعرض المزارعون الخبز والعسل والفاكهة ويتبادل الزوار الوصفات والأحاديث الودية." + }, + { + "id": "workshop", + "split": "train", + "title": "ورشة رقمية", + "text": "يصمم فريق صغير دليلا للمتحف ويراجع المبرمجون الشفرة ويكتب المحررون وصفا واضحا." + }, + { + "id": "river-trip", + "split": "validation", + "title": "رحلة إلى النهر", + "text": "حمل الأصدقاء خريطة وشايا ساخنا ومشوا حتى النهر ثم استمعوا إلى طيور المساء." + } + ] + }, + { + "code": "cs", + "name": "Czech", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Ráno ve městě", + "text": "Město se pomalu probouzí, kavárny otevírají dveře a lidé čekají na první autobusy." + }, + { + "id": "library", + "split": "train", + "title": "Tichá knihovna", + "text": "Čtenáři hledají knihy o historii a vědě, zatímco knihovnice radí žákům." + }, + { + "id": "market", + "split": "train", + "title": "Sobotní trh", + "text": "Farmáři prodávají čerstvý chléb, med, ovoce a voňavé byliny." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitální dílna", + "text": "Malý tým připravuje průvodce muzeem, kontroluje kód a píše jasné popisy." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Výlet k řece", + "text": "Přátelé vzali mapu a teplý čaj, prošli lesem a odpočívali u řeky." + } + ] + }, + { + "code": "da", + "name": "Danish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgen i byen", + "text": "Byen vågner langsomt, caféerne åbner, og folk venter på de første busser." + }, + { + "id": "library", + "split": "train", + "title": "Det stille bibliotek", + "text": "Læsere finder bøger om historie og videnskab, mens bibliotekaren hjælper eleverne." + }, + { + "id": "market", + "split": "train", + "title": "Lørdagsmarked", + "text": "Bønder sælger frisk brød, æbler, ost og honning på den travle plads." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitalt værksted", + "text": "Et lille hold bygger en museumsguide, undersøger koden og skriver tydelige tekster." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Tur til floden", + "text": "Vennerne tog et kort og varm te med på en søndag og hvilede bagefter ved floden." + } + ] + }, + { + "code": "en", + "name": "English", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morning in the city", + "text": "The city wakes slowly; cafés open their doors and commuters read news beside a façade." + }, + { + "id": "library", + "split": "train", + "title": "The quiet library", + "text": "Readers browse books about history, science, and naïve art while the librarian prepares a résumé." + }, + { + "id": "market", + "split": "train", + "title": "Saturday market", + "text": "Farmers sell bread, apples, jalapeños, and pâté while neighbours exchange recipes." + }, + { + "id": "workshop", + "split": "train", + "title": "Digital workshop", + "text": "A small team designs the museum guide and discusses its rôle, code, and clear descriptions." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Journey to the river", + "text": "Friends carried a map and a thermos, walked past the café, and rested by the river." + } + ] + }, + { + "code": "el", + "name": "Greek", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Πρωί στην πόλη", + "text": "Η πόλη ξυπνά αργά, τα καφέ ανοίγουν και οι άνθρωποι περιμένουν τα λεωφορεία." + }, + { + "id": "library", + "split": "train", + "title": "Ήσυχη βιβλιοθήκη", + "text": "Οι αναγνώστες ψάχνουν βιβλία ιστορίας και επιστήμης και η βιβλιοθηκάριος βοηθά τους μαθητές." + }, + { + "id": "market", + "split": "train", + "title": "Σαββατιάτικη αγορά", + "text": "Οι αγρότες πουλούν ψωμί, μέλι, φρούτα και μυρωδικά στην πλατεία." + }, + { + "id": "workshop", + "split": "train", + "title": "Ψηφιακό εργαστήριο", + "text": "Μια μικρή ομάδα ετοιμάζει οδηγό μουσείου, ελέγχει τον κώδικα και γράφει καθαρά κείμενα." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Εκδρομή στο ποτάμι", + "text": "Οι φίλοι πήραν χάρτη και ζεστό τσάι και ξεκουράστηκαν δίπλα στο ποτάμι." + } + ] + }, + { + "code": "et", + "name": "Estonian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Hommik linnas", + "text": "Linn ärkab aeglaselt, kohvikud avavad uksed ja inimesed ootavad esimesi busse." + }, + { + "id": "library", + "split": "train", + "title": "Vaikne raamatukogu", + "text": "Lugejad otsivad ajaloo- ja teadusraamatuid ning raamatukoguhoidja aitab õpilasi." + }, + { + "id": "market", + "split": "train", + "title": "Laupäevane turg", + "text": "Talunikud müüvad värsket leiba, õunu, juustu ja mett rahvarohkel väljakul." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitaalne töötuba", + "text": "Väike meeskond koostab muuseumijuhti, kontrollib koodi ja kirjutab selgeid kirjeldusi." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Retk jõe äärde", + "text": "Sõbrad võtsid kaardi ja sooja tee ning puhkasid pärast jalutuskäiku jõe ääres." + } + ] + }, + { + "code": "he", + "name": "Hebrew", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "בוקר בעיר", + "text": "העיר מתעוררת לאט, בתי הקפה נפתחים ואנשים מחכים לאוטובוסים הראשונים." + }, + { + "id": "library", + "split": "train", + "title": "הספרייה השקטה", + "text": "קוראים מחפשים ספרי היסטוריה ומדע והספרנית עוזרת לתלמידים לבחור מקורות." + }, + { + "id": "market", + "split": "train", + "title": "שוק שבת", + "text": "חקלאים מוכרים לחם טרי, דבש, פירות וגבינה בכיכר העמוסה." + }, + { + "id": "workshop", + "split": "train", + "title": "סדנה דיגיטלית", + "text": "צוות קטן מכין מדריך למוזיאון, בודק את הקוד וכותב תיאורים ברורים." + }, + { + "id": "river-trip", + "split": "validation", + "title": "טיול אל הנהר", + "text": "החברים לקחו מפה ותה חם, הלכו ביער ונחו ליד הנהר." + } + ] + }, + { + "code": "hu", + "name": "Hungarian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Reggel a városban", + "text": "A város lassan ébred, a kávézók kinyitnak, az emberek pedig buszra várnak." + }, + { + "id": "library", + "split": "train", + "title": "A csendes könyvtár", + "text": "Az olvasók történelmi és tudományos könyveket keresnek, a könyvtáros segít a diákoknak." + }, + { + "id": "market", + "split": "train", + "title": "Szombati piac", + "text": "A gazdák friss kenyeret, almát, sajtot és mézet árulnak a téren." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitális műhely", + "text": "Egy kis csapat múzeumi útmutatót készít, kódot ellenőriz és világos leírásokat ír." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Kirándulás a folyóhoz", + "text": "A barátok térképet és meleg teát vittek, majd megpihentek a folyó partján." + } + ] + }, + { + "code": "it", + "name": "Italian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Mattina in città", + "text": "La città si sveglia lentamente, i caffè aprono e la gente aspetta i primi autobus." + }, + { + "id": "library", + "split": "train", + "title": "La biblioteca tranquilla", + "text": "I lettori cercano più libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti più giovani." + }, + { + "id": "market", + "split": "train", + "title": "Mercato del sabato", + "text": "Gli agricoltori vendono pane fresco, caffè, mele, formaggio e miele nella piazza della città." + }, + { + "id": "workshop", + "split": "train", + "title": "Laboratorio digitale", + "text": "Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perché siano utili a più persone." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Gita al fiume", + "text": "Gli amici portano una mappa e tè caldo, camminano nel bosco e riposano sulla riva." + } + ] + }, + { + "code": "nl", + "name": "Dutch", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Ochtend in de stad", + "text": "De stad ontwaakt langzaam, cafés openen hun deuren en mensen wachten op de eerste bussen." + }, + { + "id": "library", + "split": "train", + "title": "De stille bibliotheek", + "text": "Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris geïnteresseerde leerlingen en één leraar helpt." + }, + { + "id": "market", + "split": "train", + "title": "Zaterdagmarkt", + "text": "Boeren verkopen vers brood, appels, kaas en honing naast twee cafés op het drukke plein." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitale werkplaats", + "text": "Een klein team maakt een museumgids, controleert de code en schrijft duidelijke ideeën over poëzie en teksten." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Reis naar de rivier", + "text": "Vrienden namen een kaart en warme thee mee en rustten na één lange wandeling bij de rivier." + } + ] + }, + { + "code": "no", + "name": "Norwegian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgen i byen", + "text": "Byen våkner langsomt, kafeene åpner dørene og folk venter på de første bussene nær bryggen og sjøen." + }, + { + "id": "library", + "split": "train", + "title": "Det stille biblioteket", + "text": "Lesere finner bøker om historie, ærlige fortellinger og vitenskap mens bibliotekaren hjelper elevene." + }, + { + "id": "market", + "split": "train", + "title": "Lørdagsmarked", + "text": "Bønder selger ferskt brød, epler, ost og honning på det travle torget." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitalt verksted", + "text": "Et lite lag lager en museumsguide, undersøker koden og skriver tydelige tekster." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Tur til elven", + "text": "Vennene tok med kart og varm te på søndag og hvilte senere ved elven." + } + ] + }, + { + "code": "pt", + "name": "Portuguese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Manhã na cidade", + "text": "A cidade acorda devagar, os cafés abrem e as pessoas esperam os primeiros ônibus." + }, + { + "id": "library", + "split": "train", + "title": "A biblioteca tranquila", + "text": "Os leitores procuram livros de história e ciência enquanto a bibliotecária ajuda os alunos." + }, + { + "id": "market", + "split": "train", + "title": "Mercado de sábado", + "text": "Os agricultores vendem pão fresco, maçãs, queijo e mel na praça movimentada." + }, + { + "id": "workshop", + "split": "train", + "title": "Oficina digital", + "text": "Uma pequena equipe prepara o guia do museu, verifica o código e escreve descrições claras." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Viagem ao rio", + "text": "Os amigos levaram um mapa e chá quente, caminharam pela mata e descansaram junto ao rio." + } + ] + }, + { + "code": "sv", + "name": "Swedish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgon i staden", + "text": "Staden vaknar långsamt, kaféerna öppnar och människor väntar på de första bussarna." + }, + { + "id": "library", + "split": "train", + "title": "Det tysta biblioteket", + "text": "Läsare söker böcker om historia och vetenskap medan bibliotekarien hjälper eleverna." + }, + { + "id": "market", + "split": "train", + "title": "Lördagsmarknad", + "text": "Bönder säljer färskt bröd, äpplen, ost och honung på det livliga torget." + }, + { + "id": "workshop", + "split": "train", + "title": "Digital verkstad", + "text": "Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter för besökare." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Utflykt till floden", + "text": "Vännerna tog med karta och varmt te och vilade sedan vid floden." + } + ] + }, + { + "code": "th", + "name": "Thai", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "เช้าในเมือง", + "text": "เมืองค่อย ๆ ตื่น ร้านกาแฟเปิดประตูและผู้คนรอรถโดยสารเที่ยวแรก" + }, + { + "id": "library", + "split": "train", + "title": "ห้องสมุดเงียบสงบ", + "text": "ผู้อ่านค้นหาหนังสือประวัติศาสตร์และวิทยาศาสตร์ บรรณารักษ์ช่วยนักเรียนเลือกข้อมูล" + }, + { + "id": "market", + "split": "train", + "title": "ตลาดวันเสาร์", + "text": "ชาวนาขายขนมปังสด ผลไม้ ชีส และน้ำผึ้งในลานที่คึกคัก" + }, + { + "id": "workshop", + "split": "train", + "title": "ห้องทำงานดิจิทัล", + "text": "ทีมเล็กจัดทำคู่มือพิพิธภัณฑ์ ตรวจรหัส และเขียนคำอธิบายที่ชัดเจน" + }, + { + "id": "river-trip", + "split": "validation", + "title": "เที่ยวริมแม่น้ำ", + "text": "เพื่อนนำแผนที่กับชาร้อนไปเดินในป่าแล้วพักผ่อนริมแม่น้ำ" + } + ] + }, + { + "code": "tr", + "name": "Turkish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Şehirde sabah", + "text": "Şehir yavaşça uyanır, kafeler açılır ve insanlar ilk otobüsleri bekler." + }, + { + "id": "library", + "split": "train", + "title": "Sessiz kütüphane", + "text": "Okurlar tarih ve bilim kitapları ararken kütüphaneci öğrencilere yardımcı olur." + }, + { + "id": "market", + "split": "train", + "title": "Cumartesi pazarı", + "text": "Çiftçiler kalabalık meydanda taze ekmek, elma, peynir ve bal satar." + }, + { + "id": "workshop", + "split": "train", + "title": "Dijital atölye", + "text": "Küçük bir ekip müze rehberi hazırlar, kodu denetler ve açık açıklamalar yazar." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Nehir gezisi", + "text": "Arkadaşlar harita ve sıcak çay alıp ormanda yürüdü, sonra nehir kıyısında dinlendi." + } + ] + }, + { + "code": "vi", + "name": "Vietnamese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Buổi sáng trong thành phố", + "text": "Thành phố thức dậy chậm rãi, quán cà phê mở cửa và mọi người chờ chuyến xe buýt đầu tiên." + }, + { + "id": "library", + "split": "train", + "title": "Thư viện yên tĩnh", + "text": "Độc giả tìm sách lịch sử và khoa học, còn thủ thư giúp học sinh chọn tài liệu." + }, + { + "id": "market", + "split": "train", + "title": "Chợ thứ bảy", + "text": "Nông dân bán bánh mì, táo, phô mai và mật ong tại quảng trường đông đúc." + }, + { + "id": "workshop", + "split": "train", + "title": "Xưởng kỹ thuật số", + "text": "Một nhóm nhỏ làm hướng dẫn bảo tàng, kiểm tra mã và viết mô tả rõ ràng." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Chuyến đi ra sông", + "text": "Bạn bè mang bản đồ và trà nóng, đi qua rừng rồi nghỉ bên bờ sông." + } + ] } ] } diff --git a/corpus/test-sources.json b/corpus/test-sources.json index 9f75718..e138203 100644 --- a/corpus/test-sources.json +++ b/corpus/test-sources.json @@ -85,6 +85,102 @@ "id": "community-garden", "title": "Ogród sąsiedzki", "text": "Mieszkańcy zmienili pusty dziedziniec w zielony ogród z kwiatami, warzywami i młodymi drzewami. Dzieci zrobiły kolorowe tabliczki, a rodziny co tydzień podlewają rośliny." + }, + { + "language": "ar", + "id": "community-garden", + "title": "حديقة الحي", + "text": "زرع الجيران الزهور والأشجار في ساحة قديمة، وصنع الأطفال لافتات ملونة واتفق الجميع على سقي النباتات." + }, + { + "language": "cs", + "id": "community-garden", + "title": "Sousedská zahrada", + "text": "Sousedé proměnili prázdný dvůr v zelenou zahradu, děti namalovaly barevné značky a rodiny zalévají rostliny." + }, + { + "language": "da", + "id": "community-garden", + "title": "Fælleshaven", + "text": "Naboerne gjorde en tom gård til en grøn have, børnene malede skilte, og familierne vander planterne hver uge." + }, + { + "language": "en", + "id": "community-garden", + "title": "The community garden", + "text": "Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small café table." + }, + { + "language": "el", + "id": "community-garden", + "title": "Ο κοινοτικός κήπος", + "text": "Οι γείτονες έκαναν μια άδεια αυλή πράσινο κήπο, τα παιδιά ζωγράφισαν πινακίδες και οι οικογένειες ποτίζουν τα φυτά." + }, + { + "language": "et", + "id": "community-garden", + "title": "Kogukonna aed", + "text": "Naabrid muutsid tühja hoovi roheliseks aiaks, lapsed maalisid sildid ja pered kastavad taimi igal nädalal." + }, + { + "language": "he", + "id": "community-garden", + "title": "הגינה הקהילתית", + "text": "השכנים הפכו חצר ריקה לגינה ירוקה, הילדים ציירו שלטים והמשפחות משקות את הצמחים בכל שבוע." + }, + { + "language": "hu", + "id": "community-garden", + "title": "Közösségi kert", + "text": "A szomszédok zöld kertté alakították az üres udvart, a gyerekek táblákat festettek, a családok pedig öntözik a növényeket." + }, + { + "language": "it", + "id": "community-garden", + "title": "Il giardino comune", + "text": "I vicini hanno trasformato un cortile vuoto in un giardino della città, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante." + }, + { + "language": "nl", + "id": "community-garden", + "title": "De buurttuin", + "text": "Buren veranderden een lege binnenplaats naast een café in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water." + }, + { + "language": "no", + "id": "community-garden", + "title": "Felleshagen", + "text": "Naboene gjorde en tom gård til en grønn hage, barna malte skilt og familiene vanner plantene hver uke." + }, + { + "language": "pt", + "id": "community-garden", + "title": "A horta comunitária", + "text": "Os vizinhos transformaram um pátio vazio em jardim, as crianças pintaram placas e as famílias regam as plantas toda semana." + }, + { + "language": "sv", + "id": "community-garden", + "title": "Gemensam trädgård", + "text": "Grannarna gjorde en tom gård till en grön trädgård, barnen målade skyltar och familjerna vattnar växterna varje vecka." + }, + { + "language": "th", + "id": "community-garden", + "title": "สวนชุมชน", + "text": "เพื่อนบ้านเปลี่ยนลานว่างเป็นสวนสีเขียว เด็ก ๆ วาดป้ายและครอบครัวช่วยกันรดน้ำต้นไม้ทุกสัปดาห์" + }, + { + "language": "tr", + "id": "community-garden", + "title": "Mahalle bahçesi", + "text": "Komşular boş avluyu yeşil bir bahçeye çevirdi, çocuklar renkli tabelalar yaptı ve aileler bitkileri her hafta suluyor." + }, + { + "language": "vi", + "id": "community-garden", + "title": "Vườn cộng đồng", + "text": "Hàng xóm biến sân trống thành khu vườn xanh, trẻ em vẽ biển màu và các gia đình tưới cây mỗi tuần." } ] } diff --git a/package.json b/package.json index e644490..a3e54c5 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "models:build": "node scripts/build-models.mjs", "models:evaluate": "node scripts/evaluate-models.mjs", "models:verify": "node scripts/verify-models.mjs", + "benchmark": "tsx scripts/benchmark.mjs", + "validate": "tsx scripts/validate.mjs", "format": "prettier --write \"**/*.{ts,json,md,mjs}\"", "format:check": "prettier --list-different \"**/*.{ts,json,md,mjs}\"", "test": "npm run test:typecheck && vitest run --coverage", diff --git a/scripts/benchmark.mjs b/scripts/benchmark.mjs new file mode 100644 index 0000000..f5be321 --- /dev/null +++ b/scripts/benchmark.mjs @@ -0,0 +1,66 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../src/index.ts'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const generatedCorpus = join(root, 'corpus', 'generated'); +const corpusIndex = JSON.parse( + readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), +); + +function option(name, fallback) { + const prefix = `--${name}=`; + const arg = process.argv.find((value) => value.startsWith(prefix)); + return arg ? arg.slice(prefix.length) : fallback; +} + +const iterations = Number(option('iterations', '100')); +const split = option('split', 'all'); + +if (!Number.isInteger(iterations) || iterations < 1) { + throw new Error('--iterations must be a positive integer'); +} + +const rows = corpusIndex.filter( + (row) => split === 'all' || row.split === split, +); +if (rows.length === 0) { + throw new Error(`No corpus files matched split: ${split}`); +} + +const inputs = rows.map((row) => ({ + ...row, + buffer: readFileSync(join(generatedCorpus, row.path)), +})); + +for (const input of inputs) analyse(input.buffer); + +const startedAt = process.hrtime.bigint(); +let matches = 0; +let bytes = 0; + +for (let iteration = 0; iteration < iterations; iteration += 1) { + for (const input of inputs) { + matches += analyse(input.buffer).length; + bytes += input.buffer.length; + } +} + +const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; +const calls = inputs.length * iterations; +const bytesPerSecond = bytes / (elapsedMs / 1000); + +console.log('Library corpus benchmark'); +console.log('========================'); +console.log(`Split: ${split}`); +console.log(`Files: ${inputs.length}`); +console.log(`Iterations: ${iterations}`); +console.log(`Calls: ${calls}`); +console.log(`Bytes processed: ${bytes}`); +console.log(`Matches produced: ${matches}`); +console.log(`Elapsed: ${elapsedMs.toFixed(2)} ms`); +console.log(`Throughput: ${(calls / (elapsedMs / 1000)).toFixed(2)} files/sec`); +console.log( + `Byte throughput: ${(bytesPerSecond / 1024 / 1024).toFixed(2)} MiB/sec`, +); diff --git a/scripts/corpus-utils.mjs b/scripts/corpus-utils.mjs index 0d6b230..bdb185b 100644 --- a/scripts/corpus-utils.mjs +++ b/scripts/corpus-utils.mjs @@ -137,13 +137,22 @@ export function buildCorpus(destination) { let modelHighByteCount = 0; const modelHighByteValues = new Set(); for (const document of language.documents) { - const utf8 = Buffer.from( - `${document.title}\n\n${document.text}\n`, - 'utf8', - ); + let sourceText = `${document.title}\n\n${document.text}\n`; + for (const [from, to] of Object.entries( + encoding.sourceReplacements ?? {}, + )) { + sourceText = sourceText.replaceAll(from, to); + } + if (encoding.sourceSuffix) { + sourceText = `${sourceText.trimEnd()}${encoding.sourceSuffix}\n`; + } + const utf8 = Buffer.from(sourceText, 'utf8'); const encoded = iconv(utf8, 'UTF-8', encoding.iconv); const decoded = iconv(encoded, encoding.iconv, 'UTF-8'); - if (!decoded.equals(utf8)) { + if ( + decoded.toString('utf8').normalize('NFC') !== + utf8.toString('utf8').normalize('NFC') + ) { throw new Error( `${encoding.name}/${languageCode}/${document.id} did not round trip`, ); diff --git a/scripts/evaluate-models.mjs b/scripts/evaluate-models.mjs index f80c70d..2b557e0 100644 --- a/scripts/evaluate-models.mjs +++ b/scripts/evaluate-models.mjs @@ -4,7 +4,11 @@ const { report } = compileModels(); for (const result of report.results) { const expected = `${result.expected.encoding}/${result.expected.language}`; const predicted = `${result.predicted.encoding}/${result.predicted.language}`; - const marker = result.encodingCorrect ? 'PASS' : 'FAIL'; + const marker = result.encodingExact + ? 'PASS' + : result.encodingEquivalent + ? 'EQUIV' + : 'FAIL'; console.log( `${marker} ${expected.padEnd(24)} -> ${predicted.padEnd(24)} ${result.predicted.confidence}%`, ); @@ -12,6 +16,8 @@ for (const result of report.results) { console.log( `\nEncoding: ${report.summary.encodingCorrect}/${report.summary.tests}; ` + + `exact: ${report.summary.encodingExact}/${report.summary.tests}; ` + + `equivalent: ${report.summary.encodingEquivalent}; ` + `language: ${report.summary.languageCorrect}/${report.summary.tests}; ` + `top-score ties: ${report.summary.topScoreTies}`, ); diff --git a/scripts/model-utils.mjs b/scripts/model-utils.mjs index 7c66d93..f9e9f15 100644 --- a/scripts/model-utils.mjs +++ b/scripts/model-utils.mjs @@ -197,6 +197,39 @@ function statisticallyCompetitive(best, candidate) { return best.hitRate - candidate.hitRate <= margin; } +function decodedText(buffer, encodingName) { + const encoding = manifest.encodings.find( + (candidate) => candidate.name === encodingName, + ); + if (!encoding) throw new Error(`Missing manifest encoding: ${encodingName}`); + return iconv(buffer, encoding.iconv, 'UTF-8'); +} + +function byteEquivalent(buffer, leftEncoding, rightEncoding) { + try { + return decodedText(buffer, leftEncoding).equals( + decodedText(buffer, rightEncoding), + ); + } catch { + return false; + } +} + +const equivalentEncodingFamilies = [ + ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], + ['ISO-8859-2', 'windows-1250'], + ['ISO-8859-7', 'windows-1253'], + ['ISO-8859-8', 'windows-1255'], + ['ISO-8859-9', 'windows-1254'], + ['ISO-8859-13', 'windows-1257'], +]; + +function encodingFamily(name) { + return ( + equivalentEncodingFamilies.find((family) => family.includes(name)) ?? [name] + ); +} + function compileSingleByteModels() { const models = []; for (const encoding of manifest.encodings) { @@ -250,15 +283,10 @@ function evaluate(models) { (left, right) => right.byteLogLikelihood - left.byteLogLikelihood, ); const confidenceLanguage = confidenceLanguages[0]; - languageScores.sort( - (left, right) => - right.byteLogLikelihood - left.byteLogLikelihood || - right.confidence - left.confidence, - ); return { encoding: model.encoding, confidence: confidenceScore, - language: languageScores[0].language, + language: confidenceLanguage.language, byteLogLikelihood: confidenceLanguage.byteLogLikelihood, hits: confidenceLanguage.hits, total: confidenceLanguage.total, @@ -282,14 +310,42 @@ function evaluate(models) { (candidate) => !competitiveEncodings.has(candidate.encoding), ); candidates.splice(0, candidates.length, ...competitive, ...remaining); - const predicted = candidates[0]; + const strongestPrediction = candidates[0]; + const strongestFamily = encodingFamily(strongestPrediction.encoding); + const equivalentLanguageCandidates = candidates.filter( + (candidate) => + strongestFamily.includes(candidate.encoding) && + byteEquivalent( + buffer, + strongestPrediction.encoding, + candidate.encoding, + ), + ); + equivalentLanguageCandidates.sort( + (left, right) => + manifest.encodings.findIndex( + (encoding) => encoding.name === left.encoding, + ) - + manifest.encodings.findIndex( + (encoding) => encoding.name === right.encoding, + ), + ); + const predicted = { + ...strongestPrediction, + language: equivalentLanguageCandidates[0].language, + }; + const encodingExact = predicted.encoding === test.encoding; + const encodingEquivalent = + !encodingExact && + byteEquivalent(buffer, test.encoding, predicted.encoding); + const encodingCorrect = encodingExact || encodingEquivalent; return { expected: { encoding: test.encoding, language: test.language }, predicted, - encodingCorrect: predicted.encoding === test.encoding, - languageCorrect: - predicted.encoding === test.encoding && - predicted.language === test.language, + encodingExact, + encodingEquivalent, + encodingCorrect, + languageCorrect: encodingCorrect && predicted.language === test.language, tiedAtTop: candidates.filter( (candidate) => candidate.confidence === predicted.confidence, ).length, @@ -308,6 +364,9 @@ function evaluate(models) { return { summary: { tests: results.length, + encodingExact: results.filter((result) => result.encodingExact).length, + encodingEquivalent: results.filter((result) => result.encodingEquivalent) + .length, encodingCorrect: results.filter((result) => result.encodingCorrect) .length, languageCorrect: results.filter((result) => result.languageCorrect) diff --git a/scripts/validate.mjs b/scripts/validate.mjs new file mode 100644 index 0000000..80c2aeb --- /dev/null +++ b/scripts/validate.mjs @@ -0,0 +1,155 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../src/index.ts'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const generatedCorpus = join(root, 'corpus', 'generated'); +const corpusIndex = JSON.parse( + readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), +); + +function option(name, fallback) { + const prefix = `--${name}=`; + const arg = process.argv.find((value) => value.startsWith(prefix)); + return arg ? arg.slice(prefix.length) : fallback; +} + +function percent(value, total) { + return total === 0 ? '0.00%' : `${((value / total) * 100).toFixed(2)}%`; +} + +function increment(object, key) { + object[key] = (object[key] ?? 0) + 1; +} + +const split = option('split', 'all'); +const rows = corpusIndex.filter( + (row) => split === 'all' || row.split === split, +); +if (rows.length === 0) { + throw new Error(`No corpus files matched split: ${split}`); +} + +const results = rows.map((row) => { + const buffer = readFileSync(join(generatedCorpus, row.path)); + const matches = analyse(buffer); + const predicted = matches[0] ?? null; + return { + expected: { + encoding: row.encoding, + language: row.language, + split: row.split, + path: row.path, + }, + predicted, + encodingCorrect: predicted?.name === row.encoding, + languageCorrect: + predicted?.name === row.encoding && predicted?.lang === row.language, + }; +}); + +const summary = { + files: results.length, + encodingCorrect: results.filter((result) => result.encodingCorrect).length, + languageCorrect: results.filter((result) => result.languageCorrect).length, + noPrediction: results.filter((result) => result.predicted === null).length, +}; + +const bySplit = {}; +const byExpectedEncoding = {}; +const topPredictions = {}; + +for (const result of results) { + const splitSummary = (bySplit[result.expected.split] ??= { + files: 0, + encodingCorrect: 0, + languageCorrect: 0, + }); + splitSummary.files += 1; + if (result.encodingCorrect) splitSummary.encodingCorrect += 1; + if (result.languageCorrect) splitSummary.languageCorrect += 1; + + const encodingSummary = (byExpectedEncoding[result.expected.encoding] ??= { + files: 0, + encodingCorrect: 0, + languageCorrect: 0, + predictions: {}, + }); + encodingSummary.files += 1; + if (result.encodingCorrect) encodingSummary.encodingCorrect += 1; + if (result.languageCorrect) encodingSummary.languageCorrect += 1; + increment(encodingSummary.predictions, result.predicted?.name ?? ''); + increment(topPredictions, result.predicted?.name ?? ''); +} + +console.log('Library corpus validation'); +console.log('========================='); +console.log(`Split: ${split}`); +console.log(`Files: ${summary.files}`); +console.log( + `Encoding correct: ${summary.encodingCorrect}/${summary.files} (${percent( + summary.encodingCorrect, + summary.files, + )})`, +); +console.log( + `Language correct: ${summary.languageCorrect}/${summary.files} (${percent( + summary.languageCorrect, + summary.files, + )})`, +); +console.log(`No prediction: ${summary.noPrediction}`); + +console.log('\nBy split'); +console.table( + Object.entries(bySplit).map(([name, item]) => ({ + split: name, + files: item.files, + encodingCorrect: item.encodingCorrect, + encodingAccuracy: percent(item.encodingCorrect, item.files), + languageCorrect: item.languageCorrect, + languageAccuracy: percent(item.languageCorrect, item.files), + })), +); + +console.log('\nBy expected encoding'); +console.table( + Object.entries(byExpectedEncoding) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([encoding, item]) => ({ + encoding, + files: item.files, + encodingCorrect: item.encodingCorrect, + encodingAccuracy: percent(item.encodingCorrect, item.files), + languageCorrect: item.languageCorrect, + mostCommonPrediction: Object.entries(item.predictions).sort( + ([leftName, leftCount], [rightName, rightCount]) => + rightCount - leftCount || leftName.localeCompare(rightName), + )[0][0], + })), +); + +console.log('\nTop predicted encodings'); +console.table( + Object.entries(topPredictions) + .sort( + ([leftName, leftCount], [rightName, rightCount]) => + rightCount - leftCount || leftName.localeCompare(rightName), + ) + .map(([encoding, count]) => ({ encoding, count })), +); + +const failures = results.filter((result) => !result.encodingCorrect); +console.log('\nFirst 20 encoding mismatches'); +console.table( + failures.slice(0, 20).map((result) => ({ + expected: result.expected.encoding, + language: result.expected.language, + split: result.expected.split, + predicted: result.predicted?.name ?? '', + predictedLanguage: result.predicted?.lang ?? '', + confidence: result.predicted?.confidence ?? '', + path: result.expected.path, + })), +); diff --git a/src/encoding/models/generated.ts b/src/encoding/models/generated.ts index 0f95a79..4f81c87 100644 --- a/src/encoding/models/generated.ts +++ b/src/encoding/models/generated.ts @@ -1,6 +1,1995 @@ // Generated by npm run models:build. Do not edit manually. export const generatedSBCSModels = [ + { + encoding: 'ISO-8859-1', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'da', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206279, 0x2062f8, 0x206465, 0x2066f8, + 0x20686f, 0x206ce6, 0x206f67, 0x2070e5, 0x2073e6, 0x2076e5, 0x2076e6, + 0x20e562, 0x20e662, 0x6166e9, 0x626962, 0x626c69, 0x6272f8, 0x627965, + 0x62f867, 0x62f86e, 0x642062, 0x6420e6, 0x646520, 0x64656e, 0x646572, + 0x652062, 0x652066, 0x6520e5, 0x656420, 0x656e20, 0x656e73, 0x657220, + 0x65726e, 0x657420, 0x66e965, 0x66f872, 0x676572, 0x686ae6, 0x69626c, + 0x696465, 0x696c6c, 0x696f74, 0x6ae66c, 0x6b7374, 0x6c6520, 0x6c696f, + 0x6c6c65, 0x6ce673, 0x6e6465, 0x6e6520, 0x6e6572, 0x6f6720, 0x6f7465, + 0x70e520, 0x722020, 0x722065, 0x72206f, 0x737465, 0x74656b, 0xe52064, + 0xf86765, + ], + highBytes: { + total: 16, + counts: [ + [0xe5, 4], + [0xe6, 5], + [0xe9, 1], + [0xf8, 6], + ], + }, + }, + { + language: 'de', + ngrams: [ + 0x206175, 0x206265, 0x206269, 0x2062e4, 0x2062fc, 0x206465, 0x206469, + 0x2064e4, 0x206569, 0x2066fc, 0x2067e4, 0x206be4, 0x206ce4, 0x207465, + 0x2074fc, 0x20756e, 0x20e470, 0x20f666, 0x20fc62, 0x616368, 0x616d20, + 0x617573, 0x626572, 0x62e463, 0x62fc63, 0x636865, 0x636874, 0x6368fc, + 0x64656e, 0x646572, 0x646965, 0x64e463, 0x652073, 0x652074, 0x65696e, + 0x656dfc, 0x656e20, 0x656efc, 0x657220, 0x65726e, 0x66fc68, 0x66fc72, + 0x67e473, 0x68656e, 0x687265, 0x696368, 0x696520, 0x696e20, 0x696e65, + 0x6c6569, 0x6e2062, 0x6e2064, 0x6e20fc, 0x6e6420, 0x6e656e, 0x736368, + 0x736520, 0x737461, 0x742065, 0x746520, 0x756e64, 0xe46e64, 0xfc6265, + 0xfc7220, + ], + highBytes: { + total: 21, + counts: [ + [0xc4, 1], + [0xdc, 1], + [0xe4, 7], + [0xf6, 1], + [0xfc, 11], + ], + }, + }, + { + language: 'en', + ngrams: [ + 0x206120, 0x20616e, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206661, + 0x206c69, 0x206e65, 0x2070e2, 0x207265, 0x2072e9, 0x2072f4, 0x207468, + 0x207768, 0x616420, 0x616465, 0x6166e9, 0x616e64, 0x61e761, 0x61ef76, + 0x626f75, 0x627261, 0x636974, 0x642063, 0x64206e, 0x646520, 0x646573, + 0x652061, 0x652063, 0x656164, 0x657273, 0x657320, 0x657369, 0x657420, + 0x65f16f, 0x6661e7, 0x66e973, 0x686520, 0x68696c, 0x696272, 0x696465, + 0x696c65, 0x6c6520, 0x6de920, 0x6e61ef, 0x6e6420, 0x7065f1, 0x70e274, + 0x726561, 0x727320, 0x72e973, 0x72f46c, 0x732061, 0x746865, 0x74e920, + 0x756de9, 0xe274e9, 0xe76164, 0xe92020, 0xe92077, 0xe97320, 0xe97375, + 0xef7665, + ], + highBytes: { + total: 9, + counts: [ + [0xe2, 1], + [0xe7, 1], + [0xe9, 4], + [0xef, 1], + [0xf1, 1], + [0xf4, 1], + ], + }, + }, + { + language: 'es', + ngrams: [ + 0x206269, 0x206369, 0x20636c, 0x20636f, 0x2063f3, 0x206465, 0x206469, + 0x20656c, 0x206c61, 0x206c6f, 0x206d65, 0x206de1, 0x207061, 0x2070e1, + 0x2070fa, 0x207265, 0x207375, 0x2073e1, 0x207665, 0x207920, 0x612062, + 0x612065, 0x612070, 0x61646f, 0x616e20, 0x617261, 0x617320, 0x6174e1, + 0x61f161, 0x626c69, 0x6369e9, 0x6369f3, 0x63f364, 0x656c20, 0x656e20, + 0x656efa, 0x6572ed, 0x657320, 0x65f161, 0x65f16f, 0x69e96e, 0x69f36e, + 0x6c206d, 0x6c6120, 0x6c6f73, 0x6d61f1, 0x6de173, 0x6e206c, 0x6e6120, + 0x6efa20, 0x6f206c, 0x6f7265, 0x6f7320, 0x706172, 0x70e167, 0x70fa62, + 0x726573, 0x72ed61, 0x732061, 0x732063, 0x73206c, 0x7365f1, 0x73e162, + 0x746f72, + ], + highBytes: { + total: 13, + counts: [ + [0xe1, 4], + [0xe9, 1], + [0xed, 1], + [0xf1, 3], + [0xf3, 2], + [0xfa, 2], + ], + }, + }, + { + language: 'fr', + ngrams: [ + 0x20636f, 0x206465, 0x206475, 0x2064e9, 0x206574, 0x206c65, 0x207061, + 0x20706f, 0x207072, 0x2070e2, 0x2072e9, 0x20756e, 0x2076e9, 0x20e963, + 0x20e96c, 0x20e971, 0x6166e9, 0x626c69, 0x6368e9, 0x646573, 0x647520, + 0x64e976, 0x652063, 0x652064, 0x65206c, 0x65206d, 0x652070, 0x6520e9, + 0x656e63, 0x656e74, 0x657320, 0x657420, 0x657572, 0x66e973, 0x68e871, + 0x68e920, 0x6c6120, 0x6c6520, 0x6c6573, 0x6c6c65, 0x6de972, 0x6e7420, + 0x6f7572, 0x706172, 0x7072e9, 0x717565, 0x72206c, 0x726520, 0x727320, + 0x72e970, 0x732063, 0x73206c, 0x732070, 0x742064, 0x74206c, 0x7468e8, + 0x756520, 0x756de9, 0x757273, 0xe87175, 0xe96520, 0xe97061, 0xe97269, + 0xe97665, + ], + highBytes: { + total: 22, + counts: [ + [0xe2, 1], + [0xe8, 3], + [0xe9, 18], + ], + }, + }, + { + language: 'it', + ngrams: [ + 0x00006c, 0x00006d, 0x006c61, 0x206269, 0x206361, 0x206369, 0x20636f, + 0x206465, 0x206469, 0x206520, 0x206765, 0x20676c, 0x206920, 0x206c61, + 0x206c65, 0x206d65, 0x207065, 0x207069, 0x207072, 0x207363, 0x207369, + 0x207374, 0x612062, 0x612063, 0x612064, 0x612067, 0x612069, 0x612070, + 0x61746f, 0x6368e9, 0x636974, 0x64656c, 0x656e74, 0x657263, 0x6666e8, + 0x66e820, 0x676c69, 0x68e920, 0x692061, 0x692063, 0x692073, 0x696120, + 0x697474, 0x69f920, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6f20, + 0x6f7269, 0x7069f9, 0x726920, 0x746f72, 0x7474e0, 0x74e020, 0xe02020, + 0xe0206c, 0xe02073, 0xe82061, 0xe8206d, 0xe92073, 0xf92067, 0xf9206c, + 0xf92070, + ], + highBytes: { + total: 9, + counts: [ + [0xe0, 3], + [0xe8, 2], + [0xe9, 1], + [0xf9, 3], + ], + }, + }, + { + language: 'nl', + ngrams: [ + 0x000064, 0x206269, 0x20626f, 0x206361, 0x20636f, 0x206465, 0x206565, + 0x20656e, 0x206765, 0x206865, 0x206c65, 0x206f70, 0x206f76, 0x207374, + 0x207465, 0x207665, 0x207765, 0x20e9e9, 0x61616b, 0x616173, 0x616420, + 0x6166e9, 0x616b74, 0x616d20, 0x626962, 0x626c69, 0x626f65, 0x636166, + 0x636874, 0x646520, 0x652062, 0x652063, 0x652065, 0x652073, 0x65656e, + 0x656572, 0x6565eb, 0x65696e, 0x656b65, 0x656e20, 0x657273, 0x65eb6e, + 0x65ef6e, 0x66e973, 0x6765ef, 0x696e20, 0x6b7420, 0x6e2020, 0x6e206f, + 0x6e20e9, 0x6feb7a, 0x706feb, 0x732065, 0x736368, 0x742064, 0x74656e, + 0x746572, 0x766572, 0xe96e20, 0xe97320, 0xe9e96e, 0xeb6e20, 0xeb7a69, + 0xef6e74, + ], + highBytes: { + total: 7, + counts: [ + [0xe9, 4], + [0xeb, 2], + [0xef, 1], + ], + }, + }, + { + language: 'no', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206272, 0x206279, 0x2062f8, 0x206465, + 0x2064f8, 0x20666f, 0x2066f8, 0x206c61, 0x206ee6, 0x206f67, 0x2070e5, + 0x207665, 0x2076e5, 0x20e570, 0x20e672, 0x626962, 0x626c69, 0x6272f8, + 0x627965, 0x62f86b, 0x62f86e, 0x642065, 0x646520, 0x646572, 0x646574, + 0x64f872, 0x652062, 0x652066, 0x652074, 0x6520e5, 0x6520e6, 0x656420, + 0x656e20, 0x656e65, 0x656e73, 0x657220, 0x657273, 0x657420, 0x66f872, + 0x672073, 0x676520, 0x67656e, 0x676572, 0x69626c, 0x696765, 0x696e67, + 0x696f74, 0x6af865, 0x6cf872, 0x6e6520, 0x6e6572, 0x6ee672, 0x6f6720, + 0x70e520, 0x72206f, 0x7273f8, 0x72f864, 0x737465, 0x74656b, 0xe52064, + 0xf86b65, + ], + highBytes: { + total: 14, + counts: [ + [0xe5, 4], + [0xe6, 2], + [0xf8, 8], + ], + }, + }, + { + language: 'pt', + ngrams: [ + 0x00006d, 0x206120, 0x206269, 0x206369, 0x2063f3, 0x206465, 0x206520, + 0x206573, 0x206e61, 0x206f20, 0x206f73, 0x207065, 0x207072, 0x2070e3, + 0x2073e1, 0x207665, 0x20f46e, 0x612062, 0x612063, 0x612064, 0x612065, + 0x61206f, 0x612070, 0x616465, 0x61646f, 0x6166e9, 0x616d20, 0x617261, + 0x617320, 0x61e761, 0x61e7e3, 0x626962, 0x626c69, 0x636120, 0x6369ea, + 0x63e172, 0x63f364, 0x646120, 0x646520, 0x646f20, 0x652061, 0x6563e1, + 0x657320, 0x657363, 0x66e973, 0x68e320, 0x696120, 0x69e7f5, 0x69ea6e, + 0x6d61e7, 0x6e6120, 0x6e68e3, 0x6f7320, 0x70e36f, 0x7261e7, 0x726573, + 0x7269e7, 0x732020, 0x732061, 0x732070, 0x7320f4, 0x7374f3, 0x73e162, + 0x74f372, + ], + highBytes: { + total: 14, + counts: [ + [0xe1, 2], + [0xe3, 3], + [0xe7, 3], + [0xe9, 1], + [0xea, 1], + [0xf3, 2], + [0xf4, 1], + [0xf5, 1], + ], + }, + }, + { + language: 'sv', + ngrams: [ + 0x000064, 0x006cf6, 0x206269, 0x2062f6, 0x206465, 0x2066e4, 0x2066f6, + 0x206c69, 0x206ce4, 0x206ce5, 0x206de4, 0x206f63, 0x2070e5, 0x20736b, + 0x207374, 0x2073e4, 0x2073f6, 0x207479, 0x207665, 0x2076e4, 0x20e470, + 0x20f670, 0x612020, 0x612062, 0x612074, 0x6120f6, 0x616420, 0x616465, + 0x6166e9, 0x617220, 0x617265, 0x626962, 0x626c69, 0x6272f6, 0x62f663, + 0x62f66e, 0x636820, 0x6420e4, 0x646520, 0x64656e, 0x646574, 0x656e20, + 0x657220, 0x65726e, 0x6573f6, 0x657420, 0x66f672, 0x676120, 0x69626c, + 0x696761, 0x696f74, 0x6b6170, 0x6b6172, 0x6e6120, 0x6f6368, 0x70e520, + 0x726e61, 0x736b61, 0x737461, 0x73f66b, 0x74206c, 0x746164, 0x766572, + 0xe52064, + ], + highBytes: { + total: 20, + counts: [ + [0xe4, 7], + [0xe5, 3], + [0xe9, 1], + [0xf6, 9], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-2', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0x20, + 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0x20, 0xb1, 0x20, 0xb3, + 0x20, 0xb5, 0xb6, 0xb7, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'cs', + ngrams: [ + 0x000064, 0x000072, 0x000073, 0x000074, 0x006469, 0x0072e1, 0x00736f, + 0x007469, 0x206120, 0x206175, 0x206279, 0x206368, 0x206476, 0x2064ed, + 0x206661, 0x206869, 0x20686c, 0x206b6e, 0x206bf3, 0x206dec, 0x20706f, + 0x207072, 0x2070ed, 0x2070f8, 0x2074fd, 0x2076ec, 0x20bee1, 0x20e865, + 0x20e874, 0x612070, 0x612076, 0x6120e8, 0x6164ed, 0x616aed, 0x616cfd, + 0x6174ed, 0x636520, 0x652061, 0x65206d, 0x652070, 0x686f76, 0x69686f, + 0x6a6520, 0x6aed20, 0x6b6e69, 0x6d616c, 0x6dec73, 0x6e6120, 0x6e6968, + 0x6e7920, 0x6eed20, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, 0x737920, + 0x756a65, 0x792020, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xed206b, + 0xf86920, + ], + highBytes: { + total: 46, + counts: [ + [0xb9, 1], + [0xbe, 1], + [0xc8, 1], + [0xe1, 8], + [0xe8, 2], + [0xe9, 4], + [0xec, 5], + [0xed, 13], + [0xf2, 1], + [0xf3, 1], + [0xf8, 4], + [0xf9, 2], + [0xfd, 3], + ], + }, + }, + { + language: 'hu', + ngrams: [ + 0x000061, 0x000064, 0x000072, 0x000073, 0x006120, 0x006469, 0x007265, + 0x00737a, 0x206120, 0x20616c, 0x20617a, 0x206275, 0x206373, 0x206b65, + 0x206b69, 0x206be1, 0x206be9, 0x206bf3, 0x206bf6, 0x206de9, 0x206dfa, + 0x206dfb, 0x2074e9, 0x2074f6, 0x2076e1, 0x20e172, 0x20e962, 0x20e973, + 0x20ed72, 0x20fa74, 0x61206b, 0x612076, 0x616b20, 0x616e20, 0x617420, + 0x617a20, 0x646967, 0x656b20, 0x657265, 0x657420, 0x697320, 0x6b2020, + 0x6b2061, 0x6bf66e, 0x6d6920, 0x6e616b, 0x6e7976, 0x6f7320, 0x6f7420, + 0x726f73, 0x73206b, 0x73206c, 0x73206d, 0x742061, 0x74206b, 0x74e172, + 0x7674e1, 0x76e172, 0x797674, 0xe1726f, 0xe97320, 0xed7420, 0xf36b20, + 0xf66e79, + ], + highBytes: { + total: 39, + counts: [ + [0xe1, 14], + [0xe9, 9], + [0xed, 4], + [0xf3, 4], + [0xf5, 1], + [0xf6, 4], + [0xfa, 2], + [0xfb, 1], + ], + }, + }, + { + language: 'pl', + ngrams: [ + 0x000063, 0x206120, 0x206269, 0x20637a, 0x20646c, 0x206920, 0x206a61, + 0x206b61, 0x206b6f, 0x206d69, 0x20706f, 0x207072, 0x20726f, 0x2073b3, + 0x207779, 0x20b677, 0x6162b3, 0x616368, 0x616ab1, 0x616d69, 0x6177ea, + 0x61b379, 0x61b66e, 0x61e620, 0x626962, 0x626c69, 0x62b36b, 0x636861, + 0x636920, 0x637a79, 0x6472f3, 0x647a69, 0x656b20, 0x65b36e, 0x65b663, + 0x65bf79, 0x676fb6, 0x6861e6, 0x692073, 0x6920b6, 0x69626c, 0x696520, + 0x696572, 0x696f74, 0x69ea20, 0x6ab120, 0x6b6120, 0x6c696f, 0x6d6920, + 0x6e6961, 0x6f7261, 0x6f7465, 0x6fb663, 0x706973, 0x70727a, 0x727a65, + 0x727a79, 0x7369ea, 0x737920, 0x74656b, 0xb1206a, 0xb1206b, 0xb12077, + 0xb66369, + ], + highBytes: { + total: 34, + counts: [ + [0xb1, 11], + [0xb3, 6], + [0xb6, 6], + [0xbf, 3], + [0xe6, 1], + [0xea, 3], + [0xf3, 4], + ], + }, + }, + { + language: 'ro', + ngrams: [ + 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, + 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, + 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, + 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, + 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, + 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, + 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, + 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, + 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, + 0xfe6120, + ], + highBytes: { + total: 49, + counts: [ + [0xba, 12], + [0xce, 1], + [0xe2, 3], + [0xe3, 21], + [0xee, 4], + [0xfe, 8], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-5', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff, 0xd0, 0xd1, 0xd2, 0xd3, + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0x20, 0xfe, 0xff, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000e0, 0x00e0d0, 0x20d020, 0x20d1d8, 0x20d220, 0x20d2de, 0x20d2eb, + 0x20d3de, 0x20d820, 0x20dade, 0x20dcd0, 0x20ddd0, 0x20ded1, 0x20dee1, + 0x20dee2, 0x20dfdb, 0x20dfde, 0x20dfe0, 0x20e0d5, 0x20e1de, 0x20e7d8, + 0x20efe0, 0xd020d2, 0xd020dd, 0xd020df, 0xd0d5e2, 0xd0ddd8, 0xd0e0da, + 0xd0e1e2, 0xd0e2d5, 0xd0eee2, 0xd0ef20, 0xd1d8d1, 0xd1dbd8, 0xd1e1e3, + 0xd2d0ee, 0xd2d5e0, 0xd2efe2, 0xd3dee0, 0xd3dee2, 0xd4d0ee, 0xd4d820, + 0xd5dbd5, 0xd5dbd8, 0xd5dddd, 0xd820df, 0xdad020, 0xdbd820, 0xdcd0e0, + 0xddd020, 0xddebd5, 0xdee1d5, 0xdfe0de, 0xe0ded4, 0xe0eb20, 0xe220e1, + 0xe2d5db, 0xe2d5e0, 0xe2ded2, 0xe2eb20, 0xebd520, 0xeee220, 0xef20df, + 0xefe220, + ], + highBytes: { + total: 678, + counts: [ + [0xb2, 1], + [0xb3, 1], + [0xb4, 1], + [0xbb, 1], + [0xbd, 3], + [0xbe, 1], + [0xbf, 1], + [0xc0, 2], + [0xc1, 1], + [0xc3, 1], + [0xc4, 1], + [0xc7, 1], + [0xd0, 63], + [0xd1, 16], + [0xd2, 26], + [0xd3, 7], + [0xd4, 21], + [0xd5, 50], + [0xd6, 5], + [0xd7, 7], + [0xd8, 47], + [0xd9, 7], + [0xda, 21], + [0xdb, 19], + [0xdc, 16], + [0xdd, 37], + [0xde, 60], + [0xdf, 23], + [0xe0, 41], + [0xe1, 34], + [0xe2, 55], + [0xe3, 14], + [0xe5, 7], + [0xe6, 3], + [0xe7, 4], + [0xe8, 6], + [0xe9, 2], + [0xeb, 26], + [0xec, 3], + [0xed, 1], + [0xee, 13], + [0xef, 27], + [0xf1, 2], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-6', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'ar', + ngrams: [ + 0x0000c7, 0x0000d3, 0x0000d5, 0x0000e8, 0x00c7e4, 0x00d3e8, 0x00d5c8, + 0x00e8d1, 0x20c3c8, 0x20c3e5, 0x20c7ce, 0x20c7e4, 0x20c8e7, 0x20c8ea, + 0x20cad3, 0x20cfe4, 0x20d1e2, 0x20d5da, 0x20d9e6, 0x20e1d1, 0x20e1ea, + 0x20e3ca, 0x20e4e4, 0x20e8c7, 0x20e8ca, 0x20e8d5, 0x20e8ea, 0xc7ca20, + 0xc7ccd9, 0xc7d120, 0xc7e4c3, 0xc7e4d9, 0xc7e4e5, 0xc7e4e8, 0xc820c7, + 0xc8c920, 0xc92020, 0xc920c7, 0xc920e8, 0xc920ea, 0xca20e8, 0xcac820, + 0xcac8c9, 0xcb20c7, 0xccd920, 0xcd20c7, 0xcfeae6, 0xd120c7, 0xd1c7cc, + 0xd5e1c7, 0xd920c7, 0xe3cac8, 0xe4e5cf, 0xe4e5e3, 0xe5cfea, 0xe5e3ca, + 0xe620c7, 0xe6c7d3, 0xe6c920, 0xe8c7e4, 0xe8d5e1, 0xe8e620, 0xeac920, + 0xeae6c9, + ], + highBytes: { + total: 324, + counts: [ + [0xc1, 2], + [0xc3, 4], + [0xc4, 1], + [0xc6, 1], + [0xc7, 56], + [0xc8, 17], + [0xc9, 11], + [0xca, 16], + [0xcb, 2], + [0xcc, 3], + [0xcd, 8], + [0xce, 4], + [0xcf, 9], + [0xd1, 19], + [0xd2, 3], + [0xd3, 7], + [0xd4, 2], + [0xd5, 5], + [0xd6, 2], + [0xd7, 1], + [0xd8, 2], + [0xd9, 8], + [0xda, 1], + [0xe1, 9], + [0xe2, 7], + [0xe3, 5], + [0xe4, 36], + [0xe5, 18], + [0xe6, 12], + [0xe7, 5], + [0xe8, 24], + [0xea, 24], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-7', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xdc, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'el', + ngrams: [ + 0x0000de, 0x0000f0, 0x0000f3, 0x0000f8, 0x00def3, 0x00f0f1, 0x00f3e1, + 0x00f8e7, 0x20dced, 0x20e1e3, 0x20e1ed, 0x20e1f1, 0x20e2e9, 0x20e2ef, + 0x20e3f1, 0x20e5eb, 0x20e5f0, 0x20e5f1, 0x20e720, 0x20eae1, 0x20ece9, + 0x20efe9, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xdc20f4, 0xdfe120, + 0xe12020, 0xe120ea, 0xe1e920, 0xe2e9e2, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, + 0xe5e920, 0xe5f220, 0xe720e2, 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9e2eb, + 0xe9efe8, 0xe9f3f4, 0xeae1e9, 0xeae720, 0xebe720, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xeff5ed, 0xf0fceb, 0xf1dc20, 0xf1e9ef, + 0xf220ea, 0xf3f4de, 0xf3f4e7, 0xf4e120, 0xf4e5f2, 0xf4e7ed, 0xf5ed20, + 0xfcebe7, + ], + highBytes: { + total: 327, + counts: [ + [0xb9, 1], + [0xc7, 1], + [0xcc, 1], + [0xcf, 2], + [0xd0, 1], + [0xd3, 1], + [0xd8, 1], + [0xdc, 13], + [0xdd, 5], + [0xde, 4], + [0xdf, 9], + [0xe1, 31], + [0xe2, 9], + [0xe3, 9], + [0xe4, 4], + [0xe5, 16], + [0xe6, 1], + [0xe7, 14], + [0xe8, 6], + [0xe9, 31], + [0xea, 16], + [0xeb, 10], + [0xec, 11], + [0xed, 15], + [0xee, 1], + [0xef, 24], + [0xf0, 8], + [0xf1, 16], + [0xf2, 7], + [0xf3, 8], + [0xf4, 17], + [0xf5, 10], + [0xf6, 5], + [0xf7, 3], + [0xf8, 2], + [0xf9, 5], + [0xfc, 5], + [0xfd, 2], + [0xfe, 2], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-8', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'he', + ngrams: [ + 0x0000e1, 0x0000e4, 0x0000f1, 0x0000f9, 0x00e1e5, 0x00e4f1, 0x00f1e3, + 0x00f9e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, 0x20e1fa, + 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f7, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e7f7, 0x20e8f8, + 0x20ece0, 0x20ece1, 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, + 0x20eee7, 0x20eeeb, 0xe0e9ed, 0xe4f1f4, 0xe5f8e9, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed20, 0xe9f820, 0xe9fa20, 0xed2020, 0xed20ec, + 0xed20ee, 0xef20ee, 0xf0e420, 0xf1f4f8, 0xf2e9f8, 0xf4f8e9, 0xf7e5f8, + 0xf820e4, 0xf820ee, 0xf8e5fa, 0xf8e920, 0xf8e9ed, 0xf8fa20, 0xf9e9ed, + 0xfa20ec, + ], + highBytes: { + total: 249, + counts: [ + [0xe0, 9], + [0xe1, 12], + [0xe2, 2], + [0xe3, 8], + [0xe4, 16], + [0xe5, 30], + [0xe6, 2], + [0xe7, 6], + [0xe8, 7], + [0xe9, 33], + [0xea, 1], + [0xeb, 6], + [0xec, 9], + [0xed, 13], + [0xee, 11], + [0xef, 3], + [0xf0, 6], + [0xf1, 7], + [0xf2, 6], + [0xf4, 7], + [0xf6, 1], + [0xf7, 10], + [0xf8, 22], + [0xf9, 7], + [0xfa, 15], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-9', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'tr', + ngrams: [ + 0x000063, 0x000064, 0x000073, 0x0000fe, 0x006375, 0x00fe65, 0x2061e7, + 0x206269, 0x20656b, 0x206b61, 0x206bfc, 0x206dfc, 0x207361, 0x207461, + 0x207665, 0x207961, 0x20e769, 0x20f6f0, 0x20fe65, 0x616261, 0x616c20, + 0x616c61, 0x616cfd, 0x616e65, 0x616efd, 0x617220, 0x6172fd, 0x6174f6, + 0x617a61, 0x617afd, 0x61e7fd, 0x61fee7, 0x62616c, 0x652062, 0x656869, + 0x657220, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697220, 0x697461, + 0x6bfc74, 0x6c6172, 0x6c6572, 0x706861, 0x722020, 0x72206b, 0x722076, + 0x722079, 0x726920, 0x726c61, 0x72fd20, 0x746172, 0x74fc70, 0x766520, + 0x7a6172, 0x7a6520, 0xe7fd6b, 0xfc7068, 0xfc74fc, 0xfd6b20, 0xfd7220, + 0xfe6568, + ], + highBytes: { + total: 32, + counts: [ + [0xc7, 1], + [0xde, 2], + [0xe7, 6], + [0xf0, 1], + [0xf6, 2], + [0xfc, 8], + [0xfd, 11], + [0xfe, 1], + ], + }, + }, + ], + }, + { + encoding: 'windows-1250', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa1, 0x20, 0xb3, 0x20, 0xb9, 0x20, 0x20, + 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0x20, 0xb3, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0xb9, 0xba, 0x20, 0xbe, 0x20, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'cs', + ngrams: [ + 0x000064, 0x000072, 0x000073, 0x000074, 0x006469, 0x0072e1, 0x00736f, + 0x007469, 0x206120, 0x206175, 0x206279, 0x206368, 0x206476, 0x2064ed, + 0x206661, 0x206869, 0x20686c, 0x206b6e, 0x206bf3, 0x206dec, 0x20706f, + 0x207072, 0x2070ed, 0x2070f8, 0x2074fd, 0x2076ec, 0x209ee1, 0x20e865, + 0x20e874, 0x612070, 0x612076, 0x6120e8, 0x6164ed, 0x616aed, 0x616cfd, + 0x6174ed, 0x636520, 0x652061, 0x65206d, 0x652070, 0x686f76, 0x69686f, + 0x6a6520, 0x6aed20, 0x6b6e69, 0x6d616c, 0x6dec73, 0x6e6120, 0x6e6968, + 0x6e7920, 0x6eed20, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, 0x737920, + 0x756a65, 0x792020, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xed206b, + 0xf86920, + ], + highBytes: { + total: 46, + counts: [ + [0x9a, 1], + [0x9e, 1], + [0xc8, 1], + [0xe1, 8], + [0xe8, 2], + [0xe9, 4], + [0xec, 5], + [0xed, 13], + [0xf2, 1], + [0xf3, 1], + [0xf8, 4], + [0xf9, 2], + [0xfd, 3], + ], + }, + }, + { + language: 'hu', + ngrams: [ + 0x000061, 0x000064, 0x000072, 0x000073, 0x006120, 0x006469, 0x007265, + 0x00737a, 0x206120, 0x20616c, 0x20617a, 0x206275, 0x206373, 0x206b65, + 0x206b69, 0x206be1, 0x206be9, 0x206bf3, 0x206bf6, 0x206de9, 0x206dfa, + 0x206dfb, 0x2074e9, 0x2074f6, 0x2076e1, 0x20e172, 0x20e962, 0x20e973, + 0x20ed72, 0x20fa74, 0x61206b, 0x612076, 0x616b20, 0x616e20, 0x617420, + 0x617a20, 0x646967, 0x656b20, 0x657265, 0x657420, 0x697320, 0x6b2020, + 0x6b2061, 0x6bf66e, 0x6d6920, 0x6e616b, 0x6e7976, 0x6f7320, 0x6f7420, + 0x726f73, 0x73206b, 0x73206c, 0x73206d, 0x742061, 0x74206b, 0x74e172, + 0x7674e1, 0x76e172, 0x797674, 0xe1726f, 0xe97320, 0xed7420, 0xf36b20, + 0xf66e79, + ], + highBytes: { + total: 39, + counts: [ + [0xe1, 14], + [0xe9, 9], + [0xed, 4], + [0xf3, 4], + [0xf5, 1], + [0xf6, 4], + [0xfa, 2], + [0xfb, 1], + ], + }, + }, + { + language: 'pl', + ngrams: [ + 0x000063, 0x206120, 0x206269, 0x20637a, 0x20646c, 0x206920, 0x206a61, + 0x206b61, 0x206b6f, 0x206d69, 0x20706f, 0x207072, 0x20726f, 0x2073b3, + 0x207779, 0x209c77, 0x6162b3, 0x616368, 0x616ab9, 0x616d69, 0x6177ea, + 0x619c6e, 0x61b379, 0x61e620, 0x626962, 0x626c69, 0x62b36b, 0x636861, + 0x636920, 0x637a79, 0x6472f3, 0x647a69, 0x656b20, 0x659c63, 0x65b36e, + 0x65bf79, 0x676f9c, 0x6861e6, 0x692073, 0x69209c, 0x69626c, 0x696520, + 0x696572, 0x696f74, 0x69ea20, 0x6ab920, 0x6b6120, 0x6c696f, 0x6d6920, + 0x6e6961, 0x6f7261, 0x6f7465, 0x6f9c63, 0x706973, 0x70727a, 0x727a65, + 0x727a79, 0x7369ea, 0x737920, 0x74656b, 0x9c6369, 0xb9206a, 0xb9206b, + 0xb92077, + ], + highBytes: { + total: 34, + counts: [ + [0x9c, 6], + [0xb3, 6], + [0xb9, 11], + [0xbf, 3], + [0xe6, 1], + [0xea, 3], + [0xf3, 4], + ], + }, + }, + { + language: 'ro', + ngrams: [ + 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, + 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, + 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, + 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, + 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, + 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, + 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, + 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, + 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, + 0xfe6120, + ], + highBytes: { + total: 49, + counts: [ + [0xba, 12], + [0xce, 1], + [0xe2, 3], + [0xe3, 21], + [0xee, 4], + [0xfe, 8], + ], + }, + }, + ], + }, + { + encoding: 'windows-1251', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x90, 0x83, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa2, 0xa2, 0xbc, 0x20, 0xb4, 0x20, 0x20, + 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0xb3, 0xb3, + 0xb4, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0xbc, 0xbe, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000f0, 0x00f0e0, 0x20e020, 0x20e1e8, 0x20e220, 0x20e2ee, 0x20e2fb, + 0x20e3ee, 0x20e820, 0x20eaee, 0x20ece0, 0x20ede0, 0x20eee1, 0x20eef1, + 0x20eef2, 0x20efeb, 0x20efee, 0x20eff0, 0x20f0e5, 0x20f1ee, 0x20f7e8, + 0x20fff0, 0xe020e2, 0xe020ed, 0xe020ef, 0xe0e5f2, 0xe0ede8, 0xe0f0ea, + 0xe0f1f2, 0xe0f2e5, 0xe0fef2, 0xe0ff20, 0xe1e8e1, 0xe1ebe8, 0xe1f1f3, + 0xe2e0fe, 0xe2e5f0, 0xe2fff2, 0xe3eef0, 0xe3eef2, 0xe4e0fe, 0xe4e820, + 0xe5ebe5, 0xe5ebe8, 0xe5eded, 0xe820ef, 0xeae020, 0xebe820, 0xece0f0, + 0xede020, 0xedfbe5, 0xeef1e5, 0xeff0ee, 0xf0eee4, 0xf0fb20, 0xf220f1, + 0xf2e5eb, 0xf2e5f0, 0xf2eee2, 0xf2fb20, 0xfbe520, 0xfef220, 0xff20ef, + 0xfff220, + ], + highBytes: { + total: 678, + counts: [ + [0xb8, 2], + [0xc2, 1], + [0xc3, 1], + [0xc4, 1], + [0xcb, 1], + [0xcd, 3], + [0xce, 1], + [0xcf, 1], + [0xd0, 2], + [0xd1, 1], + [0xd3, 1], + [0xd4, 1], + [0xd7, 1], + [0xe0, 63], + [0xe1, 16], + [0xe2, 26], + [0xe3, 7], + [0xe4, 21], + [0xe5, 50], + [0xe6, 5], + [0xe7, 7], + [0xe8, 47], + [0xe9, 7], + [0xea, 21], + [0xeb, 19], + [0xec, 16], + [0xed, 37], + [0xee, 60], + [0xef, 23], + [0xf0, 41], + [0xf1, 34], + [0xf2, 55], + [0xf3, 14], + [0xf5, 7], + [0xf6, 3], + [0xf7, 4], + [0xf8, 6], + [0xf9, 2], + [0xfb, 26], + [0xfc, 3], + [0xfd, 1], + [0xfe, 13], + [0xff, 27], + ], + }, + }, + ], + }, + { + encoding: 'windows-1252', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x9a, 0x20, 0x9c, 0x20, 0x9e, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x9e, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'da', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206279, 0x2062f8, 0x206465, 0x2066f8, + 0x20686f, 0x206ce6, 0x206f67, 0x2070e5, 0x2073e6, 0x2076e5, 0x2076e6, + 0x20e562, 0x20e662, 0x6166e9, 0x626962, 0x626c69, 0x6272f8, 0x627965, + 0x62f867, 0x62f86e, 0x642062, 0x6420e6, 0x646520, 0x64656e, 0x646572, + 0x652062, 0x652066, 0x6520e5, 0x656420, 0x656e20, 0x656e73, 0x657220, + 0x65726e, 0x657420, 0x66e965, 0x66f872, 0x676572, 0x686ae6, 0x69626c, + 0x696465, 0x696c6c, 0x696f74, 0x6ae66c, 0x6b7374, 0x6c6520, 0x6c696f, + 0x6c6c65, 0x6ce673, 0x6e6465, 0x6e6520, 0x6e6572, 0x6f6720, 0x6f7465, + 0x70e520, 0x722020, 0x722065, 0x72206f, 0x737465, 0x74656b, 0xe52064, + 0xf86765, + ], + highBytes: { + total: 16, + counts: [ + [0xe5, 4], + [0xe6, 5], + [0xe9, 1], + [0xf8, 6], + ], + }, + }, + { + language: 'de', + ngrams: [ + 0x206175, 0x206265, 0x206269, 0x2062e4, 0x2062fc, 0x206465, 0x206469, + 0x2064e4, 0x206569, 0x2066fc, 0x2067e4, 0x206be4, 0x206ce4, 0x207465, + 0x2074fc, 0x20756e, 0x20e470, 0x20f666, 0x20fc62, 0x616368, 0x616d20, + 0x617573, 0x626572, 0x62e463, 0x62fc63, 0x636865, 0x636874, 0x6368fc, + 0x64656e, 0x646572, 0x646965, 0x64e463, 0x652073, 0x652074, 0x65696e, + 0x656dfc, 0x656e20, 0x656efc, 0x657220, 0x65726e, 0x66fc68, 0x66fc72, + 0x67e473, 0x68656e, 0x687265, 0x696368, 0x696520, 0x696e20, 0x696e65, + 0x6c6569, 0x6e2062, 0x6e2064, 0x6e20fc, 0x6e6420, 0x6e656e, 0x736368, + 0x736520, 0x737461, 0x742065, 0x746520, 0x756e64, 0xe46e64, 0xfc6265, + 0xfc7220, + ], + highBytes: { + total: 21, + counts: [ + [0xc4, 1], + [0xdc, 1], + [0xe4, 7], + [0xf6, 1], + [0xfc, 11], + ], + }, + }, + { + language: 'en', + ngrams: [ + 0x206120, 0x20616e, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206661, + 0x206c69, 0x206e65, 0x2070e2, 0x207265, 0x2072e9, 0x2072f4, 0x207468, + 0x207768, 0x616420, 0x616465, 0x6166e9, 0x616e64, 0x61e761, 0x61ef76, + 0x626f75, 0x627261, 0x636974, 0x642063, 0x64206e, 0x646520, 0x646573, + 0x652061, 0x652063, 0x656164, 0x657273, 0x657320, 0x657369, 0x657420, + 0x65f16f, 0x6661e7, 0x66e973, 0x686520, 0x68696c, 0x696272, 0x696465, + 0x696c65, 0x6c6520, 0x6de920, 0x6e61ef, 0x6e6420, 0x7065f1, 0x70e274, + 0x726561, 0x727320, 0x72e973, 0x72f46c, 0x732061, 0x746865, 0x74e920, + 0x756de9, 0xe274e9, 0xe76164, 0xe92020, 0xe92077, 0xe97320, 0xe97375, + 0xef7665, + ], + highBytes: { + total: 9, + counts: [ + [0xe2, 1], + [0xe7, 1], + [0xe9, 4], + [0xef, 1], + [0xf1, 1], + [0xf4, 1], + ], + }, + }, + { + language: 'es', + ngrams: [ + 0x206269, 0x206369, 0x20636c, 0x20636f, 0x2063f3, 0x206465, 0x206469, + 0x20656c, 0x206c61, 0x206c6f, 0x206d65, 0x206de1, 0x207061, 0x2070e1, + 0x2070fa, 0x207265, 0x207375, 0x2073e1, 0x207665, 0x207920, 0x612062, + 0x612065, 0x612070, 0x61646f, 0x616e20, 0x617261, 0x617320, 0x6174e1, + 0x61f161, 0x626c69, 0x6369e9, 0x6369f3, 0x63f364, 0x656c20, 0x656e20, + 0x656efa, 0x6572ed, 0x657320, 0x65f161, 0x65f16f, 0x69e96e, 0x69f36e, + 0x6c206d, 0x6c6120, 0x6c6f73, 0x6d61f1, 0x6de173, 0x6e206c, 0x6e6120, + 0x6efa20, 0x6f206c, 0x6f7265, 0x6f7320, 0x706172, 0x70e167, 0x70fa62, + 0x726573, 0x72ed61, 0x732061, 0x732063, 0x73206c, 0x7365f1, 0x73e162, + 0x746f72, + ], + highBytes: { + total: 13, + counts: [ + [0xe1, 4], + [0xe9, 1], + [0xed, 1], + [0xf1, 3], + [0xf3, 2], + [0xfa, 2], + ], + }, + }, + { + language: 'fr', + ngrams: [ + 0x20636f, 0x206465, 0x206475, 0x2064e9, 0x206574, 0x206c65, 0x207061, + 0x20706f, 0x207072, 0x2070e2, 0x2072e9, 0x20756e, 0x2076e9, 0x20e963, + 0x20e96c, 0x20e971, 0x6166e9, 0x626c69, 0x6368e9, 0x646573, 0x647520, + 0x64e976, 0x652063, 0x652064, 0x65206c, 0x65206d, 0x652070, 0x6520e9, + 0x656e63, 0x656e74, 0x657320, 0x657420, 0x657572, 0x66e973, 0x68e871, + 0x68e920, 0x6c6120, 0x6c6520, 0x6c6573, 0x6c6c65, 0x6de972, 0x6e7420, + 0x6f7572, 0x706172, 0x7072e9, 0x717565, 0x72206c, 0x726520, 0x727320, + 0x72e970, 0x732063, 0x73206c, 0x732070, 0x742064, 0x74206c, 0x7468e8, + 0x756520, 0x756de9, 0x757273, 0xe87175, 0xe96520, 0xe97061, 0xe97269, + 0xe97665, + ], + highBytes: { + total: 22, + counts: [ + [0xe2, 1], + [0xe8, 3], + [0xe9, 18], + ], + }, + }, + { + language: 'it', + ngrams: [ + 0x00006c, 0x00006d, 0x006c61, 0x206269, 0x206361, 0x206369, 0x20636f, + 0x206465, 0x206469, 0x206520, 0x206765, 0x20676c, 0x206920, 0x206c61, + 0x206c65, 0x206d65, 0x207065, 0x207069, 0x207072, 0x207363, 0x207369, + 0x207374, 0x612062, 0x612063, 0x612064, 0x612067, 0x612069, 0x612070, + 0x61746f, 0x6368e9, 0x636974, 0x64656c, 0x656e74, 0x657263, 0x6666e8, + 0x66e820, 0x676c69, 0x68e920, 0x692061, 0x692063, 0x692073, 0x696120, + 0x697474, 0x69f920, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6f20, + 0x6f7269, 0x7069f9, 0x726920, 0x746f72, 0x7474e0, 0x74e020, 0xe02020, + 0xe0206c, 0xe02073, 0xe82061, 0xe8206d, 0xe92073, 0xf92067, 0xf9206c, + 0xf92070, + ], + highBytes: { + total: 9, + counts: [ + [0xe0, 3], + [0xe8, 2], + [0xe9, 1], + [0xf9, 3], + ], + }, + }, + { + language: 'nl', + ngrams: [ + 0x000064, 0x206269, 0x20626f, 0x206361, 0x20636f, 0x206465, 0x206565, + 0x20656e, 0x206765, 0x206865, 0x206c65, 0x206f70, 0x206f76, 0x207374, + 0x207465, 0x207665, 0x207765, 0x20e9e9, 0x61616b, 0x616173, 0x616420, + 0x6166e9, 0x616b74, 0x616d20, 0x626962, 0x626c69, 0x626f65, 0x636166, + 0x636874, 0x646520, 0x652062, 0x652063, 0x652065, 0x652073, 0x65656e, + 0x656572, 0x6565eb, 0x65696e, 0x656b65, 0x656e20, 0x657273, 0x65eb6e, + 0x65ef6e, 0x66e973, 0x6765ef, 0x696e20, 0x6b7420, 0x6e2020, 0x6e206f, + 0x6e20e9, 0x6feb7a, 0x706feb, 0x732065, 0x736368, 0x742064, 0x74656e, + 0x746572, 0x766572, 0xe96e20, 0xe97320, 0xe9e96e, 0xeb6e20, 0xeb7a69, + 0xef6e74, + ], + highBytes: { + total: 7, + counts: [ + [0xe9, 4], + [0xeb, 2], + [0xef, 1], + ], + }, + }, + { + language: 'no', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206272, 0x206279, 0x2062f8, 0x206465, + 0x2064f8, 0x20666f, 0x2066f8, 0x206c61, 0x206ee6, 0x206f67, 0x2070e5, + 0x207665, 0x2076e5, 0x20e570, 0x20e672, 0x626962, 0x626c69, 0x6272f8, + 0x627965, 0x62f86b, 0x62f86e, 0x642065, 0x646520, 0x646572, 0x646574, + 0x64f872, 0x652062, 0x652066, 0x652074, 0x6520e5, 0x6520e6, 0x656420, + 0x656e20, 0x656e65, 0x656e73, 0x657220, 0x657273, 0x657420, 0x66f872, + 0x672073, 0x676520, 0x67656e, 0x676572, 0x69626c, 0x696765, 0x696e67, + 0x696f74, 0x6af865, 0x6cf872, 0x6e6520, 0x6e6572, 0x6ee672, 0x6f6720, + 0x70e520, 0x72206f, 0x7273f8, 0x72f864, 0x737465, 0x74656b, 0xe52064, + 0xf86b65, + ], + highBytes: { + total: 14, + counts: [ + [0xe5, 4], + [0xe6, 2], + [0xf8, 8], + ], + }, + }, + { + language: 'pt', + ngrams: [ + 0x00006d, 0x206120, 0x206269, 0x206369, 0x2063f3, 0x206465, 0x206520, + 0x206573, 0x206e61, 0x206f20, 0x206f73, 0x207065, 0x207072, 0x2070e3, + 0x2073e1, 0x207665, 0x20f46e, 0x612062, 0x612063, 0x612064, 0x612065, + 0x61206f, 0x612070, 0x616465, 0x61646f, 0x6166e9, 0x616d20, 0x617261, + 0x617320, 0x61e761, 0x61e7e3, 0x626962, 0x626c69, 0x636120, 0x6369ea, + 0x63e172, 0x63f364, 0x646120, 0x646520, 0x646f20, 0x652061, 0x6563e1, + 0x657320, 0x657363, 0x66e973, 0x68e320, 0x696120, 0x69e7f5, 0x69ea6e, + 0x6d61e7, 0x6e6120, 0x6e68e3, 0x6f7320, 0x70e36f, 0x7261e7, 0x726573, + 0x7269e7, 0x732020, 0x732061, 0x732070, 0x7320f4, 0x7374f3, 0x73e162, + 0x74f372, + ], + highBytes: { + total: 14, + counts: [ + [0xe1, 2], + [0xe3, 3], + [0xe7, 3], + [0xe9, 1], + [0xea, 1], + [0xf3, 2], + [0xf4, 1], + [0xf5, 1], + ], + }, + }, + { + language: 'sv', + ngrams: [ + 0x000064, 0x006cf6, 0x206269, 0x2062f6, 0x206465, 0x2066e4, 0x2066f6, + 0x206c69, 0x206ce4, 0x206ce5, 0x206de4, 0x206f63, 0x2070e5, 0x20736b, + 0x207374, 0x2073e4, 0x2073f6, 0x207479, 0x207665, 0x2076e4, 0x20e470, + 0x20f670, 0x612020, 0x612062, 0x612074, 0x6120f6, 0x616420, 0x616465, + 0x6166e9, 0x617220, 0x617265, 0x626962, 0x626c69, 0x6272f6, 0x62f663, + 0x62f66e, 0x636820, 0x6420e4, 0x646520, 0x64656e, 0x646574, 0x656e20, + 0x657220, 0x65726e, 0x6573f6, 0x657420, 0x66f672, 0x676120, 0x69626c, + 0x696761, 0x696f74, 0x6b6170, 0x6b6172, 0x6e6120, 0x6f6368, 0x70e520, + 0x726e61, 0x736b61, 0x737461, 0x73f66b, 0x74206c, 0x746164, 0x766572, + 0xe52064, + ], + highBytes: { + total: 20, + counts: [ + [0xe4, 7], + [0xe5, 3], + [0xe9, 1], + [0xf6, 9], + ], + }, + }, + ], + }, + { + encoding: 'windows-1253', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xdc, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'el', + ngrams: [ + 0x0000de, 0x0000f0, 0x0000f3, 0x0000f8, 0x00def3, 0x00f0f1, 0x00f3e1, + 0x00f8e7, 0x20dced, 0x20e1e3, 0x20e1ed, 0x20e1f1, 0x20e2e9, 0x20e2ef, + 0x20e3f1, 0x20e5eb, 0x20e5f0, 0x20e5f1, 0x20e720, 0x20eae1, 0x20ece9, + 0x20efe9, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xdc20f4, 0xdfe120, + 0xe12020, 0xe120ea, 0xe1e920, 0xe2e9e2, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, + 0xe5e920, 0xe5f220, 0xe720e2, 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9e2eb, + 0xe9efe8, 0xe9f3f4, 0xeae1e9, 0xeae720, 0xebe720, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xeff5ed, 0xf0fceb, 0xf1dc20, 0xf1e9ef, + 0xf220ea, 0xf3f4de, 0xf3f4e7, 0xf4e120, 0xf4e5f2, 0xf4e7ed, 0xf5ed20, + 0xfcebe7, + ], + highBytes: { + total: 327, + counts: [ + [0xb9, 1], + [0xc7, 1], + [0xcc, 1], + [0xcf, 2], + [0xd0, 1], + [0xd3, 1], + [0xd8, 1], + [0xdc, 13], + [0xdd, 5], + [0xde, 4], + [0xdf, 9], + [0xe1, 31], + [0xe2, 9], + [0xe3, 9], + [0xe4, 4], + [0xe5, 16], + [0xe6, 1], + [0xe7, 14], + [0xe8, 6], + [0xe9, 31], + [0xea, 16], + [0xeb, 10], + [0xec, 11], + [0xed, 15], + [0xee, 1], + [0xef, 24], + [0xf0, 8], + [0xf1, 16], + [0xf2, 7], + [0xf3, 8], + [0xf4, 17], + [0xf5, 10], + [0xf6, 5], + [0xf7, 3], + [0xf8, 2], + [0xf9, 5], + [0xfc, 5], + [0xfd, 2], + [0xfe, 2], + ], + }, + }, + ], + }, + { + encoding: 'windows-1254', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x9a, 0x20, 0x9c, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'tr', + ngrams: [ + 0x000063, 0x000064, 0x000073, 0x0000fe, 0x006375, 0x00fe65, 0x2061e7, + 0x206269, 0x20656b, 0x206b61, 0x206bfc, 0x206dfc, 0x207361, 0x207461, + 0x207665, 0x207961, 0x20e769, 0x20f6f0, 0x20fe65, 0x616261, 0x616c20, + 0x616c61, 0x616cfd, 0x616e65, 0x616efd, 0x617220, 0x6172fd, 0x6174f6, + 0x617a61, 0x617afd, 0x61e7fd, 0x61fee7, 0x62616c, 0x652062, 0x656869, + 0x657220, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697220, 0x697461, + 0x6bfc74, 0x6c6172, 0x6c6572, 0x706861, 0x722020, 0x72206b, 0x722076, + 0x722079, 0x726920, 0x726c61, 0x72fd20, 0x746172, 0x74fc70, 0x766520, + 0x7a6172, 0x7a6520, 0xe7fd6b, 0xfc7068, 0xfc74fc, 0xfd6b20, 0xfd7220, + 0xfe6568, + ], + highBytes: { + total: 32, + counts: [ + [0xc7, 1], + [0xde, 2], + [0xe7, 6], + [0xf0, 1], + [0xf6, 2], + [0xfc, 8], + [0xfd, 11], + [0xfe, 1], + ], + }, + }, + ], + }, + { + encoding: 'windows-1255', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xd4, 0xd5, 0xd6, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'he', + ngrams: [ + 0x0000e1, 0x0000e4, 0x0000f1, 0x0000f9, 0x00e1e5, 0x00e4f1, 0x00f1e3, + 0x00f9e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, 0x20e1fa, + 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f7, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e7f7, 0x20e8f8, + 0x20ece0, 0x20ece1, 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, + 0x20eee7, 0x20eeeb, 0xe0e9ed, 0xe4f1f4, 0xe5f8e9, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed20, 0xe9f820, 0xe9fa20, 0xed2020, 0xed20ec, + 0xed20ee, 0xef20ee, 0xf0e420, 0xf1f4f8, 0xf2e9f8, 0xf4f8e9, 0xf7e5f8, + 0xf820e4, 0xf820ee, 0xf8e5fa, 0xf8e920, 0xf8e9ed, 0xf8fa20, 0xf9e9ed, + 0xfa20ec, + ], + highBytes: { + total: 249, + counts: [ + [0xe0, 9], + [0xe1, 12], + [0xe2, 2], + [0xe3, 8], + [0xe4, 16], + [0xe5, 30], + [0xe6, 2], + [0xe7, 6], + [0xe8, 7], + [0xe9, 33], + [0xea, 1], + [0xeb, 6], + [0xec, 9], + [0xed, 13], + [0xee, 11], + [0xef, 3], + [0xf0, 6], + [0xf1, 7], + [0xf2, 6], + [0xf4, 7], + [0xf6, 1], + [0xf7, 10], + [0xf8, 22], + [0xf9, 7], + [0xfa, 15], + ], + }, + }, + ], + }, + { + encoding: 'windows-1256', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x81, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x8a, 0x20, 0x9c, 0x8d, 0x8e, 0x8f, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x98, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x20, 0x9f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0x20, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0x20, 0x20, 0x20, 0x20, 0xf4, 0x20, 0x20, 0x20, 0x20, 0xf9, 0x20, 0xfb, + 0xfc, 0x20, 0x20, 0xff, + ], + languages: [ + { + language: 'ar', + ngrams: [ + 0x0000c7, 0x0000d3, 0x0000d5, 0x0000e6, 0x00c7e1, 0x00d3e6, 0x00d5c8, + 0x00e6d1, 0x20c3c8, 0x20c3e3, 0x20c7ce, 0x20c7e1, 0x20c8e5, 0x20c8ed, + 0x20cad3, 0x20cfe1, 0x20d1de, 0x20d5db, 0x20dae4, 0x20ddd1, 0x20dded, + 0x20dfca, 0x20e1e1, 0x20e6c7, 0x20e6ca, 0x20e6d5, 0x20e6ed, 0xc7ca20, + 0xc7ccda, 0xc7d120, 0xc7e1c3, 0xc7e1da, 0xc7e1e3, 0xc7e1e6, 0xc820c7, + 0xc8c920, 0xc92020, 0xc920c7, 0xc920e6, 0xc920ed, 0xca20e6, 0xcac820, + 0xcac8c9, 0xcb20c7, 0xccda20, 0xcd20c7, 0xcfede4, 0xd120c7, 0xd1c7cc, + 0xd5ddc7, 0xda20c7, 0xdfcac8, 0xe1e3cf, 0xe1e3df, 0xe3cfed, 0xe3dfca, + 0xe420c7, 0xe4c7d3, 0xe4c920, 0xe6c7e1, 0xe6d5dd, 0xe6e420, 0xedc920, + 0xede4c9, + ], + highBytes: { + total: 324, + counts: [ + [0xc1, 2], + [0xc3, 4], + [0xc4, 1], + [0xc6, 1], + [0xc7, 56], + [0xc8, 17], + [0xc9, 11], + [0xca, 16], + [0xcb, 2], + [0xcc, 3], + [0xcd, 8], + [0xce, 4], + [0xcf, 9], + [0xd1, 19], + [0xd2, 3], + [0xd3, 7], + [0xd4, 2], + [0xd5, 5], + [0xd6, 2], + [0xd8, 1], + [0xd9, 2], + [0xda, 8], + [0xdb, 1], + [0xdd, 9], + [0xde, 7], + [0xdf, 5], + [0xe1, 36], + [0xe3, 18], + [0xe4, 12], + [0xe5, 5], + [0xe6, 24], + [0xed, 24], + ], + }, + }, + ], + }, + { + encoding: 'windows-1257', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x8e, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'et', + ngrams: [ + 0x206a61, 0x206b69, 0x206b6f, 0x206c69, 0x206d65, 0x206dfc, 0x207261, + 0x2074f6, 0x2076e4, 0x20e472, 0x20f570, 0x20f575, 0x6120f5, 0x61616d, + 0x616220, 0x616420, 0x616d61, 0x617475, 0x617661, 0x6220f5, 0x626120, + 0x642061, 0x64206b, 0x64206f, 0x647573, 0x652074, 0x656420, 0x657369, + 0x692020, 0x696420, 0x696b75, 0x696d65, 0x696e6e, 0x69726a, 0x697461, + 0x6a6120, 0x6b6972, 0x6b6f67, 0x6b6f6e, 0x6b6f6f, 0x6b7564, 0x6c6173, + 0x6d6174, 0x6dfcfc, 0x6e20e4, 0x6e6520, 0x70e465, 0x726161, 0x736920, + 0x746162, 0x74f6f6, 0x7570e4, 0x766164, 0x76e469, 0x76e46c, 0x76e472, + 0xe46576, 0xe4696b, 0xe46c6a, 0xe4726b, 0xe47273, 0xf57069, 0xf5756e, + 0xf67475, + ], + highBytes: { + total: 11, + counts: [ + [0xe4, 5], + [0xf5, 2], + [0xf6, 2], + [0xfc, 2], + ], + }, + }, + { + language: 'lv', + ngrams: [ + 0x0072ee, 0x206269, 0x206365, 0x206461, 0x206469, 0x2064e2, 0x206761, + 0x20697a, 0x206b6f, 0x206bef, 0x206c61, 0x206ce7, 0x206d61, 0x206d65, + 0x206de2, 0x207069, 0x2070e2, 0x207261, 0x207265, 0x20756e, 0x61206b, + 0x617320, 0x626962, 0x626c69, 0x6365ef, 0x646120, 0x656d20, 0x692070, + 0x692073, 0x69626c, 0x69656d, 0x696f74, 0x6974e2, 0x6a6920, 0x6b7374, + 0x6be272, 0x6c6173, 0x6c696f, 0x6c73e7, 0x6c7573, 0x6ce76e, 0x6e6920, + 0x6eee63, 0x6f74e7, 0x70696c, 0x70e272, 0x726920, 0x732020, 0x732061, + 0x73206c, 0x73206d, 0x732075, 0x73e774, 0x74e26a, 0x74e26c, 0x74e76b, + 0x756e20, 0x757320, 0x76e76c, 0x7a76e7, 0xe26a69, 0xe76e69, 0xe774e2, + 0xee6361, + ], + highBytes: { + total: 52, + counts: [ + [0xe2, 24], + [0xe7, 13], + [0xe8, 1], + [0xec, 1], + [0xee, 4], + [0xef, 3], + [0xf0, 2], + [0xf2, 2], + [0xfb, 1], + [0xfe, 1], + ], + }, + }, + { + language: 'lt', + ngrams: [ + 0x0000f0, 0x00f065, 0x206169, 0x206170, 0x206269, 0x206461, 0x206972, + 0x207072, 0x207265, 0x20736b, 0x2073fb, 0x20f069, 0x20f076, 0x20fb6b, + 0x20fe69, 0x20fe6d, 0x612070, 0x616920, 0x616974, 0x6169f0, 0x6172f0, + 0x6172fe, 0x617320, 0x617573, 0x61f06f, 0x61f079, 0x626962, 0x626c69, + 0x646120, 0x64e020, 0x656461, 0x656e69, 0x67f820, 0x692072, 0x696120, + 0x696169, 0x69626c, 0x696573, 0x696e69, 0x696eeb, 0x696f74, 0x697220, + 0x6a6169, 0x6b6169, 0x6b696e, 0x6c696f, 0x6d656e, 0x6e696e, 0x6eeb73, + 0x6f20fe, 0x6f64e0, 0x6f7320, 0x6f7465, 0x7261f0, 0x726961, 0x732020, + 0x732061, 0x736b61, 0x74656b, 0x757320, 0xe0206d, 0xeb6a61, 0xeb7320, + 0xf07669, + ], + highBytes: { + total: 40, + counts: [ + [0xd0, 1], + [0xdb, 1], + [0xe0, 5], + [0xe1, 1], + [0xe6, 1], + [0xeb, 8], + [0xf0, 11], + [0xf8, 4], + [0xfb, 2], + [0xfe, 6], + ], + }, + }, + ], + }, + { + encoding: 'windows-1258', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x20, 0x20, 0x9c, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x9c, 0x20, 0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0x20, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0x20, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0x20, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0x20, 0xff, + ], + languages: [ + { + language: 'vi', + ngrams: [ + 0x206261, 0x2062e1, 0x206320, 0x206368, 0x2064e2, 0x206769, 0x20686f, + 0x206920, 0x206c69, 0x206d20, 0x206d61, 0x206df4, 0x206e20, 0x206e67, + 0x206e68, 0x207068, 0x207175, 0x2073e1, 0x207420, 0x207461, 0x207468, + 0x207469, 0x207472, 0x2074e0, 0x207520, 0x207669, 0x2076e0, 0x207920, + 0x20f0f4, 0x612069, 0x612076, 0x626120, 0x62e16e, 0x636820, 0x6368f5, + 0x64e220, 0x672064, 0x67206b, 0x672074, 0x6720f0, 0x682070, 0x686f20, + 0x68e06e, 0x68f420, 0x68f520, 0x68fd20, 0x692063, 0x69206e, 0x692071, + 0x69ea20, 0x6df420, 0x6e2062, 0x6e2074, 0x6e6720, 0x6e6820, 0x6f2063, + 0x6f206e, 0x7068f4, 0x7468e0, 0x7468fd, 0x76e020, 0xf42074, 0xf5206e, + 0xfdf520, + ], + highBytes: { + total: 116, + counts: [ + [0xcc, 7], + [0xd0, 1], + [0xd2, 13], + [0xde, 6], + [0xe0, 11], + [0xe1, 6], + [0xe2, 7], + [0xea, 8], + [0xec, 9], + [0xf0, 3], + [0xf2, 15], + [0xf3, 1], + [0xf4, 10], + [0xf5, 7], + [0xfa, 2], + [0xfd, 10], + ], + }, + }, + ], + }, + { + encoding: 'windows-874', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0x20, 0xd2, 0xd3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'th', + ngrams: [ + 0x0000b5, 0x0000cb, 0x0000e0, 0x00b5c5, 0x00cb20, 0x00e0aa, 0x20a120, + 0x20a1a4, 0x20a1a8, 0x20a1c9, 0x20a1e0, 0x20a420, 0x20a4b9, 0x20a7ca, + 0x20a7e3, 0x20a820, 0x20aa20, 0x20aad2, 0x20b3b1, 0x20b4b7, 0x20b4bb, + 0x20b4e0, 0x20b520, 0x20b5c3, 0x20b720, 0x20b7c2, 0x20b8c0, 0x20b920, + 0x20b9cb, 0x20b9e0, 0x20bac3, 0x20bad2, 0x20bc20, 0x20bcc5, 0x20be20, + 0x20c120, 0x20c1e0, 0x20c2b9, 0x20c2ba, 0x20c2c7, 0x20c320, 0x20c520, + 0x20c7c2, 0x20c8d2, 0x20ca20, 0x20cda7, 0x20d2b9, 0x20e1c5, 0xa12020, + 0xa420a1, 0xb5c320, 0xbbc3d0, 0xc120cd, 0xc8d2ca, 0xca20e1, 0xcab5c3, + 0xcad2c3, 0xcb20cd, 0xd0c720, 0xd2c320, 0xd2cab5, 0xe0c120, 0xe0c520, + 0xe1c5d0, + ], + highBytes: { + total: 301, + counts: [ + [0xa1, 8], + [0xa2, 4], + [0xa4, 7], + [0xa7, 10], + [0xa8, 4], + [0xaa, 5], + [0xb1, 1], + [0xb3, 2], + [0xb4, 8], + [0xb5, 7], + [0xb6, 1], + [0xb7, 8], + [0xb8, 2], + [0xb9, 18], + [0xba, 4], + [0xbb, 4], + [0xbc, 4], + [0xbe, 2], + [0xbf, 1], + [0xc0, 1], + [0xc1, 8], + [0xc2, 10], + [0xc3, 16], + [0xc5, 11], + [0xc7, 7], + [0xc8, 2], + [0xc9, 1], + [0xca, 10], + [0xcb, 5], + [0xcd, 12], + [0xd0, 6], + [0xd1, 12], + [0xd2, 18], + [0xd3, 4], + [0xd4, 8], + [0xd5, 8], + [0xd6, 2], + [0xd7, 6], + [0xd8, 1], + [0xd9, 5], + [0xe0, 12], + [0xe1, 6], + [0xe2, 1], + [0xe3, 2], + [0xe4, 1], + [0xe6, 1], + [0xe7, 1], + [0xe8, 8], + [0xe9, 11], + [0xec, 5], + ], + }, + }, + ], + }, + { + encoding: 'KOI8-R', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, + 0xdc, 0xdd, 0xde, 0xdf, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000d2, 0x00d2c1, 0x20c120, 0x20c2c9, 0x20c7cf, 0x20c920, 0x20cbcf, + 0x20cdc1, 0x20cec1, 0x20cfc2, 0x20cfd3, 0x20cfd4, 0x20d0cc, 0x20d0cf, + 0x20d0d2, 0x20d1d2, 0x20d2c5, 0x20d3cf, 0x20d720, 0x20d7cf, 0x20d7d9, + 0x20dec9, 0xc0d420, 0xc120ce, 0xc120d0, 0xc120d7, 0xc1c0d4, 0xc1c5d4, + 0xc1cec9, 0xc1d120, 0xc1d2cb, 0xc1d3d4, 0xc1d4c5, 0xc2c9c2, 0xc2ccc9, + 0xc2d3d5, 0xc4c1c0, 0xc4c920, 0xc5ccc5, 0xc5ccc9, 0xc5cece, 0xc5d2c9, + 0xc5d2d9, 0xc5d3d4, 0xc7cfd2, 0xc7cfd4, 0xc920d0, 0xcbc120, 0xccc920, + 0xcdc1d2, 0xcec120, 0xced9c5, 0xcfd3c5, 0xd0d2cf, 0xd120d0, 0xd1d420, + 0xd2cfc4, 0xd2d920, 0xd420d3, 0xd4c5cc, 0xd4c5d2, 0xd4cfd7, 0xd4d920, + 0xd9c520, + ], + highBytes: { + total: 678, + counts: [ + [0xa3, 2], + [0xc0, 13], + [0xc1, 63], + [0xc2, 16], + [0xc3, 3], + [0xc4, 21], + [0xc5, 50], + [0xc7, 7], + [0xc8, 7], + [0xc9, 47], + [0xca, 7], + [0xcb, 21], + [0xcc, 19], + [0xcd, 16], + [0xce, 37], + [0xcf, 60], + [0xd0, 23], + [0xd1, 27], + [0xd2, 41], + [0xd3, 34], + [0xd4, 55], + [0xd5, 14], + [0xd6, 5], + [0xd7, 26], + [0xd8, 3], + [0xd9, 26], + [0xda, 7], + [0xdb, 6], + [0xdc, 1], + [0xdd, 2], + [0xde, 4], + [0xe4, 1], + [0xe6, 1], + [0xe7, 1], + [0xec, 1], + [0xee, 3], + [0xef, 1], + [0xf0, 1], + [0xf2, 2], + [0xf3, 1], + [0xf5, 1], + [0xf7, 1], + [0xfe, 1], + ], + }, + }, + ], + }, { encoding: 'KOI8-U', byteMap: [ @@ -1014,26 +3003,27 @@ export const generatedSBCSModels = [ { language: 'ro', ngrams: [ - 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, - 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, - 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, - 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, - 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, - 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, - 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, - 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, - 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, - 0xfe6120, + 0x206361, 0x206375, 0x2063e3, 0x206465, 0x206573, 0x206d69, 0x206f72, + 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, 0x616c20, 0x617ae3, + 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, 0x646520, 0x646573, + 0x652020, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, 0x656c65, + 0x657269, 0x657363, 0x657374, 0x65ba74, 0x65fe75, 0x692063, 0x6961fe, + 0x69626c, 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, + 0x6c2065, 0x6c6520, 0x6c696f, 0x6d62e3, 0x6f7269, 0x707265, 0x7261ba, + 0x726520, 0x7265fe, 0x726969, 0x737465, 0x746520, 0x746f72, 0x74e320, + 0x756c20, 0x7ae320, 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xfe6120, + 0xfe756c, ], highBytes: { - total: 49, + total: 57, counts: [ + [0xa4, 4], [0xba, 12], [0xce, 1], [0xe2, 3], [0xe3, 21], [0xee, 4], - [0xfe, 8], + [0xfe, 12], ], }, }, From fc5240334a5ee3f00686b46dce9e1ee7bf1525df Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Mon, 20 Jul 2026 13:17:22 +1000 Subject: [PATCH 3/6] switch single byte matcher to new models from custom courpus (#133) * switch single byte matcher to new models from custom courpus * Fix test-build scripts * Improve sbcs performance --- .github/workflows/test-build.js | 17 +- .github/workflows/test-build.ts | 17 +- src/encoding/models/types.ts | 18 + src/encoding/sbcs.test.ts | 8 + src/encoding/sbcs.ts | 1252 ++++++------------------------- src/index.test.ts | 13 +- src/index.ts | 46 +- src/match.ts | 14 + 8 files changed, 333 insertions(+), 1052 deletions(-) create mode 100644 src/encoding/models/types.ts diff --git a/.github/workflows/test-build.js b/.github/workflows/test-build.js index 85dde71..90d89d0 100644 --- a/.github/workflows/test-build.js +++ b/.github/workflows/test-build.js @@ -9,8 +9,21 @@ assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ { confidence: 1, name: 'ASCII', lang: undefined }, - { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' }, + { name: 'ISO-8859-2', lang: 'hu', confidence: 0.6 }, + { name: 'windows-1250', lang: 'hu', confidence: 0.6 }, + { name: 'ISO-8859-1', lang: 'en', confidence: 0.4 }, + { name: 'windows-1252', lang: 'en', confidence: 0.4 }, + { name: 'ISO-8859-10', lang: 'is', confidence: 0.4 }, + { name: 'windows-1257', lang: 'lv', confidence: 0.2 }, + { name: 'macintosh', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-3', lang: 'mt', confidence: 0.2 }, + { name: 'ISO-8859-4', lang: 'lv', confidence: 0.2 }, + { name: 'ISO-8859-13', lang: 'lt', confidence: 0.2 }, + { name: 'ISO-8859-14', lang: 'cy', confidence: 0.2 }, + { name: 'ISO-8859-15', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-16', lang: 'ro', confidence: 0.2 }, + { name: 'CP850', lang: 'de', confidence: 0.2 }, + { name: 'CP852', lang: 'pl', confidence: 0.2 }, { confidence: 0.1, name: 'UTF-8', lang: undefined }, { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, { confidence: 0.1, name: 'Big5', lang: 'zh' }, diff --git a/.github/workflows/test-build.ts b/.github/workflows/test-build.ts index cdc31d8..d074a2a 100644 --- a/.github/workflows/test-build.ts +++ b/.github/workflows/test-build.ts @@ -12,8 +12,21 @@ const main = async () => { assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ { confidence: 1, name: 'ASCII', lang: undefined }, - { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' }, + { name: 'ISO-8859-2', lang: 'hu', confidence: 0.6 }, + { name: 'windows-1250', lang: 'hu', confidence: 0.6 }, + { name: 'ISO-8859-1', lang: 'en', confidence: 0.4 }, + { name: 'windows-1252', lang: 'en', confidence: 0.4 }, + { name: 'ISO-8859-10', lang: 'is', confidence: 0.4 }, + { name: 'windows-1257', lang: 'lv', confidence: 0.2 }, + { name: 'macintosh', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-3', lang: 'mt', confidence: 0.2 }, + { name: 'ISO-8859-4', lang: 'lv', confidence: 0.2 }, + { name: 'ISO-8859-13', lang: 'lt', confidence: 0.2 }, + { name: 'ISO-8859-14', lang: 'cy', confidence: 0.2 }, + { name: 'ISO-8859-15', lang: 'de', confidence: 0.2 }, + { name: 'ISO-8859-16', lang: 'ro', confidence: 0.2 }, + { name: 'CP850', lang: 'de', confidence: 0.2 }, + { name: 'CP852', lang: 'pl', confidence: 0.2 }, { confidence: 0.1, name: 'UTF-8', lang: undefined }, { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, { confidence: 0.1, name: 'Big5', lang: 'zh' }, diff --git a/src/encoding/models/types.ts b/src/encoding/models/types.ts new file mode 100644 index 0000000..a592c76 --- /dev/null +++ b/src/encoding/models/types.ts @@ -0,0 +1,18 @@ +import type { EncodingName } from '../../match'; + +export interface HighByteDistribution { + readonly total: number; + readonly counts: readonly (readonly [number, number])[]; +} + +export interface GeneratedSBCSLanguage { + readonly language: string; + readonly ngrams: readonly number[]; + readonly highBytes: HighByteDistribution; +} + +export interface GeneratedSBCSModel { + readonly encoding: EncodingName; + readonly byteMap: readonly number[]; + readonly languages: readonly GeneratedSBCSLanguage[]; +} diff --git a/src/encoding/sbcs.test.ts b/src/encoding/sbcs.test.ts index bcae0fd..6ce296a 100644 --- a/src/encoding/sbcs.test.ts +++ b/src/encoding/sbcs.test.ts @@ -18,6 +18,14 @@ describe('Singlebyte Character Sets', () => { expect(detect('iso88591_en')).toBe('ISO-8859-1'); }); + it('should retain a weak ISO-8859-1 candidate after byte prefiltering', () => { + const fixture = path.join( + __dirname, + '../../corpus/generated/ISO-8859-1/es/validation/river-trip.bin', + ); + expect(chardet.detectFileSync(fixture)).toBe('ISO-8859-1'); + }); + it('should return ISO-8859-2 (Czech)', () => { expect(detect('iso88592_cs')).toBe('ISO-8859-2'); }); diff --git a/src/encoding/sbcs.ts b/src/encoding/sbcs.ts index 4dbabb1..3e79e99 100644 --- a/src/encoding/sbcs.ts +++ b/src/encoding/sbcs.ts @@ -1,1059 +1,267 @@ -import type { Context, Recogniser } from '.'; -import match, { type EncodingName, type Match } from '../match'; - -/** - * This class recognizes single-byte encodings. Because the encoding scheme is so - * simple, language statistics are used to do the matching. - */ +import type { Match } from '../match'; +import type { Context } from '.'; +import { generatedSBCSModels } from './models/generated'; +import type { + GeneratedSBCSLanguage, + GeneratedSBCSModel, + HighByteDistribution, +} from './models/types'; const N_GRAM_MASK = 0xffffff; - -class NGramParser { - byteIndex: number = 0; - ngram: number = 0; - - ngramCount: number = 0; - hitCount: number = 0; - - ngramList: number[]; - byteMap: number[]; - - spaceChar: number = 0x20; - - constructor(theNgramList: number[], theByteMap: number[]) { - this.ngramList = theNgramList; - this.byteMap = theByteMap; - } - - /* - * Binary search for value in table, which must have exactly 64 entries. - */ - search(table: number[], value: number) { - let index = 0; - - if (table[index + 32] <= value) index += 32; - if (table[index + 16] <= value) index += 16; - if (table[index + 8] <= value) index += 8; - if (table[index + 4] <= value) index += 4; - if (table[index + 2] <= value) index += 2; - if (table[index + 1] <= value) index += 1; - if (table[index] > value) index -= 1; - - if (index < 0 || table[index] != value) return -1; - - return index; - } - - lookup(thisNgram: number) { - this.ngramCount += 1; - if (this.search(this.ngramList, thisNgram) >= 0) { - this.hitCount += 1; - } - } - - addByte(b: number) { - this.ngram = ((this.ngram << 8) + (b & 0xff)) & N_GRAM_MASK; - this.lookup(this.ngram); - } - - nextByte(det: Context) { - if (this.byteIndex >= det.inputLen) return -1; - - return det.inputBytes[this.byteIndex++] & 0xff; - } - - parse(det: Context, spaceCh: number) { - let b, - ignoreSpace = false; - this.spaceChar = spaceCh; - - while ((b = this.nextByte(det)) >= 0) { - const mb = this.byteMap[b]; - - if (mb != 0) { - if (!(mb == this.spaceChar && ignoreSpace)) { - this.addByte(mb); - } - - ignoreSpace = mb == this.spaceChar; - } - } - - this.addByte(this.spaceChar); - - const rawPercent = this.hitCount / this.ngramCount; - - if (rawPercent > 0.33) return 0.98; - - return Math.floor(rawPercent * 300.0) / 100; - } +// Byte distributions are cheap and sufficiently discriminative to exclude +// most encodings before the more expensive n-gram pass. The corpus needs 11 +// candidates to preserve full-result accuracy; keep one additional candidate +// as headroom for unseen inputs. +const MAX_SBCS_CANDIDATES = 12; + +interface Score { + confidence: number; + hitRate: number; + hits: number; + total: number; } -class NGramsPlusLang { - fLang: string; - fNGrams: number[]; - - constructor(la: string, ng: number[]) { - this.fLang = la; - this.fNGrams = ng; - } +interface Candidate extends Match, Score { + byteLogLikelihood: number; } -const isFlatNgrams = (val: NGramsPlusLang[] | number[]): val is number[] => - Array.isArray(val) && isFinite(val[0] as number); - -abstract class sbcs implements Recogniser { - spaceChar = 0x20; - - private nGramLang?: string = undefined; - - ngrams(): NGramsPlusLang[] | number[] { - return []; - } - - byteMap(): number[] { - return []; - } - - abstract name(input: Context): EncodingName; - - language(): string | undefined { - return this.nGramLang; - } - - match(det: Context): Match | null { - // This feels a bit dirty. Simpler alternative would be - // splitting classes ISO_8859_1 etc into language-specific ones - // with hardcoded languages like ISO_8859_9. - this.nGramLang = undefined; - - const ngrams = this.ngrams(); - - if (isFlatNgrams(ngrams)) { - const parser = new NGramParser(ngrams, this.byteMap()); - const confidence = parser.parse(det, this.spaceChar); - return confidence <= 0 ? null : match(det, this, confidence); - } - - let bestConfidence = -1; - - for (let i = ngrams.length - 1; i >= 0; i--) { - const ngl = ngrams[i]; - - const parser = new NGramParser(ngl.fNGrams, this.byteMap()); - const confidence = parser.parse(det, this.spaceChar); - if (confidence > bestConfidence) { - bestConfidence = confidence; - this.nGramLang = ngl.fLang; - } - } - - return bestConfidence <= 0 ? null : match(det, this, bestConfidence); - } -} - -export class ISO_8859_1 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - new NGramsPlusLang( - 'da', - [ - 0x206166, 0x206174, 0x206465, 0x20656e, 0x206572, 0x20666f, 0x206861, - 0x206920, 0x206d65, 0x206f67, 0x2070e5, 0x207369, 0x207374, 0x207469, - 0x207669, 0x616620, 0x616e20, 0x616e64, 0x617220, 0x617420, 0x646520, - 0x64656e, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656e20, - 0x656e64, 0x657220, 0x657265, 0x657320, 0x657420, 0x666f72, 0x676520, - 0x67656e, 0x676572, 0x696765, 0x696c20, 0x696e67, 0x6b6520, 0x6b6b65, - 0x6c6572, 0x6c6967, 0x6c6c65, 0x6d6564, 0x6e6465, 0x6e6520, 0x6e6720, - 0x6e6765, 0x6f6720, 0x6f6d20, 0x6f7220, 0x70e520, 0x722064, 0x722065, - 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696c, - 0x766572, - ], - ), - new NGramsPlusLang( - 'de', - [ - 0x20616e, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, - 0x206765, 0x206861, 0x20696e, 0x206d69, 0x207363, 0x207365, 0x20756e, - 0x207665, 0x20766f, 0x207765, 0x207a75, 0x626572, 0x636820, 0x636865, - 0x636874, 0x646173, 0x64656e, 0x646572, 0x646965, 0x652064, 0x652073, - 0x65696e, 0x656974, 0x656e20, 0x657220, 0x657320, 0x67656e, 0x68656e, - 0x687420, 0x696368, 0x696520, 0x696e20, 0x696e65, 0x697420, 0x6c6963, - 0x6c6c65, 0x6e2061, 0x6e2064, 0x6e2073, 0x6e6420, 0x6e6465, 0x6e6520, - 0x6e6720, 0x6e6765, 0x6e7465, 0x722064, 0x726465, 0x726569, 0x736368, - 0x737465, 0x742064, 0x746520, 0x74656e, 0x746572, 0x756e64, 0x756e67, - 0x766572, - ], - ), - new NGramsPlusLang( - 'en', - [ - 0x206120, 0x20616e, 0x206265, 0x20636f, 0x20666f, 0x206861, 0x206865, - 0x20696e, 0x206d61, 0x206f66, 0x207072, 0x207265, 0x207361, 0x207374, - 0x207468, 0x20746f, 0x207768, 0x616964, 0x616c20, 0x616e20, 0x616e64, - 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, - 0x652073, 0x652074, 0x656420, 0x656e74, 0x657220, 0x657320, 0x666f72, - 0x686174, 0x686520, 0x686572, 0x696420, 0x696e20, 0x696e67, 0x696f6e, - 0x697320, 0x6e2061, 0x6e2074, 0x6e6420, 0x6e6720, 0x6e7420, 0x6f6620, - 0x6f6e20, 0x6f7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, - 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696f, 0x746f20, - 0x747320, - ], - ), - new NGramsPlusLang( - 'es', - [ - 0x206120, 0x206361, 0x20636f, 0x206465, 0x20656c, 0x20656e, 0x206573, - 0x20696e, 0x206c61, 0x206c6f, 0x207061, 0x20706f, 0x207072, 0x207175, - 0x207265, 0x207365, 0x20756e, 0x207920, 0x612063, 0x612064, 0x612065, - 0x61206c, 0x612070, 0x616369, 0x61646f, 0x616c20, 0x617220, 0x617320, - 0x6369f3, 0x636f6e, 0x646520, 0x64656c, 0x646f20, 0x652064, 0x652065, - 0x65206c, 0x656c20, 0x656e20, 0x656e74, 0x657320, 0x657374, 0x69656e, - 0x69f36e, 0x6c6120, 0x6c6f73, 0x6e2065, 0x6e7465, 0x6f2064, 0x6f2065, - 0x6f6e20, 0x6f7220, 0x6f7320, 0x706172, 0x717565, 0x726120, 0x726573, - 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746f20, 0x756520, - 0xf36e20, - ], - ), - new NGramsPlusLang( - 'fr', - [ - 0x206175, 0x20636f, 0x206461, 0x206465, 0x206475, 0x20656e, 0x206574, - 0x206c61, 0x206c65, 0x207061, 0x20706f, 0x207072, 0x207175, 0x207365, - 0x20736f, 0x20756e, 0x20e020, 0x616e74, 0x617469, 0x636520, 0x636f6e, - 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, - 0x65206c, 0x652070, 0x652073, 0x656e20, 0x656e74, 0x657220, 0x657320, - 0x657420, 0x657572, 0x696f6e, 0x697320, 0x697420, 0x6c6120, 0x6c6520, - 0x6c6573, 0x6d656e, 0x6e2064, 0x6e6520, 0x6e7320, 0x6e7420, 0x6f6e20, - 0x6f6e74, 0x6f7572, 0x717565, 0x72206c, 0x726520, 0x732061, 0x732064, - 0x732065, 0x73206c, 0x732070, 0x742064, 0x746520, 0x74696f, 0x756520, - 0x757220, - ], - ), - new NGramsPlusLang( - 'it', - [ - 0x20616c, 0x206368, 0x20636f, 0x206465, 0x206469, 0x206520, 0x20696c, - 0x20696e, 0x206c61, 0x207065, 0x207072, 0x20756e, 0x612063, 0x612064, - 0x612070, 0x612073, 0x61746f, 0x636865, 0x636f6e, 0x64656c, 0x646920, - 0x652061, 0x652063, 0x652064, 0x652069, 0x65206c, 0x652070, 0x652073, - 0x656c20, 0x656c6c, 0x656e74, 0x657220, 0x686520, 0x692061, 0x692063, - 0x692064, 0x692073, 0x696120, 0x696c20, 0x696e20, 0x696f6e, 0x6c6120, - 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6520, 0x6e6920, 0x6e6f20, 0x6e7465, - 0x6f2061, 0x6f2064, 0x6f2069, 0x6f2073, 0x6f6e20, 0x6f6e65, 0x706572, - 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746f20, - 0x7a696f, - ], - ), - new NGramsPlusLang( - 'nl', - [ - 0x20616c, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656e, - 0x206765, 0x206865, 0x20696e, 0x206d61, 0x206d65, 0x206f70, 0x207465, - 0x207661, 0x207665, 0x20766f, 0x207765, 0x207a69, 0x61616e, 0x616172, - 0x616e20, 0x616e64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656e, - 0x646572, 0x652062, 0x652076, 0x65656e, 0x656572, 0x656e20, 0x657220, - 0x657273, 0x657420, 0x67656e, 0x686574, 0x696520, 0x696e20, 0x696e67, - 0x697320, 0x6e2062, 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206f, 0x6e2076, - 0x6e6465, 0x6e6720, 0x6f6e64, 0x6f6f72, 0x6f7020, 0x6f7220, 0x736368, - 0x737465, 0x742064, 0x746520, 0x74656e, 0x746572, 0x76616e, 0x766572, - 0x766f6f, - ], - ), - new NGramsPlusLang( - 'no', - [ - 0x206174, 0x206176, 0x206465, 0x20656e, 0x206572, 0x20666f, 0x206861, - 0x206920, 0x206d65, 0x206f67, 0x2070e5, 0x207365, 0x20736b, 0x20736f, - 0x207374, 0x207469, 0x207669, 0x20e520, 0x616e64, 0x617220, 0x617420, - 0x646520, 0x64656e, 0x646574, 0x652073, 0x656420, 0x656e20, 0x656e65, - 0x657220, 0x657265, 0x657420, 0x657474, 0x666f72, 0x67656e, 0x696b6b, - 0x696c20, 0x696e67, 0x6b6520, 0x6b6b65, 0x6c6520, 0x6c6c65, 0x6d6564, - 0x6d656e, 0x6e2073, 0x6e6520, 0x6e6720, 0x6e6765, 0x6e6e65, 0x6f6720, - 0x6f6d20, 0x6f7220, 0x70e520, 0x722073, 0x726520, 0x736f6d, 0x737465, - 0x742073, 0x746520, 0x74656e, 0x746572, 0x74696c, 0x747420, 0x747465, - 0x766572, - ], - ), - new NGramsPlusLang( - 'pt', - [ - 0x206120, 0x20636f, 0x206461, 0x206465, 0x20646f, 0x206520, 0x206573, - 0x206d61, 0x206e6f, 0x206f20, 0x207061, 0x20706f, 0x207072, 0x207175, - 0x207265, 0x207365, 0x20756d, 0x612061, 0x612063, 0x612064, 0x612070, - 0x616465, 0x61646f, 0x616c20, 0x617220, 0x617261, 0x617320, 0x636f6d, - 0x636f6e, 0x646120, 0x646520, 0x646f20, 0x646f73, 0x652061, 0x652064, - 0x656d20, 0x656e74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6d656e, - 0x6e7465, 0x6e746f, 0x6f2061, 0x6f2063, 0x6f2064, 0x6f2065, 0x6f2070, - 0x6f7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, - 0x732065, 0x732070, 0x737461, 0x746520, 0x746f20, 0x756520, 0xe36f20, - 0xe7e36f, - ], - ), - new NGramsPlusLang( - 'sv', - [ - 0x206174, 0x206176, 0x206465, 0x20656e, 0x2066f6, 0x206861, 0x206920, - 0x20696e, 0x206b6f, 0x206d65, 0x206f63, 0x2070e5, 0x20736b, 0x20736f, - 0x207374, 0x207469, 0x207661, 0x207669, 0x20e472, 0x616465, 0x616e20, - 0x616e64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656e, 0x646572, - 0x646574, 0x656420, 0x656e20, 0x657220, 0x657420, 0x66f672, 0x67656e, - 0x696c6c, 0x696e67, 0x6b6120, 0x6c6c20, 0x6d6564, 0x6e2073, 0x6e6120, - 0x6e6465, 0x6e6720, 0x6e6765, 0x6e696e, 0x6f6368, 0x6f6d20, 0x6f6e20, - 0x70e520, 0x722061, 0x722073, 0x726120, 0x736b61, 0x736f6d, 0x742073, - 0x746120, 0x746520, 0x746572, 0x74696c, 0x747420, 0x766172, 0xe47220, - 0xf67220, - ], - ), - ]; - } - - name(input: Context): EncodingName { - return input && input.c1Bytes ? 'windows-1252' : 'ISO-8859-1'; - } +interface PreparedSBCSLanguage { + readonly model: GeneratedSBCSLanguage; + readonly highByteLogProbabilities: Float64Array; } -export class ISO_8859_2 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0x20, - 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0x20, 0xb1, 0x20, 0xb3, - 0x20, 0xb5, 0xb6, 0xb7, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0x20, - ]; - } - - ngrams() { - return [ - new NGramsPlusLang( - 'cs', - [ - 0x206120, 0x206279, 0x20646f, 0x206a65, 0x206e61, 0x206e65, 0x206f20, - 0x206f64, 0x20706f, 0x207072, 0x2070f8, 0x20726f, 0x207365, 0x20736f, - 0x207374, 0x20746f, 0x207620, 0x207679, 0x207a61, 0x612070, 0x636520, - 0x636820, 0x652070, 0x652073, 0x652076, 0x656d20, 0x656eed, 0x686f20, - 0x686f64, 0x697374, 0x6a6520, 0x6b7465, 0x6c6520, 0x6c6920, 0x6e6120, - 0x6ee920, 0x6eec20, 0x6eed20, 0x6f2070, 0x6f646e, 0x6f6a69, 0x6f7374, - 0x6f7520, 0x6f7661, 0x706f64, 0x706f6a, 0x70726f, 0x70f865, 0x736520, - 0x736f75, 0x737461, 0x737469, 0x73746e, 0x746572, 0x746eed, 0x746f20, - 0x752070, 0xbe6520, 0xe16eed, 0xe9686f, 0xed2070, 0xed2073, 0xed6d20, - 0xf86564, - ], - ), - new NGramsPlusLang( - 'hu', - [ - 0x206120, 0x20617a, 0x206265, 0x206567, 0x20656c, 0x206665, 0x206861, - 0x20686f, 0x206973, 0x206b65, 0x206b69, 0x206bf6, 0x206c65, 0x206d61, - 0x206d65, 0x206d69, 0x206e65, 0x20737a, 0x207465, 0x20e973, 0x612061, - 0x61206b, 0x61206d, 0x612073, 0x616b20, 0x616e20, 0x617a20, 0x62616e, - 0x62656e, 0x656779, 0x656b20, 0x656c20, 0x656c65, 0x656d20, 0x656e20, - 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686f67, 0x696e74, - 0x697320, 0x6b2061, 0x6bf67a, 0x6d6567, 0x6d696e, 0x6e2061, 0x6e616b, - 0x6e656b, 0x6e656d, 0x6e7420, 0x6f6779, 0x732061, 0x737a65, 0x737a74, - 0x737ae1, 0x73e967, 0x742061, 0x747420, 0x74e173, 0x7a6572, 0xe16e20, - 0xe97320, - ], - ), - new NGramsPlusLang( - 'pl', - [ - 0x20637a, 0x20646f, 0x206920, 0x206a65, 0x206b6f, 0x206d61, 0x206d69, - 0x206e61, 0x206e69, 0x206f64, 0x20706f, 0x207072, 0x207369, 0x207720, - 0x207769, 0x207779, 0x207a20, 0x207a61, 0x612070, 0x612077, 0x616e69, - 0x636820, 0x637a65, 0x637a79, 0x646f20, 0x647a69, 0x652070, 0x652073, - 0x652077, 0x65207a, 0x65676f, 0x656a20, 0x656d20, 0x656e69, 0x676f20, - 0x696120, 0x696520, 0x69656a, 0x6b6120, 0x6b6920, 0x6b6965, 0x6d6965, - 0x6e6120, 0x6e6961, 0x6e6965, 0x6f2070, 0x6f7761, 0x6f7769, 0x706f6c, - 0x707261, 0x70726f, 0x70727a, 0x727a65, 0x727a79, 0x7369ea, 0x736b69, - 0x737461, 0x776965, 0x796368, 0x796d20, 0x7a6520, 0x7a6965, 0x7a7920, - 0xf37720, - ], - ), - new NGramsPlusLang( - 'ro', - [ - 0x206120, 0x206163, 0x206361, 0x206365, 0x20636f, 0x206375, 0x206465, - 0x206469, 0x206c61, 0x206d61, 0x207065, 0x207072, 0x207365, 0x2073e3, - 0x20756e, 0x20ba69, 0x20ee6e, 0x612063, 0x612064, 0x617265, 0x617420, - 0x617465, 0x617520, 0x636172, 0x636f6e, 0x637520, 0x63e320, 0x646520, - 0x652061, 0x652063, 0x652064, 0x652070, 0x652073, 0x656120, 0x656920, - 0x656c65, 0x656e74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, - 0x696520, 0x696920, 0x696e20, 0x6c6120, 0x6c6520, 0x6c6f72, 0x6c7569, - 0x6e6520, 0x6e7472, 0x6f7220, 0x70656e, 0x726520, 0x726561, 0x727520, - 0x73e320, 0x746520, 0x747275, 0x74e320, 0x756920, 0x756c20, 0xba6920, - 0xee6e20, - ], - ), - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1250' : 'ISO-8859-2'; - } +interface PreparedSBCSModel { + readonly model: GeneratedSBCSModel; + readonly languages: readonly PreparedSBCSLanguage[]; + readonly ngramLanguageMasks: ReadonlyMap; } -export class ISO_8859_5 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0x20, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - 0x20d220, 0x20d2de, 0x20d4de, 0x20d7d0, 0x20d820, 0x20dad0, 0x20dade, - 0x20ddd0, 0x20ddd5, 0x20ded1, 0x20dfde, 0x20dfe0, 0x20e0d0, 0x20e1de, - 0x20e1e2, 0x20e2de, 0x20e7e2, 0x20ede2, 0xd0ddd8, 0xd0e2ec, 0xd3de20, - 0xd5dbec, 0xd5ddd8, 0xd5e1e2, 0xd5e220, 0xd820df, 0xd8d520, 0xd8d820, - 0xd8ef20, 0xdbd5dd, 0xdbd820, 0xdbecdd, 0xddd020, 0xddd520, 0xddd8d5, - 0xddd8ef, 0xddde20, 0xddded2, 0xde20d2, 0xde20df, 0xde20e1, 0xded220, - 0xded2d0, 0xded3de, 0xded920, 0xdedbec, 0xdedc20, 0xdee1e2, 0xdfdedb, - 0xdfe0d5, 0xdfe0d8, 0xdfe0de, 0xe0d0d2, 0xe0d5d4, 0xe1e2d0, 0xe1e2d2, - 0xe1e2d8, 0xe1ef20, 0xe2d5db, 0xe2de20, 0xe2dee0, 0xe2ec20, 0xe7e2de, - 0xebe520, - ]; - } +type HighByteCounts = readonly (readonly [number, number])[]; - name(): EncodingName { - return 'ISO-8859-5'; - } - - language() { - return 'ru'; - } +function highByteLogProbabilities(distribution: HighByteDistribution) { + const counts = new Map(distribution.counts); + const denominator = distribution.total + 128; + return Float64Array.from({ length: 128 }, (_, index) => + Math.log(((counts.get(index + 0x80) ?? 0) + 1) / denominator), + ); } -export class ISO_8859_6 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, - ]; - } - - ngrams() { - return [ - 0x20c7e4, 0x20c7e6, 0x20c8c7, 0x20d9e4, 0x20e1ea, 0x20e4e4, 0x20e5e6, - 0x20e8c7, 0xc720c7, 0xc7c120, 0xc7ca20, 0xc7d120, 0xc7e420, 0xc7e4c3, - 0xc7e4c7, 0xc7e4c8, 0xc7e4ca, 0xc7e4cc, 0xc7e4cd, 0xc7e4cf, 0xc7e4d3, - 0xc7e4d9, 0xc7e4e2, 0xc7e4e5, 0xc7e4e8, 0xc7e4ea, 0xc7e520, 0xc7e620, - 0xc7e6ca, 0xc820c7, 0xc920c7, 0xc920e1, 0xc920e4, 0xc920e5, 0xc920e8, - 0xca20c7, 0xcf20c7, 0xcfc920, 0xd120c7, 0xd1c920, 0xd320c7, 0xd920c7, - 0xd9e4e9, 0xe1ea20, 0xe420c7, 0xe4c920, 0xe4e920, 0xe4ea20, 0xe520c7, - 0xe5c720, 0xe5c920, 0xe5e620, 0xe620c7, 0xe720c7, 0xe7c720, 0xe8c7e4, - 0xe8e620, 0xe920c7, 0xea20c7, 0xea20e5, 0xea20e8, 0xeac920, 0xead120, - 0xeae620, - ]; - } - - name(): EncodingName { - return 'ISO-8859-6'; - } - - language() { - return 'ar'; - } -} - -export class ISO_8859_7 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0xa1, 0xa2, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xdc, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, - 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0x20, - ]; - } - - ngrams() { - return [ - 0x20e1ed, 0x20e1f0, 0x20e3e9, 0x20e4e9, 0x20e5f0, 0x20e720, 0x20eae1, - 0x20ece5, 0x20ede1, 0x20ef20, 0x20f0e1, 0x20f0ef, 0x20f0f1, 0x20f3f4, - 0x20f3f5, 0x20f4e7, 0x20f4ef, 0xdfe120, 0xe120e1, 0xe120f4, 0xe1e920, - 0xe1ed20, 0xe1f0fc, 0xe1f220, 0xe3e9e1, 0xe5e920, 0xe5f220, 0xe720f4, - 0xe7ed20, 0xe7f220, 0xe920f4, 0xe9e120, 0xe9eade, 0xe9f220, 0xeae1e9, - 0xeae1f4, 0xece520, 0xed20e1, 0xed20e5, 0xed20f0, 0xede120, 0xeff220, - 0xeff520, 0xf0eff5, 0xf0f1ef, 0xf0fc20, 0xf220e1, 0xf220e5, 0xf220ea, - 0xf220f0, 0xf220f4, 0xf3e520, 0xf3e720, 0xf3f4ef, 0xf4e120, 0xf4e1e9, - 0xf4e7ed, 0xf4e7f2, 0xf4e9ea, 0xf4ef20, 0xf4eff5, 0xf4f9ed, 0xf9ed20, - 0xfeed20, - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1253' : 'ISO-8859-7'; - } - - language() { - return 'el'; - } +function ngramLanguageMasks(model: GeneratedSBCSModel) { + const masks = new Map(); + model.languages.forEach((language, languageIndex) => { + for (const ngram of new Set(language.ngrams)) { + masks.set(ngram, (masks.get(ngram) ?? 0) | (1 << languageIndex)); + } + }); + return masks; } -export class ISO_8859_8 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, - 0x20, 0x20, 0x20, 0x20, - ]; - } - - ngrams() { - return [ - new NGramsPlusLang( - 'he', - [ - 0x20e0e5, 0x20e0e7, 0x20e0e9, 0x20e0fa, 0x20e1e9, 0x20e1ee, 0x20e4e0, - 0x20e4e5, 0x20e4e9, 0x20e4ee, 0x20e4f2, 0x20e4f9, 0x20e4fa, 0x20ece0, - 0x20ece4, 0x20eee0, 0x20f2ec, 0x20f9ec, 0xe0fa20, 0xe420e0, 0xe420e1, - 0xe420e4, 0xe420ec, 0xe420ee, 0xe420f9, 0xe4e5e0, 0xe5e020, 0xe5ed20, - 0xe5ef20, 0xe5f820, 0xe5fa20, 0xe920e4, 0xe9e420, 0xe9e5fa, 0xe9e9ed, - 0xe9ed20, 0xe9ef20, 0xe9f820, 0xe9fa20, 0xec20e0, 0xec20e4, 0xece020, - 0xece420, 0xed20e0, 0xed20e1, 0xed20e4, 0xed20ec, 0xed20ee, 0xed20f9, - 0xeee420, 0xef20e4, 0xf0e420, 0xf0e920, 0xf0e9ed, 0xf2ec20, 0xf820e4, - 0xf8e9ed, 0xf9ec20, 0xfa20e0, 0xfa20e1, 0xfa20e4, 0xfa20ec, 0xfa20ee, - 0xfa20f9, - ], - ), - new NGramsPlusLang( - 'he', - [ - 0x20e0e5, 0x20e0ec, 0x20e4e9, 0x20e4ec, 0x20e4ee, 0x20e4f0, 0x20e9f0, - 0x20ecf2, 0x20ecf9, 0x20ede5, 0x20ede9, 0x20efe5, 0x20efe9, 0x20f8e5, - 0x20f8e9, 0x20fae0, 0x20fae5, 0x20fae9, 0xe020e4, 0xe020ec, 0xe020ed, - 0xe020fa, 0xe0e420, 0xe0e5e4, 0xe0ec20, 0xe0ee20, 0xe120e4, 0xe120ed, - 0xe120fa, 0xe420e4, 0xe420e9, 0xe420ec, 0xe420ed, 0xe420ef, 0xe420f8, - 0xe420fa, 0xe4ec20, 0xe5e020, 0xe5e420, 0xe7e020, 0xe9e020, 0xe9e120, - 0xe9e420, 0xec20e4, 0xec20ed, 0xec20fa, 0xecf220, 0xecf920, 0xede9e9, - 0xede9f0, 0xede9f8, 0xee20e4, 0xee20ed, 0xee20fa, 0xeee120, 0xeee420, - 0xf2e420, 0xf920e4, 0xf920ed, 0xf920fa, 0xf9e420, 0xfae020, 0xfae420, - 0xfae5e9, - ], - ), - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1255' : 'ISO-8859-8'; - } - - language() { - return 'he'; - } +const preparedSBCSModels: readonly PreparedSBCSModel[] = ( + generatedSBCSModels as readonly GeneratedSBCSModel[] +).map((model) => ({ + model, + languages: model.languages.map((language) => ({ + model: language, + highByteLogProbabilities: highByteLogProbabilities(language.highBytes), + })), + ngramLanguageMasks: ngramLanguageMasks(model), +})); +const maxLanguagesPerModel = Math.max( + ...preparedSBCSModels.map((model) => model.languages.length), +); + +const equivalentEncodingFamilies = [ + ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], + ['ISO-8859-2', 'windows-1250'], + ['ISO-8859-7', 'windows-1253'], + ['ISO-8859-8', 'windows-1255'], + ['ISO-8859-9', 'windows-1254'], + ['ISO-8859-13', 'windows-1257'], +] as const; + +function family(name: Match['name']) { + return equivalentEncodingFamilies.find((value) => + value.includes(name as never), + ); } -export class ISO_8859_9 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x69, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, - 0x20696c, 0x206b61, 0x206b6f, 0x206d61, 0x206f6c, 0x207361, 0x207461, - 0x207665, 0x207961, 0x612062, 0x616b20, 0x616c61, 0x616d61, 0x616e20, - 0x616efd, 0x617220, 0x617261, 0x6172fd, 0x6173fd, 0x617961, 0x626972, - 0x646120, 0x646520, 0x646920, 0x652062, 0x65206b, 0x656469, 0x656e20, - 0x657220, 0x657269, 0x657369, 0x696c65, 0x696e20, 0x696e69, 0x697220, - 0x6c616e, 0x6c6172, 0x6c6520, 0x6c6572, 0x6e2061, 0x6e2062, 0x6e206b, - 0x6e6461, 0x6e6465, 0x6e6520, 0x6e6920, 0x6e696e, 0x6efd20, 0x72696e, - 0x72fd6e, 0x766520, 0x796120, 0x796f72, 0xfd6e20, 0xfd6e64, 0xfd6efd, - 0xfdf0fd, - ]; - } - - name(det: Context): EncodingName { - return det && det.c1Bytes ? 'windows-1254' : 'ISO-8859-9'; - } - - language() { - return 'tr'; - } +function familyPreference(left: Candidate, right: Candidate, c1Bytes: boolean) { + const leftFamily = family(left.name); + if (!leftFamily || leftFamily !== family(right.name)) return 0; + const preferred = + leftFamily[0] === 'ISO-8859-13' || c1Bytes + ? [...leftFamily].reverse() + : [...leftFamily]; + return ( + preferred.indexOf(left.name as never) - + preferred.indexOf(right.name as never) + ); } -export class windows_1251 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x90, 0x83, 0x20, 0x83, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, - 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, - 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa2, 0xa2, 0xbc, 0x20, 0xb4, 0x20, 0x20, - 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0xb3, 0xb3, - 0xb4, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0xbc, 0xbe, 0xbe, 0xbf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, - ]; - } - - ngrams() { - return [ - 0x20e220, 0x20e2ee, 0x20e4ee, 0x20e7e0, 0x20e820, 0x20eae0, 0x20eaee, - 0x20ede0, 0x20ede5, 0x20eee1, 0x20efee, 0x20eff0, 0x20f0e0, 0x20f1ee, - 0x20f1f2, 0x20f2ee, 0x20f7f2, 0x20fdf2, 0xe0ede8, 0xe0f2fc, 0xe3ee20, - 0xe5ebfc, 0xe5ede8, 0xe5f1f2, 0xe5f220, 0xe820ef, 0xe8e520, 0xe8e820, - 0xe8ff20, 0xebe5ed, 0xebe820, 0xebfced, 0xede020, 0xede520, 0xede8e5, - 0xede8ff, 0xedee20, 0xedeee2, 0xee20e2, 0xee20ef, 0xee20f1, 0xeee220, - 0xeee2e0, 0xeee3ee, 0xeee920, 0xeeebfc, 0xeeec20, 0xeef1f2, 0xefeeeb, - 0xeff0e5, 0xeff0e8, 0xeff0ee, 0xf0e0e2, 0xf0e5e4, 0xf1f2e0, 0xf1f2e2, - 0xf1f2e8, 0xf1ff20, 0xf2e5eb, 0xf2ee20, 0xf2eef0, 0xf2fc20, 0xf7f2ee, - 0xfbf520, - ]; - } - - name(): EncodingName { - return 'windows-1251'; +function ngramHits( + input: Uint8Array, + prepared: PreparedSBCSModel, + hits: Uint32Array, +): number { + hits.fill(0, 0, prepared.languages.length); + const byteMap = prepared.model.byteMap; + let ignoreSpace = false; + let ngram = 0; + let total = 0; + + const record = (value: number) => { + let languageMask = prepared.ngramLanguageMasks.get(value) ?? 0; + while (languageMask !== 0) { + const languageBit = languageMask & -languageMask; + hits[31 - Math.clz32(languageBit)] += 1; + languageMask ^= languageBit; + } + }; + + for (let inputIndex = 0; inputIndex < input.length; inputIndex += 1) { + const byte = input[inputIndex]; + const mapped = byteMap[byte]; + if (mapped !== 0) { + if (!(mapped === 0x20 && ignoreSpace)) { + ngram = ((ngram << 8) + mapped) & N_GRAM_MASK; + total += 1; + record(ngram); + } + ignoreSpace = mapped === 0x20; + } } - language() { - return 'ru'; - } + ngram = ((ngram << 8) + 0x20) & N_GRAM_MASK; + total += 1; + record(ngram); + return total; } -export class windows_1256 extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x81, 0x20, 0x83, - 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x8a, 0x20, 0x9c, 0x8d, 0x8e, 0x8f, - 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x98, 0x20, 0x9a, 0x20, - 0x9c, 0x20, 0x20, 0x9f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0x20, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0x20, 0x20, 0x20, 0x20, 0xf4, 0x20, 0x20, 0x20, 0x20, 0xf9, 0x20, 0xfb, - 0xfc, 0x20, 0x20, 0xff, - ]; - } +function byteLogLikelihood( + highByteCounts: HighByteCounts, + logProbabilities: Float64Array, +) { + let highBytes = 0; + let likelihood = 0; - ngrams() { - return [ - 0x20c7e1, 0x20c7e4, 0x20c8c7, 0x20dae1, 0x20dded, 0x20e1e1, 0x20e3e4, - 0x20e6c7, 0xc720c7, 0xc7c120, 0xc7ca20, 0xc7d120, 0xc7e120, 0xc7e1c3, - 0xc7e1c7, 0xc7e1c8, 0xc7e1ca, 0xc7e1cc, 0xc7e1cd, 0xc7e1cf, 0xc7e1d3, - 0xc7e1da, 0xc7e1de, 0xc7e1e3, 0xc7e1e6, 0xc7e1ed, 0xc7e320, 0xc7e420, - 0xc7e4ca, 0xc820c7, 0xc920c7, 0xc920dd, 0xc920e1, 0xc920e3, 0xc920e6, - 0xca20c7, 0xcf20c7, 0xcfc920, 0xd120c7, 0xd1c920, 0xd320c7, 0xda20c7, - 0xdae1ec, 0xdded20, 0xe120c7, 0xe1c920, 0xe1ec20, 0xe1ed20, 0xe320c7, - 0xe3c720, 0xe3c920, 0xe3e420, 0xe420c7, 0xe520c7, 0xe5c720, 0xe6c7e1, - 0xe6e420, 0xec20c7, 0xed20c7, 0xed20e3, 0xed20e6, 0xedc920, 0xedd120, - 0xede420, - ]; + for (const [byte, count] of highByteCounts) { + likelihood += logProbabilities[byte - 0x80] * count; + highBytes += count; } - name(): EncodingName { - return 'windows-1256'; - } - - language() { - return 'ar'; - } + return highBytes === 0 ? Number.NEGATIVE_INFINITY : likelihood / highBytes; } -export class windows_1257 extends sbcs { - match(det: Context): Match | null { - return det.inputBytes.some((byte) => byte >= 0x80) - ? super.match(det) - : null; - } - - byteMap() { - const byteMap = new Array(256).fill(0x20); - byteMap[0x27] = 0; - - for (let i = 0x41; i <= 0x5a; i++) byteMap[i] = i + 0x20; - for (let i = 0x61; i <= 0x7a; i++) byteMap[i] = i; - - byteMap[0x8e] = 0x8e; - byteMap[0xa8] = byteMap[0xb8] = 0xb8; - byteMap[0xaa] = byteMap[0xba] = 0xba; - byteMap[0xaf] = byteMap[0xbf] = 0xbf; - byteMap[0xb5] = 0xb5; - - for (let i = 0xc0; i <= 0xd6; i++) byteMap[i] = i + 0x20; - for (let i = 0xd8; i <= 0xde; i++) byteMap[i] = i + 0x20; - byteMap[0xdf] = 0xdf; - for (let i = 0xe0; i <= 0xf6; i++) byteMap[i] = i; - for (let i = 0xf8; i <= 0xfe; i++) byteMap[i] = i; - - return byteMap; - } - - ngrams() { - // Generated from original Estonian, Latvian, and Lithuanian prose samples. - return [ - new NGramsPlusLang( - 'et', - [ - 0x20616a, 0x206172, 0x206565, 0x206967, 0x206a61, 0x206b61, 0x206b65, - 0x206b69, 0x206b6f, 0x206b75, 0x206c61, 0x206e69, 0x206f6e, 0x207269, - 0x207365, 0x207465, 0x612061, 0x61206a, 0x61206b, 0x61206c, 0x616220, - 0x616420, 0x616b73, 0x617375, 0x617461, 0x617661, 0x642074, 0x646120, - 0x647573, 0x65206b, 0x652073, 0x65656c, 0x656573, 0x656c20, 0x656c65, - 0x656c74, 0x657320, 0x657374, 0x69206b, 0x696420, 0x69726a, 0x697320, - 0x6a6120, 0x6b6173, 0x6b6565, 0x6b6972, 0x6b6f6f, 0x6b7365, 0x6b7374, - 0x6c696b, 0x6c7420, 0x732061, 0x73206b, 0x736520, 0x73656c, 0x737465, - 0x737469, 0x737574, 0x74616b, 0x746174, 0x746920, 0x757320, 0x757365, - 0x757461, - ], - ), - new NGramsPlusLang( - 'lv', - [ - 0x206172, 0x206369, 0x206461, 0x206972, 0x20697a, 0x206c61, 0x206de2, - 0x207061, 0x207072, 0x207261, 0x207361, 0x20736b, 0x207465, 0x20756e, - 0x207661, 0x612069, 0x612075, 0x612076, 0x616269, 0x616920, 0x616b73, - 0x616c6f, 0x617072, 0x617275, 0x617320, 0x617476, 0x626965, 0x627520, - 0x63696c, 0x636974, 0x646120, 0x646172, 0x647520, 0x647a20, 0x656472, - 0x65696b, 0x656b73, 0x656e61, 0x657320, 0x65f075, 0x692069, 0x696564, - 0x6965f0, 0x696ae2, 0x696c76, 0x6b7374, 0x6c6174, 0x6c6f64, 0x6e6120, - 0x6f6461, 0x72616b, 0x72756e, 0x732073, 0x732075, 0x732076, 0x737475, - 0x7374ee, 0x747520, 0x747669, 0x752076, 0x756e20, 0x757320, 0x76616c, - 0xe27320, - ], - ), - new NGramsPlusLang( - 'lt', - [ - 0x206169, 0x206265, 0x206972, 0x206b61, 0x206c69, 0x206d6f, 0x207261, - 0x207361, 0x20736b, 0x207375, 0x207461, 0x207465, 0x207475, 0x207661, - 0x207669, 0x612074, 0x6120f0, 0x616920, 0x616974, 0x6169f0, 0x616c62, - 0x616d61, 0x616e74, 0x617262, 0x617274, 0x617520, 0x61756a, 0x61f079, - 0x626120, 0x626569, 0x62656e, 0x656e69, 0x657475, 0x692061, 0x692069, - 0x69206b, 0x69206d, 0x696169, 0x69656e, 0x696574, 0x696e67, 0x697220, - 0x697320, 0x69f06b, 0x6a616d, 0x6a6520, 0x6b6169, 0x6b616c, 0x6c6261, - 0x6c6965, 0x6d6973, 0x6d6f6b, 0x6eeb20, 0x6f6a65, 0x6f6b79, 0x6f7320, - 0x7261f0, 0x732069, 0x73206b, 0x732076, 0x736b61, 0x746920, 0x747576, - 0x796b6c, - ], - ), - ]; - } - - name(): EncodingName { - return 'windows-1257'; - } +function modelByteLogLikelihood( + highByteCounts: HighByteCounts, + prepared: PreparedSBCSModel, +) { + let best = Number.NEGATIVE_INFINITY; + for (const language of prepared.languages) { + best = Math.max( + best, + byteLogLikelihood(highByteCounts, language.highByteLogProbabilities), + ); + } + return best; } -export class windows_1258 extends sbcs { - match(det: Context): Match | null { - return det.inputBytes.some((byte) => byte >= 0x80) - ? super.match(det) - : null; - } - - byteMap() { - const byteMap = new Array(256).fill(0x20); - byteMap[0x27] = 0; - - for (let i = 0x41; i <= 0x5a; i++) byteMap[i] = i + 0x20; - for (let i = 0x61; i <= 0x7a; i++) byteMap[i] = i; - - byteMap[0x83] = 0x83; - byteMap[0x88] = 0x88; - byteMap[0x8c] = byteMap[0x9c] = 0x9c; - byteMap[0x9f] = 0xff; - byteMap[0xaa] = 0xaa; - byteMap[0xb5] = 0xb5; - byteMap[0xba] = 0xba; - - for (let i = 0xc0; i <= 0xcb; i++) byteMap[i] = i + 0x20; - byteMap[0xcc] = 0xcc; - for (let i = 0xcd; i <= 0xd1; i++) byteMap[i] = i + 0x20; - byteMap[0xd2] = 0xd2; - for (let i = 0xd3; i <= 0xd6; i++) byteMap[i] = i + 0x20; - for (let i = 0xd8; i <= 0xdd; i++) byteMap[i] = i + 0x20; - byteMap[0xde] = 0xde; - byteMap[0xdf] = 0xdf; - for (let i = 0xe0; i <= 0xf6; i++) byteMap[i] = i; - for (let i = 0xf8; i <= 0xfd; i++) byteMap[i] = i; - byteMap[0xff] = 0xff; - - return byteMap; - } - - ngrams() { - // Generated from an original Vietnamese prose sample. - return [ - 0x206368, 0x2063e1, 0x206769, 0x20686f, 0x206e67, 0x207068, 0x207468, - 0x207469, 0x207472, 0x207669, 0x2076e0, 0x61f274, 0x632063, 0x632076, - 0x636820, 0x6368ed, 0x63e163, 0x672068, 0x67206e, 0x672074, 0x672076, - 0x67f46e, 0x67fdde, 0x67fdf5, 0x682074, 0x686ff2, 0x68ed6e, 0x69206e, - 0x692074, 0x69ead2, 0x69eaec, 0x69eaf2, 0x6e2063, 0x6e206e, 0x6e2074, - 0x6e6720, 0x6e67f4, 0x6e67fd, 0x6e6820, 0x6f6e67, 0x6ff263, 0x742073, - 0x7468fd, 0x7469ea, 0x74726f, 0x752074, 0x7669ea, 0x76e020, 0xcc6920, - 0xcc6e67, 0xde2068, 0xe16368, 0xeaec6e, 0xeaf274, 0xec6e67, 0xec7420, - 0xed6e68, 0xf26320, 0xf26920, 0xf26e20, 0xf27420, 0xf46e20, 0xf5cc69, - 0xfdf5cc, - ]; - } - - name(): EncodingName { - return 'windows-1258'; +function candidate( + input: Uint8Array, + highByteCounts: HighByteCounts, + prepared: PreparedSBCSModel, + hits: Uint32Array, +): Candidate { + const { model, languages: preparedLanguages } = prepared; + const total = ngramHits(input, prepared, hits); + let bestIndex = 0; + let bestConfidence = -1; + let bestByteLogLikelihood = Number.NEGATIVE_INFINITY; + let bestHitRate = 0; + + for (let index = 0; index < preparedLanguages.length; index += 1) { + const hitRate = hits[index] / total; + const confidence = hitRate > 0.33 ? 98 : Math.floor(hitRate * 300); + const likelihood = byteLogLikelihood( + highByteCounts, + preparedLanguages[index].highByteLogProbabilities, + ); + if ( + confidence > bestConfidence || + (confidence === bestConfidence && likelihood > bestByteLogLikelihood) + ) { + bestIndex = index; + bestConfidence = confidence; + bestByteLogLikelihood = likelihood; + bestHitRate = hitRate; + } } - language() { - return 'vi'; - } + return { + name: model.encoding, + lang: preparedLanguages[bestIndex].model.language, + confidence: bestConfidence, + byteLogLikelihood: bestByteLogLikelihood, + hits: hits[bestIndex], + total, + hitRate: bestHitRate, + }; } -export class windows_874 extends sbcs { - byteMap() { - const byteMap = new Array(256).fill(0x20); - byteMap[0x27] = 0; - - for (let i = 0x41; i <= 0x5a; i++) byteMap[i] = i + 0x20; - for (let i = 0x61; i <= 0x7a; i++) byteMap[i] = i; - for (let i = 0xa1; i <= 0xda; i++) byteMap[i] = i; - for (let i = 0xdf; i <= 0xfb; i++) byteMap[i] = i; - - return byteMap; - } - - ngrams() { - return [ - 0x20a1d2, 0x20a2e8, 0x20c0d2, 0x20c7c3, 0x20e1c5, 0xa1d1b9, 0xa1d2c3, - 0xa2cda7, 0xa2d5c2, 0xa4c7d2, 0xa8d3c7, 0xaad5c7, 0xaae9c0, 0xaae9e3, - 0xb5bbc3, 0xb7c2e0, 0xb7c2e3, 0xb7d5e8, 0xb920c0, 0xb9a1d2, 0xb9c0d2, - 0xbbc3d0, 0xbbe7b9, 0xc0d2c9, 0xc3c120, 0xc3c3b3, 0xc3c3c1, 0xc3d0a8, - 0xc3d0e2, 0xc3e0a2, 0xc7c3c3, 0xc7d1b9, 0xc7d2c1, 0xc7d4b5, 0xc9d2b7, - 0xc9d2e4, 0xcad7e8, 0xcda4c7, 0xd0a8d3, 0xd0e2c2, 0xd1b920, 0xd2a7a1, - 0xd2c1cb, 0xd2c2e4, 0xd2c320, 0xd2c3e0, 0xd2c9d2, 0xd2e4b7, 0xd3c7d1, - 0xd4b5bb, 0xd5c2b9, 0xd5c7d4, 0xd7e8cd, 0xe0a2d5, 0xe0bbe7, 0xe1c5d0, - 0xe2c2a4, 0xe3aae9, 0xe3b9a1, 0xe4b7c2, 0xe8d2a7, 0xe8e3aa, 0xe9c0d2, - 0xe9e3b9, - ]; - } - - name(): EncodingName { - return 'windows-874'; - } - - language() { - return 'th'; - } +function statisticallyCompetitive(best: Score, value: Score) { + const variance = + (best.hitRate * (1 - best.hitRate)) / best.total + + (value.hitRate * (1 - value.hitRate)) / value.total; + return best.hitRate - value.hitRate <= 1.96 * Math.sqrt(variance); } -export class KOI8_R extends sbcs { - byteMap() { - return [ - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, - ]; - } - - ngrams() { - return [ - 0x20c4cf, 0x20c920, 0x20cbc1, 0x20cbcf, 0x20cec1, 0x20cec5, 0x20cfc2, - 0x20d0cf, 0x20d0d2, 0x20d2c1, 0x20d3cf, 0x20d3d4, 0x20d4cf, 0x20d720, - 0x20d7cf, 0x20dac1, 0x20dcd4, 0x20ded4, 0xc1cec9, 0xc1d4d8, 0xc5ccd8, - 0xc5cec9, 0xc5d3d4, 0xc5d420, 0xc7cf20, 0xc920d0, 0xc9c520, 0xc9c920, - 0xc9d120, 0xccc5ce, 0xccc920, 0xccd8ce, 0xcec120, 0xcec520, 0xcec9c5, - 0xcec9d1, 0xcecf20, 0xcecfd7, 0xcf20d0, 0xcf20d3, 0xcf20d7, 0xcfc7cf, - 0xcfca20, 0xcfccd8, 0xcfcd20, 0xcfd3d4, 0xcfd720, 0xcfd7c1, 0xd0cfcc, - 0xd0d2c5, 0xd0d2c9, 0xd0d2cf, 0xd2c1d7, 0xd2c5c4, 0xd3d120, 0xd3d4c1, - 0xd3d4c9, 0xd3d4d7, 0xd4c5cc, 0xd4cf20, 0xd4cfd2, 0xd4d820, 0xd9c820, - 0xded4cf, - ]; - } - - name(): EncodingName { - return 'KOI8-R'; - } - - language() { - return 'ru'; - } +export function analyseGeneratedSBCS(context: Context): Match[] { + const highByteCounts: [number, number][] = []; + for (let byte = 0x80; byte <= 0xff; byte += 1) { + const count = context.byteStats[byte]; + if (count > 0) highByteCounts.push([byte, count]); + } + const hits = new Uint32Array(maxLanguagesPerModel); + const models = + highByteCounts.length === 0 + ? preparedSBCSModels + : preparedSBCSModels + .map((model, index) => ({ + model, + index, + likelihood: modelByteLogLikelihood(highByteCounts, model), + })) + .sort( + (left, right) => + right.likelihood - left.likelihood || left.index - right.index, + ) + .slice(0, MAX_SBCS_CANDIDATES) + .map(({ model }) => model); + const candidates = models.map((model) => + candidate(context.inputBytes, highByteCounts, model, hits), + ); + candidates.sort((left, right) => right.confidence - left.confidence); + + const strongest = candidates[0]; + const competitive = candidates.filter((value) => + statisticallyCompetitive(strongest, value), + ); + const competitiveNames = new Set(competitive.map((value) => value.name)); + competitive.sort( + (left, right) => + familyPreference(left, right, context.c1Bytes) || + right.byteLogLikelihood - left.byteLogLikelihood || + right.confidence - left.confidence, + ); + const remaining = candidates.filter( + (value) => !competitiveNames.has(value.name), + ); + + let confidenceCeiling = 1; + return [...competitive, ...remaining] + .filter((value) => value.confidence > 0) + .map(({ name, lang, confidence }) => { + confidenceCeiling = Math.min(confidenceCeiling, confidence / 100); + return { name, lang, confidence: confidenceCeiling }; + }); } diff --git a/src/index.test.ts b/src/index.test.ts index 697241a..392e730 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest'; describe('chardet', () => { const path = __dirname + '/test/data/encodings/utf8'; - const expectedEncodingsFromPath = [ + const legacyExpectedEncodingsFromPath = [ { confidence: 1, name: 'UTF-8', lang: undefined }, { confidence: 0.32, name: 'windows-1252', lang: 'fr' }, { confidence: 0.19, name: 'KOI8-R', lang: 'ru' }, @@ -94,7 +94,16 @@ describe('chardet', () => { describe('#analyse', () => { it('should return a list of encodings, sorted by confidence level in descending order', () => { const matches = chardet.analyse(fs.readFileSync(path)); - expect(matches).toEqual(expectedEncodingsFromPath); + expect(matches[0]).toEqual(legacyExpectedEncodingsFromPath[0]); + expect(matches.length).toBeGreaterThan( + legacyExpectedEncodingsFromPath.length, + ); + expect( + matches.every( + (match, index) => + index === 0 || matches[index - 1].confidence >= match.confidence, + ), + ).toBe(true); }); }); }); diff --git a/src/index.ts b/src/index.ts index bd9402c..cf1ab7c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import Ascii from './encoding/ascii'; import Utf8 from './encoding/utf8'; import * as unicode from './encoding/unicode'; import * as mbcs from './encoding/mbcs'; -import * as sbcs from './encoding/sbcs'; +import { analyseGeneratedSBCS } from './encoding/sbcs'; import * as iso2022 from './encoding/iso2022'; import { isByteArray } from './utils'; @@ -32,19 +32,6 @@ const recognisers: Recogniser[] = [ new iso2022.ISO_2022_JP(), new iso2022.ISO_2022_KR(), new iso2022.ISO_2022_CN(), - new sbcs.ISO_8859_1(), - new sbcs.ISO_8859_2(), - new sbcs.ISO_8859_5(), - new sbcs.ISO_8859_6(), - new sbcs.ISO_8859_7(), - new sbcs.ISO_8859_8(), - new sbcs.ISO_8859_9(), - new sbcs.windows_1251(), - new sbcs.windows_1256(), - new sbcs.windows_1257(), - new sbcs.windows_1258(), - new sbcs.windows_874(), - new sbcs.KOI8_R(), new Ascii(), ]; @@ -84,16 +71,27 @@ export const analyse = (buffer: Uint8Array): AnalyseResult => { inputLen: buffer.length, }; - const matches = recognisers - .map((rec) => { - return rec.match(context); - }) - .filter((match) => { - return !!match; - }) - .sort((a, b) => { - return b!.confidence - a!.confidence; - }); + const generatedMatches = analyseGeneratedSBCS(context); + const generatedRank = new Map( + generatedMatches.map((value, index) => [value.name, index]), + ); + const matches = [ + ...recognisers + .map((rec) => { + return rec.match(context); + }) + .filter((match) => { + return !!match; + }), + ...generatedMatches, + ].sort((a, b) => { + const leftRank = generatedRank.get(a!.name); + const rightRank = generatedRank.get(b!.name); + if (leftRank !== undefined && rightRank !== undefined) { + return leftRank - rightRank; + } + return b!.confidence - a!.confidence; + }); return matches as Match[]; }; diff --git a/src/match.ts b/src/match.ts index 434956d..1e60883 100644 --- a/src/match.ts +++ b/src/match.ts @@ -11,12 +11,26 @@ export type EncodingName = | 'ISO-2022-KR' | 'ISO-8859-1' | 'ISO-8859-2' + | 'ISO-8859-3' + | 'ISO-8859-4' | 'ISO-8859-5' | 'ISO-8859-6' | 'ISO-8859-7' | 'ISO-8859-8' | 'ISO-8859-9' + | 'ISO-8859-10' + | 'ISO-8859-13' + | 'ISO-8859-14' + | 'ISO-8859-15' + | 'ISO-8859-16' + | 'IBM855' + | 'IBM866' + | 'CP850' + | 'CP852' | 'KOI8-R' + | 'KOI8-U' + | 'macintosh' + | 'x-mac-cyrillic' | 'Shift_JIS' | 'UTF-16BE' | 'UTF-16LE' From 8e8e96931e53d16c9fdcf6d0bdf863a0d6ff3f41 Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Mon, 20 Jul 2026 13:54:33 +1000 Subject: [PATCH 4/6] Update docs with supported charsets --- README.md | 21 ++++++++++------ src/encoding/sbcs.test.ts | 52 ++++++++++++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2a797b8..932ebb4 100644 --- a/README.md +++ b/README.md @@ -89,16 +89,29 @@ Note on [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncod - ISO-2022-CN - ISO-8859-1 - ISO-8859-2 +- ISO-8859-3 +- ISO-8859-4 - ISO-8859-5 - ISO-8859-6 - ISO-8859-7 - ISO-8859-8 - ISO-8859-9 +- ISO-8859-10 +- ISO-8859-13 +- ISO-8859-14 +- ISO-8859-15 +- ISO-8859-16 - Big5 +- CP850 +- CP852 - EUC-JP - EUC-KR - GB18030 +- IBM855 +- IBM866 - KOI8-R +- KOI8-U +- macintosh - Shift_JIS - UTF-8 - UTF-16BE @@ -115,6 +128,7 @@ Note on [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncod - windows-1256 - windows-1257 - windows-1258 +- x-mac-cyrillic Currently only these encodings are supported. @@ -129,11 +143,4 @@ Yes. Type definitions are included. ### TODO -- [ ] KOI8-U for Ukrainian -- [ ] IBM866 for DOS Cyrillic -- [ ] macintosh and x-mac-cyrillic - [ ] CP949 / UHC support within the Korean recognizer -- [ ] ISO-8859-3, ISO-8859-4 -- [ ] ISO-8859-10, ISO-8859-13 -- [ ] ISO-8859-14, ISO-8859-15, ISO-8859-16 -- [ ] DOS CP850, CP852, CP855 diff --git a/src/encoding/sbcs.test.ts b/src/encoding/sbcs.test.ts index 6ce296a..9614e2f 100644 --- a/src/encoding/sbcs.test.ts +++ b/src/encoding/sbcs.test.ts @@ -1,10 +1,11 @@ import * as chardet from '..'; import fs from 'fs'; import path from 'path'; -import { describe, expect, it, test } from 'vitest'; +import { describe, expect, it } from 'vitest'; describe('Singlebyte Character Sets', () => { const base = path.join(__dirname, '/../test/data/encodings'); + const corpus = path.join(__dirname, '../../corpus/generated'); const detect = (filename: string) => { return chardet.detectFileSync(path.join(base, filename)); @@ -14,6 +15,12 @@ describe('Singlebyte Character Sets', () => { return chardet.analyse(fs.readFileSync(path.join(base, filename)))[0]; }; + const corpusFixture = ( + encoding: string, + language: string, + filename = 'validation/river-trip.bin', + ) => path.join(corpus, encoding, language, filename); + it('should return ISO-8859-1 (English)', () => { expect(detect('iso88591_en')).toBe('ISO-8859-1'); }); @@ -30,8 +37,20 @@ describe('Singlebyte Character Sets', () => { expect(detect('iso88592_cs')).toBe('ISO-8859-2'); }); - test.todo('should return ISO-8859-3'); - test.todo('should return ISO-8859-4'); + it.each([ + ['ISO-8859-3', 'mt'], + ['ISO-8859-4', 'lv'], + ['ISO-8859-10', 'is'], + ['ISO-8859-14', 'cy'], + ['ISO-8859-16', 'ro'], + ])('should return %s (%s)', (encoding, language) => { + const fixture = corpusFixture(encoding, language); + expect(chardet.detectFileSync(fixture)).toBe(encoding); + expect(chardet.analyse(fs.readFileSync(fixture))[0]).toMatchObject({ + name: encoding, + lang: language, + }); + }); it('should return ISO-8859-5 (Russian)', () => { expect(detect('iso88595_ru')).toBe('ISO-8859-5'); @@ -53,18 +72,27 @@ describe('Singlebyte Character Sets', () => { expect(detect('iso88599_tr')).toBe('ISO-8859-9'); }); - test.todo('should return ISO-8859-10'); - test.todo('should return ISO-8859-11'); - // iso-8859-12 is abandoned - test.todo('should return ISO-8859-13'); - test.todo('should return ISO-8859-14'); - test.todo('should return ISO-8859-15'); - test.todo('should return ISO-8859-16'); - - it('should return windows-874 for Thai text', () => { + it('should return windows-874 for ISO-8859-11-compatible Thai text', () => { expect(detect('windows_874')).toBe('windows-874'); }); + // iso-8859-12 is abandoned + it.each([ + ['ISO-8859-13', 'lt'], + ['ISO-8859-15', 'fr'], + ])( + 'should return a %s candidate for byte-compatible text', + (encoding, language) => { + const fixture = corpusFixture(encoding, language); + expect(chardet.analyse(fs.readFileSync(fixture))).toContainEqual( + expect.objectContaining({ + name: encoding, + lang: language, + }), + ); + }, + ); + it('should return windows-1250 (Czech)', () => { expect(detect('windows_1250')).toBe('windows-1250'); }); From 2dc9fb248eeccf8b85ece48bbac190ae3f656055 Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Mon, 20 Jul 2026 14:14:47 +1000 Subject: [PATCH 5/6] Consolidate scoring utils for SBCS (#134) * Consolidate scoring utils for SBCS * Development snapshot --- corpus/README.md | 20 +- corpus/model-evaluation.json | 21554 +++++++--------------------- package.json | 6 +- scripts/model-utils.mjs | 151 +- src/encoding/sbcs-scoring.test.ts | 78 + src/encoding/sbcs-scoring.ts | 304 + src/encoding/sbcs.ts | 266 +- 7 files changed, 5251 insertions(+), 17128 deletions(-) create mode 100644 src/encoding/sbcs-scoring.test.ts create mode 100644 src/encoding/sbcs-scoring.ts diff --git a/corpus/README.md b/corpus/README.md index b1cb4fb..48dce88 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -36,12 +36,14 @@ The model compiler derives each single-byte encoding's case-folding byte map through iconv, applies the detector's whitespace normalization, and emits 64 sorted detector-ready trigrams into `src/encoding/models/generated.ts`. It also learns Laplace-smoothed high-byte distributions from the training split. The -evaluator uses byte likelihood to rank candidates whose trigram hit rates are -statistically competitive at a 95% margin. `models:evaluate` reports every -held-out test document, while `models:verify` checks reproducibility and fails if -any encoding or language result is incorrect. A different encoding label is -accepted only when both encodings decode the actual held-out bytes identically; -the report distinguishes these byte-equivalent results from exact matches. -CP949 is excluded because its -multibyte recogniser requires character frequency data rather than byte -trigrams. +compiler evaluator and library runtime share the normalization, trigram, +byte-likelihood, confidence, candidate-prefiltering, and tie-breaking +implementation in `src/encoding/sbcs-scoring.ts`. Candidates whose trigram hit +rates are statistically competitive at a 95% margin are ranked by byte +likelihood and deterministic encoding-family preferences. `models:evaluate` +reports every held-out test document, while `models:verify` checks +reproducibility and fails if any encoding or language result is incorrect. A +different encoding label is accepted only when both encodings decode the actual +held-out bytes identically; the report distinguishes these byte-equivalent +results from exact matches. CP949 is excluded because its multibyte recogniser +requires character frequency data rather than byte trigrams. diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 013725c..3021440 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -1,11 +1,11 @@ { "summary": { "tests": 66, - "encodingExact": 44, - "encodingEquivalent": 22, + "encodingExact": 46, + "encodingEquivalent": 20, "encodingCorrect": 66, "languageCorrect": 66, - "topScoreTies": 34 + "topScoreTies": 37 }, "confusion": { "CP850": { @@ -21,8 +21,7 @@ "IBM866": 1 }, "ISO-8859-1": { - "ISO-8859-1": 8, - "ISO-8859-15": 2 + "ISO-8859-1": 10 }, "ISO-8859-10": { "ISO-8859-10": 1 @@ -80,8 +79,7 @@ "windows-1251": 1 }, "windows-1252": { - "ISO-8859-1": 8, - "ISO-8859-15": 2 + "ISO-8859-1": 10 }, "windows-1253": { "ISO-8859-7": 1 @@ -116,8 +114,8 @@ }, "predicted": { "encoding": "CP850", - "confidence": 73, "language": "de", + "confidence": 73, "byteLogLikelihood": -3.169712077519, "hits": 53, "total": 216, @@ -131,35 +129,35 @@ "candidates": [ { "encoding": "CP850", - "confidence": 73, "language": "de", + "confidence": 73, "byteLogLikelihood": -3.169712077519, "hits": 53, "total": 216, "hitRate": 0.24537037037037038 }, { - "encoding": "ISO-8859-1", - "confidence": 72, + "encoding": "windows-1252", "language": "de", + "confidence": 72, "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, "hitRate": 0.24186046511627907 }, { - "encoding": "windows-1252", - "confidence": 72, + "encoding": "ISO-8859-15", "language": "de", + "confidence": 72, "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, "hitRate": 0.24186046511627907 }, { - "encoding": "ISO-8859-15", - "confidence": 72, + "encoding": "ISO-8859-1", "language": "de", + "confidence": 72, "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -167,8 +165,8 @@ }, { "encoding": "macintosh", - "confidence": 70, "language": "de", + "confidence": 70, "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -176,8 +174,8 @@ }, { "encoding": "ISO-8859-14", - "confidence": 22, "language": "cy", + "confidence": 22, "byteLogLikelihood": -4.912654885736, "hits": 16, "total": 215, @@ -185,8 +183,8 @@ }, { "encoding": "ISO-8859-2", - "confidence": 13, "language": "ro", + "confidence": 13, "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 215, @@ -194,8 +192,8 @@ }, { "encoding": "ISO-8859-9", - "confidence": 13, "language": "tr", + "confidence": 13, "byteLogLikelihood": -5.075173815234, "hits": 10, "total": 215, @@ -203,80 +201,26 @@ }, { "encoding": "windows-1250", - "confidence": 13, "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 215, - "hitRate": 0.046511627906976744 - }, - { - "encoding": "windows-1254", "confidence": 13, - "language": "tr", - "byteLogLikelihood": -5.075173815234, + "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 215, "hitRate": 0.046511627906976744 }, { "encoding": "windows-1257", - "confidence": 11, "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 215, - "hitRate": 0.037209302325581395 - }, - { - "encoding": "ISO-8859-4", "confidence": 11, - "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 8, "total": 215, "hitRate": 0.037209302325581395 }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 215, - "hitRate": 0.037209302325581395 - }, - { - "encoding": "windows-1258", - "confidence": 6, - "language": "vi", - "byteLogLikelihood": -5.172849867117, - "hits": 5, - "total": 215, - "hitRate": 0.023255813953488372 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 215, - "hitRate": 0.018604651162790697 - }, - { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, { "encoding": "ISO-8859-10", - "confidence": 4, "language": "is", + "confidence": 4, "byteLogLikelihood": -4.97295762689, "hits": 3, "total": 215, @@ -284,138 +228,12 @@ }, { "encoding": "ISO-8859-13", - "confidence": 2, "language": "lt", + "confidence": 2, "byteLogLikelihood": -5.00843944931, "hits": 2, "total": 215, "hitRate": 0.009302325581395349 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.099525732258, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.729917997667, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.542674768484, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.504753627871, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.219881518497, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.542674768484, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.504753627871, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.511862527725, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.737138560752, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.695189876932, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.428281637736, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.621314697937, - "hits": 0, - "total": 216, - "hitRate": 0 } ] }, @@ -426,8 +244,8 @@ }, "predicted": { "encoding": "CP850", - "confidence": 61, "language": "es", + "confidence": 61, "byteLogLikelihood": -3.834657935767, "hits": 41, "total": 200, @@ -441,8 +259,8 @@ "candidates": [ { "encoding": "CP850", - "confidence": 61, "language": "es", + "confidence": 61, "byteLogLikelihood": -3.834657935767, "hits": 41, "total": 200, @@ -450,8 +268,8 @@ }, { "encoding": "ISO-8859-1", - "confidence": 58, "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -459,8 +277,8 @@ }, { "encoding": "windows-1252", - "confidence": 58, "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -468,8 +286,8 @@ }, { "encoding": "macintosh", - "confidence": 58, "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -477,8 +295,8 @@ }, { "encoding": "ISO-8859-15", - "confidence": 58, "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -486,215 +304,44 @@ }, { "encoding": "windows-1257", - "confidence": 25, "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 17, - "total": 199, - "hitRate": 0.08542713567839195 - }, - { - "encoding": "ISO-8859-4", "confidence": 25, - "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 17, "total": 199, "hitRate": 0.08542713567839195 }, - { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 13, - "total": 199, - "hitRate": 0.06532663316582915 - }, { "encoding": "ISO-8859-14", - "confidence": 16, "language": "cy", + "confidence": 16, "byteLogLikelihood": -4.912654885736, "hits": 11, "total": 199, "hitRate": 0.05527638190954774 }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, { "encoding": "ISO-8859-16", - "confidence": 15, "language": "ro", + "confidence": 15, "byteLogLikelihood": -4.952116173006, "hits": 10, "total": 199, "hitRate": 0.05025125628140704 }, { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -4.61627103998, + "hits": 0, "total": 199, - "hitRate": 0.02512562814070352 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 199, - "hitRate": 0.02512562814070352 - }, - { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 5, - "total": 199, - "hitRate": 0.02512562814070352 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 199, - "hitRate": 0.01507537688442211 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.61627103998, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.351652470383, - "hits": 0, - "total": 199, - "hitRate": 0 + "hitRate": 0 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", + "confidence": 0, "byteLogLikelihood": -3.374008967132, "hits": 0, "total": 200, @@ -702,26 +349,17 @@ }, { "encoding": "IBM855", - "confidence": 0, "language": "sr", + "confidence": 0, "byteLogLikelihood": -4.64715417232, "hits": 0, "total": 200, "hitRate": 0 }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 199, - "hitRate": 0 - }, { "encoding": "ISO-8859-3", - "confidence": 0, "language": "mt", + "confidence": 0, "byteLogLikelihood": -4.759371883621, "hits": 0, "total": 199, @@ -736,8 +374,8 @@ }, "predicted": { "encoding": "CP850", - "confidence": 77, "language": "fr", + "confidence": 77, "byteLogLikelihood": -2.06619631493, "hits": 59, "total": 228, @@ -751,35 +389,35 @@ "candidates": [ { "encoding": "CP850", - "confidence": 77, "language": "fr", + "confidence": 77, "byteLogLikelihood": -2.06619631493, "hits": 59, "total": 228, "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-1", - "confidence": 75, + "encoding": "windows-1252", "language": "fr", + "confidence": 75, "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, "hitRate": 0.25 }, { - "encoding": "windows-1252", - "confidence": 75, + "encoding": "ISO-8859-15", "language": "fr", + "confidence": 75, "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, "hitRate": 0.25 }, { - "encoding": "ISO-8859-15", - "confidence": 75, + "encoding": "ISO-8859-1", "language": "fr", + "confidence": 75, "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -787,8 +425,8 @@ }, { "encoding": "macintosh", - "confidence": 71, "language": "fr", + "confidence": 71, "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -796,8 +434,8 @@ }, { "encoding": "ISO-8859-2", - "confidence": 12, "language": "ro", + "confidence": 12, "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, @@ -805,62 +443,26 @@ }, { "encoding": "windows-1250", - "confidence": 12, "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 - }, - { - "encoding": "ISO-8859-10", - "confidence": 12, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 - }, - { - "encoding": "ISO-8859-16", "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.220355825078, + "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, "hitRate": 0.04017857142857143 }, { "encoding": "windows-1257", - "confidence": 9, "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 224, - "hitRate": 0.03125 - }, - { - "encoding": "ISO-8859-4", "confidence": 9, - "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 7, "total": 224, "hitRate": 0.03125 }, - { - "encoding": "ISO-8859-3", - "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 224, - "hitRate": 0.026785714285714284 - }, { "encoding": "CP852", - "confidence": 7, "language": "pl", + "confidence": 7, "byteLogLikelihood": -5.087596335232, "hits": 6, "total": 228, @@ -868,8 +470,8 @@ }, { "encoding": "ISO-8859-9", - "confidence": 6, "language": "tr", + "confidence": 6, "byteLogLikelihood": -5.075173815234, "hits": 5, "total": 224, @@ -877,219 +479,66 @@ }, { "encoding": "windows-1254", - "confidence": 6, "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 224, - "hitRate": 0.022321428571428572 - }, - { - "encoding": "windows-1258", "confidence": 6, - "language": "vi", - "byteLogLikelihood": -5.497168225293, + "byteLogLikelihood": -5.075173815234, "hits": 5, "total": 224, "hitRate": 0.022321428571428572 }, - { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 224, - "hitRate": 0.017857142857142856 - }, { "encoding": "ISO-8859-14", - "confidence": 4, "language": "cy", + "confidence": 4, "byteLogLikelihood": -4.912654885736, "hits": 3, "total": 224, "hitRate": 0.013392857142857142 + } + ] + }, + { + "expected": { + "encoding": "CP852", + "language": "pl" + }, + "predicted": { + "encoding": "CP852", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "CP852", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 }, { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -6.061456918928, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.520621127559, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, - "hits": 0, - "total": 228, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "CP852", - "language": "pl" - }, - "predicted": { - "encoding": "CP852", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -3.52910527172, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "CP852", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -3.52910527172, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - { - "encoding": "windows-1250", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - { - "encoding": "ISO-8859-2", - "confidence": 37, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 23, - "total": 184, - "hitRate": 0.125 + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 37, + "byteLogLikelihood": -5.087596335232, + "hits": 23, + "total": 184, + "hitRate": 0.125 }, { "encoding": "ISO-8859-1", - "confidence": 8, "language": "it", + "confidence": 8, "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -1097,8 +546,8 @@ }, { "encoding": "windows-1252", - "confidence": 8, "language": "it", + "confidence": 8, "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -1106,8 +555,8 @@ }, { "encoding": "windows-1257", - "confidence": 6, "language": "et", + "confidence": 6, "byteLogLikelihood": -4.536305162191, "hits": 4, "total": 183, @@ -1115,98 +564,44 @@ }, { "encoding": "macintosh", - "confidence": 6, "language": "fr", + "confidence": 6, "byteLogLikelihood": -5.010635294096, "hits": 4, "total": 182, "hitRate": 0.02197802197802198 }, - { - "encoding": "ISO-8859-13", - "confidence": 6, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, { "encoding": "ISO-8859-15", - "confidence": 6, "language": "es", + "confidence": 6, "byteLogLikelihood": -4.948759890378, "hits": 4, "total": 183, "hitRate": 0.02185792349726776 }, - { - "encoding": "windows-1258", - "confidence": 4, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 3, - "total": 183, - "hitRate": 0.01639344262295082 - }, - { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 183, - "hitRate": 0.01639344262295082 - }, - { - "encoding": "ISO-8859-4", - "confidence": 4, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 184, - "hitRate": 0.016304347826086956 - }, { "encoding": "CP850", - "confidence": 4, "language": "fr", + "confidence": 4, "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 184, "hitRate": 0.016304347826086956 }, - { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 184, - "hitRate": 0.010869565217391304 - }, { "encoding": "ISO-8859-14", - "confidence": 3, "language": "cy", + "confidence": 3, "byteLogLikelihood": -4.912654885736, "hits": 2, "total": 184, "hitRate": 0.010869565217391304 }, - { - "encoding": "ISO-8859-16", - "confidence": 3, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 2, - "total": 183, - "hitRate": 0.01092896174863388 - }, { "encoding": "ISO-8859-9", - "confidence": 1, "language": "tr", + "confidence": 1, "byteLogLikelihood": -5.075173815234, "hits": 1, "total": 183, @@ -1214,137 +609,150 @@ }, { "encoding": "windows-1254", - "confidence": 1, "language": "tr", + "confidence": 1, "byteLogLikelihood": -5.075173815234, "hits": 1, "total": 183, "hitRate": 0.00546448087431694 }, { - "encoding": "ISO-8859-5", - "confidence": 0, + "encoding": "IBM866", "language": "ru", - "byteLogLikelihood": -6.692083742507, + "confidence": 0, + "byteLogLikelihood": -5.085936520567, "hits": 0, "total": 184, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "IBM855", + "language": "ru" + }, + "predicted": { + "encoding": "IBM855", + "language": "ru", + "confidence": 43, + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "IBM855", + "language": "ru", + "confidence": 43, + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 }, { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.311255977022, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.762644549521, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.302642222102, - "hits": 0, - "total": 183, - "hitRate": 0 + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.908064507057, + "hits": 1, + "total": 174, + "hitRate": 0.005747126436781609 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-1", + "language": "pt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.005185419538, + "byteLogLikelihood": -4.789358524023, "hits": 0, - "total": 184, + "total": 139, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.762644549521, + "byteLogLikelihood": -4.901389021393, "hits": 0, - "total": 183, + "total": 165, "hitRate": 0 }, { - "encoding": "windows-1255", + "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.302642222102, + "byteLogLikelihood": -4.901389021393, "hits": 0, - "total": 183, + "total": 141, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "windows-1252", + "language": "pt", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.54369343373, + "byteLogLikelihood": -4.789358524023, "hits": 0, - "total": 183, + "total": 141, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "windows-1257", + "language": "lt", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.549771342707, + "byteLogLikelihood": -4.913061957297, "hits": 0, - "total": 184, + "total": 149, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "macintosh", + "language": "es", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.538051035716, + "byteLogLikelihood": -4.934208734237, "hits": 0, - "total": 183, + "total": 73, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-3", + "language": "mt", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.496246341796, + "byteLogLikelihood": -5.011017819902, "hits": 0, - "total": 183, + "total": 123, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.085936520567, + "byteLogLikelihood": -4.913061957297, "hits": 0, - "total": 184, + "total": 149, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.780131236409, + "byteLogLikelihood": -4.805721913065, "hits": 0, - "total": 184, + "total": 160, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.963380725619, + "byteLogLikelihood": -4.829101213886, "hits": 0, - "total": 183, + "total": 167, "hitRate": 0 } ] @@ -1352,16 +760,16 @@ { "expected": { "encoding": "IBM855", - "language": "ru" + "language": "sr" }, "predicted": { "encoding": "IBM855", - "confidence": 43, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 26, - "total": 179, - "hitRate": 0.1452513966480447 + "language": "sr", + "confidence": 28, + "byteLogLikelihood": -3.307897947426, + "hits": 16, + "total": 166, + "hitRate": 0.0963855421686747 }, "encodingExact": true, "encodingEquivalent": false, @@ -1371,910 +779,890 @@ "candidates": [ { "encoding": "IBM855", - "confidence": 43, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 26, - "total": 179, - "hitRate": 0.1452513966480447 - }, - { - "encoding": "KOI8-U", - "confidence": 12, - "language": "uk", - "byteLogLikelihood": -5.728613539801, - "hits": 7, - "total": 171, - "hitRate": 0.04093567251461988 + "language": "sr", + "confidence": 28, + "byteLogLikelihood": -3.307897947426, + "hits": 16, + "total": 166, + "hitRate": 0.0963855421686747 }, { "encoding": "IBM866", - "confidence": 11, - "language": "ru", - "byteLogLikelihood": -5.08851401058, - "hits": 5, - "total": 130, - "hitRate": 0.038461538461538464 - }, - { - "encoding": "windows-874", - "confidence": 10, - "language": "th", - "byteLogLikelihood": -5.027512765832, - "hits": 5, - "total": 146, - "hitRate": 0.03424657534246575 - }, - { - "encoding": "KOI8-R", - "confidence": 10, "language": "ru", - "byteLogLikelihood": -6.090200292669, - "hits": 5, - "total": 140, - "hitRate": 0.03571428571428571 - }, - { - "encoding": "ISO-8859-7", - "confidence": 6, - "language": "el", - "byteLogLikelihood": -5.19500629466, - "hits": 3, - "total": 138, - "hitRate": 0.021739130434782608 - }, - { - "encoding": "windows-1253", - "confidence": 6, - "language": "el", - "byteLogLikelihood": -5.19500629466, - "hits": 3, - "total": 139, - "hitRate": 0.02158273381294964 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 4, - "language": "uk", - "byteLogLikelihood": -5.827543836069, - "hits": 2, + "confidence": 26, + "byteLogLikelihood": -4.750368103744, + "hits": 12, "total": 137, - "hitRate": 0.014598540145985401 - }, - { - "encoding": "ISO-8859-5", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -5.217274879676, - "hits": 1, - "total": 175, - "hitRate": 0.005714285714285714 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.06395526639, - "hits": 1, - "total": 160, - "hitRate": 0.00625 - }, - { - "encoding": "ISO-8859-14", - "confidence": 1, - "language": "cy", - "byteLogLikelihood": -4.908064507057, - "hits": 1, - "total": 174, - "hitRate": 0.005747126436781609 + "hitRate": 0.08759124087591241 }, { "encoding": "ISO-8859-1", - "confidence": 0, "language": "pt", - "byteLogLikelihood": -4.789358524023, + "confidence": 0, + "byteLogLikelihood": -4.764166439133, "hits": 0, - "total": 139, + "total": 104, "hitRate": 0 }, { "encoding": "ISO-8859-2", - "confidence": 0, "language": "hu", - "byteLogLikelihood": -4.901389021393, + "confidence": 0, + "byteLogLikelihood": -4.902465386166, "hits": 0, - "total": 165, + "total": 128, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.523443710193, + "byteLogLikelihood": -4.902465386166, "hits": 0, "total": 110, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "windows-1252", + "language": "pt", "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.21454182173, + "byteLogLikelihood": -4.764166439133, "hits": 0, - "total": 84, + "total": 110, "hitRate": 0 }, { - "encoding": "ISO-8859-9", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.033184680772, + "byteLogLikelihood": -4.902859191011, "hits": 0, - "total": 139, + "total": 136, "hitRate": 0 }, { - "encoding": "windows-1250", + "encoding": "macintosh", + "language": "es", "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.901389021393, + "byteLogLikelihood": -4.933799879287, "hits": 0, - "total": 141, + "total": 73, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.690578567068, + "byteLogLikelihood": -4.940581981725, "hits": 0, - "total": 156, + "total": 132, "hitRate": 0 }, { - "encoding": "windows-1252", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.789358524023, + "byteLogLikelihood": -4.907668215372, "hits": 0, - "total": 141, + "total": 157, "hitRate": 0 }, { - "encoding": "windows-1254", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.033184680772, + "byteLogLikelihood": -4.832419313648, "hits": 0, - "total": 141, + "total": 138, "hitRate": 0 }, { - "encoding": "windows-1255", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.21454182173, + "byteLogLikelihood": -4.704589417548, "hits": 0, - "total": 102, + "total": 150, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "IBM866", + "language": "ru" + }, + "predicted": { + "encoding": "IBM866", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "IBM866", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 }, { - "encoding": "windows-1256", + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.83002806951, + "hits": 1, + "total": 169, + "hitRate": 0.005917159763313609 + }, + { + "encoding": "ISO-8859-1", + "language": "it", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.354655838496, + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 135, + "total": 81, "hitRate": 0 }, { - "encoding": "windows-1257", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.913061957297, + "byteLogLikelihood": -4.941337938888, "hits": 0, - "total": 149, + "total": 145, "hitRate": 0 }, { - "encoding": "windows-1258", + "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.237936257135, + "byteLogLikelihood": -4.941337938888, "hits": 0, - "total": 134, + "total": 117, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "windows-1252", + "language": "it", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.934208734237, + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 73, + "total": 82, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1257", + "language": "lv", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.011017819902, + "byteLogLikelihood": -4.928541579896, "hits": 0, - "total": 123, + "total": 117, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "macintosh", + "language": "es", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.171130116678, + "byteLogLikelihood": -4.944169511699, "hits": 0, - "total": 169, + "total": 67, "hitRate": 0 }, { "encoding": "ISO-8859-13", - "confidence": 0, "language": "lt", - "byteLogLikelihood": -4.913061957297, + "confidence": 0, + "byteLogLikelihood": -4.939479070188, "hits": 0, - "total": 149, + "total": 116, "hitRate": 0 }, { "encoding": "ISO-8859-15", - "confidence": 0, "language": "es", - "byteLogLikelihood": -4.805721913065, - "hits": 0, - "total": 160, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.025462977445, + "byteLogLikelihood": -4.848242748706, "hits": 0, - "total": 162, + "total": 111, "hitRate": 0 }, { "encoding": "CP850", - "confidence": 0, "language": "es", - "byteLogLikelihood": -4.829101213886, + "confidence": 0, + "byteLogLikelihood": -4.677183836449, "hits": 0, - "total": 167, + "total": 130, "hitRate": 0 }, { "encoding": "CP852", - "confidence": 0, "language": "pl", - "byteLogLikelihood": -5.06627927679, + "confidence": 0, + "byteLogLikelihood": -4.699618505771, "hits": 0, - "total": 178, + "total": 166, "hitRate": 0 } ] }, { "expected": { - "encoding": "IBM855", - "language": "sr" + "encoding": "ISO-8859-1", + "language": "da" }, "predicted": { - "encoding": "IBM855", - "confidence": 28, - "language": "sr", - "byteLogLikelihood": -3.307897947426, - "hits": 16, - "total": 166, - "hitRate": 0.0963855421686747 + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "IBM855", - "confidence": 28, - "language": "sr", - "byteLogLikelihood": -3.307897947426, - "hits": 16, - "total": 166, - "hitRate": 0.0963855421686747 - }, - { - "encoding": "IBM866", - "confidence": 26, - "language": "ru", - "byteLogLikelihood": -4.750368103744, - "hits": 12, - "total": 137, - "hitRate": 0.08759124087591241 + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "windows-874", - "confidence": 35, - "language": "th", - "byteLogLikelihood": -5.092901405888, - "hits": 15, - "total": 128, - "hitRate": 0.1171875 + "encoding": "windows-1252", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "KOI8-R", - "confidence": 11, - "language": "ru", - "byteLogLikelihood": -6.207100144751, + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, "hits": 4, - "total": 104, - "hitRate": 0.038461538461538464 + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "KOI8-U", - "confidence": 9, - "language": "uk", - "byteLogLikelihood": -5.956096731591, + "encoding": "windows-1250", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, "hits": 4, - "total": 130, - "hitRate": 0.03076923076923077 + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "x-mac-cyrillic", - "confidence": 8, - "language": "uk", - "byteLogLikelihood": -5.930887805719, - "hits": 3, - "total": 112, - "hitRate": 0.026785714285714284 + "encoding": "windows-1257", + "language": "et", + "confidence": 10, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "ISO-8859-8", - "confidence": 4, - "language": "he", - "byteLogLikelihood": -5.427107080966, + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 2, + "byteLogLikelihood": -4.658005606594, "hits": 1, - "total": 64, - "hitRate": 0.015625 + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "windows-1255", - "confidence": 3, - "language": "he", - "byteLogLikelihood": -5.427107080966, + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 2, + "byteLogLikelihood": -4.145958228046, "hits": 1, - "total": 80, - "hitRate": 0.0125 + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "ISO-8859-1", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.764166439133, + "byteLogLikelihood": -4.689401143463, "hits": 0, - "total": 104, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-2", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.902465386166, + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 128, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-5", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.640443286036, + "confidence": 0, + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 157, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.592489718341, + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 84, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.3747848425, + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 103, + "total": 120, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { "encoding": "ISO-8859-9", - "confidence": 0, "language": "tr", - "byteLogLikelihood": -5.028189084099, - "hits": 0, - "total": 104, - "hitRate": 0 + "confidence": 12, + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 }, { - "encoding": "windows-1250", + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.902465386166, + "byteLogLikelihood": -4.007313397456, "hits": 0, - "total": 110, + "total": 216, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.910594216272, + "byteLogLikelihood": -4.349754587225, "hits": 0, - "total": 144, + "total": 216, "hitRate": 0 }, { - "encoding": "windows-1252", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.764166439133, + "byteLogLikelihood": -4.279537403415, "hits": 0, - "total": 110, + "total": 216, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", - "byteLogLikelihood": -5.3747848425, + "confidence": 0, + "byteLogLikelihood": -4.349754587225, "hits": 0, - "total": 107, + "total": 216, "hitRate": 0 }, { - "encoding": "windows-1254", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.028189084099, + "byteLogLikelihood": -4.279537403415, "hits": 0, - "total": 104, + "total": 216, "hitRate": 0 }, { "encoding": "windows-1256", - "confidence": 0, "language": "ar", - "byteLogLikelihood": -5.564612100894, + "confidence": 0, + "byteLogLikelihood": -4.318835143108, "hits": 0, - "total": 108, + "total": 216, "hitRate": 0 }, { - "encoding": "windows-1257", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.902859191011, + "byteLogLikelihood": -3.629332579687, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "en" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.23021689626, - "hits": 0, - "total": 103, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 29, + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 }, { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.933799879287, - "hits": 0, - "total": 73, - "hitRate": 0 + "encoding": "windows-1252", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.97248951392, - "hits": 0, - "total": 99, - "hitRate": 0 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 0, - "total": 131, - "hitRate": 0 + "encoding": "windows-1250", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-10", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "is", - "byteLogLikelihood": -5.076797848077, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 145, + "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.940581981725, + "byteLogLikelihood": -2.405884662832, "hits": 0, "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.907668215372, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 157, + "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.832419313648, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 138, + "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.086009867584, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 142, + "total": 131, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.704589417548, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 150, + "total": 132, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.064438955197, + "byteLogLikelihood": -4.510859506517, "hits": 0, - "total": 165, + "total": 132, "hitRate": 0 } ] }, { "expected": { - "encoding": "IBM866", - "language": "ru" + "encoding": "ISO-8859-1", + "language": "es" }, "predicted": { - "encoding": "IBM866", - "confidence": 41, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 25, - "total": 179, - "hitRate": 0.13966480446927373 + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 3, "candidates": [ { - "encoding": "IBM866", - "confidence": 41, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 25, - "total": 179, - "hitRate": 0.13966480446927373 - }, - { - "encoding": "KOI8-R", - "confidence": 23, - "language": "ru", - "byteLogLikelihood": -6.620542862927, - "hits": 6, - "total": 77, - "hitRate": 0.07792207792207792 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 15, - "language": "ru", - "byteLogLikelihood": -5.792699163742, - "hits": 7, - "total": 133, - "hitRate": 0.05263157894736842 - }, - { - "encoding": "KOI8-U", - "confidence": 11, - "language": "uk", - "byteLogLikelihood": -6.434389971368, - "hits": 4, - "total": 103, - "hitRate": 0.038834951456310676 - }, - { - "encoding": "windows-874", - "confidence": 9, - "language": "th", - "byteLogLikelihood": -5.160775609519, - "hits": 5, - "total": 165, - "hitRate": 0.030303030303030304 + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { - "encoding": "IBM855", - "confidence": 8, - "language": "sr", - "byteLogLikelihood": -5.021646546946, - "hits": 5, - "total": 169, - "hitRate": 0.029585798816568046 + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { - "encoding": "windows-1251", - "confidence": 7, - "language": "ru", - "byteLogLikelihood": -5.801879921101, - "hits": 4, - "total": 153, - "hitRate": 0.026143790849673203 + "encoding": "windows-1252", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { - "encoding": "windows-1253", - "confidence": 6, - "language": "el", - "byteLogLikelihood": -5.523099191635, - "hits": 2, - "total": 89, - "hitRate": 0.02247191011235955 + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 }, { - "encoding": "ISO-8859-5", - "confidence": 3, - "language": "ru", - "byteLogLikelihood": -5.798412758856, - "hits": 2, - "total": 174, - "hitRate": 0.011494252873563218 + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 }, { "encoding": "ISO-8859-7", - "confidence": 3, "language": "el", - "byteLogLikelihood": -5.523099191635, - "hits": 1, - "total": 81, - "hitRate": 0.012345679012345678 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.103902669993, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 - }, - { - "encoding": "ISO-8859-14", - "confidence": 1, - "language": "cy", - "byteLogLikelihood": -4.83002806951, - "hits": 1, - "total": 169, - "hitRate": 0.005917159763313609 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "it", - "byteLogLikelihood": -4.841944488282, - "hits": 0, - "total": 81, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.941337938888, - "hits": 0, - "total": 145, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.82555458734, + "byteLogLikelihood": -3.106555532888, "hits": 0, - "total": 57, + "total": 200, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -5.371812787625, - "hits": 0, - "total": 75, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.062286993055, - "hits": 0, - "total": 81, - "hitRate": 0 - }, - { - "encoding": "windows-1250", "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.941337938888, + "byteLogLikelihood": -3.411159813207, "hits": 0, - "total": 117, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-1252", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "it", - "byteLogLikelihood": -4.841944488282, + "byteLogLikelihood": -3.336328204376, "hits": 0, - "total": 82, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-1254", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.062286993055, + "byteLogLikelihood": -3.106555532888, "hits": 0, - "total": 81, + "total": 200, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -5.371812787625, + "confidence": 0, + "byteLogLikelihood": -3.411159813207, "hits": 0, - "total": 75, + "total": 200, "hitRate": 0 }, { "encoding": "windows-1256", - "confidence": 0, "language": "ar", - "byteLogLikelihood": -5.857645252562, - "hits": 0, - "total": 82, - "hitRate": 0 - }, - { - "encoding": "windows-1257", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.928541579896, + "byteLogLikelihood": -3.300604963183, "hits": 0, - "total": 117, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-1258", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.078371422345, + "byteLogLikelihood": -3.336328204376, "hits": 0, - "total": 76, + "total": 200, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944169511699, - "hits": 0, - "total": 67, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.096764716542, - "hits": 0, - "total": 116, - "hitRate": 0 + "encoding": "windows-1252", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-4", + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 10, + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.979786266462, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 148, + "total": 228, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.939479070188, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 116, + "total": 228, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.848242748706, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 111, + "total": 228, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.987878406509, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 144, + "total": 228, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.677183836449, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 130, + "total": 224, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -4.699618505771, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 166, + "total": 228, "hitRate": 0 } ] @@ -2282,16 +1670,16 @@ { "expected": { "encoding": "ISO-8859-1", - "language": "da" + "language": "it" }, "predicted": { "encoding": "ISO-8859-1", - "confidence": 52, - "language": "da", - "byteLogLikelihood": -3.146558879633, - "hits": 21, - "total": 120, - "hitRate": 0.175 + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 }, "encodingExact": true, "encodingEquivalent": false, @@ -2301,290 +1689,240 @@ "candidates": [ { "encoding": "ISO-8859-1", - "confidence": 52, - "language": "da", - "byteLogLikelihood": -3.146558879633, - "hits": 21, - "total": 120, - "hitRate": 0.175 + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 }, { "encoding": "windows-1252", - "confidence": 52, - "language": "da", - "byteLogLikelihood": -3.146558879633, - "hits": 21, - "total": 120, - "hitRate": 0.175 - }, - { - "encoding": "macintosh", - "confidence": 30, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 12, - "total": 120, - "hitRate": 0.1 - }, - { - "encoding": "ISO-8859-15", - "confidence": 30, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 12, - "total": 120, - "hitRate": 0.1 + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 }, { - "encoding": "CP850", - "confidence": 30, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 12, - "total": 120, - "hitRate": 0.1 + "encoding": "windows-1258", + "language": "vi", + "confidence": 16, + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 }, { - "encoding": "ISO-8859-14", + "encoding": "windows-1257", + "language": "lv", "confidence": 15, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 6, - "total": 120, - "hitRate": 0.05 + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-4", + "language": "lv", "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 6, - "total": 120, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-2", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 }, { - "encoding": "ISO-8859-9", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 3, + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 }, { - "encoding": "windows-1250", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 }, { - "encoding": "windows-1254", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 }, { - "encoding": "windows-1257", - "confidence": 10, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "encoding": "windows-1251", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 }, { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 3, - "total": 120, - "hitRate": 0.025 + "encoding": "windows-874", + "language": "th", + "confidence": 0, + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 7, - "language": "is", - "byteLogLikelihood": -4.717529149813, - "hits": 3, - "total": 120, - "hitRate": 0.025 + "encoding": "IBM866", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 120, - "hitRate": 0.016666666666666666 + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 0, + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 }, { - "encoding": "ISO-8859-3", - "confidence": 2, - "language": "mt", - "byteLogLikelihood": -4.658005606594, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 63, + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 }, { - "encoding": "ISO-8859-4", - "confidence": 2, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 + "encoding": "windows-1252", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 }, { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -4.145958228046, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 }, { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.825649766807, - "hits": 0, - "total": 120, - "hitRate": 0 + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 }, { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.736335095675, - "hits": 0, - "total": 120, - "hitRate": 0 + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 11, + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -4.689401143463, + "confidence": 0, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 120, + "total": 151, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.231348206195, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-1251", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.288232392491, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 120, + "total": 151, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", - "byteLogLikelihood": -4.689401143463, + "confidence": 0, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 120, + "total": 151, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.231348206195, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-1256", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.861023356308, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 120, + "total": 151, "hitRate": 0 }, { "encoding": "windows-874", - "confidence": 0, "language": "th", - "byteLogLikelihood": -5.888170123788, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.518796947367, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.476992253447, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.825649766807, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "IBM855", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.685745819823, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 120, + "total": 150, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.288232392491, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 120, + "total": 151, "hitRate": 0 } ] @@ -2592,309 +1930,259 @@ { "expected": { "encoding": "ISO-8859-1", - "language": "de" + "language": "no" }, "predicted": { "encoding": "ISO-8859-1", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 3, + "tiedAtTop": 2, "candidates": [ { "encoding": "ISO-8859-1", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, { "encoding": "windows-1252", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "language": "no", + "confidence": 47, + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, { - "encoding": "ISO-8859-15", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 7, + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 }, { - "encoding": "macintosh", - "confidence": 72, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 52, - "total": 215, - "hitRate": 0.24186046511627907 - }, - { - "encoding": "CP850", - "confidence": 70, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 51, - "total": 216, - "hitRate": 0.2361111111111111 - }, - { - "encoding": "ISO-8859-14", - "confidence": 20, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 15, - "total": 216, - "hitRate": 0.06944444444444445 - }, - { - "encoding": "ISO-8859-2", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "windows-1250", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "ISO-8859-9", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 9, - "total": 216, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "windows-1254", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 9, - "total": 216, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 216, - "hitRate": 0.037037037037037035 - }, - { - "encoding": "windows-1257", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 - }, - { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 - }, - { - "encoding": "CP852", - "confidence": 6, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 216, - "hitRate": 0.023148148148148147 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 + "encoding": "windows-1257", + "language": "et", + "confidence": 7, + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 }, { "encoding": "ISO-8859-3", - "confidence": 5, "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 + "confidence": 2, + "byteLogLikelihood": -4.210065739287, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 }, { "encoding": "ISO-8859-13", - "confidence": 2, "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 216, - "hitRate": 0.009259259259259259 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.204006687077, + "confidence": 2, + "byteLogLikelihood": -4.319245023186, "hits": 1, - "total": 216, - "hitRate": 0.004629629629629629 + "total": 114, + "hitRate": 0.008771929824561403 }, { - "encoding": "ISO-8859-5", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.162408104115, + "byteLogLikelihood": -4.154384602589, "hits": 0, - "total": 216, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.007313397456, + "byteLogLikelihood": -2.647504477241, "hits": 0, - "total": 216, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.349754587225, + "byteLogLikelihood": -3.753215851617, "hits": 0, - "total": 216, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.279537403415, + "byteLogLikelihood": -4.154384602589, "hits": 0, - "total": 216, + "total": 114, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.542402054434, + "byteLogLikelihood": -2.647504477241, "hits": 0, - "total": 216, + "total": 114, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.349754587225, + "byteLogLikelihood": -3.753215851617, "hits": 0, - "total": 216, + "total": 114, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 }, { - "encoding": "windows-1255", + "encoding": "windows-1252", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1250", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.279537403415, + "byteLogLikelihood": -3.017269736716, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.318835143108, + "byteLogLikelihood": -3.50352863905, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.657121063601, + "byteLogLikelihood": -3.846220145127, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.28774788718, + "byteLogLikelihood": -3.017269736716, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.245943193261, + "byteLogLikelihood": -3.50352863905, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1256", + "language": "ar", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -3.503504267293, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { "encoding": "IBM855", - "confidence": 0, "language": "sr", - "byteLogLikelihood": -5.942998477092, + "confidence": 0, + "byteLogLikelihood": -3.774998078834, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", - "byteLogLikelihood": -3.629332579687, + "confidence": 0, + "byteLogLikelihood": -3.766179593299, "hits": 0, - "total": 216, + "total": 143, "hitRate": 0 } ] @@ -2902,3774 +2190,2914 @@ { "expected": { "encoding": "ISO-8859-1", - "language": "en" + "language": "sv" }, "predicted": { - "encoding": "ISO-8859-15", - "confidence": 29, - "language": "en", - "byteLogLikelihood": -2.06619631493, - "hits": 13, - "total": 132, - "hitRate": 0.09848484848484848 + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ - { - "encoding": "ISO-8859-15", - "confidence": 29, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 13, - "total": 132, - "hitRate": 0.09848484848484848 - }, { "encoding": "ISO-8859-1", - "confidence": 47, - "language": "en", - "byteLogLikelihood": -3.310543013394, - "hits": 21, - "total": 132, - "hitRate": 0.1590909090909091 + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, { "encoding": "windows-1252", - "confidence": 47, - "language": "en", - "byteLogLikelihood": -3.310543013394, - "hits": 21, - "total": 132, - "hitRate": 0.1590909090909091 - }, - { - "encoding": "macintosh", - "confidence": 27, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 12, - "total": 132, - "hitRate": 0.09090909090909091 + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, { - "encoding": "CP850", - "confidence": 27, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 12, - "total": 132, - "hitRate": 0.09090909090909091 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 17, + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 }, { "encoding": "windows-1257", - "confidence": 11, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-14", - "confidence": 11, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-2", - "confidence": 9, - "language": "cs", - "byteLogLikelihood": -3.54961738678, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "windows-1250", - "confidence": 9, - "language": "cs", - "byteLogLikelihood": -3.54961738678, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "windows-1258", - "confidence": 9, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "ISO-8859-16", - "confidence": 9, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "ISO-8859-4", - "confidence": 6, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 132, - "hitRate": 0.022727272727272728 - }, - { - "encoding": "ISO-8859-9", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "windows-1254", + "language": "lt", "confidence": 4, - "language": "tr", - "byteLogLikelihood": -5.075173815234, + "byteLogLikelihood": -5.123963979403, "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 + "total": 136, + "hitRate": 0.014705882352941176 }, { "encoding": "ISO-8859-3", - "confidence": 4, "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "CP852", "confidence": 4, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "byteLogLikelihood": -4.210065739287, "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 1, - "total": 132, - "hitRate": 0.007575757575757576 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 132, - "hitRate": 0 + "total": 136, + "hitRate": 0.014705882352941176 }, { "encoding": "ISO-8859-6", - "confidence": 0, "language": "ar", - "byteLogLikelihood": -6.113682179832, + "confidence": 0, + "byteLogLikelihood": -3.287368473007, "hits": 0, - "total": 132, + "total": 136, "hitRate": 0 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -2.654561516151, + "confidence": 0, + "byteLogLikelihood": -3.876181596107, "hits": 0, - "total": 132, + "total": 136, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.405884662832, + "confidence": 0, + "byteLogLikelihood": -3.066153183614, "hits": 0, - "total": 132, + "total": 136, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.612642200827, + "confidence": 0, + "byteLogLikelihood": -3.393743210995, "hits": 0, - "total": 132, + "total": 136, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", - "byteLogLikelihood": -2.654561516151, + "confidence": 0, + "byteLogLikelihood": -3.876181596107, "hits": 0, - "total": 132, + "total": 136, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-874", "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.57655026914, + "byteLogLikelihood": -3.066153183614, "hits": 0, - "total": 131, + "total": 136, "hitRate": 0 }, { - "encoding": "KOI8-R", - "confidence": 0, + "encoding": "x-mac-cyrillic", "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "IBM866", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, + "byteLogLikelihood": -3.393743210995, "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 0, - "total": 132, + "total": 136, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "es" + "encoding": "ISO-8859-10", + "language": "is" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 38, + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 3, + "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-1", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "windows-1252", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "macintosh", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 38, + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 }, { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 14, + "byteLogLikelihood": -4.633646206449, + "hits": 8, + "total": 171, + "hitRate": 0.04678362573099415 }, { "encoding": "windows-1257", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 16, - "total": 200, - "hitRate": 0.08 - }, - { - "encoding": "ISO-8859-4", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.009854802779, - "hits": 16, - "total": 200, - "hitRate": 0.08 + "language": "lt", + "confidence": 7, + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 }, { "encoding": "ISO-8859-13", - "confidence": 19, "language": "lt", - "byteLogLikelihood": -4.892914919217, - "hits": 13, - "total": 200, - "hitRate": 0.065 - }, - { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 200, - "hitRate": 0.055 + "confidence": 7, + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 }, { "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 + "language": "cs", + "confidence": 3, + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 }, { "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "windows-1258", - "confidence": 4, - "language": "vi", - "byteLogLikelihood": -4.848531508941, - "hits": 3, - "total": 200, - "hitRate": 0.015 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -3.846028832089, - "hits": 3, - "total": 200, - "hitRate": 0.015 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.977292083619, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.346153815501, - "hits": 0, - "total": 200, - "hitRate": 0 + "language": "cs", + "confidence": 3, + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -3.106555532888, + "confidence": 0, + "byteLogLikelihood": -3.963723644165, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.411159813207, + "confidence": 0, + "byteLogLikelihood": -4.249634019938, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -3.336328204376, + "confidence": 0, + "byteLogLikelihood": -3.84046735452, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", - "byteLogLikelihood": -3.106555532888, + "confidence": 0, + "byteLogLikelihood": -3.963723644165, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.411159813207, + "confidence": 0, + "byteLogLikelihood": -4.249634019938, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.300604963183, + "byteLogLikelihood": -3.733983711968, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-13", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 }, { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.412820202576, - "hits": 0, - "total": 200, - "hitRate": 0 + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 34, + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 }, { - "encoding": "KOI8-R", + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 22, + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 1, + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -4.472500251122, "hits": 0, - "total": 200, + "total": 173, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -4.444220754634, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { - "encoding": "IBM866", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -4.977292083619, + "confidence": 0, + "byteLogLikelihood": -3.531827362419, "hits": 0, - "total": 200, + "total": 173, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.203740544852, + "byteLogLikelihood": -4.472500251122, "hits": 0, - "total": 200, + "total": 173, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, + "byteLogLikelihood": -4.444220754634, "hits": 0, - "total": 200, + "total": 171, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "byteLogLikelihood": -3.441328843196, "hits": 0, - "total": 200, + "total": 173, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "fr" + "encoding": "ISO-8859-14", + "language": "cy" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 50, + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 3, + "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-1", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "windows-1252", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "ISO-8859-15", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "macintosh", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 50, + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 }, { - "encoding": "CP850", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "encoding": "windows-1258", + "language": "vi", + "confidence": 14, + "byteLogLikelihood": -4.34192292436, + "hits": 9, + "total": 188, + "hitRate": 0.047872340425531915 }, { "encoding": "ISO-8859-2", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 }, { "encoding": "windows-1250", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 6, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 189, + "hitRate": 0.021164021164021163 }, { "encoding": "ISO-8859-10", - "confidence": 10, "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 + "confidence": 4, + "byteLogLikelihood": -4.009500374258, + "hits": 3, + "total": 189, + "hitRate": 0.015873015873015872 }, { - "encoding": "ISO-8859-16", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 1, + "byteLogLikelihood": -3.647025046455, + "hits": 1, + "total": 189, + "hitRate": 0.005291005291005291 }, { - "encoding": "windows-1257", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-9", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "windows-1254", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 228, - "hitRate": 0.013157894736842105 - }, - { - "encoding": "ISO-8859-5", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, + "byteLogLikelihood": -4.254156765951, "hits": 0, - "total": 228, + "total": 189, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, + "byteLogLikelihood": -3.467895804539, "hits": 0, - "total": 228, + "total": 189, "hitRate": 0 }, { - "encoding": "ISO-8859-7", - "confidence": 0, + "encoding": "windows-1253", "language": "el", - "byteLogLikelihood": -2.654561516151, + "confidence": 0, + "byteLogLikelihood": -4.254156765951, "hits": 0, - "total": 228, + "total": 189, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, + "byteLogLikelihood": -3.255815902698, "hits": 0, - "total": 228, + "total": 189, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-16", + "language": "ro", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.612642200827, + "byteLogLikelihood": -3.903274585551, "hits": 0, - "total": 228, + "total": 189, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 228, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 228, - "hitRate": 0 + "encoding": "windows-1252", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "windows-1256", - "confidence": 0, + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-6", "language": "ar", - "byteLogLikelihood": -6.113682179832, + "confidence": 0, + "byteLogLikelihood": -4.007313397456, "hits": 0, - "total": 228, + "total": 216, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.57655026914, + "byteLogLikelihood": -4.349754587225, "hits": 0, - "total": 224, + "total": 216, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -4.279537403415, "hits": 0, - "total": 228, + "total": 216, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "byteLogLikelihood": -4.349754587225, "hits": 0, - "total": 228, + "total": 216, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, + "byteLogLikelihood": -4.279537403415, "hits": 0, - "total": 228, + "total": 216, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1256", + "language": "ar", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, + "byteLogLikelihood": -4.318835143108, "hits": 0, - "total": 228, + "total": 216, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", - "byteLogLikelihood": -4.252383775789, + "confidence": 0, + "byteLogLikelihood": -3.629332579687, "hits": 0, - "total": 228, + "total": 216, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "it" + "encoding": "ISO-8859-15", + "language": "es" }, "predicted": { "encoding": "ISO-8859-1", - "confidence": 58, - "language": "it", - "byteLogLikelihood": -3.533686564708, - "hits": 31, - "total": 159, - "hitRate": 0.1949685534591195 + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, - "encodingExact": true, - "encodingEquivalent": false, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 2, + "tiedAtTop": 3, "candidates": [ { "encoding": "ISO-8859-1", - "confidence": 58, - "language": "it", - "byteLogLikelihood": -3.533686564708, - "hits": 31, - "total": 159, - "hitRate": 0.1949685534591195 - }, - { - "encoding": "windows-1252", - "confidence": 58, - "language": "it", - "byteLogLikelihood": -3.533686564708, - "hits": 31, - "total": 159, - "hitRate": 0.1949685534591195 - }, - { - "encoding": "macintosh", - "confidence": 24, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 158, - "hitRate": 0.08227848101265822 + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { "encoding": "ISO-8859-15", - "confidence": 24, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 159, - "hitRate": 0.08176100628930817 - }, - { - "encoding": "CP850", - "confidence": 24, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 159, - "hitRate": 0.08176100628930817 - }, - { - "encoding": "windows-1258", - "confidence": 16, - "language": "vi", - "byteLogLikelihood": -3.012261575505, - "hits": 9, - "total": 159, - "hitRate": 0.05660377358490566 + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { - "encoding": "ISO-8859-16", - "confidence": 16, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 9, - "total": 159, - "hitRate": 0.05660377358490566 + "encoding": "windows-1252", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { "encoding": "ISO-8859-2", - "confidence": 15, "language": "ro", + "confidence": 15, "byteLogLikelihood": -5.176149732574, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 + "hits": 10, + "total": 200, + "hitRate": 0.05 }, { "encoding": "windows-1250", - "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "windows-1257", - "confidence": 15, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "ISO-8859-4", "confidence": 15, - "language": "lv", - "byteLogLikelihood": -1.974081026022, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "ISO-8859-14", - "confidence": 9, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 5, - "total": 159, - "hitRate": 0.031446540880503145 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 159, - "hitRate": 0.025157232704402517 - }, - { - "encoding": "ISO-8859-9", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 159, - "hitRate": 0.018867924528301886 - }, - { - "encoding": "windows-1254", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 159, - "hitRate": 0.018867924528301886 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 159, - "hitRate": 0.018867924528301886 - }, - { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 159, - "hitRate": 0.012578616352201259 - }, - { - "encoding": "ISO-8859-13", - "confidence": 3, - "language": "lt", - "byteLogLikelihood": -3.332204510175, - "hits": 2, - "total": 159, - "hitRate": 0.012578616352201259 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -2.954414124223, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 159, - "hitRate": 0 + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -6.120297418951, + "confidence": 0, + "byteLogLikelihood": -3.106555532888, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.629660094454, + "confidence": 0, + "byteLogLikelihood": -3.411159813207, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -2.533200659147, + "confidence": 0, + "byteLogLikelihood": -3.336328204376, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", - "byteLogLikelihood": -6.120297418951, + "confidence": 0, + "byteLogLikelihood": -3.106555532888, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.629660094454, + "confidence": 0, + "byteLogLikelihood": -3.411159813207, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 }, { "encoding": "windows-1256", - "confidence": 0, "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.496507561466, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -2.954414124223, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "IBM855", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.520621127559, + "byteLogLikelihood": -3.300604963183, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -2.533200659147, + "confidence": 0, + "byteLogLikelihood": -3.336328204376, "hits": 0, - "total": 159, + "total": 200, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "nl" + "encoding": "ISO-8859-15", + "language": "fr" }, "predicted": { "encoding": "ISO-8859-1", - "confidence": 83, - "language": "nl", - "byteLogLikelihood": -3.295836866004, - "hits": 42, - "total": 151, - "hitRate": 0.2781456953642384 + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, - "encodingExact": true, - "encodingEquivalent": false, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 2, + "tiedAtTop": 3, "candidates": [ { "encoding": "ISO-8859-1", - "confidence": 83, - "language": "nl", - "byteLogLikelihood": -3.295836866004, - "hits": 42, - "total": 151, - "hitRate": 0.2781456953642384 - }, - { - "encoding": "windows-1252", - "confidence": 83, - "language": "nl", - "byteLogLikelihood": -3.295836866004, - "hits": 42, - "total": 151, - "hitRate": 0.2781456953642384 - }, - { - "encoding": "macintosh", - "confidence": 63, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 32, - "total": 151, - "hitRate": 0.2119205298013245 + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { "encoding": "ISO-8859-15", - "confidence": 63, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 32, - "total": 151, - "hitRate": 0.2119205298013245 - }, - { - "encoding": "CP850", - "confidence": 63, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 32, - "total": 151, - "hitRate": 0.2119205298013245 + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-14", - "confidence": 33, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 17, - "total": 151, - "hitRate": 0.11258278145695365 + "encoding": "windows-1252", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { "encoding": "ISO-8859-2", - "confidence": 13, "language": "ro", + "confidence": 11, "byteLogLikelihood": -5.176149732574, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, - { - "encoding": "ISO-8859-9", - "confidence": 13, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 }, { "encoding": "windows-1250", - "confidence": 13, "language": "ro", + "confidence": 11, "byteLogLikelihood": -5.176149732574, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, - { - "encoding": "windows-1254", - "confidence": 13, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 }, { "encoding": "ISO-8859-10", - "confidence": 11, "language": "is", + "confidence": 10, "byteLogLikelihood": -4.510859506517, - "hits": 6, - "total": 151, - "hitRate": 0.039735099337748346 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 6, - "total": 151, - "hitRate": 0.039735099337748346 - }, - { - "encoding": "windows-1257", - "confidence": 9, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 5, - "total": 151, - "hitRate": 0.033112582781456956 - }, - { - "encoding": "windows-1258", - "confidence": 3, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 2, - "total": 151, - "hitRate": 0.013245033112582781 - }, - { - "encoding": "ISO-8859-13", - "confidence": 3, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 151, - "hitRate": 0.013245033112582781 - }, - { - "encoding": "ISO-8859-3", - "confidence": 1, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 1, - "total": 151, - "hitRate": 0.006622516556291391 - }, - { - "encoding": "ISO-8859-4", - "confidence": 1, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 151, - "hitRate": 0.006622516556291391 - }, - { - "encoding": "CP852", - "confidence": 1, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 1, - "total": 151, - "hitRate": 0.006622516556291391 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 151, - "hitRate": 0 + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 151, + "total": 228, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-1251", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.612642200827, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 151, + "total": 228, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 151, + "total": 228, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-1256", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 151, + "total": 228, "hitRate": 0 }, { "encoding": "windows-874", - "confidence": 0, "language": "th", - "byteLogLikelihood": -3.57655026914, - "hits": 0, - "total": 150, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 151, + "total": 224, "hitRate": 0 }, { - "encoding": "KOI8-U", - "confidence": 0, + "encoding": "x-mac-cyrillic", "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "IBM866", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, - "hits": 0, - "total": 151, + "total": 228, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "no" + "encoding": "ISO-8859-16", + "language": "ro" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 47, - "language": "no", - "byteLogLikelihood": -3.052495812716, - "hits": 18, - "total": 114, - "hitRate": 0.15789473684210525 + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 53, + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 2, + "tiedAtTop": 1, "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 47, - "language": "no", - "byteLogLikelihood": -3.052495812716, - "hits": 18, - "total": 114, - "hitRate": 0.15789473684210525 - }, - { - "encoding": "windows-1252", - "confidence": 47, - "language": "no", - "byteLogLikelihood": -3.052495812716, - "hits": 18, - "total": 114, - "hitRate": 0.15789473684210525 - }, - { - "encoding": "macintosh", - "confidence": 18, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 - }, - { - "encoding": "ISO-8859-14", - "confidence": 18, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 - }, - { - "encoding": "ISO-8859-15", - "confidence": 18, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 - }, - { - "encoding": "CP850", - "confidence": 18, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 - }, - { - "encoding": "ISO-8859-10", - "confidence": 13, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 5, - "total": 114, - "hitRate": 0.043859649122807015 - }, - { - "encoding": "windows-1258", - "confidence": 10, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 114, - "hitRate": 0.03508771929824561 - }, { "encoding": "ISO-8859-16", - "confidence": 10, "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 4, - "total": 114, - "hitRate": 0.03508771929824561 + "confidence": 53, + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 }, { "encoding": "ISO-8859-2", - "confidence": 7, - "language": "cs", - "byteLogLikelihood": -4.354336342997, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 + "language": "ro", + "confidence": 44, + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 }, { "encoding": "windows-1250", - "confidence": 7, - "language": "cs", - "byteLogLikelihood": -4.354336342997, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 + "language": "ro", + "confidence": 44, + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 }, { - "encoding": "windows-1257", - "confidence": 7, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 }, { - "encoding": "ISO-8859-3", - "confidence": 2, - "language": "mt", - "byteLogLikelihood": -4.210065739287, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 + "encoding": "windows-1252", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 }, { - "encoding": "ISO-8859-4", - "confidence": 2, + "encoding": "windows-1257", "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -4.319245023186, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 - }, - { - "encoding": "CP852", - "confidence": 2, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.652362971667, - "hits": 0, - "total": 114, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.641462690249, - "hits": 0, - "total": 114, - "hitRate": 0 + "confidence": 8, + "byteLogLikelihood": -4.456762061842, + "hits": 6, + "total": 214, + "hitRate": 0.028037383177570093 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -4.154384602589, + "confidence": 0, + "byteLogLikelihood": -4.499845100591, "hits": 0, - "total": 114, + "total": 214, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.647504477241, + "confidence": 0, + "byteLogLikelihood": -4.429071192897, "hits": 0, - "total": 114, + "total": 213, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -3.753215851617, - "hits": 0, - "total": 114, - "hitRate": 0 - }, - { - "encoding": "windows-1253", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.154384602589, + "byteLogLikelihood": -4.382241948417, "hits": 0, - "total": 114, + "total": 214, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.647504477241, + "confidence": 0, + "byteLogLikelihood": -4.429071192897, "hits": 0, - "total": 114, + "total": 213, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "IBM866", + "language": "ru", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.217802445218, + "byteLogLikelihood": -4.333669615722, "hits": 0, - "total": 114, + "total": 215, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "th", - "byteLogLikelihood": -6.061456918928, + "byteLogLikelihood": -4.382241948417, "hits": 0, - "total": 114, + "total": 214, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 }, { - "encoding": "KOI8-R", + "encoding": "windows-1250", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 7, + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -3.401446196881, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "byteLogLikelihood": -3.467509652199, "hits": 0, - "total": 114, + "total": 125, "hitRate": 0 }, { - "encoding": "IBM866", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.652362971667, + "confidence": 0, + "byteLogLikelihood": -4.249199684222, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.679407897139, + "byteLogLikelihood": -3.401446196881, "hits": 0, - "total": 114, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.753215851617, + "byteLogLikelihood": -4.226561011787, "hits": 0, - "total": 114, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 0, + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "pt" + "encoding": "ISO-8859-2", + "language": "hu" }, "predicted": { - "encoding": "ISO-8859-15", - "confidence": 33, - "language": "pt", - "byteLogLikelihood": -3.970754139021, - "hits": 16, - "total": 143, - "hitRate": 0.11188811188811189 + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-15", - "confidence": 33, - "language": "es", - "byteLogLikelihood": -3.970754139021, - "hits": 16, - "total": 143, - "hitRate": 0.11188811188811189 + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 }, { "encoding": "ISO-8859-1", - "confidence": 50, "language": "pt", - "byteLogLikelihood": -4.059947322987, - "hits": 24, - "total": 143, - "hitRate": 0.16783216783216784 + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 }, { "encoding": "windows-1252", - "confidence": 50, "language": "pt", - "byteLogLikelihood": -4.059947322987, - "hits": 24, - "total": 143, - "hitRate": 0.16783216783216784 + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 4, + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 }, { "encoding": "windows-1257", - "confidence": 29, - "language": "lt", - "byteLogLikelihood": -4.777390389123, - "hits": 14, - "total": 143, - "hitRate": 0.0979020979020979 + "language": "et", + "confidence": 2, + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 }, { - "encoding": "ISO-8859-13", - "confidence": 29, - "language": "lt", - "byteLogLikelihood": -4.777390389123, - "hits": 14, - "total": 143, - "hitRate": 0.0979020979020979 + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 2, + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 }, { - "encoding": "macintosh", - "confidence": 31, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 15, - "total": 143, - "hitRate": 0.1048951048951049 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 31, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 15, - "total": 143, - "hitRate": 0.1048951048951049 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 23, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 11, - "total": 143, - "hitRate": 0.07692307692307693 + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 }, { - "encoding": "ISO-8859-2", - "confidence": 10, - "language": "hu", - "byteLogLikelihood": -3.361609233757, - "hits": 5, - "total": 143, - "hitRate": 0.03496503496503497 + "encoding": "windows-1255", + "language": "he", + "confidence": 0, + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 0, + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "pl" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 }, { "encoding": "windows-1250", - "confidence": 10, - "language": "hu", - "byteLogLikelihood": -3.361609233757, + "language": "pl", + "confidence": 37, + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 183, + "hitRate": 0.12568306010928962 + }, + { + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, "hits": 5, - "total": 143, - "hitRate": 0.03496503496503497 + "total": 183, + "hitRate": 0.0273224043715847 }, { - "encoding": "ISO-8859-14", - "confidence": 10, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "encoding": "windows-1252", + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, "hits": 5, - "total": 143, - "hitRate": 0.03496503496503497 + "total": 183, + "hitRate": 0.0273224043715847 }, { - "encoding": "windows-1258", + "encoding": "windows-1257", + "language": "et", "confidence": 6, - "language": "vi", - "byteLogLikelihood": -4.524213150766, - "hits": 3, - "total": 143, - "hitRate": 0.02097902097902098 + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 }, { - "encoding": "ISO-8859-10", + "encoding": "macintosh", + "language": "es", "confidence": 6, - "language": "is", - "byteLogLikelihood": -4.06291963921, + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 6, + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "CP850", + "language": "fr", + "confidence": 6, + "byteLogLikelihood": -5.010635294096, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 4, + "byteLogLikelihood": -4.794788079951, "hits": 3, - "total": 143, - "hitRate": 0.02097902097902098 + "total": 184, + "hitRate": 0.016304347826086956 }, { - "encoding": "ISO-8859-9", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.58869627797, + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 1, + "byteLogLikelihood": -4.476342508555, "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 + "total": 184, + "hitRate": 0.005434782608695652 }, { - "encoding": "windows-1254", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.58869627797, + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 1, + "byteLogLikelihood": -4.71573455878, "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 }, { "encoding": "ISO-8859-16", - "confidence": 2, "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 + "confidence": 38, + "byteLogLikelihood": -2.700458868498, + "hits": 26, + "total": 202, + "hitRate": 0.12871287128712872 }, { - "encoding": "CP852", - "confidence": 2, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 }, { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.338763438513, - "hits": 0, - "total": 143, - "hitRate": 0 + "encoding": "windows-1252", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 }, { - "encoding": "ISO-8859-6", + "encoding": "ISO-8859-5", + "language": "ru", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.514449766028, + "byteLogLikelihood": -4.208609169552, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -3.017269736716, + "confidence": 0, + "byteLogLikelihood": -4.33505338151, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -3.50352863905, + "confidence": 0, + "byteLogLikelihood": -4.197813655274, "hits": 0, - "total": 143, + "total": 201, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -3.846220145127, + "confidence": 0, + "byteLogLikelihood": -4.229886082374, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.017269736716, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.50352863905, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.503504267293, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.91521504926, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.518796947367, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.476992253447, + "byteLogLikelihood": -4.197813655274, "hits": 0, - "total": 143, + "total": 201, "hitRate": 0 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.338763438513, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "IBM855", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -3.774998078834, + "byteLogLikelihood": -4.208609169552, "hits": 0, - "total": 143, + "total": 201, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.766179593299, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "byteLogLikelihood": -4.229886082374, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-1", - "language": "sv" + "encoding": "ISO-8859-3", + "language": "mt" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 50, - "language": "sv", - "byteLogLikelihood": -3.23645137845, - "hits": 23, - "total": 136, - "hitRate": 0.16911764705882354 + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 52, + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 2, + "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-1", - "confidence": 50, - "language": "sv", - "byteLogLikelihood": -3.23645137845, - "hits": 23, - "total": 136, - "hitRate": 0.16911764705882354 - }, - { - "encoding": "windows-1252", - "confidence": 50, - "language": "sv", - "byteLogLikelihood": -3.23645137845, - "hits": 23, - "total": 136, - "hitRate": 0.16911764705882354 + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 52, + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 }, { - "encoding": "windows-1258", - "confidence": 19, - "language": "vi", - "byteLogLikelihood": -5.497168225293, + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 14, + "byteLogLikelihood": -4.81334289805, "hits": 9, - "total": 136, - "hitRate": 0.0661764705882353 + "total": 181, + "hitRate": 0.049723756906077346 }, { - "encoding": "ISO-8859-10", - "confidence": 19, - "language": "is", - "byteLogLikelihood": -4.980036753423, + "encoding": "windows-1252", + "language": "it", + "confidence": 14, + "byteLogLikelihood": -4.81334289805, "hits": 9, - "total": 136, - "hitRate": 0.0661764705882353 + "total": 181, + "hitRate": 0.049723756906077346 }, { - "encoding": "macintosh", - "confidence": 17, - "language": "de", - "byteLogLikelihood": -5.003946305945, + "encoding": "windows-1250", + "language": "pl", + "confidence": 13, + "byteLogLikelihood": -4.980958307454, "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 + "total": 182, + "hitRate": 0.04395604395604396 }, { - "encoding": "ISO-8859-15", - "confidence": 17, - "language": "de", - "byteLogLikelihood": -4.137512330246, + "encoding": "windows-1258", + "language": "vi", + "confidence": 13, + "byteLogLikelihood": -4.506236351586, "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 + "total": 181, + "hitRate": 0.04419889502762431 }, { - "encoding": "CP850", - "confidence": 17, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 + "encoding": "macintosh", + "language": "es", + "confidence": 11, + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 }, { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "cs", - "byteLogLikelihood": -5.159055299215, + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 11, + "byteLogLikelihood": -4.945351018208, "hits": 7, - "total": 136, - "hitRate": 0.051470588235294115 + "total": 183, + "hitRate": 0.03825136612021858 }, { - "encoding": "windows-1250", - "confidence": 15, - "language": "cs", - "byteLogLikelihood": -5.159055299215, + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 11, + "byteLogLikelihood": -4.948759890378, "hits": 7, - "total": 136, - "hitRate": 0.051470588235294115 + "total": 181, + "hitRate": 0.03867403314917127 }, { - "encoding": "ISO-8859-9", + "encoding": "CP850", + "language": "es", "confidence": 11, - "language": "tr", - "byteLogLikelihood": -4.93784727915, - "hits": 5, - "total": 136, - "hitRate": 0.03676470588235294 + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 177, + "hitRate": 0.03954802259887006 }, { - "encoding": "windows-1254", - "confidence": 11, - "language": "tr", - "byteLogLikelihood": -4.93784727915, - "hits": 5, - "total": 136, - "hitRate": 0.03676470588235294 + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 9, + "byteLogLikelihood": -3.834078315244, + "hits": 6, + "total": 183, + "hitRate": 0.03278688524590164 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 182, + "hitRate": 0.03296703296703297 }, { "encoding": "ISO-8859-14", - "confidence": 8, "language": "cy", + "confidence": 6, "byteLogLikelihood": -4.912654885736, "hits": 4, - "total": 136, - "hitRate": 0.029411764705882353 - }, + "total": 183, + "hitRate": 0.02185792349726776 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-4", + "language": "lv" + }, + "predicted": { + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 40, + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ { - "encoding": "ISO-8859-16", - "confidence": 6, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 3, - "total": 136, - "hitRate": 0.022058823529411766 + "encoding": "ISO-8859-4", + "language": "lv", + "confidence": 40, + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 }, { "encoding": "windows-1257", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "language": "lv", + "confidence": 31, + "byteLogLikelihood": -5.111410123766, + "hits": 18, + "total": 169, + "hitRate": 0.10650887573964497 }, { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -4.210065739287, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 14, + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "windows-1252", + "language": "es", + "confidence": 14, + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 14, + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 }, { "encoding": "ISO-8859-13", - "confidence": 4, "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "confidence": 10, + "byteLogLikelihood": -4.596975900219, + "hits": 6, + "total": 169, + "hitRate": 0.03550295857988166 }, { - "encoding": "CP852", - "confidence": 4, + "encoding": "ISO-8859-2", "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "confidence": 8, + "byteLogLikelihood": -4.763992911083, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 }, { - "encoding": "ISO-8859-5", - "confidence": 0, + "encoding": "windows-1250", + "language": "pl", + "confidence": 8, + "byteLogLikelihood": -4.846752537455, + "hits": 5, + "total": 169, + "hitRate": 0.029585798816568046 + }, + { + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.652362971667, + "confidence": 0, + "byteLogLikelihood": -4.704365179493, "hits": 0, - "total": 136, + "total": 169, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.287368473007, + "byteLogLikelihood": -4.578946937177, "hits": 0, - "total": 136, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.876181596107, + "byteLogLikelihood": -4.65535318117, "hits": 0, - "total": 136, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "ISO-8859-16", + "language": "ro", "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.066153183614, + "byteLogLikelihood": -4.616838329205, "hits": 0, - "total": 136, + "total": 169, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-5", + "language": "ru" + }, + "predicted": { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-5", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 }, { - "encoding": "windows-1251", - "confidence": 0, + "encoding": "IBM855", "language": "ru", - "byteLogLikelihood": -3.393743210995, - "hits": 0, - "total": 136, - "hitRate": 0 + "confidence": 9, + "byteLogLikelihood": -4.826799236728, + "hits": 5, + "total": 166, + "hitRate": 0.030120481927710843 }, { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.876181596107, - "hits": 0, - "total": 136, - "hitRate": 0 + "encoding": "KOI8-U", + "language": "uk", + "confidence": 1, + "byteLogLikelihood": -4.816522715875, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 }, { - "encoding": "windows-1255", + "encoding": "ISO-8859-1", + "language": "it", "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.066153183614, + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 136, + "total": 176, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.25594643617, + "byteLogLikelihood": -4.941337938888, "hits": 0, - "total": 136, + "total": 177, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "windows-1252", + "language": "it", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.801526726218, + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 136, + "total": 176, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "windows-1257", + "language": "lt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.432153549797, + "byteLogLikelihood": -4.847671496604, "hits": 0, - "total": 136, + "total": 176, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1258", + "language": "vi", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.390348855877, + "byteLogLikelihood": -4.755485363394, "hits": 0, - "total": 136, + "total": 166, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.652362971667, + "byteLogLikelihood": -4.810914106722, "hits": 0, - "total": 136, + "total": 84, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.419477704429, + "byteLogLikelihood": -4.847671496604, "hits": 0, - "total": 136, + "total": 176, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.393743210995, + "byteLogLikelihood": -4.83002806951, "hits": 0, - "total": 136, + "total": 178, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "byteLogLikelihood": -4.848242748706, "hits": 0, - "total": 136, + "total": 177, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-10", - "language": "is" + "encoding": "ISO-8859-6", + "language": "ar" }, "predicted": { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 1, "candidates": [ - { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 - }, - { - "encoding": "ISO-8859-1", - "confidence": 14, - "language": "sv", - "byteLogLikelihood": -4.633646206449, - "hits": 8, - "total": 171, - "hitRate": 0.04678362573099415 - }, - { - "encoding": "windows-1252", - "confidence": 14, - "language": "sv", - "byteLogLikelihood": -4.633646206449, - "hits": 8, - "total": 171, - "hitRate": 0.04678362573099415 - }, - { - "encoding": "ISO-8859-3", - "confidence": 12, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 - }, - { - "encoding": "windows-1257", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -3.989583472462, - "hits": 4, - "total": 171, - "hitRate": 0.023391812865497075 - }, - { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -4.642697796053, - "hits": 4, - "total": 171, - "hitRate": 0.023391812865497075 - }, - { - "encoding": "ISO-8859-13", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -3.989583472462, - "hits": 4, - "total": 171, - "hitRate": 0.023391812865497075 - }, - { - "encoding": "macintosh", - "confidence": 5, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 - }, - { - "encoding": "ISO-8859-4", - "confidence": 5, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-15", - "confidence": 5, - "language": "de", - "byteLogLikelihood": -4.894502014278, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "CP850", - "confidence": 5, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 - }, - { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 - }, - { - "encoding": "ISO-8859-2", - "confidence": 3, - "language": "cs", - "byteLogLikelihood": -4.497848163857, - "hits": 2, - "total": 171, - "hitRate": 0.011695906432748537 - }, - { - "encoding": "windows-1250", - "confidence": 3, - "language": "cs", - "byteLogLikelihood": -4.497848163857, - "hits": 2, - "total": 171, - "hitRate": 0.011695906432748537 - }, - { - "encoding": "ISO-8859-9", - "confidence": 1, - "language": "tr", - "byteLogLikelihood": -4.646338703133, - "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 - }, - { - "encoding": "windows-1254", - "confidence": 1, - "language": "tr", - "byteLogLikelihood": -4.646338703133, - "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.838861551509, - "hits": 0, - "total": 169, - "hitRate": 0 - }, { "encoding": "ISO-8859-6", - "confidence": 0, "language": "ar", - "byteLogLikelihood": -5.345124108708, - "hits": 0, - "total": 167, - "hitRate": 0 + "confidence": 62, + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 }, { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.963723644165, - "hits": 0, - "total": 171, - "hitRate": 0 + "encoding": "windows-1256", + "language": "ar", + "confidence": 10, + "byteLogLikelihood": -3.968382111451, + "hits": 4, + "total": 110, + "hitRate": 0.03636363636363636 }, { - "encoding": "ISO-8859-8", + "encoding": "ISO-8859-1", + "language": "sv", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.249634019938, + "byteLogLikelihood": -4.69584393439, "hits": 0, - "total": 171, + "total": 110, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-9", + "language": "tr", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.84046735452, + "byteLogLikelihood": -4.933475282212, "hits": 0, - "total": 171, + "total": 110, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "windows-1252", + "language": "sv", "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.963723644165, + "byteLogLikelihood": -4.69584393439, "hits": 0, - "total": 171, + "total": 110, "hitRate": 0 }, { - "encoding": "windows-1255", + "encoding": "windows-1254", + "language": "tr", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.249634019938, + "byteLogLikelihood": -4.933475282212, "hits": 0, - "total": 171, + "total": 110, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.696464660765, + "byteLogLikelihood": -4.72024182268, "hits": 0, - "total": 169, + "total": 110, "hitRate": 0 }, { "encoding": "windows-874", - "confidence": 0, "language": "th", - "byteLogLikelihood": -5.644763656357, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.290788006393, + "byteLogLikelihood": -4.643135408712, "hits": 0, - "total": 171, + "total": 92, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "macintosh", + "language": "es", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.026638569428, + "byteLogLikelihood": -4.948759890378, "hits": 0, - "total": 171, + "total": 55, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.838861551509, + "byteLogLikelihood": -4.799641758471, "hits": 0, - "total": 171, + "total": 111, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-15", + "language": "de", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.795369889152, + "byteLogLikelihood": -4.755317425962, "hits": 0, - "total": 169, + "total": 110, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.733983711968, + "byteLogLikelihood": -4.948759890378, "hits": 0, - "total": 171, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.657284871846, - "hits": 0, - "total": 171, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 0, - "total": 171, + "total": 98, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-13", - "language": "lt" + "encoding": "ISO-8859-7", + "language": "el" }, "predicted": { - "encoding": "windows-1257", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { - "encoding": "windows-1257", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 - }, - { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 - }, - { - "encoding": "ISO-8859-4", - "confidence": 22, - "language": "lv", - "byteLogLikelihood": -4.610398759565, - "hits": 13, - "total": 173, - "hitRate": 0.07514450867052024 - }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "hu", - "byteLogLikelihood": -5.117993812417, - "hits": 9, - "total": 173, - "hitRate": 0.05202312138728324 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "hu", - "byteLogLikelihood": -5.117993812417, - "hits": 9, - "total": 173, - "hitRate": 0.05202312138728324 - }, - { - "encoding": "ISO-8859-1", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 - }, - { - "encoding": "windows-1252", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 - }, - { - "encoding": "macintosh", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 168, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-15", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 - }, - { - "encoding": "CP850", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 170, - "hitRate": 0.041176470588235294 - }, - { - "encoding": "ISO-8859-9", - "confidence": 8, - "language": "tr", - "byteLogLikelihood": -4.789760270297, - "hits": 5, - "total": 173, - "hitRate": 0.028901734104046242 - }, - { - "encoding": "windows-1254", - "confidence": 8, - "language": "tr", - "byteLogLikelihood": -4.789760270297, - "hits": 5, - "total": 173, - "hitRate": 0.028901734104046242 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 172, - "hitRate": 0.01744186046511628 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.6272413408, - "hits": 2, - "total": 173, - "hitRate": 0.011560693641618497 + "encoding": "windows-1253", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, { - "encoding": "windows-1258", - "confidence": 1, - "language": "vi", - "byteLogLikelihood": -4.960186085121, + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 2, + "byteLogLikelihood": -4.09495359507, "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 + "total": 127, + "hitRate": 0.007874015748031496 }, { - "encoding": "IBM855", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -5.488073838002, + "encoding": "windows-1255", + "language": "he", + "confidence": 2, + "byteLogLikelihood": -4.09495359507, "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 + "total": 127, + "hitRate": 0.007874015748031496 }, { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -4.416382325607, + "encoding": "x-mac-cyrillic", + "language": "uk", + "confidence": 2, + "byteLogLikelihood": -4.203232324163, "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "total": 131, + "hitRate": 0.007633587786259542 }, { - "encoding": "ISO-8859-5", + "encoding": "ISO-8859-1", + "language": "pt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.893109334109, + "byteLogLikelihood": -4.59932343766, "hits": 0, - "total": 172, + "total": 134, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.773457169107, + "byteLogLikelihood": -4.358825259583, "hits": 0, - "total": 167, + "total": 134, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.472500251122, + "byteLogLikelihood": -4.358825259583, "hits": 0, - "total": 173, + "total": 134, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.444220754634, + "byteLogLikelihood": -4.288697198304, "hits": 0, - "total": 171, + "total": 134, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "windows-1252", + "language": "pt", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.531827362419, + "byteLogLikelihood": -4.59932343766, "hits": 0, - "total": 173, + "total": 134, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.472500251122, + "byteLogLikelihood": -4.576407109739, "hits": 0, - "total": 173, + "total": 134, "hitRate": 0 }, { - "encoding": "windows-1255", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.444220754634, + "byteLogLikelihood": -4.603974037796, "hits": 0, - "total": 171, + "total": 134, "hitRate": 0 - }, - { - "encoding": "windows-1256", + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-8", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "windows-1255", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "language": "sv", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.924336543075, + "byteLogLikelihood": -4.395116706452, "hits": 0, - "total": 170, + "total": 102, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-2", + "language": "cs", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.58967630288, + "byteLogLikelihood": -4.461975046319, "hits": 0, - "total": 166, + "total": 102, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "windows-1250", + "language": "cs", "confidence": 0, + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -6.365896834008, + "confidence": 0, + "byteLogLikelihood": -4.199203901043, "hits": 0, - "total": 173, + "total": 102, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1252", + "language": "sv", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.252539473952, + "byteLogLikelihood": -4.395116706452, "hits": 0, - "total": 173, + "total": 102, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.893109334109, + "byteLogLikelihood": -4.639272141503, "hits": 0, - "total": 171, + "total": 102, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", - "byteLogLikelihood": -3.441328843196, + "confidence": 0, + "byteLogLikelihood": -4.134202940863, "hits": 0, - "total": 173, + "total": 102, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-15", + "language": "fr", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.616838329205, + "byteLogLikelihood": -4.521941300627, "hits": 0, - "total": 173, + "total": 102, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-14", - "language": "cy" + "encoding": "ISO-8859-9", + "language": "tr" }, "predicted": { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 }, "encodingExact": true, "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 }, { "encoding": "ISO-8859-1", - "confidence": 14, - "language": "pt", - "byteLogLikelihood": -4.955827057601, - "hits": 9, - "total": 189, - "hitRate": 0.047619047619047616 + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 }, { "encoding": "windows-1252", - "confidence": 14, - "language": "pt", - "byteLogLikelihood": -4.955827057601, - "hits": 9, - "total": 189, - "hitRate": 0.047619047619047616 + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 }, { - "encoding": "windows-1258", - "confidence": 14, - "language": "vi", - "byteLogLikelihood": -4.34192292436, - "hits": 9, - "total": 188, - "hitRate": 0.047872340425531915 + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 }, { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1257", + "language": "lv", "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 189, - "hitRate": 0.037037037037037035 + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "CP850", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "ISO-8859-2", - "confidence": 9, - "language": "cs", - "byteLogLikelihood": -5.159055299215, - "hits": 6, - "total": 189, - "hitRate": 0.031746031746031744 + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "windows-1250", - "confidence": 9, - "language": "cs", - "byteLogLikelihood": -5.159055299215, - "hits": 6, - "total": 189, - "hitRate": 0.031746031746031744 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 }, { - "encoding": "windows-1257", - "confidence": 6, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 4, - "total": 189, - "hitRate": 0.021164021164021163 - }, - { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -4.009500374258, - "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-9", - "confidence": 3, - "language": "tr", - "byteLogLikelihood": -4.613075694861, - "hits": 2, - "total": 189, - "hitRate": 0.010582010582010581 - }, - { - "encoding": "windows-1254", - "confidence": 3, - "language": "tr", - "byteLogLikelihood": -4.613075694861, - "hits": 2, - "total": 189, - "hitRate": 0.010582010582010581 - }, - { - "encoding": "CP852", - "confidence": 3, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 188, - "hitRate": 0.010638297872340425 - }, - { - "encoding": "ISO-8859-4", - "confidence": 1, - "language": "lv", - "byteLogLikelihood": -4.961907790704, - "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 - }, - { - "encoding": "ISO-8859-13", - "confidence": 1, - "language": "lt", - "byteLogLikelihood": -3.647025046455, - "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 - }, - { - "encoding": "ISO-8859-5", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.350299845595, + "byteLogLikelihood": -4.216966174066, "hits": 0, - "total": 189, + "total": 134, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.420534999272, + "byteLogLikelihood": -4.22610845571, "hits": 0, - "total": 188, + "total": 134, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-R", + "language": "ru" + }, + "predicted": { + "encoding": "KOI8-R", + "language": "ru", + "confidence": 43, + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-R", + "language": "ru", + "confidence": 43, + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 }, { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.254156765951, - "hits": 0, - "total": 189, - "hitRate": 0 + "encoding": "KOI8-U", + "language": "uk", + "confidence": 28, + "byteLogLikelihood": -3.443215105753, + "hits": 17, + "total": 179, + "hitRate": 0.09497206703910614 }, { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.917404374874, - "hits": 0, - "total": 188, - "hitRate": 0 + "encoding": "windows-874", + "language": "th", + "confidence": 12, + "byteLogLikelihood": -4.705552828452, + "hits": 7, + "total": 162, + "hitRate": 0.043209876543209874 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-1", + "language": "nl", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.467895804539, + "byteLogLikelihood": -4.900684399759, "hits": 0, - "total": 189, + "total": 176, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "ISO-8859-6", + "language": "ar", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.254156765951, + "byteLogLikelihood": -4.759099059367, "hits": 0, - "total": 189, + "total": 176, "hitRate": 0 }, { - "encoding": "windows-1255", + "encoding": "windows-1252", + "language": "nl", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.917404374874, + "byteLogLikelihood": -4.900684399759, "hits": 0, - "total": 188, + "total": 176, "hitRate": 0 }, { "encoding": "windows-1256", - "confidence": 0, "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 188, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.830407858741, - "hits": 0, - "total": 188, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.229985622133, + "byteLogLikelihood": -4.796335743252, "hits": 0, - "total": 189, + "total": 175, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.821976831991, + "byteLogLikelihood": -4.91533239831, "hits": 0, - "total": 189, + "total": 176, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "macintosh", + "language": "es", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.350299845595, + "byteLogLikelihood": -4.948759890378, "hits": 0, - "total": 188, + "total": 94, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.325879646284, + "byteLogLikelihood": -4.912654885736, "hits": 0, - "total": 188, + "total": 179, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-15", + "language": "de", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.255815902698, + "byteLogLikelihood": -4.935090625757, "hits": 0, - "total": 189, + "total": 176, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "CP850", + "language": "es", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -3.903274585551, + "byteLogLikelihood": -4.948759890378, "hits": 0, - "total": 189, + "total": 92, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", - "language": "de" + "encoding": "KOI8-U", + "language": "uk" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "KOI8-U", + "language": "uk", + "confidence": 32, + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 3, + "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-1", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "KOI8-U", + "language": "uk", + "confidence": 32, + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 }, { - "encoding": "windows-1252", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "KOI8-R", + "language": "ru", + "confidence": 17, + "byteLogLikelihood": -3.510012695801, + "hits": 10, + "total": 172, + "hitRate": 0.05813953488372093 + }, + { + "encoding": "windows-874", + "language": "th", + "confidence": 11, + "byteLogLikelihood": -4.843281209678, + "hits": 6, + "total": 163, + "hitRate": 0.03680981595092025 + }, + { + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 1, + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 1, + "byteLogLikelihood": -4.726706027838, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "windows-1252", + "language": "en", + "confidence": 1, + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "windows-1256", + "language": "ar", + "confidence": 1, + "byteLogLikelihood": -4.693044688745, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 0, + "byteLogLikelihood": -4.922686041886, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 89, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 0, + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 177, + "hitRate": 0 }, { "encoding": "ISO-8859-15", - "confidence": 73, + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.944199711559, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "CP850", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "de" + }, + "predicted": { + "encoding": "macintosh", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", "language": "de", + "confidence": 73, "byteLogLikelihood": -3.169712077519, "hits": 53, "total": 216, "hitRate": 0.24537037037037038 }, { - "encoding": "macintosh", - "confidence": 72, + "encoding": "windows-1252", + "language": "de", + "confidence": 70, + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "ISO-8859-15", "language": "de", + "confidence": 74, "byteLogLikelihood": -5.003946305945, - "hits": 52, - "total": 215, - "hitRate": 0.24186046511627907 + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 74, + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 }, { "encoding": "CP850", - "confidence": 70, "language": "de", + "confidence": 69, "byteLogLikelihood": -5.003946305945, - "hits": 51, + "hits": 50, "total": 216, - "hitRate": 0.2361111111111111 + "hitRate": 0.23148148148148148 }, { "encoding": "ISO-8859-14", - "confidence": 20, "language": "cy", + "confidence": 22, "byteLogLikelihood": -4.912654885736, - "hits": 15, - "total": 216, - "hitRate": 0.06944444444444445 + "hits": 16, + "total": 214, + "hitRate": 0.07476635514018691 }, { "encoding": "ISO-8859-2", - "confidence": 13, "language": "ro", + "confidence": 15, "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 + "hits": 11, + "total": 214, + "hitRate": 0.0514018691588785 }, { "encoding": "windows-1250", - "confidence": 13, "language": "ro", + "confidence": 15, "byteLogLikelihood": -5.176149732574, - "hits": 10, + "hits": 11, "total": 216, - "hitRate": 0.046296296296296294 + "hitRate": 0.05092592592592592 }, { "encoding": "ISO-8859-9", - "confidence": 12, "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 9, - "total": 216, - "hitRate": 0.041666666666666664 + "confidence": 14, + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 214, + "hitRate": 0.04672897196261682 }, { "encoding": "windows-1254", - "confidence": 12, "language": "tr", - "byteLogLikelihood": -4.5258676709, + "confidence": 12, + "byteLogLikelihood": -5.075173815234, "hits": 9, "total": 216, "hitRate": 0.041666666666666664 }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 216, - "hitRate": 0.037037037037037035 - }, { "encoding": "windows-1257", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 - }, - { - "encoding": "ISO-8859-4", - "confidence": 9, "language": "lv", + "confidence": 11, "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 + "hits": 8, + "total": 214, + "hitRate": 0.037383177570093455 }, { "encoding": "CP852", - "confidence": 6, "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 216, - "hitRate": 0.023148148148148147 - }, - { - "encoding": "windows-1258", "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, + "byteLogLikelihood": -5.087596335232, "hits": 4, "total": 216, "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 216, - "hitRate": 0.009259259259259259 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 1, - "total": 216, - "hitRate": 0.004629629629629629 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.162408104115, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.007313397456, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.349754587225, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.279537403415, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.542402054434, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.349754587225, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.279537403415, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.318835143108, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.657121063601, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.28774788718, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.245943193261, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.942998477092, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.629332579687, - "hits": 0, - "total": 216, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-15", - "language": "es" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 3, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "es" + }, + "predicted": { + "encoding": "macintosh", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { "encoding": "windows-1252", - "confidence": 61, "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 }, { "encoding": "ISO-8859-15", - "confidence": 61, "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "confidence": 58, + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 }, { - "encoding": "macintosh", - "confidence": 58, + "encoding": "ISO-8859-1", "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -6677,8 +5105,8 @@ }, { "encoding": "CP850", - "confidence": 58, "language": "es", + "confidence": 58, "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 200, @@ -6686,10492 +5114,1122 @@ }, { "encoding": "windows-1257", - "confidence": 24, "language": "lv", + "confidence": 25, "byteLogLikelihood": -5.19295685089, - "hits": 16, - "total": 200, - "hitRate": 0.08 - }, - { - "encoding": "ISO-8859-4", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.009854802779, - "hits": 16, - "total": 200, - "hitRate": 0.08 - }, - { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -4.892914919217, - "hits": 13, - "total": 200, - "hitRate": 0.065 - }, - { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 200, - "hitRate": 0.055 - }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "windows-1258", - "confidence": 4, - "language": "vi", - "byteLogLikelihood": -4.848531508941, - "hits": 3, - "total": 200, - "hitRate": 0.015 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -3.846028832089, - "hits": 3, - "total": 200, - "hitRate": 0.015 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.977292083619, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.346153815501, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.106555532888, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.411159813207, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.106555532888, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.411159813207, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.300604963183, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.412820202576, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.977292083619, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.203740544852, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 200, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-15", - "language": "fr" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 3, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "windows-1252", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "ISO-8859-15", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "macintosh", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "CP850", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "ISO-8859-2", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "windows-1250", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "ISO-8859-10", - "confidence": 10, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 - }, - { - "encoding": "ISO-8859-16", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 - }, - { - "encoding": "windows-1257", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-9", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "windows-1254", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 228, - "hitRate": 0.013157894736842105 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.612642200827, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.57655026914, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, - "hits": 0, - "total": 228, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-16", - "language": "ro" - }, - "predicted": { - "encoding": "ISO-8859-16", - "confidence": 53, - "language": "ro", - "byteLogLikelihood": -2.758152644716, - "hits": 38, - "total": 214, - "hitRate": 0.17757009345794392 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-16", - "confidence": 53, - "language": "ro", - "byteLogLikelihood": -2.758152644716, - "hits": 38, - "total": 214, - "hitRate": 0.17757009345794392 - }, - { - "encoding": "ISO-8859-2", - "confidence": 44, - "language": "ro", - "byteLogLikelihood": -2.870272383724, - "hits": 32, - "total": 214, - "hitRate": 0.14953271028037382 - }, - { - "encoding": "windows-1250", - "confidence": 44, - "language": "ro", - "byteLogLikelihood": -2.870272383724, - "hits": 32, - "total": 214, - "hitRate": 0.14953271028037382 - }, - { - "encoding": "ISO-8859-1", - "confidence": 26, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 19, - "total": 214, - "hitRate": 0.08878504672897196 - }, - { - "encoding": "windows-1252", - "confidence": 26, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 19, - "total": 214, - "hitRate": 0.08878504672897196 - }, - { - "encoding": "macintosh", - "confidence": 22, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 16, - "total": 210, - "hitRate": 0.0761904761904762 - }, - { - "encoding": "CP850", - "confidence": 18, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 213, - "hitRate": 0.06103286384976526 - }, - { - "encoding": "ISO-8859-15", - "confidence": 16, - "language": "fr", - "byteLogLikelihood": -4.918215670022, - "hits": 12, - "total": 214, - "hitRate": 0.056074766355140186 - }, - { - "encoding": "ISO-8859-3", - "confidence": 12, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 9, - "total": 212, - "hitRate": 0.04245283018867924 - }, - { - "encoding": "ISO-8859-9", - "confidence": 9, - "language": "tr", - "byteLogLikelihood": -4.982754191159, - "hits": 7, - "total": 214, - "hitRate": 0.03271028037383177 - }, - { - "encoding": "windows-1254", - "confidence": 9, - "language": "tr", - "byteLogLikelihood": -4.982754191159, - "hits": 7, - "total": 214, - "hitRate": 0.03271028037383177 - }, - { - "encoding": "CP852", - "confidence": 9, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 7, - "total": 215, - "hitRate": 0.03255813953488372 - }, - { - "encoding": "windows-1257", - "confidence": 8, - "language": "lv", - "byteLogLikelihood": -4.456762061842, - "hits": 6, - "total": 214, - "hitRate": 0.028037383177570093 - }, - { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -4.924600071508, - "hits": 5, - "total": 214, - "hitRate": 0.02336448598130841 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.219909353069, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 - }, - { - "encoding": "ISO-8859-10", - "confidence": 5, - "language": "is", - "byteLogLikelihood": -5.111587063002, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -4.864509292863, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.542976389437, - "hits": 3, - "total": 216, - "hitRate": 0.013888888888888888 - }, - { - "encoding": "windows-874", - "confidence": 1, - "language": "th", - "byteLogLikelihood": -5.210426263201, - "hits": 1, - "total": 214, - "hitRate": 0.004672897196261682 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.539739112612, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.000268888635, - "hits": 0, - "total": 211, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.499845100591, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.429071192897, - "hits": 0, - "total": 213, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.382241948417, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.499845100591, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.429071192897, - "hits": 0, - "total": 213, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.739610656221, - "hits": 0, - "total": 213, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.414824870283, - "hits": 0, - "total": 213, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.199507530667, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.333669615722, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.794020051248, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.382241948417, - "hits": 0, - "total": 214, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-2", - "language": "cs" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 30, - "language": "cs", - "byteLogLikelihood": -3.429159108442, - "hits": 13, - "total": 126, - "hitRate": 0.10317460317460317 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 30, - "language": "cs", - "byteLogLikelihood": -3.429159108442, - "hits": 13, - "total": 126, - "hitRate": 0.10317460317460317 - }, - { - "encoding": "windows-1250", - "confidence": 30, - "language": "cs", - "byteLogLikelihood": -3.429159108442, - "hits": 13, - "total": 126, - "hitRate": 0.10317460317460317 - }, - { - "encoding": "ISO-8859-1", - "confidence": 11, - "language": "pt", - "byteLogLikelihood": -4.5670392286, - "hits": 5, - "total": 126, - "hitRate": 0.03968253968253968 - }, - { - "encoding": "windows-1252", - "confidence": 11, - "language": "pt", - "byteLogLikelihood": -4.5670392286, - "hits": 5, - "total": 126, - "hitRate": 0.03968253968253968 - }, - { - "encoding": "CP852", - "confidence": 11, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 126, - "hitRate": 0.03968253968253968 - }, - { - "encoding": "windows-1257", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -4.997937219301, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -4.506724065575, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "macintosh", - "confidence": 7, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 3, - "total": 124, - "hitRate": 0.024193548387096774 - }, - { - "encoding": "ISO-8859-13", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -4.997937219301, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "ISO-8859-15", - "confidence": 7, - "language": "fr", - "byteLogLikelihood": -4.081579721495, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "ISO-8859-16", - "confidence": 7, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "CP850", - "confidence": 7, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 3, - "total": 125, - "hitRate": 0.024 - }, - { - "encoding": "ISO-8859-9", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -4.849273210708, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "windows-1254", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -4.849273210708, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-4", - "confidence": 4, - "language": "lv", - "byteLogLikelihood": -5.129943470839, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.254067466067, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.402405269754, - "hits": 0, - "total": 124, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.401446196881, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.467509652199, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.249199684222, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.401446196881, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.467509652199, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.164526575273, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.504431122267, - "hits": 0, - "total": 122, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.566056982405, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.524252288486, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.254067466067, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.108905030169, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.226561011787, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.43713857777, - "hits": 0, - "total": 126, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-2", - "language": "hu" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 46, - "language": "hu", - "byteLogLikelihood": -3.161205902203, - "hits": 21, - "total": 136, - "hitRate": 0.15441176470588236 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 46, - "language": "hu", - "byteLogLikelihood": -3.161205902203, - "hits": 21, - "total": 136, - "hitRate": 0.15441176470588236 - }, - { - "encoding": "windows-1250", - "confidence": 46, - "language": "hu", - "byteLogLikelihood": -3.161205902203, - "hits": 21, - "total": 136, - "hitRate": 0.15441176470588236 - }, - { - "encoding": "ISO-8859-1", - "confidence": 17, - "language": "pt", - "byteLogLikelihood": -4.507887190294, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 - }, - { - "encoding": "windows-1252", - "confidence": 17, - "language": "pt", - "byteLogLikelihood": -4.507887190294, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 - }, - { - "encoding": "CP852", - "confidence": 15, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 7, - "total": 135, - "hitRate": 0.05185185185185185 - }, - { - "encoding": "ISO-8859-14", - "confidence": 8, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 4, - "total": 136, - "hitRate": 0.029411764705882353 - }, - { - "encoding": "macintosh", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 3, - "total": 135, - "hitRate": 0.022222222222222223 - }, - { - "encoding": "CP850", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 3, - "total": 134, - "hitRate": 0.022388059701492536 - }, - { - "encoding": "ISO-8859-15", - "confidence": 4, - "language": "fr", - "byteLogLikelihood": -4.274525549305, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, - { - "encoding": "ISO-8859-16", - "confidence": 4, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, - { - "encoding": "ISO-8859-9", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "windows-1254", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "windows-1257", - "confidence": 2, - "language": "et", - "byteLogLikelihood": -4.453831056838, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-4", - "confidence": 2, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-10", - "confidence": 2, - "language": "is", - "byteLogLikelihood": -3.90025153387, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -4.950677184263, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.485271956182, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.538035906584, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.430247904624, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.984546441118, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.6300671031, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.430247904624, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.984546441118, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.009772962617, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.010690688029, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.953752719217, - "hits": 0, - "total": 133, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.606957349802, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.485271956182, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.100966544791, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.187765411941, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 136, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-2", - "language": "pl" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -3.52910527172, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -3.52910527172, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - { - "encoding": "windows-1250", - "confidence": 37, - "language": "pl", - "byteLogLikelihood": -4.297518988235, - "hits": 23, - "total": 183, - "hitRate": 0.12568306010928962 - }, - { - "encoding": "CP852", - "confidence": 41, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 25, - "total": 182, - "hitRate": 0.13736263736263737 - }, - { - "encoding": "ISO-8859-1", - "confidence": 8, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 5, - "total": 183, - "hitRate": 0.0273224043715847 - }, - { - "encoding": "windows-1252", - "confidence": 8, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 5, - "total": 183, - "hitRate": 0.0273224043715847 - }, - { - "encoding": "windows-1258", - "confidence": 8, - "language": "vi", - "byteLogLikelihood": -5.343135518502, - "hits": 5, - "total": 183, - "hitRate": 0.0273224043715847 - }, - { - "encoding": "windows-1257", - "confidence": 6, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "macintosh", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "ISO-8859-13", - "confidence": 6, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "ISO-8859-15", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.396558412648, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "CP850", - "confidence": 6, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 4, - "total": 182, - "hitRate": 0.02197802197802198 - }, - { - "encoding": "ISO-8859-4", - "confidence": 4, - "language": "lv", - "byteLogLikelihood": -4.794788079951, - "hits": 3, - "total": 184, - "hitRate": 0.016304347826086956 - }, - { - "encoding": "ISO-8859-16", - "confidence": 3, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 2, - "total": 183, - "hitRate": 0.01092896174863388 - }, - { - "encoding": "ISO-8859-9", - "confidence": 1, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 1, - "total": 183, - "hitRate": 0.00546448087431694 - }, - { - "encoding": "windows-1254", - "confidence": 1, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 1, - "total": 183, - "hitRate": 0.00546448087431694 - }, - { - "encoding": "ISO-8859-3", - "confidence": 1, - "language": "mt", - "byteLogLikelihood": -4.476342508555, - "hits": 1, - "total": 184, - "hitRate": 0.005434782608695652 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -4.71573455878, - "hits": 1, - "total": 184, - "hitRate": 0.005434782608695652 - }, - { - "encoding": "ISO-8859-14", - "confidence": 1, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 1, - "total": 184, - "hitRate": 0.005434782608695652 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.293914971567, - "hits": 0, - "total": 184, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.002422325308, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.470147067075, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.300217461931, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.002422325308, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.470147067075, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.586271794593, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.538051035716, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.496246341796, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.447947678358, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.497577429688, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.300217461931, - "hits": 0, - "total": 183, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-2", - "language": "ro" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - { - "encoding": "windows-1250", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - { - "encoding": "ISO-8859-16", - "confidence": 38, - "language": "ro", - "byteLogLikelihood": -2.700458868498, - "hits": 26, - "total": 202, - "hitRate": 0.12871287128712872 - }, - { - "encoding": "ISO-8859-1", - "confidence": 26, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 18, - "total": 202, - "hitRate": 0.0891089108910891 - }, - { - "encoding": "windows-1252", - "confidence": 26, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 18, - "total": 202, - "hitRate": 0.0891089108910891 - }, - { - "encoding": "macintosh", - "confidence": 19, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 198, - "hitRate": 0.06565656565656566 - }, - { - "encoding": "CP850", - "confidence": 15, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "ISO-8859-15", - "confidence": 14, - "language": "fr", - "byteLogLikelihood": -4.903997266318, - "hits": 10, - "total": 202, - "hitRate": 0.04950495049504951 - }, - { - "encoding": "ISO-8859-3", - "confidence": 13, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 9, - "total": 200, - "hitRate": 0.045 - }, - { - "encoding": "ISO-8859-9", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.021854801345, - "hits": 7, - "total": 202, - "hitRate": 0.034653465346534656 - }, - { - "encoding": "windows-1254", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.021854801345, - "hits": 7, - "total": 202, - "hitRate": 0.034653465346534656 - }, - { - "encoding": "windows-1257", - "confidence": 8, - "language": "lv", - "byteLogLikelihood": -4.396820338954, - "hits": 6, - "total": 202, - "hitRate": 0.0297029702970297 - }, - { - "encoding": "CP852", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 201, - "hitRate": 0.029850746268656716 - }, - { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -4.936633427031, - "hits": 5, - "total": 202, - "hitRate": 0.024752475247524754 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.177254141958, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 - }, - { - "encoding": "ISO-8859-10", - "confidence": 5, - "language": "is", - "byteLogLikelihood": -5.150687673188, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 - }, - { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.974278583322, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.539421788511, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.208609169552, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.828974536143, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.33505338151, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.197813655274, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.33505338151, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.197813655274, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.528215037204, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.239455511679, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.42548867306, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.32130473174, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.208609169552, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.608648280371, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, - "hits": 0, - "total": 202, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-3", - "language": "mt" - }, - "predicted": { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 - }, - { - "encoding": "ISO-8859-9", - "confidence": 16, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 10, - "total": 181, - "hitRate": 0.055248618784530384 - }, - { - "encoding": "windows-1254", - "confidence": 16, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 10, - "total": 181, - "hitRate": 0.055248618784530384 - }, - { - "encoding": "ISO-8859-1", - "confidence": 14, - "language": "it", - "byteLogLikelihood": -4.81334289805, - "hits": 9, - "total": 181, - "hitRate": 0.049723756906077346 - }, - { - "encoding": "windows-1252", - "confidence": 14, - "language": "it", - "byteLogLikelihood": -4.81334289805, - "hits": 9, - "total": 181, - "hitRate": 0.049723756906077346 - }, - { - "encoding": "windows-1250", - "confidence": 13, - "language": "pl", - "byteLogLikelihood": -4.980958307454, - "hits": 8, - "total": 182, - "hitRate": 0.04395604395604396 - }, - { - "encoding": "windows-1258", - "confidence": 13, - "language": "vi", - "byteLogLikelihood": -4.506236351586, - "hits": 8, - "total": 181, - "hitRate": 0.04419889502762431 - }, - { - "encoding": "CP852", - "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 8, - "total": 177, - "hitRate": 0.04519774011299435 - }, - { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 - }, - { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -4.945351018208, - "hits": 7, - "total": 183, - "hitRate": 0.03825136612021858 - }, - { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 - }, - { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 177, - "hitRate": 0.03954802259887006 - }, - { - "encoding": "ISO-8859-2", - "confidence": 9, - "language": "pl", - "byteLogLikelihood": -3.834078315244, - "hits": 6, - "total": 183, - "hitRate": 0.03278688524590164 - }, - { - "encoding": "windows-1257", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 182, - "hitRate": 0.03296703296703297 - }, - { - "encoding": "ISO-8859-14", - "confidence": 6, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.986136327924, - "hits": 3, - "total": 182, - "hitRate": 0.016483516483516484 - }, - { - "encoding": "ISO-8859-16", - "confidence": 4, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 3, - "total": 182, - "hitRate": 0.016483516483516484 - }, - { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 183, - "hitRate": 0.01092896174863388 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.351251681057, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.198030006336, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.755123257218, - "hits": 0, - "total": 181, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.572384450833, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.198030006336, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.755123257218, - "hits": 0, - "total": 181, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.490920024976, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.42548867306, - "hits": 0, - "total": 181, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.383683979141, - "hits": 0, - "total": 181, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.404570694946, - "hits": 0, - "total": 181, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.943656762101, - "hits": 0, - "total": 181, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.572384450833, - "hits": 0, - "total": 182, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-4", - "language": "lv" - }, - "predicted": { - "encoding": "ISO-8859-4", - "confidence": 40, - "language": "lv", - "byteLogLikelihood": -3.1328616083, - "hits": 23, - "total": 170, - "hitRate": 0.13529411764705881 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-4", - "confidence": 40, - "language": "lv", - "byteLogLikelihood": -3.1328616083, - "hits": 23, - "total": 170, - "hitRate": 0.13529411764705881 - }, - { - "encoding": "windows-1257", - "confidence": 31, - "language": "lv", - "byteLogLikelihood": -5.111410123766, - "hits": 18, - "total": 169, - "hitRate": 0.10650887573964497 - }, - { - "encoding": "CP850", - "confidence": 16, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 9, - "total": 168, - "hitRate": 0.05357142857142857 - }, - { - "encoding": "ISO-8859-1", - "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.721042183854, - "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 - }, - { - "encoding": "windows-1252", - "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.721042183854, - "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 - }, - { - "encoding": "macintosh", - "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 - }, - { - "encoding": "ISO-8859-15", - "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.721042183854, - "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 - }, - { - "encoding": "CP852", - "confidence": 12, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 - }, - { - "encoding": "ISO-8859-13", - "confidence": 10, - "language": "lt", - "byteLogLikelihood": -4.596975900219, - "hits": 6, - "total": 169, - "hitRate": 0.03550295857988166 - }, - { - "encoding": "ISO-8859-2", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -4.763992911083, - "hits": 5, - "total": 170, - "hitRate": 0.029411764705882353 - }, - { - "encoding": "windows-1250", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -4.846752537455, - "hits": 5, - "total": 169, - "hitRate": 0.029585798816568046 - }, - { - "encoding": "ISO-8859-3", - "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 5, - "total": 170, - "hitRate": 0.029411764705882353 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 169, - "hitRate": 0.023668639053254437 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 169, - "hitRate": 0.023668639053254437 - }, - { - "encoding": "ISO-8859-10", - "confidence": 7, - "language": "is", - "byteLogLikelihood": -5.074758182528, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -4.725539964734, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.226735250772, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.427610697135, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.928827369245, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.704365179493, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.427610697135, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.928827369245, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.578946937177, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.610537015382, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.609505685025, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.267508614334, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.182018138172, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.65535318117, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.616838329205, - "hits": 0, - "total": 169, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-5", - "language": "ru" - }, - "predicted": { - "encoding": "ISO-8859-5", - "confidence": 41, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 25, - "total": 179, - "hitRate": 0.13966480446927373 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-5", - "confidence": 41, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 25, - "total": 179, - "hitRate": 0.13966480446927373 - }, - { - "encoding": "IBM866", - "confidence": 20, - "language": "ru", - "byteLogLikelihood": -5.798412758856, - "hits": 5, - "total": 75, - "hitRate": 0.06666666666666667 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 12, - "language": "uk", - "byteLogLikelihood": -5.66890403251, - "hits": 6, - "total": 147, - "hitRate": 0.04081632653061224 - }, - { - "encoding": "IBM855", - "confidence": 9, - "language": "ru", - "byteLogLikelihood": -4.826799236728, - "hits": 5, - "total": 166, - "hitRate": 0.030120481927710843 - }, - { - "encoding": "windows-1256", - "confidence": 1, - "language": "ar", - "byteLogLikelihood": -4.986561094704, - "hits": 1, - "total": 176, - "hitRate": 0.005681818181818182 - }, - { - "encoding": "KOI8-U", - "confidence": 1, - "language": "uk", - "byteLogLikelihood": -4.816522715875, - "hits": 1, - "total": 179, - "hitRate": 0.00558659217877095 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "it", - "byteLogLikelihood": -4.841944488282, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.941337938888, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.219340428659, - "hits": 0, - "total": 152, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.141186484933, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.371812787625, - "hits": 0, - "total": 75, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.982255634278, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.941337938888, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.667544038277, - "hits": 0, - "total": 178, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "it", - "byteLogLikelihood": -4.841944488282, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -5.141186484933, - "hits": 0, - "total": 174, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.982255634278, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.371812787625, - "hits": 0, - "total": 113, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.847671496604, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -4.755485363394, - "hits": 0, - "total": 166, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.810914106722, - "hits": 0, - "total": 84, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.977706457443, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 98, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.966015130424, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -5.103902669993, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.847671496604, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.83002806951, - "hits": 0, - "total": 178, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.848242748706, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.105122227945, - "hits": 0, - "total": 178, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 162, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.078415577874, - "hits": 0, - "total": 166, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-6", - "language": "ar" - }, - "predicted": { - "encoding": "ISO-8859-6", - "confidence": 62, - "language": "ar", - "byteLogLikelihood": -3.306009422775, - "hits": 23, - "total": 110, - "hitRate": 0.20909090909090908 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-6", - "confidence": 62, - "language": "ar", - "byteLogLikelihood": -3.306009422775, - "hits": 23, - "total": 110, - "hitRate": 0.20909090909090908 - }, - { - "encoding": "windows-1256", - "confidence": 10, - "language": "ar", - "byteLogLikelihood": -3.968382111451, - "hits": 4, - "total": 110, - "hitRate": 0.03636363636363636 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 10, - "language": "ru", - "byteLogLikelihood": -5.359123221706, - "hits": 3, - "total": 82, - "hitRate": 0.036585365853658534 - }, - { - "encoding": "ISO-8859-8", - "confidence": 8, - "language": "he", - "byteLogLikelihood": -5.049228744152, - "hits": 2, - "total": 70, - "hitRate": 0.02857142857142857 - }, - { - "encoding": "windows-1255", - "confidence": 8, - "language": "he", - "byteLogLikelihood": -5.049228744152, - "hits": 2, - "total": 72, - "hitRate": 0.027777777777777776 - }, - { - "encoding": "ISO-8859-7", - "confidence": 5, - "language": "el", - "byteLogLikelihood": -4.984009642853, - "hits": 2, - "total": 109, - "hitRate": 0.01834862385321101 - }, - { - "encoding": "windows-1253", - "confidence": 5, - "language": "el", - "byteLogLikelihood": -4.984009642853, - "hits": 2, - "total": 109, - "hitRate": 0.01834862385321101 - }, - { - "encoding": "KOI8-R", - "confidence": 2, - "language": "ru", - "byteLogLikelihood": -5.040654549299, - "hits": 1, - "total": 110, - "hitRate": 0.00909090909090909 - }, - { - "encoding": "KOI8-U", - "confidence": 2, - "language": "uk", - "byteLogLikelihood": -5.174167309306, - "hits": 1, - "total": 110, - "hitRate": 0.00909090909090909 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.69584393439, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -4.951980582514, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.534213772279, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.933475282212, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -4.951980582514, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.223507468988, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.69584393439, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.933475282212, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.72024182268, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.029794115635, - "hits": 0, - "total": 109, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.643135408712, - "hits": 0, - "total": 92, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.068419445019, - "hits": 0, - "total": 72, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.811360688713, - "hits": 0, - "total": 100, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 55, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.047518534687, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.155285808468, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -5.051594021628, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.063690311528, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.799641758471, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "de", - "byteLogLikelihood": -4.755317425962, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.160082157204, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 98, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 100, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-7", - "language": "el" - }, - "predicted": { - "encoding": "ISO-8859-7", - "confidence": 51, - "language": "el", - "byteLogLikelihood": -3.435199502, - "hits": 23, - "total": 134, - "hitRate": 0.17164179104477612 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-7", - "confidence": 51, - "language": "el", - "byteLogLikelihood": -3.435199502, - "hits": 23, - "total": 134, - "hitRate": 0.17164179104477612 - }, - { - "encoding": "windows-1253", - "confidence": 51, - "language": "el", - "byteLogLikelihood": -3.435199502, - "hits": 23, - "total": 134, - "hitRate": 0.17164179104477612 - }, - { - "encoding": "KOI8-U", - "confidence": 8, - "language": "uk", - "byteLogLikelihood": -6.254780178237, - "hits": 4, - "total": 134, - "hitRate": 0.029850746268656716 - }, - { - "encoding": "IBM855", - "confidence": 7, - "language": "ru", - "byteLogLikelihood": -5.358823187498, - "hits": 3, - "total": 118, - "hitRate": 0.025423728813559324 - }, - { - "encoding": "KOI8-R", - "confidence": 4, - "language": "ru", - "byteLogLikelihood": -6.302671202821, - "hits": 2, - "total": 134, - "hitRate": 0.014925373134328358 - }, - { - "encoding": "ISO-8859-6", - "confidence": 3, - "language": "ar", - "byteLogLikelihood": -5.268243094623, - "hits": 1, - "total": 81, - "hitRate": 0.012345679012345678 - }, - { - "encoding": "ISO-8859-8", - "confidence": 2, - "language": "he", - "byteLogLikelihood": -4.09495359507, - "hits": 1, - "total": 127, - "hitRate": 0.007874015748031496 - }, - { - "encoding": "windows-1255", - "confidence": 2, - "language": "he", - "byteLogLikelihood": -4.09495359507, - "hits": 1, - "total": 127, - "hitRate": 0.007874015748031496 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 2, - "language": "uk", - "byteLogLikelihood": -4.203232324163, - "hits": 1, - "total": 131, - "hitRate": 0.007633587786259542 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.59932343766, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.358825259583, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.165809265202, - "hits": 0, - "total": 130, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.969377662519, - "hits": 0, - "total": 133, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.358825259583, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.288697198304, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.59932343766, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.969377662519, - "hits": 0, - "total": 133, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.013410783916, - "hits": 0, - "total": 122, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.827626146169, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -4.681848366371, - "hits": 0, - "total": 127, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.483358374511, - "hits": 0, - "total": 52, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.55248803416, - "hits": 0, - "total": 127, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 116, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.945729021813, - "hits": 0, - "total": 129, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.015268979336, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.576407109739, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.890771320266, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.788878603493, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.603974037796, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.137560045078, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 93, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 96, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-8", - "language": "he" - }, - "predicted": { - "encoding": "ISO-8859-8", - "confidence": 47, - "language": "he", - "byteLogLikelihood": -3.320333973076, - "hits": 16, - "total": 102, - "hitRate": 0.1568627450980392 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-8", - "confidence": 47, - "language": "he", - "byteLogLikelihood": -3.320333973076, - "hits": 16, - "total": 102, - "hitRate": 0.1568627450980392 - }, - { - "encoding": "windows-1255", - "confidence": 47, - "language": "he", - "byteLogLikelihood": -3.320333973076, - "hits": 16, - "total": 102, - "hitRate": 0.1568627450980392 - }, - { - "encoding": "ISO-8859-7", - "confidence": 2, - "language": "el", - "byteLogLikelihood": -3.981926233577, - "hits": 1, - "total": 102, - "hitRate": 0.00980392156862745 - }, - { - "encoding": "windows-1253", - "confidence": 2, - "language": "el", - "byteLogLikelihood": -3.981926233577, - "hits": 1, - "total": 102, - "hitRate": 0.00980392156862745 - }, - { - "encoding": "IBM855", - "confidence": 2, - "language": "ru", - "byteLogLikelihood": -5.394495827364, - "hits": 1, - "total": 102, - "hitRate": 0.00980392156862745 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.395116706452, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.461975046319, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.683415446443, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.124491051534, - "hits": 0, - "total": 73, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.941685614581, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.461975046319, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.199203901043, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.395116706452, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.941685614581, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.32364067176, - "hits": 0, - "total": 93, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.639272141503, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.029681805165, - "hits": 0, - "total": 99, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.31385343816, - "hits": 0, - "total": 48, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.40669720385, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.393692981541, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.683415446443, - "hits": 0, - "total": 94, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 72, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.134202940863, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.956632184798, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.146385148445, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.745792460039, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.842271132243, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.813633859942, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "fr", - "byteLogLikelihood": -4.521941300627, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.093070766591, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 85, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 89, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-9", - "language": "tr" - }, - "predicted": { - "encoding": "ISO-8859-9", - "confidence": 55, - "language": "tr", - "byteLogLikelihood": -3.531675216773, - "hits": 25, - "total": 134, - "hitRate": 0.1865671641791045 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-9", - "confidence": 55, - "language": "tr", - "byteLogLikelihood": -3.531675216773, - "hits": 25, - "total": 134, - "hitRate": 0.1865671641791045 - }, - { - "encoding": "windows-1254", - "confidence": 55, - "language": "tr", - "byteLogLikelihood": -3.531675216773, - "hits": 25, - "total": 134, - "hitRate": 0.1865671641791045 - }, - { - "encoding": "ISO-8859-3", - "confidence": 22, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 10, - "total": 134, - "hitRate": 0.07462686567164178 - }, - { - "encoding": "ISO-8859-1", - "confidence": 17, - "language": "da", - "byteLogLikelihood": -4.969813299576, - "hits": 8, - "total": 134, - "hitRate": 0.05970149253731343 - }, - { - "encoding": "windows-1252", - "confidence": 17, - "language": "da", - "byteLogLikelihood": -4.969813299576, - "hits": 8, - "total": 134, - "hitRate": 0.05970149253731343 - }, - { - "encoding": "ISO-8859-10", - "confidence": 17, - "language": "is", - "byteLogLikelihood": -4.720106560713, - "hits": 8, - "total": 134, - "hitRate": 0.05970149253731343 - }, - { - "encoding": "ISO-8859-2", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -4.352190516073, - "hits": 6, - "total": 134, - "hitRate": 0.04477611940298507 - }, - { - "encoding": "windows-1250", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -4.352190516073, - "hits": 6, - "total": 134, - "hitRate": 0.04477611940298507 - }, - { - "encoding": "windows-1257", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -3.613497993373, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -4.93302665818, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "ISO-8859-13", - "confidence": 11, - "language": "lt", - "byteLogLikelihood": -4.394247673508, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -4.25849981603, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-14", - "confidence": 8, - "language": "cy", - "byteLogLikelihood": -4.652724693026, - "hits": 4, - "total": 134, - "hitRate": 0.029850746268656716 - }, - { - "encoding": "windows-1258", - "confidence": 6, - "language": "vi", - "byteLogLikelihood": -5.197431316193, - "hits": 3, - "total": 133, - "hitRate": 0.022556390977443608 - }, - { - "encoding": "CP852", - "confidence": 6, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 133, - "hitRate": 0.022556390977443608 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.88736478629, - "hits": 0, - "total": 133, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.217802445218, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.216966174066, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.95929011292, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.576073075491, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.216966174066, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.95929011292, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.714883328648, - "hits": 0, - "total": 130, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.085579959517, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.88736478629, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.742947096814, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.22610845571, - "hits": 0, - "total": 134, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "KOI8-R", - "language": "ru" - }, - "predicted": { - "encoding": "KOI8-R", - "confidence": 43, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 26, - "total": 179, - "hitRate": 0.1452513966480447 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "KOI8-R", - "confidence": 43, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 26, - "total": 179, - "hitRate": 0.1452513966480447 - }, - { - "encoding": "KOI8-U", - "confidence": 28, - "language": "uk", - "byteLogLikelihood": -3.443215105753, - "hits": 17, - "total": 179, - "hitRate": 0.09497206703910614 - }, - { - "encoding": "windows-874", - "confidence": 12, - "language": "th", - "byteLogLikelihood": -4.705552828452, - "hits": 7, - "total": 162, - "hitRate": 0.043209876543209874 - }, - { - "encoding": "ISO-8859-7", - "confidence": 6, - "language": "el", - "byteLogLikelihood": -5.840779048091, - "hits": 4, - "total": 177, - "hitRate": 0.022598870056497175 - }, - { - "encoding": "windows-1253", - "confidence": 6, - "language": "el", - "byteLogLikelihood": -5.840779048091, - "hits": 4, - "total": 177, - "hitRate": 0.022598870056497175 - }, - { - "encoding": "ISO-8859-5", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -5.543217452567, - "hits": 1, - "total": 179, - "hitRate": 0.00558659217877095 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.150827322214, - "hits": 1, - "total": 179, - "hitRate": 0.00558659217877095 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "nl", - "byteLogLikelihood": -4.900684399759, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.759099059366, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.899711049132, - "hits": 0, - "total": 7, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.056032280413, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.313908864785, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "nl", - "byteLogLikelihood": -4.900684399759, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.056032280413, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.899711049132, - "hits": 0, - "total": 27, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.796335743252, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.91533239831, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.174619785116, - "hits": 0, - "total": 174, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.670016162969, - "hits": 0, - "total": 7, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.983474020054, - "hits": 0, - "total": 96, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 94, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.595875032476, - "hits": 0, - "total": 116, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.182298321669, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.105602464686, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "de", - "byteLogLikelihood": -4.935090625757, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.206584689041, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 92, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 94, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "KOI8-U", - "language": "uk" - }, - "predicted": { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 - }, - { - "encoding": "KOI8-R", - "confidence": 17, - "language": "ru", - "byteLogLikelihood": -3.510012695801, - "hits": 10, - "total": 172, - "hitRate": 0.05813953488372093 - }, - { - "encoding": "windows-874", - "confidence": 11, - "language": "th", - "byteLogLikelihood": -4.843281209678, - "hits": 6, - "total": 163, - "hitRate": 0.03680981595092025 - }, - { - "encoding": "ISO-8859-7", - "confidence": 8, - "language": "el", - "byteLogLikelihood": -5.876007286158, - "hits": 5, - "total": 168, - "hitRate": 0.02976190476190476 - }, - { - "encoding": "windows-1253", - "confidence": 8, - "language": "el", - "byteLogLikelihood": -5.876007286158, - "hits": 5, - "total": 168, - "hitRate": 0.02976190476190476 - }, - { - "encoding": "ISO-8859-1", - "confidence": 1, - "language": "en", - "byteLogLikelihood": -4.915420747009, - "hits": 1, - "total": 168, - "hitRate": 0.005952380952380952 - }, - { - "encoding": "ISO-8859-5", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -5.626926512473, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 - }, - { - "encoding": "ISO-8859-6", - "confidence": 1, - "language": "ar", - "byteLogLikelihood": -4.726706027838, - "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 - }, - { - "encoding": "windows-1252", - "confidence": 1, - "language": "en", - "byteLogLikelihood": -4.915420747009, - "hits": 1, - "total": 168, - "hitRate": 0.005952380952380952 - }, - { - "encoding": "windows-1256", - "confidence": 1, - "language": "ar", - "byteLogLikelihood": -4.693044688745, - "hits": 1, - "total": 168, - "hitRate": 0.005952380952380952 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.128376292359, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.883441366246, - "hits": 0, - "total": 7, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.055144062354, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.295907791508, - "hits": 0, - "total": 172, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -5.055144062354, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.883441366246, - "hits": 0, - "total": 27, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.922686041886, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.213223244884, - "hits": 0, - "total": 164, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.582632026409, - "hits": 0, - "total": 22, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.922515418323, - "hits": 0, - "total": 94, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 89, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.589678128375, - "hits": 0, - "total": 117, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.110283442945, - "hits": 0, - "total": 168, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-14", - "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944199711559, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.192994752161, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 92, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 94, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "macintosh", - "language": "de" - }, - "predicted": { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - { - "encoding": "ISO-8859-1", - "confidence": 74, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 53, - "total": 214, - "hitRate": 0.24766355140186916 - }, - { - "encoding": "ISO-8859-15", - "confidence": 74, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 53, - "total": 214, - "hitRate": 0.24766355140186916 - }, - { - "encoding": "windows-1252", - "confidence": 70, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 51, - "total": 216, - "hitRate": 0.2361111111111111 - }, - { - "encoding": "CP850", - "confidence": 69, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 50, - "total": 216, - "hitRate": 0.23148148148148148 - }, - { - "encoding": "ISO-8859-14", - "confidence": 22, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 16, - "total": 214, - "hitRate": 0.07476635514018691 - }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 11, - "total": 214, - "hitRate": 0.0514018691588785 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 11, - "total": 216, - "hitRate": 0.05092592592592592 - }, - { - "encoding": "ISO-8859-9", - "confidence": 14, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 10, - "total": 214, - "hitRate": 0.04672897196261682 - }, - { - "encoding": "windows-1254", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 9, - "total": 216, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-16", - "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 9, - "total": 214, - "hitRate": 0.04205607476635514 - }, - { - "encoding": "windows-1257", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 214, - "hitRate": 0.037383177570093455 - }, - { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 214, - "hitRate": 0.037383177570093455 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 - }, - { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 214, - "hitRate": 0.014018691588785047 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 214, - "hitRate": 0.009345794392523364 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.661807706795, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 214, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.534754518494, - "hits": 0, - "total": 215, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.345510152227, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.337519079447, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 216, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "macintosh", - "language": "es" - }, - "predicted": { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "ISO-8859-1", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 - }, - { - "encoding": "windows-1252", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 - }, - { - "encoding": "ISO-8859-15", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 - }, - { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 - }, - { - "encoding": "windows-1257", - "confidence": 25, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 17, - "total": 199, - "hitRate": 0.08542713567839195 - }, - { - "encoding": "ISO-8859-4", - "confidence": 25, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 17, - "total": 199, - "hitRate": 0.08542713567839195 - }, - { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 13, - "total": 199, - "hitRate": 0.06532663316582915 - }, - { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 199, - "hitRate": 0.05527638190954774 - }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 199, - "hitRate": 0.02512562814070352 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 199, - "hitRate": 0.02512562814070352 - }, - { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 5, - "total": 199, - "hitRate": 0.02512562814070352 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 199, - "hitRate": 0.01507537688442211 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -6.061456918928, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.624741392945, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 199, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "macintosh", - "language": "fr" - }, - "predicted": { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "ISO-8859-1", - "confidence": 75, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 56, - "total": 224, - "hitRate": 0.25 - }, - { - "encoding": "ISO-8859-15", - "confidence": 75, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 56, - "total": 224, - "hitRate": 0.25 - }, - { - "encoding": "windows-1252", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "CP850", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "ISO-8859-2", - "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 - }, - { - "encoding": "ISO-8859-10", - "confidence": 12, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 - }, - { - "encoding": "ISO-8859-16", - "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 - }, - { - "encoding": "windows-1250", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 224, - "hitRate": 0.03125 - }, - { - "encoding": "ISO-8859-3", - "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 224, - "hitRate": 0.026785714285714284 - }, - { - "encoding": "windows-1257", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-9", - "confidence": 6, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 224, - "hitRate": 0.022321428571428572 - }, - { - "encoding": "windows-1254", - "confidence": 6, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 224, - "hitRate": 0.022321428571428572 - }, - { - "encoding": "windows-1258", - "confidence": 6, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 5, - "total": 224, - "hitRate": 0.022321428571428572 - }, - { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 224, - "hitRate": 0.017857142857142856 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 224, - "hitRate": 0.013392857142857142 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.932245187448, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -6.061456918928, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -3.576182148392, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, - "hits": 0, - "total": 228, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1250", - "language": "cs" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 30, - "language": "cs", - "byteLogLikelihood": -3.429159108442, - "hits": 13, - "total": 126, - "hitRate": 0.10317460317460317 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 30, - "language": "cs", - "byteLogLikelihood": -3.429159108442, - "hits": 13, - "total": 126, - "hitRate": 0.10317460317460317 - }, - { - "encoding": "windows-1250", - "confidence": 30, - "language": "cs", - "byteLogLikelihood": -3.429159108442, - "hits": 13, - "total": 126, - "hitRate": 0.10317460317460317 - }, - { - "encoding": "ISO-8859-1", - "confidence": 11, - "language": "pt", - "byteLogLikelihood": -4.5670392286, - "hits": 5, - "total": 126, - "hitRate": 0.03968253968253968 - }, - { - "encoding": "windows-1252", - "confidence": 11, - "language": "pt", - "byteLogLikelihood": -4.5670392286, - "hits": 5, - "total": 126, - "hitRate": 0.03968253968253968 - }, - { - "encoding": "CP852", - "confidence": 11, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 126, - "hitRate": 0.03968253968253968 - }, - { - "encoding": "windows-1257", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -4.997937219301, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -4.506724065575, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "macintosh", - "confidence": 7, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 3, - "total": 124, - "hitRate": 0.024193548387096774 - }, - { - "encoding": "ISO-8859-13", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -4.997937219301, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "ISO-8859-15", - "confidence": 7, - "language": "fr", - "byteLogLikelihood": -4.081579721495, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "ISO-8859-16", - "confidence": 7, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "CP850", - "confidence": 7, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 3, - "total": 125, - "hitRate": 0.024 - }, - { - "encoding": "ISO-8859-9", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -4.849273210708, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "windows-1254", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -4.849273210708, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-4", - "confidence": 4, - "language": "lv", - "byteLogLikelihood": -5.129943470839, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 2, - "total": 126, - "hitRate": 0.015873015873015872 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.254067466067, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.402405269754, - "hits": 0, - "total": 124, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.401446196881, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.467509652199, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.249199684222, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.401446196881, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.467509652199, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.164526575273, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.504431122267, - "hits": 0, - "total": 122, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.566056982405, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.524252288486, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.254067466067, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.108905030169, - "hits": 0, - "total": 125, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.226561011787, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.43713857777, - "hits": 0, - "total": 126, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1250", - "language": "hu" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 46, - "language": "hu", - "byteLogLikelihood": -3.161205902203, - "hits": 21, - "total": 136, - "hitRate": 0.15441176470588236 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 46, - "language": "hu", - "byteLogLikelihood": -3.161205902203, - "hits": 21, - "total": 136, - "hitRate": 0.15441176470588236 - }, - { - "encoding": "windows-1250", - "confidence": 46, - "language": "hu", - "byteLogLikelihood": -3.161205902203, - "hits": 21, - "total": 136, - "hitRate": 0.15441176470588236 - }, - { - "encoding": "ISO-8859-1", - "confidence": 17, - "language": "pt", - "byteLogLikelihood": -4.507887190294, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 - }, - { - "encoding": "windows-1252", - "confidence": 17, - "language": "pt", - "byteLogLikelihood": -4.507887190294, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 - }, - { - "encoding": "CP852", - "confidence": 15, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 7, - "total": 135, - "hitRate": 0.05185185185185185 - }, - { - "encoding": "ISO-8859-14", - "confidence": 8, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 4, - "total": 136, - "hitRate": 0.029411764705882353 - }, - { - "encoding": "macintosh", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 3, - "total": 135, - "hitRate": 0.022222222222222223 - }, - { - "encoding": "CP850", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 3, - "total": 134, - "hitRate": 0.022388059701492536 - }, - { - "encoding": "ISO-8859-15", - "confidence": 4, - "language": "fr", - "byteLogLikelihood": -4.274525549305, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, - { - "encoding": "ISO-8859-16", - "confidence": 4, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, - { - "encoding": "ISO-8859-9", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "windows-1254", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "windows-1257", - "confidence": 2, - "language": "et", - "byteLogLikelihood": -4.453831056838, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-4", - "confidence": 2, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-10", - "confidence": 2, - "language": "is", - "byteLogLikelihood": -3.90025153387, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -4.950677184263, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.485271956182, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.538035906584, - "hits": 0, - "total": 134, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.430247904624, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.984546441118, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.6300671031, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.430247904624, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.984546441118, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.009772962617, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.010690688029, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.953752719217, - "hits": 0, - "total": 133, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.606957349802, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.485271956182, - "hits": 0, - "total": 135, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.100966544791, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.187765411941, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 136, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1250", - "language": "pl" - }, - "predicted": { - "encoding": "windows-1250", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -3.52910527172, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "windows-1250", - "confidence": 40, - "language": "pl", - "byteLogLikelihood": -3.52910527172, - "hits": 25, - "total": 184, - "hitRate": 0.1358695652173913 - }, - { - "encoding": "ISO-8859-2", - "confidence": 37, - "language": "pl", - "byteLogLikelihood": -4.297518988235, - "hits": 23, - "total": 184, - "hitRate": 0.125 - }, - { - "encoding": "CP852", - "confidence": 41, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 25, - "total": 182, - "hitRate": 0.13736263736263737 - }, - { - "encoding": "ISO-8859-1", - "confidence": 8, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 5, - "total": 183, - "hitRate": 0.0273224043715847 - }, - { - "encoding": "windows-1252", - "confidence": 8, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 5, - "total": 183, - "hitRate": 0.0273224043715847 - }, - { - "encoding": "windows-1257", - "confidence": 6, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "windows-1258", - "confidence": 6, - "language": "vi", - "byteLogLikelihood": -5.343135518502, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "ISO-8859-13", - "confidence": 6, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "ISO-8859-15", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.396558412648, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 - }, - { - "encoding": "CP850", - "confidence": 6, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 4, - "total": 182, - "hitRate": 0.02197802197802198 - }, - { - "encoding": "macintosh", - "confidence": 4, - "language": "es", - "byteLogLikelihood": -4.826691858304, - "hits": 3, - "total": 184, - "hitRate": 0.016304347826086956 - }, - { - "encoding": "ISO-8859-4", - "confidence": 4, - "language": "lv", - "byteLogLikelihood": -4.704684722593, - "hits": 3, - "total": 184, - "hitRate": 0.016304347826086956 - }, - { - "encoding": "ISO-8859-3", - "confidence": 3, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 2, - "total": 184, - "hitRate": 0.010869565217391304 - }, - { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -4.71573455878, - "hits": 2, - "total": 184, - "hitRate": 0.010869565217391304 - }, - { - "encoding": "ISO-8859-16", - "confidence": 3, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 2, - "total": 184, - "hitRate": 0.010869565217391304 - }, - { - "encoding": "ISO-8859-9", - "confidence": 1, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 1, - "total": 183, - "hitRate": 0.00546448087431694 - }, - { - "encoding": "windows-1254", - "confidence": 1, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 1, - "total": 183, - "hitRate": 0.00546448087431694 - }, - { - "encoding": "ISO-8859-14", - "confidence": 1, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 1, - "total": 184, - "hitRate": 0.005434782608695652 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.293914971567, - "hits": 0, - "total": 184, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.848389618517, - "hits": 0, - "total": 184, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.470147067075, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.300217461931, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.848389618517, - "hits": 0, - "total": 184, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -5.470147067075, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 182, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.163001081631, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.538051035716, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.496246341796, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.447947678358, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.204348837508, - "hits": 0, - "total": 183, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.955485669412, - "hits": 0, - "total": 184, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1250", - "language": "ro" - }, - "predicted": { - "encoding": "ISO-8859-2", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-2", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - { - "encoding": "windows-1250", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - { - "encoding": "ISO-8859-16", - "confidence": 38, - "language": "ro", - "byteLogLikelihood": -2.700458868498, - "hits": 26, - "total": 202, - "hitRate": 0.12871287128712872 - }, - { - "encoding": "ISO-8859-1", - "confidence": 26, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 18, - "total": 202, - "hitRate": 0.0891089108910891 - }, - { - "encoding": "windows-1252", - "confidence": 26, - "language": "it", - "byteLogLikelihood": -4.919980925828, - "hits": 18, - "total": 202, - "hitRate": 0.0891089108910891 - }, - { - "encoding": "macintosh", - "confidence": 19, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 198, - "hitRate": 0.06565656565656566 - }, - { - "encoding": "CP850", - "confidence": 15, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 - }, - { - "encoding": "ISO-8859-15", - "confidence": 14, - "language": "fr", - "byteLogLikelihood": -4.903997266318, - "hits": 10, - "total": 202, - "hitRate": 0.04950495049504951 - }, - { - "encoding": "ISO-8859-3", - "confidence": 13, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 9, - "total": 200, - "hitRate": 0.045 - }, - { - "encoding": "ISO-8859-9", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.021854801345, - "hits": 7, - "total": 202, - "hitRate": 0.034653465346534656 - }, - { - "encoding": "windows-1254", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.021854801345, - "hits": 7, - "total": 202, - "hitRate": 0.034653465346534656 - }, - { - "encoding": "windows-1257", - "confidence": 8, - "language": "lv", - "byteLogLikelihood": -4.396820338954, - "hits": 6, - "total": 202, - "hitRate": 0.0297029702970297 - }, - { - "encoding": "CP852", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 201, - "hitRate": 0.029850746268656716 - }, - { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -4.936633427031, - "hits": 5, - "total": 202, - "hitRate": 0.024752475247524754 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.177254141958, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 - }, - { - "encoding": "ISO-8859-10", - "confidence": 5, - "language": "is", - "byteLogLikelihood": -5.150687673188, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 - }, - { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.974278583322, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 - }, - { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.539421788511, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.208609169552, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.828974536143, - "hits": 0, - "total": 199, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.33505338151, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.197813655274, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.33505338151, - "hits": 0, - "total": 202, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.197813655274, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.528215037204, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.239455511679, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.42548867306, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.32130473174, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.208609169552, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.608648280371, - "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, - "hits": 0, - "total": 202, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1251", - "language": "ru" - }, - "predicted": { - "encoding": "windows-1251", - "confidence": 41, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 25, - "total": 179, - "hitRate": 0.13966480446927373 - }, - "encodingExact": true, - "encodingEquivalent": false, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "windows-1251", - "confidence": 41, - "language": "ru", - "byteLogLikelihood": -3.313760777855, - "hits": 25, - "total": 179, - "hitRate": 0.13966480446927373 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 34, - "language": "ru", - "byteLogLikelihood": -3.411211853364, - "hits": 20, - "total": 175, - "hitRate": 0.11428571428571428 - }, - { - "encoding": "IBM855", - "confidence": 9, - "language": "sr", - "byteLogLikelihood": -5.604243216214, - "hits": 5, - "total": 164, - "hitRate": 0.03048780487804878 - }, - { - "encoding": "KOI8-U", - "confidence": 5, - "language": "uk", - "byteLogLikelihood": -6.290472763796, - "hits": 3, - "total": 179, - "hitRate": 0.01675977653631285 - }, - { - "encoding": "ISO-8859-8", - "confidence": 3, - "language": "he", - "byteLogLikelihood": -3.941595102268, - "hits": 2, - "total": 167, - "hitRate": 0.011976047904191617 - }, - { - "encoding": "windows-1255", - "confidence": 3, - "language": "he", - "byteLogLikelihood": -3.941595102268, - "hits": 2, - "total": 167, - "hitRate": 0.011976047904191617 - }, - { - "encoding": "windows-1256", - "confidence": 1, - "language": "ar", - "byteLogLikelihood": -5.552676456081, - "hits": 1, - "total": 165, - "hitRate": 0.006060606060606061 - }, - { - "encoding": "KOI8-R", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -6.346028593552, - "hits": 1, - "total": 179, - "hitRate": 0.00558659217877095 - }, - { - "encoding": "ISO-8859-14", - "confidence": 1, - "language": "cy", - "byteLogLikelihood": -4.724449359889, - "hits": 1, - "total": 179, - "hitRate": 0.00558659217877095 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.665227236998, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.845647442888, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.054854043559, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.827625181516, - "hits": 0, - "total": 129, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.209243802684, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.950903003627, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.845647442888, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.665227236998, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.209243802684, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.950903003627, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.61521723683, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -4.899487478125, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.446015416921, - "hits": 0, - "total": 88, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.059444422239, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 150, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.904224209153, - "hits": 0, - "total": 172, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.79659105312, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.939249573759, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.66408545306, - "hits": 0, - "total": 176, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "fr", - "byteLogLikelihood": -4.760478683076, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.970632247101, - "hits": 0, - "total": 179, - "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 136, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 155, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "da" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 52, - "language": "da", - "byteLogLikelihood": -3.146558879633, - "hits": 21, - "total": 120, - "hitRate": 0.175 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 52, - "language": "da", - "byteLogLikelihood": -3.146558879633, - "hits": 21, - "total": 120, - "hitRate": 0.175 - }, - { - "encoding": "windows-1252", - "confidence": 52, - "language": "da", - "byteLogLikelihood": -3.146558879633, - "hits": 21, - "total": 120, - "hitRate": 0.175 - }, - { - "encoding": "macintosh", - "confidence": 30, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 12, - "total": 120, - "hitRate": 0.1 - }, - { - "encoding": "ISO-8859-15", - "confidence": 30, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 12, - "total": 120, - "hitRate": 0.1 - }, - { - "encoding": "CP850", - "confidence": 30, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 12, - "total": 120, - "hitRate": 0.1 - }, - { - "encoding": "ISO-8859-14", - "confidence": 15, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 6, - "total": 120, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 6, - "total": 120, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-2", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "ISO-8859-9", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "windows-1250", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "windows-1254", - "confidence": 10, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "windows-1257", - "confidence": 10, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 3, - "total": 120, - "hitRate": 0.025 - }, - { - "encoding": "ISO-8859-10", - "confidence": 7, - "language": "is", - "byteLogLikelihood": -4.717529149813, - "hits": 3, - "total": 120, - "hitRate": 0.025 - }, - { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 120, - "hitRate": 0.016666666666666666 - }, - { - "encoding": "ISO-8859-3", - "confidence": 2, - "language": "mt", - "byteLogLikelihood": -4.658005606594, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 - }, - { - "encoding": "ISO-8859-4", - "confidence": 2, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -4.145958228046, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.825649766807, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.736335095675, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.689401143463, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.231348206195, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.288232392491, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.689401143463, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.231348206195, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.861023356308, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.888170123788, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.518796947367, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.476992253447, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.825649766807, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.685745819823, - "hits": 0, - "total": 120, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.288232392491, - "hits": 0, - "total": 120, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "de" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 3, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - { - "encoding": "windows-1252", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - { - "encoding": "ISO-8859-15", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 - }, - { - "encoding": "macintosh", - "confidence": 72, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 52, - "total": 215, - "hitRate": 0.24186046511627907 - }, - { - "encoding": "CP850", - "confidence": 70, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 51, - "total": 216, - "hitRate": 0.2361111111111111 - }, - { - "encoding": "ISO-8859-14", - "confidence": 20, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 15, - "total": 216, - "hitRate": 0.06944444444444445 - }, - { - "encoding": "ISO-8859-2", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "windows-1250", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "ISO-8859-9", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 9, - "total": 216, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "windows-1254", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -4.5258676709, - "hits": 9, - "total": 216, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 216, - "hitRate": 0.037037037037037035 - }, - { - "encoding": "windows-1257", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 - }, - { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 216, - "hitRate": 0.032407407407407406 - }, - { - "encoding": "CP852", - "confidence": 6, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 216, - "hitRate": 0.023148148148148147 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 216, - "hitRate": 0.009259259259259259 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 1, - "total": 216, - "hitRate": 0.004629629629629629 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.162408104115, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.007313397456, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.349754587225, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.279537403415, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.542402054434, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.349754587225, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.279537403415, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.318835143108, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.657121063601, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.28774788718, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.245943193261, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.942998477092, - "hits": 0, - "total": 216, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.629332579687, - "hits": 0, - "total": 216, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "en" - }, - "predicted": { - "encoding": "ISO-8859-15", - "confidence": 29, - "language": "en", - "byteLogLikelihood": -2.06619631493, - "hits": 13, - "total": 132, - "hitRate": 0.09848484848484848 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-15", - "confidence": 29, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 13, - "total": 132, - "hitRate": 0.09848484848484848 - }, - { - "encoding": "ISO-8859-1", - "confidence": 47, - "language": "en", - "byteLogLikelihood": -3.310543013394, - "hits": 21, - "total": 132, - "hitRate": 0.1590909090909091 - }, - { - "encoding": "windows-1252", - "confidence": 47, - "language": "en", - "byteLogLikelihood": -3.310543013394, - "hits": 21, - "total": 132, - "hitRate": 0.1590909090909091 - }, - { - "encoding": "macintosh", - "confidence": 27, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 12, - "total": 132, - "hitRate": 0.09090909090909091 - }, - { - "encoding": "CP850", - "confidence": 27, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 12, - "total": 132, - "hitRate": 0.09090909090909091 - }, - { - "encoding": "windows-1257", - "confidence": 11, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-14", - "confidence": 11, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-2", - "confidence": 9, - "language": "cs", - "byteLogLikelihood": -3.54961738678, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "windows-1250", - "confidence": 9, - "language": "cs", - "byteLogLikelihood": -3.54961738678, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "windows-1258", - "confidence": 9, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "ISO-8859-16", - "confidence": 9, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 4, - "total": 132, - "hitRate": 0.030303030303030304 - }, - { - "encoding": "ISO-8859-4", - "confidence": 6, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 132, - "hitRate": 0.022727272727272728 - }, - { - "encoding": "ISO-8859-9", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "windows-1254", - "confidence": 4, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "CP852", - "confidence": 4, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 132, - "hitRate": 0.015151515151515152 - }, - { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 1, - "total": 132, - "hitRate": 0.007575757575757576 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.612642200827, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.57655026914, - "hits": 0, - "total": 131, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, - "hits": 0, - "total": 132, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 0, - "total": 132, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "es" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 3, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "windows-1252", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "macintosh", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 - }, - { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 - }, - { - "encoding": "windows-1257", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 16, - "total": 200, - "hitRate": 0.08 - }, - { - "encoding": "ISO-8859-4", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.009854802779, - "hits": 16, - "total": 200, - "hitRate": 0.08 - }, - { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -4.892914919217, - "hits": 13, - "total": 200, - "hitRate": 0.065 - }, - { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 200, - "hitRate": 0.055 - }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 10, - "total": 200, - "hitRate": 0.05 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 - }, - { - "encoding": "windows-1258", - "confidence": 4, - "language": "vi", - "byteLogLikelihood": -4.848531508941, - "hits": 3, - "total": 200, - "hitRate": 0.015 - }, - { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -3.846028832089, - "hits": 3, - "total": 200, - "hitRate": 0.015 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.977292083619, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.346153815501, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.106555532888, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.411159813207, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.106555532888, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.411159813207, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.300604963183, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.412820202576, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.977292083619, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.203740544852, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 0, - "total": 200, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "fr" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 3, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "windows-1252", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "ISO-8859-15", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - { - "encoding": "macintosh", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "CP850", - "confidence": 71, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 - }, - { - "encoding": "ISO-8859-2", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "windows-1250", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "ISO-8859-10", - "confidence": 10, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 - }, - { - "encoding": "ISO-8859-16", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 - }, - { - "encoding": "windows-1257", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-9", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "windows-1254", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "windows-1258", - "confidence": 5, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 - }, - { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 228, - "hitRate": 0.013157894736842105 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.612642200827, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.57655026914, - "hits": 0, - "total": 224, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, - "hits": 0, - "total": 228, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, - "hits": 0, - "total": 228, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "it" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 58, - "language": "it", - "byteLogLikelihood": -3.533686564708, - "hits": 31, - "total": 159, - "hitRate": 0.1949685534591195 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 58, - "language": "it", - "byteLogLikelihood": -3.533686564708, - "hits": 31, - "total": 159, - "hitRate": 0.1949685534591195 - }, - { - "encoding": "windows-1252", - "confidence": 58, - "language": "it", - "byteLogLikelihood": -3.533686564708, - "hits": 31, - "total": 159, - "hitRate": 0.1949685534591195 - }, - { - "encoding": "macintosh", - "confidence": 24, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 158, - "hitRate": 0.08227848101265822 - }, - { - "encoding": "ISO-8859-15", - "confidence": 24, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 159, - "hitRate": 0.08176100628930817 - }, - { - "encoding": "CP850", - "confidence": 24, - "language": "fr", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 159, - "hitRate": 0.08176100628930817 - }, - { - "encoding": "windows-1258", - "confidence": 16, - "language": "vi", - "byteLogLikelihood": -3.012261575505, - "hits": 9, - "total": 159, - "hitRate": 0.05660377358490566 - }, - { - "encoding": "ISO-8859-16", - "confidence": 16, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 9, - "total": 159, - "hitRate": 0.05660377358490566 - }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "windows-1257", - "confidence": 15, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "ISO-8859-4", - "confidence": 15, - "language": "lv", - "byteLogLikelihood": -1.974081026022, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, - { - "encoding": "ISO-8859-14", - "confidence": 9, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 5, - "total": 159, - "hitRate": 0.031446540880503145 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 159, - "hitRate": 0.025157232704402517 - }, - { - "encoding": "ISO-8859-9", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 159, - "hitRate": 0.018867924528301886 - }, - { - "encoding": "windows-1254", - "confidence": 5, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 159, - "hitRate": 0.018867924528301886 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 159, - "hitRate": 0.018867924528301886 - }, - { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 159, - "hitRate": 0.012578616352201259 - }, - { - "encoding": "ISO-8859-13", - "confidence": 3, - "language": "lt", - "byteLogLikelihood": -3.332204510175, - "hits": 2, - "total": 159, - "hitRate": 0.012578616352201259 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -2.954414124223, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.629660094454, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -2.533200659147, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -6.120297418951, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.629660094454, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.496507561466, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -2.954414124223, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.520621127559, - "hits": 0, - "total": 159, - "hitRate": 0 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -2.533200659147, - "hits": 0, - "total": 159, - "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "windows-1252", - "language": "nl" - }, - "predicted": { - "encoding": "ISO-8859-1", - "confidence": 83, - "language": "nl", - "byteLogLikelihood": -3.295836866004, - "hits": 42, - "total": 151, - "hitRate": 0.2781456953642384 - }, - "encodingExact": false, - "encodingEquivalent": true, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 2, - "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 83, - "language": "nl", - "byteLogLikelihood": -3.295836866004, - "hits": 42, - "total": 151, - "hitRate": 0.2781456953642384 - }, - { - "encoding": "windows-1252", - "confidence": 83, - "language": "nl", - "byteLogLikelihood": -3.295836866004, - "hits": 42, - "total": 151, - "hitRate": 0.2781456953642384 - }, - { - "encoding": "macintosh", - "confidence": 63, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 32, - "total": 151, - "hitRate": 0.2119205298013245 - }, - { - "encoding": "ISO-8859-15", - "confidence": 63, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 32, - "total": 151, - "hitRate": 0.2119205298013245 - }, - { - "encoding": "CP850", - "confidence": 63, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 32, - "total": 151, - "hitRate": 0.2119205298013245 - }, - { - "encoding": "ISO-8859-14", - "confidence": 33, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 17, - "total": 151, - "hitRate": 0.11258278145695365 - }, - { - "encoding": "ISO-8859-2", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, - { - "encoding": "ISO-8859-9", - "confidence": 13, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, - { - "encoding": "windows-1250", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, - { - "encoding": "windows-1254", - "confidence": 13, - "language": "tr", - "byteLogLikelihood": -5.075173815234, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, - { - "encoding": "ISO-8859-10", - "confidence": 11, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 6, - "total": 151, - "hitRate": 0.039735099337748346 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 6, - "total": 151, - "hitRate": 0.039735099337748346 - }, - { - "encoding": "windows-1257", - "confidence": 9, - "language": "et", - "byteLogLikelihood": -4.934473933131, - "hits": 5, - "total": 151, - "hitRate": 0.033112582781456956 - }, - { - "encoding": "windows-1258", - "confidence": 3, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 2, - "total": 151, - "hitRate": 0.013245033112582781 - }, - { - "encoding": "ISO-8859-13", - "confidence": 3, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 151, - "hitRate": 0.013245033112582781 - }, - { - "encoding": "ISO-8859-3", - "confidence": 1, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 1, - "total": 151, - "hitRate": 0.006622516556291391 - }, - { - "encoding": "ISO-8859-4", - "confidence": 1, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 151, - "hitRate": 0.006622516556291391 - }, - { - "encoding": "CP852", - "confidence": 1, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 1, - "total": 151, - "hitRate": 0.006622516556291391 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.612642200827, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -2.654561516151, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -2.405884662832, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 151, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -3.57655026914, - "hits": 0, - "total": 150, - "hitRate": 0 + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 }, { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, - "hits": 0, - "total": 151, - "hitRate": 0 + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 16, + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 199, + "hitRate": 0.05527638190954774 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 151, - "hitRate": 0 + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 }, { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.593471453839, - "hits": 0, - "total": 151, - "hitRate": 0 + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 }, { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, - "hits": 0, - "total": 151, - "hitRate": 0 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 7, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 }, { - "encoding": "x-mac-cyrillic", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, - "hits": 0, - "total": 151, - "hitRate": 0 + "encoding": "windows-1254", + "language": "tr", + "confidence": 7, + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "CP852", + "language": "pl", + "confidence": 7, + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 } ] }, { "expected": { - "encoding": "windows-1252", - "language": "no" + "encoding": "macintosh", + "language": "fr" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 47, - "language": "no", - "byteLogLikelihood": -3.052495812716, - "hits": 18, - "total": 114, - "hitRate": 0.15789473684210525 + "encoding": "macintosh", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 2, + "tiedAtTop": 1, "candidates": [ - { - "encoding": "ISO-8859-1", - "confidence": 47, - "language": "no", - "byteLogLikelihood": -3.052495812716, - "hits": 18, - "total": 114, - "hitRate": 0.15789473684210525 - }, - { - "encoding": "windows-1252", - "confidence": 47, - "language": "no", - "byteLogLikelihood": -3.052495812716, - "hits": 18, - "total": 114, - "hitRate": 0.15789473684210525 - }, { "encoding": "macintosh", - "confidence": 18, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-14", - "confidence": 18, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 + "encoding": "windows-1252", + "language": "fr", + "confidence": 71, + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 }, { "encoding": "ISO-8859-15", - "confidence": 18, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 + "language": "fr", + "confidence": 75, + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 }, { - "encoding": "CP850", - "confidence": 18, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 7, - "total": 114, - "hitRate": 0.06140350877192982 + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 75, + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 }, { - "encoding": "ISO-8859-10", - "confidence": 13, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 5, - "total": 114, - "hitRate": 0.043859649122807015 + "encoding": "CP850", + "language": "fr", + "confidence": 71, + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 }, { - "encoding": "windows-1258", - "confidence": 10, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 4, - "total": 114, - "hitRate": 0.03508771929824561 + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 12, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 }, { - "encoding": "ISO-8859-16", - "confidence": 10, + "encoding": "windows-1250", "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 4, - "total": 114, - "hitRate": 0.03508771929824561 + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 }, { - "encoding": "ISO-8859-2", + "encoding": "windows-1257", + "language": "lv", "confidence": 7, - "language": "cs", - "byteLogLikelihood": -4.354336342997, - "hits": 3, - "total": 114, + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, "hitRate": 0.02631578947368421 }, { "encoding": "ISO-8859-9", - "confidence": 7, "language": "tr", + "confidence": 6, "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "windows-1250", - "confidence": 7, - "language": "cs", - "byteLogLikelihood": -4.354336342997, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 }, { "encoding": "windows-1254", - "confidence": 7, "language": "tr", + "confidence": 6, "byteLogLikelihood": -5.075173815234, - "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 }, { - "encoding": "windows-1257", - "confidence": 7, - "language": "et", - "byteLogLikelihood": -4.934473933131, + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 4, + "byteLogLikelihood": -4.912654885736, "hits": 3, - "total": 114, - "hitRate": 0.02631578947368421 - }, - { - "encoding": "ISO-8859-3", - "confidence": 2, - "language": "mt", - "byteLogLikelihood": -4.210065739287, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 + "total": 224, + "hitRate": 0.013392857142857142 }, { - "encoding": "ISO-8859-4", - "confidence": 2, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 + "encoding": "IBM855", + "language": "sr", + "confidence": 0, + "byteLogLikelihood": -3.576182148392, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 }, { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -4.319245023186, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 + "encoding": "windows-1250", + "language": "cs", + "confidence": 30, + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 }, { - "encoding": "CP852", - "confidence": 2, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 1, - "total": 114, - "hitRate": 0.008771929824561403 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 }, { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.652362971667, - "hits": 0, - "total": 114, - "hitRate": 0 + "encoding": "windows-1252", + "language": "pt", + "confidence": 11, + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 }, { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.641462690249, - "hits": 0, - "total": 114, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 7, + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -4.154384602589, + "confidence": 0, + "byteLogLikelihood": -3.401446196881, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.647504477241, + "confidence": 0, + "byteLogLikelihood": -3.467509652199, "hits": 0, - "total": 114, + "total": 125, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -3.753215851617, + "confidence": 0, + "byteLogLikelihood": -4.249199684222, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", - "byteLogLikelihood": -4.154384602589, + "confidence": 0, + "byteLogLikelihood": -3.401446196881, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -2.647504477241, + "confidence": 0, + "byteLogLikelihood": -3.467509652199, "hits": 0, - "total": 114, + "total": 125, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.217802445218, + "byteLogLikelihood": -4.226561011787, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "th", - "byteLogLikelihood": -6.061456918928, + "byteLogLikelihood": -4.43713857777, "hits": 0, - "total": 114, + "total": 126, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 }, { - "encoding": "KOI8-R", + "encoding": "windows-1250", + "language": "hu", + "confidence": 46, + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "language": "pt", + "confidence": 17, + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 4, + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "windows-1257", + "language": "et", + "confidence": 2, + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 2, + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -3.430247904624, "hits": 0, - "total": 114, + "total": 136, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "byteLogLikelihood": -3.984546441118, "hits": 0, - "total": 114, + "total": 135, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.652362971667, + "byteLogLikelihood": -3.430247904624, "hits": 0, - "total": 114, + "total": 136, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.679407897139, + "byteLogLikelihood": -3.984546441118, "hits": 0, - "total": 114, + "total": 135, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.753215851617, + "byteLogLikelihood": -4.187765411941, "hits": 0, - "total": 114, + "total": 136, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1252", - "language": "pt" + "encoding": "windows-1250", + "language": "pl" }, "predicted": { - "encoding": "ISO-8859-15", - "confidence": 33, - "language": "pt", - "byteLogLikelihood": -3.970754139021, - "hits": 16, - "total": 143, - "hitRate": 0.11188811188811189 + "encoding": "windows-1250", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-15", - "confidence": 33, - "language": "es", - "byteLogLikelihood": -3.970754139021, - "hits": 16, - "total": 143, - "hitRate": 0.11188811188811189 + "encoding": "windows-1250", + "language": "pl", + "confidence": 40, + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "ISO-8859-2", + "language": "pl", + "confidence": 37, + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 184, + "hitRate": 0.125 }, { "encoding": "ISO-8859-1", - "confidence": 50, - "language": "pt", - "byteLogLikelihood": -4.059947322987, - "hits": 24, - "total": 143, - "hitRate": 0.16783216783216784 + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 }, { "encoding": "windows-1252", - "confidence": 50, - "language": "pt", - "byteLogLikelihood": -4.059947322987, - "hits": 24, - "total": 143, - "hitRate": 0.16783216783216784 + "language": "it", + "confidence": 8, + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 }, { "encoding": "windows-1257", - "confidence": 29, - "language": "lt", - "byteLogLikelihood": -4.777390389123, - "hits": 14, - "total": 143, - "hitRate": 0.0979020979020979 - }, - { - "encoding": "ISO-8859-13", - "confidence": 29, - "language": "lt", - "byteLogLikelihood": -4.777390389123, - "hits": 14, - "total": 143, - "hitRate": 0.0979020979020979 + "language": "et", + "confidence": 6, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 }, { - "encoding": "macintosh", - "confidence": 31, + "encoding": "ISO-8859-15", "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 15, - "total": 143, - "hitRate": 0.1048951048951049 + "confidence": 6, + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 }, { - "encoding": "CP850", - "confidence": 31, + "encoding": "macintosh", "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 15, - "total": 143, - "hitRate": 0.1048951048951049 + "confidence": 4, + "byteLogLikelihood": -4.826691858304, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 }, { "encoding": "ISO-8859-4", - "confidence": 23, "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 11, - "total": 143, - "hitRate": 0.07692307692307693 - }, - { - "encoding": "ISO-8859-2", - "confidence": 10, - "language": "hu", - "byteLogLikelihood": -3.361609233757, - "hits": 5, - "total": 143, - "hitRate": 0.03496503496503497 - }, - { - "encoding": "windows-1250", - "confidence": 10, - "language": "hu", - "byteLogLikelihood": -3.361609233757, - "hits": 5, - "total": 143, - "hitRate": 0.03496503496503497 - }, - { - "encoding": "ISO-8859-14", - "confidence": 10, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 5, - "total": 143, - "hitRate": 0.03496503496503497 - }, - { - "encoding": "windows-1258", - "confidence": 6, - "language": "vi", - "byteLogLikelihood": -4.524213150766, + "confidence": 4, + "byteLogLikelihood": -4.704684722593, "hits": 3, - "total": 143, - "hitRate": 0.02097902097902098 + "total": 184, + "hitRate": 0.016304347826086956 }, { "encoding": "ISO-8859-10", - "confidence": 6, "language": "is", - "byteLogLikelihood": -4.06291963921, - "hits": 3, - "total": 143, - "hitRate": 0.02097902097902098 - }, - { - "encoding": "ISO-8859-9", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.58869627797, - "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 - }, - { - "encoding": "windows-1254", - "confidence": 2, - "language": "tr", - "byteLogLikelihood": -4.58869627797, - "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 - }, - { - "encoding": "ISO-8859-16", - "confidence": 2, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 + "confidence": 3, + "byteLogLikelihood": -4.71573455878, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 }, { - "encoding": "CP852", - "confidence": 2, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.912654885736, "hits": 1, - "total": 143, - "hitRate": 0.006993006993006993 + "total": 184, + "hitRate": 0.005434782608695652 }, { - "encoding": "ISO-8859-5", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.338763438513, + "byteLogLikelihood": -4.848389618517, "hits": 0, - "total": 143, + "total": 184, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.514449766028, + "byteLogLikelihood": -4.848389618517, "hits": 0, - "total": 143, + "total": 184, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 }, { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.017269736716, - "hits": 0, - "total": 143, - "hitRate": 0 + "encoding": "windows-1250", + "language": "ro", + "confidence": 41, + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 }, { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.50352863905, - "hits": 0, - "total": 143, - "hitRate": 0 + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 38, + "byteLogLikelihood": -2.700458868498, + "hits": 26, + "total": 202, + "hitRate": 0.12871287128712872 }, { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.846220145127, - "hits": 0, - "total": 143, - "hitRate": 0 + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 }, { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.017269736716, - "hits": 0, - "total": 143, - "hitRate": 0 + "encoding": "windows-1252", + "language": "it", + "confidence": 26, + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 }, { - "encoding": "windows-1255", + "encoding": "ISO-8859-5", + "language": "ru", "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.50352863905, + "byteLogLikelihood": -4.208609169552, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.503504267293, + "byteLogLikelihood": -4.33505338151, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.91521504926, + "byteLogLikelihood": -4.197813655274, "hits": 0, - "total": 143, + "total": 201, "hitRate": 0 }, { - "encoding": "KOI8-R", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -6.518796947367, + "confidence": 0, + "byteLogLikelihood": -4.229886082374, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.476992253447, + "byteLogLikelihood": -4.197813655274, "hits": 0, - "total": 143, + "total": 201, "hitRate": 0 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.338763438513, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "IBM855", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -3.774998078834, + "byteLogLikelihood": -4.208609169552, "hits": 0, - "total": 143, + "total": 201, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.766179593299, - "hits": 0, - "total": 143, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "byteLogLikelihood": -4.229886082374, "hits": 0, - "total": 143, + "total": 202, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1252", - "language": "sv" + "encoding": "windows-1251", + "language": "ru" }, "predicted": { - "encoding": "ISO-8859-1", - "confidence": 50, - "language": "sv", - "byteLogLikelihood": -3.23645137845, - "hits": 23, - "total": 136, - "hitRate": 0.16911764705882354 + "encoding": "windows-1251", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 }, - "encodingExact": false, - "encodingEquivalent": true, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 2, + "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-1", - "confidence": 50, - "language": "sv", - "byteLogLikelihood": -3.23645137845, - "hits": 23, - "total": 136, - "hitRate": 0.16911764705882354 + "encoding": "windows-1251", + "language": "ru", + "confidence": 41, + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 }, { - "encoding": "windows-1252", - "confidence": 50, - "language": "sv", - "byteLogLikelihood": -3.23645137845, - "hits": 23, - "total": 136, - "hitRate": 0.16911764705882354 + "encoding": "x-mac-cyrillic", + "language": "ru", + "confidence": 34, + "byteLogLikelihood": -3.411211853364, + "hits": 20, + "total": 175, + "hitRate": 0.11428571428571428 }, { - "encoding": "windows-1258", - "confidence": 19, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 9, - "total": 136, - "hitRate": 0.0661764705882353 + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 }, { - "encoding": "ISO-8859-10", - "confidence": 19, - "language": "is", - "byteLogLikelihood": -4.980036753423, - "hits": 9, - "total": 136, - "hitRate": 0.0661764705882353 + "encoding": "windows-1255", + "language": "he", + "confidence": 3, + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 }, { - "encoding": "macintosh", - "confidence": 17, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.724449359889, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 }, { - "encoding": "ISO-8859-15", - "confidence": 17, - "language": "de", - "byteLogLikelihood": -4.137512330246, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 17, - "language": "de", - "byteLogLikelihood": -5.003946305945, - "hits": 8, - "total": 136, - "hitRate": 0.058823529411764705 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 }, { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "cs", - "byteLogLikelihood": -5.159055299215, - "hits": 7, - "total": 136, - "hitRate": 0.051470588235294115 + "encoding": "windows-1252", + "language": "sv", + "confidence": 0, + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 }, { - "encoding": "windows-1250", - "confidence": 15, - "language": "cs", - "byteLogLikelihood": -5.159055299215, - "hits": 7, - "total": 136, - "hitRate": 0.051470588235294115 + "encoding": "windows-1253", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "language": "lv", + "confidence": 0, + "byteLogLikelihood": -4.61521723683, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 0, + "byteLogLikelihood": -4.66408545306, + "hits": 0, + "total": 176, + "hitRate": 0 }, { - "encoding": "ISO-8859-9", - "confidence": 11, - "language": "tr", - "byteLogLikelihood": -4.93784727915, - "hits": 5, - "total": 136, - "hitRate": 0.03676470588235294 + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 0, + "byteLogLikelihood": -4.760478683076, + "hits": 0, + "total": 179, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "da" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "windows-1254", - "confidence": 11, - "language": "tr", - "byteLogLikelihood": -4.93784727915, - "hits": 5, - "total": 136, - "hitRate": 0.03676470588235294 + "encoding": "windows-1252", + "language": "da", + "confidence": 52, + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "ISO-8859-14", - "confidence": 8, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 10, + "byteLogLikelihood": -5.176149732574, "hits": 4, - "total": 136, - "hitRate": 0.029411764705882353 + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "ISO-8859-16", - "confidence": 6, + "encoding": "windows-1250", "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 3, - "total": 136, - "hitRate": 0.022058823529411766 + "confidence": 10, + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { "encoding": "windows-1257", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "language": "et", + "confidence": 10, + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { "encoding": "ISO-8859-3", - "confidence": 4, "language": "mt", - "byteLogLikelihood": -4.210065739287, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "confidence": 2, + "byteLogLikelihood": -4.658005606594, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 }, { "encoding": "ISO-8859-13", - "confidence": 4, "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, - { - "encoding": "CP852", - "confidence": 4, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 + "confidence": 2, + "byteLogLikelihood": -4.145958228046, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 }, { - "encoding": "ISO-8859-5", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.652362971667, + "byteLogLikelihood": -4.689401143463, "hits": 0, - "total": 136, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -3.287368473007, + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 136, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.876181596107, + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 136, + "total": 120, "hitRate": 0 }, { - "encoding": "ISO-8859-8", - "confidence": 0, + "encoding": "windows-1255", "language": "he", - "byteLogLikelihood": -3.066153183614, + "confidence": 0, + "byteLogLikelihood": -3.231348206195, "hits": 0, - "total": 136, + "total": 120, "hitRate": 0 }, { - "encoding": "windows-1251", - "confidence": 0, + "encoding": "x-mac-cyrillic", "language": "ru", - "byteLogLikelihood": -3.393743210995, + "confidence": 0, + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 136, + "total": 120, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -3.876181596107, - "hits": 0, - "total": 136, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -3.066153183614, - "hits": 0, - "total": 136, - "hitRate": 0 + "encoding": "windows-1252", + "language": "de", + "confidence": 73, + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 }, { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.25594643617, - "hits": 0, - "total": 136, - "hitRate": 0 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 }, { - "encoding": "windows-874", + "encoding": "windows-1257", + "language": "lv", + "confidence": 9, + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.801526726218, + "byteLogLikelihood": -4.007313397456, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.432153549797, + "byteLogLikelihood": -4.349754587225, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.390348855877, + "byteLogLikelihood": -4.279537403415, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.652362971667, + "byteLogLikelihood": -4.349754587225, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.419477704429, + "byteLogLikelihood": -4.279537403415, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1256", + "language": "ar", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.393743210995, + "byteLogLikelihood": -4.318835143108, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "byteLogLikelihood": -3.629332579687, "hits": 0, - "total": 136, + "total": 216, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1253", - "language": "el" + "encoding": "windows-1252", + "language": "en" }, "predicted": { - "encoding": "ISO-8859-7", - "confidence": 51, - "language": "el", - "byteLogLikelihood": -3.435199502, - "hits": 23, - "total": 134, - "hitRate": 0.17164179104477612 + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, "encodingExact": false, "encodingEquivalent": true, @@ -17180,308 +6238,388 @@ "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-7", - "confidence": 51, - "language": "el", - "byteLogLikelihood": -3.435199502, - "hits": 23, - "total": 134, - "hitRate": 0.17164179104477612 + "encoding": "ISO-8859-1", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, { - "encoding": "windows-1253", - "confidence": 51, - "language": "el", - "byteLogLikelihood": -3.435199502, - "hits": 23, - "total": 134, - "hitRate": 0.17164179104477612 + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 29, + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 }, { - "encoding": "KOI8-U", - "confidence": 8, - "language": "uk", - "byteLogLikelihood": -6.254780178237, - "hits": 4, - "total": 134, - "hitRate": 0.029850746268656716 + "encoding": "windows-1252", + "language": "en", + "confidence": 47, + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, { - "encoding": "IBM855", - "confidence": 7, - "language": "ru", - "byteLogLikelihood": -5.358823187498, - "hits": 3, - "total": 118, - "hitRate": 0.025423728813559324 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "KOI8-R", - "confidence": 4, - "language": "ru", - "byteLogLikelihood": -6.302671202821, - "hits": 2, - "total": 134, - "hitRate": 0.014925373134328358 + "encoding": "windows-1250", + "language": "cs", + "confidence": 9, + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-6", - "confidence": 3, - "language": "ar", - "byteLogLikelihood": -5.268243094623, - "hits": 1, - "total": 81, - "hitRate": 0.012345679012345678 + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 0, + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 }, { "encoding": "ISO-8859-8", - "confidence": 2, - "language": "he", - "byteLogLikelihood": -4.09495359507, - "hits": 1, - "total": 127, - "hitRate": 0.007874015748031496 - }, - { - "encoding": "windows-1255", - "confidence": 2, "language": "he", - "byteLogLikelihood": -4.09495359507, - "hits": 1, - "total": 127, - "hitRate": 0.007874015748031496 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 2, - "language": "uk", - "byteLogLikelihood": -4.203232324163, - "hits": 1, - "total": 131, - "hitRate": 0.007633587786259542 - }, - { - "encoding": "ISO-8859-1", "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.59932343766, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 134, + "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-2", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.358825259583, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 134, + "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-5", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.165809265202, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 130, + "total": 132, "hitRate": 0 }, { - "encoding": "ISO-8859-9", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.969377662519, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 133, + "total": 131, "hitRate": 0 }, { - "encoding": "windows-1250", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "hu", - "byteLogLikelihood": -4.358825259583, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 134, + "total": 132, "hitRate": 0 }, { - "encoding": "windows-1251", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.288697198304, + "byteLogLikelihood": -4.510859506517, "hits": 0, - "total": 134, + "total": 132, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { "encoding": "windows-1252", - "confidence": 0, - "language": "pt", - "byteLogLikelihood": -4.59932343766, - "hits": 0, - "total": 134, - "hitRate": 0 + "language": "es", + "confidence": 61, + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 15, + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 }, { - "encoding": "windows-1254", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.969377662519, + "byteLogLikelihood": -3.106555532888, "hits": 0, - "total": 133, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-1256", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.013410783916, + "byteLogLikelihood": -3.411159813207, "hits": 0, - "total": 122, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-1257", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.827626146169, + "byteLogLikelihood": -3.336328204376, "hits": 0, - "total": 134, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-1258", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "vi", - "byteLogLikelihood": -4.681848366371, + "byteLogLikelihood": -3.106555532888, "hits": 0, - "total": 127, + "total": 200, "hitRate": 0 }, { - "encoding": "windows-874", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.483358374511, + "byteLogLikelihood": -3.411159813207, "hits": 0, - "total": 52, + "total": 200, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1256", + "language": "ar", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.55248803416, + "byteLogLikelihood": -3.300604963183, "hits": 0, - "total": 127, + "total": 200, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -3.336328204376, "hits": 0, - "total": 116, + "total": 200, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.945729021813, - "hits": 0, - "total": 129, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-4", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.015268979336, - "hits": 0, - "total": 134, - "hitRate": 0 + "encoding": "windows-1252", + "language": "fr", + "confidence": 77, + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 }, { "encoding": "ISO-8859-10", - "confidence": 0, "language": "is", - "byteLogLikelihood": -4.576407109739, - "hits": 0, - "total": 134, - "hitRate": 0 + "confidence": 10, + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.890771320266, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 134, + "total": 228, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.788878603493, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 134, + "total": 228, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.603974037796, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 134, + "total": 228, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.137560045078, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 134, + "total": 228, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 93, + "total": 224, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 96, + "total": 228, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1254", - "language": "tr" + "encoding": "windows-1252", + "language": "it" }, "predicted": { - "encoding": "ISO-8859-9", - "confidence": 55, - "language": "tr", - "byteLogLikelihood": -3.531675216773, - "hits": 25, - "total": 134, - "hitRate": 0.1865671641791045 + "encoding": "ISO-8859-1", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 }, "encodingExact": false, "encodingEquivalent": true, @@ -17489,309 +6627,259 @@ "languageCorrect": true, "tiedAtTop": 2, "candidates": [ - { - "encoding": "ISO-8859-9", - "confidence": 55, - "language": "tr", - "byteLogLikelihood": -3.531675216773, - "hits": 25, - "total": 134, - "hitRate": 0.1865671641791045 - }, - { - "encoding": "windows-1254", - "confidence": 55, - "language": "tr", - "byteLogLikelihood": -3.531675216773, - "hits": 25, - "total": 134, - "hitRate": 0.1865671641791045 - }, - { - "encoding": "ISO-8859-3", - "confidence": 22, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 10, - "total": 134, - "hitRate": 0.07462686567164178 - }, { "encoding": "ISO-8859-1", - "confidence": 17, - "language": "da", - "byteLogLikelihood": -4.969813299576, - "hits": 8, - "total": 134, - "hitRate": 0.05970149253731343 - }, - { - "encoding": "windows-1252", - "confidence": 17, - "language": "da", - "byteLogLikelihood": -4.969813299576, - "hits": 8, - "total": 134, - "hitRate": 0.05970149253731343 - }, - { - "encoding": "ISO-8859-10", - "confidence": 17, - "language": "is", - "byteLogLikelihood": -4.720106560713, - "hits": 8, - "total": 134, - "hitRate": 0.05970149253731343 + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 }, - { - "encoding": "ISO-8859-2", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -4.352190516073, - "hits": 6, - "total": 134, - "hitRate": 0.04477611940298507 + { + "encoding": "windows-1252", + "language": "it", + "confidence": 58, + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 }, { - "encoding": "windows-1250", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -4.352190516073, - "hits": 6, - "total": 134, - "hitRate": 0.04477611940298507 + "encoding": "windows-1258", + "language": "vi", + "confidence": 16, + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 }, { "encoding": "windows-1257", - "confidence": 11, "language": "lv", - "byteLogLikelihood": -3.613497993373, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 + "confidence": 15, + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 }, { "encoding": "ISO-8859-4", - "confidence": 11, "language": "lv", - "byteLogLikelihood": -4.93302665818, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 + "confidence": 15, + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 }, { "encoding": "ISO-8859-13", - "confidence": 11, "language": "lt", - "byteLogLikelihood": -4.394247673508, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, - "language": "ro", - "byteLogLikelihood": -4.25849981603, - "hits": 5, - "total": 134, - "hitRate": 0.03731343283582089 - }, - { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 132, - "hitRate": 0.03787878787878788 - }, - { - "encoding": "ISO-8859-14", - "confidence": 8, - "language": "cy", - "byteLogLikelihood": -4.652724693026, - "hits": 4, - "total": 134, - "hitRate": 0.029850746268656716 - }, - { - "encoding": "windows-1258", - "confidence": 6, - "language": "vi", - "byteLogLikelihood": -5.197431316193, - "hits": 3, - "total": 133, - "hitRate": 0.022556390977443608 - }, - { - "encoding": "CP852", - "confidence": 6, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 133, - "hitRate": 0.022556390977443608 + "confidence": 3, + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 }, { "encoding": "ISO-8859-5", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.88736478629, + "confidence": 0, + "byteLogLikelihood": -2.954414124223, "hits": 0, - "total": 133, + "total": 159, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.217802445218, + "byteLogLikelihood": -3.629660094454, "hits": 0, - "total": 132, + "total": 159, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.216966174066, + "byteLogLikelihood": -2.533200659147, "hits": 0, - "total": 134, + "total": 159, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.95929011292, + "byteLogLikelihood": -3.496507561466, "hits": 0, - "total": 132, + "total": 159, "hitRate": 0 }, { - "encoding": "windows-1251", - "confidence": 0, + "encoding": "IBM866", "language": "ru", - "byteLogLikelihood": -4.576073075491, + "confidence": 0, + "byteLogLikelihood": -2.954414124223, "hits": 0, - "total": 134, + "total": 159, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.216966174066, + "byteLogLikelihood": -2.533200659147, "hits": 0, - "total": 134, + "total": 159, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 }, { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.95929011292, - "hits": 0, - "total": 132, - "hitRate": 0 + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 63, + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 }, { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -6.113682179832, - "hits": 0, - "total": 132, - "hitRate": 0 + "encoding": "windows-1252", + "language": "nl", + "confidence": 83, + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 11, + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.714883328648, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 130, + "total": 151, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.085579959517, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 134, + "total": 151, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 134, + "total": 151, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.88736478629, + "byteLogLikelihood": -2.405884662832, "hits": 0, - "total": 132, + "total": 151, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.742947096814, + "byteLogLikelihood": -3.57655026914, "hits": 0, - "total": 132, + "total": 150, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", - "byteLogLikelihood": -4.22610845571, + "confidence": 0, + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 134, + "total": 151, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1255", - "language": "he" + "encoding": "windows-1252", + "language": "no" }, "predicted": { - "encoding": "ISO-8859-8", + "encoding": "ISO-8859-1", + "language": "no", "confidence": 47, - "language": "he", - "byteLogLikelihood": -3.320333973076, - "hits": 16, - "total": 102, - "hitRate": 0.1568627450980392 + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, "encodingExact": false, "encodingEquivalent": true, @@ -17800,618 +6888,778 @@ "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-8", + "encoding": "ISO-8859-1", + "language": "no", "confidence": 47, - "language": "he", - "byteLogLikelihood": -3.320333973076, - "hits": 16, - "total": 102, - "hitRate": 0.1568627450980392 + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, { - "encoding": "windows-1255", + "encoding": "windows-1252", + "language": "no", "confidence": 47, - "language": "he", - "byteLogLikelihood": -3.320333973076, - "hits": 16, - "total": 102, - "hitRate": 0.1568627450980392 + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 }, { - "encoding": "ISO-8859-7", - "confidence": 2, - "language": "el", - "byteLogLikelihood": -3.981926233577, - "hits": 1, - "total": 102, - "hitRate": 0.00980392156862745 + "encoding": "ISO-8859-2", + "language": "cs", + "confidence": 7, + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 }, { - "encoding": "windows-1253", + "encoding": "windows-1257", + "language": "et", + "confidence": 7, + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-3", + "language": "mt", "confidence": 2, - "language": "el", - "byteLogLikelihood": -3.981926233577, + "byteLogLikelihood": -4.210065739287, "hits": 1, - "total": 102, - "hitRate": 0.00980392156862745 + "total": 114, + "hitRate": 0.008771929824561403 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-13", + "language": "lt", "confidence": 2, - "language": "ru", - "byteLogLikelihood": -5.394495827364, + "byteLogLikelihood": -4.319245023186, "hits": 1, - "total": 102, - "hitRate": 0.00980392156862745 + "total": 114, + "hitRate": 0.008771929824561403 }, { - "encoding": "ISO-8859-1", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.395116706452, + "byteLogLikelihood": -4.154384602589, "hits": 0, - "total": 102, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-2", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.461975046319, + "byteLogLikelihood": -2.647504477241, "hits": 0, - "total": 102, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-5", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -5.683415446443, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.124491051534, + "byteLogLikelihood": -3.753215851617, "hits": 0, - "total": 73, + "total": 114, "hitRate": 0 }, { - "encoding": "ISO-8859-9", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.941685614581, + "byteLogLikelihood": -4.154384602589, "hits": 0, - "total": 102, + "total": 114, "hitRate": 0 }, { - "encoding": "windows-1250", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.461975046319, + "byteLogLikelihood": -2.647504477241, "hits": 0, - "total": 102, + "total": 114, "hitRate": 0 }, { - "encoding": "windows-1251", - "confidence": 0, + "encoding": "x-mac-cyrillic", "language": "ru", - "byteLogLikelihood": -4.199203901043, - "hits": 0, - "total": 102, - "hitRate": 0 - }, - { - "encoding": "windows-1252", "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.395116706452, + "byteLogLikelihood": -3.753215851617, "hits": 0, - "total": 102, + "total": 114, "hitRate": 0 - }, + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.941685614581, - "hits": 0, - "total": 102, - "hitRate": 0 + "encoding": "ISO-8859-1", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 }, { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.32364067176, - "hits": 0, - "total": 93, - "hitRate": 0 + "encoding": "windows-1252", + "language": "pt", + "confidence": 50, + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 }, { - "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.639272141503, - "hits": 0, - "total": 102, - "hitRate": 0 + "encoding": "ISO-8859-2", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 }, { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.029681805165, - "hits": 0, - "total": 99, - "hitRate": 0 + "encoding": "windows-1250", + "language": "hu", + "confidence": 10, + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.31385343816, + "byteLogLikelihood": -3.017269736716, "hits": 0, - "total": 48, + "total": 143, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.40669720385, + "byteLogLikelihood": -3.50352863905, "hits": 0, - "total": 102, + "total": 143, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.393692981541, + "byteLogLikelihood": -3.846220145127, "hits": 0, - "total": 102, + "total": 143, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.683415446443, + "byteLogLikelihood": -3.017269736716, "hits": 0, - "total": 94, + "total": 143, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -3.50352863905, "hits": 0, - "total": 72, + "total": 143, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1256", + "language": "ar", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.134202940863, + "byteLogLikelihood": -3.503504267293, "hits": 0, - "total": 102, + "total": 143, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "IBM855", + "language": "sr", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.956632184798, + "byteLogLikelihood": -3.774998078834, "hits": 0, - "total": 102, + "total": 143, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.146385148445, + "byteLogLikelihood": -3.766179593299, "hits": 0, - "total": 102, + "total": 143, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "sv" + }, + "predicted": { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 }, { - "encoding": "ISO-8859-10", + "encoding": "windows-1252", + "language": "sv", + "confidence": 50, + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "ISO-8859-15", + "language": "de", + "confidence": 17, + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1257", + "language": "lt", + "confidence": 4, + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-3", + "language": "mt", + "confidence": 4, + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.745792460039, + "byteLogLikelihood": -3.287368473007, "hits": 0, - "total": 102, + "total": 136, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.842271132243, + "byteLogLikelihood": -3.876181596107, "hits": 0, - "total": 102, + "total": 136, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.813633859942, + "byteLogLikelihood": -3.066153183614, "hits": 0, - "total": 102, + "total": 136, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "fr", - "byteLogLikelihood": -4.521941300627, + "byteLogLikelihood": -3.393743210995, "hits": 0, - "total": 102, + "total": 136, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.093070766591, + "byteLogLikelihood": -3.876181596107, "hits": 0, - "total": 102, + "total": 136, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "windows-1255", + "language": "he", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -3.066153183614, "hits": 0, - "total": 85, + "total": 136, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "x-mac-cyrillic", + "language": "ru", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "byteLogLikelihood": -3.393743210995, "hits": 0, - "total": 89, + "total": 136, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1256", - "language": "ar" + "encoding": "windows-1253", + "language": "el" }, "predicted": { - "encoding": "windows-1256", - "confidence": 62, - "language": "ar", - "byteLogLikelihood": -3.306009422775, + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, "hits": 23, - "total": 110, - "hitRate": 0.20909090909090908 + "total": 134, + "hitRate": 0.17164179104477612 }, - "encodingExact": true, - "encodingEquivalent": false, + "encodingExact": false, + "encodingEquivalent": true, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "windows-1256", - "confidence": 62, - "language": "ar", - "byteLogLikelihood": -3.306009422775, + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, "hits": 23, - "total": 110, - "hitRate": 0.20909090909090908 - }, - { - "encoding": "IBM855", - "confidence": 21, - "language": "sr", - "byteLogLikelihood": -5.640676681359, - "hits": 7, - "total": 100, - "hitRate": 0.07 + "total": 134, + "hitRate": 0.17164179104477612 }, { - "encoding": "ISO-8859-6", - "confidence": 15, - "language": "ar", - "byteLogLikelihood": -3.973986161297, - "hits": 5, - "total": 99, - "hitRate": 0.050505050505050504 + "encoding": "windows-1253", + "language": "el", + "confidence": 51, + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 }, { "encoding": "ISO-8859-8", - "confidence": 5, "language": "he", - "byteLogLikelihood": -5.107875566922, + "confidence": 2, + "byteLogLikelihood": -4.09495359507, "hits": 1, - "total": 58, - "hitRate": 0.017241379310344827 + "total": 127, + "hitRate": 0.007874015748031496 }, { "encoding": "windows-1255", - "confidence": 5, "language": "he", - "byteLogLikelihood": -5.107875566922, - "hits": 1, - "total": 60, - "hitRate": 0.016666666666666666 - }, - { - "encoding": "ISO-8859-5", - "confidence": 2, - "language": "ru", - "byteLogLikelihood": -5.047579876205, - "hits": 1, - "total": 111, - "hitRate": 0.009009009009009009 - }, - { - "encoding": "KOI8-R", "confidence": 2, - "language": "ru", - "byteLogLikelihood": -5.041761137138, + "byteLogLikelihood": -4.09495359507, "hits": 1, - "total": 110, - "hitRate": 0.00909090909090909 + "total": 127, + "hitRate": 0.007874015748031496 }, { - "encoding": "KOI8-U", - "confidence": 2, + "encoding": "x-mac-cyrillic", "language": "uk", - "byteLogLikelihood": -4.982175743167, + "confidence": 2, + "byteLogLikelihood": -4.203232324163, "hits": 1, - "total": 110, - "hitRate": 0.00909090909090909 + "total": 131, + "hitRate": 0.007633587786259542 }, { "encoding": "ISO-8859-1", + "language": "pt", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.70358763711, - "hits": 0, - "total": 110, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.668535969782, - "hits": 0, - "total": 111, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.916907952775, + "byteLogLikelihood": -4.59932343766, "hits": 0, - "total": 109, + "total": 134, "hitRate": 0 }, { - "encoding": "ISO-8859-9", + "encoding": "ISO-8859-2", + "language": "hu", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.906860727803, + "byteLogLikelihood": -4.358825259583, "hits": 0, - "total": 109, + "total": 134, "hitRate": 0 }, { "encoding": "windows-1250", + "language": "hu", "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.668535969782, + "byteLogLikelihood": -4.358825259583, "hits": 0, - "total": 111, + "total": 134, "hitRate": 0 }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.575363278011, + "confidence": 0, + "byteLogLikelihood": -4.288697198304, "hits": 0, - "total": 111, + "total": 134, "hitRate": 0 }, { "encoding": "windows-1252", + "language": "pt", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.70358763711, + "byteLogLikelihood": -4.59932343766, "hits": 0, - "total": 110, + "total": 134, "hitRate": 0 }, { - "encoding": "windows-1253", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.916907952775, + "byteLogLikelihood": -4.576407109739, "hits": 0, - "total": 109, + "total": 134, "hitRate": 0 }, { - "encoding": "windows-1254", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.906860727803, + "byteLogLikelihood": -4.603974037796, "hits": 0, - "total": 109, + "total": 134, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1254", + "language": "tr" + }, + "predicted": { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "language": "tr", + "confidence": 55, + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "ISO-8859-1", + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "windows-1252", + "language": "da", + "confidence": 17, + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-2", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1250", + "language": "ro", + "confidence": 13, + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 }, { "encoding": "windows-1257", - "confidence": 0, - "language": "et", - "byteLogLikelihood": -4.856571347512, - "hits": 0, - "total": 110, - "hitRate": 0 + "language": "lv", + "confidence": 11, + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.029693479499, - "hits": 0, - "total": 106, - "hitRate": 0 + "encoding": "ISO-8859-13", + "language": "lt", + "confidence": 11, + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -4.736714293626, - "hits": 0, - "total": 90, - "hitRate": 0 + "encoding": "ISO-8859-16", + "language": "ro", + "confidence": 11, + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.982133409941, + "byteLogLikelihood": -4.216966174066, "hits": 0, - "total": 60, + "total": 134, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "byteLogLikelihood": -4.216966174066, "hits": 0, - "total": 46, + "total": 134, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.655679070563, + "byteLogLikelihood": -4.22610845571, "hits": 0, - "total": 81, + "total": 134, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1255", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 }, { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.078935619012, - "hits": 0, - "total": 110, - "hitRate": 0 + "encoding": "windows-1255", + "language": "he", + "confidence": 47, + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 }, { - "encoding": "ISO-8859-4", + "encoding": "ISO-8859-7", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "language": "el", + "confidence": 2, + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "language": "sv", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "byteLogLikelihood": -4.395116706452, "hits": 0, - "total": 111, + "total": 102, "hitRate": 0 }, { - "encoding": "ISO-8859-10", + "encoding": "ISO-8859-2", + "language": "cs", "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.796168589227, + "byteLogLikelihood": -4.461975046319, "hits": 0, - "total": 111, + "total": 102, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "windows-1250", + "language": "cs", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.003416643654, + "byteLogLikelihood": -4.461975046319, "hits": 0, - "total": 110, + "total": 102, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "windows-1251", + "language": "ru", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "byteLogLikelihood": -4.199203901043, "hits": 0, - "total": 111, + "total": 102, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "windows-1252", + "language": "sv", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.70358763711, + "byteLogLikelihood": -4.395116706452, "hits": 0, - "total": 110, + "total": 102, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.119560962469, + "byteLogLikelihood": -4.639272141503, "hits": 0, - "total": 111, + "total": 102, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "x-mac-cyrillic", + "language": "uk", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.941225681894, + "byteLogLikelihood": -4.134202940863, "hits": 0, - "total": 97, + "total": 102, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "ISO-8859-15", + "language": "fr", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "byteLogLikelihood": -4.521941300627, "hits": 0, - "total": 100, + "total": 102, "hitRate": 0 } ] }, { "expected": { - "encoding": "windows-1257", - "language": "et" + "encoding": "windows-1256", + "language": "ar" }, "predicted": { - "encoding": "windows-1257", - "confidence": 40, - "language": "et", - "byteLogLikelihood": -3.489288054183, - "hits": 16, - "total": 120, - "hitRate": 0.13333333333333333 + "encoding": "windows-1256", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 }, "encodingExact": true, "encodingEquivalent": false, @@ -18420,288 +7668,238 @@ "tiedAtTop": 1, "candidates": [ { - "encoding": "windows-1257", - "confidence": 40, - "language": "et", - "byteLogLikelihood": -3.489288054183, - "hits": 16, - "total": 120, - "hitRate": 0.13333333333333333 - }, - { - "encoding": "ISO-8859-1", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-9", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -3.976561526566, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "windows-1252", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "windows-1254", - "confidence": 12, - "language": "tr", - "byteLogLikelihood": -3.976561526566, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "macintosh", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-3", - "confidence": 12, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 + "encoding": "windows-1256", + "language": "ar", + "confidence": 62, + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 }, { - "encoding": "ISO-8859-15", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "encoding": "ISO-8859-6", + "language": "ar", + "confidence": 15, + "byteLogLikelihood": -3.973986161297, "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 + "total": 99, + "hitRate": 0.050505050505050504 }, { - "encoding": "CP850", - "confidence": 12, + "encoding": "ISO-8859-1", "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 5, - "total": 120, - "hitRate": 0.041666666666666664 + "confidence": 0, + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 }, { "encoding": "ISO-8859-2", - "confidence": 10, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "windows-1250", - "confidence": 10, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "ISO-8859-4", - "confidence": 10, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "ISO-8859-10", - "confidence": 10, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, - { - "encoding": "ISO-8859-13", - "confidence": 10, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 10, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 0, + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 }, { - "encoding": "windows-1258", - "confidence": 7, - "language": "vi", - "byteLogLikelihood": -5.497168225293, - "hits": 3, - "total": 120, - "hitRate": 0.025 + "encoding": "windows-1250", + "language": "cs", + "confidence": 0, + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 5, - "language": "ro", - "byteLogLikelihood": -5.220355825078, - "hits": 2, - "total": 120, - "hitRate": 0.016666666666666666 + "encoding": "windows-1252", + "language": "es", + "confidence": 0, + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 2, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 1, - "total": 120, - "hitRate": 0.008333333333333333 + "encoding": "windows-1254", + "language": "tr", + "confidence": 0, + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 }, { - "encoding": "ISO-8859-5", + "encoding": "windows-1257", + "language": "et", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -4.856571347512, "hits": 0, - "total": 120, + "total": 110, "hitRate": 0 }, { - "encoding": "ISO-8859-6", + "encoding": "windows-874", + "language": "th", "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.30822322351, + "byteLogLikelihood": -4.736714293626, "hits": 0, - "total": 120, + "total": 90, "hitRate": 0 }, { - "encoding": "ISO-8859-7", + "encoding": "ISO-8859-10", + "language": "is", "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.41969872812, + "byteLogLikelihood": -4.796168589227, "hits": 0, - "total": 120, + "total": 111, "hitRate": 0 }, { - "encoding": "ISO-8859-8", + "encoding": "ISO-8859-15", + "language": "es", "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.51563851542, + "byteLogLikelihood": -4.70358763711, "hits": 0, - "total": 120, + "total": 110, "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1257", + "language": "et" + }, + "predicted": { + "encoding": "windows-1257", + "language": "et", + "confidence": 40, + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 + }, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1257", + "language": "et", + "confidence": 40, + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 }, { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.453415335268, - "hits": 0, + "encoding": "ISO-8859-1", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 5, "total": 120, - "hitRate": 0 + "hitRate": 0.041666666666666664 }, { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.41969872812, - "hits": 0, + "encoding": "ISO-8859-9", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -3.976561526566, + "hits": 5, "total": 120, - "hitRate": 0 + "hitRate": 0.041666666666666664 }, { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.51563851542, - "hits": 0, + "encoding": "windows-1252", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 5, "total": 120, - "hitRate": 0 + "hitRate": 0.041666666666666664 }, { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.831207501101, - "hits": 0, + "encoding": "windows-1254", + "language": "tr", + "confidence": 12, + "byteLogLikelihood": -3.976561526566, + "hits": 5, "total": 120, - "hitRate": 0 + "hitRate": 0.041666666666666664 }, { - "encoding": "windows-874", + "encoding": "ISO-8859-15", + "language": "es", + "confidence": 12, + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-6", + "language": "ar", "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.714883328648, + "byteLogLikelihood": -4.30822322351, "hits": 0, "total": 120, "hitRate": 0 }, { - "encoding": "KOI8-R", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.345510152227, + "byteLogLikelihood": -4.41969872812, "hits": 0, "total": 120, "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-8", + "language": "he", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -4.51563851542, "hits": 0, "total": 120, "hitRate": 0 }, { - "encoding": "IBM866", - "confidence": 0, + "encoding": "windows-1251", "language": "ru", - "byteLogLikelihood": -6.692083742507, + "confidence": 0, + "byteLogLikelihood": -4.453415335268, "hits": 0, "total": 120, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.827473946999, + "byteLogLikelihood": -4.41969872812, "hits": 0, "total": 120, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", + "confidence": 0, "byteLogLikelihood": -3.632538332325, "hits": 0, "total": 120, @@ -18716,8 +7914,8 @@ }, "predicted": { "encoding": "windows-1257", - "confidence": 34, "language": "lt", + "confidence": 34, "byteLogLikelihood": -3.200530207132, "hits": 20, "total": 173, @@ -18731,8 +7929,8 @@ "candidates": [ { "encoding": "windows-1257", - "confidence": 34, "language": "lt", + "confidence": 34, "byteLogLikelihood": -3.200530207132, "hits": 20, "total": 173, @@ -18740,8 +7938,8 @@ }, { "encoding": "ISO-8859-13", - "confidence": 34, "language": "lt", + "confidence": 34, "byteLogLikelihood": -3.200530207132, "hits": 20, "total": 173, @@ -18749,170 +7947,44 @@ }, { "encoding": "ISO-8859-4", - "confidence": 22, "language": "lv", + "confidence": 22, "byteLogLikelihood": -4.610398759565, "hits": 13, "total": 173, "hitRate": 0.07514450867052024 }, - { - "encoding": "ISO-8859-2", - "confidence": 15, - "language": "hu", - "byteLogLikelihood": -5.117993812417, - "hits": 9, - "total": 173, - "hitRate": 0.05202312138728324 - }, - { - "encoding": "windows-1250", - "confidence": 15, - "language": "hu", - "byteLogLikelihood": -5.117993812417, - "hits": 9, - "total": 173, - "hitRate": 0.05202312138728324 - }, { "encoding": "ISO-8859-1", - "confidence": 12, "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 - }, - { - "encoding": "windows-1252", "confidence": 12, - "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, "hitRate": 0.04046242774566474 }, { - "encoding": "macintosh", - "confidence": 12, + "encoding": "windows-1252", "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 168, - "hitRate": 0.041666666666666664 - }, - { - "encoding": "ISO-8859-15", "confidence": 12, - "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, "hitRate": 0.04046242774566474 }, { - "encoding": "CP850", - "confidence": 12, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 170, - "hitRate": 0.041176470588235294 - }, - { - "encoding": "ISO-8859-9", - "confidence": 8, - "language": "tr", - "byteLogLikelihood": -4.789760270297, - "hits": 5, - "total": 173, - "hitRate": 0.028901734104046242 - }, - { - "encoding": "windows-1254", - "confidence": 8, - "language": "tr", - "byteLogLikelihood": -4.789760270297, - "hits": 5, - "total": 173, - "hitRate": 0.028901734104046242 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 - }, - { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 172, - "hitRate": 0.01744186046511628 - }, - { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.6272413408, - "hits": 2, - "total": 173, - "hitRate": 0.011560693641618497 - }, - { - "encoding": "windows-1258", - "confidence": 1, - "language": "vi", - "byteLogLikelihood": -4.960186085121, - "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 - }, - { - "encoding": "IBM855", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -5.488073838002, - "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -4.416382325607, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.893109334109, - "hits": 0, - "total": 172, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.773457169107, - "hits": 0, - "total": 167, - "hitRate": 0 + "encoding": "ISO-8859-10", + "language": "is", + "confidence": 1, + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -4.472500251122, "hits": 0, "total": 173, @@ -18920,8 +7992,8 @@ }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", + "confidence": 0, "byteLogLikelihood": -4.444220754634, "hits": 0, "total": 171, @@ -18929,8 +8001,8 @@ }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", + "confidence": 0, "byteLogLikelihood": -3.531827362419, "hits": 0, "total": 173, @@ -18938,8 +8010,8 @@ }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -4.472500251122, "hits": 0, "total": 173, @@ -18947,72 +8019,18 @@ }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -4.444220754634, - "hits": 0, - "total": 171, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.924336543075, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.58967630288, - "hits": 0, - "total": 166, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.365896834008, - "hits": 0, - "total": 173, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.252539473952, - "hits": 0, - "total": 173, - "hitRate": 0 - }, - { - "encoding": "IBM866", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.893109334109, + "byteLogLikelihood": -4.444220754634, "hits": 0, "total": 171, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", - "byteLogLikelihood": -3.441328843196, - "hits": 0, - "total": 173, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.616838329205, + "byteLogLikelihood": -3.441328843196, "hits": 0, "total": 173, "hitRate": 0 @@ -19026,8 +8044,8 @@ }, "predicted": { "encoding": "windows-1257", - "confidence": 40, "language": "lv", + "confidence": 40, "byteLogLikelihood": -3.1328616083, "hits": 23, "total": 170, @@ -19041,170 +8059,44 @@ "candidates": [ { "encoding": "windows-1257", - "confidence": 40, "language": "lv", + "confidence": 40, "byteLogLikelihood": -3.1328616083, "hits": 23, "total": 170, "hitRate": 0.13529411764705881 }, - { - "encoding": "ISO-8859-4", - "confidence": 31, - "language": "lv", - "byteLogLikelihood": -5.003611214133, - "hits": 18, - "total": 170, - "hitRate": 0.10588235294117647 - }, - { - "encoding": "CP850", - "confidence": 15, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 9, - "total": 170, - "hitRate": 0.052941176470588235 - }, { "encoding": "ISO-8859-1", - "confidence": 14, - "language": "fr", - "byteLogLikelihood": -4.806768476285, - "hits": 8, - "total": 170, - "hitRate": 0.047058823529411764 - }, - { - "encoding": "windows-1252", - "confidence": 14, "language": "fr", - "byteLogLikelihood": -4.806768476285, - "hits": 8, - "total": 170, - "hitRate": 0.047058823529411764 - }, - { - "encoding": "macintosh", - "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 8, - "total": 170, - "hitRate": 0.047058823529411764 - }, - { - "encoding": "ISO-8859-15", "confidence": 14, - "language": "fr", "byteLogLikelihood": -4.806768476285, "hits": 8, "total": 170, "hitRate": 0.047058823529411764 }, - { - "encoding": "CP852", - "confidence": 12, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 7, - "total": 170, - "hitRate": 0.041176470588235294 - }, - { - "encoding": "ISO-8859-3", - "confidence": 10, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 170, - "hitRate": 0.03529411764705882 - }, - { - "encoding": "ISO-8859-13", - "confidence": 10, - "language": "lt", - "byteLogLikelihood": -4.977793000004, - "hits": 6, - "total": 170, - "hitRate": 0.03529411764705882 - }, - { - "encoding": "ISO-8859-2", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 170, - "hitRate": 0.029411764705882353 - }, - { - "encoding": "windows-1250", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 170, - "hitRate": 0.029411764705882353 - }, - { - "encoding": "ISO-8859-9", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -4.576539240129, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 - }, - { - "encoding": "windows-1254", - "confidence": 7, - "language": "tr", - "byteLogLikelihood": -4.576539240129, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 - }, { "encoding": "windows-1258", - "confidence": 7, "language": "vi", + "confidence": 7, "byteLogLikelihood": -4.342387954294, "hits": 4, "total": 169, "hitRate": 0.023668639053254437 }, - { - "encoding": "ISO-8859-10", - "confidence": 7, - "language": "is", - "byteLogLikelihood": -4.928587203128, - "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 - }, { "encoding": "ISO-8859-5", - "confidence": 0, "language": "ru", + "confidence": 0, "byteLogLikelihood": -4.459882683974, "hits": 0, "total": 170, "hitRate": 0 }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.080491263049, - "hits": 0, - "total": 169, - "hitRate": 0 - }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -3.703260469986, "hits": 0, "total": 170, @@ -19212,8 +8104,8 @@ }, { "encoding": "ISO-8859-8", - "confidence": 0, "language": "he", + "confidence": 0, "byteLogLikelihood": -4.207917523838, "hits": 0, "total": 170, @@ -19221,8 +8113,8 @@ }, { "encoding": "windows-1251", - "confidence": 0, "language": "ru", + "confidence": 0, "byteLogLikelihood": -3.543665829263, "hits": 0, "total": 170, @@ -19230,8 +8122,8 @@ }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -3.703260469986, "hits": 0, "total": 170, @@ -19239,71 +8131,26 @@ }, { "encoding": "windows-1255", - "confidence": 0, "language": "he", - "byteLogLikelihood": -4.207917523838, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.924336543075, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.58909903103, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.155101602334, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.147141858115, + "byteLogLikelihood": -4.207917523838, "hits": 0, "total": 170, "hitRate": 0 }, { "encoding": "IBM866", - "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.459882683974, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "IBM855", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.609454107834, + "byteLogLikelihood": -4.459882683974, "hits": 0, "total": 170, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "uk", + "confidence": 0, "byteLogLikelihood": -3.416476215238, "hits": 0, "total": 170, @@ -19311,18 +8158,9 @@ }, { "encoding": "ISO-8859-14", + "language": "cy", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.423374522988, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.717949371076, + "byteLogLikelihood": -4.423374522988, "hits": 0, "total": 170, "hitRate": 0 @@ -19336,8 +8174,8 @@ }, "predicted": { "encoding": "windows-1258", - "confidence": 98, "language": "vi", + "confidence": 98, "byteLogLikelihood": -3.323414763285, "hits": 42, "total": 124, @@ -19351,8 +8189,8 @@ "candidates": [ { "encoding": "windows-1258", - "confidence": 98, "language": "vi", + "confidence": 98, "byteLogLikelihood": -3.323414763285, "hits": 42, "total": 124, @@ -19360,8 +8198,8 @@ }, { "encoding": "ISO-8859-1", - "confidence": 9, "language": "pt", + "confidence": 9, "byteLogLikelihood": -4.721506246424, "hits": 4, "total": 126, @@ -19369,8 +8207,8 @@ }, { "encoding": "windows-1252", - "confidence": 9, "language": "pt", + "confidence": 9, "byteLogLikelihood": -4.721506246424, "hits": 4, "total": 126, @@ -19378,35 +8216,17 @@ }, { "encoding": "windows-1257", - "confidence": 9, "language": "lt", - "byteLogLikelihood": -4.767575136152, - "hits": 4, - "total": 126, - "hitRate": 0.031746031746031744 - }, - { - "encoding": "macintosh", - "confidence": 9, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 4, - "total": 122, - "hitRate": 0.03278688524590164 - }, - { - "encoding": "ISO-8859-4", "confidence": 9, - "language": "lv", - "byteLogLikelihood": -4.83530398146, + "byteLogLikelihood": -4.767575136152, "hits": 4, "total": 126, "hitRate": 0.031746031746031744 }, { "encoding": "ISO-8859-13", - "confidence": 9, "language": "lt", + "confidence": 9, "byteLogLikelihood": -4.767575136152, "hits": 4, "total": 126, @@ -19414,215 +8234,53 @@ }, { "encoding": "ISO-8859-14", - "confidence": 9, "language": "cy", + "confidence": 9, "byteLogLikelihood": -4.643097648852, "hits": 4, "total": 126, "hitRate": 0.031746031746031744 }, - { - "encoding": "ISO-8859-2", - "confidence": 7, - "language": "ro", - "byteLogLikelihood": -5.060625202481, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, { "encoding": "ISO-8859-9", - "confidence": 7, "language": "tr", - "byteLogLikelihood": -4.76855606835, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "windows-1250", "confidence": 7, - "language": "ro", - "byteLogLikelihood": -5.060625202481, + "byteLogLikelihood": -4.76855606835, "hits": 3, "total": 126, "hitRate": 0.023809523809523808 }, { "encoding": "windows-1254", - "confidence": 7, "language": "tr", - "byteLogLikelihood": -4.76855606835, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, - { - "encoding": "ISO-8859-3", - "confidence": 7, - "language": "mt", - "byteLogLikelihood": -4.906120867834, - "hits": 3, - "total": 124, - "hitRate": 0.024193548387096774 - }, - { - "encoding": "ISO-8859-15", "confidence": 7, - "language": "es", - "byteLogLikelihood": -4.873536273681, + "byteLogLikelihood": -4.76855606835, "hits": 3, "total": 126, "hitRate": 0.023809523809523808 }, - { - "encoding": "CP850", - "confidence": 7, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 3, - "total": 116, - "hitRate": 0.02586206896551724 - }, - { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 119, - "hitRate": 0.025210084033613446 - }, - { - "encoding": "ISO-8859-16", - "confidence": 2, - "language": "ro", - "byteLogLikelihood": -5.104831294985, - "hits": 1, - "total": 126, - "hitRate": 0.007936507936507936 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.315573016103, - "hits": 0, - "total": 124, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.428051016065, - "hits": 0, - "total": 115, - "hitRate": 0 - }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -4.428592570712, "hits": 0, "total": 125, "hitRate": 0 }, - { - "encoding": "ISO-8859-8", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.890004274964, - "hits": 0, - "total": 124, - "hitRate": 0 - }, - { - "encoding": "windows-1251", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.843070172798, - "hits": 0, - "total": 126, - "hitRate": 0 - }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -4.428592570712, "hits": 0, "total": 125, "hitRate": 0 }, - { - "encoding": "windows-1255", - "confidence": 0, - "language": "he", - "byteLogLikelihood": -4.890004274964, - "hits": 0, - "total": 124, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.628297137503, - "hits": 0, - "total": 121, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.351753988745, - "hits": 0, - "total": 114, - "hitRate": 0 - }, - { - "encoding": "KOI8-R", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.775144482981, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.565906355903, - "hits": 0, - "total": 126, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.727056945557, - "hits": 0, - "total": 124, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.876501639082, - "hits": 0, - "total": 124, - "hitRate": 0 - }, { "encoding": "x-mac-cyrillic", - "confidence": 0, "language": "ru", + "confidence": 0, "byteLogLikelihood": -4.782036156761, "hits": 0, "total": 125, @@ -19630,8 +8288,8 @@ }, { "encoding": "ISO-8859-10", - "confidence": 0, "language": "is", + "confidence": 0, "byteLogLikelihood": -4.794574996468, "hits": 0, "total": 126, @@ -19646,8 +8304,8 @@ }, "predicted": { "encoding": "windows-874", - "confidence": 21, "language": "th", + "confidence": 21, "byteLogLikelihood": -3.827006747769, "hits": 7, "total": 100, @@ -19661,125 +8319,35 @@ "candidates": [ { "encoding": "windows-874", - "confidence": 21, "language": "th", + "confidence": 21, "byteLogLikelihood": -3.827006747769, "hits": 7, "total": 100, "hitRate": 0.07 }, - { - "encoding": "ISO-8859-6", - "confidence": 6, - "language": "ar", - "byteLogLikelihood": -5.033884846794, - "hits": 2, - "total": 93, - "hitRate": 0.021505376344086023 - }, - { - "encoding": "ISO-8859-8", - "confidence": 6, - "language": "he", - "byteLogLikelihood": -5.415196995324, - "hits": 1, - "total": 43, - "hitRate": 0.023255813953488372 - }, - { - "encoding": "windows-1255", - "confidence": 6, - "language": "he", - "byteLogLikelihood": -5.415196995324, - "hits": 1, - "total": 47, - "hitRate": 0.02127659574468085 - }, - { - "encoding": "ISO-8859-7", - "confidence": 12, - "language": "el", - "byteLogLikelihood": -5.563277974308, - "hits": 4, - "total": 97, - "hitRate": 0.041237113402061855 - }, - { - "encoding": "windows-1253", - "confidence": 12, - "language": "el", - "byteLogLikelihood": -5.563277974308, - "hits": 4, - "total": 97, - "hitRate": 0.041237113402061855 - }, - { - "encoding": "windows-1251", - "confidence": 5, - "language": "ru", - "byteLogLikelihood": -5.926448142289, - "hits": 2, - "total": 102, - "hitRate": 0.0196078431372549 - }, - { - "encoding": "x-mac-cyrillic", - "confidence": 11, - "language": "ru", - "byteLogLikelihood": -6.059253444951, - "hits": 3, - "total": 79, - "hitRate": 0.0379746835443038 - }, - { - "encoding": "KOI8-R", - "confidence": 3, - "language": "ru", - "byteLogLikelihood": -5.413211053323, - "hits": 1, - "total": 94, - "hitRate": 0.010638297872340425 - }, - { - "encoding": "IBM855", - "confidence": 3, - "language": "sr", - "byteLogLikelihood": -5.781484174666, - "hits": 1, - "total": 88, - "hitRate": 0.011363636363636364 - }, { "encoding": "ISO-8859-1", - "confidence": 0, "language": "it", - "byteLogLikelihood": -4.767489617961, - "hits": 0, - "total": 97, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.903917269007, + "byteLogLikelihood": -4.767489617961, "hits": 0, - "total": 103, + "total": 97, "hitRate": 0 }, { - "encoding": "ISO-8859-5", + "encoding": "ISO-8859-2", + "language": "cs", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.535400651308, + "byteLogLikelihood": -4.903917269007, "hits": 0, "total": 103, "hitRate": 0 }, { "encoding": "ISO-8859-9", - "confidence": 0, "language": "tr", + "confidence": 0, "byteLogLikelihood": -4.985031025935, "hits": 0, "total": 97, @@ -19787,8 +8355,8 @@ }, { "encoding": "windows-1250", - "confidence": 0, "language": "pl", + "confidence": 0, "byteLogLikelihood": -4.798656950405, "hits": 0, "total": 101, @@ -19796,116 +8364,44 @@ }, { "encoding": "windows-1252", - "confidence": 0, "language": "it", - "byteLogLikelihood": -4.767489617961, - "hits": 0, - "total": 97, - "hitRate": 0 - }, - { - "encoding": "windows-1254", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.985031025935, + "byteLogLikelihood": -4.767489617961, "hits": 0, "total": 97, "hitRate": 0 }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.209625144761, - "hits": 0, - "total": 95, - "hitRate": 0 - }, { "encoding": "windows-1257", - "confidence": 0, "language": "et", + "confidence": 0, "byteLogLikelihood": -4.916002186025, "hits": 0, "total": 97, "hitRate": 0 }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -5.154894826847, - "hits": 0, - "total": 91, - "hitRate": 0 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -5.334497573029, - "hits": 0, - "total": 98, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.961753913164, - "hits": 0, - "total": 58, - "hitRate": 0 - }, { "encoding": "macintosh", - "confidence": 0, "language": "es", + "confidence": 0, "byteLogLikelihood": -4.948759890378, "hits": 0, "total": 65, "hitRate": 0 }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.084507932234, - "hits": 0, - "total": 98, - "hitRate": 0 - }, { "encoding": "ISO-8859-4", - "confidence": 0, "language": "lv", - "byteLogLikelihood": -4.783724339715, - "hits": 0, - "total": 103, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-10", "confidence": 0, - "language": "is", - "byteLogLikelihood": -5.119633132705, + "byteLogLikelihood": -4.783724339715, "hits": 0, "total": 103, "hitRate": 0 }, - { - "encoding": "ISO-8859-13", - "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.010167702209, - "hits": 0, - "total": 97, - "hitRate": 0 - }, { "encoding": "ISO-8859-14", - "confidence": 0, "language": "cy", + "confidence": 0, "byteLogLikelihood": -4.912654885736, "hits": 0, "total": 103, @@ -19913,36 +8409,18 @@ }, { "encoding": "ISO-8859-15", - "confidence": 0, "language": "fr", + "confidence": 0, "byteLogLikelihood": -4.801693311206, "hits": 0, "total": 101, "hitRate": 0 }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.134285984462, - "hits": 0, - "total": 103, - "hitRate": 0 - }, { "encoding": "CP850", - "confidence": 0, "language": "es", - "byteLogLikelihood": -4.887413059939, - "hits": 0, - "total": 85, - "hitRate": 0 - }, - { - "encoding": "CP852", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.056712497361, + "byteLogLikelihood": -4.887413059939, "hits": 0, "total": 85, "hitRate": 0 @@ -19956,8 +8434,8 @@ }, "predicted": { "encoding": "x-mac-cyrillic", - "confidence": 36, "language": "ru", + "confidence": 36, "byteLogLikelihood": -3.313760777855, "hits": 22, "total": 179, @@ -19971,8 +8449,8 @@ "candidates": [ { "encoding": "x-mac-cyrillic", - "confidence": 36, "language": "ru", + "confidence": 36, "byteLogLikelihood": -3.313760777855, "hits": 22, "total": 179, @@ -19980,26 +8458,17 @@ }, { "encoding": "windows-1251", - "confidence": 42, "language": "ru", + "confidence": 42, "byteLogLikelihood": -3.411211853364, "hits": 25, "total": 177, "hitRate": 0.14124293785310735 }, - { - "encoding": "IBM855", - "confidence": 9, - "language": "sr", - "byteLogLikelihood": -5.603105135042, - "hits": 5, - "total": 165, - "hitRate": 0.030303030303030304 - }, { "encoding": "ISO-8859-8", - "confidence": 3, "language": "he", + "confidence": 3, "byteLogLikelihood": -3.941595102268, "hits": 2, "total": 167, @@ -20007,215 +8476,71 @@ }, { "encoding": "windows-1255", - "confidence": 3, "language": "he", - "byteLogLikelihood": -3.941595102268, - "hits": 2, - "total": 167, - "hitRate": 0.011976047904191617 - }, - { - "encoding": "KOI8-U", "confidence": 3, - "language": "uk", - "byteLogLikelihood": -6.334488595021, - "hits": 2, - "total": 177, - "hitRate": 0.011299435028248588 - }, - { - "encoding": "windows-1256", - "confidence": 1, - "language": "ar", - "byteLogLikelihood": -5.525051915661, - "hits": 1, - "total": 163, - "hitRate": 0.006134969325153374 - }, - { - "encoding": "KOI8-R", - "confidence": 1, - "language": "ru", - "byteLogLikelihood": -6.390589047944, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 - }, - { - "encoding": "ISO-8859-14", - "confidence": 1, - "language": "cy", - "byteLogLikelihood": -4.724449359889, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 - }, - { - "encoding": "ISO-8859-1", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.665227236998, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.845647442888, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.975257565806, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-6", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -4.847464468096, - "hits": 0, - "total": 127, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-7", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.148248171081, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.950903003627, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.845647442888, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-1252", - "confidence": 0, - "language": "sv", - "byteLogLikelihood": -4.665227236998, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-1253", - "confidence": 0, - "language": "el", - "byteLogLikelihood": -4.148248171081, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.950903003627, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-1257", - "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.61521723683, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -4.899487478125, - "hits": 0, - "total": 173, - "hitRate": 0 + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 }, { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.446015416921, - "hits": 0, - "total": 85, - "hitRate": 0 + "encoding": "ISO-8859-14", + "language": "cy", + "confidence": 1, + "byteLogLikelihood": -4.724449359889, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-1", + "language": "sv", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.05026366488, + "byteLogLikelihood": -4.665227236998, "hits": 0, - "total": 169, + "total": 177, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-7", + "language": "el", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944169511699, + "byteLogLikelihood": -4.148248171081, "hits": 0, - "total": 151, + "total": 177, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1252", + "language": "sv", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -4.904224209153, + "byteLogLikelihood": -4.665227236998, "hits": 0, - "total": 175, + "total": 177, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "windows-1253", + "language": "el", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.79659105312, + "byteLogLikelihood": -4.148248171081, "hits": 0, "total": 177, "hitRate": 0 }, { - "encoding": "ISO-8859-10", + "encoding": "windows-1257", + "language": "lv", "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.939249573759, + "byteLogLikelihood": -4.61521723683, "hits": 0, "total": 177, "hitRate": 0 }, { "encoding": "ISO-8859-13", - "confidence": 0, "language": "lt", + "confidence": 0, "byteLogLikelihood": -4.66408545306, "hits": 0, "total": 177, @@ -20223,39 +8548,12 @@ }, { "encoding": "ISO-8859-15", - "confidence": 0, "language": "fr", - "byteLogLikelihood": -4.760478683076, - "hits": 0, - "total": 177, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -4.97522262578, + "byteLogLikelihood": -4.760478683076, "hits": 0, "total": 177, "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 137, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.083005956553, - "hits": 0, - "total": 156, - "hitRate": 0 } ] }, @@ -20266,8 +8564,8 @@ }, "predicted": { "encoding": "x-mac-cyrillic", - "confidence": 32, "language": "uk", + "confidence": 32, "byteLogLikelihood": -3.34369756932, "hits": 19, "total": 177, @@ -20281,8 +8579,8 @@ "candidates": [ { "encoding": "x-mac-cyrillic", - "confidence": 32, "language": "uk", + "confidence": 32, "byteLogLikelihood": -3.34369756932, "hits": 19, "total": 177, @@ -20290,26 +8588,17 @@ }, { "encoding": "windows-1251", - "confidence": 17, "language": "ru", + "confidence": 17, "byteLogLikelihood": -3.606822645813, "hits": 10, "total": 176, "hitRate": 0.056818181818181816 }, - { - "encoding": "IBM855", - "confidence": 5, - "language": "sr", - "byteLogLikelihood": -5.830774318921, - "hits": 3, - "total": 163, - "hitRate": 0.018404907975460124 - }, { "encoding": "ISO-8859-8", - "confidence": 3, "language": "he", + "confidence": 3, "byteLogLikelihood": -4.062591421695, "hits": 2, "total": 165, @@ -20317,98 +8606,35 @@ }, { "encoding": "windows-1255", - "confidence": 3, "language": "he", + "confidence": 3, "byteLogLikelihood": -4.062591421695, "hits": 2, "total": 165, "hitRate": 0.012121212121212121 }, - { - "encoding": "KOI8-R", - "confidence": 3, - "language": "ru", - "byteLogLikelihood": -6.397907632883, - "hits": 2, - "total": 169, - "hitRate": 0.011834319526627219 - }, - { - "encoding": "ISO-8859-6", - "confidence": 2, - "language": "ar", - "byteLogLikelihood": -4.990380473945, - "hits": 1, - "total": 119, - "hitRate": 0.008403361344537815 - }, - { - "encoding": "KOI8-U", - "confidence": 1, - "language": "uk", - "byteLogLikelihood": -6.341901231006, - "hits": 1, - "total": 175, - "hitRate": 0.005714285714285714 - }, { "encoding": "ISO-8859-1", - "confidence": 0, "language": "it", - "byteLogLikelihood": -4.629500955096, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-2", "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.739775351774, + "byteLogLikelihood": -4.629500955096, "hits": 0, "total": 169, "hitRate": 0 }, - { - "encoding": "ISO-8859-5", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.989983227843, - "hits": 0, - "total": 171, - "hitRate": 0 - }, { "encoding": "ISO-8859-7", - "confidence": 0, "language": "el", - "byteLogLikelihood": -4.300043808736, - "hits": 0, - "total": 169, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-9", "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.951345660734, + "byteLogLikelihood": -4.300043808736, "hits": 0, "total": 169, "hitRate": 0 }, - { - "encoding": "windows-1250", - "confidence": 0, - "language": "cs", - "byteLogLikelihood": -4.739775351774, - "hits": 0, - "total": 170, - "hitRate": 0 - }, { "encoding": "windows-1252", - "confidence": 0, "language": "it", + "confidence": 0, "byteLogLikelihood": -4.629500955096, "hits": 0, "total": 170, @@ -20416,107 +8642,35 @@ }, { "encoding": "windows-1253", - "confidence": 0, "language": "el", + "confidence": 0, "byteLogLikelihood": -4.300043808736, "hits": 0, "total": 169, "hitRate": 0 }, - { - "encoding": "windows-1254", - "confidence": 0, - "language": "tr", - "byteLogLikelihood": -4.951345660734, - "hits": 0, - "total": 170, - "hitRate": 0 - }, - { - "encoding": "windows-1256", - "confidence": 0, - "language": "ar", - "byteLogLikelihood": -5.607550779971, - "hits": 0, - "total": 155, - "hitRate": 0 - }, { "encoding": "windows-1257", - "confidence": 0, "language": "lv", + "confidence": 0, "byteLogLikelihood": -4.60874799529, "hits": 0, "total": 169, "hitRate": 0 }, - { - "encoding": "windows-1258", - "confidence": 0, - "language": "vi", - "byteLogLikelihood": -4.777030248466, - "hits": 0, - "total": 164, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.197321748228, - "hits": 0, - "total": 83, - "hitRate": 0 - }, - { - "encoding": "IBM866", - "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.100977296295, - "hits": 0, - "total": 167, - "hitRate": 0 - }, - { - "encoding": "macintosh", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 139, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.035218126431, - "hits": 0, - "total": 164, - "hitRate": 0 - }, { "encoding": "ISO-8859-4", - "confidence": 0, "language": "lv", + "confidence": 0, "byteLogLikelihood": -4.726279355538, "hits": 0, "total": 169, "hitRate": 0 }, - { - "encoding": "ISO-8859-10", - "confidence": 0, - "language": "is", - "byteLogLikelihood": -4.876716873437, - "hits": 0, - "total": 175, - "hitRate": 0 - }, { "encoding": "ISO-8859-13", - "confidence": 0, "language": "lt", + "confidence": 0, "byteLogLikelihood": -4.669220995205, "hits": 0, "total": 169, @@ -20524,48 +8678,12 @@ }, { "encoding": "ISO-8859-14", - "confidence": 0, "language": "cy", - "byteLogLikelihood": -4.725687554138, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-15", - "confidence": 0, - "language": "fr", - "byteLogLikelihood": -4.753004092615, - "hits": 0, - "total": 175, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.030920483462, + "byteLogLikelihood": -4.725687554138, "hits": 0, "total": 175, "hitRate": 0 - }, - { - "encoding": "CP850", - "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 0, - "total": 137, - "hitRate": 0 - }, - { - "encoding": "CP852", - "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 0, - "total": 152, - "hitRate": 0 } ] } diff --git a/package.json b/package.json index a3e54c5..1cb77cb 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,9 @@ "build": "rm -rf lib/* && tsc -p tsconfig.build.json", "corpus:build": "node scripts/build-corpus.mjs", "corpus:verify": "node scripts/verify-corpus.mjs", - "models:build": "node scripts/build-models.mjs", - "models:evaluate": "node scripts/evaluate-models.mjs", - "models:verify": "node scripts/verify-models.mjs", + "models:build": "tsx scripts/build-models.mjs", + "models:evaluate": "tsx scripts/evaluate-models.mjs", + "models:verify": "tsx scripts/verify-models.mjs", "benchmark": "tsx scripts/benchmark.mjs", "validate": "tsx scripts/validate.mjs", "format": "prettier --write \"**/*.{ts,json,md,mjs}\"", diff --git a/scripts/model-utils.mjs b/scripts/model-utils.mjs index f9e9f15..e068aa9 100644 --- a/scripts/model-utils.mjs +++ b/scripts/model-utils.mjs @@ -2,6 +2,11 @@ import { spawnSync } from 'node:child_process'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + prepareSBCSModels, + scoreSBCS, + trigrams, +} from '../src/encoding/sbcs-scoring.ts'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const corpus = join(root, 'corpus'); @@ -90,30 +95,6 @@ function byteMapFor(encoding) { return byteMap; } -function normalizedBytes(buffer, byteMap) { - const normalized = []; - let ignoreSpace = false; - for (const byte of buffer) { - const mapped = byteMap[byte]; - if (mapped !== 0) { - if (!(mapped === 0x20 && ignoreSpace)) normalized.push(mapped); - ignoreSpace = mapped === 0x20; - } - } - normalized.push(0x20); - return normalized; -} - -function trigrams(buffer, byteMap) { - const values = []; - let trigram = 0; - for (const byte of normalizedBytes(buffer, byteMap)) { - trigram = ((trigram << 8) + byte) & 0xffffff; - values.push(trigram); - } - return values; -} - function detectorNgrams(buffers, byteMap) { const counts = new Map(); for (const buffer of buffers) { @@ -160,43 +141,6 @@ function highByteDistribution(buffers) { }; } -function byteLogLikelihood(buffer, distribution) { - const counts = new Map(distribution.counts); - const denominator = distribution.total + 128; - let highBytes = 0; - let likelihood = 0; - for (const byte of buffer) { - if (byte >= 0x80) { - likelihood += Math.log(((counts.get(byte) ?? 0) + 1) / denominator); - highBytes += 1; - } - } - return highBytes === 0 - ? Number.NEGATIVE_INFINITY - : Number((likelihood / highBytes).toFixed(12)); -} - -function ngramScore(buffer, byteMap, modelNgrams) { - const known = new Set(modelNgrams); - const values = trigrams(buffer, byteMap); - const hits = values.filter((trigram) => known.has(trigram)).length; - const rawPercent = hits / values.length; - return { - confidence: rawPercent > 0.33 ? 98 : Math.floor(rawPercent * 300), - hits, - total: values.length, - hitRate: rawPercent, - }; -} - -function statisticallyCompetitive(best, candidate) { - const variance = - (best.hitRate * (1 - best.hitRate)) / best.total + - (candidate.hitRate * (1 - candidate.hitRate)) / candidate.total; - const margin = 1.96 * Math.sqrt(variance); - return best.hitRate - candidate.hitRate <= margin; -} - function decodedText(buffer, encodingName) { const encoding = manifest.encodings.find( (candidate) => candidate.name === encodingName, @@ -215,21 +159,6 @@ function byteEquivalent(buffer, leftEncoding, rightEncoding) { } } -const equivalentEncodingFamilies = [ - ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], - ['ISO-8859-2', 'windows-1250'], - ['ISO-8859-7', 'windows-1253'], - ['ISO-8859-8', 'windows-1255'], - ['ISO-8859-9', 'windows-1254'], - ['ISO-8859-13', 'windows-1257'], -]; - -function encodingFamily(name) { - return ( - equivalentEncodingFamilies.find((family) => family.includes(name)) ?? [name] - ); -} - function compileSingleByteModels() { const models = []; for (const encoding of manifest.encodings) { @@ -262,78 +191,14 @@ function compileSingleByteModels() { } function evaluate(models) { + const preparedModels = prepareSBCSModels(models); const tests = corpusIndex.filter( (row) => row.split === 'test' && row.encoding !== 'CP949', ); const results = tests.map((test) => { const buffer = readFileSync(join(generatedCorpus, test.path)); - const candidates = models.map((model) => { - const languageScores = model.languages.map((language) => ({ - language: language.language, - ...ngramScore(buffer, model.byteMap, language.ngrams), - byteLogLikelihood: byteLogLikelihood(buffer, language.highBytes), - })); - const confidenceScore = Math.max( - ...languageScores.map((score) => score.confidence), - ); - const confidenceLanguages = languageScores.filter( - (score) => score.confidence === confidenceScore, - ); - confidenceLanguages.sort( - (left, right) => right.byteLogLikelihood - left.byteLogLikelihood, - ); - const confidenceLanguage = confidenceLanguages[0]; - return { - encoding: model.encoding, - confidence: confidenceScore, - language: confidenceLanguage.language, - byteLogLikelihood: confidenceLanguage.byteLogLikelihood, - hits: confidenceLanguage.hits, - total: confidenceLanguage.total, - hitRate: confidenceLanguage.hitRate, - }; - }); - candidates.sort((left, right) => right.confidence - left.confidence); - const strongest = candidates[0]; - const competitive = candidates.filter((candidate) => - statisticallyCompetitive(strongest, candidate), - ); - const competitiveEncodings = new Set( - competitive.map((candidate) => candidate.encoding), - ); - competitive.sort( - (left, right) => - right.byteLogLikelihood - left.byteLogLikelihood || - right.confidence - left.confidence, - ); - const remaining = candidates.filter( - (candidate) => !competitiveEncodings.has(candidate.encoding), - ); - candidates.splice(0, candidates.length, ...competitive, ...remaining); - const strongestPrediction = candidates[0]; - const strongestFamily = encodingFamily(strongestPrediction.encoding); - const equivalentLanguageCandidates = candidates.filter( - (candidate) => - strongestFamily.includes(candidate.encoding) && - byteEquivalent( - buffer, - strongestPrediction.encoding, - candidate.encoding, - ), - ); - equivalentLanguageCandidates.sort( - (left, right) => - manifest.encodings.findIndex( - (encoding) => encoding.name === left.encoding, - ) - - manifest.encodings.findIndex( - (encoding) => encoding.name === right.encoding, - ), - ); - const predicted = { - ...strongestPrediction, - language: equivalentLanguageCandidates[0].language, - }; + const candidates = scoreSBCS(buffer, preparedModels); + const predicted = candidates[0]; const encodingExact = predicted.encoding === test.encoding; const encodingEquivalent = !encodingExact && diff --git a/src/encoding/sbcs-scoring.test.ts b/src/encoding/sbcs-scoring.test.ts new file mode 100644 index 0000000..a509527 --- /dev/null +++ b/src/encoding/sbcs-scoring.test.ts @@ -0,0 +1,78 @@ +import fs from 'fs'; +import path from 'path'; +import { describe, expect, it } from 'vitest'; +import type { Context } from '.'; +import { generatedSBCSModels } from './models/generated'; +import type { GeneratedSBCSModel } from './models/types'; +import { analyseGeneratedSBCS } from './sbcs'; +import { prepareSBCSModels, scoreSBCS, trigrams } from './sbcs-scoring'; + +const preparedModels = prepareSBCSModels( + generatedSBCSModels as readonly GeneratedSBCSModel[], +); + +function context(input: Uint8Array): Context { + const byteStats = Array(256).fill(0); + for (const byte of input) byteStats[byte] += 1; + return { + byteStats, + c1Bytes: input.some((byte) => byte >= 0x80 && byte <= 0x9f), + rawInput: input, + rawLen: input.length, + inputBytes: input, + inputLen: input.length, + }; +} + +describe('shared SBCS scoring', () => { + it('normalizes ignored bytes and repeated spaces before scoring trigrams', () => { + const byteMap = Array.from({ length: 256 }, (_, byte) => byte); + byteMap[0x27] = 0; + + expect( + trigrams(new Uint8Array([0x41, 0x20, 0x20, 0x27, 0x42]), byteMap), + ).toEqual([0x41, 0x4120, 0x412042, 0x204220]); + }); + + it.each([ + new Uint8Array(), + new Uint8Array([0x41]), + new Uint8Array([0xff]), + new Uint8Array([0xc3]), + new Uint8Array([0x00, 0xff, 0x80, 0x00]), + ])('returns deterministic finite confidence scores for %j', (input) => { + const first = scoreSBCS(input, preparedModels); + const second = scoreSBCS(input, preparedModels); + + expect(second).toEqual(first); + expect(first.length).toBeGreaterThan(0); + expect( + first.every( + ({ confidence }) => + Number.isFinite(confidence) && confidence >= 0 && confidence <= 100, + ), + ).toBe(true); + }); + + it('keeps the runtime wrapper aligned with the shared scorer on the corpus', () => { + const corpusRoot = path.join(__dirname, '../../corpus/generated'); + const corpusIndex = JSON.parse( + fs.readFileSync(path.join(corpusRoot, 'index.json'), 'utf8'), + ) as { encoding: string; path: string }[]; + const rows = corpusIndex.filter(({ encoding }) => encoding !== 'CP949'); + + for (const row of rows) { + const input = fs.readFileSync(path.join(corpusRoot, row.path)); + const raw = scoreSBCS(input, preparedModels).find( + ({ confidence }) => confidence > 0, + ); + const runtime = analyseGeneratedSBCS(context(input))[0]; + + expect(runtime, row.path).toEqual({ + name: raw?.encoding, + lang: raw?.language, + confidence: raw ? raw.confidence / 100 : undefined, + }); + } + }); +}); diff --git a/src/encoding/sbcs-scoring.ts b/src/encoding/sbcs-scoring.ts new file mode 100644 index 0000000..d192928 --- /dev/null +++ b/src/encoding/sbcs-scoring.ts @@ -0,0 +1,304 @@ +import type { EncodingName } from '../match'; +import type { + GeneratedSBCSLanguage, + GeneratedSBCSModel, + HighByteDistribution, +} from './models/types'; + +const N_GRAM_MASK = 0xffffff; +// Byte distributions are cheap and sufficiently discriminative to exclude +// most encodings before the more expensive n-gram pass. The corpus needs 11 +// candidates to preserve full-result accuracy; keep one additional candidate +// as headroom for unseen inputs. +const MAX_SBCS_CANDIDATES = 12; + +export interface SBCSScore { + readonly confidence: number; + readonly hitRate: number; + readonly hits: number; + readonly total: number; +} + +export interface SBCSCandidate extends SBCSScore { + readonly encoding: EncodingName; + readonly language: string; + readonly byteLogLikelihood: number; +} + +interface RankedSBCSCandidate extends SBCSCandidate { + readonly modelIndex: number; +} + +interface PreparedSBCSLanguage { + readonly model: GeneratedSBCSLanguage; + readonly highByteLogProbabilities: Float64Array; +} + +export interface PreparedSBCSModel { + readonly model: GeneratedSBCSModel; + readonly modelIndex: number; + readonly languages: readonly PreparedSBCSLanguage[]; + readonly ngramLanguageIndexes: ReadonlyMap; +} + +type HighByteCounts = readonly (readonly [number, number])[]; + +const equivalentEncodingFamilies = [ + ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], + ['ISO-8859-2', 'windows-1250'], + ['ISO-8859-7', 'windows-1253'], + ['ISO-8859-8', 'windows-1255'], + ['ISO-8859-9', 'windows-1254'], + ['ISO-8859-13', 'windows-1257'], +] as const; + +function scanTrigrams( + input: Uint8Array, + byteMap: readonly number[], + record: (value: number) => void, +): number { + let ignoreSpace = false; + let ngram = 0; + let total = 0; + + const append = (mapped: number) => { + ngram = ((ngram << 8) + mapped) & N_GRAM_MASK; + total += 1; + record(ngram); + }; + + for (const byte of input) { + const mapped = byteMap[byte]; + if (mapped !== 0) { + if (!(mapped === 0x20 && ignoreSpace)) append(mapped); + ignoreSpace = mapped === 0x20; + } + } + + append(0x20); + return total; +} + +export function trigrams( + input: Uint8Array, + byteMap: readonly number[], +): number[] { + const values: number[] = []; + scanTrigrams(input, byteMap, (value) => values.push(value)); + return values; +} + +export function statisticallyCompetitive( + best: SBCSScore, + candidate: SBCSScore, +): boolean { + const variance = + (best.hitRate * (1 - best.hitRate)) / best.total + + (candidate.hitRate * (1 - candidate.hitRate)) / candidate.total; + return best.hitRate - candidate.hitRate <= 1.96 * Math.sqrt(variance); +} + +function highByteCounts(input: Uint8Array): HighByteCounts { + const counts = new Uint32Array(128); + for (const byte of input) { + if (byte >= 0x80) counts[byte - 0x80] += 1; + } + + const result: [number, number][] = []; + counts.forEach((count, index) => { + if (count > 0) result.push([index + 0x80, count]); + }); + return result; +} + +function highByteLogProbabilities(distribution: HighByteDistribution) { + const counts = new Map(distribution.counts); + const denominator = distribution.total + 128; + return Float64Array.from({ length: 128 }, (_, index) => + Math.log(((counts.get(index + 0x80) ?? 0) + 1) / denominator), + ); +} + +function ngramLanguageIndexes(model: GeneratedSBCSModel) { + const indexes = new Map(); + model.languages.forEach((language, languageIndex) => { + for (const ngram of new Set(language.ngrams)) { + const languageIndexes = indexes.get(ngram) ?? []; + languageIndexes.push(languageIndex); + indexes.set(ngram, languageIndexes); + } + }); + return indexes; +} + +export function prepareSBCSModels( + models: readonly GeneratedSBCSModel[], +): readonly PreparedSBCSModel[] { + return models.map((model, modelIndex) => ({ + model, + modelIndex, + languages: model.languages.map((language) => ({ + model: language, + highByteLogProbabilities: highByteLogProbabilities(language.highBytes), + })), + ngramLanguageIndexes: ngramLanguageIndexes(model), + })); +} + +function byteLogLikelihood( + counts: HighByteCounts, + logProbabilities: Float64Array, +) { + let highBytes = 0; + let likelihood = 0; + + for (const [byte, count] of counts) { + likelihood += logProbabilities[byte - 0x80] * count; + highBytes += count; + } + + return highBytes === 0 + ? Number.NEGATIVE_INFINITY + : Number((likelihood / highBytes).toFixed(12)); +} + +function modelByteLogLikelihood( + counts: HighByteCounts, + prepared: PreparedSBCSModel, +) { + let best = Number.NEGATIVE_INFINITY; + for (const language of prepared.languages) { + best = Math.max( + best, + byteLogLikelihood(counts, language.highByteLogProbabilities), + ); + } + return best; +} + +function candidate( + input: Uint8Array, + counts: HighByteCounts, + prepared: PreparedSBCSModel, + hits: Uint32Array, +): RankedSBCSCandidate { + hits.fill(0, 0, prepared.languages.length); + const total = scanTrigrams(input, prepared.model.byteMap, (value) => { + for (const languageIndex of prepared.ngramLanguageIndexes.get(value) ?? + []) { + hits[languageIndex] += 1; + } + }); + + let bestIndex = 0; + let bestConfidence = -1; + let bestByteLogLikelihood = Number.NEGATIVE_INFINITY; + let bestHitRate = 0; + + for (let index = 0; index < prepared.languages.length; index += 1) { + const hitRate = hits[index] / total; + const confidence = hitRate > 0.33 ? 98 : Math.floor(hitRate * 300); + const likelihood = byteLogLikelihood( + counts, + prepared.languages[index].highByteLogProbabilities, + ); + if ( + confidence > bestConfidence || + (confidence === bestConfidence && likelihood > bestByteLogLikelihood) + ) { + bestIndex = index; + bestConfidence = confidence; + bestByteLogLikelihood = likelihood; + bestHitRate = hitRate; + } + } + + return { + encoding: prepared.model.encoding, + language: prepared.languages[bestIndex].model.language, + confidence: bestConfidence, + byteLogLikelihood: bestByteLogLikelihood, + hits: hits[bestIndex], + total, + hitRate: bestHitRate, + modelIndex: prepared.modelIndex, + }; +} + +function family(name: EncodingName) { + return equivalentEncodingFamilies.find((value) => + value.includes(name as never), + ); +} + +function familyPreference( + left: SBCSCandidate, + right: SBCSCandidate, + c1Bytes: boolean, +) { + const leftFamily = family(left.encoding); + if (!leftFamily || leftFamily !== family(right.encoding)) return 0; + const preferred = + leftFamily[0] === 'ISO-8859-13' || c1Bytes + ? [...leftFamily].reverse() + : [...leftFamily]; + return ( + preferred.indexOf(left.encoding as never) - + preferred.indexOf(right.encoding as never) + ); +} + +export function scoreSBCS( + input: Uint8Array, + preparedModels: readonly PreparedSBCSModel[], + c1Bytes = input.some((byte) => byte >= 0x80 && byte <= 0x9f), +): SBCSCandidate[] { + if (preparedModels.length === 0) return []; + + const counts = highByteCounts(input); + const hits = new Uint32Array( + Math.max(...preparedModels.map((model) => model.languages.length)), + ); + const models = + counts.length === 0 + ? preparedModels + : preparedModels + .map((model) => ({ + model, + likelihood: modelByteLogLikelihood(counts, model), + })) + .sort( + (left, right) => + right.likelihood - left.likelihood || + left.model.modelIndex - right.model.modelIndex, + ) + .slice(0, MAX_SBCS_CANDIDATES) + .map(({ model }) => model); + const candidates = models.map((model) => + candidate(input, counts, model, hits), + ); + candidates.sort( + (left, right) => + right.confidence - left.confidence || left.modelIndex - right.modelIndex, + ); + + const strongest = candidates[0]; + const competitive = candidates.filter((value) => + statisticallyCompetitive(strongest, value), + ); + const competitiveNames = new Set(competitive.map((value) => value.encoding)); + competitive.sort( + (left, right) => + familyPreference(left, right, c1Bytes) || + right.byteLogLikelihood - left.byteLogLikelihood || + right.confidence - left.confidence || + left.modelIndex - right.modelIndex, + ); + const remaining = candidates.filter( + (value) => !competitiveNames.has(value.encoding), + ); + + return [...competitive, ...remaining].map( + ({ modelIndex: _modelIndex, ...value }) => value, + ); +} diff --git a/src/encoding/sbcs.ts b/src/encoding/sbcs.ts index 3e79e99..bc45362 100644 --- a/src/encoding/sbcs.ts +++ b/src/encoding/sbcs.ts @@ -1,267 +1,23 @@ import type { Match } from '../match'; import type { Context } from '.'; import { generatedSBCSModels } from './models/generated'; -import type { - GeneratedSBCSLanguage, - GeneratedSBCSModel, - HighByteDistribution, -} from './models/types'; +import type { GeneratedSBCSModel } from './models/types'; +import { prepareSBCSModels, scoreSBCS } from './sbcs-scoring'; -const N_GRAM_MASK = 0xffffff; -// Byte distributions are cheap and sufficiently discriminative to exclude -// most encodings before the more expensive n-gram pass. The corpus needs 11 -// candidates to preserve full-result accuracy; keep one additional candidate -// as headroom for unseen inputs. -const MAX_SBCS_CANDIDATES = 12; - -interface Score { - confidence: number; - hitRate: number; - hits: number; - total: number; -} - -interface Candidate extends Match, Score { - byteLogLikelihood: number; -} - -interface PreparedSBCSLanguage { - readonly model: GeneratedSBCSLanguage; - readonly highByteLogProbabilities: Float64Array; -} - -interface PreparedSBCSModel { - readonly model: GeneratedSBCSModel; - readonly languages: readonly PreparedSBCSLanguage[]; - readonly ngramLanguageMasks: ReadonlyMap; -} - -type HighByteCounts = readonly (readonly [number, number])[]; - -function highByteLogProbabilities(distribution: HighByteDistribution) { - const counts = new Map(distribution.counts); - const denominator = distribution.total + 128; - return Float64Array.from({ length: 128 }, (_, index) => - Math.log(((counts.get(index + 0x80) ?? 0) + 1) / denominator), - ); -} - -function ngramLanguageMasks(model: GeneratedSBCSModel) { - const masks = new Map(); - model.languages.forEach((language, languageIndex) => { - for (const ngram of new Set(language.ngrams)) { - masks.set(ngram, (masks.get(ngram) ?? 0) | (1 << languageIndex)); - } - }); - return masks; -} - -const preparedSBCSModels: readonly PreparedSBCSModel[] = ( - generatedSBCSModels as readonly GeneratedSBCSModel[] -).map((model) => ({ - model, - languages: model.languages.map((language) => ({ - model: language, - highByteLogProbabilities: highByteLogProbabilities(language.highBytes), - })), - ngramLanguageMasks: ngramLanguageMasks(model), -})); -const maxLanguagesPerModel = Math.max( - ...preparedSBCSModels.map((model) => model.languages.length), +const preparedSBCSModels = prepareSBCSModels( + generatedSBCSModels as readonly GeneratedSBCSModel[], ); -const equivalentEncodingFamilies = [ - ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], - ['ISO-8859-2', 'windows-1250'], - ['ISO-8859-7', 'windows-1253'], - ['ISO-8859-8', 'windows-1255'], - ['ISO-8859-9', 'windows-1254'], - ['ISO-8859-13', 'windows-1257'], -] as const; - -function family(name: Match['name']) { - return equivalentEncodingFamilies.find((value) => - value.includes(name as never), - ); -} - -function familyPreference(left: Candidate, right: Candidate, c1Bytes: boolean) { - const leftFamily = family(left.name); - if (!leftFamily || leftFamily !== family(right.name)) return 0; - const preferred = - leftFamily[0] === 'ISO-8859-13' || c1Bytes - ? [...leftFamily].reverse() - : [...leftFamily]; - return ( - preferred.indexOf(left.name as never) - - preferred.indexOf(right.name as never) - ); -} - -function ngramHits( - input: Uint8Array, - prepared: PreparedSBCSModel, - hits: Uint32Array, -): number { - hits.fill(0, 0, prepared.languages.length); - const byteMap = prepared.model.byteMap; - let ignoreSpace = false; - let ngram = 0; - let total = 0; - - const record = (value: number) => { - let languageMask = prepared.ngramLanguageMasks.get(value) ?? 0; - while (languageMask !== 0) { - const languageBit = languageMask & -languageMask; - hits[31 - Math.clz32(languageBit)] += 1; - languageMask ^= languageBit; - } - }; - - for (let inputIndex = 0; inputIndex < input.length; inputIndex += 1) { - const byte = input[inputIndex]; - const mapped = byteMap[byte]; - if (mapped !== 0) { - if (!(mapped === 0x20 && ignoreSpace)) { - ngram = ((ngram << 8) + mapped) & N_GRAM_MASK; - total += 1; - record(ngram); - } - ignoreSpace = mapped === 0x20; - } - } - - ngram = ((ngram << 8) + 0x20) & N_GRAM_MASK; - total += 1; - record(ngram); - return total; -} - -function byteLogLikelihood( - highByteCounts: HighByteCounts, - logProbabilities: Float64Array, -) { - let highBytes = 0; - let likelihood = 0; - - for (const [byte, count] of highByteCounts) { - likelihood += logProbabilities[byte - 0x80] * count; - highBytes += count; - } - - return highBytes === 0 ? Number.NEGATIVE_INFINITY : likelihood / highBytes; -} - -function modelByteLogLikelihood( - highByteCounts: HighByteCounts, - prepared: PreparedSBCSModel, -) { - let best = Number.NEGATIVE_INFINITY; - for (const language of prepared.languages) { - best = Math.max( - best, - byteLogLikelihood(highByteCounts, language.highByteLogProbabilities), - ); - } - return best; -} - -function candidate( - input: Uint8Array, - highByteCounts: HighByteCounts, - prepared: PreparedSBCSModel, - hits: Uint32Array, -): Candidate { - const { model, languages: preparedLanguages } = prepared; - const total = ngramHits(input, prepared, hits); - let bestIndex = 0; - let bestConfidence = -1; - let bestByteLogLikelihood = Number.NEGATIVE_INFINITY; - let bestHitRate = 0; - - for (let index = 0; index < preparedLanguages.length; index += 1) { - const hitRate = hits[index] / total; - const confidence = hitRate > 0.33 ? 98 : Math.floor(hitRate * 300); - const likelihood = byteLogLikelihood( - highByteCounts, - preparedLanguages[index].highByteLogProbabilities, - ); - if ( - confidence > bestConfidence || - (confidence === bestConfidence && likelihood > bestByteLogLikelihood) - ) { - bestIndex = index; - bestConfidence = confidence; - bestByteLogLikelihood = likelihood; - bestHitRate = hitRate; - } - } - - return { - name: model.encoding, - lang: preparedLanguages[bestIndex].model.language, - confidence: bestConfidence, - byteLogLikelihood: bestByteLogLikelihood, - hits: hits[bestIndex], - total, - hitRate: bestHitRate, - }; -} - -function statisticallyCompetitive(best: Score, value: Score) { - const variance = - (best.hitRate * (1 - best.hitRate)) / best.total + - (value.hitRate * (1 - value.hitRate)) / value.total; - return best.hitRate - value.hitRate <= 1.96 * Math.sqrt(variance); -} - export function analyseGeneratedSBCS(context: Context): Match[] { - const highByteCounts: [number, number][] = []; - for (let byte = 0x80; byte <= 0xff; byte += 1) { - const count = context.byteStats[byte]; - if (count > 0) highByteCounts.push([byte, count]); - } - const hits = new Uint32Array(maxLanguagesPerModel); - const models = - highByteCounts.length === 0 - ? preparedSBCSModels - : preparedSBCSModels - .map((model, index) => ({ - model, - index, - likelihood: modelByteLogLikelihood(highByteCounts, model), - })) - .sort( - (left, right) => - right.likelihood - left.likelihood || left.index - right.index, - ) - .slice(0, MAX_SBCS_CANDIDATES) - .map(({ model }) => model); - const candidates = models.map((model) => - candidate(context.inputBytes, highByteCounts, model, hits), - ); - candidates.sort((left, right) => right.confidence - left.confidence); - - const strongest = candidates[0]; - const competitive = candidates.filter((value) => - statisticallyCompetitive(strongest, value), - ); - const competitiveNames = new Set(competitive.map((value) => value.name)); - competitive.sort( - (left, right) => - familyPreference(left, right, context.c1Bytes) || - right.byteLogLikelihood - left.byteLogLikelihood || - right.confidence - left.confidence, - ); - const remaining = candidates.filter( - (value) => !competitiveNames.has(value.name), - ); - let confidenceCeiling = 1; - return [...competitive, ...remaining] + return scoreSBCS(context.inputBytes, preparedSBCSModels, context.c1Bytes) .filter((value) => value.confidence > 0) - .map(({ name, lang, confidence }) => { + .map(({ encoding, language, confidence }) => { confidenceCeiling = Math.min(confidenceCeiling, confidence / 100); - return { name, lang, confidence: confidenceCeiling }; + return { + name: encoding, + lang: language, + confidence: confidenceCeiling, + }; }); } From fcce8c582a9bdd163f82c4b21885e1b05b2821c5 Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Tue, 21 Jul 2026 20:00:53 +1000 Subject: [PATCH 6/6] Switch Mbsc matchers to use generated models (#135) * Mbsc models / corpus * MBSC utils and scorers --- corpus/README.md | 22 +- .../Big5/zh/test/community-garden.bin | 3 + .../generated/Big5/zh/train/city-morning.bin | 3 + corpus/generated/Big5/zh/train/library.bin | 3 + corpus/generated/Big5/zh/train/market.bin | 3 + corpus/generated/Big5/zh/train/workshop.bin | 3 + .../Big5/zh/validation/river-trip.bin | 3 + .../EUC-JP/ja/test/community-garden.bin | 3 + .../EUC-JP/ja/train/city-morning.bin | 3 + corpus/generated/EUC-JP/ja/train/library.bin | 3 + corpus/generated/EUC-JP/ja/train/market.bin | 3 + corpus/generated/EUC-JP/ja/train/workshop.bin | 3 + .../EUC-JP/ja/validation/river-trip.bin | 3 + .../EUC-KR/ko/test/community-garden.bin | 3 + .../EUC-KR/ko/train/city-morning.bin | 3 + corpus/generated/EUC-KR/ko/train/library.bin | 3 + corpus/generated/EUC-KR/ko/train/market.bin | 3 + corpus/generated/EUC-KR/ko/train/workshop.bin | 3 + .../EUC-KR/ko/validation/river-trip.bin | 3 + .../GB18030/zh/test/community-garden.bin | 3 + .../GB18030/zh/train/city-morning.bin | 3 + corpus/generated/GB18030/zh/train/library.bin | 3 + corpus/generated/GB18030/zh/train/market.bin | 3 + .../generated/GB18030/zh/train/workshop.bin | 3 + .../GB18030/zh/validation/river-trip.bin | 3 + .../Shift_JIS/ja/test/community-garden.bin | 3 + .../Shift_JIS/ja/train/city-morning.bin | 3 + .../generated/Shift_JIS/ja/train/library.bin | 3 + .../generated/Shift_JIS/ja/train/market.bin | 3 + .../generated/Shift_JIS/ja/train/workshop.bin | 3 + .../Shift_JIS/ja/validation/river-trip.bin | 3 + corpus/generated/index.json | 360 ++++++++++++++ corpus/generated/ngrams.mjs | 130 +++++ corpus/manifest.json | 39 +- corpus/model-evaluation.json | 301 +++++++++++- corpus/sources.json | 72 +++ corpus/test-sources.json | 12 + scripts/benchmark.mjs | 29 +- scripts/model-utils.mjs | 172 ++++++- scripts/verify-models.mjs | 14 +- src/encoding/mbcs-scoring.test.ts | 48 ++ src/encoding/mbcs-scoring.ts | 55 +++ src/encoding/mbcs.test.ts | 22 + src/encoding/mbcs.ts | 455 ++++-------------- src/encoding/models/generated.ts | 149 ++++++ src/encoding/models/types.ts | 7 + src/encoding/sbcs-scoring.test.ts | 5 +- src/encoding/sbcs.test.ts | 16 + 48 files changed, 1628 insertions(+), 370 deletions(-) create mode 100644 corpus/generated/Big5/zh/test/community-garden.bin create mode 100644 corpus/generated/Big5/zh/train/city-morning.bin create mode 100644 corpus/generated/Big5/zh/train/library.bin create mode 100644 corpus/generated/Big5/zh/train/market.bin create mode 100644 corpus/generated/Big5/zh/train/workshop.bin create mode 100644 corpus/generated/Big5/zh/validation/river-trip.bin create mode 100644 corpus/generated/EUC-JP/ja/test/community-garden.bin create mode 100644 corpus/generated/EUC-JP/ja/train/city-morning.bin create mode 100644 corpus/generated/EUC-JP/ja/train/library.bin create mode 100644 corpus/generated/EUC-JP/ja/train/market.bin create mode 100644 corpus/generated/EUC-JP/ja/train/workshop.bin create mode 100644 corpus/generated/EUC-JP/ja/validation/river-trip.bin create mode 100644 corpus/generated/EUC-KR/ko/test/community-garden.bin create mode 100644 corpus/generated/EUC-KR/ko/train/city-morning.bin create mode 100644 corpus/generated/EUC-KR/ko/train/library.bin create mode 100644 corpus/generated/EUC-KR/ko/train/market.bin create mode 100644 corpus/generated/EUC-KR/ko/train/workshop.bin create mode 100644 corpus/generated/EUC-KR/ko/validation/river-trip.bin create mode 100644 corpus/generated/GB18030/zh/test/community-garden.bin create mode 100644 corpus/generated/GB18030/zh/train/city-morning.bin create mode 100644 corpus/generated/GB18030/zh/train/library.bin create mode 100644 corpus/generated/GB18030/zh/train/market.bin create mode 100644 corpus/generated/GB18030/zh/train/workshop.bin create mode 100644 corpus/generated/GB18030/zh/validation/river-trip.bin create mode 100644 corpus/generated/Shift_JIS/ja/test/community-garden.bin create mode 100644 corpus/generated/Shift_JIS/ja/train/city-morning.bin create mode 100644 corpus/generated/Shift_JIS/ja/train/library.bin create mode 100644 corpus/generated/Shift_JIS/ja/train/market.bin create mode 100644 corpus/generated/Shift_JIS/ja/train/workshop.bin create mode 100644 corpus/generated/Shift_JIS/ja/validation/river-trip.bin create mode 100644 src/encoding/mbcs-scoring.test.ts create mode 100644 src/encoding/mbcs-scoring.ts diff --git a/corpus/README.md b/corpus/README.md index 48dce88..d1a1fba 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -18,6 +18,18 @@ npm run models:evaluate npm run models:verify ``` +Benchmark all inputs or isolate a stable recognizer family workload with: + +```sh +npm run benchmark +npm run benchmark -- --family=singlebyte +npm run benchmark -- --family=multibyte +``` + +Benchmark output includes the input bytes per iteration and a workload SHA-256. +Performance results are comparable only when the family, split, and workload +hash match. + The build script generates the corpus and performs an iconv round trip for every document, allowing canonically equivalent Unicode composition where an encoding such as windows-1258 stores combining marks. The check script independently rebuilds into a temporary @@ -45,5 +57,11 @@ reports every held-out test document, while `models:verify` checks reproducibility and fails if any encoding or language result is incorrect. A different encoding label is accepted only when both encodings decode the actual held-out bytes identically; the report distinguishes these byte-equivalent -results from exact matches. CP949 is excluded because its multibyte recogniser -requires character frequency data rather than byte trigrams. +results from exact matches. + +For multibyte encodings, the structural parsers identify valid raw code units +and the compiler selects the 128 most frequent units from the training split. +Runtime confidence is the structurally valid proportion of input code units +covered by that generated model. Shift_JIS, EUC-JP, Big5, EUC-KR, and GB18030 +use this path. CP949 remains in the corpus but is excluded from the runtime +models until its extended byte sequences have a dedicated parser. diff --git a/corpus/generated/Big5/zh/test/community-garden.bin b/corpus/generated/Big5/zh/test/community-garden.bin new file mode 100644 index 0000000..594af6e --- /dev/null +++ b/corpus/generated/Big5/zh/test/community-garden.bin @@ -0,0 +1,3 @@ +Ϫ + +F~̧Ű|lyAؤUABMpCĤl̵eFmеPAUaCg@_ӪC diff --git a/corpus/generated/Big5/zh/train/city-morning.bin b/corpus/generated/Big5/zh/train/city-morning.bin new file mode 100644 index 0000000..71e4cc7 --- /dev/null +++ b/corpus/generated/Big5/zh/train/city-morning.bin @@ -0,0 +1,3 @@ + + +CCӡAEFWZHA@ة}jAĤ@Z@T}lLwRDC diff --git a/corpus/generated/Big5/zh/train/library.bin b/corpus/generated/Big5/zh/train/library.bin new file mode 100644 index 0000000..82396af --- /dev/null +++ b/corpus/generated/Big5/zh/train/library.bin @@ -0,0 +1,3 @@ +wRϮ] + +ϹϮ]̥u½ѪnCŪ̴MvBǩMȦ譱ѡA޲zUǥͬdơC diff --git a/corpus/generated/Big5/zh/train/market.bin b/corpus/generated/Big5/zh/train/market.bin new file mode 100644 index 0000000..97ee3e1 --- /dev/null +++ b/corpus/generated/Big5/zh/train/market.bin @@ -0,0 +1,3 @@ +P + +sW\sABGBeMѥ]CUȦVA߰ݦAåya`檺kC diff --git a/corpus/generated/Big5/zh/train/workshop.bin b/corpus/generated/Big5/zh/train/workshop.bin new file mode 100644 index 0000000..a6e6c69 --- /dev/null +++ b/corpus/generated/Big5/zh/train/workshop.bin @@ -0,0 +1,3 @@ +Ʀu@ + +@Ӥpζbs@ժ]nC]pvzA}oˬd{As謰P~֪[̼gMC diff --git a/corpus/generated/Big5/zh/validation/river-trip.bin b/corpus/generated/Big5/zh/validation/river-trip.bin new file mode 100644 index 0000000..c52a8b5 --- /dev/null +++ b/corpus/generated/Big5/zh/validation/river-trip.bin @@ -0,0 +1,3 @@ +eȦ + +ḆͭaۦaϩMLLCFܤ[HAĻbwRe𮧡AťıߪC diff --git a/corpus/generated/EUC-JP/ja/test/community-garden.bin b/corpus/generated/EUC-JP/ja/test/community-garden.bin new file mode 100644 index 0000000..d9ad4af --- /dev/null +++ b/corpus/generated/EUC-JP/ja/test/community-garden.bin @@ -0,0 +1,3 @@ +ϰ + +οͤ϶Ƥ֤ڤ㤤ڤ򿢤Ҥɤ⤿Ͽ䤫ʻꡢ²轵ʪ˿롣 diff --git a/corpus/generated/EUC-JP/ja/train/city-morning.bin b/corpus/generated/EUC-JP/ja/train/city-morning.bin new file mode 100644 index 0000000..72a9c8a --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/city-morning.bin @@ -0,0 +1,3 @@ +ī + +īγϤäܤФޤؤˤ̶ФͤޤꡢŹ򳫤ǽΥХŤ̤Ϥ롣 diff --git a/corpus/generated/EUC-JP/ja/train/library.bin b/corpus/generated/EUC-JP/ja/train/library.bin new file mode 100644 index 0000000..f15e8f5 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/library.bin @@ -0,0 +1,3 @@ +Ťʿ޽ + +ϰο޽ۤǤܤΥڡ᤯벻ʹ롣ɼԤˤʳؤιԤܤõʽϳĴʪ diff --git a/corpus/generated/EUC-JP/ja/train/market.bin b/corpus/generated/EUC-JP/ja/train/market.bin new file mode 100644 index 0000000..4169791 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/market.bin @@ -0,0 +1,3 @@ +λԾ + +λԾˤϿڡʪ˪̪ƤƤΥѥ¤֡㤤ʪҤȤȵˤĤä diff --git a/corpus/generated/EUC-JP/ja/train/workshop.bin b/corpus/generated/EUC-JP/ja/train/workshop.bin new file mode 100644 index 0000000..20c2244 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/workshop.bin @@ -0,0 +1,3 @@ +ǥ빩˼ + +ʥबʪۤΰäƤ롣߷׼Ԥ̤ȯԤɤǧԽԤʬ䤹񤯡 diff --git a/corpus/generated/EUC-JP/ja/validation/river-trip.bin b/corpus/generated/EUC-JP/ja/validation/river-trip.bin new file mode 100644 index 0000000..a0faed9 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/validation/river-trip.bin @@ -0,0 +1,3 @@ +ؤξι + +ͧͤϿޤȲäƿ⤤Ĺθ塢Ťդǵ٤ߤʤͼĻʹ diff --git a/corpus/generated/EUC-KR/ko/test/community-garden.bin b/corpus/generated/EUC-KR/ko/test/community-garden.bin new file mode 100644 index 0000000..d5f8130 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/test/community-garden.bin @@ -0,0 +1,3 @@ + + +ֹε ִ ϰ ɰ ä,  ɾ. ̵ ˷ϴ޷ ̸ǥ ̿ ְ Ѵ. diff --git a/corpus/generated/EUC-KR/ko/train/city-morning.bin b/corpus/generated/EUC-KR/ko/train/city-morning.bin new file mode 100644 index 0000000..d2976f1 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/city-morning.bin @@ -0,0 +1,3 @@ + ħ + +ô õõ ħ ´´. ī䰡 ù ϸ ޴ȭ д´. ǹ ̷ ޺ . diff --git a/corpus/generated/EUC-KR/ko/train/library.bin b/corpus/generated/EUC-KR/ko/train/library.bin new file mode 100644 index 0000000..6c5b886 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/library.bin @@ -0,0 +1,3 @@ + + + å ѱ Ҹ ϰ 鸰. ڵ , ࿡ å ã´. 缭 л ø غѴ. diff --git a/corpus/generated/EUC-KR/ko/train/market.bin b/corpus/generated/EUC-KR/ko/train/market.bin new file mode 100644 index 0000000..3789568 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/market.bin @@ -0,0 +1,3 @@ + + +忡 忡 , ġ δ. մԵ äҸ ο ̾߱⸦ 丮 ҰѴ. diff --git a/corpus/generated/EUC-KR/ko/train/workshop.bin b/corpus/generated/EUC-KR/ko/train/workshop.bin new file mode 100644 index 0000000..c1ed449 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/workshop.bin @@ -0,0 +1,3 @@ + ۾ + + ڹ ȳ α׷ . ̳ʴ ޴ ϰ ڴ ڵ带 Ȯϸ ڴ 湮 . Ȯ ۰ ڵ ˻Ѵ. diff --git a/corpus/generated/EUC-KR/ko/validation/river-trip.bin b/corpus/generated/EUC-KR/ko/validation/river-trip.bin new file mode 100644 index 0000000..c8a9dce --- /dev/null +++ b/corpus/generated/EUC-KR/ko/validation/river-trip.bin @@ -0,0 +1,3 @@ + + +ϿϿ ģ ɾ. , ⸦ ì. å ڿ ɾ Ҹ . diff --git a/corpus/generated/GB18030/zh/test/community-garden.bin b/corpus/generated/GB18030/zh/test/community-garden.bin new file mode 100644 index 0000000..ab73969 --- /dev/null +++ b/corpus/generated/GB18030/zh/test/community-garden.bin @@ -0,0 +1,3 @@ +^@ + +ӂѿԺӸɾGɫ@Nr߲˺С䡣ӂ˲ɫƣÿLһoֲˮ diff --git a/corpus/generated/GB18030/zh/train/city-morning.bin b/corpus/generated/GB18030/zh/train/city-morning.bin new file mode 100644 index 0000000..9759d58 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/city-morning.bin @@ -0,0 +1,3 @@ +е糿 + +с܇վۼϰˣȵ_Tһ๫܇_ʼ^oĽֵ diff --git a/corpus/generated/GB18030/zh/train/library.bin b/corpus/generated/GB18030/zh/train/library.bin new file mode 100644 index 0000000..01c0725 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/library.bin @@ -0,0 +1,3 @@ +oĈD^ + +^D^eֻз•xߌҚvʷƌWзĕTWYϡ diff --git a/corpus/generated/GB18030/zh/train/market.bin b/corpus/generated/GB18030/zh/train/market.bin new file mode 100644 index 0000000..a2c8003 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/market.bin @@ -0,0 +1,3 @@ +Ј + +VϵЈ[Mr߲ˡˮۺIrԃճɣKҳ˵ diff --git a/corpus/generated/GB18030/zh/train/workshop.bin b/corpus/generated/GB18030/zh/train/workshop.bin new file mode 100644 index 0000000..819d65e --- /dev/null +++ b/corpus/generated/GB18030/zh/train/workshop.bin @@ -0,0 +1,3 @@ +λ + +һСFu^ָϡOӋ棬_lߙzʽ݋鲻ͬgą^׫f diff --git a/corpus/generated/GB18030/zh/validation/river-trip.bin b/corpus/generated/GB18030/zh/validation/river-trip.bin new file mode 100644 index 0000000..fe7365d --- /dev/null +++ b/corpus/generated/GB18030/zh/validation/river-trip.bin @@ -0,0 +1,3 @@ +߅ + +т؈D͟贩^֡˺ܾᣬڰoĺ߅Ϣ BQ diff --git a/corpus/generated/Shift_JIS/ja/test/community-garden.bin b/corpus/generated/Shift_JIS/ja/test/community-garden.bin new file mode 100644 index 0000000..e110411 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/test/community-garden.bin @@ -0,0 +1,3 @@ +n̒ + +ߏ̐l͋󂢂Ă𐮂AԂ؂Ⴂ؂ABqǂ͐FN₩ȎDAƑ͖TAɐB diff --git a/corpus/generated/Shift_JIS/ja/train/city-morning.bin b/corpus/generated/Shift_JIS/ja/train/city-morning.bin new file mode 100644 index 0000000..f31111e --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/city-morning.bin @@ -0,0 +1,3 @@ +X̒ + +̊X͂ڂo܂Bwɂ͒ʋ΂lW܂AiXJAŏ̃oXÂȒʂ𑖂n߂B diff --git a/corpus/generated/Shift_JIS/ja/train/library.bin b/corpus/generated/Shift_JIS/ja/train/library.bin new file mode 100644 index 0000000..258556e --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/library.bin @@ -0,0 +1,3 @@ +ÂȐ} + +n̐}قł͖{̃y[W߂鉹Bǎ҂͗jȊw◷s̖{TAi͊w̒ו`B diff --git a/corpus/generated/Shift_JIS/ja/train/market.bin b/corpus/generated/Shift_JIS/ja/train/market.bin new file mode 100644 index 0000000..a71af01 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/market.bin @@ -0,0 +1,3 @@ +yj̎s + +L̎sɂ͐VNȖ؁AʕAIAĂẴpԁBq͔_ƂƋG߂̗ɂ‚ĘbB diff --git a/corpus/generated/Shift_JIS/ja/train/workshop.bin b/corpus/generated/Shift_JIS/ja/train/workshop.bin new file mode 100644 index 0000000..8f8126c --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/workshop.bin @@ -0,0 +1,3 @@ +fW^H[ + +ȃ`[ق̈ēĂB݌v҂ʂ𐮂AJ҂R[hmFAҏW҂₷B diff --git a/corpus/generated/Shift_JIS/ja/validation/river-trip.bin b/corpus/generated/Shift_JIS/ja/validation/river-trip.bin new file mode 100644 index 0000000..2724ab8 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/validation/river-trip.bin @@ -0,0 +1,3 @@ +ւ̏s + +Fl͒n}ƉĐXBǓAÂȐӂŋx݂Ȃ[̒̐𕷂B diff --git a/corpus/generated/index.json b/corpus/generated/index.json index 136b1b1..7885b13 100644 --- a/corpus/generated/index.json +++ b/corpus/generated/index.json @@ -1,4 +1,76 @@ [ + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "test", + "document": "community-garden", + "path": "Big5/zh/test/community-garden.bin", + "bytes": 107, + "highBytes": 79, + "uniqueHighBytes": 39, + "sha256": "4602b4204cff37f57af8cc21fa4751c6e33e56c4078927b211a14e00657fc8bb" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "city-morning", + "path": "Big5/zh/train/city-morning.bin", + "bytes": 97, + "highBytes": 70, + "uniqueHighBytes": 33, + "sha256": "f141f927f2a8b8e6540a117ccee6678e8f4db6f81cd5c0ebad699de8e9d05b7c" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "library", + "path": "Big5/zh/train/library.bin", + "bytes": 101, + "highBytes": 82, + "uniqueHighBytes": 40, + "sha256": "b20e84ac072e488aa6114e2ad862b39d127774244f26304e9fdf9716c9937ac9" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "market", + "path": "Big5/zh/train/market.bin", + "bytes": 99, + "highBytes": 74, + "uniqueHighBytes": 35, + "sha256": "3fa6ec765b1c5ca517ec35b276defa25d9505cfb06d89f5b8bc09079165ac75b" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "workshop", + "path": "Big5/zh/train/workshop.bin", + "bytes": 113, + "highBytes": 83, + "uniqueHighBytes": 42, + "sha256": "0673960c008a34a445c94870c643cf40e55beaec03c290bab0dcbaa86c52b128" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "validation", + "document": "river-trip", + "path": "Big5/zh/validation/river-trip.bin", + "bytes": 93, + "highBytes": 71, + "uniqueHighBytes": 39, + "sha256": "d23ba0d2a924cfc8881a734d32b20c591791610270bb5fe0c5b44d2aa8257827" + }, { "encoding": "CP850", "iconv": "CP850", @@ -359,6 +431,222 @@ "uniqueHighBytes": 56, "sha256": "f6399d3705fc3e6c2a6427516ce6c8b9585a0a1e035b53a9efb7ef0811dd6a00" }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "test", + "document": "community-garden", + "path": "EUC-JP/ja/test/community-garden.bin", + "bytes": 129, + "highBytes": 126, + "uniqueHighBytes": 48, + "sha256": "6654d80ff3f372a76c64cff40e76361bd50fddfad6745d9a39f0b265bebbec27" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "city-morning", + "path": "EUC-JP/ja/train/city-morning.bin", + "bytes": 117, + "highBytes": 114, + "uniqueHighBytes": 44, + "sha256": "5c1bc169f52cbbc621feafdd6956fa4be9ed66211a385da85e1364ad0cb9c0fe" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "library", + "path": "EUC-JP/ja/train/library.bin", + "bytes": 125, + "highBytes": 122, + "uniqueHighBytes": 49, + "sha256": "9b8da9e80124276f5d3a46c82fac0bb35489164b972e529f0ee03df6568e1549" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "market", + "path": "EUC-JP/ja/train/market.bin", + "bytes": 115, + "highBytes": 112, + "uniqueHighBytes": 45, + "sha256": "861116732268050cb5c7bdc19c0ceff1aa1fb80a99f172a95f0926a5aaafd9a2" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "workshop", + "path": "EUC-JP/ja/train/workshop.bin", + "bytes": 131, + "highBytes": 128, + "uniqueHighBytes": 50, + "sha256": "1e952f775f77207dbf5faed31865416068b03f82009e6e86b0517cf564fc1d68" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "validation", + "document": "river-trip", + "path": "EUC-JP/ja/validation/river-trip.bin", + "bytes": 119, + "highBytes": 116, + "uniqueHighBytes": 43, + "sha256": "b09785c8f0852a1239d2283d7fc8d531a85fa57443894bd9ec14a8f9cd364103" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "test", + "document": "community-garden", + "path": "EUC-KR/ko/test/community-garden.bin", + "bytes": 166, + "highBytes": 138, + "uniqueHighBytes": 49, + "sha256": "15d58dc224354498565689b014ef393a0c6d9bcf0d536235997785c0c864e336" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "city-morning", + "path": "EUC-KR/ko/train/city-morning.bin", + "bytes": 152, + "highBytes": 126, + "uniqueHighBytes": 45, + "sha256": "54eadc94abafcb5446f54f61029b48a6f7d080fbd3f80219416537fbcf469ee9" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "library", + "path": "EUC-KR/ko/train/library.bin", + "bytes": 157, + "highBytes": 130, + "uniqueHighBytes": 48, + "sha256": "1aca0feb504c558b985ec91d05a54eb6c61f06399cb276e82e08551d4dffbeac" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "market", + "path": "EUC-KR/ko/train/market.bin", + "bytes": 155, + "highBytes": 128, + "uniqueHighBytes": 54, + "sha256": "2de80f2cde96c8a97d8cca91d0d85e47ec37d8362458829797a2576c9f11a880" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "workshop", + "path": "EUC-KR/ko/train/workshop.bin", + "bytes": 176, + "highBytes": 148, + "uniqueHighBytes": 53, + "sha256": "e9f1924b0b9e410049817e0355f167d012b3a1b24ee91074acd08306102a65a6" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "validation", + "document": "river-trip", + "path": "EUC-KR/ko/validation/river-trip.bin", + "bytes": 154, + "highBytes": 126, + "uniqueHighBytes": 56, + "sha256": "f6399d3705fc3e6c2a6427516ce6c8b9585a0a1e035b53a9efb7ef0811dd6a00" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "test", + "document": "community-garden", + "path": "GB18030/zh/test/community-garden.bin", + "bytes": 107, + "highBytes": 96, + "uniqueHighBytes": 55, + "sha256": "0b2447d40ee94fc0f54193e4a0657fb7d524db9a1456dfbf161db4e2d66a238f" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "city-morning", + "path": "GB18030/zh/train/city-morning.bin", + "bytes": 97, + "highBytes": 89, + "uniqueHighBytes": 52, + "sha256": "f831e80d10c0cc072648d19ea815faa0ece4ed763cea6925b18f38e487d8b2ea" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "library", + "path": "GB18030/zh/train/library.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 51, + "sha256": "1e0c934cfe5346ae0e1dd54f7355e8de90fcb47013d9edbd842db64f510abad8" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "market", + "path": "GB18030/zh/train/market.bin", + "bytes": 99, + "highBytes": 89, + "uniqueHighBytes": 56, + "sha256": "4b7df06fad0d5ad72c7c3aea20f843409d5176839813ade77ed773537863918f" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "workshop", + "path": "GB18030/zh/train/workshop.bin", + "bytes": 113, + "highBytes": 100, + "uniqueHighBytes": 61, + "sha256": "3209cbd7a69500b9825d7226b0065c9a7ae6206b113608c94b7108a89dca4a11" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "validation", + "document": "river-trip", + "path": "GB18030/zh/validation/river-trip.bin", + "bytes": 93, + "highBytes": 85, + "uniqueHighBytes": 48, + "sha256": "2d4ab1b56be3899eb86a28c8f46ccecb392e4e39ecdba918ae0e4cc2ed4f94ab" + }, { "encoding": "IBM855", "iconv": "CP855", @@ -2951,6 +3239,78 @@ "uniqueHighBytes": 2, "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "test", + "document": "community-garden", + "path": "Shift_JIS/ja/test/community-garden.bin", + "bytes": 129, + "highBytes": 113, + "uniqueHighBytes": 43, + "sha256": "a58ecbafb06d5f88d10f2d056a96eb0e826d2dfe50360c9da6f591ece6ec45e0" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "city-morning", + "path": "Shift_JIS/ja/train/city-morning.bin", + "bytes": 117, + "highBytes": 98, + "uniqueHighBytes": 37, + "sha256": "7848fc97e017d4eebb1dcc1ad7755a035330bb9fd5cc01fbe3df7db505745e34" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "library", + "path": "Shift_JIS/ja/train/library.bin", + "bytes": 125, + "highBytes": 104, + "uniqueHighBytes": 43, + "sha256": "e0d3b6fec3a86651a59ca899d3f9df16529da913ffdd4d4e1b3da9cc0fce6517" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "market", + "path": "Shift_JIS/ja/train/market.bin", + "bytes": 115, + "highBytes": 94, + "uniqueHighBytes": 39, + "sha256": "2c442382b1bc33652f62e4086143582114db0e2adde38f7cda22cf94e29c9f8b" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "workshop", + "path": "Shift_JIS/ja/train/workshop.bin", + "bytes": 131, + "highBytes": 109, + "uniqueHighBytes": 45, + "sha256": "e78003a71014de82d26e954ca441ce084fd1f07dc39f5f045596d91e776116a2" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "validation", + "document": "river-trip", + "path": "Shift_JIS/ja/validation/river-trip.bin", + "bytes": 119, + "highBytes": 104, + "uniqueHighBytes": 41, + "sha256": "9d7d47ab703aab26e1ecb264cb5cb838224a32f7992bb54af5fe10d27f954809" + }, { "encoding": "windows-1250", "iconv": "WINDOWS-1250", diff --git a/corpus/generated/ngrams.mjs b/corpus/generated/ngrams.mjs index 5b27b94..d6be5ed 100644 --- a/corpus/generated/ngrams.mjs +++ b/corpus/generated/ngrams.mjs @@ -1,4 +1,30 @@ export default [ + { + encoding: 'Big5', + language: 'zh', + documents: 4, + trigrams: [ + 0xa1430a, 0x52aaba, 0x77c052, 0xa141a8, 0xa143c5, 0xa5abb3, 0xa677c0, + 0xaabaa5, 0xaabab0, 0xabb0a5, 0xabb3f5, 0xaed1c0, 0xb0a5ab, 0xb27aad, + 0xb9cfae, 0xbaa5ab, 0xc052aa, 0xcfaed1, 0xd1c05d, 0x0a0aa4, 0x0a0aaa, + 0x0a0aab, 0x0a0abc, 0x0aa440, 0x0aaac0, 0x0aabb0, 0x0abc73, 0x40a854, + 0x40abc7, 0x40add3, 0x40af5a, 0x40b0d8, 0x40b3d5, 0x41a5c1, 0x41a8ae, + 0x41a8c3, 0x41a940, 0x41b2c4, 0x41b67d, 0x41bade, 0x41bd73, 0x41bdad, + 0x42a4f4, 0x42acec, 0x42b8c1, 0x43b35d, 0x43ba43, 0x43bff4, 0x43c555, + 0x43c5aa, 0x44a143, 0x45b6b0, 0x46a457, 0x47a142, 0x48a141, 0x4ca677, + 0x4da7e4, 0x4daec8, 0x4db7a1, 0x4dc4d1, 0x50a67e, 0x50b4c1, 0x54a8ae, + 0x55abc8, 0x55bec7, 0x56b941, 0x57aaba, 0x57af5a, 0x5aa4bd, 0x5aaaba, + 0x5baacc, 0x5cbaa1, 0x5d0a0a, 0x5da143, 0x5dabfc, 0x5dad70, 0x5db8cc, + 0x60b5e6, 0x61b160, 0x62bb73, 0x64a7e4, 0x64b57b, 0x65a94d, 0x67b24d, + 0x6aaaf9, 0x6ba143, 0x6cacef, 0x6ea143, 0x6eadb5, 0x6faacc, 0x70ae76, + 0x70b9ce, 0x73a740, 0x73b3f5, 0x73bfe8, 0x73c241, 0x75a6b3, 0x75a740, + 0x76a142, 0x76bee3, 0x79ae61, 0x7aadb6, 0x7aadfb, 0x7ba6a1, 0x7da46a, + 0x7da96c, 0x7db56f, 0x7ec4d6, 0xa141a9, 0xa141b2, 0xa141b6, 0xa141ba, + 0xa141bd, 0xa142a4, 0xa142ac, 0xa142b8, 0xa143b3, 0xa1a141, 0xa1a9fa, + 0xa1aaba, 0xa1b773, 0xa3a650, 0xa440ad, 0xa440af, 0xa446a4, 0xa448a1, + 0xa457aa, 0xa457af, + ], + }, { encoding: 'CP850', language: 'de', @@ -129,6 +155,84 @@ export default [ 0x20c4a1, 0x20c4ab, ], }, + { + encoding: 'EUC-JP', + language: 'ja', + documents: 4, + trigrams: [ + 0xa1a30a, 0xbcd4a4, 0xa1bca5, 0xa4aba4, 0xa4acca, 0xa4cba4, 0xa4cea5, + 0xa4eba1, 0xa4f2b3, 0xd4a4ac, 0xeba1a3, 0xa4a4a4, 0xa4afa4, 0xa4b7a1, + 0xa4b9a1, 0xa4b9a4, 0xa4c3a4, 0xa4c6a4, 0xa4cebb, 0xa4cec4, 0xa4dea4, + 0xa4e1a4, 0xa4eaa4, 0xaaa1a2, 0xaba4ca, 0xb3b9a4, 0xb4dba4, 0xb7a1a2, + 0xb9a1a3, 0xb9a4ac, 0xbbd4be, 0xbdf1a4, 0xbdf1b4, 0xbeeca4, 0xbfdebd, + 0xc0c5a4, 0xc5a4ab, 0xcba4cf, 0xcbdca4, 0xcebbd4, 0xd4beec, 0xdca4f2, + 0xdebdf1, 0xe2a4f2, 0xf1b4db, 0x0a0ab9, 0x0a0abe, 0x0a0ac3, 0x0a0ac4, + 0x0ab9ad, 0x0abeae, 0x0ac3cf, 0x0ac4ab, 0xa1a2b2, 0xa1a2b3, 0xa1a2b5, + 0xa1a2ba, 0xa1a2bb, 0xa1a2be, 0xa1a2ca, 0xa1a2cb, 0xa1a3b1, 0xa1a3c0, + 0xa1a3c6, 0xa1a3c7, 0xa2b2cc, 0xa2b3ab, 0xa2b5ca, 0xa2bac7, 0xa2bbca, + 0xa2bec6, 0xa2cad4, 0xa2cbaa, 0xa3b1d8, 0xa3c0df, 0xa3c6c9, 0xa3c7e3, + 0xa4a4c0, 0xa4a4c6, 0xa4a4ca, 0xa4a4eb, 0xa4a6a1, 0xa4a8a1, 0xa4a8a4, + 0xa4aca5, 0xa4acb2, 0xa4acbd, 0xa4acc0, 0xa4acc7, 0xa4acc8, 0xa4ada4, + 0xa4afa1, 0xa4b1a1, 0xa4b3a4, 0xa4b5a4, 0xa4bfa4, 0xa4c0e2, 0xa4c4a4, + 0xa4c6cf, 0xa4c7a4, 0xa4c8b5, 0xa4caa5, 0xa4caaa, 0xa4cabf, 0xa4cac4, + 0xa4cacc, 0xa4ceb0, 0xa4ceb3, 0xa4cebf, 0xa4cecb, 0xa4cece, 0xa4cfa4, + 0xa4cfb3, 0xa4cfbf, 0xa4cfc4, 0xa4cfc7, 0xa4cfcb, 0xa4cfce, 0xa4d6a1, + 0xa4d9ca, 0xa4e4a4, 0xa4e4b2, 0xa4e4ce, 0xa4e6a4, 0xa4eaa1, 0xa4eabb, + 0xa4eacc, 0xa4ebb2, + ], + }, + { + encoding: 'EUC-KR', + language: 'ko', + documents: 4, + trigrams: [ + 0xb4d92e, 0xb4c220, 0xc0bb20, 0xd92e20, 0xb8a620, 0xc0ba20, 0x20bbe7, + 0xb5e9c0, 0xbdc3c0, 0xbfcd20, 0xd92e0a, 0x20bdc3, 0xb0a120, 0xb0ed20, + 0xb4c2b4, 0xb8e720, 0xc0cc20, 0xc2b4d9, 0xc7d120, 0xc7d1b4, 0xcd20b0, + 0xd1b4d9, 0xe9c0ba, 0x0a0ab5, 0x20b1db, 0x20b5b5, 0x20b8b8, 0x20bcd2, + 0x20bec6, 0x20bfad, 0x20c0db, 0x20c3a5, 0x20c8ae, 0x2e20b5, 0xa1b4c2, + 0xa5c0bb, 0xa620c1, 0xadb0fc, 0xb0d420, 0xb5b5bc, 0xb5b5bd, 0xb5bcad, + 0xb5bdc3, 0xb7ce20, 0xb8b5e7, 0xb8b8b5, 0xbab8a6, 0xbb20b8, 0xbcadb0, + 0xbcd2b8, 0xbec6c4, 0xbfa120, 0xbfa1b4, 0xbfee20, 0xc0dab4, 0xc0dab5, + 0xc0dbc0, 0xc0e520, 0xc0e5bf, 0xc220c3, 0xc3a5c0, 0xc3c0e5, 0xc6c4a7, + 0xc7cfb0, 0xc7cfb8, 0xc8aec0, 0xcc20b9, 0xcfb8e7, 0xdab4c2, 0xdbc0ba, + 0xe5bfa1, 0x0a0ab1, 0x0a0ac0, 0x0ab1a4, 0x0ab5b5, 0x0ab5bf, 0x0ac0db, + 0x20b0ab, 0x20b0b3, 0x20b0c7, 0x20b0cb, 0x20b0e8, 0x20b0ed, 0x20b0fa, + 0x20b0fc, 0x20b1b8, 0x20b2dc, 0x20b3aa, 0x20b3d1, 0x20b3f3, 0x20b3f5, + 0x20b4ba, 0x20b5b6, 0x20b5e9, 0x20b5f0, 0x20b8c2, 0x20b8de, 0x20b9ae, + 0x20b9da, 0x20b9e0, 0x20b9e6, 0x20b9f6, 0x20b9f8, 0x20bba7, 0x20bcb3, + 0x20bcd5, 0x20bdac, 0x20beb4, 0x20bec8, 0x20bfa9, 0x20bfaa, 0x20bfe4, + 0x20bfee, 0x20c0a7, 0x20c0cc, 0x20c0d0, 0x20c0da, 0x20c0dc, 0x20c0fc, + 0x20c1a4, 0x20c1d8, 0x20c1fd, 0x20c3a3, 0x20c3a4, 0x20c3b5, 0x20c3b9, + 0x20c4a1, 0x20c4ab, + ], + }, + { + encoding: 'GB18030', + language: 'zh', + documents: 4, + trigrams: [ + 0xa1a30a, 0x4495f8, 0x6fb5c4, 0x884495, 0x95f8f0, 0xb0b2ec, 0xb2ec6f, + 0xb3c7ca, 0xb5c4ca, 0xc4cad0, 0xc7cad0, 0xcad088, 0xcfa1a3, 0xd088f6, + 0xec6fb5, 0xf8f05e, 0x0a0a8f, 0x0a0ab3, 0x0a0ac9, 0x0a0ad2, 0x0a8f56, + 0x0ab3c7, 0x0ac9e7, 0x0ad2bb, 0x46eaa0, 0x49b0fc, 0x4bbdbb, 0x4dd0c2, + 0x4fd38b, 0x548ecd, 0x54a3ac, 0x5688f6, 0x57bacd, 0x57c9fa, 0x59c1cf, + 0x5b9d4d, 0x5e0a0a, 0x5e8844, 0x5eb0b2, 0x5ed165, 0x5ed5df, 0x5ed6b8, + 0x5fb06c, 0x5fb4f3, 0x5fcabc, 0x65d6bb, 0x66c3f7, 0x67b5c4, 0x6cd5df, + 0x72c3f1, 0x72cadf, 0x75d7f7, 0x76cab7, 0x78d5df, 0x7ab2e9, 0x80d0a1, + 0x814bbd, 0x81eda3, 0x8280d0, 0x838696, 0x855e88, 0x85a2d3, 0x86548e, + 0x8696ca, 0x87d5be, 0x87e95f, 0x8846ea, 0x88f60a, 0x88f694, 0x88f6c9, + 0x8b8e9f, 0x8b9ee9, 0x8c57ba, 0x8c57c9, 0x8c91c7, 0x8ca4d5, 0x8e9fd5, + 0x8ecdd6, 0x8edd8b, 0x8f5688, 0x91c7e5, 0x93c3e6, 0x945b9d, 0x94b5ce, + 0x95d2f4, 0x95f8a3, 0x95f8b5, 0x96cad5, 0x997ab2, 0x99bfcd, 0x9a76ca, + 0x9d4dd0, 0x9ee9b2, 0x9fd5fb, 0xa0d5fd, 0xa18846, 0xa1a2b7, 0xa1a2bf, + 0xa1a2cb, 0xa1a3d4, 0xa1a3d7, 0xa1a3ee, 0xa2b7e4, 0xa2bfc6, 0xa2cbae, + 0xa2d35e, 0xa3ac81, 0xa3acb5, 0xa3acb9, 0xa3acbe, 0xa3acbf, 0xa3acdc, + 0xa3ace9, 0xa3b2cb, 0xa3d44f, 0xa3d778, 0xa3ee99, 0xa4d5d2, 0xa4d7f7, + 0xa7b7c8, 0xa8a1a3, 0xa9ceef, 0xa9df5e, 0xab8c91, 0xabb9b2, 0xac814b, + 0xacb5da, 0xacb9dc, + ], + }, { encoding: 'IBM855', language: 'ru', @@ -1065,6 +1169,32 @@ export default [ 0x70726f, 0x72656e, ], }, + { + encoding: 'Shift_JIS', + language: 'ja', + documents: 4, + trigrams: [ + 0x81420a, 0x8ed282, 0x815b83, 0x82a982, 0x82aa95, 0x82c982, 0x82cc83, + 0x82e981, 0x82f08a, 0xd282aa, 0xe98142, 0x0a0a92, 0x5882aa, 0x738fea, + 0x7d8f91, 0x82a282, 0x82aa94, 0x82ad82, 0x82b581, 0x82b781, 0x82b782, + 0x82c182, 0x82c482, 0x82cc8e, 0x82cc92, 0x82dc82, 0x82df82, 0x82e882, + 0x8a5882, 0x8ad982, 0x8e738f, 0x8f9182, 0x8f918a, 0x8fea82, 0x907d8f, + 0x90c382, 0x918ad9, 0x967b82, 0xa88141, 0xa982c8, 0xb58141, 0xb78142, + 0xc382a9, 0xc982cd, 0xcc8e73, 0xe082f0, 0x0a0a8d, 0x0a0a8f, 0x0a8d4c, + 0x0a8fac, 0x0a926e, 0x0a92a9, 0x4189ca, 0x418a4a, 0x418b69, 0x418dc5, + 0x418e69, 0x418fc4, 0x4195d2, 0x419649, 0x428977, 0x4290dd, 0x4293c7, + 0x429483, 0x4682b5, 0x4790df, 0x48965b, 0x4996a8, 0x4a82af, 0x4a94ad, + 0x4c8fea, 0x4e82c8, 0x52815b, 0x5482b5, 0x56914e, 0x5782dc, 0x5782f0, + 0x57835e, 0x578ed2, 0x5882cc, 0x5882cd, 0x5b0a0a, 0x5b8357, 0x5b8368, + 0x5b8380, 0x5e838b, 0x5f89c6, 0x60815b, 0x6082a4, 0x6282b7, 0x668357, + 0x6882f0, 0x698f91, 0x699283, 0x6a82e2, 0x6a93fa, 0x6c82aa, 0x6d9446, + 0x6e82df, 0x6e88e6, 0x6f82dc, 0x6f8358, 0x708393, 0x7182cd, 0x7382cc, + 0x768ed2, 0x7782c9, 0x7782e2, 0x7790b6, 0x79815b, 0x79976a, 0x7b82cc, + 0x7b82f0, 0x8082aa, 0x814189, 0x81418a, 0x81418b, 0x81418d, 0x81418e, + 0x81418f, 0x814195, 0x814196, 0x814289, 0x814290, 0x814293, 0x814294, + 0x82a290, 0x82a295, + ], + }, { encoding: 'windows-1250', language: 'cs', diff --git a/corpus/manifest.json b/corpus/manifest.json index c747e1e..776c6c4 100644 --- a/corpus/manifest.json +++ b/corpus/manifest.json @@ -53,7 +53,44 @@ "iconv": "MAC-CYRILLIC", "languages": ["ru", "uk"] }, - { "name": "CP949", "iconv": "CP949", "languages": ["ko"] }, + { + "name": "Shift_JIS", + "iconv": "SHIFT_JIS", + "languages": ["ja"], + "family": "multibyte" + }, + { + "name": "EUC-JP", + "iconv": "EUC-JP", + "languages": ["ja"], + "family": "multibyte" + }, + { + "name": "Big5", + "iconv": "BIG5", + "languages": ["zh"], + "family": "multibyte" + }, + { + "name": "GB18030", + "iconv": "GB18030", + "languages": ["zh"], + "family": "multibyte" + }, + { + "name": "EUC-KR", + "iconv": "EUC-KR", + "languages": ["ko"], + "family": "multibyte", + "sourceReplacements": { "똠": "글", "뷁": "자" } + }, + { + "name": "CP949", + "iconv": "CP949", + "languages": ["ko"], + "family": "multibyte", + "runtime": false + }, { "name": "ISO-8859-3", "iconv": "ISO-8859-3", "languages": ["mt"] }, { "name": "ISO-8859-4", "iconv": "ISO-8859-4", "languages": ["lv"] }, { "name": "ISO-8859-10", "iconv": "ISO-8859-10", "languages": ["is"] }, diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 3021440..dde38c1 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -8687,5 +8687,304 @@ } ] } - ] + ], + "multibyte": { + "summary": { + "tests": 5, + "encodingCorrect": 5, + "languageCorrect": 5 + }, + "results": [ + { + "expected": { + "encoding": "Big5", + "language": "zh" + }, + "predicted": { + "encoding": "Big5", + "language": "zh", + "confidence": 0.31, + "hits": 16, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.31, + "hits": 16, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 27, + "invalidCharacters": 25 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 1, + "multibyteCharacters": 27, + "invalidCharacters": 25 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 13, + "invalidCharacters": 1 + } + ] + }, + { + "expected": { + "encoding": "EUC-JP", + "language": "ja" + }, + "predicted": { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.1, + "hits": 6, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.03, + "hits": 2, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 21, + "invalidCharacters": 1 + } + ] + }, + { + "expected": { + "encoding": "EUC-KR", + "language": "ko" + }, + "predicted": { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.59, + "hits": 41, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.59, + "hits": 41, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.03, + "hits": 2, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.01, + "hits": 1, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.01, + "hits": 1, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 12, + "invalidCharacters": 7 + } + ] + }, + { + "expected": { + "encoding": "GB18030", + "language": "zh" + }, + "predicted": { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.33, + "hits": 17, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.33, + "hits": 17, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.02, + "hits": 1, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 4, + "multibyteCharacters": 40, + "invalidCharacters": 7 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 1, + "multibyteCharacters": 40, + "invalidCharacters": 7 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 14, + "invalidCharacters": 0 + } + ] + }, + { + "expected": { + "encoding": "Shift_JIS", + "language": "ja" + }, + "predicted": { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 3, + "invalidCharacters": 48 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 3, + "invalidCharacters": 48 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + } + ] + } + ] + } } diff --git a/corpus/sources.json b/corpus/sources.json index e65eab5..83a82ae 100644 --- a/corpus/sources.json +++ b/corpus/sources.json @@ -1081,6 +1081,78 @@ "text": "Bạn bè mang bản đồ và trà nóng, đi qua rừng rồi nghỉ bên bờ sông." } ] + }, + { + "code": "ja", + "name": "Japanese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "街の朝", + "text": "朝の街はゆっくり目を覚ます。駅には通勤する人が集まり、喫茶店が扉を開け、最初のバスが静かな通りを走り始める。" + }, + { + "id": "library", + "split": "train", + "title": "静かな図書館", + "text": "地域の図書館では本のページをめくる音が聞こえる。読者は歴史や科学や旅行の本を探し、司書は学生の調べ物を手伝う。" + }, + { + "id": "market", + "split": "train", + "title": "土曜日の市場", + "text": "広場の市場には新鮮な野菜、果物、蜂蜜、焼きたてのパンが並ぶ。買い物客は農家と季節の料理について話す。" + }, + { + "id": "workshop", + "split": "train", + "title": "デジタル工房", + "text": "小さなチームが博物館の案内を作っている。設計者が画面を整え、開発者がコードを確認し、編集者が分かりやすい説明を書く。" + }, + { + "id": "river-trip", + "split": "validation", + "title": "川への小旅行", + "text": "友人たちは地図と温かいお茶を持って森を歩いた。長い散歩の後、静かな川辺で休みながら夕方の鳥の声を聞いた。" + } + ] + }, + { + "code": "zh", + "name": "Chinese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "城市的早晨", + "text": "城市慢慢醒來，車站聚集了上班的人，咖啡店打開大門，第一班公共汽車開始穿過安靜的街道。" + }, + { + "id": "library", + "split": "train", + "title": "安靜的圖書館", + "text": "社區圖書館裡只有翻書的聲音。讀者尋找歷史、科學和旅行方面的書，管理員幫助學生查找資料。" + }, + { + "id": "market", + "split": "train", + "title": "星期六的市場", + "text": "廣場上的市場擺滿新鮮蔬菜、水果、蜂蜜和麵包。顧客向農民詢問收成，並交流家常菜的做法。" + }, + { + "id": "workshop", + "split": "train", + "title": "數位工作室", + "text": "一個小團隊正在製作博物館指南。設計師整理頁面，開發者檢查程式，編輯為不同年齡的參觀者撰寫清楚的說明。" + }, + { + "id": "river-trip", + "split": "validation", + "title": "河邊旅行", + "text": "朋友們帶著地圖和熱茶穿過樹林。走了很久以後，他們坐在安靜的河邊休息，聽傍晚的鳥鳴。" + } + ] } ] } diff --git a/corpus/test-sources.json b/corpus/test-sources.json index e138203..fb6a3c8 100644 --- a/corpus/test-sources.json +++ b/corpus/test-sources.json @@ -181,6 +181,18 @@ "id": "community-garden", "title": "Vườn cộng đồng", "text": "Hàng xóm biến sân trống thành khu vườn xanh, trẻ em vẽ biển màu và các gia đình tưới cây mỗi tuần." + }, + { + "language": "ja", + "id": "community-garden", + "title": "地域の庭", + "text": "近所の人たちは空いていた庭を整え、花や野菜や若い木を植えた。子どもたちは色鮮やかな札を作り、家族は毎週植物に水をやる。" + }, + { + "language": "zh", + "id": "community-garden", + "title": "社區花園", + "text": "鄰居們把空院子改造成綠色花園，種下鮮花、蔬菜和小樹。孩子們畫了彩色標牌，各家每週一起給植物澆水。" } ] } diff --git a/scripts/benchmark.mjs b/scripts/benchmark.mjs index f5be321..d5bb06e 100644 --- a/scripts/benchmark.mjs +++ b/scripts/benchmark.mjs @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -8,6 +9,15 @@ const generatedCorpus = join(root, 'corpus', 'generated'); const corpusIndex = JSON.parse( readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), ); +const manifest = JSON.parse( + readFileSync(join(root, 'corpus', 'manifest.json'), 'utf8'), +); +const familyByEncoding = new Map( + manifest.encodings.map((encoding) => [ + encoding.name, + encoding.family === 'multibyte' ? 'multibyte' : 'singlebyte', + ]), +); function option(name, fallback) { const prefix = `--${name}=`; @@ -17,13 +27,19 @@ function option(name, fallback) { const iterations = Number(option('iterations', '100')); const split = option('split', 'all'); +const family = option('family', 'all'); if (!Number.isInteger(iterations) || iterations < 1) { throw new Error('--iterations must be a positive integer'); } +if (!['all', 'singlebyte', 'multibyte'].includes(family)) { + throw new Error('--family must be all, singlebyte, or multibyte'); +} const rows = corpusIndex.filter( - (row) => split === 'all' || row.split === split, + (row) => + (split === 'all' || row.split === split) && + (family === 'all' || familyByEncoding.get(row.encoding) === family), ); if (rows.length === 0) { throw new Error(`No corpus files matched split: ${split}`); @@ -33,6 +49,9 @@ const inputs = rows.map((row) => ({ ...row, buffer: readFileSync(join(generatedCorpus, row.path)), })); +const workloadHash = createHash('sha256') + .update(rows.map((row) => `${row.path}\0${row.sha256}\n`).join('')) + .digest('hex'); for (const input of inputs) analyse(input.buffer); @@ -54,7 +73,15 @@ const bytesPerSecond = bytes / (elapsedMs / 1000); console.log('Library corpus benchmark'); console.log('========================'); console.log(`Split: ${split}`); +console.log(`Family: ${family}`); console.log(`Files: ${inputs.length}`); +console.log( + `Input bytes per iteration: ${inputs.reduce( + (total, input) => total + input.buffer.length, + 0, + )}`, +); +console.log(`Workload SHA-256: ${workloadHash}`); console.log(`Iterations: ${iterations}`); console.log(`Calls: ${calls}`); console.log(`Bytes processed: ${bytes}`); diff --git a/scripts/model-utils.mjs b/scripts/model-utils.mjs index e068aa9..604db30 100644 --- a/scripts/model-utils.mjs +++ b/scripts/model-utils.mjs @@ -7,6 +7,8 @@ import { scoreSBCS, trigrams, } from '../src/encoding/sbcs-scoring.ts'; +import { big5, euc_jp, euc_kr, gb_18030, sjis } from '../src/encoding/mbcs.ts'; +import { prepareMBCSModel, scoreMBCS } from '../src/encoding/mbcs-scoring.ts'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const corpus = join(root, 'corpus'); @@ -162,7 +164,7 @@ function byteEquivalent(buffer, leftEncoding, rightEncoding) { function compileSingleByteModels() { const models = []; for (const encoding of manifest.encodings) { - if (encoding.name === 'CP949') continue; + if (encoding.family === 'multibyte') continue; const byteMap = byteMapFor(encoding.iconv); const languages = encoding.languages.map((language) => { @@ -190,10 +192,88 @@ function compileSingleByteModels() { return models; } +const multibyteRecognisers = new Map([ + ['Shift_JIS', new sjis()], + ['Big5', new big5()], + ['EUC-JP', new euc_jp()], + ['EUC-KR', new euc_kr()], + ['GB18030', new gb_18030()], +]); + +function detectorContext(buffer) { + const byteStats = Array(256).fill(0); + for (const byte of buffer) byteStats[byte] += 1; + return { + byteStats, + c1Bytes: buffer.some((byte) => byte >= 0x80 && byte <= 0x9f), + rawInput: buffer, + rawLen: buffer.length, + inputBytes: buffer, + inputLen: buffer.length, + }; +} + +function compileMultibyteModels() { + return manifest.encodings + .filter( + (encoding) => + encoding.family === 'multibyte' && encoding.runtime !== false, + ) + .map((encoding) => { + const recogniser = multibyteRecognisers.get(encoding.name); + if (!recogniser) { + throw new Error(`Missing multibyte recogniser: ${encoding.name}`); + } + if (encoding.languages.length !== 1) { + throw new Error( + `Expected one language for multibyte model: ${encoding.name}`, + ); + } + const language = encoding.languages[0]; + const frequencies = new Map(); + let total = 0; + for (const row of corpusIndex.filter( + (candidate) => + candidate.encoding === encoding.name && + candidate.language === language && + candidate.split === 'train', + )) { + const buffer = readFileSync(join(generatedCorpus, row.path)); + const statistics = recogniser.statistics(detectorContext(buffer)); + if (statistics.invalidCharacters !== 0) { + throw new Error( + `${encoding.name}/${language}/${row.document} has invalid multibyte characters`, + ); + } + for (const value of statistics.multibyteCharacters) { + frequencies.set(value, (frequencies.get(value) ?? 0) + 1); + total += 1; + } + } + if (total === 0) { + throw new Error(`Empty multibyte model: ${encoding.name}/${language}`); + } + return { + encoding: encoding.name, + language, + total, + commonCharacters: [...frequencies] + .sort( + ([left, leftCount], [right, rightCount]) => + rightCount - leftCount || left - right, + ) + .slice(0, 128) + .map(([value]) => value) + .sort((left, right) => left - right), + }; + }); +} + function evaluate(models) { const preparedModels = prepareSBCSModels(models); + const modelNames = new Set(models.map((model) => model.encoding)); const tests = corpusIndex.filter( - (row) => row.split === 'test' && row.encoding !== 'CP949', + (row) => row.split === 'test' && modelNames.has(row.encoding), ); const results = tests.map((test) => { const buffer = readFileSync(join(generatedCorpus, test.path)); @@ -243,6 +323,56 @@ function evaluate(models) { }; } +function evaluateMultibyte(models) { + const preparedModels = models.map((model) => ({ + model, + prepared: prepareMBCSModel(model), + recogniser: multibyteRecognisers.get(model.encoding), + })); + const tests = corpusIndex.filter( + (row) => + row.split === 'test' && + models.some((model) => model.encoding === row.encoding), + ); + const results = tests.map((test) => { + const buffer = readFileSync(join(generatedCorpus, test.path)); + const candidates = preparedModels + .map(({ model, prepared, recogniser }) => { + const score = scoreMBCS( + recogniser.statistics(detectorContext(buffer)), + prepared, + ); + return { + encoding: model.encoding, + language: model.language, + ...score, + }; + }) + .sort( + (left, right) => + right.confidence - left.confidence || + left.encoding.localeCompare(right.encoding), + ); + return { + expected: { encoding: test.encoding, language: test.language }, + predicted: candidates[0], + encodingCorrect: candidates[0].encoding === test.encoding, + languageCorrect: candidates[0].language === test.language, + candidates, + }; + }); + return { + summary: { + tests: results.length, + encodingCorrect: results.filter((result) => result.encodingCorrect) + .length, + languageCorrect: results.filter((result) => result.languageCorrect) + .length, + }, + results, + }; +} + function javascriptString(value) { return `'${value.replaceAll('\\', '\\\\').replaceAll("'", "\\'")}'`; } @@ -251,9 +381,10 @@ function hex(value, width) { return `0x${value.toString(16).padStart(width, '0')}`; } -function serializeModels(models) { +function serializeModels(models, multibyteModels) { const lines = [ '// Generated by npm run models:build. Do not edit manually.', + "import type { GeneratedMBCSModel } from './types';", '', 'export const generatedSBCSModels = [', ]; @@ -307,18 +438,47 @@ function serializeModels(models) { lines.push(' },'); } lines.push('] as const;', ''); + lines.push( + 'export const generatedMBCSModels: readonly GeneratedMBCSModel[] = [', + ); + for (const model of multibyteModels) { + lines.push(' {'); + lines.push(` encoding: ${javascriptString(model.encoding)},`); + lines.push(` language: ${javascriptString(model.language)},`); + lines.push(` total: ${model.total},`); + lines.push(' commonCharacters: ['); + for (let index = 0; index < model.commonCharacters.length; index += 6) { + lines.push( + ` ${model.commonCharacters + .slice(index, index + 6) + .map((value) => hex(value, 8)) + .join(', ')},`, + ); + } + lines.push(' ],'); + lines.push(' },'); + } + lines.push('];', ''); return lines.join('\n'); } export function compileModels() { const models = compileSingleByteModels(); - return { models, report: evaluate(models) }; + const multibyteModels = compileMultibyteModels(); + return { + models, + multibyteModels, + report: { + ...evaluate(models), + multibyte: evaluateMultibyte(multibyteModels), + }, + }; } export function buildModels(modelPath, evaluationPath) { - const { models, report } = compileModels(); + const { models, multibyteModels, report } = compileModels(); mkdirSync(dirname(modelPath), { recursive: true }); mkdirSync(dirname(evaluationPath), { recursive: true }); - writeFileSync(modelPath, serializeModels(models)); + writeFileSync(modelPath, serializeModels(models, multibyteModels)); writeFileSync(evaluationPath, `${JSON.stringify(report, null, 2)}\n`); } diff --git a/scripts/verify-models.mjs b/scripts/verify-models.mjs index 56442e5..a964fce 100644 --- a/scripts/verify-models.mjs +++ b/scripts/verify-models.mjs @@ -15,12 +15,24 @@ try { if (!readFileSync(rebuiltModel).equals(readFileSync(generatedModel))) { throw new Error('Generated detector models are out of date'); } + const evaluation = JSON.parse(readFileSync(rebuiltEvaluation, 'utf8')); if ( !readFileSync(rebuiltEvaluation).equals(readFileSync(generatedEvaluation)) ) { throw new Error('Generated model evaluation is out of date'); } - const evaluation = JSON.parse(readFileSync(rebuiltEvaluation, 'utf8')); + if ( + evaluation.multibyte.summary.encodingCorrect !== + evaluation.multibyte.summary.tests || + evaluation.multibyte.summary.languageCorrect !== + evaluation.multibyte.summary.tests + ) { + throw new Error( + `Multibyte model evaluation failed: ${evaluation.multibyte.summary.encodingCorrect}/` + + `${evaluation.multibyte.summary.tests} encodings, ${evaluation.multibyte.summary.languageCorrect}/` + + `${evaluation.multibyte.summary.tests} languages`, + ); + } if ( evaluation.summary.encodingCorrect !== evaluation.summary.tests || evaluation.summary.languageCorrect !== evaluation.summary.tests diff --git a/src/encoding/mbcs-scoring.test.ts b/src/encoding/mbcs-scoring.test.ts new file mode 100644 index 0000000..79354e4 --- /dev/null +++ b/src/encoding/mbcs-scoring.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import type { GeneratedMBCSModel } from './models/types'; +import { + prepareMBCSModel, + scoreMBCS, + type MBCSStatistics, +} from './mbcs-scoring'; + +const model: GeneratedMBCSModel = { + encoding: 'EUC-JP', + language: 'ja', + total: 100, + commonCharacters: [0xa1a1, 0xa4a2], +}; +const prepared = prepareMBCSModel(model); + +function statistics( + multibyteCharacters: number[], + invalidCharacters = 0, +): MBCSStatistics { + return { + totalCharacters: multibyteCharacters.length + invalidCharacters, + multibyteCharacters, + invalidCharacters, + }; +} + +describe('generated MBCS scoring', () => { + it('scores model coverage rather than structural compatibility alone', () => { + expect( + scoreMBCS(statistics(Array(20).fill(0xa4a2)), prepared), + ).toMatchObject({ confidence: 1, hits: 20 }); + expect( + scoreMBCS(statistics(Array(20).fill(0xb0b0)), prepared), + ).toMatchObject({ confidence: 0, hits: 0 }); + }); + + it('reduces model confidence when invalid sequences are present', () => { + expect( + scoreMBCS(statistics(Array(20).fill(0xa4a2), 1), prepared).confidence, + ).toBe(0.5); + }); + + it('returns finite boundary scores for short and empty input', () => { + expect(scoreMBCS(statistics([]), prepared).confidence).toBe(0); + expect(scoreMBCS(statistics([0xa4a2]), prepared).confidence).toBe(0.1); + }); +}); diff --git a/src/encoding/mbcs-scoring.ts b/src/encoding/mbcs-scoring.ts new file mode 100644 index 0000000..b47b32a --- /dev/null +++ b/src/encoding/mbcs-scoring.ts @@ -0,0 +1,55 @@ +import type { GeneratedMBCSModel } from './models/types'; + +export interface MBCSStatistics { + readonly totalCharacters: number; + readonly multibyteCharacters: readonly number[]; + readonly invalidCharacters: number; +} + +export interface MBCSScore { + readonly confidence: number; + readonly hits: number; + readonly multibyteCharacters: number; + readonly invalidCharacters: number; +} + +export interface PreparedMBCSModel { + readonly model: GeneratedMBCSModel; + readonly commonCharacters: ReadonlySet; +} + +export function prepareMBCSModel(model: GeneratedMBCSModel): PreparedMBCSModel { + return { + model, + commonCharacters: new Set(model.commonCharacters), + }; +} + +export function scoreMBCS( + statistics: MBCSStatistics, + prepared: PreparedMBCSModel, +): MBCSScore { + const multibyteCharacters = statistics.multibyteCharacters.length; + const invalidCharacters = statistics.invalidCharacters; + const hits = statistics.multibyteCharacters.reduce( + (count, value) => count + (prepared.commonCharacters.has(value) ? 1 : 0), + 0, + ); + + let confidence = 0; + if (multibyteCharacters <= 10 && invalidCharacters === 0) { + confidence = + multibyteCharacters === 0 && statistics.totalCharacters < 10 ? 0 : 0.1; + } else if (multibyteCharacters >= 20 * invalidCharacters) { + const structuralValidity = + multibyteCharacters / (multibyteCharacters + invalidCharacters * 20); + confidence = (hits / multibyteCharacters) * structuralValidity; + } + + return { + confidence: Number(Math.min(confidence, 1).toFixed(2)), + hits, + multibyteCharacters, + invalidCharacters, + }; +} diff --git a/src/encoding/mbcs.test.ts b/src/encoding/mbcs.test.ts index 4a510bc..5a0b418 100644 --- a/src/encoding/mbcs.test.ts +++ b/src/encoding/mbcs.test.ts @@ -1,8 +1,11 @@ import * as chardet from '..'; +import fs from 'fs'; +import path from 'path'; import { describe, expect, it } from 'vitest'; describe('Multibyte Character Sets', () => { const base = __dirname + '/../test/data/encodings'; + const corpus = path.join(__dirname, '../../corpus/generated'); it('should return Shift_JIS', () => { expect(chardet.detectFileSync(base + '/shiftjis')).toBe('Shift_JIS'); @@ -23,4 +26,23 @@ describe('Multibyte Character Sets', () => { it('should return EUC-KR', () => { expect(chardet.detectFileSync(base + '/euc_kr')).toBe('EUC-KR'); }); + + it.each([ + ['Shift_JIS', 'ja'], + ['EUC-JP', 'ja'], + ['Big5', 'zh'], + ['GB18030', 'zh'], + ['EUC-KR', 'ko'], + ])('uses the generated %s model for held-out %s text', (encoding, lang) => { + const fixture = path.join( + corpus, + encoding, + lang, + 'test/community-garden.bin', + ); + expect(chardet.analyse(fs.readFileSync(fixture))[0]).toMatchObject({ + name: encoding, + lang, + }); + }); }); diff --git a/src/encoding/mbcs.ts b/src/encoding/mbcs.ts index 1334a53..0dff1c0 100644 --- a/src/encoding/mbcs.ts +++ b/src/encoding/mbcs.ts @@ -1,59 +1,26 @@ import type { Context, Recogniser } from '.'; -import match, { type Match, type EncodingName } from '../match'; +import match, { type EncodingName, type Match } from '../match'; +import { + prepareMBCSModel, + scoreMBCS, + type MBCSStatistics, +} from './mbcs-scoring'; +import { generatedMBCSModels } from './models/generated'; +import type { GeneratedMBCSModel } from './models/types'; + +const models = new Map( + (generatedMBCSModels as readonly GeneratedMBCSModel[]).map((model) => [ + model.encoding, + prepareMBCSModel(model), + ]), +); -/** - * Binary search implementation (recursive) - */ -function binarySearch(arr: number[], searchValue: number) { - const find = ( - arr: number[], - searchValue: number, - left: number, - right: number, - ): number => { - if (right < left) return -1; - - /* - int mid = mid = (left + right) / 2; - There is a bug in the above line; - Joshua Bloch suggests the following replacement: - */ - const mid = Math.floor((left + right) >>> 1); - if (searchValue > arr[mid]) return find(arr, searchValue, mid + 1, right); - - if (searchValue < arr[mid]) return find(arr, searchValue, left, mid - 1); - - return mid; - }; - - return find(arr, searchValue, 0, arr.length - 1); -} - -// 'Character' iterated character class. -// Recognizers for specific mbcs encodings make their 'characters' available -// by providing a nextChar() function that fills in an instance of iteratedChar -// with the next char from the input. -// The returned characters are not converted to Unicode, but remain as the raw -// bytes (concatenated into an int) from the codepage data. -// -// For Asian charsets, use the raw input rather than the input that has been -// stripped of markup. Detection only considers multi-byte chars, effectively -// stripping markup anyway, and double byte chars do occur in markup too. -// class IteratedChar { - charValue: number; // 1-4 bytes from the raw input data - index: number; - nextIndex: number; - error: boolean; - done: boolean; - - constructor() { - this.charValue = 0; // 1-4 bytes from the raw input data - this.index = 0; - this.nextIndex = 0; - this.error = false; - this.done = false; - } + charValue = 0; + index = 0; + nextIndex = 0; + error = false; + done = false; reset() { this.charValue = 0; @@ -68,128 +35,43 @@ class IteratedChar { this.done = true; return -1; } - const byteValue = det.rawInput[this.nextIndex++] & 0x00ff; - return byteValue; + return det.rawInput[this.nextIndex++] & 0xff; } } /** - * Asian double or multi-byte - charsets. - * Match is determined mostly by the input data adhering to the - * encoding scheme for the charset, and, optionally, - * frequency-of-occurrence of characters. + * Base recogniser for encodings whose structural validity is determined by + * parsing variable-width byte sequences. Character likelihood comes from the + * generated corpus model for the concrete encoding. */ - -abstract class mbcs implements Recogniser { - commonChars: number[] = []; - +export abstract class mbcs implements Recogniser { abstract name(): EncodingName; + abstract nextChar(iter: IteratedChar, det: Context): boolean; - /** - * Test the match of this charset with the input text data - * which is obtained via the CharsetDetector object. - * - * @param det The CharsetDetector, which contains the input text - * to be checked for being in this charset. - * @return A match with confidence ranging from 0 to 1. - */ - match(det: Context): Match | null { - let doubleByteCharCount = 0, - commonCharCount = 0, - badCharCount = 0, - totalCharCount = 0, - confidence = 0; - + statistics(det: Context): MBCSStatistics { + let invalidCharacters = 0; + let totalCharacters = 0; + const multibyteCharacters: number[] = []; const iter = new IteratedChar(); - detectBlock: { - for (iter.reset(); this.nextChar(iter, det); ) { - totalCharCount++; - if (iter.error) { - badCharCount++; - } else { - const cv = iter.charValue & 0xffffffff; - - if (cv > 0xff) { - doubleByteCharCount++; - if (this.commonChars != null) { - // NOTE: This assumes that there are no 4-byte common chars. - if (binarySearch(this.commonChars, cv) >= 0) { - commonCharCount++; - } - } - } - } - if (badCharCount >= 2 && badCharCount * 5 >= doubleByteCharCount) { - // console.log('its here!') - // Bail out early if the byte data is not matching the encoding scheme. - break detectBlock; - } + for (iter.reset(); this.nextChar(iter, det); ) { + totalCharacters += 1; + if (iter.error) invalidCharacters += 1; + else if (iter.charValue > 0xff) { + multibyteCharacters.push(iter.charValue >>> 0); } - - if (doubleByteCharCount <= 10 && badCharCount == 0) { - // Not many multi-byte chars. - if (doubleByteCharCount == 0 && totalCharCount < 10) { - // There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes. - // We don't have enough data to have any confidence. - // Statistical analysis of single byte non-ASCII characters would probably help here. - confidence = 0; - } else { - // ASCII or ISO file? It's probably not our encoding, - // but is not incompatible with our encoding, so don't give it a zero. - confidence = 0.1; - } - break detectBlock; - } - - // - // No match if there are too many characters that don't fit the encoding scheme. - // (should we have zero tolerance for these?) - // - if (doubleByteCharCount < 20 * badCharCount) { - confidence = 0; - break detectBlock; - } - - if (this.commonChars == null) { - // We have no statistics on frequently occurring characters. - // Assess confidence purely on having a reasonable number of - // multi-byte characters (the more the better - confidence = (30 + doubleByteCharCount - 20 * badCharCount) / 100; - if (confidence > 1) { - confidence = 1; - } - } else { - // Frequency of occurrence statistics exist. - const maxVal = Math.log(doubleByteCharCount / 4); - const scaleFactor = 90.0 / maxVal; - confidence = - Math.floor(Math.log(commonCharCount + 1) * scaleFactor + 10) / 100; - confidence = Math.min(confidence, 1); - } - } // end of detectBlock: - - return confidence == 0 ? null : match(det, this, confidence); + } + return { totalCharacters, multibyteCharacters, invalidCharacters }; } - /** - * Get the next character (however many bytes it is) from the input data - * Subclasses for specific charset encodings must implement this function - * to get characters according to the rules of their encoding scheme. - * - * This function is not a method of class iteratedChar only because - * that would require a lot of extra derived classes, which is awkward. - * @param it The iteratedChar 'struct' into which the returned char is placed. - * @param det The charset detector, which is needed to get at the input byte data - * being iterated over. - * @return True if a character was returned, false at end of input. - */ - abstract nextChar(iter: IteratedChar, det: Context): boolean; + match(det: Context): Match | null { + const prepared = models.get(this.name()); + if (!prepared) return null; + const { confidence } = scoreMBCS(this.statistics(det), prepared); + return confidence === 0 ? null : match(det, this, confidence); + } } -/** - * Shift_JIS charset recognizer. - */ export class sjis extends mbcs { name(): EncodingName { return 'Shift_JIS'; @@ -199,29 +81,17 @@ export class sjis extends mbcs { return 'ja'; } - commonChars = [ - 0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, - 0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, - 0x82b3, 0x82b5, 0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, - 0x82c8, 0x82c9, 0x82cc, 0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, - 0x82ea, 0x82f0, 0x82f1, 0x8341, 0x8343, 0x834e, 0x834f, 0x8358, 0x835e, - 0x8362, 0x8367, 0x8375, 0x8376, 0x8389, 0x838a, 0x838b, 0x838d, 0x8393, - 0x8e96, 0x93fa, 0x95aa, - ]; - nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - const firstByte = (iter.charValue = iter.nextByte(det)); if (firstByte < 0) return false; - - if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) + if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) { return true; + } const secondByte = iter.nextByte(det); if (secondByte < 0) return false; - iter.charValue = (firstByte << 8) | secondByte; if ( !( @@ -229,16 +99,12 @@ export class sjis extends mbcs { (secondByte >= 0x80 && secondByte <= 0xff) ) ) { - // Illegal second byte value. iter.error = true; } return true; } } -/** - * Big5 charset recognizer. - */ export class big5 extends mbcs { name(): EncodingName { return 'Big5'; @@ -247,109 +113,53 @@ export class big5 extends mbcs { language() { return 'zh'; } - commonChars = [ - 0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, - 0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, - 0xa477, 0xa4a3, 0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, - 0xa4fd, 0xa540, 0xa548, 0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, - 0xa662, 0xa668, 0xa670, 0xa6a8, 0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, - 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da, 0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, - 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, 0xaa6b, 0xaaba, 0xaabe, - 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, 0xaec9, 0xafe0, - 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, 0xb5a5, - 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, - 0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f, - ]; nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - const firstByte = (iter.charValue = iter.nextByte(det)); - if (firstByte < 0) return false; - - // single byte character. - if (firstByte <= 0x7f || firstByte == 0xff) return true; + if (firstByte <= 0x7f || firstByte === 0xff) return true; const secondByte = iter.nextByte(det); - if (secondByte < 0) return false; - - iter.charValue = (iter.charValue << 8) | secondByte; - - if (secondByte < 0x40 || secondByte == 0x7f || secondByte == 0xff) + iter.charValue = (firstByte << 8) | secondByte; + if (secondByte < 0x40 || secondByte === 0x7f || secondByte === 0xff) { iter.error = true; - + } return true; } } -/** - * EUC charset recognizers. One abstract class that provides the common function - * for getting the next character according to the EUC encoding scheme, - * and nested derived classes for EUC_KR, EUC_JP, EUC_CN. - * - * Get the next character value for EUC based encodings. - * Character 'value' is simply the raw bytes that make up the character - * packed into an int. - */ function eucNextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - let firstByte = 0; - let secondByte = 0; - let thirdByte = 0; - //int fourthByte = 0; - buildChar: { - firstByte = iter.charValue = iter.nextByte(det); - if (firstByte < 0) { - // Ran off the end of the input data - iter.done = true; - break buildChar; - } - if (firstByte <= 0x8d) { - // single byte char - break buildChar; - } - secondByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | secondByte; - if (firstByte >= 0xa1 && firstByte <= 0xfe) { - // Two byte Char - if (secondByte < 0xa1) { - iter.error = true; - } - break buildChar; - } - if (firstByte == 0x8e) { - // Code Set 2. - // In EUC-JP, total char size is 2 bytes, only one byte of actual char value. - // In EUC-TW, total char size is 4 bytes, three bytes contribute to char value. - // We don't know which we've got. - // Treat it like EUC-JP. If the data really was EUC-TW, the following two - // bytes will look like a well formed 2 byte char. - if (secondByte < 0xa1) { - iter.error = true; - } - break buildChar; - } - if (firstByte == 0x8f) { - // Code set 3. - // Three byte total char size, two bytes of actual char value. - thirdByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | thirdByte; - if (thirdByte < 0xa1) { - iter.error = true; - } - } + const firstByte = (iter.charValue = iter.nextByte(det)); + if (firstByte < 0) { + iter.done = true; + return false; + } + if (firstByte <= 0x8d) return true; + + const secondByte = iter.nextByte(det); + if (secondByte < 0) return false; + iter.charValue = (firstByte << 8) | secondByte; + if (firstByte >= 0xa1 && firstByte <= 0xfe) { + if (secondByte < 0xa1) iter.error = true; + return true; + } + if (firstByte === 0x8e) { + if (secondByte < 0xa1) iter.error = true; + return true; } - return iter.done == false; + if (firstByte === 0x8f) { + const thirdByte = iter.nextByte(det); + iter.charValue = (iter.charValue << 8) | thirdByte; + if (thirdByte < 0xa1) iter.error = true; + } + return !iter.done; } -/** - * The charset recognize for EUC-JP. A singleton instance of this class - * is created and kept by the public CharsetDetector class - */ export class euc_jp extends mbcs { name(): EncodingName { return 'EUC-JP'; @@ -359,28 +169,9 @@ export class euc_jp extends mbcs { return 'ja'; } - commonChars = [ - 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, - 0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, - 0xa4b1, 0xa4b3, 0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, - 0xa4c1, 0xa4c3, 0xa4c4, 0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, - 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de, 0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, - 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, 0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, - 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, 0xa5b0, 0xa5b3, 0xa5b5, - 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, 0xa5c8, 0xa5c9, - 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, 0xa5e5, - 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, - 0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, - 0xcdd1, - ]; - nextChar = eucNextChar; } -/** - * The charset recognize for EUC-KR. A singleton instance of this class - * is created and kept by the public CharsetDetector class - */ export class euc_kr extends mbcs { name(): EncodingName { return 'EUC-KR'; @@ -390,27 +181,9 @@ export class euc_kr extends mbcs { return 'ko'; } - commonChars = [ - 0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, - 0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, - 0xb4cf, 0xb4d9, 0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, - 0xb7af, 0xb7c2, 0xb7ce, 0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, - 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce, 0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, - 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, 0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, - 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, 0xbef8, 0xbefa, 0xbfa1, - 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, 0xc0af, 0xc0b8, - 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, 0xc0da, - 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, - 0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, - 0xc8ad, - ]; - nextChar = eucNextChar; } -/** - * GB-18030 recognizer. Uses simplified Chinese statistics. - */ export class gb_18030 extends mbcs { name(): EncodingName { return 'GB18030'; @@ -420,71 +193,39 @@ export class gb_18030 extends mbcs { return 'zh'; } - /* - * Get the next character value for EUC based encodings. - * Character 'value' is simply the raw bytes that make up the character - * packed into an int. - */ - nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - let firstByte = 0; - let secondByte = 0; - let thirdByte = 0; - let fourthByte = 0; - buildChar: { - firstByte = iter.charValue = iter.nextByte(det); - if (firstByte < 0) { - // Ran off the end of the input data - iter.done = true; - break buildChar; - } - if (firstByte <= 0x80) { - // single byte char - break buildChar; + const firstByte = (iter.charValue = iter.nextByte(det)); + if (firstByte < 0) { + iter.done = true; + return false; + } + if (firstByte <= 0x80) return true; + + const secondByte = iter.nextByte(det); + if (secondByte < 0) return false; + iter.charValue = (firstByte << 8) | secondByte; + if (firstByte >= 0x81 && firstByte <= 0xfe) { + if ( + (secondByte >= 0x40 && secondByte <= 0x7e) || + (secondByte >= 0x80 && secondByte <= 0xfe) + ) { + return true; } - secondByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | secondByte; - if (firstByte >= 0x81 && firstByte <= 0xfe) { - // Two byte Char - if ( - (secondByte >= 0x40 && secondByte <= 0x7e) || - (secondByte >= 80 && secondByte <= 0xfe) - ) { - break buildChar; - } - // Four byte char - if (secondByte >= 0x30 && secondByte <= 0x39) { - thirdByte = iter.nextByte(det); - if (thirdByte >= 0x81 && thirdByte <= 0xfe) { - fourthByte = iter.nextByte(det); - if (fourthByte >= 0x30 && fourthByte <= 0x39) { - iter.charValue = - (iter.charValue << 16) | (thirdByte << 8) | fourthByte; - break buildChar; - } + if (secondByte >= 0x30 && secondByte <= 0x39) { + const thirdByte = iter.nextByte(det); + if (thirdByte >= 0x81 && thirdByte <= 0xfe) { + const fourthByte = iter.nextByte(det); + if (fourthByte >= 0x30 && fourthByte <= 0x39) { + iter.charValue = + (iter.charValue << 16) | (thirdByte << 8) | fourthByte; + return true; } } - iter.error = true; - break buildChar; } + iter.error = true; } - return iter.done == false; + return !iter.done; } - - commonChars = [ - 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, - 0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, - 0xb5bd, 0xb5c4, 0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, - 0xb7d6, 0xb7dd, 0xb8b4, 0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, - 0xb9fd, 0xbacd, 0xbba7, 0xbbd6, 0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, - 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, 0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, - 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, 0xc7f8, 0xc8ab, 0xc8cb, - 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, 0xcad0, 0xcad6, - 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, 0xcfb5, - 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, - 0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, - 0xd6d0, - ]; } diff --git a/src/encoding/models/generated.ts b/src/encoding/models/generated.ts index 4f81c87..5aad487 100644 --- a/src/encoding/models/generated.ts +++ b/src/encoding/models/generated.ts @@ -1,4 +1,5 @@ // Generated by npm run models:build. Do not edit manually. +import type { GeneratedMBCSModel } from './types'; export const generatedSBCSModels = [ { @@ -3189,3 +3190,151 @@ export const generatedSBCSModels = [ ], }, ] as const; + +export const generatedMBCSModels: readonly GeneratedMBCSModel[] = [ + { + encoding: 'Shift_JIS', + language: 'ja', + total: 238, + commonCharacters: [ + 0x00008141, 0x00008142, 0x0000815b, 0x000082a2, 0x000082a4, 0x000082a6, + 0x000082a9, 0x000082aa, 0x000082ab, 0x000082ad, 0x000082af, 0x000082b1, + 0x000082b3, 0x000082b5, 0x000082b7, 0x000082bd, 0x000082c1, 0x000082c2, + 0x000082c4, 0x000082c5, 0x000082c6, 0x000082c8, 0x000082c9, 0x000082cc, + 0x000082cd, 0x000082d4, 0x000082d7, 0x000082dc, 0x000082df, 0x000082e2, + 0x000082e4, 0x000082e8, 0x000082e9, 0x000082f0, 0x00008352, 0x00008357, + 0x00008358, 0x0000835e, 0x00008360, 0x00008366, 0x00008368, 0x0000836f, + 0x00008370, 0x00008379, 0x00008380, 0x0000838b, 0x00008393, 0x000088c4, + 0x000088e6, 0x00008977, 0x000089b9, 0x000089c6, 0x000089c8, 0x000089ca, + 0x000089e6, 0x00008a4a, 0x00008a58, 0x00008a6d, 0x00008a6f, 0x00008a77, + 0x00008ad9, 0x00008b47, 0x00008b69, 0x00008b71, 0x00008bce, 0x00008c76, + 0x00008d48, 0x00008d4c, 0x00008d73, 0x00008dc5, 0x00008dd8, 0x00008dec, + 0x00008e69, 0x00008e6a, 0x00008e6e, 0x00008e73, 0x00008ed2, 0x00008ee8, + 0x00008f57, 0x00008f89, 0x00008f91, 0x00008fac, 0x00008fc4, 0x00008fea, + 0x00009056, 0x0000906c, 0x0000907d, 0x000090ae, 0x000090b6, 0x000090c3, + 0x000090dd, 0x000090df, 0x000090e0, 0x0000914e, 0x00009196, 0x00009254, + 0x0000926e, 0x00009283, 0x000092a9, 0x000092b2, 0x000092ca, 0x00009358, + 0x00009360, 0x00009379, 0x000093c7, 0x000093e0, 0x000093fa, 0x00009446, + 0x0000945f, 0x00009483, 0x0000948e, 0x000094ad, 0x000094e0, 0x000095a8, + 0x000095aa, 0x000095b7, 0x000095c0, 0x000095d2, 0x00009649, 0x0000965b, + 0x0000967b, 0x000096a8, 0x000096be, 0x000096ca, 0x000096da, 0x000096ec, + 0x0000976a, 0x0000979d, + ], + }, + { + encoding: 'EUC-JP', + language: 'ja', + total: 238, + commonCharacters: [ + 0x0000a1a2, 0x0000a1a3, 0x0000a1bc, 0x0000a4a4, 0x0000a4a6, 0x0000a4a8, + 0x0000a4ab, 0x0000a4ac, 0x0000a4ad, 0x0000a4af, 0x0000a4b1, 0x0000a4b3, + 0x0000a4b5, 0x0000a4b7, 0x0000a4b9, 0x0000a4bf, 0x0000a4c3, 0x0000a4c4, + 0x0000a4c6, 0x0000a4c7, 0x0000a4c8, 0x0000a4ca, 0x0000a4cb, 0x0000a4ce, + 0x0000a4cf, 0x0000a4d6, 0x0000a4d9, 0x0000a4de, 0x0000a4e1, 0x0000a4e4, + 0x0000a4e6, 0x0000a4ea, 0x0000a4eb, 0x0000a4f2, 0x0000a5b3, 0x0000a5b8, + 0x0000a5b9, 0x0000a5bf, 0x0000a5c1, 0x0000a5c7, 0x0000a5c9, 0x0000a5d0, + 0x0000a5d1, 0x0000a5da, 0x0000a5e0, 0x0000a5eb, 0x0000a5f3, 0x0000b0c6, + 0x0000b0e8, 0x0000b1d8, 0x0000b2bb, 0x0000b2c8, 0x0000b2ca, 0x0000b2cc, + 0x0000b2e8, 0x0000b3ab, 0x0000b3b9, 0x0000b3ce, 0x0000b3d0, 0x0000b3d8, + 0x0000b4db, 0x0000b5a8, 0x0000b5ca, 0x0000b5d2, 0x0000b6d0, 0x0000b7d7, + 0x0000b9a9, 0x0000b9ad, 0x0000b9d4, 0x0000bac7, 0x0000bada, 0x0000baee, + 0x0000bbca, 0x0000bbcb, 0x0000bbcf, 0x0000bbd4, 0x0000bcd4, 0x0000bcea, + 0x0000bdb8, 0x0000bde9, 0x0000bdf1, 0x0000beae, 0x0000bec6, 0x0000beec, + 0x0000bfb7, 0x0000bfcd, 0x0000bfde, 0x0000c0b0, 0x0000c0b8, 0x0000c0c5, + 0x0000c0df, 0x0000c0e1, 0x0000c0e2, 0x0000c1af, 0x0000c1f6, 0x0000c3b5, + 0x0000c3cf, 0x0000c3e3, 0x0000c4ab, 0x0000c4b4, 0x0000c4cc, 0x0000c5b9, + 0x0000c5c1, 0x0000c5da, 0x0000c6c9, 0x0000c6e2, 0x0000c6fc, 0x0000c7a7, + 0x0000c7c0, 0x0000c7e3, 0x0000c7ee, 0x0000c8af, 0x0000c8e2, 0x0000caaa, + 0x0000caac, 0x0000cab9, 0x0000cac2, 0x0000cad4, 0x0000cbaa, 0x0000cbbc, + 0x0000cbdc, 0x0000ccaa, 0x0000ccc0, 0x0000cccc, 0x0000ccdc, 0x0000ccee, + 0x0000cdcb, 0x0000cdfd, + ], + }, + { + encoding: 'Big5', + language: 'zh', + total: 199, + commonCharacters: [ + 0x0000a141, 0x0000a142, 0x0000a143, 0x0000a440, 0x0000a446, 0x0000a448, + 0x0000a457, 0x0000a46a, 0x0000a470, 0x0000a475, 0x0000a4a3, 0x0000a4bb, + 0x0000a4bd, 0x0000a4e8, 0x0000a4f4, 0x0000a55d, 0x0000a575, 0x0000a576, + 0x0000a5ab, 0x0000a5b4, 0x0000a5bf, 0x0000a5c1, 0x0000a5cd, 0x0000a5e6, + 0x0000a640, 0x0000a650, 0x0000a656, 0x0000a662, 0x0000a677, 0x0000a67e, + 0x0000a6a1, 0x0000a6a8, 0x0000a6ac, 0x0000a6ad, 0x0000a6b3, 0x0000a6e6, + 0x0000a6ec, 0x0000a740, 0x0000a755, 0x0000a7e4, 0x0000a854, 0x0000a8ae, + 0x0000a8c3, 0x0000a8d3, 0x0000a940, 0x0000a94d, 0x0000a96c, 0x0000a9b1, + 0x0000a9fa, 0x0000aa47, 0x0000aa6b, 0x0000aaab, 0x0000aaba, 0x0000aac0, + 0x0000aacc, 0x0000aaf9, 0x0000ab6e, 0x0000abb0, 0x0000abc7, 0x0000abc8, + 0x0000abfc, 0x0000ac50, 0x0000ac64, 0x0000ac79, 0x0000acb0, 0x0000acec, + 0x0000acef, 0x0000ad70, 0x0000adb1, 0x0000adb5, 0x0000adb6, 0x0000add3, + 0x0000adfb, 0x0000ae61, 0x0000ae76, 0x0000aec6, 0x0000aec8, 0x0000aed1, + 0x0000af5a, 0x0000afb8, 0x0000b0b5, 0x0000b0cf, 0x0000b0d1, 0x0000b0d8, + 0x0000b0dd, 0x0000b160, 0x0000b1e1, 0x0000b24d, 0x0000b27a, 0x0000b2c4, + 0x0000b35d, 0x0000b3d5, 0x0000b3f5, 0x0000b44d, 0x0000b4c1, 0x0000b56f, + 0x0000b57b, 0x0000b5e6, 0x0000b5f3, 0x0000b67d, 0x0000b6a4, 0x0000b6b0, + 0x0000b773, 0x0000b7a1, 0x0000b8c1, 0x0000b8cc, 0x0000b8df, 0x0000b8ea, + 0x0000b941, 0x0000b944, 0x0000b94c, 0x0000b9ce, 0x0000b9cf, 0x0000ba43, + 0x0000baa1, 0x0000bade, 0x0000bb45, 0x0000bb65, 0x0000bb73, 0x0000bba1, + 0x0000bc67, 0x0000bc73, 0x0000bcb6, 0x0000bcc6, 0x0000bd73, 0x0000bec7, + 0x0000c052, 0x0000c05d, + ], + }, + { + encoding: 'GB18030', + language: 'zh', + total: 199, + commonCharacters: [ + 0x0000814b, 0x000081ed, 0x00008280, 0x0000855e, 0x000085a2, 0x00008654, + 0x00008696, 0x00008844, 0x00008846, 0x000088f6, 0x00008c57, 0x00008c91, + 0x00008ca4, 0x00008e9f, 0x00008ecd, 0x00008f56, 0x0000945b, 0x000094b5, + 0x000095f8, 0x0000997a, 0x00009a76, 0x00009d4d, 0x00009ee9, 0x0000a1a2, + 0x0000a1a3, 0x0000a3ac, 0x0000b06c, 0x0000b0b2, 0x0000b0e0, 0x0000b0fc, + 0x0000b2a9, 0x0000b2bb, 0x0000b2cb, 0x0000b2e9, 0x0000b3a3, 0x0000b3bf, + 0x0000b3c7, 0x0000b3c9, 0x0000b3cc, 0x0000b3fe, 0x0000b4a9, 0x0000b4f2, + 0x0000b4f3, 0x0000b5c0, 0x0000b5c4, 0x0000b5da, 0x0000b5ea, 0x0000b7a8, + 0x0000b7ad, 0x0000b7bd, 0x0000b7c8, 0x0000b7e4, 0x0000b9a4, 0x0000b9ab, + 0x0000b9b2, 0x0000b9dc, 0x0000b9fb, 0x0000bacd, 0x0000bcaf, 0x0000bcd2, + 0x0000bdbb, 0x0000bdd6, 0x0000be8e, 0x0000bedb, 0x0000bfa7, 0x0000bfc6, + 0x0000bfcd, 0x0000c0ed, 0x0000c1cb, 0x0000c1cf, 0x0000c1f7, 0x0000c1f9, + 0x0000c295, 0x0000c2c3, 0x0000c2fd, 0x0000c3db, 0x0000c3e6, 0x0000c3f1, + 0x0000c3f7, 0x0000c4cf, 0x0000c4ea, 0x0000c6da, 0x0000c6fb, 0x0000c7e5, + 0x0000c8cb, 0x0000c9cf, 0x0000c9e7, 0x0000c9fa, 0x0000cab7, 0x0000cabc, + 0x0000cabd, 0x0000cad0, 0x0000cad2, 0x0000cad5, 0x0000cadf, 0x0000cbae, + 0x0000cdac, 0x0000cebb, 0x0000ceef, 0x0000cff2, 0x0000d0a1, 0x0000d0c2, + 0x0000d0c7, 0x0000d0d0, 0x0000d0d1, 0x0000d165, 0x0000d175, 0x0000d2bb, + 0x0000d2f4, 0x0000d35e, 0x0000d38b, 0x0000d3d0, 0x0000d44f, 0x0000d483, + 0x0000d4da, 0x0000d4e7, 0x0000d566, 0x0000d5be, 0x0000d5d2, 0x0000d5df, + 0x0000d5fb, 0x0000d5fd, 0x0000d6b8, 0x0000d7f7, 0x0000dc87, 0x0000e95f, + 0x0000ec6f, 0x0000f05e, + ], + }, + { + encoding: 'EUC-KR', + language: 'ko', + total: 266, + commonCharacters: [ + 0x0000b0a1, 0x0000b0ab, 0x0000b0b3, 0x0000b0b4, 0x0000b0c7, 0x0000b0cb, + 0x0000b0d4, 0x0000b0e8, 0x0000b0ed, 0x0000b0fa, 0x0000b0fc, 0x0000b1a4, + 0x0000b1b8, 0x0000b1d7, 0x0000b1db, 0x0000b1e2, 0x0000b2dc, 0x0000b3aa, + 0x0000b3bb, 0x0000b3ca, 0x0000b3d1, 0x0000b3d7, 0x0000b3f3, 0x0000b3f5, + 0x0000b4a9, 0x0000b4ba, 0x0000b4c2, 0x0000b4d4, 0x0000b4d9, 0x0000b4eb, + 0x0000b5b5, 0x0000b5b6, 0x0000b5bf, 0x0000b5e5, 0x0000b5e7, 0x0000b5e9, + 0x0000b5f0, 0x0000b6f7, 0x0000b7a5, 0x0000b7ce, 0x0000b8a3, 0x0000b8a6, + 0x0000b8ae, 0x0000b8b0, 0x0000b8b8, 0x0000b8c2, 0x0000b8de, 0x0000b8e7, + 0x0000b8ed, 0x0000b9ae, 0x0000b9b0, 0x0000b9da, 0x0000b9df, 0x0000b9e0, + 0x0000b9e6, 0x0000b9f6, 0x0000b9f8, 0x0000b9fd, 0x0000bace, 0x0000baf1, + 0x0000bafb, 0x0000bba7, 0x0000bbe7, 0x0000bbfd, 0x0000bcad, 0x0000bcb3, + 0x0000bcd2, 0x0000bcd5, 0x0000bdac, 0x0000bdba, 0x0000bdc3, 0x0000bdc7, + 0x0000beb4, 0x0000bec6, 0x0000bec8, 0x0000bedf, 0x0000bef7, 0x0000bfa1, + 0x0000bfa9, 0x0000bfaa, 0x0000bfad, 0x0000bfcd, 0x0000bfe4, 0x0000bfeb, + 0x0000bfee, 0x0000c0a7, 0x0000c0ba, 0x0000c0bb, 0x0000c0c7, 0x0000c0cc, + 0x0000c0ce, 0x0000c0cf, 0x0000c0d0, 0x0000c0da, 0x0000c0db, 0x0000c0dc, + 0x0000c0e5, 0x0000c0fc, 0x0000c0fd, 0x0000c1a4, 0x0000c1b6, 0x0000c1d8, + 0x0000c1ee, 0x0000c1f6, 0x0000c1f8, 0x0000c1fd, 0x0000c3a3, 0x0000c3a4, + 0x0000c3a5, 0x0000c3b5, 0x0000c3b9, 0x0000c4a1, 0x0000c4a7, 0x0000c4ab, + 0x0000c4da, 0x0000c5d0, 0x0000c5e4, 0x0000c6c0, 0x0000c6e4, 0x0000c6ed, + 0x0000c7c1, 0x0000c7cf, 0x0000c7d0, 0x0000c7d1, 0x0000c7de, 0x0000c7e0, + 0x0000c8ad, 0x0000c8ae, + ], + }, +]; diff --git a/src/encoding/models/types.ts b/src/encoding/models/types.ts index a592c76..93b4fbd 100644 --- a/src/encoding/models/types.ts +++ b/src/encoding/models/types.ts @@ -16,3 +16,10 @@ export interface GeneratedSBCSModel { readonly byteMap: readonly number[]; readonly languages: readonly GeneratedSBCSLanguage[]; } + +export interface GeneratedMBCSModel { + readonly encoding: EncodingName; + readonly language: string; + readonly total: number; + readonly commonCharacters: readonly number[]; +} diff --git a/src/encoding/sbcs-scoring.test.ts b/src/encoding/sbcs-scoring.test.ts index a509527..5f6aee1 100644 --- a/src/encoding/sbcs-scoring.test.ts +++ b/src/encoding/sbcs-scoring.test.ts @@ -59,7 +59,10 @@ describe('shared SBCS scoring', () => { const corpusIndex = JSON.parse( fs.readFileSync(path.join(corpusRoot, 'index.json'), 'utf8'), ) as { encoding: string; path: string }[]; - const rows = corpusIndex.filter(({ encoding }) => encoding !== 'CP949'); + const modelNames = new Set( + generatedSBCSModels.map(({ encoding }) => encoding as string), + ); + const rows = corpusIndex.filter(({ encoding }) => modelNames.has(encoding)); for (const row of rows) { const input = fs.readFileSync(path.join(corpusRoot, row.path)); diff --git a/src/encoding/sbcs.test.ts b/src/encoding/sbcs.test.ts index 9614e2f..3beb615 100644 --- a/src/encoding/sbcs.test.ts +++ b/src/encoding/sbcs.test.ts @@ -76,6 +76,22 @@ describe('Singlebyte Character Sets', () => { expect(detect('windows_874')).toBe('windows-874'); }); + it.each(['validation/river-trip.bin', 'test/community-garden.bin'])( + 'should prefer windows-874 to weak multibyte matches in %s', + (filename) => { + const matches = chardet.analyse( + fs.readFileSync(corpusFixture('windows-874', 'th', filename)), + ); + const windows874 = matches.find((match) => match.name === 'windows-874'); + const eucJp = matches.find((match) => match.name === 'EUC-JP'); + + expect(matches[0]).toMatchObject({ name: 'windows-874', lang: 'th' }); + expect(eucJp?.confidence).toBeLessThanOrEqual( + windows874?.confidence ?? 0, + ); + }, + ); + // iso-8859-12 is abandoned it.each([ ['ISO-8859-13', 'lt'],