From df1e8dd0088f5d4b71790a54ee2fc1f2afe6cb0a Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Fri, 26 Jun 2026 12:13:43 +1000 Subject: [PATCH 1/4] chore: bench and validate scripts --- package.json | 2 + scripts/benchmark.mjs | 66 ++++++++++++++++++ scripts/validate.mjs | 155 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+) create mode 100644 scripts/benchmark.mjs create mode 100644 scripts/validate.mjs diff --git a/package.json b/package.json index e644490..a3e54c5 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "models:build": "node scripts/build-models.mjs", "models:evaluate": "node scripts/evaluate-models.mjs", "models:verify": "node scripts/verify-models.mjs", + "benchmark": "tsx scripts/benchmark.mjs", + "validate": "tsx scripts/validate.mjs", "format": "prettier --write \"**/*.{ts,json,md,mjs}\"", "format:check": "prettier --list-different \"**/*.{ts,json,md,mjs}\"", "test": "npm run test:typecheck && vitest run --coverage", diff --git a/scripts/benchmark.mjs b/scripts/benchmark.mjs new file mode 100644 index 0000000..f5be321 --- /dev/null +++ b/scripts/benchmark.mjs @@ -0,0 +1,66 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../src/index.ts'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const generatedCorpus = join(root, 'corpus', 'generated'); +const corpusIndex = JSON.parse( + readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), +); + +function option(name, fallback) { + const prefix = `--${name}=`; + const arg = process.argv.find((value) => value.startsWith(prefix)); + return arg ? arg.slice(prefix.length) : fallback; +} + +const iterations = Number(option('iterations', '100')); +const split = option('split', 'all'); + +if (!Number.isInteger(iterations) || iterations < 1) { + throw new Error('--iterations must be a positive integer'); +} + +const rows = corpusIndex.filter( + (row) => split === 'all' || row.split === split, +); +if (rows.length === 0) { + throw new Error(`No corpus files matched split: ${split}`); +} + +const inputs = rows.map((row) => ({ + ...row, + buffer: readFileSync(join(generatedCorpus, row.path)), +})); + +for (const input of inputs) analyse(input.buffer); + +const startedAt = process.hrtime.bigint(); +let matches = 0; +let bytes = 0; + +for (let iteration = 0; iteration < iterations; iteration += 1) { + for (const input of inputs) { + matches += analyse(input.buffer).length; + bytes += input.buffer.length; + } +} + +const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; +const calls = inputs.length * iterations; +const bytesPerSecond = bytes / (elapsedMs / 1000); + +console.log('Library corpus benchmark'); +console.log('========================'); +console.log(`Split: ${split}`); +console.log(`Files: ${inputs.length}`); +console.log(`Iterations: ${iterations}`); +console.log(`Calls: ${calls}`); +console.log(`Bytes processed: ${bytes}`); +console.log(`Matches produced: ${matches}`); +console.log(`Elapsed: ${elapsedMs.toFixed(2)} ms`); +console.log(`Throughput: ${(calls / (elapsedMs / 1000)).toFixed(2)} files/sec`); +console.log( + `Byte throughput: ${(bytesPerSecond / 1024 / 1024).toFixed(2)} MiB/sec`, +); diff --git a/scripts/validate.mjs b/scripts/validate.mjs new file mode 100644 index 0000000..80c2aeb --- /dev/null +++ b/scripts/validate.mjs @@ -0,0 +1,155 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../src/index.ts'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const generatedCorpus = join(root, 'corpus', 'generated'); +const corpusIndex = JSON.parse( + readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), +); + +function option(name, fallback) { + const prefix = `--${name}=`; + const arg = process.argv.find((value) => value.startsWith(prefix)); + return arg ? arg.slice(prefix.length) : fallback; +} + +function percent(value, total) { + return total === 0 ? '0.00%' : `${((value / total) * 100).toFixed(2)}%`; +} + +function increment(object, key) { + object[key] = (object[key] ?? 0) + 1; +} + +const split = option('split', 'all'); +const rows = corpusIndex.filter( + (row) => split === 'all' || row.split === split, +); +if (rows.length === 0) { + throw new Error(`No corpus files matched split: ${split}`); +} + +const results = rows.map((row) => { + const buffer = readFileSync(join(generatedCorpus, row.path)); + const matches = analyse(buffer); + const predicted = matches[0] ?? null; + return { + expected: { + encoding: row.encoding, + language: row.language, + split: row.split, + path: row.path, + }, + predicted, + encodingCorrect: predicted?.name === row.encoding, + languageCorrect: + predicted?.name === row.encoding && predicted?.lang === row.language, + }; +}); + +const summary = { + files: results.length, + encodingCorrect: results.filter((result) => result.encodingCorrect).length, + languageCorrect: results.filter((result) => result.languageCorrect).length, + noPrediction: results.filter((result) => result.predicted === null).length, +}; + +const bySplit = {}; +const byExpectedEncoding = {}; +const topPredictions = {}; + +for (const result of results) { + const splitSummary = (bySplit[result.expected.split] ??= { + files: 0, + encodingCorrect: 0, + languageCorrect: 0, + }); + splitSummary.files += 1; + if (result.encodingCorrect) splitSummary.encodingCorrect += 1; + if (result.languageCorrect) splitSummary.languageCorrect += 1; + + const encodingSummary = (byExpectedEncoding[result.expected.encoding] ??= { + files: 0, + encodingCorrect: 0, + languageCorrect: 0, + predictions: {}, + }); + encodingSummary.files += 1; + if (result.encodingCorrect) encodingSummary.encodingCorrect += 1; + if (result.languageCorrect) encodingSummary.languageCorrect += 1; + increment(encodingSummary.predictions, result.predicted?.name ?? ''); + increment(topPredictions, result.predicted?.name ?? ''); +} + +console.log('Library corpus validation'); +console.log('========================='); +console.log(`Split: ${split}`); +console.log(`Files: ${summary.files}`); +console.log( + `Encoding correct: ${summary.encodingCorrect}/${summary.files} (${percent( + summary.encodingCorrect, + summary.files, + )})`, +); +console.log( + `Language correct: ${summary.languageCorrect}/${summary.files} (${percent( + summary.languageCorrect, + summary.files, + )})`, +); +console.log(`No prediction: ${summary.noPrediction}`); + +console.log('\nBy split'); +console.table( + Object.entries(bySplit).map(([name, item]) => ({ + split: name, + files: item.files, + encodingCorrect: item.encodingCorrect, + encodingAccuracy: percent(item.encodingCorrect, item.files), + languageCorrect: item.languageCorrect, + languageAccuracy: percent(item.languageCorrect, item.files), + })), +); + +console.log('\nBy expected encoding'); +console.table( + Object.entries(byExpectedEncoding) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([encoding, item]) => ({ + encoding, + files: item.files, + encodingCorrect: item.encodingCorrect, + encodingAccuracy: percent(item.encodingCorrect, item.files), + languageCorrect: item.languageCorrect, + mostCommonPrediction: Object.entries(item.predictions).sort( + ([leftName, leftCount], [rightName, rightCount]) => + rightCount - leftCount || leftName.localeCompare(rightName), + )[0][0], + })), +); + +console.log('\nTop predicted encodings'); +console.table( + Object.entries(topPredictions) + .sort( + ([leftName, leftCount], [rightName, rightCount]) => + rightCount - leftCount || leftName.localeCompare(rightName), + ) + .map(([encoding, count]) => ({ encoding, count })), +); + +const failures = results.filter((result) => !result.encodingCorrect); +console.log('\nFirst 20 encoding mismatches'); +console.table( + failures.slice(0, 20).map((result) => ({ + expected: result.expected.encoding, + language: result.expected.language, + split: result.expected.split, + predicted: result.predicted?.name ?? '', + predictedLanguage: result.predicted?.lang ?? '', + confidence: result.predicted?.confidence ?? '', + path: result.expected.path, + })), +); From 0f4d311207f0ac2dbf1fa1efd2cd032fd3c919d5 Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Sun, 12 Jul 2026 09:50:17 +1000 Subject: [PATCH 2/4] baseline --- BASELINE.md | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 BASELINE.md diff --git a/BASELINE.md b/BASELINE.md new file mode 100644 index 0000000..f45cd81 --- /dev/null +++ b/BASELINE.md @@ -0,0 +1,108 @@ +# Generated model migration baseline + +Recorded on 2026-07-12 at commit +`df1e8dd0088f5d4b71790a54ee2fc1f2afe6cb0a` (`v3`), using Node.js v24.10.0 +and npm 11.6.1. + +`TODO.md` is locally present but ignored by the repository. There were no +tracked or unignored changes when this baseline was recorded. Generated `lib/` +and coverage output are also ignored. + +## Checks + +| Check | Result | +| --------------------------------- | ------------------------------------------------------------------ | +| `npm test` | Pass: 8 files, 48 tests passed, 8 todo | +| TypeScript | Pass | +| Coverage | 93.04% statements, 85.44% branches, 95.86% functions, 93.13% lines | +| `npm run models:verify` | Pass; models and evaluation are reproducible | +| `npm run validate -- --split=all` | Completes; 0/138 encoding and language predictions correct | +| `npm run benchmark` | 7,848.13 files/sec; 1.42 MiB/sec | +| `npm run build` | Pass | +| Built `lib/` | 341,661 bytes across files; 448 KiB allocated on disk | +| `npm pack --dry-run` | 42,359 bytes packed; 348,964 bytes unpacked; 42 entries | + +The benchmark processed 13,800 calls and 2,622,400 bytes in 1,758.38 ms, +producing 100,200 matches. These figures are a local performance reference, not +a hard CI threshold; comparisons should use the same machine and runtime. + +## Runtime corpus validation + +The validation corpus currently contains only newly generated models that are +not registered with the runtime. The 0% result is therefore the expected +pre-migration baseline, not a model-evaluator failure. Every input still +received at least one prediction. + +### By split + +| Split | Files | Encoding correct | Language correct | +| ---------- | ------: | ---------------: | ---------------: | +| train | 92 | 0 (0.00%) | 0 (0.00%) | +| validation | 23 | 0 (0.00%) | 0 (0.00%) | +| test | 23 | 0 (0.00%) | 0 (0.00%) | +| **Total** | **138** | **0 (0.00%)** | **0 (0.00%)** | + +### By expected encoding + +| Expected encoding | Files | Encoding correct | Most common current prediction | +| ----------------- | ----: | ---------------: | ------------------------------ | +| CP850 | 18 | 0 | windows-1252 | +| CP852 | 6 | 0 | windows-1250 | +| CP949 | 6 | 0 | EUC-KR | +| IBM855 | 12 | 0 | windows-1252 | +| IBM866 | 6 | 0 | windows-1252 | +| ISO-8859-10 | 6 | 0 | ISO-8859-1 | +| ISO-8859-13 | 6 | 0 | windows-1257 | +| ISO-8859-14 | 6 | 0 | ISO-8859-1 | +| ISO-8859-15 | 18 | 0 | ISO-8859-1 | +| ISO-8859-16 | 6 | 0 | ISO-8859-2 | +| ISO-8859-3 | 6 | 0 | ISO-8859-1 | +| ISO-8859-4 | 6 | 0 | windows-1257 | +| KOI8-U | 6 | 0 | KOI8-R | +| macintosh | 18 | 0 | windows-1252 | +| x-mac-cyrillic | 12 | 0 | windows-1251 | + +The most common predictions above are useful collision targets, but do not +describe every document. Notably, weak IBM855 input can also be classified as +Shift_JIS or windows-1255. + +## Public naming decisions + +Existing names remain exactly as declared by `EncodingName`; the migration must +not silently recase or rename them. The new SBCS canonical result names are: + +```text +KOI8-U +IBM866 +IBM855 +macintosh +x-mac-cyrillic +ISO-8859-3 +ISO-8859-4 +ISO-8859-10 +ISO-8859-13 +ISO-8859-14 +ISO-8859-15 +ISO-8859-16 +CP850 +CP852 +``` + +CP949 is reserved as the canonical future result name for the separate +multibyte follow-up. + +The current public API does not accept encoding-name filters and does not expose +alias normalization. Consequently, the initial migration adds only these +canonical `EncodingName` values. Iconv/compiler input labels such as `CP866`, +`CP855`, `MACINTOSH`, and `MAC-CYRILLIC` are implementation aliases and must not +be returned by `detect` or `analyse`. + +## Known pre-migration limitations + +- Generated SBCS models are not consumed or registered by the detector. +- New encodings therefore lose to the closest existing recognizer. +- Confidence values are not yet calibrated between generated SBCS and existing + UTF, ISO-2022, or MBCS recognizers. +- The existing test suite has eight explicitly pending tests. +- CP949 is represented in the corpus but intentionally excluded from the SBCS + model compiler and requires separate MBCS work. From 6cd2ff6331082ea950fa8cc9ea3cdf2d6166ba22 Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Sun, 12 Jul 2026 10:12:09 +1000 Subject: [PATCH 3/4] Development snapshot --- .gitignore | 1 + BASELINE.md | 7 + corpus/README.md | 10 +- .../ISO-8859-1/da/test/community-garden.bin | 3 + .../ISO-8859-1/da/train/city-morning.bin | 3 + .../generated/ISO-8859-1/da/train/library.bin | 3 + .../generated/ISO-8859-1/da/train/market.bin | 3 + .../ISO-8859-1/da/train/workshop.bin | 3 + .../ISO-8859-1/da/validation/river-trip.bin | 3 + .../ISO-8859-1/de/test/community-garden.bin | 3 + .../ISO-8859-1/de/train/city-morning.bin | 3 + .../generated/ISO-8859-1/de/train/library.bin | 3 + .../generated/ISO-8859-1/de/train/market.bin | 3 + .../ISO-8859-1/de/train/workshop.bin | 3 + .../ISO-8859-1/de/validation/river-trip.bin | 3 + .../ISO-8859-1/en/test/community-garden.bin | 3 + .../ISO-8859-1/en/train/city-morning.bin | 3 + .../generated/ISO-8859-1/en/train/library.bin | 3 + .../generated/ISO-8859-1/en/train/market.bin | 3 + .../ISO-8859-1/en/train/workshop.bin | 3 + .../ISO-8859-1/en/validation/river-trip.bin | 3 + .../ISO-8859-1/es/test/community-garden.bin | 3 + .../ISO-8859-1/es/train/city-morning.bin | 3 + .../generated/ISO-8859-1/es/train/library.bin | 3 + .../generated/ISO-8859-1/es/train/market.bin | 3 + .../ISO-8859-1/es/train/workshop.bin | 3 + .../ISO-8859-1/es/validation/river-trip.bin | 3 + .../ISO-8859-1/fr/test/community-garden.bin | 3 + .../ISO-8859-1/fr/train/city-morning.bin | 3 + .../generated/ISO-8859-1/fr/train/library.bin | 3 + .../generated/ISO-8859-1/fr/train/market.bin | 3 + .../ISO-8859-1/fr/train/workshop.bin | 3 + .../ISO-8859-1/fr/validation/river-trip.bin | 3 + .../ISO-8859-1/it/test/community-garden.bin | 3 + .../ISO-8859-1/it/train/city-morning.bin | 3 + .../generated/ISO-8859-1/it/train/library.bin | 3 + .../generated/ISO-8859-1/it/train/market.bin | 3 + .../ISO-8859-1/it/train/workshop.bin | 3 + .../ISO-8859-1/it/validation/river-trip.bin | 3 + .../ISO-8859-1/nl/test/community-garden.bin | 3 + .../ISO-8859-1/nl/train/city-morning.bin | 3 + .../generated/ISO-8859-1/nl/train/library.bin | 3 + .../generated/ISO-8859-1/nl/train/market.bin | 3 + .../ISO-8859-1/nl/train/workshop.bin | 3 + .../ISO-8859-1/nl/validation/river-trip.bin | 3 + .../ISO-8859-1/no/test/community-garden.bin | 3 + .../ISO-8859-1/no/train/city-morning.bin | 3 + .../generated/ISO-8859-1/no/train/library.bin | 3 + .../generated/ISO-8859-1/no/train/market.bin | 3 + .../ISO-8859-1/no/train/workshop.bin | 3 + .../ISO-8859-1/no/validation/river-trip.bin | 3 + .../ISO-8859-1/pt/test/community-garden.bin | 3 + .../ISO-8859-1/pt/train/city-morning.bin | 3 + .../generated/ISO-8859-1/pt/train/library.bin | 3 + .../generated/ISO-8859-1/pt/train/market.bin | 3 + .../ISO-8859-1/pt/train/workshop.bin | 3 + .../ISO-8859-1/pt/validation/river-trip.bin | 3 + .../ISO-8859-1/sv/test/community-garden.bin | 3 + .../ISO-8859-1/sv/train/city-morning.bin | 3 + .../generated/ISO-8859-1/sv/train/library.bin | 3 + .../generated/ISO-8859-1/sv/train/market.bin | 3 + .../ISO-8859-1/sv/train/workshop.bin | 3 + .../ISO-8859-1/sv/validation/river-trip.bin | 3 + .../ISO-8859-2/cs/test/community-garden.bin | 3 + .../ISO-8859-2/cs/train/city-morning.bin | 3 + .../generated/ISO-8859-2/cs/train/library.bin | 3 + .../generated/ISO-8859-2/cs/train/market.bin | 3 + .../ISO-8859-2/cs/train/workshop.bin | 3 + .../ISO-8859-2/cs/validation/river-trip.bin | 3 + .../ISO-8859-2/hu/test/community-garden.bin | 3 + .../ISO-8859-2/hu/train/city-morning.bin | 3 + .../generated/ISO-8859-2/hu/train/library.bin | 3 + .../generated/ISO-8859-2/hu/train/market.bin | 3 + .../ISO-8859-2/hu/train/workshop.bin | 3 + .../ISO-8859-2/hu/validation/river-trip.bin | 3 + .../ISO-8859-2/pl/test/community-garden.bin | 3 + .../ISO-8859-2/pl/train/city-morning.bin | 3 + .../generated/ISO-8859-2/pl/train/library.bin | 3 + .../generated/ISO-8859-2/pl/train/market.bin | 3 + .../ISO-8859-2/pl/train/workshop.bin | 3 + .../ISO-8859-2/pl/validation/river-trip.bin | 3 + .../ISO-8859-2/ro/test/community-garden.bin | 3 + .../ISO-8859-2/ro/train/city-morning.bin | 3 + .../generated/ISO-8859-2/ro/train/library.bin | 3 + .../generated/ISO-8859-2/ro/train/market.bin | 3 + .../ISO-8859-2/ro/train/workshop.bin | 3 + .../ISO-8859-2/ro/validation/river-trip.bin | 3 + .../ISO-8859-5/ru/test/community-garden.bin | 3 + .../ISO-8859-5/ru/train/city-morning.bin | 3 + .../generated/ISO-8859-5/ru/train/library.bin | 3 + .../generated/ISO-8859-5/ru/train/market.bin | 3 + .../ISO-8859-5/ru/train/workshop.bin | 3 + .../ISO-8859-5/ru/validation/river-trip.bin | 3 + .../ISO-8859-6/ar/test/community-garden.bin | 3 + .../ISO-8859-6/ar/train/city-morning.bin | 3 + .../generated/ISO-8859-6/ar/train/library.bin | 3 + .../generated/ISO-8859-6/ar/train/market.bin | 3 + .../ISO-8859-6/ar/train/workshop.bin | 3 + .../ISO-8859-6/ar/validation/river-trip.bin | 3 + .../ISO-8859-7/el/test/community-garden.bin | 3 + .../ISO-8859-7/el/train/city-morning.bin | 3 + .../generated/ISO-8859-7/el/train/library.bin | 3 + .../generated/ISO-8859-7/el/train/market.bin | 3 + .../ISO-8859-7/el/train/workshop.bin | 3 + .../ISO-8859-7/el/validation/river-trip.bin | 3 + .../ISO-8859-8/he/test/community-garden.bin | 3 + .../ISO-8859-8/he/train/city-morning.bin | 3 + .../generated/ISO-8859-8/he/train/library.bin | 3 + .../generated/ISO-8859-8/he/train/market.bin | 3 + .../ISO-8859-8/he/train/workshop.bin | 3 + .../ISO-8859-8/he/validation/river-trip.bin | 3 + .../ISO-8859-9/tr/test/community-garden.bin | 3 + .../ISO-8859-9/tr/train/city-morning.bin | 3 + .../generated/ISO-8859-9/tr/train/library.bin | 3 + .../generated/ISO-8859-9/tr/train/market.bin | 3 + .../ISO-8859-9/tr/train/workshop.bin | 3 + .../ISO-8859-9/tr/validation/river-trip.bin | 3 + .../KOI8-R/ru/test/community-garden.bin | 3 + .../KOI8-R/ru/train/city-morning.bin | 3 + corpus/generated/KOI8-R/ru/train/library.bin | 3 + corpus/generated/KOI8-R/ru/train/market.bin | 3 + corpus/generated/KOI8-R/ru/train/workshop.bin | 3 + .../KOI8-R/ru/validation/river-trip.bin | 3 + corpus/generated/index.json | 4308 +++- corpus/generated/ngrams.mjs | 1144 + .../windows-1250/cs/test/community-garden.bin | 3 + .../windows-1250/cs/train/city-morning.bin | 3 + .../windows-1250/cs/train/library.bin | 3 + .../windows-1250/cs/train/market.bin | 3 + .../windows-1250/cs/train/workshop.bin | 3 + .../windows-1250/cs/validation/river-trip.bin | 3 + .../windows-1250/hu/test/community-garden.bin | 3 + .../windows-1250/hu/train/city-morning.bin | 3 + .../windows-1250/hu/train/library.bin | 3 + .../windows-1250/hu/train/market.bin | 3 + .../windows-1250/hu/train/workshop.bin | 3 + .../windows-1250/hu/validation/river-trip.bin | 3 + .../windows-1250/pl/test/community-garden.bin | 3 + .../windows-1250/pl/train/city-morning.bin | 3 + .../windows-1250/pl/train/library.bin | 3 + .../windows-1250/pl/train/market.bin | 3 + .../windows-1250/pl/train/workshop.bin | 3 + .../windows-1250/pl/validation/river-trip.bin | 3 + .../windows-1250/ro/test/community-garden.bin | 3 + .../windows-1250/ro/train/city-morning.bin | 3 + .../windows-1250/ro/train/library.bin | 3 + .../windows-1250/ro/train/market.bin | 3 + .../windows-1250/ro/train/workshop.bin | 3 + .../windows-1250/ro/validation/river-trip.bin | 3 + .../windows-1251/ru/test/community-garden.bin | 3 + .../windows-1251/ru/train/city-morning.bin | 3 + .../windows-1251/ru/train/library.bin | 3 + .../windows-1251/ru/train/market.bin | 3 + .../windows-1251/ru/train/workshop.bin | 3 + .../windows-1251/ru/validation/river-trip.bin | 3 + .../windows-1252/da/test/community-garden.bin | 3 + .../windows-1252/da/train/city-morning.bin | 3 + .../windows-1252/da/train/library.bin | 3 + .../windows-1252/da/train/market.bin | 3 + .../windows-1252/da/train/workshop.bin | 3 + .../windows-1252/da/validation/river-trip.bin | 3 + .../windows-1252/de/test/community-garden.bin | 3 + .../windows-1252/de/train/city-morning.bin | 3 + .../windows-1252/de/train/library.bin | 3 + .../windows-1252/de/train/market.bin | 3 + .../windows-1252/de/train/workshop.bin | 3 + .../windows-1252/de/validation/river-trip.bin | 3 + .../windows-1252/en/test/community-garden.bin | 3 + .../windows-1252/en/train/city-morning.bin | 3 + .../windows-1252/en/train/library.bin | 3 + .../windows-1252/en/train/market.bin | 3 + .../windows-1252/en/train/workshop.bin | 3 + .../windows-1252/en/validation/river-trip.bin | 3 + .../windows-1252/es/test/community-garden.bin | 3 + .../windows-1252/es/train/city-morning.bin | 3 + .../windows-1252/es/train/library.bin | 3 + .../windows-1252/es/train/market.bin | 3 + .../windows-1252/es/train/workshop.bin | 3 + .../windows-1252/es/validation/river-trip.bin | 3 + .../windows-1252/fr/test/community-garden.bin | 3 + .../windows-1252/fr/train/city-morning.bin | 3 + .../windows-1252/fr/train/library.bin | 3 + .../windows-1252/fr/train/market.bin | 3 + .../windows-1252/fr/train/workshop.bin | 3 + .../windows-1252/fr/validation/river-trip.bin | 3 + .../windows-1252/it/test/community-garden.bin | 3 + .../windows-1252/it/train/city-morning.bin | 3 + .../windows-1252/it/train/library.bin | 3 + .../windows-1252/it/train/market.bin | 3 + .../windows-1252/it/train/workshop.bin | 3 + .../windows-1252/it/validation/river-trip.bin | 3 + .../windows-1252/nl/test/community-garden.bin | 3 + .../windows-1252/nl/train/city-morning.bin | 3 + .../windows-1252/nl/train/library.bin | 3 + .../windows-1252/nl/train/market.bin | 3 + .../windows-1252/nl/train/workshop.bin | 3 + .../windows-1252/nl/validation/river-trip.bin | 3 + .../windows-1252/no/test/community-garden.bin | 3 + .../windows-1252/no/train/city-morning.bin | 3 + .../windows-1252/no/train/library.bin | 3 + .../windows-1252/no/train/market.bin | 3 + .../windows-1252/no/train/workshop.bin | 3 + .../windows-1252/no/validation/river-trip.bin | 3 + .../windows-1252/pt/test/community-garden.bin | 3 + .../windows-1252/pt/train/city-morning.bin | 3 + .../windows-1252/pt/train/library.bin | 3 + .../windows-1252/pt/train/market.bin | 3 + .../windows-1252/pt/train/workshop.bin | 3 + .../windows-1252/pt/validation/river-trip.bin | 3 + .../windows-1252/sv/test/community-garden.bin | 3 + .../windows-1252/sv/train/city-morning.bin | 3 + .../windows-1252/sv/train/library.bin | 3 + .../windows-1252/sv/train/market.bin | 3 + .../windows-1252/sv/train/workshop.bin | 3 + .../windows-1252/sv/validation/river-trip.bin | 3 + .../windows-1253/el/test/community-garden.bin | 3 + .../windows-1253/el/train/city-morning.bin | 3 + .../windows-1253/el/train/library.bin | 3 + .../windows-1253/el/train/market.bin | 3 + .../windows-1253/el/train/workshop.bin | 3 + .../windows-1253/el/validation/river-trip.bin | 3 + .../windows-1254/tr/test/community-garden.bin | 3 + .../windows-1254/tr/train/city-morning.bin | 3 + .../windows-1254/tr/train/library.bin | 3 + .../windows-1254/tr/train/market.bin | 3 + .../windows-1254/tr/train/workshop.bin | 3 + .../windows-1254/tr/validation/river-trip.bin | 3 + .../windows-1255/he/test/community-garden.bin | 3 + .../windows-1255/he/train/city-morning.bin | 3 + .../windows-1255/he/train/library.bin | 3 + .../windows-1255/he/train/market.bin | 3 + .../windows-1255/he/train/workshop.bin | 3 + .../windows-1255/he/validation/river-trip.bin | 3 + .../windows-1256/ar/test/community-garden.bin | 3 + .../windows-1256/ar/train/city-morning.bin | 3 + .../windows-1256/ar/train/library.bin | 3 + .../windows-1256/ar/train/market.bin | 3 + .../windows-1256/ar/train/workshop.bin | 3 + .../windows-1256/ar/validation/river-trip.bin | 3 + .../windows-1257/et/test/community-garden.bin | 3 + .../windows-1257/et/train/city-morning.bin | 3 + .../windows-1257/et/train/library.bin | 3 + .../windows-1257/et/train/market.bin | 3 + .../windows-1257/et/train/workshop.bin | 3 + .../windows-1257/et/validation/river-trip.bin | 3 + .../windows-1257/lt/test/community-garden.bin | 3 + .../windows-1257/lt/train/city-morning.bin | 3 + .../windows-1257/lt/train/library.bin | 3 + .../windows-1257/lt/train/market.bin | 3 + .../windows-1257/lt/train/workshop.bin | 3 + .../windows-1257/lt/validation/river-trip.bin | 3 + .../windows-1257/lv/test/community-garden.bin | 3 + .../windows-1257/lv/train/city-morning.bin | 3 + .../windows-1257/lv/train/library.bin | 3 + .../windows-1257/lv/train/market.bin | 3 + .../windows-1257/lv/train/workshop.bin | 3 + .../windows-1257/lv/validation/river-trip.bin | 3 + .../windows-1258/vi/test/community-garden.bin | 3 + .../windows-1258/vi/train/city-morning.bin | 3 + .../windows-1258/vi/train/library.bin | 3 + .../windows-1258/vi/train/market.bin | 3 + .../windows-1258/vi/train/workshop.bin | 3 + .../windows-1258/vi/validation/river-trip.bin | 3 + .../windows-874/th/test/community-garden.bin | 3 + .../windows-874/th/train/city-morning.bin | 3 + .../windows-874/th/train/library.bin | 3 + .../generated/windows-874/th/train/market.bin | 3 + .../windows-874/th/train/workshop.bin | 3 + .../windows-874/th/validation/river-trip.bin | 3 + corpus/manifest.json | 40 + corpus/model-evaluation.json | 19483 +++++++++++++++- corpus/sources.json | 576 + corpus/test-sources.json | 96 + scripts/corpus-utils.mjs | 16 +- src/encoding/models/generated.ts | 1989 ++ 275 files changed, 26728 insertions(+), 1734 deletions(-) create mode 100644 corpus/generated/ISO-8859-1/da/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/da/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/da/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/de/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/de/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/de/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/en/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/en/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/en/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/es/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/es/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/es/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/fr/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/fr/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/fr/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/it/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/it/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/it/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/nl/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/nl/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/nl/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/no/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/no/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/no/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/pt/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/pt/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/pt/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-1/sv/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/library.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/market.bin create mode 100644 corpus/generated/ISO-8859-1/sv/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-1/sv/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/cs/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/cs/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/cs/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/hu/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/hu/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/hu/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/pl/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/pl/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/pl/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-2/ro/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/library.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/market.bin create mode 100644 corpus/generated/ISO-8859-2/ro/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-2/ro/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-5/ru/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/library.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/market.bin create mode 100644 corpus/generated/ISO-8859-5/ru/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-5/ru/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-6/ar/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/library.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/market.bin create mode 100644 corpus/generated/ISO-8859-6/ar/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-6/ar/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-7/el/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/library.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/market.bin create mode 100644 corpus/generated/ISO-8859-7/el/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-7/el/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-8/he/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/library.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/market.bin create mode 100644 corpus/generated/ISO-8859-8/he/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-8/he/validation/river-trip.bin create mode 100644 corpus/generated/ISO-8859-9/tr/test/community-garden.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/city-morning.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/library.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/market.bin create mode 100644 corpus/generated/ISO-8859-9/tr/train/workshop.bin create mode 100644 corpus/generated/ISO-8859-9/tr/validation/river-trip.bin create mode 100644 corpus/generated/KOI8-R/ru/test/community-garden.bin create mode 100644 corpus/generated/KOI8-R/ru/train/city-morning.bin create mode 100644 corpus/generated/KOI8-R/ru/train/library.bin create mode 100644 corpus/generated/KOI8-R/ru/train/market.bin create mode 100644 corpus/generated/KOI8-R/ru/train/workshop.bin create mode 100644 corpus/generated/KOI8-R/ru/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/cs/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/cs/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/cs/train/library.bin create mode 100644 corpus/generated/windows-1250/cs/train/market.bin create mode 100644 corpus/generated/windows-1250/cs/train/workshop.bin create mode 100644 corpus/generated/windows-1250/cs/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/hu/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/hu/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/hu/train/library.bin create mode 100644 corpus/generated/windows-1250/hu/train/market.bin create mode 100644 corpus/generated/windows-1250/hu/train/workshop.bin create mode 100644 corpus/generated/windows-1250/hu/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/pl/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/pl/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/pl/train/library.bin create mode 100644 corpus/generated/windows-1250/pl/train/market.bin create mode 100644 corpus/generated/windows-1250/pl/train/workshop.bin create mode 100644 corpus/generated/windows-1250/pl/validation/river-trip.bin create mode 100644 corpus/generated/windows-1250/ro/test/community-garden.bin create mode 100644 corpus/generated/windows-1250/ro/train/city-morning.bin create mode 100644 corpus/generated/windows-1250/ro/train/library.bin create mode 100644 corpus/generated/windows-1250/ro/train/market.bin create mode 100644 corpus/generated/windows-1250/ro/train/workshop.bin create mode 100644 corpus/generated/windows-1250/ro/validation/river-trip.bin create mode 100644 corpus/generated/windows-1251/ru/test/community-garden.bin create mode 100644 corpus/generated/windows-1251/ru/train/city-morning.bin create mode 100644 corpus/generated/windows-1251/ru/train/library.bin create mode 100644 corpus/generated/windows-1251/ru/train/market.bin create mode 100644 corpus/generated/windows-1251/ru/train/workshop.bin create mode 100644 corpus/generated/windows-1251/ru/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/da/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/da/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/da/train/library.bin create mode 100644 corpus/generated/windows-1252/da/train/market.bin create mode 100644 corpus/generated/windows-1252/da/train/workshop.bin create mode 100644 corpus/generated/windows-1252/da/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/de/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/de/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/de/train/library.bin create mode 100644 corpus/generated/windows-1252/de/train/market.bin create mode 100644 corpus/generated/windows-1252/de/train/workshop.bin create mode 100644 corpus/generated/windows-1252/de/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/en/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/en/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/en/train/library.bin create mode 100644 corpus/generated/windows-1252/en/train/market.bin create mode 100644 corpus/generated/windows-1252/en/train/workshop.bin create mode 100644 corpus/generated/windows-1252/en/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/es/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/es/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/es/train/library.bin create mode 100644 corpus/generated/windows-1252/es/train/market.bin create mode 100644 corpus/generated/windows-1252/es/train/workshop.bin create mode 100644 corpus/generated/windows-1252/es/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/fr/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/fr/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/fr/train/library.bin create mode 100644 corpus/generated/windows-1252/fr/train/market.bin create mode 100644 corpus/generated/windows-1252/fr/train/workshop.bin create mode 100644 corpus/generated/windows-1252/fr/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/it/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/it/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/it/train/library.bin create mode 100644 corpus/generated/windows-1252/it/train/market.bin create mode 100644 corpus/generated/windows-1252/it/train/workshop.bin create mode 100644 corpus/generated/windows-1252/it/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/nl/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/nl/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/nl/train/library.bin create mode 100644 corpus/generated/windows-1252/nl/train/market.bin create mode 100644 corpus/generated/windows-1252/nl/train/workshop.bin create mode 100644 corpus/generated/windows-1252/nl/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/no/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/no/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/no/train/library.bin create mode 100644 corpus/generated/windows-1252/no/train/market.bin create mode 100644 corpus/generated/windows-1252/no/train/workshop.bin create mode 100644 corpus/generated/windows-1252/no/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/pt/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/pt/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/pt/train/library.bin create mode 100644 corpus/generated/windows-1252/pt/train/market.bin create mode 100644 corpus/generated/windows-1252/pt/train/workshop.bin create mode 100644 corpus/generated/windows-1252/pt/validation/river-trip.bin create mode 100644 corpus/generated/windows-1252/sv/test/community-garden.bin create mode 100644 corpus/generated/windows-1252/sv/train/city-morning.bin create mode 100644 corpus/generated/windows-1252/sv/train/library.bin create mode 100644 corpus/generated/windows-1252/sv/train/market.bin create mode 100644 corpus/generated/windows-1252/sv/train/workshop.bin create mode 100644 corpus/generated/windows-1252/sv/validation/river-trip.bin create mode 100644 corpus/generated/windows-1253/el/test/community-garden.bin create mode 100644 corpus/generated/windows-1253/el/train/city-morning.bin create mode 100644 corpus/generated/windows-1253/el/train/library.bin create mode 100644 corpus/generated/windows-1253/el/train/market.bin create mode 100644 corpus/generated/windows-1253/el/train/workshop.bin create mode 100644 corpus/generated/windows-1253/el/validation/river-trip.bin create mode 100644 corpus/generated/windows-1254/tr/test/community-garden.bin create mode 100644 corpus/generated/windows-1254/tr/train/city-morning.bin create mode 100644 corpus/generated/windows-1254/tr/train/library.bin create mode 100644 corpus/generated/windows-1254/tr/train/market.bin create mode 100644 corpus/generated/windows-1254/tr/train/workshop.bin create mode 100644 corpus/generated/windows-1254/tr/validation/river-trip.bin create mode 100644 corpus/generated/windows-1255/he/test/community-garden.bin create mode 100644 corpus/generated/windows-1255/he/train/city-morning.bin create mode 100644 corpus/generated/windows-1255/he/train/library.bin create mode 100644 corpus/generated/windows-1255/he/train/market.bin create mode 100644 corpus/generated/windows-1255/he/train/workshop.bin create mode 100644 corpus/generated/windows-1255/he/validation/river-trip.bin create mode 100644 corpus/generated/windows-1256/ar/test/community-garden.bin create mode 100644 corpus/generated/windows-1256/ar/train/city-morning.bin create mode 100644 corpus/generated/windows-1256/ar/train/library.bin create mode 100644 corpus/generated/windows-1256/ar/train/market.bin create mode 100644 corpus/generated/windows-1256/ar/train/workshop.bin create mode 100644 corpus/generated/windows-1256/ar/validation/river-trip.bin create mode 100644 corpus/generated/windows-1257/et/test/community-garden.bin create mode 100644 corpus/generated/windows-1257/et/train/city-morning.bin create mode 100644 corpus/generated/windows-1257/et/train/library.bin create mode 100644 corpus/generated/windows-1257/et/train/market.bin create mode 100644 corpus/generated/windows-1257/et/train/workshop.bin create mode 100644 corpus/generated/windows-1257/et/validation/river-trip.bin create mode 100644 corpus/generated/windows-1257/lt/test/community-garden.bin create mode 100644 corpus/generated/windows-1257/lt/train/city-morning.bin create mode 100644 corpus/generated/windows-1257/lt/train/library.bin create mode 100644 corpus/generated/windows-1257/lt/train/market.bin create mode 100644 corpus/generated/windows-1257/lt/train/workshop.bin create mode 100644 corpus/generated/windows-1257/lt/validation/river-trip.bin create mode 100644 corpus/generated/windows-1257/lv/test/community-garden.bin create mode 100644 corpus/generated/windows-1257/lv/train/city-morning.bin create mode 100644 corpus/generated/windows-1257/lv/train/library.bin create mode 100644 corpus/generated/windows-1257/lv/train/market.bin create mode 100644 corpus/generated/windows-1257/lv/train/workshop.bin create mode 100644 corpus/generated/windows-1257/lv/validation/river-trip.bin create mode 100644 corpus/generated/windows-1258/vi/test/community-garden.bin create mode 100644 corpus/generated/windows-1258/vi/train/city-morning.bin create mode 100644 corpus/generated/windows-1258/vi/train/library.bin create mode 100644 corpus/generated/windows-1258/vi/train/market.bin create mode 100644 corpus/generated/windows-1258/vi/train/workshop.bin create mode 100644 corpus/generated/windows-1258/vi/validation/river-trip.bin create mode 100644 corpus/generated/windows-874/th/test/community-garden.bin create mode 100644 corpus/generated/windows-874/th/train/city-morning.bin create mode 100644 corpus/generated/windows-874/th/train/library.bin create mode 100644 corpus/generated/windows-874/th/train/market.bin create mode 100644 corpus/generated/windows-874/th/train/workshop.bin create mode 100644 corpus/generated/windows-874/th/validation/river-trip.bin diff --git a/.gitignore b/.gitignore index 6c52297..5358bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ lib TODO.md package-lock.json .vscode +BASELINE.md diff --git a/BASELINE.md b/BASELINE.md index f45cd81..c2790dc 100644 --- a/BASELINE.md +++ b/BASELINE.md @@ -97,6 +97,13 @@ canonical `EncodingName` values. Iconv/compiler input labels such as `CP866`, `CP855`, `MACINTOSH`, and `MAC-CYRILLIC` are implementation aliases and must not be returned by `detect` or `analyse`. +## Migration scope decision + +All existing SBCS recognizers will move to generated models. The legacy byte +maps and n-gram tables in `src/encoding/sbcs.ts` will not remain as a parallel +model system. Existing encodings must first be added to the corpus/model inputs, +generated, and validated before their hard-coded tables are removed. + ## Known pre-migration limitations - Generated SBCS models are not consumed or registered by the detector. diff --git a/corpus/README.md b/corpus/README.md index 2878450..f2a6b52 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -4,8 +4,9 @@ This directory contains a deterministic seed corpus for developing additional encoding recognisers. The prose is synthetic and should not be treated as a replacement for a large, independently sourced natural-language corpus. -`sources.json` stores UTF-8 originals. `manifest.json` maps languages to iconv -encodings. Generated byte samples and statistics live under `generated/`. +`sources.json` stores UTF-8 originals. Held-out material is kept separately in +`test-sources.json`. `manifest.json` maps languages to iconv encodings. +Generated byte samples and statistics live under `generated/`. Build and verify the corpus with: @@ -17,8 +18,9 @@ npm run models:evaluate npm run models:verify ``` -The build script generates the corpus and performs an exact iconv round trip for -every document. The check script independently rebuilds into a temporary +The build script generates the corpus and performs an iconv round trip for every +document, allowing canonically equivalent Unicode composition where an encoding +such as windows-1258 stores combining marks. The check script independently rebuilds into a temporary directory and byte-compares the result with `generated/`. Four documents per language are assigned to training, one to validation, and one independent document to testing. Statistics are calculated from training documents only. diff --git a/corpus/generated/ISO-8859-1/da/test/community-garden.bin b/corpus/generated/ISO-8859-1/da/test/community-garden.bin new file mode 100644 index 0000000..bbb5980 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/test/community-garden.bin @@ -0,0 +1,3 @@ +Fælleshaven + +Naboerne gjorde en tom gård til en grøn have, børnene malede skilte, og familierne vander planterne hver uge. diff --git a/corpus/generated/ISO-8859-1/da/train/city-morning.bin b/corpus/generated/ISO-8859-1/da/train/city-morning.bin new file mode 100644 index 0000000..aafe557 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen vågner langsomt, caféerne åbner, og folk venter på de første busser. diff --git a/corpus/generated/ISO-8859-1/da/train/library.bin b/corpus/generated/ISO-8859-1/da/train/library.bin new file mode 100644 index 0000000..dfcd24b --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/library.bin @@ -0,0 +1,3 @@ +Det stille bibliotek + +Læsere finder bøger om historie og videnskab, mens bibliotekaren hjælper eleverne. diff --git a/corpus/generated/ISO-8859-1/da/train/market.bin b/corpus/generated/ISO-8859-1/da/train/market.bin new file mode 100644 index 0000000..e156a58 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/market.bin @@ -0,0 +1,3 @@ +Lørdagsmarked + +Bønder sælger frisk brød, æbler, ost og honning på den travle plads. diff --git a/corpus/generated/ISO-8859-1/da/train/workshop.bin b/corpus/generated/ISO-8859-1/da/train/workshop.bin new file mode 100644 index 0000000..04a0b0c --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt værksted + +Et lille hold bygger en museumsguide, undersøger koden og skriver tydelige tekster. diff --git a/corpus/generated/ISO-8859-1/da/validation/river-trip.bin b/corpus/generated/ISO-8859-1/da/validation/river-trip.bin new file mode 100644 index 0000000..386c495 --- /dev/null +++ b/corpus/generated/ISO-8859-1/da/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til floden + +Vennerne tog et kort og varm te med på en søndag og hvilede bagefter ved floden. diff --git a/corpus/generated/ISO-8859-1/de/test/community-garden.bin b/corpus/generated/ISO-8859-1/de/test/community-garden.bin new file mode 100644 index 0000000..df4bbec --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/test/community-garden.bin @@ -0,0 +1,3 @@ +Der Gemeinschaftsgarten + +Die Nachbarn haben einen leeren Hof begrünt und dort Kräuter, Gemüse und junge Bäume gepflanzt. Die Schüler malten bunte Schilder, während ältere Bewohner regelmäßig gießen und Geräte aufräumen. diff --git a/corpus/generated/ISO-8859-1/de/train/city-morning.bin b/corpus/generated/ISO-8859-1/de/train/city-morning.bin new file mode 100644 index 0000000..3ccc201 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen in der Stadt + +Die Stadt erwacht langsam. Bäckereien öffnen ihre Türen, Busse beginnen ihre Fahrt und viele Menschen lesen unterwegs die Nachrichten. Über den Dächern wird der Himmel allmählich heller. diff --git a/corpus/generated/ISO-8859-1/de/train/library.bin b/corpus/generated/ISO-8859-1/de/train/library.bin new file mode 100644 index 0000000..992cca1 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/library.bin @@ -0,0 +1,3 @@ +Die ruhige Bibliothek + +In der kleinen Bibliothek rascheln leise die Seiten. Besucher suchen Bücher über Geschichte, Technik und ferne Länder. Eine Mitarbeiterin bereitet eine Ausstellung für Schüler vor. diff --git a/corpus/generated/ISO-8859-1/de/train/market.bin b/corpus/generated/ISO-8859-1/de/train/market.bin new file mode 100644 index 0000000..db29233 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/market.bin @@ -0,0 +1,3 @@ +Markt am Samstag + +Auf dem Platz bieten Bauern Äpfel, Käse, Honig und frisches Brot an. Die Kunden vergleichen Gemüse, sprechen über die Ernte und tauschen einfache Rezepte aus. diff --git a/corpus/generated/ISO-8859-1/de/train/workshop.bin b/corpus/generated/ISO-8859-1/de/train/workshop.bin new file mode 100644 index 0000000..17ea61c --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale Werkstatt + +Ein kleines Team entwickelt einen digitalen Museumsführer. Designer ordnen das Menü, Entwickler prüfen den Code und Redakteure schreiben verständliche Texte für Gäste. diff --git a/corpus/generated/ISO-8859-1/de/validation/river-trip.bin b/corpus/generated/ISO-8859-1/de/validation/river-trip.bin new file mode 100644 index 0000000..09bc118 --- /dev/null +++ b/corpus/generated/ISO-8859-1/de/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ausflug an den Fluss + +Am Sonntag wandern Freunde zu einem ruhigen Fluss. Sie nehmen eine Karte, heißen Tee und eine Kamera mit. Nach dem langen Weg sitzen sie am Ufer und hören den Vögeln zu. diff --git a/corpus/generated/ISO-8859-1/en/test/community-garden.bin b/corpus/generated/ISO-8859-1/en/test/community-garden.bin new file mode 100644 index 0000000..549b374 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/test/community-garden.bin @@ -0,0 +1,3 @@ +The community garden + +Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small café table. diff --git a/corpus/generated/ISO-8859-1/en/train/city-morning.bin b/corpus/generated/ISO-8859-1/en/train/city-morning.bin new file mode 100644 index 0000000..f7199ae --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/city-morning.bin @@ -0,0 +1,3 @@ +Morning in the city + +The city wakes slowly; cafés open their doors and commuters read news beside a façade. diff --git a/corpus/generated/ISO-8859-1/en/train/library.bin b/corpus/generated/ISO-8859-1/en/train/library.bin new file mode 100644 index 0000000..20400f3 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/library.bin @@ -0,0 +1,3 @@ +The quiet library + +Readers browse books about history, science, and naïve art while the librarian prepares a résumé. diff --git a/corpus/generated/ISO-8859-1/en/train/market.bin b/corpus/generated/ISO-8859-1/en/train/market.bin new file mode 100644 index 0000000..c760513 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/market.bin @@ -0,0 +1,3 @@ +Saturday market + +Farmers sell bread, apples, jalapeños, and pâté while neighbours exchange recipes. diff --git a/corpus/generated/ISO-8859-1/en/train/workshop.bin b/corpus/generated/ISO-8859-1/en/train/workshop.bin new file mode 100644 index 0000000..788536f --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/train/workshop.bin @@ -0,0 +1,3 @@ +Digital workshop + +A small team designs the museum guide and discusses its rôle, code, and clear descriptions. diff --git a/corpus/generated/ISO-8859-1/en/validation/river-trip.bin b/corpus/generated/ISO-8859-1/en/validation/river-trip.bin new file mode 100644 index 0000000..b686334 --- /dev/null +++ b/corpus/generated/ISO-8859-1/en/validation/river-trip.bin @@ -0,0 +1,3 @@ +Journey to the river + +Friends carried a map and a thermos, walked past the café, and rested by the river. diff --git a/corpus/generated/ISO-8859-1/es/test/community-garden.bin b/corpus/generated/ISO-8859-1/es/test/community-garden.bin new file mode 100644 index 0000000..3c82a6f --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/test/community-garden.bin @@ -0,0 +1,3 @@ +El jardín comunitario + +Los vecinos convirtieron un patio vacío en un jardín lleno de árboles, hierbas y verduras. Los niños pintaron carteles de colores y cada sábado varias familias riegan las plantas. diff --git a/corpus/generated/ISO-8859-1/es/train/city-morning.bin b/corpus/generated/ISO-8859-1/es/train/city-morning.bin new file mode 100644 index 0000000..518bc97 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/city-morning.bin @@ -0,0 +1,3 @@ +La mañana en la ciudad + +La ciudad despierta poco a poco. Las cafeterías abren sus puertas, los autobuses inician la ruta y los vecinos leen las noticias. El cielo se vuelve más claro sobre los tejados. diff --git a/corpus/generated/ISO-8859-1/es/train/library.bin b/corpus/generated/ISO-8859-1/es/train/library.bin new file mode 100644 index 0000000..4d6010f --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/library.bin @@ -0,0 +1,3 @@ +Una biblioteca tranquila + +En la biblioteca del barrio se oyen páginas y pasos suaves. Los lectores buscan libros de historia, ciencia y viajes. La bibliotecaria prepara una exposición para los alumnos. diff --git a/corpus/generated/ISO-8859-1/es/train/market.bin b/corpus/generated/ISO-8859-1/es/train/market.bin new file mode 100644 index 0000000..31af767 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/market.bin @@ -0,0 +1,3 @@ +El mercado del sábado + +El mercado llena la plaza de colores. Los agricultores venden manzanas, miel, queso y pan recién hecho. Los clientes comparan las verduras y comparten recetas de sus familias. diff --git a/corpus/generated/ISO-8859-1/es/train/workshop.bin b/corpus/generated/ISO-8859-1/es/train/workshop.bin new file mode 100644 index 0000000..92a2f60 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/train/workshop.bin @@ -0,0 +1,3 @@ +Taller digital + +Un equipo pequeño crea el catálogo digital del museo. Los diseñadores ordenan el menú, los programadores revisan el código y los editores escriben textos claros para el público. diff --git a/corpus/generated/ISO-8859-1/es/validation/river-trip.bin b/corpus/generated/ISO-8859-1/es/validation/river-trip.bin new file mode 100644 index 0000000..022d6f6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/es/validation/river-trip.bin @@ -0,0 +1,3 @@ +Paseo junto al río + +El domingo unos amigos caminan hacia un río tranquilo. Llevan un mapa, té caliente y una cámara. Después del paseo descansan en la orilla y escuchan a los pájaros. diff --git a/corpus/generated/ISO-8859-1/fr/test/community-garden.bin b/corpus/generated/ISO-8859-1/fr/test/community-garden.bin new file mode 100644 index 0000000..2dce62d --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/test/community-garden.bin @@ -0,0 +1,3 @@ +Le jardin partagé + +Les habitants ont transformé une cour oubliée en jardin fleuri. Les enfants ont préparé des étiquettes colorées, tandis que les voisins se réunissent chaque semaine pour arroser les légumes et les jeunes arbres. diff --git a/corpus/generated/ISO-8859-1/fr/train/city-morning.bin b/corpus/generated/ISO-8859-1/fr/train/city-morning.bin new file mode 100644 index 0000000..411550d --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/city-morning.bin @@ -0,0 +1,3 @@ +Le matin en ville + +La ville se réveille lentement. Les cafés ouvrent leurs portes, les autobus commencent leur trajet et les passants consultent les nouvelles. Chacun prépare sa journée sous un ciel encore pâle. diff --git a/corpus/generated/ISO-8859-1/fr/train/library.bin b/corpus/generated/ISO-8859-1/fr/train/library.bin new file mode 100644 index 0000000..e06a9ef --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/library.bin @@ -0,0 +1,3 @@ +La bibliothèque calme + +Dans la bibliothèque du quartier, les pages tournent en silence. Les lecteurs cherchent des livres sur l'histoire, les sciences et les voyages. Une bibliothécaire prépare une exposition pour les élèves. diff --git a/corpus/generated/ISO-8859-1/fr/train/market.bin b/corpus/generated/ISO-8859-1/fr/train/market.bin new file mode 100644 index 0000000..9717faa --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/market.bin @@ -0,0 +1,3 @@ +Le marché du samedi + +Le marché remplit la place de couleurs et de parfums. Les producteurs proposent des pommes, du miel, du fromage et du pain frais. Les clients discutent des saisons et échangent des recettes familiales. diff --git a/corpus/generated/ISO-8859-1/fr/train/workshop.bin b/corpus/generated/ISO-8859-1/fr/train/workshop.bin new file mode 100644 index 0000000..4dea00e --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/train/workshop.bin @@ -0,0 +1,3 @@ +Un atelier numérique + +Une petite équipe construit le catalogue numérique du musée. Les graphistes organisent le menu, les développeurs vérifient le code et les rédacteurs préparent des textes clairs pour le public. diff --git a/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin b/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin new file mode 100644 index 0000000..48f3a71 --- /dev/null +++ b/corpus/generated/ISO-8859-1/fr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Une promenade près de la rivière + +Dimanche, des amis partent vers une rivière tranquille. Ils emportent une carte, du thé chaud et un appareil photo. Après une longue marche, ils se reposent sur la rive et écoutent les oiseaux. diff --git a/corpus/generated/ISO-8859-1/it/test/community-garden.bin b/corpus/generated/ISO-8859-1/it/test/community-garden.bin new file mode 100644 index 0000000..76a54ea --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/test/community-garden.bin @@ -0,0 +1,3 @@ +Il giardino comune + +I vicini hanno trasformato un cortile vuoto in un giardino della città, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante. diff --git a/corpus/generated/ISO-8859-1/it/train/city-morning.bin b/corpus/generated/ISO-8859-1/it/train/city-morning.bin new file mode 100644 index 0000000..a8faca7 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/city-morning.bin @@ -0,0 +1,3 @@ +Mattina in città + +La città si sveglia lentamente, i caffè aprono e la gente aspetta i primi autobus. diff --git a/corpus/generated/ISO-8859-1/it/train/library.bin b/corpus/generated/ISO-8859-1/it/train/library.bin new file mode 100644 index 0000000..e9b65ab --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/library.bin @@ -0,0 +1,3 @@ +La biblioteca tranquilla + +I lettori cercano più libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti più giovani. diff --git a/corpus/generated/ISO-8859-1/it/train/market.bin b/corpus/generated/ISO-8859-1/it/train/market.bin new file mode 100644 index 0000000..780f149 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/market.bin @@ -0,0 +1,3 @@ +Mercato del sabato + +Gli agricoltori vendono pane fresco, caffè, mele, formaggio e miele nella piazza della città. diff --git a/corpus/generated/ISO-8859-1/it/train/workshop.bin b/corpus/generated/ISO-8859-1/it/train/workshop.bin new file mode 100644 index 0000000..21734d6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/train/workshop.bin @@ -0,0 +1,3 @@ +Laboratorio digitale + +Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perché siano utili a più persone. diff --git a/corpus/generated/ISO-8859-1/it/validation/river-trip.bin b/corpus/generated/ISO-8859-1/it/validation/river-trip.bin new file mode 100644 index 0000000..0ff2d62 --- /dev/null +++ b/corpus/generated/ISO-8859-1/it/validation/river-trip.bin @@ -0,0 +1,3 @@ +Gita al fiume + +Gli amici portano una mappa e tè caldo, camminano nel bosco e riposano sulla riva. diff --git a/corpus/generated/ISO-8859-1/nl/test/community-garden.bin b/corpus/generated/ISO-8859-1/nl/test/community-garden.bin new file mode 100644 index 0000000..89526b9 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/test/community-garden.bin @@ -0,0 +1,3 @@ +De buurttuin + +Buren veranderden een lege binnenplaats naast een café in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water. diff --git a/corpus/generated/ISO-8859-1/nl/train/city-morning.bin b/corpus/generated/ISO-8859-1/nl/train/city-morning.bin new file mode 100644 index 0000000..13f5ea7 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/city-morning.bin @@ -0,0 +1,3 @@ +Ochtend in de stad + +De stad ontwaakt langzaam, cafés openen hun deuren en mensen wachten op de eerste bussen. diff --git a/corpus/generated/ISO-8859-1/nl/train/library.bin b/corpus/generated/ISO-8859-1/nl/train/library.bin new file mode 100644 index 0000000..cdf34ac --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/library.bin @@ -0,0 +1,3 @@ +De stille bibliotheek + +Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris geïnteresseerde leerlingen en één leraar helpt. diff --git a/corpus/generated/ISO-8859-1/nl/train/market.bin b/corpus/generated/ISO-8859-1/nl/train/market.bin new file mode 100644 index 0000000..9c584bb --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/market.bin @@ -0,0 +1,3 @@ +Zaterdagmarkt + +Boeren verkopen vers brood, appels, kaas en honing naast twee cafés op het drukke plein. diff --git a/corpus/generated/ISO-8859-1/nl/train/workshop.bin b/corpus/generated/ISO-8859-1/nl/train/workshop.bin new file mode 100644 index 0000000..53ff667 --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale werkplaats + +Een klein team maakt een museumgids, controleert de code en schrijft duidelijke ideeën over poëzie en teksten. diff --git a/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin b/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin new file mode 100644 index 0000000..27db29e --- /dev/null +++ b/corpus/generated/ISO-8859-1/nl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Reis naar de rivier + +Vrienden namen een kaart en warme thee mee en rustten na één lange wandeling bij de rivier. diff --git a/corpus/generated/ISO-8859-1/no/test/community-garden.bin b/corpus/generated/ISO-8859-1/no/test/community-garden.bin new file mode 100644 index 0000000..b82814c --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/test/community-garden.bin @@ -0,0 +1,3 @@ +Felleshagen + +Naboene gjorde en tom gård til en grønn hage, barna malte skilt og familiene vanner plantene hver uke. diff --git a/corpus/generated/ISO-8859-1/no/train/city-morning.bin b/corpus/generated/ISO-8859-1/no/train/city-morning.bin new file mode 100644 index 0000000..972847d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen våkner langsomt, kafeene åpner dørene og folk venter på de første bussene nær bryggen og sjøen. diff --git a/corpus/generated/ISO-8859-1/no/train/library.bin b/corpus/generated/ISO-8859-1/no/train/library.bin new file mode 100644 index 0000000..f972c3d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/library.bin @@ -0,0 +1,3 @@ +Det stille biblioteket + +Lesere finner bøker om historie, ærlige fortellinger og vitenskap mens bibliotekaren hjelper elevene. diff --git a/corpus/generated/ISO-8859-1/no/train/market.bin b/corpus/generated/ISO-8859-1/no/train/market.bin new file mode 100644 index 0000000..c020f8d --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/market.bin @@ -0,0 +1,3 @@ +Lørdagsmarked + +Bønder selger ferskt brød, epler, ost og honning på det travle torget. diff --git a/corpus/generated/ISO-8859-1/no/train/workshop.bin b/corpus/generated/ISO-8859-1/no/train/workshop.bin new file mode 100644 index 0000000..954e228 --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt verksted + +Et lite lag lager en museumsguide, undersøker koden og skriver tydelige tekster. diff --git a/corpus/generated/ISO-8859-1/no/validation/river-trip.bin b/corpus/generated/ISO-8859-1/no/validation/river-trip.bin new file mode 100644 index 0000000..1489859 --- /dev/null +++ b/corpus/generated/ISO-8859-1/no/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til elven + +Vennene tok med kart og varm te på søndag og hvilte senere ved elven. diff --git a/corpus/generated/ISO-8859-1/pt/test/community-garden.bin b/corpus/generated/ISO-8859-1/pt/test/community-garden.bin new file mode 100644 index 0000000..65848b6 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/test/community-garden.bin @@ -0,0 +1,3 @@ +A horta comunitária + +Os vizinhos transformaram um pátio vazio em jardim, as crianças pintaram placas e as famílias regam as plantas toda semana. diff --git a/corpus/generated/ISO-8859-1/pt/train/city-morning.bin b/corpus/generated/ISO-8859-1/pt/train/city-morning.bin new file mode 100644 index 0000000..18b861f --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/city-morning.bin @@ -0,0 +1,3 @@ +Manhã na cidade + +A cidade acorda devagar, os cafés abrem e as pessoas esperam os primeiros ônibus. diff --git a/corpus/generated/ISO-8859-1/pt/train/library.bin b/corpus/generated/ISO-8859-1/pt/train/library.bin new file mode 100644 index 0000000..50b8034 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/library.bin @@ -0,0 +1,3 @@ +A biblioteca tranquila + +Os leitores procuram livros de história e ciência enquanto a bibliotecária ajuda os alunos. diff --git a/corpus/generated/ISO-8859-1/pt/train/market.bin b/corpus/generated/ISO-8859-1/pt/train/market.bin new file mode 100644 index 0000000..3810190 --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/market.bin @@ -0,0 +1,3 @@ +Mercado de sábado + +Os agricultores vendem pão fresco, maçãs, queijo e mel na praça movimentada. diff --git a/corpus/generated/ISO-8859-1/pt/train/workshop.bin b/corpus/generated/ISO-8859-1/pt/train/workshop.bin new file mode 100644 index 0000000..ac78c4e --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/train/workshop.bin @@ -0,0 +1,3 @@ +Oficina digital + +Uma pequena equipe prepara o guia do museu, verifica o código e escreve descrições claras. diff --git a/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin b/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin new file mode 100644 index 0000000..de2590a --- /dev/null +++ b/corpus/generated/ISO-8859-1/pt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Viagem ao rio + +Os amigos levaram um mapa e chá quente, caminharam pela mata e descansaram junto ao rio. diff --git a/corpus/generated/ISO-8859-1/sv/test/community-garden.bin b/corpus/generated/ISO-8859-1/sv/test/community-garden.bin new file mode 100644 index 0000000..0acb233 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/test/community-garden.bin @@ -0,0 +1,3 @@ +Gemensam trädgård + +Grannarna gjorde en tom gård till en grön trädgård, barnen målade skyltar och familjerna vattnar växterna varje vecka. diff --git a/corpus/generated/ISO-8859-1/sv/train/city-morning.bin b/corpus/generated/ISO-8859-1/sv/train/city-morning.bin new file mode 100644 index 0000000..f0ba24b --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgon i staden + +Staden vaknar långsamt, kaféerna öppnar och människor väntar på de första bussarna. diff --git a/corpus/generated/ISO-8859-1/sv/train/library.bin b/corpus/generated/ISO-8859-1/sv/train/library.bin new file mode 100644 index 0000000..a7319ac --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/library.bin @@ -0,0 +1,3 @@ +Det tysta biblioteket + +Läsare söker böcker om historia och vetenskap medan bibliotekarien hjälper eleverna. diff --git a/corpus/generated/ISO-8859-1/sv/train/market.bin b/corpus/generated/ISO-8859-1/sv/train/market.bin new file mode 100644 index 0000000..0bd6cfb --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/market.bin @@ -0,0 +1,3 @@ +Lördagsmarknad + +Bönder säljer färskt bröd, äpplen, ost och honung på det livliga torget. diff --git a/corpus/generated/ISO-8859-1/sv/train/workshop.bin b/corpus/generated/ISO-8859-1/sv/train/workshop.bin new file mode 100644 index 0000000..81db320 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/train/workshop.bin @@ -0,0 +1,3 @@ +Digital verkstad + +Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter för besökare. diff --git a/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin b/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin new file mode 100644 index 0000000..4704973 --- /dev/null +++ b/corpus/generated/ISO-8859-1/sv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Utflykt till floden + +Vännerna tog med karta och varmt te och vilade sedan vid floden. diff --git a/corpus/generated/ISO-8859-2/cs/test/community-garden.bin b/corpus/generated/ISO-8859-2/cs/test/community-garden.bin new file mode 100644 index 0000000..61bfe36 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/test/community-garden.bin @@ -0,0 +1,3 @@ +Sousedská zahrada + +Sousedé promìnili prázdný dvùr v zelenou zahradu, dìti namalovaly barevné znaèky a rodiny zalévají rostliny. diff --git a/corpus/generated/ISO-8859-2/cs/train/city-morning.bin b/corpus/generated/ISO-8859-2/cs/train/city-morning.bin new file mode 100644 index 0000000..6efcbfe --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/city-morning.bin @@ -0,0 +1,3 @@ +Ráno ve mìstì + +Mìsto se pomalu probouzí, kavárny otevírají dveøe a lidé èekají na první autobusy. diff --git a/corpus/generated/ISO-8859-2/cs/train/library.bin b/corpus/generated/ISO-8859-2/cs/train/library.bin new file mode 100644 index 0000000..31bd1a9 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/library.bin @@ -0,0 +1,3 @@ +Tichá knihovna + +Ètenáøi hledají knihy o historii a vìdì, zatímco knihovnice radí ¾ákùm. diff --git a/corpus/generated/ISO-8859-2/cs/train/market.bin b/corpus/generated/ISO-8859-2/cs/train/market.bin new file mode 100644 index 0000000..f274a31 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/market.bin @@ -0,0 +1,3 @@ +Sobotní trh + +Farmáøi prodávají èerstvý chléb, med, ovoce a voòavé byliny. diff --git a/corpus/generated/ISO-8859-2/cs/train/workshop.bin b/corpus/generated/ISO-8859-2/cs/train/workshop.bin new file mode 100644 index 0000000..d1dd133 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/train/workshop.bin @@ -0,0 +1,3 @@ +Digitální dílna + +Malý tým pøipravuje prùvodce muzeem, kontroluje kód a pí¹e jasné popisy. diff --git a/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin b/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin new file mode 100644 index 0000000..09eba90 --- /dev/null +++ b/corpus/generated/ISO-8859-2/cs/validation/river-trip.bin @@ -0,0 +1,3 @@ +Výlet k øece + +Pøátelé vzali mapu a teplý èaj, pro¹li lesem a odpoèívali u øeky. diff --git a/corpus/generated/ISO-8859-2/hu/test/community-garden.bin b/corpus/generated/ISO-8859-2/hu/test/community-garden.bin new file mode 100644 index 0000000..251d1d3 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/test/community-garden.bin @@ -0,0 +1,3 @@ +Közösségi kert + +A szomszédok zöld kertté alakították az üres udvart, a gyerekek táblákat festettek, a családok pedig öntözik a növényeket. diff --git a/corpus/generated/ISO-8859-2/hu/train/city-morning.bin b/corpus/generated/ISO-8859-2/hu/train/city-morning.bin new file mode 100644 index 0000000..74c3adc --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/city-morning.bin @@ -0,0 +1,3 @@ +Reggel a városban + +A város lassan ébred, a kávézók kinyitnak, az emberek pedig buszra várnak. diff --git a/corpus/generated/ISO-8859-2/hu/train/library.bin b/corpus/generated/ISO-8859-2/hu/train/library.bin new file mode 100644 index 0000000..4bbeea1 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/library.bin @@ -0,0 +1,3 @@ +A csendes könyvtár + +Az olvasók történelmi és tudományos könyveket keresnek, a könyvtáros segít a diákoknak. diff --git a/corpus/generated/ISO-8859-2/hu/train/market.bin b/corpus/generated/ISO-8859-2/hu/train/market.bin new file mode 100644 index 0000000..b95088b --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/market.bin @@ -0,0 +1,3 @@ +Szombati piac + +A gazdák friss kenyeret, almát, sajtot és mézet árulnak a téren. diff --git a/corpus/generated/ISO-8859-2/hu/train/workshop.bin b/corpus/generated/ISO-8859-2/hu/train/workshop.bin new file mode 100644 index 0000000..819901d --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/train/workshop.bin @@ -0,0 +1,3 @@ +Digitális mûhely + +Egy kis csapat múzeumi útmutatót készít, kódot ellenõriz és világos leírásokat ír. diff --git a/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin b/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin new file mode 100644 index 0000000..f807c95 --- /dev/null +++ b/corpus/generated/ISO-8859-2/hu/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kirándulás a folyóhoz + +A barátok térképet és meleg teát vittek, majd megpihentek a folyó partján. diff --git a/corpus/generated/ISO-8859-2/pl/test/community-garden.bin b/corpus/generated/ISO-8859-2/pl/test/community-garden.bin new file mode 100644 index 0000000..804f020 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/test/community-garden.bin @@ -0,0 +1,3 @@ +Ogród s±siedzki + +Mieszkañcy zmienili pusty dziedziniec w zielony ogród z kwiatami, warzywami i m³odymi drzewami. Dzieci zrobi³y kolorowe tabliczki, a rodziny co tydzieñ podlewaj± ro¶liny. diff --git a/corpus/generated/ISO-8859-2/pl/train/city-morning.bin b/corpus/generated/ISO-8859-2/pl/train/city-morning.bin new file mode 100644 index 0000000..a1106b8 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/city-morning.bin @@ -0,0 +1,3 @@ +Poranek w mie¶cie + +Miasto budzi siê powoli. Kawiarnie otwieraj± drzwi, autobusy rozpoczynaj± pierwsze kursy, a ludzie czytaj± wiadomo¶ci. Nad dachami robi siê coraz ja¶niej. diff --git a/corpus/generated/ISO-8859-2/pl/train/library.bin b/corpus/generated/ISO-8859-2/pl/train/library.bin new file mode 100644 index 0000000..dc98317 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/library.bin @@ -0,0 +1,3 @@ +Cicha biblioteka + +W osiedlowej bibliotece s³ychaæ szelest kartek. Czytelnicy szukaj± ksi±¿ek o historii, nauce i podró¿ach. Bibliotekarka przygotowuje wystawê dla uczniów. diff --git a/corpus/generated/ISO-8859-2/pl/train/market.bin b/corpus/generated/ISO-8859-2/pl/train/market.bin new file mode 100644 index 0000000..fcebf79 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/market.bin @@ -0,0 +1,3 @@ +Sobotni targ + +Targ wype³nia rynek kolorami i zapachami. Rolnicy sprzedaj± jab³ka, miód, ser i ¶wie¿y chleb. Klienci wybieraj± warzywa i wymieniaj± rodzinne przepisy. diff --git a/corpus/generated/ISO-8859-2/pl/train/workshop.bin b/corpus/generated/ISO-8859-2/pl/train/workshop.bin new file mode 100644 index 0000000..9de1523 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/train/workshop.bin @@ -0,0 +1,3 @@ +Cyfrowa pracownia + +Ma³y zespó³ tworzy cyfrowy przewodnik po muzeum. Projektanci uk³adaj± menu, programi¶ci sprawdzaj± kod, a redaktorzy pisz± jasne opisy dla go¶ci. diff --git a/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin b/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin new file mode 100644 index 0000000..b715445 --- /dev/null +++ b/corpus/generated/ISO-8859-2/pl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Spacer nad rzek± + +W niedzielê przyjaciele poszli nad spokojn± rzekê za miastem. Zabrali mapê, ciep³± herbatê i aparat. Po d³ugim spacerze odpoczywali na brzegu. diff --git a/corpus/generated/ISO-8859-2/ro/test/community-garden.bin b/corpus/generated/ISO-8859-2/ro/test/community-garden.bin new file mode 100644 index 0000000..54f8405 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/test/community-garden.bin @@ -0,0 +1,3 @@ +Grãdina comunitãþii + +Vecinii au transformat o curte goalã într-o grãdinã cu flori, legume ºi pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se întâlnesc sãptãmânal pentru a uda plantele. diff --git a/corpus/generated/ISO-8859-2/ro/train/city-morning.bin b/corpus/generated/ISO-8859-2/ro/train/city-morning.bin new file mode 100644 index 0000000..08d7469 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/city-morning.bin @@ -0,0 +1,3 @@ +Dimineaþa în oraº + +Oraºul se trezeºte încet. Cafenelele deschid uºile, autobuzele pornesc pe traseu, iar oamenii citesc ºtirile. Cerul devine mai luminos deasupra clãdirilor. diff --git a/corpus/generated/ISO-8859-2/ro/train/library.bin b/corpus/generated/ISO-8859-2/ro/train/library.bin new file mode 100644 index 0000000..1afc415 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/library.bin @@ -0,0 +1,3 @@ +Biblioteca liniºtitã + +În biblioteca de cartier se aud pagini întoarse încet. Cititorii cautã cãrþi despre istorie, ºtiinþã ºi cãlãtorii. Bibliotecara pregãteºte o expoziþie pentru elevi. diff --git a/corpus/generated/ISO-8859-2/ro/train/market.bin b/corpus/generated/ISO-8859-2/ro/train/market.bin new file mode 100644 index 0000000..15a0338 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/market.bin @@ -0,0 +1,3 @@ +Piaþa de sâmbãtã + +Piaþa umple centrul cu mirosuri ºi culori. Fermierii aduc mere, miere, brânzã ºi pâine proaspãtã. Cumpãrãtorii aleg legume ºi schimbã reþete de familie. diff --git a/corpus/generated/ISO-8859-2/ro/train/workshop.bin b/corpus/generated/ISO-8859-2/ro/train/workshop.bin new file mode 100644 index 0000000..132f010 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/train/workshop.bin @@ -0,0 +1,3 @@ +Atelier digital + +O echipã micã realizeazã ghidul digital al muzeului. Designerii organizeazã meniul, programatorii verificã sursa, iar redactorii scriu explicaþii clare. diff --git a/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin b/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin new file mode 100644 index 0000000..c125be0 --- /dev/null +++ b/corpus/generated/ISO-8859-2/ro/validation/river-trip.bin @@ -0,0 +1,3 @@ +Plimbare lângã râu + +Duminicã, prietenii au mers spre un râu liniºtit din afara oraºului. Au luat o hartã, ceai cald ºi un aparat foto. Dupã plimbare s-au odihnit pe mal. diff --git a/corpus/generated/ISO-8859-5/ru/test/community-garden.bin b/corpus/generated/ISO-8859-5/ru/test/community-garden.bin new file mode 100644 index 0000000..1ed1849 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/test/community-garden.bin @@ -0,0 +1,3 @@ +¾ÑéØÙ áÐÔ + +¶ØâÕÛØ ÒÜÕáâÕ ßàØÒÕÛØ Ò ßÞàïÔÞÚ áâÐàëÙ ÔÒÞà Ø ßÞáÐÔØÛØ æÒÕâë, ×ÕÛÕÝì Ø ÜÞÛÞÔëÕ ÔÕàÕÒìï. ´ÕâØ áÔÕÛÐÛØ ïàÚØÕ âÐÑÛØçÚØ, Ð áÞáÕÔØ ÔÞÓÞÒÞàØÛØáì ÚÐÖÔãî ÝÕÔÕÛî ßÞÛØÒÐâì àÐáâÕÝØï. diff --git a/corpus/generated/ISO-8859-5/ru/train/city-morning.bin b/corpus/generated/ISO-8859-5/ru/train/city-morning.bin new file mode 100644 index 0000000..e329ef5 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/city-morning.bin @@ -0,0 +1,3 @@ +ÃâàÞ Ò ÓÞàÞÔÕ + +³ÞàÞÔ ßàÞáëßÐÕâáï ßÞáâÕßÕÝÝÞ. ½Ð ÞáâÐÝÞÒÚÐå ßÞïÒÛïîâáï ßÐááÐÖØàë, ÑãÛÞçÝÐï ÞâÚàëÒÐÕâ ÔÒÕàØ, Ð ßÕàÒëÕ ÐÒâÞÑãáë ÒëåÞÔïâ ÝÐ ÜÐàèàãâ. »îÔØ çØâÐîâ áÞÞÑéÕÝØï Ø ÞÑáãÖÔÐîâ ßÛÐÝë ÝÐ ÝÞÒëÙ ÔÕÝì. diff --git a/corpus/generated/ISO-8859-5/ru/train/library.bin b/corpus/generated/ISO-8859-5/ru/train/library.bin new file mode 100644 index 0000000..a05a46f --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/library.bin @@ -0,0 +1,3 @@ +ÀÐÙÞÝÝÐï ÑØÑÛØÞâÕÚÐ + +² ÑØÑÛØÞâÕÚÕ âØåÞ èÕÛÕáâïâ áâàÐÝØæë. ÇØâÐâÕÛØ ÒëÑØàÐîâ ÚÝØÓØ ÞÑ ØáâÞàØØ, ÝÐãÚÕ Ø ßãâÕèÕáâÒØïå. ÁÞâàãÔÝØÚØ ÓÞâÞÒïâ ÒëáâÐÒÚã, ÞâÒÕçÐîâ ÝÐ ÒÞßàÞáë Ø ßÞÜÞÓÐîâ èÚÞÛìÝØÚÐÜ ÝÐÙâØ ÝãÖÝëÕ ÜÐâÕàØÐÛë. diff --git a/corpus/generated/ISO-8859-5/ru/train/market.bin b/corpus/generated/ISO-8859-5/ru/train/market.bin new file mode 100644 index 0000000..3ff323b --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/market.bin @@ -0,0 +1,3 @@ +¾áÕÝÝïï ïàÜÐàÚÐ + +½Ð ßÛÞéÐÔØ ßàÞåÞÔØâ ÞáÕÝÝïï ïàÜÐàÚÐ. ÄÕàÜÕàë ßàØÒÕ×ÛØ ïÑÛÞÚØ, ÜñÔ, áëà Ø áÒÕÖØÙ åÛÕÑ. ¿ÞÚãßÐâÕÛØ áàÐÒÝØÒÐîâ ßàÞÔãÚâë, ÑÕáÕÔãîâ á ßàÞÔÐÒæÐÜØ Ø ×ÐßØáëÒÐîâ ØÝâÕàÕáÝëÕ ÔÞÜÐèÝØÕ àÕæÕßâë. diff --git a/corpus/generated/ISO-8859-5/ru/train/workshop.bin b/corpus/generated/ISO-8859-5/ru/train/workshop.bin new file mode 100644 index 0000000..54ff5e6 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/train/workshop.bin @@ -0,0 +1,3 @@ +ÀÐÑÞçÐï ÜÐáâÕàáÚÐï + +½ÕÑÞÛìèÐï ÚÞÜÐÝÔÐ áÞ×ÔÐñâ íÛÕÚâàÞÝÝëÙ ÚÐâÐÛÞÓ Üã×Õï. ´Ø×ÐÙÝÕàë ÞÑáãÖÔÐîâ ÜÕÝî, ßàÞÓàÐÜÜØáâë ßàÞÒÕàïîâ ÚÞÔ, Ð àÕÔÐÚâÞàë ÓÞâÞÒïâ ïáÝëÕ ÞßØáÐÝØï ÔÛï ßÞáÕâØâÕÛÕÙ àÐ×Ýëå ÒÞ×àÐáâÞÒ. diff --git a/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin b/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin new file mode 100644 index 0000000..1c97742 --- /dev/null +++ b/corpus/generated/ISO-8859-5/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ +¿ÞÕ×ÔÚÐ Ú àÕÚÕ + +² ÒÞáÚàÕáÕÝìÕ Ôàã×ìï ÞâßàÐÒØÛØáì Ú áßÞÚÞÙÝÞÙ àÕÚÕ ×Ð ÓÞàÞÔÞÜ. ¾ÝØ Ò×ïÛØ ÚÐàâã, ÓÞàïçØÙ çÐÙ Ø äÞâÞÐßßÐàÐâ. ¿ÞáÛÕ ÔÞÛÓÞÙ ßàÞÓãÛÚØ ßãâÕèÕáâÒÕÝÝØÚØ ÞâÔëåÐÛØ ÝÐ ÑÕàÕÓã Ø áÛãèÐÛØ ÒÕçÕàÝØå ßâØæ. diff --git a/corpus/generated/ISO-8859-6/ar/test/community-garden.bin b/corpus/generated/ISO-8859-6/ar/test/community-garden.bin new file mode 100644 index 0000000..d2a9d0c --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/test/community-garden.bin @@ -0,0 +1,3 @@ +ÍÏêâÉ ÇäÍê + +ÒÑÙ ÇäÌêÑÇæ ÇäÒçèÑ èÇäÃÔÌÇÑ áê ÓÇÍÉ âÏêåɬ èÕæÙ ÇäÃ×áÇä äÇáÊÇÊ åäèæÉ èÇÊáâ ÇäÌåêÙ Ùäé Óâê ÇäæÈÇÊÇÊ. diff --git a/corpus/generated/ISO-8859-6/ar/train/city-morning.bin b/corpus/generated/ISO-8859-6/ar/train/city-morning.bin new file mode 100644 index 0000000..50c1ce8 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/city-morning.bin @@ -0,0 +1,3 @@ +ÕÈÇÍ ÇäåÏêæÉ + +ÊÓÊêâØ ÇäåÏêæÉ ÈçÏèÁ èÊáÊÍ ÇäåâÇçê ÃÈèÇÈçÇ ÈêæåÇ êæÊØÑ ÇäæÇÓ ÇäÍÇáäÇÊ èêâÑÄèæ ÇäÃÎÈÇÑ. diff --git a/corpus/generated/ISO-8859-6/ar/train/library.bin b/corpus/generated/ISO-8859-6/ar/train/library.bin new file mode 100644 index 0000000..ddd9db9 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/library.bin @@ -0,0 +1,3 @@ +ÇäåãÊÈÉ ÇäçÇÏÆÉ + +êÈÍË ÇäâÑÇÁ Ùæ ãÊÈ ÇäÊÇÑêÎ èÇäÙäèå èêÓÇÙÏ Ãåêæ ÇäåãÊÈÉ Çä×äÇÈ áê ÇÎÊêÇÑ ÇäåÑÇÌÙ ÇäåæÇÓÈÉ. diff --git a/corpus/generated/ISO-8859-6/ar/train/market.bin b/corpus/generated/ISO-8859-6/ar/train/market.bin new file mode 100644 index 0000000..67ef9f6 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/market.bin @@ -0,0 +1,3 @@ +Óèâ ÇäÓÈÊ + +êÙÑÖ ÇäåÒÇÑÙèæ ÇäÎÈÒ èÇäÙÓä èÇäáÇãçÉ èêÊÈÇÏä ÇäÒèÇÑ ÇäèÕáÇÊ èÇäÃÍÇÏêË ÇäèÏêÉ. diff --git a/corpus/generated/ISO-8859-6/ar/train/workshop.bin b/corpus/generated/ISO-8859-6/ar/train/workshop.bin new file mode 100644 index 0000000..27001c3 --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/train/workshop.bin @@ -0,0 +1,3 @@ +èÑÔÉ ÑâåêÉ + +êÕåå áÑêâ ÕÚêÑ ÏäêäÇ ääåÊÍá èêÑÇÌÙ ÇäåÈÑåÌèæ ÇäÔáÑÉ èêãÊÈ ÇäåÍÑÑèæ èÕáÇ èÇÖÍÇ. diff --git a/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin b/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin new file mode 100644 index 0000000..2d416cb --- /dev/null +++ b/corpus/generated/ISO-8859-6/ar/validation/river-trip.bin @@ -0,0 +1,3 @@ +ÑÍäÉ Åäé ÇäæçÑ + +Íåä ÇäÃÕÏâÇÁ ÎÑê×É èÔÇêÇ ÓÇÎæÇ èåÔèÇ ÍÊé ÇäæçÑ Ëå ÇÓÊåÙèÇ Åäé ×êèÑ ÇäåÓÇÁ. diff --git a/corpus/generated/ISO-8859-7/el/test/community-garden.bin b/corpus/generated/ISO-8859-7/el/test/community-garden.bin new file mode 100644 index 0000000..2e8196c --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/test/community-garden.bin @@ -0,0 +1,3 @@ +Ï êïéíïôéêüò êÞðïò + +Ïé ãåßôïíåò Ýêáíáí ìéá Üäåéá áõëÞ ðñÜóéíï êÞðï, ôá ðáéäéÜ æùãñÜöéóáí ðéíáêßäåò êáé ïé ïéêïãÝíåéåò ðïôßæïõí ôá öõôÜ. diff --git a/corpus/generated/ISO-8859-7/el/train/city-morning.bin b/corpus/generated/ISO-8859-7/el/train/city-morning.bin new file mode 100644 index 0000000..f58abba --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/city-morning.bin @@ -0,0 +1,3 @@ +Ðñùß óôçí ðüëç + +Ç ðüëç îõðíÜ áñãÜ, ôá êáöÝ áíïßãïõí êáé ïé Üíèñùðïé ðåñéìÝíïõí ôá ëåùöïñåßá. diff --git a/corpus/generated/ISO-8859-7/el/train/library.bin b/corpus/generated/ISO-8859-7/el/train/library.bin new file mode 100644 index 0000000..6724218 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/library.bin @@ -0,0 +1,3 @@ +¹óõ÷ç âéâëéïèÞêç + +Ïé áíáãíþóôåò øÜ÷íïõí âéâëßá éóôïñßáò êáé åðéóôÞìçò êáé ç âéâëéïèçêÜñéïò âïçèÜ ôïõò ìáèçôÝò. diff --git a/corpus/generated/ISO-8859-7/el/train/market.bin b/corpus/generated/ISO-8859-7/el/train/market.bin new file mode 100644 index 0000000..40995f7 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/market.bin @@ -0,0 +1,3 @@ +ÓáââáôéÜôéêç áãïñÜ + +Ïé áãñüôåò ðïõëïýí øùìß, ìÝëé, öñïýôá êáé ìõñùäéêÜ óôçí ðëáôåßá. diff --git a/corpus/generated/ISO-8859-7/el/train/workshop.bin b/corpus/generated/ISO-8859-7/el/train/workshop.bin new file mode 100644 index 0000000..4f44381 --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/train/workshop.bin @@ -0,0 +1,3 @@ +Øçöéáêü åñãáóôÞñéï + +Ìéá ìéêñÞ ïìÜäá åôïéìÜæåé ïäçãü ìïõóåßïõ, åëÝã÷åé ôïí êþäéêá êáé ãñÜöåé êáèáñÜ êåßìåíá. diff --git a/corpus/generated/ISO-8859-7/el/validation/river-trip.bin b/corpus/generated/ISO-8859-7/el/validation/river-trip.bin new file mode 100644 index 0000000..dcea70f --- /dev/null +++ b/corpus/generated/ISO-8859-7/el/validation/river-trip.bin @@ -0,0 +1,3 @@ +ÅêäñïìÞ óôï ðïôÜìé + +Ïé ößëïé ðÞñáí ÷Üñôç êáé æåóôü ôóÜé êáé îåêïõñÜóôçêáí äßðëá óôï ðïôÜìé. diff --git a/corpus/generated/ISO-8859-8/he/test/community-garden.bin b/corpus/generated/ISO-8859-8/he/test/community-garden.bin new file mode 100644 index 0000000..28d54b2 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/test/community-garden.bin @@ -0,0 +1,3 @@ +äâéðä ä÷äéìúéú + +äùëðéí äôëå çöø øé÷ä ìâéðä éøå÷ä, äéìãéí öééøå ùìèéí åäîùôçåú îù÷åú àú äöîçéí áëì ùáåò. diff --git a/corpus/generated/ISO-8859-8/he/train/city-morning.bin b/corpus/generated/ISO-8859-8/he/train/city-morning.bin new file mode 100644 index 0000000..90ce4a0 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/city-morning.bin @@ -0,0 +1,3 @@ +áå÷ø áòéø + +äòéø îúòåøøú ìàè, áúé ä÷ôä ðôúçéí åàðùéí îçëéí ìàåèåáåñéí äøàùåðéí. diff --git a/corpus/generated/ISO-8859-8/he/train/library.bin b/corpus/generated/ISO-8859-8/he/train/library.bin new file mode 100644 index 0000000..d4abd33 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/library.bin @@ -0,0 +1,3 @@ +äñôøééä äù÷èä + +÷åøàéí îçôùéí ñôøé äéñèåøéä åîãò åäñôøðéú òåæøú ìúìîéãéí ìáçåø î÷åøåú. diff --git a/corpus/generated/ISO-8859-8/he/train/market.bin b/corpus/generated/ISO-8859-8/he/train/market.bin new file mode 100644 index 0000000..d4e1672 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/market.bin @@ -0,0 +1,3 @@ +ùå÷ ùáú + +ç÷ìàéí îåëøéí ìçí èøé, ãáù, ôéøåú åâáéðä áëéëø äòîåñä. diff --git a/corpus/generated/ISO-8859-8/he/train/workshop.bin b/corpus/generated/ISO-8859-8/he/train/workshop.bin new file mode 100644 index 0000000..9120483 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/train/workshop.bin @@ -0,0 +1,3 @@ +ñãðä ãéâéèìéú + +öååú ÷èï îëéï îãøéê ìîåæéàåï, áåã÷ àú ä÷åã åëåúá úéàåøéí áøåøéí. diff --git a/corpus/generated/ISO-8859-8/he/validation/river-trip.bin b/corpus/generated/ISO-8859-8/he/validation/river-trip.bin new file mode 100644 index 0000000..f2a8f51 --- /dev/null +++ b/corpus/generated/ISO-8859-8/he/validation/river-trip.bin @@ -0,0 +1,3 @@ +èéåì àì äðäø + +äçáøéí ì÷çå îôä åúä çí, äìëå áéòø åðçå ìéã äðäø. diff --git a/corpus/generated/ISO-8859-9/tr/test/community-garden.bin b/corpus/generated/ISO-8859-9/tr/test/community-garden.bin new file mode 100644 index 0000000..831e0ea --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/test/community-garden.bin @@ -0,0 +1,3 @@ +Mahalle bahçesi + +Komþular boþ avluyu yeþil bir bahçeye çevirdi, çocuklar renkli tabelalar yaptý ve aileler bitkileri her hafta suluyor. diff --git a/corpus/generated/ISO-8859-9/tr/train/city-morning.bin b/corpus/generated/ISO-8859-9/tr/train/city-morning.bin new file mode 100644 index 0000000..e8458f6 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/city-morning.bin @@ -0,0 +1,3 @@ +Þehirde sabah + +Þehir yavaþça uyanýr, kafeler açýlýr ve insanlar ilk otobüsleri bekler. diff --git a/corpus/generated/ISO-8859-9/tr/train/library.bin b/corpus/generated/ISO-8859-9/tr/train/library.bin new file mode 100644 index 0000000..20c465c --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/library.bin @@ -0,0 +1,3 @@ +Sessiz kütüphane + +Okurlar tarih ve bilim kitaplarý ararken kütüphaneci öðrencilere yardýmcý olur. diff --git a/corpus/generated/ISO-8859-9/tr/train/market.bin b/corpus/generated/ISO-8859-9/tr/train/market.bin new file mode 100644 index 0000000..0856260 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/market.bin @@ -0,0 +1,3 @@ +Cumartesi pazarý + +Çiftçiler kalabalýk meydanda taze ekmek, elma, peynir ve bal satar. diff --git a/corpus/generated/ISO-8859-9/tr/train/workshop.bin b/corpus/generated/ISO-8859-9/tr/train/workshop.bin new file mode 100644 index 0000000..844ca1e --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/train/workshop.bin @@ -0,0 +1,3 @@ +Dijital atölye + +Küçük bir ekip müze rehberi hazýrlar, kodu denetler ve açýk açýklamalar yazar. diff --git a/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin b/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin new file mode 100644 index 0000000..2568000 --- /dev/null +++ b/corpus/generated/ISO-8859-9/tr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Nehir gezisi + +Arkadaþlar harita ve sýcak çay alýp ormanda yürüdü, sonra nehir kýyýsýnda dinlendi. diff --git a/corpus/generated/KOI8-R/ru/test/community-garden.bin b/corpus/generated/KOI8-R/ru/test/community-garden.bin new file mode 100644 index 0000000..17de462 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/test/community-garden.bin @@ -0,0 +1,3 @@ +ïÂÝÉÊ ÓÁÄ + +öÉÔÅÌÉ ×ÍÅÓÔÅ ÐÒÉ×ÅÌÉ × ÐÏÒÑÄÏË ÓÔÁÒÙÊ Ä×ÏÒ É ÐÏÓÁÄÉÌÉ Ã×ÅÔÙ, ÚÅÌÅÎØ É ÍÏÌÏÄÙÅ ÄÅÒÅ×ØÑ. äÅÔÉ ÓÄÅÌÁÌÉ ÑÒËÉÅ ÔÁÂÌÉÞËÉ, Á ÓÏÓÅÄÉ ÄÏÇÏ×ÏÒÉÌÉÓØ ËÁÖÄÕÀ ÎÅÄÅÌÀ ÐÏÌÉ×ÁÔØ ÒÁÓÔÅÎÉÑ. diff --git a/corpus/generated/KOI8-R/ru/train/city-morning.bin b/corpus/generated/KOI8-R/ru/train/city-morning.bin new file mode 100644 index 0000000..2bf62de --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/city-morning.bin @@ -0,0 +1,3 @@ +õÔÒÏ × ÇÏÒÏÄÅ + +çÏÒÏÄ ÐÒÏÓÙÐÁÅÔÓÑ ÐÏÓÔÅÐÅÎÎÏ. îÁ ÏÓÔÁÎÏ×ËÁÈ ÐÏÑ×ÌÑÀÔÓÑ ÐÁÓÓÁÖÉÒÙ, ÂÕÌÏÞÎÁÑ ÏÔËÒÙ×ÁÅÔ Ä×ÅÒÉ, Á ÐÅÒ×ÙÅ Á×ÔÏÂÕÓÙ ×ÙÈÏÄÑÔ ÎÁ ÍÁÒÛÒÕÔ. ìÀÄÉ ÞÉÔÁÀÔ ÓÏÏÂÝÅÎÉÑ É ÏÂÓÕÖÄÁÀÔ ÐÌÁÎÙ ÎÁ ÎÏ×ÙÊ ÄÅÎØ. diff --git a/corpus/generated/KOI8-R/ru/train/library.bin b/corpus/generated/KOI8-R/ru/train/library.bin new file mode 100644 index 0000000..5e2b1e4 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/library.bin @@ -0,0 +1,3 @@ +òÁÊÏÎÎÁÑ ÂÉÂÌÉÏÔÅËÁ + +÷ ÂÉÂÌÉÏÔÅËÅ ÔÉÈÏ ÛÅÌÅÓÔÑÔ ÓÔÒÁÎÉÃÙ. þÉÔÁÔÅÌÉ ×ÙÂÉÒÁÀÔ ËÎÉÇÉ Ï ÉÓÔÏÒÉÉ, ÎÁÕËÅ É ÐÕÔÅÛÅÓÔ×ÉÑÈ. óÏÔÒÕÄÎÉËÉ ÇÏÔÏ×ÑÔ ×ÙÓÔÁ×ËÕ, ÏÔ×ÅÞÁÀÔ ÎÁ ×ÏÐÒÏÓÙ É ÐÏÍÏÇÁÀÔ ÛËÏÌØÎÉËÁÍ ÎÁÊÔÉ ÎÕÖÎÙÅ ÍÁÔÅÒÉÁÌÙ. diff --git a/corpus/generated/KOI8-R/ru/train/market.bin b/corpus/generated/KOI8-R/ru/train/market.bin new file mode 100644 index 0000000..689aa9b --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/market.bin @@ -0,0 +1,3 @@ +ïÓÅÎÎÑÑ ÑÒÍÁÒËÁ + +îÁ ÐÌÏÝÁÄÉ ÐÒÏÈÏÄÉÔ ÏÓÅÎÎÑÑ ÑÒÍÁÒËÁ. æÅÒÍÅÒÙ ÐÒÉ×ÅÚÌÉ ÑÂÌÏËÉ, Í£Ä, ÓÙÒ É Ó×ÅÖÉÊ ÈÌÅÂ. ðÏËÕÐÁÔÅÌÉ ÓÒÁ×ÎÉ×ÁÀÔ ÐÒÏÄÕËÔÙ, ÂÅÓÅÄÕÀÔ Ó ÐÒÏÄÁ×ÃÁÍÉ É ÚÁÐÉÓÙ×ÁÀÔ ÉÎÔÅÒÅÓÎÙÅ ÄÏÍÁÛÎÉÅ ÒÅÃÅÐÔÙ. diff --git a/corpus/generated/KOI8-R/ru/train/workshop.bin b/corpus/generated/KOI8-R/ru/train/workshop.bin new file mode 100644 index 0000000..9133544 --- /dev/null +++ b/corpus/generated/KOI8-R/ru/train/workshop.bin @@ -0,0 +1,3 @@ +òÁÂÏÞÁÑ ÍÁÓÔÅÒÓËÁÑ + +îÅÂÏÌØÛÁÑ ËÏÍÁÎÄÁ ÓÏÚÄÁ£Ô ÜÌÅËÔÒÏÎÎÙÊ ËÁÔÁÌÏÇ ÍÕÚÅÑ. äÉÚÁÊÎÅÒÙ ÏÂÓÕÖÄÁÀÔ ÍÅÎÀ, ÐÒÏÇÒÁÍÍÉÓÔÙ ÐÒÏ×ÅÒÑÀÔ ËÏÄ, Á ÒÅÄÁËÔÏÒÙ ÇÏÔÏ×ÑÔ ÑÓÎÙÅ ÏÐÉÓÁÎÉÑ ÄÌÑ ÐÏÓÅÔÉÔÅÌÅÊ ÒÁÚÎÙÈ ×ÏÚÒÁÓÔÏ×. diff --git a/corpus/generated/KOI8-R/ru/validation/river-trip.bin b/corpus/generated/KOI8-R/ru/validation/river-trip.bin new file mode 100644 index 0000000..cb9b8fa --- /dev/null +++ b/corpus/generated/KOI8-R/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ +ðÏÅÚÄËÁ Ë ÒÅËÅ + +÷ ×ÏÓËÒÅÓÅÎØÅ ÄÒÕÚØÑ ÏÔÐÒÁ×ÉÌÉÓØ Ë ÓÐÏËÏÊÎÏÊ ÒÅËÅ ÚÁ ÇÏÒÏÄÏÍ. ïÎÉ ×ÚÑÌÉ ËÁÒÔÕ, ÇÏÒÑÞÉÊ ÞÁÊ É ÆÏÔÏÁÐÐÁÒÁÔ. ðÏÓÌÅ ÄÏÌÇÏÊ ÐÒÏÇÕÌËÉ ÐÕÔÅÛÅÓÔ×ÅÎÎÉËÉ ÏÔÄÙÈÁÌÉ ÎÁ ÂÅÒÅÇÕ É ÓÌÕÛÁÌÉ ×ÅÞÅÒÎÉÈ ÐÔÉÃ. diff --git a/corpus/generated/index.json b/corpus/generated/index.json index cc7e572..3467f44 100644 --- a/corpus/generated/index.json +++ b/corpus/generated/index.json @@ -576,940 +576,4108 @@ "sha256": "7f39f7fb36cbabea98997b9c390d71e0742ef2fad5c7deca432ee6444e164c49" }, { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "test", - "document": "community-garden", - "path": "ISO-8859-10/is/test/community-garden.bin", - "bytes": 174, - "highBytes": 19, - "uniqueHighBytes": 6, - "sha256": "11bbdafb4917a3935f98940af20e169c4a7eef5642d4a159297b44cd8b565d78" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "city-morning", - "path": "ISO-8859-10/is/train/city-morning.bin", - "bytes": 161, - "highBytes": 11, - "uniqueHighBytes": 8, - "sha256": "3c8a3358a8195de52877343b34b5a71606732b3df110594862cebeb9c76aa3cf" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "library", - "path": "ISO-8859-10/is/train/library.bin", - "bytes": 166, - "highBytes": 18, - "uniqueHighBytes": 7, - "sha256": "a78d39ae9c97ed64273f11792d1e5a353080658d28ab6a85cc248f28b91d18a0" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "market", - "path": "ISO-8859-10/is/train/market.bin", - "bytes": 168, - "highBytes": 10, - "uniqueHighBytes": 5, - "sha256": "c15c77b0cb233941ca1b038ddd269d4d866e4b5b48d61989220169b6cd851c9b" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "train", - "document": "workshop", - "path": "ISO-8859-10/is/train/workshop.bin", - "bytes": 167, - "highBytes": 15, - "uniqueHighBytes": 6, - "sha256": "1bdd4e4aa76ecbd47f82533ec096fe3ee5c637debdefad8bc10251e8f3867af0" - }, - { - "encoding": "ISO-8859-10", - "iconv": "ISO-8859-10", - "language": "is", - "split": "validation", - "document": "river-trip", - "path": "ISO-8859-10/is/validation/river-trip.bin", - "bytes": 168, - "highBytes": 17, - "uniqueHighBytes": 9, - "sha256": "ce28fe8d7fe2ee466f03495c48f00a6da6fa7d3df530c7052873c444e1dd6f59" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "test", "document": "community-garden", - "path": "ISO-8859-13/lt/test/community-garden.bin", - "bytes": 177, - "highBytes": 17, - "uniqueHighBytes": 6, - "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + "path": "ISO-8859-1/da/test/community-garden.bin", + "bytes": 123, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "d742c49869d5f97ff0dc9bed58a7e819cf5a9f3cde419ff785f2201591fa7991" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "city-morning", - "path": "ISO-8859-13/lt/train/city-morning.bin", - "bytes": 156, - "highBytes": 11, - "uniqueHighBytes": 4, - "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + "path": "ISO-8859-1/da/train/city-morning.bin", + "bytes": 89, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "75320357a51c6716244970d7c802f42af40a6a65fa1602e4f38e239e16ccd887" }, { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "library", - "path": "ISO-8859-13/lt/train/library.bin", - "bytes": 163, - "highBytes": 8, - "uniqueHighBytes": 5, - "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", - "split": "train", - "document": "market", - "path": "ISO-8859-13/lt/train/market.bin", - "bytes": 171, - "highBytes": 12, - "uniqueHighBytes": 8, - "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", - "split": "train", - "document": "workshop", - "path": "ISO-8859-13/lt/train/workshop.bin", - "bytes": 170, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" - }, - { - "encoding": "ISO-8859-13", - "iconv": "ISO-8859-13", - "language": "lt", - "split": "validation", - "document": "river-trip", - "path": "ISO-8859-13/lt/validation/river-trip.bin", - "bytes": 165, - "highBytes": 16, - "uniqueHighBytes": 5, - "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "test", - "document": "community-garden", - "path": "ISO-8859-14/cy/test/community-garden.bin", - "bytes": 194, + "path": "ISO-8859-1/da/train/library.bin", + "bytes": 105, "highBytes": 3, - "uniqueHighBytes": 3, - "sha256": "11475514e6d6789bb154d27be8a381507f5c136b1ec88219820a77d3f11a352a" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "city-morning", - "path": "ISO-8859-14/cy/train/city-morning.bin", - "bytes": 195, - "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "5310db3607ef3fcadf53799acdd8836356af51cf840f1f391c4c31acdb3188ce" - }, - { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", - "split": "train", - "document": "library", - "path": "ISO-8859-14/cy/train/library.bin", - "bytes": 194, - "highBytes": 1, - "uniqueHighBytes": 1, - "sha256": "c8445d5b31a86bb8a0f295792afe5d4632e9c30182460cb0bc590c6fa6156afb" + "sha256": "4b0f38943f7eb950af8b04b16ac6ce581c48f70e933f66bd03725842d9e1bd58" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "market", - "path": "ISO-8859-14/cy/train/market.bin", - "bytes": 183, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "c2b02e1a6e22d7f21483955ebf4d00e4b0500c3ee91dcb67e67cb07f015932e3" + "path": "ISO-8859-1/da/train/market.bin", + "bytes": 84, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "61d9285e5b354f3cffb320f3bb62af7d4ce86fe34ba0e5067d5bc26aa9cd010c" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "train", "document": "workshop", - "path": "ISO-8859-14/cy/train/workshop.bin", - "bytes": 173, + "path": "ISO-8859-1/da/train/workshop.bin", + "bytes": 103, "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "2fe7199c4f96f4c5941e8f6b95aa44ae6f8d85863552550993b3b019acdeee6f" + "sha256": "aca6cded29cf164fc9445de75292a45f007c971ea9407d7ef0d794413a7cd975" }, { - "encoding": "ISO-8859-14", - "iconv": "ISO-8859-14", - "language": "cy", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "da", "split": "validation", "document": "river-trip", - "path": "ISO-8859-14/cy/validation/river-trip.bin", - "bytes": 161, + "path": "ISO-8859-1/da/validation/river-trip.bin", + "bytes": 97, "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "318d863b3f7efa4a716e75ba4cd138fe9b9c337fb6f88eafe7ee1c671629e054" + "sha256": "0edb6dd9323283e842dea2112213cf7b86608b257a36cfd318b8c71b3c01c8e0" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/de/test/community-garden.bin", + "path": "ISO-8859-1/de/test/community-garden.bin", "bytes": 220, "highBytes": 12, "uniqueHighBytes": 3, "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/de/train/city-morning.bin", + "path": "ISO-8859-1/de/train/city-morning.bin", "bytes": 208, "highBytes": 6, "uniqueHighBytes": 4, "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "library", - "path": "ISO-8859-15/de/train/library.bin", + "path": "ISO-8859-1/de/train/library.bin", "bytes": 204, "highBytes": 5, "uniqueHighBytes": 2, "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "market", - "path": "ISO-8859-15/de/train/market.bin", + "path": "ISO-8859-1/de/train/market.bin", "bytes": 177, "highBytes": 4, "uniqueHighBytes": 3, "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "train", "document": "workshop", - "path": "ISO-8859-15/de/train/workshop.bin", + "path": "ISO-8859-1/de/train/workshop.bin", "bytes": 188, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "de", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/de/validation/river-trip.bin", + "path": "ISO-8859-1/de/validation/river-trip.bin", "bytes": 192, "highBytes": 3, "uniqueHighBytes": 2, "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-1/en/test/community-garden.bin", + "bytes": 136, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8574d38336d96e1a48fd42b012b3924bc36ee57550bc58a28ed6d5c17d514c1d" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-1/en/train/city-morning.bin", + "bytes": 108, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "ff22d70d940981c1ea141576fb34c74d841dbf2e2140d80ee55f51dd63bd9ef0" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "library", + "path": "ISO-8859-1/en/train/library.bin", + "bytes": 117, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "12ea1dc5452af3ca116753189e184ef0a10645eee17f33211dc5f7d182318f38" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "market", + "path": "ISO-8859-1/en/train/market.bin", + "bytes": 100, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "cbaa356df57d8557b3d2c3ca4d6ec5845077db27583e102aebdaedcacf9f2aee" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "train", + "document": "workshop", + "path": "ISO-8859-1/en/train/workshop.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "157d773566f7cd87f7bafb87780ba98dcc090adbdaaea2c03483e7a3b6a93a70" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "en", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-1/en/validation/river-trip.bin", + "bytes": 106, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c4e405fd6466194b2ddc91f07f8bcb624c904531d6d96f37e35d04847c6abbb9" + }, + { + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/es/test/community-garden.bin", + "path": "ISO-8859-1/es/test/community-garden.bin", "bytes": 203, "highBytes": 6, "uniqueHighBytes": 3, "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/es/train/city-morning.bin", + "path": "ISO-8859-1/es/train/city-morning.bin", "bytes": 202, "highBytes": 3, "uniqueHighBytes": 3, "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "library", - "path": "ISO-8859-15/es/train/library.bin", + "path": "ISO-8859-1/es/train/library.bin", "bytes": 202, "highBytes": 2, "uniqueHighBytes": 2, "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "market", - "path": "ISO-8859-15/es/train/market.bin", + "path": "ISO-8859-1/es/train/market.bin", "bytes": 199, "highBytes": 2, "uniqueHighBytes": 2, "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "train", "document": "workshop", - "path": "ISO-8859-15/es/train/workshop.bin", + "path": "ISO-8859-1/es/train/workshop.bin", "bytes": 194, "highBytes": 6, "uniqueHighBytes": 4, "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "es", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/es/validation/river-trip.bin", + "path": "ISO-8859-1/es/validation/river-trip.bin", "bytes": 184, "highBytes": 6, "uniqueHighBytes": 3, "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "test", "document": "community-garden", - "path": "ISO-8859-15/fr/test/community-garden.bin", + "path": "ISO-8859-1/fr/test/community-garden.bin", "bytes": 231, "highBytes": 9, "uniqueHighBytes": 1, "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "city-morning", - "path": "ISO-8859-15/fr/train/city-morning.bin", + "path": "ISO-8859-1/fr/train/city-morning.bin", "bytes": 212, "highBytes": 5, "uniqueHighBytes": 2, "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "library", - "path": "ISO-8859-15/fr/train/library.bin", + "path": "ISO-8859-1/fr/train/library.bin", "bytes": 226, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "market", - "path": "ISO-8859-15/fr/train/market.bin", + "path": "ISO-8859-1/fr/train/market.bin", "bytes": 223, "highBytes": 3, "uniqueHighBytes": 1, "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "train", "document": "workshop", - "path": "ISO-8859-15/fr/train/workshop.bin", + "path": "ISO-8859-1/fr/train/workshop.bin", "bytes": 215, "highBytes": 8, "uniqueHighBytes": 1, "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" }, { - "encoding": "ISO-8859-15", - "iconv": "ISO-8859-15", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", "language": "fr", "split": "validation", "document": "river-trip", - "path": "ISO-8859-15/fr/validation/river-trip.bin", + "path": "ISO-8859-1/fr/validation/river-trip.bin", "bytes": 228, "highBytes": 6, "uniqueHighBytes": 2, "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "test", "document": "community-garden", - "path": "ISO-8859-16/ro/test/community-garden.bin", - "bytes": 206, - "highBytes": 13, - "uniqueHighBytes": 5, - "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + "path": "ISO-8859-1/it/test/community-garden.bin", + "bytes": 161, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "53bb7b070795cd69fba5bcc79a4958b2ac5bb23677ebdac99c80623f96d501dd" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "city-morning", - "path": "ISO-8859-16/ro/train/city-morning.bin", - "bytes": 175, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + "path": "ISO-8859-1/it/train/city-morning.bin", + "bytes": 101, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "7c59ad9c3c9ac0b84c02c4829bffabdc83817989f08836d9ee44afd8a392e43d" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "library", - "path": "ISO-8859-16/ro/train/library.bin", - "bytes": 187, - "highBytes": 17, - "uniqueHighBytes": 5, - "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + "path": "ISO-8859-1/it/train/library.bin", + "bytes": 141, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "bb28d91b3f10f0208886078bf0fafbc85c7e03148069ddf5a9762809f28f8981" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "market", - "path": "ISO-8859-16/ro/train/market.bin", - "bytes": 171, - "highBytes": 17, - "uniqueHighBytes": 4, - "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + "path": "ISO-8859-1/it/train/market.bin", + "bytes": 114, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "40ce81cfc5ec4c5b655c2fb9ba519022a77da2c319950c868bc16c7f69278b11" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "train", "document": "workshop", - "path": "ISO-8859-16/ro/train/workshop.bin", - "bytes": 170, - "highBytes": 6, + "path": "ISO-8859-1/it/train/workshop.bin", + "bytes": 152, + "highBytes": 2, "uniqueHighBytes": 2, - "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + "sha256": "5b5704c6191b4179ee6966a2302452eaaf99c348b34e26bc210879c77def38bc" }, { - "encoding": "ISO-8859-16", - "iconv": "ISO-8859-16", - "language": "ro", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "it", "split": "validation", "document": "river-trip", - "path": "ISO-8859-16/ro/validation/river-trip.bin", - "bytes": 170, - "highBytes": 10, - "uniqueHighBytes": 3, - "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + "path": "ISO-8859-1/it/validation/river-trip.bin", + "bytes": 98, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8fdde80f1933a9dfb82cc0181c6770e9947ccf5bfd745b8251ee02f54e6a19a7" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "test", "document": "community-garden", - "path": "ISO-8859-3/mt/test/community-garden.bin", - "bytes": 186, - "highBytes": 13, - "uniqueHighBytes": 4, - "sha256": "d6874f4f630507a619f841cad55a2046be0ee930fb9d450e20bcf0e9525759d5" + "path": "ISO-8859-1/nl/test/community-garden.bin", + "bytes": 153, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "26b1ccb49c443abe7580037a8a836bf99e9d5a5135b1751b77a73e3f61c1a9bf" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "city-morning", - "path": "ISO-8859-3/mt/train/city-morning.bin", - "bytes": 171, - "highBytes": 7, - "uniqueHighBytes": 3, - "sha256": "e40f845da038bcae4f7c6edb026ca823788410f8895cd7ae2765ca4afde49bfb" + "path": "ISO-8859-1/nl/train/city-morning.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "201fbe0db619c5d0d4ccecd723c27c68e0807437531553e8c17ba84cfc4335b2" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "library", - "path": "ISO-8859-3/mt/train/library.bin", - "bytes": 181, - "highBytes": 6, + "path": "ISO-8859-1/nl/train/library.bin", + "bytes": 150, + "highBytes": 3, "uniqueHighBytes": 2, - "sha256": "b16045c76dede00ff236e27cd647ea45e25a98d4cf04530c9baf947dbb1a9cb2" + "sha256": "5d9cdae67a5a7cf6cb319750006ea3cf6afaf379b11c35aa241ef63b203cb17b" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "market", - "path": "ISO-8859-3/mt/train/market.bin", - "bytes": 172, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "be0fbd5745a025d769bb128a55d8c80bb1397590edde4d79d28db1310589af8e" + "path": "ISO-8859-1/nl/train/market.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "15466fb33051e25a441363879415fe1c6ec949d753105ad851a75b2054d8c0da" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "train", "document": "workshop", - "path": "ISO-8859-3/mt/train/workshop.bin", - "bytes": 164, - "highBytes": 15, - "uniqueHighBytes": 5, - "sha256": "433e386a7ccaeaeef6e9fbd1daf1f586cc61788cd13973523c2d679026964420" + "path": "ISO-8859-1/nl/train/workshop.bin", + "bytes": 132, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "4fa51f6d8f9543c9df0acef5a2711a5d805fd18286c1164d786bd8ac715c183b" }, { - "encoding": "ISO-8859-3", - "iconv": "ISO-8859-3", - "language": "mt", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "nl", "split": "validation", "document": "river-trip", - "path": "ISO-8859-3/mt/validation/river-trip.bin", - "bytes": 176, - "highBytes": 9, - "uniqueHighBytes": 2, - "sha256": "9c73d965fbad70cf830b5a3db04e12c4f0b3bd50ead049f7d6703b5c19cf4eb1" + "path": "ISO-8859-1/nl/validation/river-trip.bin", + "bytes": 113, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "d9448db92c3fe1270814d7798759eca44290f94fd76c0ff9012dc2401b426253" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "test", "document": "community-garden", - "path": "ISO-8859-4/lv/test/community-garden.bin", - "bytes": 174, - "highBytes": 17, - "uniqueHighBytes": 8, - "sha256": "c0382f3b40081b8a5d30ca7d26699a2f9e9eb2e453edac9beb97b4a4a9c3051b" + "path": "ISO-8859-1/no/test/community-garden.bin", + "bytes": 116, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b86618ac6687d3769cca67ec9cb49a0d4d1dea45476592a316ae6eef40d576db" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "city-morning", - "path": "ISO-8859-4/lv/train/city-morning.bin", - "bytes": 163, - "highBytes": 15, - "uniqueHighBytes": 7, - "sha256": "d16cbb8016fdba887337226f08a30e96f2752b657dacbabec3a00b36d5538f60" + "path": "ISO-8859-1/no/train/city-morning.bin", + "bytes": 116, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "445f262b0f2ea6f724271b8486b0cfc861cf4e081e4617db895bc36a33dea2ee" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "library", - "path": "ISO-8859-4/lv/train/library.bin", - "bytes": 161, - "highBytes": 15, - "uniqueHighBytes": 5, - "sha256": "c6757b519e310d300ab70b574b32056a85d7bdf140afd265d5e44101cca573e0" + "path": "ISO-8859-1/no/train/library.bin", + "bytes": 126, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "96c146e3d2b86817371ba6ca5d07361d31519cb0d16fbbecfaf7bba36f0ff2eb" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "market", - "path": "ISO-8859-4/lv/train/market.bin", - "bytes": 170, - "highBytes": 12, - "uniqueHighBytes": 5, - "sha256": "33999e249def18a8469c731d110dbdb9614103b3dce69d8005bc6993b878fc88" + "path": "ISO-8859-1/no/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "a0279dbf28e14a6d9841789f50751ec9cdb09c2743f4ac1c61e72f310e916732" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "train", "document": "workshop", - "path": "ISO-8859-4/lv/train/workshop.bin", - "bytes": 163, - "highBytes": 10, - "uniqueHighBytes": 4, - "sha256": "1429abf14fb2e4144c64dd59657aec2615da7795c92f4e6f5e5f5534ee4be9a0" + "path": "ISO-8859-1/no/train/workshop.bin", + "bytes": 100, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "9c70c92214b0f0ac9000a131773f1ed1756d630037293a759982edcb621e056a" }, { - "encoding": "ISO-8859-4", - "iconv": "ISO-8859-4", - "language": "lv", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "no", "split": "validation", "document": "river-trip", - "path": "ISO-8859-4/lv/validation/river-trip.bin", - "bytes": 156, - "highBytes": 14, - "uniqueHighBytes": 4, - "sha256": "282b08484ad284a0a069c2d8113e09bc80de8163fee60568042c538b0541fc3f" + "path": "ISO-8859-1/no/validation/river-trip.bin", + "bytes": 85, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "931a303e54cbfe3ccddf45aa0dd3956be07e4cf16bf1fe5ee0874bd4ad2bab9a" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "test", "document": "community-garden", - "path": "KOI8-U/uk/test/community-garden.bin", - "bytes": 181, - "highBytes": 152, - "uniqueHighBytes": 29, - "sha256": "25d71bdf3678270bf3a539771f8e429faaef3157c5649c0f2edc4ec06e0c80fa" + "path": "ISO-8859-1/pt/test/community-garden.bin", + "bytes": 145, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "a8e56da45e10d94745a21dd36a2721410e1b0d04b5bfee15b2c451a27c8ba62d" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "city-morning", - "path": "KOI8-U/uk/train/city-morning.bin", - "bytes": 205, - "highBytes": 168, - "uniqueHighBytes": 31, - "sha256": "853b422f64ef06d7a64528db49d4eca0f11240852392e51cb2700b1b8b9fd122" + "path": "ISO-8859-1/pt/train/city-morning.bin", + "bytes": 99, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "c9a43be7ac9a1af0be31146446c7cea5142ba97545c304e9fdfef166c0dfabe1" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "library", - "path": "KOI8-U/uk/train/library.bin", - "bytes": 182, - "highBytes": 151, - "uniqueHighBytes": 32, - "sha256": "42455a09e6cafcd694b38198b1f172e44e63774cde6c9e760181334974536fe5" + "path": "ISO-8859-1/pt/train/library.bin", + "bytes": 116, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "634c4f0d24e7f1a8bbf9b1fd68d0fe414376d2efeff0fbfc7637d921709f2a62" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "market", - "path": "KOI8-U/uk/train/market.bin", - "bytes": 199, - "highBytes": 165, - "uniqueHighBytes": 33, - "sha256": "0d8b40da8ad30cc4d6d849f2439f472db8092d622e428747aa55e1fa1797800b" + "path": "ISO-8859-1/pt/train/market.bin", + "bytes": 96, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "79f397f7b2ac2cca51091f006462be7777befb52d36c54237a8cfb0cb20cd924" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "train", "document": "workshop", - "path": "KOI8-U/uk/train/workshop.bin", - "bytes": 189, - "highBytes": 161, - "uniqueHighBytes": 30, - "sha256": "b4b92fa64c4b9252511ed0240a7e1cb565a0b1158f07b50fefb93fab1bd5f0f7" + "path": "ISO-8859-1/pt/train/workshop.bin", + "bytes": 108, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "fbb52707258ca57000887397d2bd8220907fc178ed36b929c62e72bf211848e1" }, { - "encoding": "KOI8-U", - "iconv": "KOI8-U", - "language": "uk", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "pt", "split": "validation", "document": "river-trip", - "path": "KOI8-U/uk/validation/river-trip.bin", - "bytes": 195, - "highBytes": 160, - "uniqueHighBytes": 31, - "sha256": "1b015aa68443a92670b3e73e37fa96d70d1e2dea1a9ed736bebfcf077bde9dfe" + "path": "ISO-8859-1/pt/validation/river-trip.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "db583dead964fc7fe8a257ebd69f9dd7388c89384bac132955a76b02150c753f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "test", "document": "community-garden", - "path": "macintosh/de/test/community-garden.bin", - "bytes": 220, - "highBytes": 12, + "path": "ISO-8859-1/sv/test/community-garden.bin", + "bytes": 138, + "highBytes": 8, "uniqueHighBytes": 3, - "sha256": "9a35ac96816b79920d3076818a106719d95169d017b89832ff735652fdbc45d5" + "sha256": "7229bf29e0adc2b37cdb6663bb696e500e13000633665f0f226915f5f74c4853" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "city-morning", - "path": "macintosh/de/train/city-morning.bin", - "bytes": 208, - "highBytes": 6, + "path": "ISO-8859-1/sv/train/city-morning.bin", + "bytes": 101, + "highBytes": 7, "uniqueHighBytes": 4, - "sha256": "d826a59c9fe0ea77efe5554f688a96a25ba31edd3b502177f88b229237bfafff" + "sha256": "5ac24bf0fe3bfb1d64aa066ff78e3af67bcf0b4a6f8d142fefeff0b8ad4ae72c" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "library", - "path": "macintosh/de/train/library.bin", - "bytes": 204, - "highBytes": 5, + "path": "ISO-8859-1/sv/train/library.bin", + "bytes": 108, + "highBytes": 4, "uniqueHighBytes": 2, - "sha256": "29829a1ac13b7f7ad2ecd8abd7d0f2454b3beb4ede31af4ab1a8901cb73103b9" + "sha256": "fc05c9ad957334217feaa333f6cfe7cd206b250fbae9ea51a19d9c69f72158e7" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "market", - "path": "macintosh/de/train/market.bin", - "bytes": 177, - "highBytes": 4, + "path": "ISO-8859-1/sv/train/market.bin", + "bytes": 89, + "highBytes": 7, "uniqueHighBytes": 3, - "sha256": "f135969c32b723540e69570512179a361f12aca33782130567cafc8b8cb40974" + "sha256": "9f4f10ac20699e493cdd99b8c1ee6caa29d4a9359962aab15ac23bb773f1d26f" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "train", "document": "workshop", - "path": "macintosh/de/train/workshop.bin", - "bytes": 188, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "a77aa45de75ac7b572174e1c99aa8aab3d7a3cb0278b2f68932d1c7fe4598218" + "path": "ISO-8859-1/sv/train/workshop.bin", + "bytes": 110, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "989f41c9b83ba4e3475a6902c33340fa7b07bc965776a09e9db165321bdcb3cb" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "de", + "encoding": "ISO-8859-1", + "iconv": "ISO-8859-1", + "language": "sv", "split": "validation", "document": "river-trip", - "path": "macintosh/de/validation/river-trip.bin", - "bytes": 192, - "highBytes": 3, - "uniqueHighBytes": 2, - "sha256": "b2edf27b09f5aeaf742baed4ebc17e0a890da6a9f0aa52a40adbcb16104e5538" + "path": "ISO-8859-1/sv/validation/river-trip.bin", + "bytes": 86, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "84cc4d3ab5abaa2d2575d868f0056375043c7e17d6c34ecd4112fca4632bc321" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "test", "document": "community-garden", - "path": "macintosh/es/test/community-garden.bin", - "bytes": 203, - "highBytes": 6, - "uniqueHighBytes": 3, - "sha256": "b675b0218b3db70d7c6f6f43cd7eb00fe7b45713f6e23e3fbac03a176e95935a" + "path": "ISO-8859-10/is/test/community-garden.bin", + "bytes": 174, + "highBytes": 19, + "uniqueHighBytes": 6, + "sha256": "11bbdafb4917a3935f98940af20e169c4a7eef5642d4a159297b44cd8b565d78" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "city-morning", - "path": "macintosh/es/train/city-morning.bin", - "bytes": 202, - "highBytes": 3, - "uniqueHighBytes": 3, - "sha256": "15f13077700d48818a5a5050c47683e1c14260cc22ce956abd29bd07eddf10eb" + "path": "ISO-8859-10/is/train/city-morning.bin", + "bytes": 161, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "3c8a3358a8195de52877343b34b5a71606732b3df110594862cebeb9c76aa3cf" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "library", - "path": "macintosh/es/train/library.bin", - "bytes": 202, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "a489cbb2f2f276564faf85fd6f86795a1ec5a1e5b1cfeda35874e2302e9d3d1b" + "path": "ISO-8859-10/is/train/library.bin", + "bytes": 166, + "highBytes": 18, + "uniqueHighBytes": 7, + "sha256": "a78d39ae9c97ed64273f11792d1e5a353080658d28ab6a85cc248f28b91d18a0" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "market", - "path": "macintosh/es/train/market.bin", - "bytes": 199, - "highBytes": 2, - "uniqueHighBytes": 2, - "sha256": "80770be98c0f1e40ee59e9de08615620aa8a0af1ff73ef8b6a3cc483b14a3a4e" + "path": "ISO-8859-10/is/train/market.bin", + "bytes": 168, + "highBytes": 10, + "uniqueHighBytes": 5, + "sha256": "c15c77b0cb233941ca1b038ddd269d4d866e4b5b48d61989220169b6cd851c9b" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "train", "document": "workshop", - "path": "macintosh/es/train/workshop.bin", - "bytes": 194, - "highBytes": 6, - "uniqueHighBytes": 4, - "sha256": "5b25ecfaa0df015d5c4c1fb52fce66af137588ac8677acd50c468b65b5112018" + "path": "ISO-8859-10/is/train/workshop.bin", + "bytes": 167, + "highBytes": 15, + "uniqueHighBytes": 6, + "sha256": "1bdd4e4aa76ecbd47f82533ec096fe3ee5c637debdefad8bc10251e8f3867af0" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "es", + "encoding": "ISO-8859-10", + "iconv": "ISO-8859-10", + "language": "is", "split": "validation", "document": "river-trip", - "path": "macintosh/es/validation/river-trip.bin", - "bytes": 184, - "highBytes": 6, - "uniqueHighBytes": 3, - "sha256": "d68dda4671b8158cbff67ab7b476d2b819ac45ad00719d4dbe48f22ba1862baa" + "path": "ISO-8859-10/is/validation/river-trip.bin", + "bytes": 168, + "highBytes": 17, + "uniqueHighBytes": 9, + "sha256": "ce28fe8d7fe2ee466f03495c48f00a6da6fa7d3df530c7052873c444e1dd6f59" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "test", "document": "community-garden", - "path": "macintosh/fr/test/community-garden.bin", - "bytes": 231, - "highBytes": 9, - "uniqueHighBytes": 1, - "sha256": "50363a67b8df69b6fbdb3b7a5a25c22c40cdac6ccca71d3bfafeff490e122c58" + "path": "ISO-8859-13/lt/test/community-garden.bin", + "bytes": 177, + "highBytes": 17, + "uniqueHighBytes": 6, + "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "city-morning", - "path": "macintosh/fr/train/city-morning.bin", - "bytes": 212, - "highBytes": 5, - "uniqueHighBytes": 2, - "sha256": "5f8dbd5bcfe437962ef5943c707e1695edce736abace97d5b29ef68945c42415" + "path": "ISO-8859-13/lt/train/city-morning.bin", + "bytes": 156, + "highBytes": 11, + "uniqueHighBytes": 4, + "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "library", - "path": "macintosh/fr/train/library.bin", - "bytes": 226, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "66ecc14482e9af6c7d2c813d11462760e1a70b98a6d6c0c966821cd03f07eee3" + "path": "ISO-8859-13/lt/train/library.bin", + "bytes": 163, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "market", - "path": "macintosh/fr/train/market.bin", - "bytes": 223, - "highBytes": 3, - "uniqueHighBytes": 1, - "sha256": "5e81aa00b93ef4299cde155e0d215906411a1b0566a676ea307403ef658274d5" + "path": "ISO-8859-13/lt/train/market.bin", + "bytes": 171, + "highBytes": 12, + "uniqueHighBytes": 8, + "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "train", "document": "workshop", - "path": "macintosh/fr/train/workshop.bin", - "bytes": 215, - "highBytes": 8, - "uniqueHighBytes": 1, - "sha256": "ee23ebed42809949b51365120200f135cb051471430331b7c6ccf8282841075a" + "path": "ISO-8859-13/lt/train/workshop.bin", + "bytes": 170, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" }, { - "encoding": "macintosh", - "iconv": "MACINTOSH", - "language": "fr", + "encoding": "ISO-8859-13", + "iconv": "ISO-8859-13", + "language": "lt", "split": "validation", "document": "river-trip", - "path": "macintosh/fr/validation/river-trip.bin", - "bytes": 228, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" + "path": "ISO-8859-13/lt/validation/river-trip.bin", + "bytes": 165, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-14/cy/test/community-garden.bin", + "bytes": 194, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "11475514e6d6789bb154d27be8a381507f5c136b1ec88219820a77d3f11a352a" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-14/cy/train/city-morning.bin", + "bytes": 195, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "5310db3607ef3fcadf53799acdd8836356af51cf840f1f391c4c31acdb3188ce" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "library", + "path": "ISO-8859-14/cy/train/library.bin", + "bytes": 194, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c8445d5b31a86bb8a0f295792afe5d4632e9c30182460cb0bc590c6fa6156afb" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "market", + "path": "ISO-8859-14/cy/train/market.bin", + "bytes": 183, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "c2b02e1a6e22d7f21483955ebf4d00e4b0500c3ee91dcb67e67cb07f015932e3" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "train", + "document": "workshop", + "path": "ISO-8859-14/cy/train/workshop.bin", + "bytes": 173, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "2fe7199c4f96f4c5941e8f6b95aa44ae6f8d85863552550993b3b019acdeee6f" + }, + { + "encoding": "ISO-8859-14", + "iconv": "ISO-8859-14", + "language": "cy", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-14/cy/validation/river-trip.bin", + "bytes": 161, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "318d863b3f7efa4a716e75ba4cd138fe9b9c337fb6f88eafe7ee1c671629e054" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "library", + "path": "ISO-8859-15/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "market", + "path": "ISO-8859-15/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "library", + "path": "ISO-8859-15/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "market", + "path": "ISO-8859-15/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-15/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-15/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "library", + "path": "ISO-8859-15/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "market", + "path": "ISO-8859-15/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "ISO-8859-15/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" + }, + { + "encoding": "ISO-8859-15", + "iconv": "ISO-8859-15", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-15/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-16/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-16/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "library", + "path": "ISO-8859-16/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "market", + "path": "ISO-8859-16/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "ISO-8859-16/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "ISO-8859-16", + "iconv": "ISO-8859-16", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-16/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/cs/test/community-garden.bin", + "bytes": 128, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "1eba2d29ae0f73a2f12f43469971a65f81c58779530b1b753285cb2d8c29200d" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/cs/train/city-morning.bin", + "bytes": 98, + "highBytes": 13, + "uniqueHighBytes": 6, + "sha256": "a0f0dfa252459d39f725fee0b0a13818681efd1c28e854e4eaf32509e03f5a55" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "library", + "path": "ISO-8859-2/cs/train/library.bin", + "bytes": 88, + "highBytes": 12, + "uniqueHighBytes": 7, + "sha256": "cd1af77dd042d83bb4106f82c6a213eb32babb8c8be59d4b3b07ece358a651a4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "market", + "path": "ISO-8859-2/cs/train/market.bin", + "bytes": 74, + "highBytes": 10, + "uniqueHighBytes": 7, + "sha256": "083f02c5caabf56bf78ab98fc83cabb38dfc79337e33d3a0f694de80c2403e0e" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/cs/train/workshop.bin", + "bytes": 90, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "05d2ab8033b5d71b2c863a14f4c54ca237af31f71158a2370622417e783788cf" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "cs", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/cs/validation/river-trip.bin", + "bytes": 80, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "3eba80db3fa617e843e8165ca7dd7ee1332d362e9024b85ffa3f9dcedf4f4651" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/hu/test/community-garden.bin", + "bytes": 139, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "d55e643af71274694d74cff7dbf12589bfd13a2b86f2ddcfb53476578a3132b8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/hu/train/city-morning.bin", + "bytes": 94, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "97945a679eae0390b0b3c65608b9cffe4cd8ba6196b8af4be0cec9272bf25ae4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "library", + "path": "ISO-8859-2/hu/train/library.bin", + "bytes": 108, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "bb34de899f6968d695ac39ed18a9f72077d309da97d9721d4e6551562d644427" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "market", + "path": "ISO-8859-2/hu/train/market.bin", + "bytes": 80, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a6a508b4927acd6751a091826a4cc4945330021699819d1c956c8ea6b440a56b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/hu/train/workshop.bin", + "bytes": 101, + "highBytes": 14, + "uniqueHighBytes": 7, + "sha256": "2a294d4fef2aeeee691b74de2293bdf7a7bae7854b3021607fc7df7d17a1a3a4" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "hu", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/hu/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "ce35a0b3ccaa4b07727c0b9e925c958ad050a5d774d91c3f25962c3b7291ffab" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/pl/test/community-garden.bin", + "bytes": 188, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "f9e88a60cbcc42a798c28f8e9b56ee86c8f657f6c8bf8487b8ce2c61c1f02a8b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/pl/train/city-morning.bin", + "bytes": 174, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "7aa1f6ba90080099f3a458933c75a4d32aa5d1f7687cbbc2900c9da738361402" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "library", + "path": "ISO-8859-2/pl/train/library.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 6, + "sha256": "4533cb3ea7bd91bf69478c029c0d4e71755df177c96a18d5363289a80aa08df8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "market", + "path": "ISO-8859-2/pl/train/market.bin", + "bytes": 166, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "14e939af68d4b46232b52c9527d0ec87ab4a5bfb2d178fb4f7a58f8fdcd45f6a" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/pl/train/workshop.bin", + "bytes": 165, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "f749532f6c64c2ff857f54f2e4ef7a0b42ac1dfb6a60bfeeb518fdc75d966c6a" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "pl", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/pl/validation/river-trip.bin", + "bytes": 161, + "highBytes": 9, + "uniqueHighBytes": 3, + "sha256": "9b6685ebb982c76027a1f70a1fb36ad3913d4400c76dae5f54b460f61704e2d7" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-2/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-2/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "library", + "path": "ISO-8859-2/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "market", + "path": "ISO-8859-2/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "ISO-8859-2/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "ISO-8859-2", + "iconv": "ISO-8859-2", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-2/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-3/mt/test/community-garden.bin", + "bytes": 186, + "highBytes": 13, + "uniqueHighBytes": 4, + "sha256": "d6874f4f630507a619f841cad55a2046be0ee930fb9d450e20bcf0e9525759d5" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-3/mt/train/city-morning.bin", + "bytes": 171, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "e40f845da038bcae4f7c6edb026ca823788410f8895cd7ae2765ca4afde49bfb" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "library", + "path": "ISO-8859-3/mt/train/library.bin", + "bytes": 181, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "b16045c76dede00ff236e27cd647ea45e25a98d4cf04530c9baf947dbb1a9cb2" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "market", + "path": "ISO-8859-3/mt/train/market.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "be0fbd5745a025d769bb128a55d8c80bb1397590edde4d79d28db1310589af8e" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "train", + "document": "workshop", + "path": "ISO-8859-3/mt/train/workshop.bin", + "bytes": 164, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "433e386a7ccaeaeef6e9fbd1daf1f586cc61788cd13973523c2d679026964420" + }, + { + "encoding": "ISO-8859-3", + "iconv": "ISO-8859-3", + "language": "mt", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-3/mt/validation/river-trip.bin", + "bytes": 176, + "highBytes": 9, + "uniqueHighBytes": 2, + "sha256": "9c73d965fbad70cf830b5a3db04e12c4f0b3bd50ead049f7d6703b5c19cf4eb1" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-4/lv/test/community-garden.bin", + "bytes": 174, + "highBytes": 17, + "uniqueHighBytes": 8, + "sha256": "c0382f3b40081b8a5d30ca7d26699a2f9e9eb2e453edac9beb97b4a4a9c3051b" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-4/lv/train/city-morning.bin", + "bytes": 163, + "highBytes": 15, + "uniqueHighBytes": 7, + "sha256": "d16cbb8016fdba887337226f08a30e96f2752b657dacbabec3a00b36d5538f60" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "library", + "path": "ISO-8859-4/lv/train/library.bin", + "bytes": 161, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "c6757b519e310d300ab70b574b32056a85d7bdf140afd265d5e44101cca573e0" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "market", + "path": "ISO-8859-4/lv/train/market.bin", + "bytes": 170, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "33999e249def18a8469c731d110dbdb9614103b3dce69d8005bc6993b878fc88" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "train", + "document": "workshop", + "path": "ISO-8859-4/lv/train/workshop.bin", + "bytes": 163, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "1429abf14fb2e4144c64dd59657aec2615da7795c92f4e6f5e5f5534ee4be9a0" + }, + { + "encoding": "ISO-8859-4", + "iconv": "ISO-8859-4", + "language": "lv", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-4/lv/validation/river-trip.bin", + "bytes": 156, + "highBytes": 14, + "uniqueHighBytes": 4, + "sha256": "282b08484ad284a0a069c2d8113e09bc80de8163fee60568042c538b0541fc3f" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-5/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "1bb1e574a38f791fa40a6e3742960a95e6576dc9016efa256a7a9d7e869d3fb4" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-5/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "aeb268c03cb92a112177352ea91fa6f13691d0bbd28729397e25ca9e36a63b30" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "library", + "path": "ISO-8859-5/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "61e10db337d7b99d3290cc4d0e83412621abf39f97bfa5f28a9e1249117d74f0" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "market", + "path": "ISO-8859-5/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "8d252284135141f156b8483f0eaea527549221996fdd2034b53337d8fe37782a" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "ISO-8859-5/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "7f22de320a26b19ac6054b59dbe79cc7e8e82857d0529c227c948ca33ef309b4" + }, + { + "encoding": "ISO-8859-5", + "iconv": "ISO-8859-5", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-5/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "9bbfd60e8c09ca445265bae5c320ca4565c3800acb17f3fe18af74f0d4b53f66" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-6/ar/test/community-garden.bin", + "bytes": 112, + "highBytes": 92, + "uniqueHighBytes": 25, + "sha256": "e49c5c7a78e9e802de85f4e3b075486c68fa32625c0682b0bc19267616009b42" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-6/ar/train/city-morning.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 22, + "sha256": "e5a89c48b2108cf6a4e864621eb2a7894e83566bceeeac7ea4458904751cb4d4" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "library", + "path": "ISO-8859-6/ar/train/library.bin", + "bytes": 107, + "highBytes": 89, + "uniqueHighBytes": 25, + "sha256": "f91cf3a719aa42bb5d2c16831a0e01a37fbf699e3d394b0b417b43a03494c26f" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "market", + "path": "ISO-8859-6/ar/train/market.bin", + "bytes": 89, + "highBytes": 75, + "uniqueHighBytes": 24, + "sha256": "08be46278b902f4cf10072e2506c7d2d29cb075c4bc087fb632cdd29553233bc" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "train", + "document": "workshop", + "path": "ISO-8859-6/ar/train/workshop.bin", + "bytes": 91, + "highBytes": 75, + "uniqueHighBytes": 21, + "sha256": "dccb5be7cce959474487e4c395c238144116f43076dc650e4e3be5a56a38dc02" + }, + { + "encoding": "ISO-8859-6", + "iconv": "ISO-8859-6", + "language": "ar", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-6/ar/validation/river-trip.bin", + "bytes": 91, + "highBytes": 73, + "uniqueHighBytes": 24, + "sha256": "b9afd70647f0be078fd19fa1040fddfe01cd90170a944ffeaf4e529112b45684" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-7/el/test/community-garden.bin", + "bytes": 136, + "highBytes": 112, + "uniqueHighBytes": 25, + "sha256": "8f7531c39aa8efa5f4e38e8eb948f713d69cbe72badea15391e5c62870179370" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-7/el/train/city-morning.bin", + "bytes": 93, + "highBytes": 74, + "uniqueHighBytes": 25, + "sha256": "80e97194d91d40aeae9020cd96d22ac6ad01e3c39020377e32bdf63c2b9622e0" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "library", + "path": "ISO-8859-7/el/train/library.bin", + "bytes": 111, + "highBytes": 94, + "uniqueHighBytes": 27, + "sha256": "eb0448b06be644629054c0f94e0c6dccb7458d80a6c477eda1ab321f378c51bb" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "market", + "path": "ISO-8859-7/el/train/market.bin", + "bytes": 85, + "highBytes": 69, + "uniqueHighBytes": 28, + "sha256": "ee94d165652f333d57c5be996825a8723cfa76aba286f1f22a0b0d40312eece9" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "train", + "document": "workshop", + "path": "ISO-8859-7/el/train/workshop.bin", + "bytes": 108, + "highBytes": 90, + "uniqueHighBytes": 27, + "sha256": "2466d051c231cbb22d9fa5a962bb180a88dbe805ad1b609467bd06afe5de3b02" + }, + { + "encoding": "ISO-8859-7", + "iconv": "ISO-8859-7", + "language": "el", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-7/el/validation/river-trip.bin", + "bytes": 92, + "highBytes": 75, + "uniqueHighBytes": 25, + "sha256": "4dc6ccfd555071ba0df444d4f2fd5b0eb96cf44e389dc872279e8ac1096b359e" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-8/he/test/community-garden.bin", + "bytes": 104, + "highBytes": 84, + "uniqueHighBytes": 21, + "sha256": "6044aa4ee9d7f7517ca965d0a83029c5569a83e8efad7b9f39974fa95fd61298" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-8/he/train/city-morning.bin", + "bytes": 79, + "highBytes": 64, + "uniqueHighBytes": 19, + "sha256": "5c0eddaa74015d9d135cd9c9d80b117cd82a3aa3effe6298deaca2c06ac1305b" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "library", + "path": "ISO-8859-8/he/train/library.bin", + "bytes": 86, + "highBytes": 72, + "uniqueHighBytes": 20, + "sha256": "b9b2605035a9017f18f9de847e03e623581ffed30ea1623fe332efcac420637b" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "market", + "path": "ISO-8859-8/he/train/market.bin", + "bytes": 64, + "highBytes": 49, + "uniqueHighBytes": 21, + "sha256": "ed2975da284ad08b0261f536f69344981747bdd3b37d0ae6a4ec0a36ed1d57ca" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "train", + "document": "workshop", + "path": "ISO-8859-8/he/train/workshop.bin", + "bytes": 80, + "highBytes": 64, + "uniqueHighBytes": 21, + "sha256": "763e4f5501fa3308c2862f48f5173d29d5fd1ccf730664a9bc4c40a714dfcbc6" + }, + { + "encoding": "ISO-8859-8", + "iconv": "ISO-8859-8", + "language": "he", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-8/he/validation/river-trip.bin", + "bytes": 63, + "highBytes": 47, + "uniqueHighBytes": 18, + "sha256": "5319026034eb165464faad8f2de9c2e6c97f01c92e71d2fc3441310417d70b28" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "test", + "document": "community-garden", + "path": "ISO-8859-9/tr/test/community-garden.bin", + "bytes": 136, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "a2224b89fee790662cf9c3cac7200da0f8c5842b4ce5d26512d76e522c6db437" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "city-morning", + "path": "ISO-8859-9/tr/train/city-morning.bin", + "bytes": 87, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "a1231eabb26c3d7d770c885972062ee78365f871d1f62be5977332949e8b1a80" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "library", + "path": "ISO-8859-9/tr/train/library.bin", + "bytes": 98, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "432b50510bf2a5ab8b7b51aaf608341bea95afbf43edff4fa4d404f4a2d87a70" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "market", + "path": "ISO-8859-9/tr/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "25354256b7d0e3f936eb224f6343e5d3d73f78a504d5b2d3c260a2015a62e356" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "train", + "document": "workshop", + "path": "ISO-8859-9/tr/train/workshop.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "0bc87de2045407c5c4db2589ab89cb9faf40cf5556d17caf0bfa916a1b4a0b03" + }, + { + "encoding": "ISO-8859-9", + "iconv": "ISO-8859-9", + "language": "tr", + "split": "validation", + "document": "river-trip", + "path": "ISO-8859-9/tr/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "b94d965d212deb815bd73eabf712bb673f632c88c0a0c9f772202109e4d85e8d" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "KOI8-R/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "7be1a487f774fe73d3dd7a96f9c443d194edf77b1316e89483096c6e0c815e6c" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "KOI8-R/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "42e7be1a39fd943782012cc3c291fa4cf0d922c51b300ec7b5a3d7f3cfc5cc51" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "library", + "path": "KOI8-R/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "9ec8dd417b12bf60f2141ea177caf67a444bbd9b53cb031596df8fe60ea398c6" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "market", + "path": "KOI8-R/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "6096b9d90fa7ab1e9ec13727c3e8d92d5c57841066b4c795b1609cee03dc3110" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "KOI8-R/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "23a3947cc61c1418df7b89c74fe7eb940ef773ab0d04723cdd3c6857cc09a448" + }, + { + "encoding": "KOI8-R", + "iconv": "KOI8-R", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "KOI8-R/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "893c13722e2ba6f81610cbebf9abb72940b3ef07a639613b2ca87cd993c925d8" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "test", + "document": "community-garden", + "path": "KOI8-U/uk/test/community-garden.bin", + "bytes": 181, + "highBytes": 152, + "uniqueHighBytes": 29, + "sha256": "25d71bdf3678270bf3a539771f8e429faaef3157c5649c0f2edc4ec06e0c80fa" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "city-morning", + "path": "KOI8-U/uk/train/city-morning.bin", + "bytes": 205, + "highBytes": 168, + "uniqueHighBytes": 31, + "sha256": "853b422f64ef06d7a64528db49d4eca0f11240852392e51cb2700b1b8b9fd122" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "library", + "path": "KOI8-U/uk/train/library.bin", + "bytes": 182, + "highBytes": 151, + "uniqueHighBytes": 32, + "sha256": "42455a09e6cafcd694b38198b1f172e44e63774cde6c9e760181334974536fe5" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "market", + "path": "KOI8-U/uk/train/market.bin", + "bytes": 199, + "highBytes": 165, + "uniqueHighBytes": 33, + "sha256": "0d8b40da8ad30cc4d6d849f2439f472db8092d622e428747aa55e1fa1797800b" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "train", + "document": "workshop", + "path": "KOI8-U/uk/train/workshop.bin", + "bytes": 189, + "highBytes": 161, + "uniqueHighBytes": 30, + "sha256": "b4b92fa64c4b9252511ed0240a7e1cb565a0b1158f07b50fefb93fab1bd5f0f7" + }, + { + "encoding": "KOI8-U", + "iconv": "KOI8-U", + "language": "uk", + "split": "validation", + "document": "river-trip", + "path": "KOI8-U/uk/validation/river-trip.bin", + "bytes": 195, + "highBytes": 160, + "uniqueHighBytes": 31, + "sha256": "1b015aa68443a92670b3e73e37fa96d70d1e2dea1a9ed736bebfcf077bde9dfe" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "macintosh/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "9a35ac96816b79920d3076818a106719d95169d017b89832ff735652fdbc45d5" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "macintosh/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "d826a59c9fe0ea77efe5554f688a96a25ba31edd3b502177f88b229237bfafff" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "library", + "path": "macintosh/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "29829a1ac13b7f7ad2ecd8abd7d0f2454b3beb4ede31af4ab1a8901cb73103b9" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "market", + "path": "macintosh/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "f135969c32b723540e69570512179a361f12aca33782130567cafc8b8cb40974" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "train", + "document": "workshop", + "path": "macintosh/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a77aa45de75ac7b572174e1c99aa8aab3d7a3cb0278b2f68932d1c7fe4598218" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "macintosh/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "b2edf27b09f5aeaf742baed4ebc17e0a890da6a9f0aa52a40adbcb16104e5538" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "macintosh/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "b675b0218b3db70d7c6f6f43cd7eb00fe7b45713f6e23e3fbac03a176e95935a" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "macintosh/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "15f13077700d48818a5a5050c47683e1c14260cc22ce956abd29bd07eddf10eb" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "library", + "path": "macintosh/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "a489cbb2f2f276564faf85fd6f86795a1ec5a1e5b1cfeda35874e2302e9d3d1b" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "market", + "path": "macintosh/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "80770be98c0f1e40ee59e9de08615620aa8a0af1ff73ef8b6a3cc483b14a3a4e" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "train", + "document": "workshop", + "path": "macintosh/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "5b25ecfaa0df015d5c4c1fb52fce66af137588ac8677acd50c468b65b5112018" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "macintosh/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "d68dda4671b8158cbff67ab7b476d2b819ac45ad00719d4dbe48f22ba1862baa" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "macintosh/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "50363a67b8df69b6fbdb3b7a5a25c22c40cdac6ccca71d3bfafeff490e122c58" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "macintosh/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "5f8dbd5bcfe437962ef5943c707e1695edce736abace97d5b29ef68945c42415" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "library", + "path": "macintosh/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "66ecc14482e9af6c7d2c813d11462760e1a70b98a6d6c0c966821cd03f07eee3" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "market", + "path": "macintosh/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "5e81aa00b93ef4299cde155e0d215906411a1b0566a676ea307403ef658274d5" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "macintosh/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "ee23ebed42809949b51365120200f135cb051471430331b7c6ccf8282841075a" + }, + { + "encoding": "macintosh", + "iconv": "MACINTOSH", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "macintosh/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "test", + "document": "community-garden", + "path": "windows-1250/cs/test/community-garden.bin", + "bytes": 128, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "1eba2d29ae0f73a2f12f43469971a65f81c58779530b1b753285cb2d8c29200d" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "city-morning", + "path": "windows-1250/cs/train/city-morning.bin", + "bytes": 98, + "highBytes": 13, + "uniqueHighBytes": 6, + "sha256": "a0f0dfa252459d39f725fee0b0a13818681efd1c28e854e4eaf32509e03f5a55" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "library", + "path": "windows-1250/cs/train/library.bin", + "bytes": 88, + "highBytes": 12, + "uniqueHighBytes": 7, + "sha256": "1469e1d9d57e6f147a56fb9caf1f5b0eab61dedd7c5482c19199de6853ade2ef" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "market", + "path": "windows-1250/cs/train/market.bin", + "bytes": 74, + "highBytes": 10, + "uniqueHighBytes": 7, + "sha256": "083f02c5caabf56bf78ab98fc83cabb38dfc79337e33d3a0f694de80c2403e0e" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "train", + "document": "workshop", + "path": "windows-1250/cs/train/workshop.bin", + "bytes": 90, + "highBytes": 11, + "uniqueHighBytes": 8, + "sha256": "98ad6215f5c5e7ebaa4324bd133c34bd89e63abc0fcffaac9bfa73ecd3175f17" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "cs", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/cs/validation/river-trip.bin", + "bytes": 80, + "highBytes": 11, + "uniqueHighBytes": 7, + "sha256": "7934c52e77661f3ba94150b262a68510c2da7c9a6d493fe1d72c2ea7658a97b5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "test", + "document": "community-garden", + "path": "windows-1250/hu/test/community-garden.bin", + "bytes": 139, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "d55e643af71274694d74cff7dbf12589bfd13a2b86f2ddcfb53476578a3132b8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "city-morning", + "path": "windows-1250/hu/train/city-morning.bin", + "bytes": 94, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "97945a679eae0390b0b3c65608b9cffe4cd8ba6196b8af4be0cec9272bf25ae4" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "library", + "path": "windows-1250/hu/train/library.bin", + "bytes": 108, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "bb34de899f6968d695ac39ed18a9f72077d309da97d9721d4e6551562d644427" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "market", + "path": "windows-1250/hu/train/market.bin", + "bytes": 80, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a6a508b4927acd6751a091826a4cc4945330021699819d1c956c8ea6b440a56b" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "train", + "document": "workshop", + "path": "windows-1250/hu/train/workshop.bin", + "bytes": 101, + "highBytes": 14, + "uniqueHighBytes": 7, + "sha256": "2a294d4fef2aeeee691b74de2293bdf7a7bae7854b3021607fc7df7d17a1a3a4" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "hu", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/hu/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "ce35a0b3ccaa4b07727c0b9e925c958ad050a5d774d91c3f25962c3b7291ffab" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "test", + "document": "community-garden", + "path": "windows-1250/pl/test/community-garden.bin", + "bytes": 188, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "bfb9b6bd673324ae75dba20653fe290162c78cc26d13be88fa4fd57ee3a7b101" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "city-morning", + "path": "windows-1250/pl/train/city-morning.bin", + "bytes": 174, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "809b42296a766532c1dedb56aaf287db6538397f3f221c8164a2c627871187d5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "library", + "path": "windows-1250/pl/train/library.bin", + "bytes": 172, + "highBytes": 9, + "uniqueHighBytes": 6, + "sha256": "d9f13cd5121d5f68350e48835968eee29e76da5406a7e6de2b3ed5481c7f51f0" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "market", + "path": "windows-1250/pl/train/market.bin", + "bytes": 166, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "bda4333ebf0ad7c3f37cf924b23be19ab53ddc3117f6031d8075c846a76299b5" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "train", + "document": "workshop", + "path": "windows-1250/pl/train/workshop.bin", + "bytes": 165, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "3344ae802a8cccebcb4561ccc3320991aa1c3c789bcf6172b7427d9bbf7fa6a0" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "pl", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/pl/validation/river-trip.bin", + "bytes": 161, + "highBytes": 9, + "uniqueHighBytes": 3, + "sha256": "aac1576aea489691f339c2cce058bb908effa44befab53e4f04a82698896fc54" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "test", + "document": "community-garden", + "path": "windows-1250/ro/test/community-garden.bin", + "bytes": 206, + "highBytes": 13, + "uniqueHighBytes": 5, + "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "city-morning", + "path": "windows-1250/ro/train/city-morning.bin", + "bytes": 175, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "library", + "path": "windows-1250/ro/train/library.bin", + "bytes": 187, + "highBytes": 17, + "uniqueHighBytes": 5, + "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "market", + "path": "windows-1250/ro/train/market.bin", + "bytes": 171, + "highBytes": 17, + "uniqueHighBytes": 4, + "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "train", + "document": "workshop", + "path": "windows-1250/ro/train/workshop.bin", + "bytes": 170, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + }, + { + "encoding": "windows-1250", + "iconv": "WINDOWS-1250", + "language": "ro", + "split": "validation", + "document": "river-trip", + "path": "windows-1250/ro/validation/river-trip.bin", + "bytes": 170, + "highBytes": 10, + "uniqueHighBytes": 3, + "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "test", + "document": "community-garden", + "path": "windows-1251/ru/test/community-garden.bin", + "bytes": 183, + "highBytes": 151, + "uniqueHighBytes": 30, + "sha256": "b02d0279ed09a186f1a89564cc9e00cf02fd68b0db74a1be7529ee85c8481256" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "city-morning", + "path": "windows-1251/ru/train/city-morning.bin", + "bytes": 200, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "74c61cbdff726b61e50b34c19a5777c7c13fef80f82ff10677136da50064ada8" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "library", + "path": "windows-1251/ru/train/library.bin", + "bytes": 211, + "highBytes": 178, + "uniqueHighBytes": 31, + "sha256": "52bcdfd21b84cf735a555d8a7669e5dc36c12509459df325f3eb3ce5b5891377" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "market", + "path": "windows-1251/ru/train/market.bin", + "bytes": 199, + "highBytes": 166, + "uniqueHighBytes": 31, + "sha256": "ca18ba887b386031b0a5dc1fc1731c94314ad760c063947f18513da039200773" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "train", + "document": "workshop", + "path": "windows-1251/ru/train/workshop.bin", + "bytes": 196, + "highBytes": 168, + "uniqueHighBytes": 32, + "sha256": "cc7dfbf50476621ae34c5f6de6eae7ba3e13fc650f507a17fc6c07e77bf45ee8" + }, + { + "encoding": "windows-1251", + "iconv": "WINDOWS-1251", + "language": "ru", + "split": "validation", + "document": "river-trip", + "path": "windows-1251/ru/validation/river-trip.bin", + "bytes": 204, + "highBytes": 169, + "uniqueHighBytes": 30, + "sha256": "78219367c687aac640cc10f65cb672bcc274a0418acb42669d050d81c849c329" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "test", + "document": "community-garden", + "path": "windows-1252/da/test/community-garden.bin", + "bytes": 123, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "d742c49869d5f97ff0dc9bed58a7e819cf5a9f3cde419ff785f2201591fa7991" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "city-morning", + "path": "windows-1252/da/train/city-morning.bin", + "bytes": 89, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "75320357a51c6716244970d7c802f42af40a6a65fa1602e4f38e239e16ccd887" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "library", + "path": "windows-1252/da/train/library.bin", + "bytes": 105, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "4b0f38943f7eb950af8b04b16ac6ce581c48f70e933f66bd03725842d9e1bd58" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "market", + "path": "windows-1252/da/train/market.bin", + "bytes": 84, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "61d9285e5b354f3cffb320f3bb62af7d4ce86fe34ba0e5067d5bc26aa9cd010c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "train", + "document": "workshop", + "path": "windows-1252/da/train/workshop.bin", + "bytes": 103, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "aca6cded29cf164fc9445de75292a45f007c971ea9407d7ef0d794413a7cd975" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "da", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/da/validation/river-trip.bin", + "bytes": 97, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "0edb6dd9323283e842dea2112213cf7b86608b257a36cfd318b8c71b3c01c8e0" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "test", + "document": "community-garden", + "path": "windows-1252/de/test/community-garden.bin", + "bytes": 220, + "highBytes": 12, + "uniqueHighBytes": 3, + "sha256": "93771ce6b07c49e90e7930e9c08952affe7064fa95627b10dffa95c75e632ac8" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "city-morning", + "path": "windows-1252/de/train/city-morning.bin", + "bytes": 208, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "e7aaf10ff4602c59e9bed5f5e68cc83c05e865adac1c2bed6d521f5be047da25" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "library", + "path": "windows-1252/de/train/library.bin", + "bytes": 204, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "d860713e1ada51f340a09ce22e48e4f4d5476bee3532e3e1aa1567b791f48ad7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "market", + "path": "windows-1252/de/train/market.bin", + "bytes": 177, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "049131e4a8a900762ced0040a0b23cf30f1eb0b551a66c81370cac9209ee3155" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "train", + "document": "workshop", + "path": "windows-1252/de/train/workshop.bin", + "bytes": 188, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "a15644f9dd3ab9b5b6cdeee2adb60de77817351c47b3e5b8c31d30c97dbc48a2" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "de", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/de/validation/river-trip.bin", + "bytes": 192, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "3e0bec3503375f52460604a2e116bb52c4513254f2bc90ddfd7a8dbe0bb2d66d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "test", + "document": "community-garden", + "path": "windows-1252/en/test/community-garden.bin", + "bytes": 136, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8574d38336d96e1a48fd42b012b3924bc36ee57550bc58a28ed6d5c17d514c1d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "city-morning", + "path": "windows-1252/en/train/city-morning.bin", + "bytes": 108, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "ff22d70d940981c1ea141576fb34c74d841dbf2e2140d80ee55f51dd63bd9ef0" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "library", + "path": "windows-1252/en/train/library.bin", + "bytes": 117, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "12ea1dc5452af3ca116753189e184ef0a10645eee17f33211dc5f7d182318f38" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "market", + "path": "windows-1252/en/train/market.bin", + "bytes": 100, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "cbaa356df57d8557b3d2c3ca4d6ec5845077db27583e102aebdaedcacf9f2aee" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "train", + "document": "workshop", + "path": "windows-1252/en/train/workshop.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "157d773566f7cd87f7bafb87780ba98dcc090adbdaaea2c03483e7a3b6a93a70" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "en", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/en/validation/river-trip.bin", + "bytes": 106, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "c4e405fd6466194b2ddc91f07f8bcb624c904531d6d96f37e35d04847c6abbb9" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "test", + "document": "community-garden", + "path": "windows-1252/es/test/community-garden.bin", + "bytes": 203, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "323efc9ab1e3d5da572b5ed4268cf3d1cbe7ac3feb83aceb983ac67a57e090cf" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "city-morning", + "path": "windows-1252/es/train/city-morning.bin", + "bytes": 202, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "1ed542551aeea32c52e599b941675dfa4f3ba675429570b27121a1453fa6fd00" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "library", + "path": "windows-1252/es/train/library.bin", + "bytes": 202, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "afc3c8ac066dc1b054f07924d3b662d2a3ec33eb804220453ad1ff20f0f9bc7c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "market", + "path": "windows-1252/es/train/market.bin", + "bytes": 199, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "d9aea19b39cbaf4d6ca3c3ee06074d42814c483f11f8bacf47555658b15fc441" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "train", + "document": "workshop", + "path": "windows-1252/es/train/workshop.bin", + "bytes": 194, + "highBytes": 6, + "uniqueHighBytes": 4, + "sha256": "89b5fa7e0bb63ae6723ed7e5590f08f68ff442a84454cec16c9c08d8369f5888" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "es", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/es/validation/river-trip.bin", + "bytes": 184, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "bd673948b09654c6fc56b3c877db37b1257583d00d73b93521c20d89d9ed808b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "test", + "document": "community-garden", + "path": "windows-1252/fr/test/community-garden.bin", + "bytes": 231, + "highBytes": 9, + "uniqueHighBytes": 1, + "sha256": "dd7571c675bccf0dbeaf92ea945a99093e726495a5145d9a2665a8e9cc429db8" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "city-morning", + "path": "windows-1252/fr/train/city-morning.bin", + "bytes": 212, + "highBytes": 5, + "uniqueHighBytes": 2, + "sha256": "f228bb1d1c13f09a39488b5503ccfebb0d3dc8dd7bc92904e737dfe638eee9e4" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "library", + "path": "windows-1252/fr/train/library.bin", + "bytes": 226, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "ee562b84861246640549a83a34323df58769b539e3236f3112007d61af80d9c3" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "market", + "path": "windows-1252/fr/train/market.bin", + "bytes": 223, + "highBytes": 3, + "uniqueHighBytes": 1, + "sha256": "48129ba7a91065d81d839eafb41b97654265791ba2ec8747019391f00f2fc1eb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "train", + "document": "workshop", + "path": "windows-1252/fr/train/workshop.bin", + "bytes": 215, + "highBytes": 8, + "uniqueHighBytes": 1, + "sha256": "489b62ae5920a6d11bc053f0e5e0dccee89f79e38f768e39fe7567b0e84e8f76" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "fr", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/fr/validation/river-trip.bin", + "bytes": 228, + "highBytes": 6, + "uniqueHighBytes": 2, + "sha256": "17811916bf5ba155a8fa7f327fb635915e128dacca07dc6c9125ee89ba07b606" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "test", + "document": "community-garden", + "path": "windows-1252/it/test/community-garden.bin", + "bytes": 161, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "53bb7b070795cd69fba5bcc79a4958b2ac5bb23677ebdac99c80623f96d501dd" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "city-morning", + "path": "windows-1252/it/train/city-morning.bin", + "bytes": 101, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "7c59ad9c3c9ac0b84c02c4829bffabdc83817989f08836d9ee44afd8a392e43d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "library", + "path": "windows-1252/it/train/library.bin", + "bytes": 141, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "bb28d91b3f10f0208886078bf0fafbc85c7e03148069ddf5a9762809f28f8981" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "market", + "path": "windows-1252/it/train/market.bin", + "bytes": 114, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "40ce81cfc5ec4c5b655c2fb9ba519022a77da2c319950c868bc16c7f69278b11" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "train", + "document": "workshop", + "path": "windows-1252/it/train/workshop.bin", + "bytes": 152, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "5b5704c6191b4179ee6966a2302452eaaf99c348b34e26bc210879c77def38bc" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "it", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/it/validation/river-trip.bin", + "bytes": 98, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "8fdde80f1933a9dfb82cc0181c6770e9947ccf5bfd745b8251ee02f54e6a19a7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "test", + "document": "community-garden", + "path": "windows-1252/nl/test/community-garden.bin", + "bytes": 153, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "26b1ccb49c443abe7580037a8a836bf99e9d5a5135b1751b77a73e3f61c1a9bf" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "city-morning", + "path": "windows-1252/nl/train/city-morning.bin", + "bytes": 110, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "201fbe0db619c5d0d4ccecd723c27c68e0807437531553e8c17ba84cfc4335b2" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "library", + "path": "windows-1252/nl/train/library.bin", + "bytes": 150, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "5d9cdae67a5a7cf6cb319750006ea3cf6afaf379b11c35aa241ef63b203cb17b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "market", + "path": "windows-1252/nl/train/market.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "15466fb33051e25a441363879415fe1c6ec949d753105ad851a75b2054d8c0da" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "train", + "document": "workshop", + "path": "windows-1252/nl/train/workshop.bin", + "bytes": 132, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "4fa51f6d8f9543c9df0acef5a2711a5d805fd18286c1164d786bd8ac715c183b" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "nl", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/nl/validation/river-trip.bin", + "bytes": 113, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "d9448db92c3fe1270814d7798759eca44290f94fd76c0ff9012dc2401b426253" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "test", + "document": "community-garden", + "path": "windows-1252/no/test/community-garden.bin", + "bytes": 116, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b86618ac6687d3769cca67ec9cb49a0d4d1dea45476592a316ae6eef40d576db" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "city-morning", + "path": "windows-1252/no/train/city-morning.bin", + "bytes": 116, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "445f262b0f2ea6f724271b8486b0cfc861cf4e081e4617db895bc36a33dea2ee" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "library", + "path": "windows-1252/no/train/library.bin", + "bytes": 126, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "96c146e3d2b86817371ba6ca5d07361d31519cb0d16fbbecfaf7bba36f0ff2eb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "market", + "path": "windows-1252/no/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "a0279dbf28e14a6d9841789f50751ec9cdb09c2743f4ac1c61e72f310e916732" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "train", + "document": "workshop", + "path": "windows-1252/no/train/workshop.bin", + "bytes": 100, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "9c70c92214b0f0ac9000a131773f1ed1756d630037293a759982edcb621e056a" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "no", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/no/validation/river-trip.bin", + "bytes": 85, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "931a303e54cbfe3ccddf45aa0dd3956be07e4cf16bf1fe5ee0874bd4ad2bab9a" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "test", + "document": "community-garden", + "path": "windows-1252/pt/test/community-garden.bin", + "bytes": 145, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "a8e56da45e10d94745a21dd36a2721410e1b0d04b5bfee15b2c451a27c8ba62d" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "city-morning", + "path": "windows-1252/pt/train/city-morning.bin", + "bytes": 99, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "c9a43be7ac9a1af0be31146446c7cea5142ba97545c304e9fdfef166c0dfabe1" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "library", + "path": "windows-1252/pt/train/library.bin", + "bytes": 116, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "634c4f0d24e7f1a8bbf9b1fd68d0fe414376d2efeff0fbfc7637d921709f2a62" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "market", + "path": "windows-1252/pt/train/market.bin", + "bytes": 96, + "highBytes": 5, + "uniqueHighBytes": 3, + "sha256": "79f397f7b2ac2cca51091f006462be7777befb52d36c54237a8cfb0cb20cd924" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "train", + "document": "workshop", + "path": "windows-1252/pt/train/workshop.bin", + "bytes": 108, + "highBytes": 3, + "uniqueHighBytes": 3, + "sha256": "fbb52707258ca57000887397d2bd8220907fc178ed36b929c62e72bf211848e1" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "pt", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/pt/validation/river-trip.bin", + "bytes": 104, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "db583dead964fc7fe8a257ebd69f9dd7388c89384bac132955a76b02150c753f" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "test", + "document": "community-garden", + "path": "windows-1252/sv/test/community-garden.bin", + "bytes": 138, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "7229bf29e0adc2b37cdb6663bb696e500e13000633665f0f226915f5f74c4853" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "city-morning", + "path": "windows-1252/sv/train/city-morning.bin", + "bytes": 101, + "highBytes": 7, + "uniqueHighBytes": 4, + "sha256": "5ac24bf0fe3bfb1d64aa066ff78e3af67bcf0b4a6f8d142fefeff0b8ad4ae72c" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "library", + "path": "windows-1252/sv/train/library.bin", + "bytes": 108, + "highBytes": 4, + "uniqueHighBytes": 2, + "sha256": "fc05c9ad957334217feaa333f6cfe7cd206b250fbae9ea51a19d9c69f72158e7" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "market", + "path": "windows-1252/sv/train/market.bin", + "bytes": 89, + "highBytes": 7, + "uniqueHighBytes": 3, + "sha256": "9f4f10ac20699e493cdd99b8c1ee6caa29d4a9359962aab15ac23bb773f1d26f" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "train", + "document": "workshop", + "path": "windows-1252/sv/train/workshop.bin", + "bytes": 110, + "highBytes": 2, + "uniqueHighBytes": 1, + "sha256": "989f41c9b83ba4e3475a6902c33340fa7b07bc965776a09e9db165321bdcb3cb" + }, + { + "encoding": "windows-1252", + "iconv": "WINDOWS-1252", + "language": "sv", + "split": "validation", + "document": "river-trip", + "path": "windows-1252/sv/validation/river-trip.bin", + "bytes": 86, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "84cc4d3ab5abaa2d2575d868f0056375043c7e17d6c34ecd4112fca4632bc321" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "test", + "document": "community-garden", + "path": "windows-1253/el/test/community-garden.bin", + "bytes": 136, + "highBytes": 112, + "uniqueHighBytes": 25, + "sha256": "8f7531c39aa8efa5f4e38e8eb948f713d69cbe72badea15391e5c62870179370" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "city-morning", + "path": "windows-1253/el/train/city-morning.bin", + "bytes": 93, + "highBytes": 74, + "uniqueHighBytes": 25, + "sha256": "80e97194d91d40aeae9020cd96d22ac6ad01e3c39020377e32bdf63c2b9622e0" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "library", + "path": "windows-1253/el/train/library.bin", + "bytes": 111, + "highBytes": 94, + "uniqueHighBytes": 27, + "sha256": "eb0448b06be644629054c0f94e0c6dccb7458d80a6c477eda1ab321f378c51bb" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "market", + "path": "windows-1253/el/train/market.bin", + "bytes": 85, + "highBytes": 69, + "uniqueHighBytes": 28, + "sha256": "ee94d165652f333d57c5be996825a8723cfa76aba286f1f22a0b0d40312eece9" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "train", + "document": "workshop", + "path": "windows-1253/el/train/workshop.bin", + "bytes": 108, + "highBytes": 90, + "uniqueHighBytes": 27, + "sha256": "2466d051c231cbb22d9fa5a962bb180a88dbe805ad1b609467bd06afe5de3b02" + }, + { + "encoding": "windows-1253", + "iconv": "WINDOWS-1253", + "language": "el", + "split": "validation", + "document": "river-trip", + "path": "windows-1253/el/validation/river-trip.bin", + "bytes": 92, + "highBytes": 75, + "uniqueHighBytes": 25, + "sha256": "4dc6ccfd555071ba0df444d4f2fd5b0eb96cf44e389dc872279e8ac1096b359e" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "test", + "document": "community-garden", + "path": "windows-1254/tr/test/community-garden.bin", + "bytes": 136, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "a2224b89fee790662cf9c3cac7200da0f8c5842b4ce5d26512d76e522c6db437" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "city-morning", + "path": "windows-1254/tr/train/city-morning.bin", + "bytes": 87, + "highBytes": 9, + "uniqueHighBytes": 5, + "sha256": "a1231eabb26c3d7d770c885972062ee78365f871d1f62be5977332949e8b1a80" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "library", + "path": "windows-1254/tr/train/library.bin", + "bytes": 98, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "432b50510bf2a5ab8b7b51aaf608341bea95afbf43edff4fa4d404f4a2d87a70" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "market", + "path": "windows-1254/tr/train/market.bin", + "bytes": 86, + "highBytes": 4, + "uniqueHighBytes": 3, + "sha256": "25354256b7d0e3f936eb224f6343e5d3d73f78a504d5b2d3c260a2015a62e356" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "train", + "document": "workshop", + "path": "windows-1254/tr/train/workshop.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "0bc87de2045407c5c4db2589ab89cb9faf40cf5556d17caf0bfa916a1b4a0b03" + }, + { + "encoding": "windows-1254", + "iconv": "WINDOWS-1254", + "language": "tr", + "split": "validation", + "document": "river-trip", + "path": "windows-1254/tr/validation/river-trip.bin", + "bytes": 98, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "b94d965d212deb815bd73eabf712bb673f632c88c0a0c9f772202109e4d85e8d" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "test", + "document": "community-garden", + "path": "windows-1255/he/test/community-garden.bin", + "bytes": 104, + "highBytes": 84, + "uniqueHighBytes": 21, + "sha256": "6044aa4ee9d7f7517ca965d0a83029c5569a83e8efad7b9f39974fa95fd61298" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "city-morning", + "path": "windows-1255/he/train/city-morning.bin", + "bytes": 79, + "highBytes": 64, + "uniqueHighBytes": 19, + "sha256": "5c0eddaa74015d9d135cd9c9d80b117cd82a3aa3effe6298deaca2c06ac1305b" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "library", + "path": "windows-1255/he/train/library.bin", + "bytes": 86, + "highBytes": 72, + "uniqueHighBytes": 20, + "sha256": "b9b2605035a9017f18f9de847e03e623581ffed30ea1623fe332efcac420637b" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "market", + "path": "windows-1255/he/train/market.bin", + "bytes": 64, + "highBytes": 49, + "uniqueHighBytes": 21, + "sha256": "ed2975da284ad08b0261f536f69344981747bdd3b37d0ae6a4ec0a36ed1d57ca" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "train", + "document": "workshop", + "path": "windows-1255/he/train/workshop.bin", + "bytes": 80, + "highBytes": 64, + "uniqueHighBytes": 21, + "sha256": "763e4f5501fa3308c2862f48f5173d29d5fd1ccf730664a9bc4c40a714dfcbc6" + }, + { + "encoding": "windows-1255", + "iconv": "WINDOWS-1255", + "language": "he", + "split": "validation", + "document": "river-trip", + "path": "windows-1255/he/validation/river-trip.bin", + "bytes": 63, + "highBytes": 47, + "uniqueHighBytes": 18, + "sha256": "5319026034eb165464faad8f2de9c2e6c97f01c92e71d2fc3441310417d70b28" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "test", + "document": "community-garden", + "path": "windows-1256/ar/test/community-garden.bin", + "bytes": 112, + "highBytes": 92, + "uniqueHighBytes": 25, + "sha256": "a36cd96bbd0a9597105cb4292371e4c2998aabd844ba3c9b6cd7a0d85dfe23e8" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "city-morning", + "path": "windows-1256/ar/train/city-morning.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 22, + "sha256": "fd5a2fe9debabe95e8ca2e20fc6a95db0d4f7ed301bfb285a49f3c4e84e7409c" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "library", + "path": "windows-1256/ar/train/library.bin", + "bytes": 107, + "highBytes": 89, + "uniqueHighBytes": 25, + "sha256": "7d22becffc494a5a0cca03a06f66167f909ceb67d914125e0668bb5db170ec9e" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "market", + "path": "windows-1256/ar/train/market.bin", + "bytes": 89, + "highBytes": 75, + "uniqueHighBytes": 24, + "sha256": "153ade153ba47b62ef5a64e3b1215b54f3cd01607fd44708a288eb78889faf82" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "train", + "document": "workshop", + "path": "windows-1256/ar/train/workshop.bin", + "bytes": 91, + "highBytes": 75, + "uniqueHighBytes": 21, + "sha256": "3aa1c63ed62f9fd08039dcc50fa8705a239a1b133a96fdb81c7340d5d4cd9a79" + }, + { + "encoding": "windows-1256", + "iconv": "WINDOWS-1256", + "language": "ar", + "split": "validation", + "document": "river-trip", + "path": "windows-1256/ar/validation/river-trip.bin", + "bytes": 91, + "highBytes": 73, + "uniqueHighBytes": 24, + "sha256": "dd2d2c110d7753296192722e41de23e67c3350eea9217a1f5935829742bd2c69" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "test", + "document": "community-garden", + "path": "windows-1257/et/test/community-garden.bin", + "bytes": 122, + "highBytes": 2, + "uniqueHighBytes": 2, + "sha256": "b09cf914e826b9230579833e56b483b71e75865e26b0cfdd921b9f386dafb448" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "city-morning", + "path": "windows-1257/et/train/city-morning.bin", + "bytes": 94, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "68d9d20af9ad1104bdfa39df3ce31e24acc520754c942351d037bc9378a9c5bc" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "library", + "path": "windows-1257/et/train/library.bin", + "bytes": 101, + "highBytes": 1, + "uniqueHighBytes": 1, + "sha256": "60b30c22898450c95e27d36443d3984c00987bbd4933d796c92a6e3ca3a38d5c" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "market", + "path": "windows-1257/et/train/market.bin", + "bytes": 92, + "highBytes": 6, + "uniqueHighBytes": 3, + "sha256": "fd5fc0e9d16f48c3a3f750929689afe78e5da298455fce90960e402427a64d9d" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "train", + "document": "workshop", + "path": "windows-1257/et/train/workshop.bin", + "bytes": 107, + "highBytes": 3, + "uniqueHighBytes": 2, + "sha256": "731b2b0230faadc87c3a981fc8ae34ec858d99970a1f73dbf0aefdcb4b58f3d0" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "et", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/et/validation/river-trip.bin", + "bytes": 95, + "highBytes": 10, + "uniqueHighBytes": 2, + "sha256": "81df0820697d38b422b3a2e8954782c17d2e93d39be7111c48e0d6f7f9b4c07e" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "test", + "document": "community-garden", + "path": "windows-1257/lt/test/community-garden.bin", + "bytes": 177, + "highBytes": 17, + "uniqueHighBytes": 6, + "sha256": "80b70659af618020af9f2c1794142df619dbcd227562bedb75d66d470583f714" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "city-morning", + "path": "windows-1257/lt/train/city-morning.bin", + "bytes": 156, + "highBytes": 11, + "uniqueHighBytes": 4, + "sha256": "d94a96e0004bc37143c160367be78385afd0d79399650c8164019338d9bd413d" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "library", + "path": "windows-1257/lt/train/library.bin", + "bytes": 163, + "highBytes": 8, + "uniqueHighBytes": 5, + "sha256": "00f464452edd344abec665695ea2b124318b01503ae7dc2873a7ec46c4b0fa1a" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "market", + "path": "windows-1257/lt/train/market.bin", + "bytes": 171, + "highBytes": 12, + "uniqueHighBytes": 8, + "sha256": "33acb1aeffced8891b03f168be73b488da3639b73ec9e31029793d2d74a00aab" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "train", + "document": "workshop", + "path": "windows-1257/lt/train/workshop.bin", + "bytes": 170, + "highBytes": 9, + "uniqueHighBytes": 4, + "sha256": "7e8f7590df8128997dba53c2fd357760e6d11685e148612d124d0978f808f264" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lt", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/lt/validation/river-trip.bin", + "bytes": 165, + "highBytes": 16, + "uniqueHighBytes": 5, + "sha256": "71dc9bb4eeeb296c5ac3cc2eb200227e3100abff60cc14087a0a05a33ff0b4dd" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "test", + "document": "community-garden", + "path": "windows-1257/lv/test/community-garden.bin", + "bytes": 174, + "highBytes": 17, + "uniqueHighBytes": 8, + "sha256": "a42bdb992cc4b4c21108bc9a5850cb2ddff8d6b568c17a7505a69efdd8a815ba" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "city-morning", + "path": "windows-1257/lv/train/city-morning.bin", + "bytes": 163, + "highBytes": 15, + "uniqueHighBytes": 7, + "sha256": "bbd09e02d22432ac8e2295607379bdd17cf7ec9ea40d8f86c44d7bcce144c221" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "library", + "path": "windows-1257/lv/train/library.bin", + "bytes": 161, + "highBytes": 15, + "uniqueHighBytes": 5, + "sha256": "05ef86295a393e71f052f9932d4c2492f44fae597764429f188ab3ddcd0be5bd" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "market", + "path": "windows-1257/lv/train/market.bin", + "bytes": 170, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "1fbdc5105ca4e06c0c87edfe63a807d1543944a170f8e41f3bd50c0523753ba4" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "train", + "document": "workshop", + "path": "windows-1257/lv/train/workshop.bin", + "bytes": 163, + "highBytes": 10, + "uniqueHighBytes": 4, + "sha256": "14a636582f0cdc34bcb262c63e52f58d31a81b04c91504839032ec88764f82df" + }, + { + "encoding": "windows-1257", + "iconv": "WINDOWS-1257", + "language": "lv", + "split": "validation", + "document": "river-trip", + "path": "windows-1257/lv/validation/river-trip.bin", + "bytes": 156, + "highBytes": 14, + "uniqueHighBytes": 4, + "sha256": "58b262baaa8f97950fb68db83d3a55a43ad3f140bc74010a9bb2a57fb1b86cc4" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "test", + "document": "community-garden", + "path": "windows-1258/vi/test/community-garden.bin", + "bytes": 128, + "highBytes": 36, + "uniqueHighBytes": 14, + "sha256": "9e50ec1534c1b7fc362908307bc6890705ecf794c0bb1e159917ec107fd77875" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "city-morning", + "path": "windows-1258/vi/train/city-morning.bin", + "bytes": 132, + "highBytes": 37, + "uniqueHighBytes": 13, + "sha256": "c2568706a878d17ba6f98ffd6881f24da76d012ee67d8469aed7556fadcb92bb" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "library", + "path": "windows-1258/vi/train/library.bin", + "bytes": 111, + "highBytes": 25, + "uniqueHighBytes": 11, + "sha256": "f319f96fe6fcbca8b0d120b30065f27299f9a356733c406bca99fa1e0d8a4d60" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "market", + "path": "windows-1258/vi/train/market.bin", + "bytes": 94, + "highBytes": 24, + "uniqueHighBytes": 12, + "sha256": "6d5fad1cf1c98c236cbe21b8ce770cf07998b21eb7a0cb84fc2723f432388a18" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "train", + "document": "workshop", + "path": "windows-1258/vi/train/workshop.bin", + "bytes": 105, + "highBytes": 30, + "uniqueHighBytes": 11, + "sha256": "f4c388a881bcb3bde3a1c02e1b371de703b0d2c75425be1206428cb828c9d7a1" + }, + { + "encoding": "windows-1258", + "iconv": "WINDOWS-1258", + "language": "vi", + "split": "validation", + "document": "river-trip", + "path": "windows-1258/vi/validation/river-trip.bin", + "bytes": 93, + "highBytes": 23, + "uniqueHighBytes": 12, + "sha256": "5379d3a9a9d3a72910233391382978a455ef5c80afb3cd31931bfe08a112fcd8" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "test", + "document": "community-garden", + "path": "windows-874/th/test/community-garden.bin", + "bytes": 103, + "highBytes": 97, + "uniqueHighBytes": 35, + "sha256": "77d8715caa7248f93f1e2a9518680d6026d3688a1711b6fe5e65722fc030771a" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "city-morning", + "path": "windows-874/th/train/city-morning.bin", + "bytes": 75, + "highBytes": 69, + "uniqueHighBytes": 32, + "sha256": "69a85d6da187e6944204dc80897bd3ecfe407193e3bb7964acc21b964bc0f58e" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "library", + "path": "windows-874/th/train/library.bin", + "bytes": 99, + "highBytes": 95, + "uniqueHighBytes": 36, + "sha256": "5e22a173b7c6cc9a867f0b13cecf42da7fae43ad4a15e0affade46786443bcd6" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "market", + "path": "windows-874/th/train/market.bin", + "bytes": 66, + "highBytes": 60, + "uniqueHighBytes": 30, + "sha256": "9299c007d348b83f096f7ceca669b27be5bc82f8bf2dd5f006604fb8302cc2ce" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "train", + "document": "workshop", + "path": "windows-874/th/train/workshop.bin", + "bytes": 82, + "highBytes": 77, + "uniqueHighBytes": 38, + "sha256": "6ece49322ff6115b207f13799d415787aeaa84d251b8a8b7f5e318d7acca9530" + }, + { + "encoding": "windows-874", + "iconv": "WINDOWS-874", + "language": "th", + "split": "validation", + "document": "river-trip", + "path": "windows-874/th/validation/river-trip.bin", + "bytes": 72, + "highBytes": 69, + "uniqueHighBytes": 27, + "sha256": "257c08098491bde6fa4830723d074c608082f28073699548f1242c9e0565366e" }, { "encoding": "x-mac-cyrillic", diff --git a/corpus/generated/ngrams.mjs b/corpus/generated/ngrams.mjs index 8fb8a41..ab72c86 100644 --- a/corpus/generated/ngrams.mjs +++ b/corpus/generated/ngrams.mjs @@ -207,6 +207,266 @@ export default [ 0x0a83ae, 0x0a8da0, ], }, + { + encoding: 'ISO-8859-1', + language: 'da', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x676572, 0x6f6720, 0x64656e, 0x646572, + 0x6c6520, 0x6e6465, 0x737465, 0x74656b, 0x0a0a42, 0x206269, 0x206279, + 0x206465, 0x20686f, 0x2070e5, 0x2c206f, 0x626962, 0x626c69, 0x640a0a, + 0x652062, 0x652066, 0x65640a, 0x656e73, 0x65722c, 0x65722e, 0x65726e, + 0x69626c, 0x696465, 0x696c6c, 0x696f74, 0x6b7374, 0x6c696f, 0x6c6c65, + 0x6e6572, 0x6f7465, 0x70e520, 0x722065, 0x722c20, 0x722e0a, 0x726e65, + 0x736572, 0x746572, 0x766572, 0x79656e, 0xe52064, 0xf86765, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4ce6, 0x206272, 0x206275, + 0x2062f8, 0x206361, 0x20656c, 0x20656e, 0x206669, 0x20666f, 0x206672, + 0x2066f8, 0x206869, 0x20686a, 0x206920, 0x206b6f, 0x206c61, 0x206c69, + 0x206d65, 0x206d75, 0x206f6d, 0x206f73, 0x20706c, 0x20736b, 0x207374, + 0x2073e6, 0x207465, 0x207472, 0x207479, 0x20756e, 0x207665, 0x207669, + 0x2076e5, 0x2076e6, 0x20e562, 0x20e662, 0x2c2063, 0x2c206d, 0x2c2075, + 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, 0x457420, 0x4ce673, + 0x4cf872, 0x4d6f72, 0x61622c, 0x616473, 0x6166e9, 0x616773, 0x616c74, + 0x616e67, 0x617265, 0x61726b, 0x61766c, 0x622c20, 0x626c65, 0x626e65, + 0x6272f8, 0x627573, 0x627965, 0x627967, 0x62f867, 0x636166, 0x642062, + 0x642c20, 0x646167, 0x646520, 0x64652c, 0x64656c, 0x64732e, 0x652068, + 0x65206f, 0x652070, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'de', + documents: 4, + trigrams: [ + 0x656e20, 0x636865, 0x657220, 0x206465, 0x20756e, 0x696520, 0x736368, + 0x756e64, 0x65696e, 0x68656e, 0x696368, 0x696e65, 0x6e2064, 0x6e6420, + 0x6e656e, 0x206469, 0x626572, 0x646572, 0x687265, 0x6c6569, 0x205465, + 0x206569, 0x446965, 0x616368, 0x636874, 0x64656e, 0x646965, 0x652c20, + 0x656974, 0x65722e, 0x65726e, 0x686572, 0x696e20, 0x697461, 0x697465, + 0x6b6c65, 0x6c6572, 0x6e2042, 0x6e2069, 0x6e2e20, 0x6e6520, 0x726520, + 0x726569, 0x742065, 0x746520, 0x74656e, 0x204269, 0x204765, 0x204d65, + 0x205265, 0x205374, 0x206265, 0x2066fc, 0x206968, 0x206b6c, 0x206c65, + 0x207665, 0x20fc62, 0x2e2042, 0x2e2044, 0x426962, 0x45696e, 0x4d656e, + 0x537461, 0x616474, 0x616c65, 0x616d20, 0x617573, 0x626c69, 0x636872, + 0x636b65, 0x642066, 0x65204d, 0x652053, 0x652054, 0x652075, 0x656368, + 0x656c6c, 0x656e2e, 0x657265, 0x657277, 0x657320, 0x66fc72, 0x676974, + 0x686520, 0x68656b, 0x68656c, 0x687465, 0x69626c, 0x69636b, 0x696769, + 0x696872, 0x696f74, 0x6c6520, 0x6c6963, 0x6c696f, 0x6e206c, 0x6e2076, + 0x6e6465, 0x6e7465, 0x6e7477, 0x6f7468, 0x722047, 0x722053, 0x722064, + 0x722e0a, 0x722e20, 0x726e20, 0x736520, 0x73652c, 0x737461, 0x737465, + 0x737563, 0x740a0a, 0x742061, 0x746164, 0x74616c, 0x746572, 0x746865, + 0x747769, 0x756368, 0x757373, 0x766572, 0x776963, 0xe46e64, 0xfc6265, + 0xfc7220, 0x0a0a41, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'en', + documents: 4, + trigrams: [ + 0x20616e, 0x616e64, 0x686520, 0x6e6420, 0x727320, 0x207468, 0x2c2061, + 0x746865, 0x652061, 0x652c20, 0x656164, 0x657273, 0x657320, 0x732061, + 0x206120, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206c69, 0x206e65, + 0x207265, 0x207768, 0x546865, 0x616465, 0x626f75, 0x627261, 0x636974, + 0x642063, 0x64206e, 0x646520, 0x646573, 0x652063, 0x657369, 0x68696c, + 0x696272, 0x696465, 0x696c65, 0x697479, 0x6c6520, 0x6c6962, 0x6c6c20, + 0x6e2074, 0x722064, 0x726172, 0x726561, 0x732062, 0x732072, 0x732073, + 0x732c20, 0x732e0a, 0x776869, 0x790a0a, 0x0a0a41, 0x0a0a46, 0x0a0a52, + 0x0a0a54, 0x0a4120, 0x0a4661, 0x0a5265, 0x0a5468, 0x206162, 0x206170, + 0x206172, 0x206265, 0x20626f, 0x206361, 0x20636c, 0x206469, 0x20646f, + 0x206578, 0x206661, 0x206775, 0x206869, 0x20696e, 0x206974, 0x206a61, + 0x206d61, 0x206d75, 0x206e61, 0x206f70, 0x207072, 0x2070e2, 0x207175, + 0x2072e9, 0x2072f4, 0x207363, 0x207365, 0x20736c, 0x20736d, 0x207465, + 0x207761, 0x20776f, 0x2c2063, 0x2c206a, 0x2c2073, 0x3b2063, 0x412073, + 0x446967, 0x466172, 0x4d6f72, 0x526561, 0x536174, 0x612066, 0x612072, + 0x61626f, 0x616420, 0x61642c, 0x6166e9, 0x616b65, 0x616c20, 0x616c61, + 0x616c6c, 0x616d20, 0x616e20, 0x616e67, 0x617065, 0x617070, 0x617220, + 0x617265, 0x617269, 0x61726b, 0x61726d, 0x617274, 0x617279, 0x617475, + 0x617920, 0x61e761, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'es', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x617320, 0x656c20, 0x656e20, 0x657320, 0x206c61, + 0x206c6f, 0x207920, 0x2e204c, 0x61646f, 0x616e20, 0x6c6f73, 0x6e206c, + 0x6f7265, 0x726573, 0x706172, 0x732063, 0x204c6f, 0x206369, 0x20656c, + 0x207061, 0x4c6f73, 0x612065, 0x612070, 0x617261, 0x626c69, 0x6c206d, + 0x6c6120, 0x6e6120, 0x732061, 0x732e20, 0x746f72, 0x206269, 0x20636c, + 0x20636f, 0x206469, 0x206d65, 0x207265, 0x207375, 0x207665, 0x456c20, + 0x4c6120, 0x612062, 0x626962, 0x636961, 0x646520, 0x64656c, 0x646967, + 0x656361, 0x65732e, 0x69626c, 0x696369, 0x696f74, 0x6c2063, 0x6c696f, + 0x6e2065, 0x6f2073, 0x6f2e20, 0x6f7465, 0x726120, 0x732064, 0x732070, + 0x732076, 0x732e0a, 0x746563, 0x0a0a45, 0x204c61, 0x206361, 0x206c65, + 0x206d61, 0x20706f, 0x207072, 0x207365, 0x207465, 0x2c206c, 0x612063, + 0x612064, 0x61206c, 0x612079, 0x616e61, 0x61726f, 0x61732c, 0x61732e, + 0x627265, 0x627573, 0x636120, 0x636164, 0x636965, 0x636975, 0x636c61, + 0x636f2e, 0x636f6d, 0x646164, 0x64656e, 0x646f20, 0x646f72, 0x656369, + 0x656e61, 0x657175, 0x657263, 0x657274, 0x676974, 0x676f20, 0x696120, + 0x696173, 0x69656c, 0x69656e, 0x696769, 0x697461, 0x697564, 0x6c6172, + 0x6c6173, 0x6c6c65, 0x6d6572, 0x6d7061, 0x6e2070, 0x6e2072, 0x6e6173, + 0x6e6f73, 0x6f2064, 0x6f2079, 0x6f636f, 0x6f6d70, 0x6f732e, 0x706f63, + 0x717565, 0x717569, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'fr', + documents: 4, + trigrams: [ + 0x657320, 0x206c65, 0x656e74, 0x6c6573, 0x6e7420, 0x74206c, 0x732070, + 0x206465, 0x657420, 0x657572, 0x727320, 0x742064, 0x206475, 0x206574, + 0x647520, 0x6c6520, 0x732063, 0x757273, 0x204c65, 0x20636f, 0x207072, + 0x2e204c, 0x4c6573, 0x646573, 0x652063, 0x652070, 0x207061, 0x20706f, + 0x2c206c, 0x626c69, 0x65206d, 0x656e63, 0x65732e, 0x6c6c65, 0x6f7572, + 0x706172, 0x717565, 0x726520, 0x732e20, 0x746573, 0x756520, 0x757220, + 0x206269, 0x206361, 0x20656e, 0x206d61, 0x207361, 0x4c6520, 0x616765, + 0x617265, 0x626962, 0x637465, 0x646520, 0x650a0a, 0x652064, 0x652065, + 0x652073, 0x69626c, 0x69656e, 0x696c6c, 0x696f74, 0x6c6575, 0x6c696f, + 0x6d656e, 0x6e6365, 0x6e6520, 0x6f6e73, 0x6f7468, 0x7072e9, 0x72206c, + 0x726368, 0x72e970, 0x732065, 0x73206c, 0x732073, 0x746575, 0xe97061, + 0xe97269, 0x0a0a4c, 0x20636c, 0x206672, 0x206c61, 0x206e75, 0x207265, + 0x2072e9, 0x20756e, 0x207669, 0x2c2064, 0x4c6120, 0x556e65, 0x612062, + 0x616972, 0x616973, 0x617263, 0x636865, 0x6368e9, 0x636965, 0x636f6e, + 0x652e20, 0x656e20, 0x65732c, 0x676573, 0x686973, 0x68e871, 0x68e920, + 0x69656c, 0x696572, 0x696e20, 0x697175, 0x697265, 0x697374, 0x697420, + 0x6c6120, 0x6c656e, 0x6c6965, 0x6d6172, 0x6d6d65, 0x6de972, 0x6e2070, + 0x6e7320, 0x6e7473, 0x6e756d, 0x6f6d6d, 0x6f7576, 0x706f73, 0x706f75, + 0x70726f, 0x72656e, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'it', + documents: 4, + trigrams: [ + 0x6c6120, 0x207069, 0x656e74, 0x206465, 0x206520, 0x612070, 0x6c6c61, + 0x6e6f20, 0x6f7269, 0x746f72, 0x206369, 0x206c61, 0x612067, 0x612069, + 0x61746f, 0x636974, 0x64656c, 0x657263, 0x692061, 0x692063, 0x692073, + 0x696120, 0x697474, 0x69f920, 0x6c6920, 0x7069f9, 0x726920, 0x7474e0, + 0x206269, 0x206361, 0x20636f, 0x206469, 0x206765, 0x206920, 0x206c65, + 0x206d65, 0x207065, 0x207072, 0x207363, 0x207369, 0x207374, 0x2c2063, + 0x4c6120, 0x612062, 0x612063, 0x612064, 0x61206c, 0x616666, 0x616e6f, + 0x626962, 0x626c69, 0x636166, 0x636f6c, 0x637269, 0x65206c, 0x652073, + 0x652c20, 0x656361, 0x656c20, 0x656c65, 0x656c6c, 0x657363, 0x657474, + 0x6666e8, 0x67696f, 0x676c69, 0x692070, 0x69626c, 0x696f20, 0x696f74, + 0x6c696f, 0x6d656e, 0x6e6120, 0x6e7465, 0x6e7472, 0x6f2064, 0x6f2065, + 0x6f2070, 0x6f2c20, 0x6f6e6f, 0x6f7465, 0x706572, 0x726120, 0x726361, + 0x726520, 0x726961, 0x736372, 0x746120, 0x746563, 0x7a6120, 0x0a0a47, + 0x0a0a49, 0x0a0a4c, 0x0a0a55, 0x0a476c, 0x0a4920, 0x0a4c61, 0x0a556e, + 0x206120, 0x206167, 0x206169, 0x206170, 0x206173, 0x206175, 0x206365, + 0x206368, 0x20666f, 0x206672, 0x206769, 0x20676c, 0x206775, 0x20696c, + 0x20696e, 0x206c69, 0x206d69, 0x206d75, 0x206e65, 0x207061, 0x207361, + 0x207371, 0x207376, 0x207472, 0x207574, 0x207665, 0x2c2066, 0x2c2067, + 0x2c2069, 0x2c206d, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'nl', + documents: 4, + trigrams: [ + 0x656e20, 0x20656e, 0x646520, 0x206465, 0x656572, 0x74656e, 0x766572, + 0x206f70, 0x207374, 0x207465, 0x652062, 0x652065, 0x652073, 0x657273, + 0x6e206f, 0x6e2e0a, 0x736368, 0x742064, 0x746572, 0x206269, 0x206361, + 0x20636f, 0x206565, 0x206765, 0x206865, 0x206c65, 0x206f76, 0x207665, + 0x207765, 0x2c2063, 0x446520, 0x61616b, 0x616173, 0x6166e9, 0x616b74, + 0x626962, 0x626c69, 0x636166, 0x636874, 0x652063, 0x65696e, 0x656b65, + 0x656e2e, 0x656e73, 0x657220, 0x657264, 0x657265, 0x65726b, 0x66e973, + 0x687465, 0x69626c, 0x696465, 0x696e20, 0x696e67, 0x696f74, 0x697320, + 0x6b6520, 0x6b656e, 0x6b7420, 0x6c6520, 0x6c6565, 0x6c6569, 0x6c696f, + 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206d, 0x6e2074, 0x6e2076, 0x6e2077, + 0x6f656b, 0x6f6e74, 0x6f7020, 0x6f7065, 0x6f7468, 0x6f7665, 0x70656e, + 0x72656e, 0x727320, 0x732065, 0x73206f, 0x732c20, 0x73656e, 0x737365, + 0x737461, 0x737465, 0x746164, 0x746865, 0xe97320, 0x0a0a42, 0x0a0a44, + 0x0a0a45, 0x0a0a4c, 0x0a426f, 0x0a4465, 0x0a4565, 0x0a4c65, 0x206170, + 0x20626f, 0x206272, 0x206275, 0x206472, 0x206475, 0x20686f, 0x206875, + 0x206964, 0x20696e, 0x206b61, 0x206b6c, 0x206c61, 0x206d61, 0x206d65, + 0x206d75, 0x206e61, 0x206f6e, 0x20706c, 0x20706f, 0x207363, 0x207477, + 0x207761, 0x207a6f, 0x20e9e9, 0x2c2061, 0x2c206b, 0x426f65, 0x446967, + 0x45656e, 0x4c657a, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'no', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x6f6720, 0x656e65, 0x206c61, 0x652066, + 0x676572, 0x6e6520, 0x6e6572, 0x737465, 0x74656b, 0x0a0a42, 0x206269, + 0x206272, 0x206465, 0x20666f, 0x2070e5, 0x207665, 0x626962, 0x626c69, + 0x640a0a, 0x646572, 0x652062, 0x652074, 0x652c20, 0x65640a, 0x656e73, + 0x657273, 0x657420, 0x672073, 0x676520, 0x67656e, 0x69626c, 0x696765, + 0x696e67, 0x696f74, 0x697465, 0x6b6572, 0x6b7374, 0x6c6167, 0x6c6520, + 0x6c6967, 0x6c696f, 0x6e206f, 0x6e6465, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x72206f, 0x72656e, 0x726765, 0x746520, 0x746572, + 0x746f72, 0x76656e, 0x766572, 0x79656e, 0xe52064, 0xf86b65, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4c65, 0x206275, 0x206279, + 0x2062f8, 0x2064f8, 0x20656c, 0x20656e, 0x206570, 0x206665, 0x206669, + 0x2066f8, 0x206869, 0x20686a, 0x20686f, 0x206920, 0x206b61, 0x206b6f, + 0x206c69, 0x206d65, 0x206d75, 0x206ee6, 0x206f6d, 0x206f73, 0x207365, + 0x20736a, 0x20736b, 0x207374, 0x207465, 0x20746f, 0x207472, 0x207479, + 0x20756e, 0x207669, 0x2076e5, 0x20e570, 0x20e672, 0x2c2065, 0x2c206b, + 0x2c206f, 0x2c2075, 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, + 0x457420, 0x4c6573, 0x4cf872, 0x4d6f72, 0x616665, 0x616720, 0x616765, + 0x616773, 0x616c74, 0x616e67, 0x617020, 0x617265, 0x61726b, 0x61766c, + 0x627279, 0x6272f8, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'pt', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x206520, 0x207072, 0x696120, 0x6e6120, 0x206369, + 0x206f73, 0x612064, 0x612065, 0x61206f, 0x646520, 0x657320, 0x657363, + 0x726573, 0x732061, 0x732070, 0x732e0a, 0x0a0a4f, 0x0a4f73, 0x206269, + 0x206573, 0x206e61, 0x206f20, 0x207065, 0x207665, 0x4f7320, 0x612070, + 0x616465, 0x61646f, 0x616d20, 0x617261, 0x617320, 0x626962, 0x626c69, + 0x636120, 0x636964, 0x646120, 0x646164, 0x646967, 0x646f20, 0x652061, + 0x656d20, 0x657175, 0x666963, 0x69626c, 0x696461, 0x696d65, 0x696f74, + 0x6c696f, 0x6e7175, 0x6f2065, 0x6f7265, 0x6f7465, 0x717565, 0x717569, + 0x72616d, 0x726961, 0x726f73, 0x732063, 0x736372, 0x746563, 0x746f72, + 0x0a0a41, 0x0a0a55, 0x0a4120, 0x0a556d, 0x206120, 0x206162, 0x206163, + 0x206167, 0x20616a, 0x20616c, 0x206173, 0x206361, 0x20636c, 0x2063f3, + 0x206469, 0x20646f, 0x20656e, 0x206571, 0x206672, 0x206775, 0x206869, + 0x206c65, 0x206c69, 0x206d61, 0x206d65, 0x206d6f, 0x206d75, 0x2070e3, + 0x207175, 0x2073e1, 0x207472, 0x20f46e, 0x2c206d, 0x2c206f, 0x2c2071, + 0x2c2076, 0x412062, 0x412063, 0x4d616e, 0x4d6572, 0x4f6669, 0x556d61, + 0x610a0a, 0x612061, 0x612062, 0x612063, 0x61206d, 0x612074, 0x612e0a, + 0x616272, 0x61636f, 0x616461, 0x6166e9, 0x616761, 0x616772, 0x616a75, + 0x616c0a, 0x616c75, 0x616e68, 0x616e71, 0x616e74, 0x61722c, 0x61732e, + 0x61e761, 0x61e7e3, + ], + }, + { + encoding: 'ISO-8859-1', + language: 'sv', + documents: 4, + trigrams: [ + 0x657220, 0x617220, 0x206f63, 0x636820, 0x656e20, 0x6f6368, 0x737461, + 0x64656e, 0x657420, 0x6b6172, 0x726e61, 0x736b61, 0x74206c, 0x746164, + 0x766572, 0x206269, 0x206465, 0x2066f6, 0x206c69, 0x2070e5, 0x20736b, + 0x207479, 0x207665, 0x612062, 0x612074, 0x612e0a, 0x61640a, 0x616465, + 0x617265, 0x626962, 0x626c69, 0x640a0a, 0x65726e, 0x66f672, 0x676120, + 0x69626c, 0x696761, 0x696f74, 0x6b6170, 0x6b6572, 0x6b6e61, 0x6c6967, + 0x6c696f, 0x6e612e, 0x6e6172, 0x6e736b, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x722066, 0x72206f, 0x736172, 0x73f66b, 0x746120, + 0x74656b, 0x746f72, 0xe52064, 0x0a0a42, 0x0a0a45, 0x0a0a4c, 0x0a0a53, + 0x0a42f6, 0x0a4574, 0x0a4ce4, 0x0a5374, 0x206265, 0x206272, 0x206275, + 0x2062f6, 0x20656c, 0x20656e, 0x2066e4, 0x206772, 0x206869, 0x20686a, + 0x20686f, 0x206920, 0x206b61, 0x206b6f, 0x206c61, 0x206ce5, 0x206d65, + 0x206d75, 0x206de4, 0x206f6d, 0x206f73, 0x207374, 0x2073e4, 0x2073f6, + 0x207465, 0x20746f, 0x207661, 0x2076e4, 0x20e470, 0x20f670, 0x2c2067, + 0x2c206b, 0x2c206f, 0x2c20e4, 0x42f66e, 0x446574, 0x446967, 0x457474, + 0x4ce473, 0x4cf672, 0x4d6f72, 0x537461, 0x61206f, 0x6120f6, 0x6166e9, + 0x616720, 0x616773, 0x616b6e, 0x616c20, 0x616d74, 0x616e20, 0x616e73, + 0x617020, 0x617061, 0x617269, 0x61726b, 0x61726e, 0x626573, 0x6272f6, + 0x627573, 0x62f663, + ], + }, { encoding: 'ISO-8859-10', language: 'is', @@ -389,6 +649,110 @@ export default [ 0xba7465, 0xe30a0a, ], }, + { + encoding: 'ISO-8859-2', + language: 'cs', + documents: 4, + trigrams: [ + 0x206120, 0x207072, 0x616aed, 0x6aed20, 0x206b6e, 0x636520, 0x6b6e69, + 0x6e6968, 0x6eed20, 0x792e0a, 0x0a0a4d, 0x20706f, 0x20e865, 0x2c206b, + 0x610a0a, 0x612070, 0x612076, 0x652061, 0x65206d, 0x652070, 0x686f76, + 0x69686f, 0x6a6520, 0x6e610a, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, + 0x73792e, 0x756a65, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xf86920, + 0x0a0a46, 0x0a0ac8, 0x0a4661, 0x0a4d61, 0x0a4dec, 0x0ac874, 0x206175, + 0x206279, 0x206368, 0x206476, 0x2064ed, 0x206869, 0x20686c, 0x206a61, + 0x206b61, 0x206b6f, 0x206bf3, 0x206c69, 0x206d65, 0x206d75, 0x206dec, + 0x206e61, 0x206f20, 0x206f74, 0x206f76, 0x2070ed, 0x2070f8, 0x207261, + 0x207365, 0x207472, 0x2074fd, 0x207665, 0x20766f, 0x2076ec, 0x207a61, + 0x20bee1, 0x2c206d, 0x2c206f, 0x2c207a, 0x446967, 0x466172, 0x4d616c, + 0x4dec73, 0x52e16e, 0x536f62, 0x546963, 0x61206c, 0x6164ed, 0x616c75, + 0x616cfd, 0x61726d, 0x61736e, 0x6174ed, 0x617574, 0x617675, 0x6176e1, + 0x6176e9, 0x622c20, 0x626f74, 0x626f75, 0x627573, 0x62796c, 0x63686c, + 0x6368e1, 0x636f20, 0x642061, 0x642c20, 0x64616a, 0x646365, 0x647665, + 0x64e176, 0x64e920, 0x64ec2c, 0x64ed20, 0x64ed6c, 0x65206a, 0x65206b, + 0x652072, 0x65642c, 0x656461, 0x65656d, 0x656b61, 0x656d2c, 0x656ee1, + 0x657273, 0x6576ed, 0x65f865, 0x676974, 0x680a0a, 0x686973, 0x686c65, + 0x686ce9, 0x687920, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'hu', + documents: 4, + trigrams: [ + 0x206120, 0x2c2061, 0x6e616b, 0x6f7320, 0x0a0a41, 0x206bf6, 0x2076e1, + 0x20e973, 0x657265, 0x6bf66e, 0x6e7976, 0x726f73, 0x73206b, 0x742c20, + 0x76e172, 0xe1726f, 0xe97320, 0xf66e79, 0x0a4120, 0x206373, 0x206b65, + 0x206b69, 0x61206b, 0x612076, 0x616b2e, 0x617420, 0x657420, 0x697320, + 0x6b2c20, 0x6b2e0a, 0x6d6920, 0x6f7420, 0x73206c, 0x73206d, 0x74206b, + 0x74e172, 0x7674e1, 0x797674, 0xf36b20, 0x0a0a45, 0x0a417a, 0x0a4567, + 0x20616c, 0x20617a, 0x206275, 0x206469, 0x20656c, 0x20656d, 0x206672, + 0x206761, 0x206be1, 0x206be9, 0x206bf3, 0x206c61, 0x206c65, 0x206de9, + 0x206dfa, 0x206dfb, 0x206f6c, 0x207065, 0x207069, 0x207361, 0x207365, + 0x207475, 0x2074e9, 0x2074f6, 0x207669, 0x20e172, 0x20e962, 0x20ed72, + 0x20fa74, 0x2c206b, 0x2c2073, 0x412063, 0x412067, 0x412076, 0x417a20, + 0x446967, 0x456779, 0x526567, 0x537a6f, 0x612064, 0x612074, 0x61630a, + 0x616a74, 0x616b20, 0x616b2c, 0x616c6d, 0x616e0a, 0x616e20, 0x617061, + 0x617373, 0x6173f3, 0x617469, 0x6174f3, 0x617a20, 0x617a64, 0x62616e, + 0x626174, 0x626572, 0x627265, 0x627573, 0x630a0a, 0x637361, 0x637365, + 0x642c20, 0x646573, 0x646967, 0x6469e1, 0x646f6d, 0x646f74, 0x64e16b, + 0x65642c, 0x656469, 0x656767, 0x6567ed, 0x656b20, 0x656b2c, 0x656b65, + 0x656c20, 0x656c6c, 0x656c6d, 0x656c79, 0x656d62, 0x656e2e, 0x656e64, + 0x656e79, 0x656ef5, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'pl', + documents: 4, + trigrams: [ + 0x616ab1, 0x6ab120, 0x207072, 0x206920, 0x207779, 0x616d69, 0x636861, + 0x70727a, 0x727a79, 0xb66369, 0x206a61, 0x20706f, 0x20726f, 0x2c2061, + 0x616368, 0x626c69, 0x636920, 0x647a69, 0x656b20, 0x692073, 0x692e20, + 0x69626c, 0x696572, 0x696f74, 0x6c696f, 0x6e6961, 0x6f7261, 0x6f7465, + 0x706973, 0x727a65, 0x74656b, 0x0a0a4d, 0x206120, 0x206269, 0x20646c, + 0x206b6f, 0x206d69, 0x207069, 0x207369, 0x207370, 0x20737a, 0x2e204b, + 0x610a0a, 0x612070, 0x612072, 0x617267, 0x626962, 0x636520, 0x63692e, + 0x637920, 0x637a79, 0x642c20, 0x64616a, 0x646c61, 0x65206f, 0x656461, + 0x656b61, 0x657261, 0x66726f, 0x68616d, 0x692077, 0x692c20, 0x696379, + 0x696520, 0x69656e, 0x697379, 0x69ea20, 0x6b6172, 0x6c6120, 0x6c6e69, + 0x6d6920, 0x6d6965, 0x6e6369, 0x6e6520, 0x6e656b, 0x6e6963, 0x6e6965, + 0x6f727a, 0x6fb663, 0x707261, 0x72616a, 0x72616d, 0x726f77, 0x7369ea, + 0x737072, 0x73746f, 0x737920, 0x737a65, 0x746f72, 0x75647a, 0x776120, + 0x776961, 0x776965, 0x792063, 0x792070, 0x792073, 0x796672, 0x7a7920, + 0x7a7974, 0xb1206a, 0xb1206b, 0xb12077, 0x0a0a54, 0x0a0a57, 0x0a4d61, + 0x0a4d69, 0x0a5461, 0x0a5720, 0x204269, 0x20437a, 0x204b61, 0x204b6c, + 0x204e61, 0x205072, 0x20526f, 0x206175, 0x206275, 0x206368, 0x20636f, + 0x206379, 0x20637a, 0x206461, 0x206472, 0x20676f, 0x206869, 0x206b61, + 0x206b73, 0x206b75, + ], + }, + { + encoding: 'ISO-8859-2', + language: 'ro', + documents: 4, + trigrams: [ + 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, + 0x206d69, 0x207072, 0x207265, 0x61fe61, 0x626c69, 0x636869, 0x646520, + 0x652070, 0x656361, 0x657269, 0x657363, 0x692e20, 0x69626c, 0x696e65, + 0x696f74, 0x6c6520, 0x6c696f, 0x6f7465, 0x736520, 0x746520, 0x746563, + 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, 0x20616c, 0x206175, 0x206361, + 0x20636c, 0x206375, 0x2063e3, 0x206469, 0x206578, 0x206961, 0x206d65, + 0x206f72, 0x207065, 0x207363, 0x207365, 0x207472, 0x20ba74, 0x2c2069, + 0x426962, 0x506961, 0x612064, 0x616c20, 0x617220, 0x617574, 0x617ae3, + 0x636120, 0x636172, 0x636574, 0x63e320, 0x646573, 0x646967, 0x652063, + 0x652064, 0x652074, 0x6520ee, 0x652e0a, 0x65617a, 0x656e69, 0x656e74, + 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, 0x65ba74, 0x676974, + 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, 0x6963e3, 0x696769, + 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, 0x6c2064, 0x6c6567, + 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, 0x6d696e, 0x6e6365, + 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, 0x7261ba, 0x72652c, + 0x72696c, 0x72756c, 0x736320, 0x736368, 0x737572, 0x742e20, 0x74616c, + 0x746974, 0x747275, 0x74e30a, 0x752065, 0x756d70, 0x757a65, 0x7a6561, + 0xba7465, 0xe30a0a, + ], + }, { encoding: 'ISO-8859-3', language: 'mt', @@ -441,6 +805,162 @@ export default [ 0x206b6c, 0x206b72, ], }, + { + encoding: 'ISO-8859-5', + language: 'ru', + documents: 4, + trigrams: [ + 0xeee220, 0xd0eee2, 0x20dfe0, 0xdfe0de, 0x20d820, 0x20ddd0, 0x20dfde, + 0xd0ef20, 0xe0ded4, 0xebd520, 0xefe220, 0x20d2eb, 0x20d3de, 0x20dcd0, + 0x20ded1, 0xd0e2d5, 0xd5dddd, 0xd820df, 0xd82c20, 0xdbd820, 0xdcd0e0, + 0xddd020, 0xddebd5, 0xe0eb20, 0xe220e1, 0xe2d5db, 0xe2d5e0, 0xe2ded2, + 0xef20df, 0x0a0abd, 0x20d020, 0x20d1d8, 0x20d2de, 0x20dade, 0x20dee1, + 0x20dee2, 0x20dfdb, 0x20e0d5, 0x20e1de, 0x20efe0, 0x2c20d0, 0x2c20d1, + 0xbdd020, 0xd00a0a, 0xd020df, 0xd0d5e2, 0xd0ddd8, 0xd0e0da, 0xd0e1e2, + 0xd1d8d1, 0xd1dbd8, 0xd1e1e3, 0xd2d0ee, 0xd2d5e0, 0xd2efe2, 0xd3dee2, + 0xd42c20, 0xd4d0ee, 0xd4d820, 0xd5dbd5, 0xd5dbd8, 0xd5e0d8, 0xd5e0eb, + 0xd5e1e2, 0xd6d4d0, 0xd820de, 0xd820e1, 0xd8d1db, 0xd8dee2, 0xd8e1e2, + 0xd8e2d0, 0xd8ef20, 0xdad00a, 0xdad520, 0xdbd8de, 0xddd0ef, 0xddd8da, + 0xddd8ef, 0xddddef, 0xddded2, 0xddefef, 0xded1e1, 0xded2ef, 0xdedbec, + 0xdedcd0, 0xdedddd, 0xdee0de, 0xdee1d5, 0xdee1e2, 0xdee1eb, 0xdee2d5, + 0xdee2de, 0xdfd8e1, 0xdfdee1, 0xe0dad0, 0xe0dcd0, 0xe0dee1, 0xe1d5dd, + 0xe1ddeb, 0xe1e2d0, 0xe1e2d5, 0xe1e2de, 0xe1e3d6, 0xe1eb20, 0xe1ef20, + 0xe220da, 0xe220dd, 0xe220df, 0xe2d5da, 0xe2dee0, 0xe2e0de, 0xe2e1ef, + 0xe3d6d4, 0xe5ded4, 0xeb20df, 0xeb2c20, 0xeb2e0a, 0xebd2d0, 0xebd920, + 0xef20ef, 0xefe0dc, 0xefeee2, 0xefef20, 0x0a0ab2, 0x0a0ab3, 0x0ab220, + 0x0ab3de, 0x0abdd0, + ], + }, + { + encoding: 'ISO-8859-6', + language: 'ar', + documents: 4, + trigrams: [ + 0x20c7e4, 0xc7e4e5, 0x20e8c7, 0x20e8ea, 0xe3cac8, 0xe620c7, 0xe8c7e4, + 0xe8e620, 0x0a0aea, 0xc90a0a, 0xd120c7, 0xc7ca20, 0xc7ccd9, 0xc7d120, + 0xc7e4c3, 0xc7e4d9, 0xc7e4e8, 0xc820c7, 0xc8c920, 0xc920c7, 0xc920e8, + 0xc92e0a, 0xca20e8, 0xcac820, 0xcac8c9, 0xcb20c7, 0xccd920, 0xcd20c7, + 0xcfeae6, 0xd1c7cc, 0xd5e1c7, 0xd920c7, 0xe4e5cf, 0xe4e5e3, 0xe5cfea, + 0xe5e3ca, 0xe6c7d3, 0xe8d5e1, 0xeae6c9, 0x0a0aca, 0x0acad3, 0x0aeac8, + 0x0aead5, 0x0aead9, 0x20c3c8, 0x20c3e5, 0x20c7ce, 0x20c8e7, 0x20c8ea, + 0x20cfe4, 0x20d1e2, 0x20d5da, 0x20d9e6, 0x20e1d1, 0x20e1ea, 0x20e3ca, + 0x20e4e4, 0x20e8ca, 0x20e8d5, 0x20eae6, 0xc120d9, 0xc120e8, 0xc3c8e8, + 0xc3cdc7, 0xc3cec8, 0xc3e5ea, 0xc4e8e6, 0xc6c90a, 0xc720c8, 0xc720e4, + 0xc720e8, 0xc720ea, 0xc72e0a, 0xc7c120, 0xc7c820, 0xc7c8e7, 0xc7cd20, + 0xc7ceca, 0xc7cfc6, 0xc7cfe4, 0xc7cfea, 0xc7d12e, 0xc7d1d9, 0xc7d1ea, + 0xc7d320, 0xc7d3c8, 0xc7d6cd, 0xc7d9cf, 0xc7e1e4, 0xc7e3e7, 0xc7e4ca, + 0xc7e4cd, 0xc7e4ce, 0xc7e4d2, 0xc7e4d3, 0xc7e4d4, 0xc7e4d7, 0xc7e4e1, + 0xc7e4e2, 0xc7e4e6, 0xc7e4e7, 0xc7e7ea, 0xc820e1, 0xc8c7cd, 0xc8c7cf, + 0xc8c7d1, 0xc8c92e, 0xc8ca0a, 0xc8cdcb, 0xc8d1e5, 0xc8d220, 0xc8e7c7, + 0xc8e7cf, 0xc8e8c7, 0xc8eae6, 0xc920c8, 0xc920d1, 0xca0a0a, 0xcac7d1, + 0xcac8c7, 0xcacd20, 0xcacde1, 0xcad3ca, 0xcad8d1, 0xcae1ca, 0xcaeac7, + 0xcaeae2, 0xcce8e6, + ], + }, + { + encoding: 'ISO-8859-7', + language: 'el', + documents: 4, + trigrams: [ + 0x20eae1, 0xe1e920, 0xeae1e9, 0x20e2e9, 0xe120ea, 0xe12e0a, 0xe2e9e2, + 0xe5e920, 0xe9e2eb, 0xeff5ed, 0xf4e120, 0xf5ed20, 0x0a0acf, 0x0acfe9, + 0x20e1e3, 0x20e1ed, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xcfe920, + 0xdfe12e, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, 0xe5f220, 0xe70a0a, 0xe720e2, + 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9efe8, 0xe9f3f4, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xf0fceb, 0xf1e9ef, 0xf220ea, 0xf3f4de, + 0xf3f4e7, 0xf4e5f2, 0xf4e7ed, 0xfcebe7, 0x0a0ac7, 0x0a0acc, 0x0ac720, + 0x0acce9, 0x20dced, 0x20e1f1, 0x20e2ef, 0x20e3f1, 0x20e5eb, 0x20e5f0, + 0x20e5f1, 0x20e5f4, 0x20e720, 0x20e9f3, 0x20eae5, 0x20eafe, 0x20ebe5, + 0x20ecdd, 0x20ece1, 0x20ece9, 0x20ecef, 0x20ecf5, 0x20eef5, 0x20efe4, + 0x20efe9, 0x20efec, 0x20f0e5, 0x20f0eb, 0x20f0ef, 0x20f6f1, 0x20f8dc, + 0x20f8f9, 0x2c20e5, 0x2c20ec, 0x2c20f4, 0x2c20f6, 0xb9f3f5, 0xc720f0, + 0xcce9e1, 0xd0f1f9, 0xd3e1e2, 0xd8e7f6, 0xdc0a0a, 0xdc20e1, 0xdc20ea, + 0xdc20f3, 0xdc20f4, 0xdc2c20, 0xdce4e1, 0xdce6e5, 0xdcede8, 0xdcf1e9, + 0xdcf4e9, 0xdcf6e5, 0xdcf7ed, 0xdd20e1, 0xdde3f7, 0xddebe9, 0xddedef, + 0xddf22e, 0xde20ef, 0xdeeae7, 0xdeece7, 0xdef1e9, 0xdf20f3, 0xdf2c20, + 0xdfe120, 0xdfe1f2, 0xdfe3ef, 0xdfece5, 0xdfeff5, 0xe120e5, 0xe120e9, + 0xe120eb, 0xe120ec, 0xe1e2e2, 0xe1e3ed, 0xe1e3ef, 0xe1e3f1, 0xe1e8e1, + 0xe1e8e7, 0xe1eafc, + ], + }, + { + encoding: 'ISO-8859-8', + language: 'he', + documents: 4, + trigrams: [ + 0xe9ed20, 0xe5f8e9, 0xed20ec, 0xed20ee, 0xf1f4f8, 0xf8e9ed, 0x20e4f7, + 0x20ece0, 0x20eee7, 0x2c20e1, 0xe0e9ed, 0xe4f1f4, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed2e, 0xed2e0a, 0xef20ee, 0xf0e420, 0xf2e9f8, + 0xf4f8e9, 0xf7e5f8, 0xf820ee, 0xf8e5fa, 0xf8fa20, 0xf9e9ed, 0xfa0a0a, + 0xfa20ec, 0x0a0ae4, 0x0a0ae7, 0x0a0af6, 0x0a0af7, 0x0ae4f2, 0x0ae7f7, + 0x0af6e5, 0x0af7e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, + 0x20e1fa, 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e8f8, 0x20ece1, + 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, 0x20eeeb, 0x20eef7, + 0x20eefa, 0x20f0f4, 0x20f1f4, 0x20f2e5, 0x20f4e9, 0x20f7e8, 0x20f9e1, + 0x20fae9, 0x2c20e3, 0x2c20f4, 0xe0e5e8, 0xe0e5ef, 0xe0e5f8, 0xe0e82c, + 0xe0f0f9, 0xe0f9e5, 0xe0fa20, 0xe120fa, 0xe1e5e3, 0xe1e5f1, 0xe1e5f7, + 0xe1e7e5, 0xe1e9f0, 0xe1ebe9, 0xe1f2e9, 0xe1f8e5, 0xe1f92c, 0xe1fa0a, + 0xe1fae9, 0xe2e1e9, 0xe2e9e8, 0xe320e5, 0xe3e1f9, 0xe3e9e2, 0xe3e9ed, + 0xe3f0e4, 0xe3f220, 0xe3f720, 0xe3f8e9, 0xe40a0a, 0xe420e1, 0xe420e3, + 0xe420e4, 0xe420e5, 0xe420f0, 0xe42e0a, 0xe4e9f1, 0xe4f2e9, 0xe4f2ee, + 0xe4f7e5, 0xe4f7f4, 0xe4f8e0, 0xe4f9f7, 0xe5e0f0, 0xe5e1e5, 0xe5e2e1, + 0xe5e320, 0xe5e3f7, 0xe5e4f1, 0xe5e5fa, 0xe5e6e9, 0xe5e6f8, 0xe5e8e5, + 0xe5ebe5, 0xe5ebf8, + ], + }, + { + encoding: 'ISO-8859-9', + language: 'tr', + documents: 4, + trigrams: [ + 0x6c6572, 0x6c6172, 0x207665, 0x722e0a, 0x766520, 0x2061e7, 0x207961, + 0x617220, 0x61e7fd, 0x657220, 0x697220, 0x722076, 0x206269, 0x20656b, + 0x206b61, 0x206bfc, 0x207361, 0x207461, 0x2c206b, 0x616261, 0x616c20, + 0x616c61, 0x616e65, 0x61722e, 0x6172fd, 0x617a61, 0x62616c, 0x650a0a, + 0x652062, 0x656869, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697461, + 0x6bfc74, 0x706861, 0x722079, 0x722c20, 0x726920, 0x726c61, 0x746172, + 0x74fc70, 0x7a6172, 0x7a6520, 0xde6568, 0xe7fd6b, 0xfc7068, 0xfc74fc, + 0xfd6b20, 0x0a0a4b, 0x0a0a4f, 0x0a0ac7, 0x0a0ade, 0x0a4bfc, 0x0a4f6b, + 0x0ac769, 0x0ade65, 0x206172, 0x206174, 0x206261, 0x206265, 0x206465, + 0x20656c, 0x206861, 0x20696c, 0x20696e, 0x206b69, 0x206b6f, 0x206d65, + 0x206dfc, 0x206f6c, 0x206f74, 0x207061, 0x207065, 0x207265, 0x207579, + 0x20f6f0, 0x2c2065, 0x2c2070, 0x43756d, 0x44696a, 0x4bfce7, 0x4f6b75, + 0x536573, 0x612074, 0x612075, 0x612c20, 0x616665, 0x61680a, 0x616cfd, + 0x616d61, 0x616e64, 0x616e6c, 0x616efd, 0x61706c, 0x61722c, 0x617261, + 0x617264, 0x617269, 0x61726b, 0x617274, 0x617461, 0x6174f6, 0x617661, + 0x617a65, 0x617afd, 0x61fee7, 0x626168, 0x62656b, 0x626572, 0x62696c, + 0x626972, 0x62fc73, 0x636920, 0x63696c, 0x63fd20, 0x646120, 0x64616e, + 0x646520, 0x64656e, 0x647520, 0x64fd6d, 0x652061, 0x652065, 0x652069, + 0x652072, 0x652073, + ], + }, + { + encoding: 'KOI8-R', + language: 'ru', + documents: 4, + trigrams: [ + 0xc0d420, 0xc1c0d4, 0x20d0d2, 0xd0d2cf, 0x20c920, 0x20cec1, 0x20d0cf, + 0xc1d120, 0xd1d420, 0xd2cfc4, 0xd9c520, 0x20c7cf, 0x20cdc1, 0x20cfc2, + 0x20d7d9, 0xc1d4c5, 0xc5cece, 0xc920d0, 0xc92c20, 0xccc920, 0xcdc1d2, + 0xcec120, 0xced9c5, 0xd120d0, 0xd2d920, 0xd420d3, 0xd4c5cc, 0xd4c5d2, + 0xd4cfd7, 0x0a0aee, 0x20c120, 0x20c2c9, 0x20cbcf, 0x20cfd3, 0x20cfd4, + 0x20d0cc, 0x20d1d2, 0x20d2c5, 0x20d3cf, 0x20d7cf, 0x2c20c1, 0x2c20c2, + 0xc10a0a, 0xc120d0, 0xc1c5d4, 0xc1cec9, 0xc1d2cb, 0xc1d3d4, 0xc2c9c2, + 0xc2ccc9, 0xc2d3d5, 0xc42c20, 0xc4c1c0, 0xc4c920, 0xc5ccc5, 0xc5ccc9, + 0xc5d2c9, 0xc5d2d9, 0xc5d3d4, 0xc7cfd4, 0xc8cfc4, 0xc920cf, 0xc920d3, + 0xc9c2cc, 0xc9cfd4, 0xc9d120, 0xc9d3d4, 0xc9d4c1, 0xcbc10a, 0xcbc520, + 0xccc9cf, 0xcec1d1, 0xcec9cb, 0xcec9d1, 0xceced1, 0xcecfd7, 0xced1d1, + 0xcfc2d3, 0xcfccd8, 0xcfcdc1, 0xcfcece, 0xcfd2cf, 0xcfd3c5, 0xcfd3d4, + 0xcfd3d9, 0xcfd4c5, 0xcfd4cf, 0xcfd7d1, 0xd0c9d3, 0xd0cfd3, 0xd120d1, + 0xd1c0d4, 0xd1d120, 0xd1d2cd, 0xd2cbc1, 0xd2cdc1, 0xd2cfd3, 0xd3c5ce, + 0xd3ced9, 0xd3d120, 0xd3d4c1, 0xd3d4c5, 0xd3d4cf, 0xd3d5d6, 0xd3d920, + 0xd420cb, 0xd420ce, 0xd420d0, 0xd4c5cb, 0xd4cfd2, 0xd4d2cf, 0xd4d3d1, + 0xd5d6c4, 0xd6c4c1, 0xd7c1c0, 0xd7c5d2, 0xd7d1d4, 0xd920d0, 0xd92c20, + 0xd92e0a, 0xd9ca20, 0xd9d7c1, 0xeec120, 0x0a0ae7, 0x0a0af7, 0x0ae7cf, + 0x0aeec1, 0x0aeec5, + ], + }, { encoding: 'KOI8-U', language: 'uk', @@ -545,6 +1065,630 @@ export default [ 0x70726f, 0x72656e, ], }, + { + encoding: 'windows-1250', + language: 'cs', + documents: 4, + trigrams: [ + 0x206120, 0x207072, 0x616aed, 0x6aed20, 0x206b6e, 0x636520, 0x6b6e69, + 0x6e6968, 0x6eed20, 0x792e0a, 0x0a0a4d, 0x20706f, 0x20e865, 0x2c206b, + 0x610a0a, 0x612070, 0x612076, 0x652061, 0x65206d, 0x652070, 0x686f76, + 0x69686f, 0x6a6520, 0x6e610a, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, + 0x73792e, 0x756a65, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xf86920, + 0x0a0a46, 0x0a0ac8, 0x0a4661, 0x0a4d61, 0x0a4dec, 0x0ac874, 0x206175, + 0x206279, 0x206368, 0x206476, 0x2064ed, 0x206869, 0x20686c, 0x206a61, + 0x206b61, 0x206b6f, 0x206bf3, 0x206c69, 0x206d65, 0x206d75, 0x206dec, + 0x206e61, 0x206f20, 0x206f74, 0x206f76, 0x2070ed, 0x2070f8, 0x207261, + 0x207365, 0x207472, 0x2074fd, 0x207665, 0x20766f, 0x2076ec, 0x207a61, + 0x209ee1, 0x2c206d, 0x2c206f, 0x2c207a, 0x446967, 0x466172, 0x4d616c, + 0x4dec73, 0x52e16e, 0x536f62, 0x546963, 0x61206c, 0x6164ed, 0x616c75, + 0x616cfd, 0x61726d, 0x61736e, 0x6174ed, 0x617574, 0x617675, 0x6176e1, + 0x6176e9, 0x622c20, 0x626f74, 0x626f75, 0x627573, 0x62796c, 0x63686c, + 0x6368e1, 0x636f20, 0x642061, 0x642c20, 0x64616a, 0x646365, 0x647665, + 0x64e176, 0x64e920, 0x64ec2c, 0x64ed20, 0x64ed6c, 0x65206a, 0x65206b, + 0x652072, 0x65642c, 0x656461, 0x65656d, 0x656b61, 0x656d2c, 0x656ee1, + 0x657273, 0x6576ed, 0x65f865, 0x676974, 0x680a0a, 0x686973, 0x686c65, + 0x686ce9, 0x687920, + ], + }, + { + encoding: 'windows-1250', + language: 'hu', + documents: 4, + trigrams: [ + 0x206120, 0x2c2061, 0x6e616b, 0x6f7320, 0x0a0a41, 0x206bf6, 0x2076e1, + 0x20e973, 0x657265, 0x6bf66e, 0x6e7976, 0x726f73, 0x73206b, 0x742c20, + 0x76e172, 0xe1726f, 0xe97320, 0xf66e79, 0x0a4120, 0x206373, 0x206b65, + 0x206b69, 0x61206b, 0x612076, 0x616b2e, 0x617420, 0x657420, 0x697320, + 0x6b2c20, 0x6b2e0a, 0x6d6920, 0x6f7420, 0x73206c, 0x73206d, 0x74206b, + 0x74e172, 0x7674e1, 0x797674, 0xf36b20, 0x0a0a45, 0x0a417a, 0x0a4567, + 0x20616c, 0x20617a, 0x206275, 0x206469, 0x20656c, 0x20656d, 0x206672, + 0x206761, 0x206be1, 0x206be9, 0x206bf3, 0x206c61, 0x206c65, 0x206de9, + 0x206dfa, 0x206dfb, 0x206f6c, 0x207065, 0x207069, 0x207361, 0x207365, + 0x207475, 0x2074e9, 0x2074f6, 0x207669, 0x20e172, 0x20e962, 0x20ed72, + 0x20fa74, 0x2c206b, 0x2c2073, 0x412063, 0x412067, 0x412076, 0x417a20, + 0x446967, 0x456779, 0x526567, 0x537a6f, 0x612064, 0x612074, 0x61630a, + 0x616a74, 0x616b20, 0x616b2c, 0x616c6d, 0x616e0a, 0x616e20, 0x617061, + 0x617373, 0x6173f3, 0x617469, 0x6174f3, 0x617a20, 0x617a64, 0x62616e, + 0x626174, 0x626572, 0x627265, 0x627573, 0x630a0a, 0x637361, 0x637365, + 0x642c20, 0x646573, 0x646967, 0x6469e1, 0x646f6d, 0x646f74, 0x64e16b, + 0x65642c, 0x656469, 0x656767, 0x6567ed, 0x656b20, 0x656b2c, 0x656b65, + 0x656c20, 0x656c6c, 0x656c6d, 0x656c79, 0x656d62, 0x656e2e, 0x656e64, + 0x656e79, 0x656ef5, + ], + }, + { + encoding: 'windows-1250', + language: 'pl', + documents: 4, + trigrams: [ + 0x616ab9, 0x6ab920, 0x207072, 0x206920, 0x207779, 0x616d69, 0x636861, + 0x70727a, 0x727a79, 0x9c6369, 0x206a61, 0x20706f, 0x20726f, 0x2c2061, + 0x616368, 0x626c69, 0x636920, 0x647a69, 0x656b20, 0x692073, 0x692e20, + 0x69626c, 0x696572, 0x696f74, 0x6c696f, 0x6e6961, 0x6f7261, 0x6f7465, + 0x706973, 0x727a65, 0x74656b, 0x0a0a4d, 0x206120, 0x206269, 0x20646c, + 0x206b6f, 0x206d69, 0x207069, 0x207369, 0x207370, 0x20737a, 0x2e204b, + 0x610a0a, 0x612070, 0x612072, 0x617267, 0x626962, 0x636520, 0x63692e, + 0x637920, 0x637a79, 0x642c20, 0x64616a, 0x646c61, 0x65206f, 0x656461, + 0x656b61, 0x657261, 0x66726f, 0x68616d, 0x692077, 0x692c20, 0x696379, + 0x696520, 0x69656e, 0x697379, 0x69ea20, 0x6b6172, 0x6c6120, 0x6c6e69, + 0x6d6920, 0x6d6965, 0x6e6369, 0x6e6520, 0x6e656b, 0x6e6963, 0x6e6965, + 0x6f727a, 0x6f9c63, 0x707261, 0x72616a, 0x72616d, 0x726f77, 0x7369ea, + 0x737072, 0x73746f, 0x737920, 0x737a65, 0x746f72, 0x75647a, 0x776120, + 0x776961, 0x776965, 0x792063, 0x792070, 0x792073, 0x796672, 0x7a7920, + 0x7a7974, 0xb9206a, 0xb9206b, 0xb92077, 0x0a0a54, 0x0a0a57, 0x0a4d61, + 0x0a4d69, 0x0a5461, 0x0a5720, 0x204269, 0x20437a, 0x204b61, 0x204b6c, + 0x204e61, 0x205072, 0x20526f, 0x206175, 0x206275, 0x206368, 0x20636f, + 0x206379, 0x20637a, 0x206461, 0x206472, 0x20676f, 0x206869, 0x206b61, + 0x206b73, 0x206b75, + ], + }, + { + encoding: 'windows-1250', + language: 'ro', + documents: 4, + trigrams: [ + 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, + 0x206d69, 0x207072, 0x207265, 0x61fe61, 0x626c69, 0x636869, 0x646520, + 0x652070, 0x656361, 0x657269, 0x657363, 0x692e20, 0x69626c, 0x696e65, + 0x696f74, 0x6c6520, 0x6c696f, 0x6f7465, 0x736520, 0x746520, 0x746563, + 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, 0x20616c, 0x206175, 0x206361, + 0x20636c, 0x206375, 0x2063e3, 0x206469, 0x206578, 0x206961, 0x206d65, + 0x206f72, 0x207065, 0x207363, 0x207365, 0x207472, 0x20ba74, 0x2c2069, + 0x426962, 0x506961, 0x612064, 0x616c20, 0x617220, 0x617574, 0x617ae3, + 0x636120, 0x636172, 0x636574, 0x63e320, 0x646573, 0x646967, 0x652063, + 0x652064, 0x652074, 0x6520ee, 0x652e0a, 0x65617a, 0x656e69, 0x656e74, + 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, 0x65ba74, 0x676974, + 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, 0x6963e3, 0x696769, + 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, 0x6c2064, 0x6c6567, + 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, 0x6d696e, 0x6e6365, + 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, 0x7261ba, 0x72652c, + 0x72696c, 0x72756c, 0x736320, 0x736368, 0x737572, 0x742e20, 0x74616c, + 0x746974, 0x747275, 0x74e30a, 0x752065, 0x756d70, 0x757a65, 0x7a6561, + 0xba7465, 0xe30a0a, + ], + }, + { + encoding: 'windows-1251', + language: 'ru', + documents: 4, + trigrams: [ + 0xfef220, 0xe0fef2, 0x20eff0, 0xeff0ee, 0x20e820, 0x20ede0, 0x20efee, + 0xe0ff20, 0xf0eee4, 0xfbe520, 0xfff220, 0x20e2fb, 0x20e3ee, 0x20ece0, + 0x20eee1, 0xe0f2e5, 0xe5eded, 0xe820ef, 0xe82c20, 0xebe820, 0xece0f0, + 0xede020, 0xedfbe5, 0xf0fb20, 0xf220f1, 0xf2e5eb, 0xf2e5f0, 0xf2eee2, + 0xff20ef, 0x0a0acd, 0x20e020, 0x20e1e8, 0x20e2ee, 0x20eaee, 0x20eef1, + 0x20eef2, 0x20efeb, 0x20f0e5, 0x20f1ee, 0x20fff0, 0x2c20e0, 0x2c20e1, + 0xcde020, 0xe00a0a, 0xe020ef, 0xe0e5f2, 0xe0ede8, 0xe0f0ea, 0xe0f1f2, + 0xe1e8e1, 0xe1ebe8, 0xe1f1f3, 0xe2e0fe, 0xe2e5f0, 0xe2fff2, 0xe3eef2, + 0xe42c20, 0xe4e0fe, 0xe4e820, 0xe5ebe5, 0xe5ebe8, 0xe5f0e8, 0xe5f0fb, + 0xe5f1f2, 0xe6e4e0, 0xe820ee, 0xe820f1, 0xe8e1eb, 0xe8eef2, 0xe8f1f2, + 0xe8f2e0, 0xe8ff20, 0xeae00a, 0xeae520, 0xebe8ee, 0xede0ff, 0xede8ea, + 0xede8ff, 0xededff, 0xedeee2, 0xedffff, 0xeee1f1, 0xeee2ff, 0xeeebfc, + 0xeeece0, 0xeeeded, 0xeef0ee, 0xeef1e5, 0xeef1f2, 0xeef1fb, 0xeef2e5, + 0xeef2ee, 0xefe8f1, 0xefeef1, 0xf0eae0, 0xf0ece0, 0xf0eef1, 0xf1e5ed, + 0xf1edfb, 0xf1f2e0, 0xf1f2e5, 0xf1f2ee, 0xf1f3e6, 0xf1fb20, 0xf1ff20, + 0xf220ea, 0xf220ed, 0xf220ef, 0xf2e5ea, 0xf2eef0, 0xf2f0ee, 0xf2f1ff, + 0xf3e6e4, 0xf5eee4, 0xfb20ef, 0xfb2c20, 0xfb2e0a, 0xfbe2e0, 0xfbe920, + 0xff20ff, 0xfff0ec, 0xfffef2, 0xffff20, 0x0a0ac2, 0x0a0ac3, 0x0ac220, + 0x0ac3ee, 0x0acde0, + ], + }, + { + encoding: 'windows-1252', + language: 'da', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x676572, 0x6f6720, 0x64656e, 0x646572, + 0x6c6520, 0x6e6465, 0x737465, 0x74656b, 0x0a0a42, 0x206269, 0x206279, + 0x206465, 0x20686f, 0x2070e5, 0x2c206f, 0x626962, 0x626c69, 0x640a0a, + 0x652062, 0x652066, 0x65640a, 0x656e73, 0x65722c, 0x65722e, 0x65726e, + 0x69626c, 0x696465, 0x696c6c, 0x696f74, 0x6b7374, 0x6c696f, 0x6c6c65, + 0x6e6572, 0x6f7465, 0x70e520, 0x722065, 0x722c20, 0x722e0a, 0x726e65, + 0x736572, 0x746572, 0x766572, 0x79656e, 0xe52064, 0xf86765, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4ce6, 0x206272, 0x206275, + 0x2062f8, 0x206361, 0x20656c, 0x20656e, 0x206669, 0x20666f, 0x206672, + 0x2066f8, 0x206869, 0x20686a, 0x206920, 0x206b6f, 0x206c61, 0x206c69, + 0x206d65, 0x206d75, 0x206f6d, 0x206f73, 0x20706c, 0x20736b, 0x207374, + 0x2073e6, 0x207465, 0x207472, 0x207479, 0x20756e, 0x207665, 0x207669, + 0x2076e5, 0x2076e6, 0x20e562, 0x20e662, 0x2c2063, 0x2c206d, 0x2c2075, + 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, 0x457420, 0x4ce673, + 0x4cf872, 0x4d6f72, 0x61622c, 0x616473, 0x6166e9, 0x616773, 0x616c74, + 0x616e67, 0x617265, 0x61726b, 0x61766c, 0x622c20, 0x626c65, 0x626e65, + 0x6272f8, 0x627573, 0x627965, 0x627967, 0x62f867, 0x636166, 0x642062, + 0x642c20, 0x646167, 0x646520, 0x64652c, 0x64656c, 0x64732e, 0x652068, + 0x65206f, 0x652070, + ], + }, + { + encoding: 'windows-1252', + language: 'de', + documents: 4, + trigrams: [ + 0x656e20, 0x636865, 0x657220, 0x206465, 0x20756e, 0x696520, 0x736368, + 0x756e64, 0x65696e, 0x68656e, 0x696368, 0x696e65, 0x6e2064, 0x6e6420, + 0x6e656e, 0x206469, 0x626572, 0x646572, 0x687265, 0x6c6569, 0x205465, + 0x206569, 0x446965, 0x616368, 0x636874, 0x64656e, 0x646965, 0x652c20, + 0x656974, 0x65722e, 0x65726e, 0x686572, 0x696e20, 0x697461, 0x697465, + 0x6b6c65, 0x6c6572, 0x6e2042, 0x6e2069, 0x6e2e20, 0x6e6520, 0x726520, + 0x726569, 0x742065, 0x746520, 0x74656e, 0x204269, 0x204765, 0x204d65, + 0x205265, 0x205374, 0x206265, 0x2066fc, 0x206968, 0x206b6c, 0x206c65, + 0x207665, 0x20fc62, 0x2e2042, 0x2e2044, 0x426962, 0x45696e, 0x4d656e, + 0x537461, 0x616474, 0x616c65, 0x616d20, 0x617573, 0x626c69, 0x636872, + 0x636b65, 0x642066, 0x65204d, 0x652053, 0x652054, 0x652075, 0x656368, + 0x656c6c, 0x656e2e, 0x657265, 0x657277, 0x657320, 0x66fc72, 0x676974, + 0x686520, 0x68656b, 0x68656c, 0x687465, 0x69626c, 0x69636b, 0x696769, + 0x696872, 0x696f74, 0x6c6520, 0x6c6963, 0x6c696f, 0x6e206c, 0x6e2076, + 0x6e6465, 0x6e7465, 0x6e7477, 0x6f7468, 0x722047, 0x722053, 0x722064, + 0x722e0a, 0x722e20, 0x726e20, 0x736520, 0x73652c, 0x737461, 0x737465, + 0x737563, 0x740a0a, 0x742061, 0x746164, 0x74616c, 0x746572, 0x746865, + 0x747769, 0x756368, 0x757373, 0x766572, 0x776963, 0xe46e64, 0xfc6265, + 0xfc7220, 0x0a0a41, + ], + }, + { + encoding: 'windows-1252', + language: 'en', + documents: 4, + trigrams: [ + 0x20616e, 0x616e64, 0x686520, 0x6e6420, 0x727320, 0x207468, 0x2c2061, + 0x746865, 0x652061, 0x652c20, 0x656164, 0x657273, 0x657320, 0x732061, + 0x206120, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206c69, 0x206e65, + 0x207265, 0x207768, 0x546865, 0x616465, 0x626f75, 0x627261, 0x636974, + 0x642063, 0x64206e, 0x646520, 0x646573, 0x652063, 0x657369, 0x68696c, + 0x696272, 0x696465, 0x696c65, 0x697479, 0x6c6520, 0x6c6962, 0x6c6c20, + 0x6e2074, 0x722064, 0x726172, 0x726561, 0x732062, 0x732072, 0x732073, + 0x732c20, 0x732e0a, 0x776869, 0x790a0a, 0x0a0a41, 0x0a0a46, 0x0a0a52, + 0x0a0a54, 0x0a4120, 0x0a4661, 0x0a5265, 0x0a5468, 0x206162, 0x206170, + 0x206172, 0x206265, 0x20626f, 0x206361, 0x20636c, 0x206469, 0x20646f, + 0x206578, 0x206661, 0x206775, 0x206869, 0x20696e, 0x206974, 0x206a61, + 0x206d61, 0x206d75, 0x206e61, 0x206f70, 0x207072, 0x2070e2, 0x207175, + 0x2072e9, 0x2072f4, 0x207363, 0x207365, 0x20736c, 0x20736d, 0x207465, + 0x207761, 0x20776f, 0x2c2063, 0x2c206a, 0x2c2073, 0x3b2063, 0x412073, + 0x446967, 0x466172, 0x4d6f72, 0x526561, 0x536174, 0x612066, 0x612072, + 0x61626f, 0x616420, 0x61642c, 0x6166e9, 0x616b65, 0x616c20, 0x616c61, + 0x616c6c, 0x616d20, 0x616e20, 0x616e67, 0x617065, 0x617070, 0x617220, + 0x617265, 0x617269, 0x61726b, 0x61726d, 0x617274, 0x617279, 0x617475, + 0x617920, 0x61e761, + ], + }, + { + encoding: 'windows-1252', + language: 'es', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x617320, 0x656c20, 0x656e20, 0x657320, 0x206c61, + 0x206c6f, 0x207920, 0x2e204c, 0x61646f, 0x616e20, 0x6c6f73, 0x6e206c, + 0x6f7265, 0x726573, 0x706172, 0x732063, 0x204c6f, 0x206369, 0x20656c, + 0x207061, 0x4c6f73, 0x612065, 0x612070, 0x617261, 0x626c69, 0x6c206d, + 0x6c6120, 0x6e6120, 0x732061, 0x732e20, 0x746f72, 0x206269, 0x20636c, + 0x20636f, 0x206469, 0x206d65, 0x207265, 0x207375, 0x207665, 0x456c20, + 0x4c6120, 0x612062, 0x626962, 0x636961, 0x646520, 0x64656c, 0x646967, + 0x656361, 0x65732e, 0x69626c, 0x696369, 0x696f74, 0x6c2063, 0x6c696f, + 0x6e2065, 0x6f2073, 0x6f2e20, 0x6f7465, 0x726120, 0x732064, 0x732070, + 0x732076, 0x732e0a, 0x746563, 0x0a0a45, 0x204c61, 0x206361, 0x206c65, + 0x206d61, 0x20706f, 0x207072, 0x207365, 0x207465, 0x2c206c, 0x612063, + 0x612064, 0x61206c, 0x612079, 0x616e61, 0x61726f, 0x61732c, 0x61732e, + 0x627265, 0x627573, 0x636120, 0x636164, 0x636965, 0x636975, 0x636c61, + 0x636f2e, 0x636f6d, 0x646164, 0x64656e, 0x646f20, 0x646f72, 0x656369, + 0x656e61, 0x657175, 0x657263, 0x657274, 0x676974, 0x676f20, 0x696120, + 0x696173, 0x69656c, 0x69656e, 0x696769, 0x697461, 0x697564, 0x6c6172, + 0x6c6173, 0x6c6c65, 0x6d6572, 0x6d7061, 0x6e2070, 0x6e2072, 0x6e6173, + 0x6e6f73, 0x6f2064, 0x6f2079, 0x6f636f, 0x6f6d70, 0x6f732e, 0x706f63, + 0x717565, 0x717569, + ], + }, + { + encoding: 'windows-1252', + language: 'fr', + documents: 4, + trigrams: [ + 0x657320, 0x206c65, 0x656e74, 0x6c6573, 0x6e7420, 0x74206c, 0x732070, + 0x206465, 0x657420, 0x657572, 0x727320, 0x742064, 0x206475, 0x206574, + 0x647520, 0x6c6520, 0x732063, 0x757273, 0x204c65, 0x20636f, 0x207072, + 0x2e204c, 0x4c6573, 0x646573, 0x652063, 0x652070, 0x207061, 0x20706f, + 0x2c206c, 0x626c69, 0x65206d, 0x656e63, 0x65732e, 0x6c6c65, 0x6f7572, + 0x706172, 0x717565, 0x726520, 0x732e20, 0x746573, 0x756520, 0x757220, + 0x206269, 0x206361, 0x20656e, 0x206d61, 0x207361, 0x4c6520, 0x616765, + 0x617265, 0x626962, 0x637465, 0x646520, 0x650a0a, 0x652064, 0x652065, + 0x652073, 0x69626c, 0x69656e, 0x696c6c, 0x696f74, 0x6c6575, 0x6c696f, + 0x6d656e, 0x6e6365, 0x6e6520, 0x6f6e73, 0x6f7468, 0x7072e9, 0x72206c, + 0x726368, 0x72e970, 0x732065, 0x73206c, 0x732073, 0x746575, 0xe97061, + 0xe97269, 0x0a0a4c, 0x20636c, 0x206672, 0x206c61, 0x206e75, 0x207265, + 0x2072e9, 0x20756e, 0x207669, 0x2c2064, 0x4c6120, 0x556e65, 0x612062, + 0x616972, 0x616973, 0x617263, 0x636865, 0x6368e9, 0x636965, 0x636f6e, + 0x652e20, 0x656e20, 0x65732c, 0x676573, 0x686973, 0x68e871, 0x68e920, + 0x69656c, 0x696572, 0x696e20, 0x697175, 0x697265, 0x697374, 0x697420, + 0x6c6120, 0x6c656e, 0x6c6965, 0x6d6172, 0x6d6d65, 0x6de972, 0x6e2070, + 0x6e7320, 0x6e7473, 0x6e756d, 0x6f6d6d, 0x6f7576, 0x706f73, 0x706f75, + 0x70726f, 0x72656e, + ], + }, + { + encoding: 'windows-1252', + language: 'it', + documents: 4, + trigrams: [ + 0x6c6120, 0x207069, 0x656e74, 0x206465, 0x206520, 0x612070, 0x6c6c61, + 0x6e6f20, 0x6f7269, 0x746f72, 0x206369, 0x206c61, 0x612067, 0x612069, + 0x61746f, 0x636974, 0x64656c, 0x657263, 0x692061, 0x692063, 0x692073, + 0x696120, 0x697474, 0x69f920, 0x6c6920, 0x7069f9, 0x726920, 0x7474e0, + 0x206269, 0x206361, 0x20636f, 0x206469, 0x206765, 0x206920, 0x206c65, + 0x206d65, 0x207065, 0x207072, 0x207363, 0x207369, 0x207374, 0x2c2063, + 0x4c6120, 0x612062, 0x612063, 0x612064, 0x61206c, 0x616666, 0x616e6f, + 0x626962, 0x626c69, 0x636166, 0x636f6c, 0x637269, 0x65206c, 0x652073, + 0x652c20, 0x656361, 0x656c20, 0x656c65, 0x656c6c, 0x657363, 0x657474, + 0x6666e8, 0x67696f, 0x676c69, 0x692070, 0x69626c, 0x696f20, 0x696f74, + 0x6c696f, 0x6d656e, 0x6e6120, 0x6e7465, 0x6e7472, 0x6f2064, 0x6f2065, + 0x6f2070, 0x6f2c20, 0x6f6e6f, 0x6f7465, 0x706572, 0x726120, 0x726361, + 0x726520, 0x726961, 0x736372, 0x746120, 0x746563, 0x7a6120, 0x0a0a47, + 0x0a0a49, 0x0a0a4c, 0x0a0a55, 0x0a476c, 0x0a4920, 0x0a4c61, 0x0a556e, + 0x206120, 0x206167, 0x206169, 0x206170, 0x206173, 0x206175, 0x206365, + 0x206368, 0x20666f, 0x206672, 0x206769, 0x20676c, 0x206775, 0x20696c, + 0x20696e, 0x206c69, 0x206d69, 0x206d75, 0x206e65, 0x207061, 0x207361, + 0x207371, 0x207376, 0x207472, 0x207574, 0x207665, 0x2c2066, 0x2c2067, + 0x2c2069, 0x2c206d, + ], + }, + { + encoding: 'windows-1252', + language: 'nl', + documents: 4, + trigrams: [ + 0x656e20, 0x20656e, 0x646520, 0x206465, 0x656572, 0x74656e, 0x766572, + 0x206f70, 0x207374, 0x207465, 0x652062, 0x652065, 0x652073, 0x657273, + 0x6e206f, 0x6e2e0a, 0x736368, 0x742064, 0x746572, 0x206269, 0x206361, + 0x20636f, 0x206565, 0x206765, 0x206865, 0x206c65, 0x206f76, 0x207665, + 0x207765, 0x2c2063, 0x446520, 0x61616b, 0x616173, 0x6166e9, 0x616b74, + 0x626962, 0x626c69, 0x636166, 0x636874, 0x652063, 0x65696e, 0x656b65, + 0x656e2e, 0x656e73, 0x657220, 0x657264, 0x657265, 0x65726b, 0x66e973, + 0x687465, 0x69626c, 0x696465, 0x696e20, 0x696e67, 0x696f74, 0x697320, + 0x6b6520, 0x6b656e, 0x6b7420, 0x6c6520, 0x6c6565, 0x6c6569, 0x6c696f, + 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206d, 0x6e2074, 0x6e2076, 0x6e2077, + 0x6f656b, 0x6f6e74, 0x6f7020, 0x6f7065, 0x6f7468, 0x6f7665, 0x70656e, + 0x72656e, 0x727320, 0x732065, 0x73206f, 0x732c20, 0x73656e, 0x737365, + 0x737461, 0x737465, 0x746164, 0x746865, 0xe97320, 0x0a0a42, 0x0a0a44, + 0x0a0a45, 0x0a0a4c, 0x0a426f, 0x0a4465, 0x0a4565, 0x0a4c65, 0x206170, + 0x20626f, 0x206272, 0x206275, 0x206472, 0x206475, 0x20686f, 0x206875, + 0x206964, 0x20696e, 0x206b61, 0x206b6c, 0x206c61, 0x206d61, 0x206d65, + 0x206d75, 0x206e61, 0x206f6e, 0x20706c, 0x20706f, 0x207363, 0x207477, + 0x207761, 0x207a6f, 0x20e9e9, 0x2c2061, 0x2c206b, 0x426f65, 0x446967, + 0x45656e, 0x4c657a, + ], + }, + { + encoding: 'windows-1252', + language: 'no', + documents: 4, + trigrams: [ + 0x657220, 0x656e20, 0x206f67, 0x6f6720, 0x656e65, 0x206c61, 0x652066, + 0x676572, 0x6e6520, 0x6e6572, 0x737465, 0x74656b, 0x0a0a42, 0x206269, + 0x206272, 0x206465, 0x20666f, 0x2070e5, 0x207665, 0x626962, 0x626c69, + 0x640a0a, 0x646572, 0x652062, 0x652074, 0x652c20, 0x65640a, 0x656e73, + 0x657273, 0x657420, 0x672073, 0x676520, 0x67656e, 0x69626c, 0x696765, + 0x696e67, 0x696f74, 0x697465, 0x6b6572, 0x6b7374, 0x6c6167, 0x6c6520, + 0x6c6967, 0x6c696f, 0x6e206f, 0x6e6465, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x72206f, 0x72656e, 0x726765, 0x746520, 0x746572, + 0x746f72, 0x76656e, 0x766572, 0x79656e, 0xe52064, 0xf86b65, 0x0a0a45, + 0x0a0a4c, 0x0a4279, 0x0a42f8, 0x0a4574, 0x0a4c65, 0x206275, 0x206279, + 0x2062f8, 0x2064f8, 0x20656c, 0x20656e, 0x206570, 0x206665, 0x206669, + 0x2066f8, 0x206869, 0x20686a, 0x20686f, 0x206920, 0x206b61, 0x206b6f, + 0x206c69, 0x206d65, 0x206d75, 0x206ee6, 0x206f6d, 0x206f73, 0x207365, + 0x20736a, 0x20736b, 0x207374, 0x207465, 0x20746f, 0x207472, 0x207479, + 0x20756e, 0x207669, 0x2076e5, 0x20e570, 0x20e672, 0x2c2065, 0x2c206b, + 0x2c206f, 0x2c2075, 0x2c20e6, 0x427965, 0x42f86e, 0x446574, 0x446967, + 0x457420, 0x4c6573, 0x4cf872, 0x4d6f72, 0x616665, 0x616720, 0x616765, + 0x616773, 0x616c74, 0x616e67, 0x617020, 0x617265, 0x61726b, 0x61766c, + 0x627279, 0x6272f8, + ], + }, + { + encoding: 'windows-1252', + language: 'pt', + documents: 4, + trigrams: [ + 0x6f7320, 0x206465, 0x206520, 0x207072, 0x696120, 0x6e6120, 0x206369, + 0x206f73, 0x612064, 0x612065, 0x61206f, 0x646520, 0x657320, 0x657363, + 0x726573, 0x732061, 0x732070, 0x732e0a, 0x0a0a4f, 0x0a4f73, 0x206269, + 0x206573, 0x206e61, 0x206f20, 0x207065, 0x207665, 0x4f7320, 0x612070, + 0x616465, 0x61646f, 0x616d20, 0x617261, 0x617320, 0x626962, 0x626c69, + 0x636120, 0x636964, 0x646120, 0x646164, 0x646967, 0x646f20, 0x652061, + 0x656d20, 0x657175, 0x666963, 0x69626c, 0x696461, 0x696d65, 0x696f74, + 0x6c696f, 0x6e7175, 0x6f2065, 0x6f7265, 0x6f7465, 0x717565, 0x717569, + 0x72616d, 0x726961, 0x726f73, 0x732063, 0x736372, 0x746563, 0x746f72, + 0x0a0a41, 0x0a0a55, 0x0a4120, 0x0a556d, 0x206120, 0x206162, 0x206163, + 0x206167, 0x20616a, 0x20616c, 0x206173, 0x206361, 0x20636c, 0x2063f3, + 0x206469, 0x20646f, 0x20656e, 0x206571, 0x206672, 0x206775, 0x206869, + 0x206c65, 0x206c69, 0x206d61, 0x206d65, 0x206d6f, 0x206d75, 0x2070e3, + 0x207175, 0x2073e1, 0x207472, 0x20f46e, 0x2c206d, 0x2c206f, 0x2c2071, + 0x2c2076, 0x412062, 0x412063, 0x4d616e, 0x4d6572, 0x4f6669, 0x556d61, + 0x610a0a, 0x612061, 0x612062, 0x612063, 0x61206d, 0x612074, 0x612e0a, + 0x616272, 0x61636f, 0x616461, 0x6166e9, 0x616761, 0x616772, 0x616a75, + 0x616c0a, 0x616c75, 0x616e68, 0x616e71, 0x616e74, 0x61722c, 0x61732e, + 0x61e761, 0x61e7e3, + ], + }, + { + encoding: 'windows-1252', + language: 'sv', + documents: 4, + trigrams: [ + 0x657220, 0x617220, 0x206f63, 0x636820, 0x656e20, 0x6f6368, 0x737461, + 0x64656e, 0x657420, 0x6b6172, 0x726e61, 0x736b61, 0x74206c, 0x746164, + 0x766572, 0x206269, 0x206465, 0x2066f6, 0x206c69, 0x2070e5, 0x20736b, + 0x207479, 0x207665, 0x612062, 0x612074, 0x612e0a, 0x61640a, 0x616465, + 0x617265, 0x626962, 0x626c69, 0x640a0a, 0x65726e, 0x66f672, 0x676120, + 0x69626c, 0x696761, 0x696f74, 0x6b6170, 0x6b6572, 0x6b6e61, 0x6c6967, + 0x6c696f, 0x6e612e, 0x6e6172, 0x6e736b, 0x6f7267, 0x6f7465, 0x70e520, + 0x722062, 0x722065, 0x722066, 0x72206f, 0x736172, 0x73f66b, 0x746120, + 0x74656b, 0x746f72, 0xe52064, 0x0a0a42, 0x0a0a45, 0x0a0a4c, 0x0a0a53, + 0x0a42f6, 0x0a4574, 0x0a4ce4, 0x0a5374, 0x206265, 0x206272, 0x206275, + 0x2062f6, 0x20656c, 0x20656e, 0x2066e4, 0x206772, 0x206869, 0x20686a, + 0x20686f, 0x206920, 0x206b61, 0x206b6f, 0x206c61, 0x206ce5, 0x206d65, + 0x206d75, 0x206de4, 0x206f6d, 0x206f73, 0x207374, 0x2073e4, 0x2073f6, + 0x207465, 0x20746f, 0x207661, 0x2076e4, 0x20e470, 0x20f670, 0x2c2067, + 0x2c206b, 0x2c206f, 0x2c20e4, 0x42f66e, 0x446574, 0x446967, 0x457474, + 0x4ce473, 0x4cf672, 0x4d6f72, 0x537461, 0x61206f, 0x6120f6, 0x6166e9, + 0x616720, 0x616773, 0x616b6e, 0x616c20, 0x616d74, 0x616e20, 0x616e73, + 0x617020, 0x617061, 0x617269, 0x61726b, 0x61726e, 0x626573, 0x6272f6, + 0x627573, 0x62f663, + ], + }, + { + encoding: 'windows-1253', + language: 'el', + documents: 4, + trigrams: [ + 0x20eae1, 0xe1e920, 0xeae1e9, 0x20e2e9, 0xe120ea, 0xe12e0a, 0xe2e9e2, + 0xe5e920, 0xe9e2eb, 0xeff5ed, 0xf4e120, 0xf5ed20, 0x0a0acf, 0x0acfe9, + 0x20e1e3, 0x20e1ed, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xcfe920, + 0xdfe12e, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, 0xe5f220, 0xe70a0a, 0xe720e2, + 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9efe8, 0xe9f3f4, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xf0fceb, 0xf1e9ef, 0xf220ea, 0xf3f4de, + 0xf3f4e7, 0xf4e5f2, 0xf4e7ed, 0xfcebe7, 0x0a0ac7, 0x0a0acc, 0x0ac720, + 0x0acce9, 0x20dced, 0x20e1f1, 0x20e2ef, 0x20e3f1, 0x20e5eb, 0x20e5f0, + 0x20e5f1, 0x20e5f4, 0x20e720, 0x20e9f3, 0x20eae5, 0x20eafe, 0x20ebe5, + 0x20ecdd, 0x20ece1, 0x20ece9, 0x20ecef, 0x20ecf5, 0x20eef5, 0x20efe4, + 0x20efe9, 0x20efec, 0x20f0e5, 0x20f0eb, 0x20f0ef, 0x20f6f1, 0x20f8dc, + 0x20f8f9, 0x2c20e5, 0x2c20ec, 0x2c20f4, 0x2c20f6, 0xb9f3f5, 0xc720f0, + 0xcce9e1, 0xd0f1f9, 0xd3e1e2, 0xd8e7f6, 0xdc0a0a, 0xdc20e1, 0xdc20ea, + 0xdc20f3, 0xdc20f4, 0xdc2c20, 0xdce4e1, 0xdce6e5, 0xdcede8, 0xdcf1e9, + 0xdcf4e9, 0xdcf6e5, 0xdcf7ed, 0xdd20e1, 0xdde3f7, 0xddebe9, 0xddedef, + 0xddf22e, 0xde20ef, 0xdeeae7, 0xdeece7, 0xdef1e9, 0xdf20f3, 0xdf2c20, + 0xdfe120, 0xdfe1f2, 0xdfe3ef, 0xdfece5, 0xdfeff5, 0xe120e5, 0xe120e9, + 0xe120eb, 0xe120ec, 0xe1e2e2, 0xe1e3ed, 0xe1e3ef, 0xe1e3f1, 0xe1e8e1, + 0xe1e8e7, 0xe1eafc, + ], + }, + { + encoding: 'windows-1254', + language: 'tr', + documents: 4, + trigrams: [ + 0x6c6572, 0x6c6172, 0x207665, 0x722e0a, 0x766520, 0x2061e7, 0x207961, + 0x617220, 0x61e7fd, 0x657220, 0x697220, 0x722076, 0x206269, 0x20656b, + 0x206b61, 0x206bfc, 0x207361, 0x207461, 0x2c206b, 0x616261, 0x616c20, + 0x616c61, 0x616e65, 0x61722e, 0x6172fd, 0x617a61, 0x62616c, 0x650a0a, + 0x652062, 0x656869, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697461, + 0x6bfc74, 0x706861, 0x722079, 0x722c20, 0x726920, 0x726c61, 0x746172, + 0x74fc70, 0x7a6172, 0x7a6520, 0xde6568, 0xe7fd6b, 0xfc7068, 0xfc74fc, + 0xfd6b20, 0x0a0a4b, 0x0a0a4f, 0x0a0ac7, 0x0a0ade, 0x0a4bfc, 0x0a4f6b, + 0x0ac769, 0x0ade65, 0x206172, 0x206174, 0x206261, 0x206265, 0x206465, + 0x20656c, 0x206861, 0x20696c, 0x20696e, 0x206b69, 0x206b6f, 0x206d65, + 0x206dfc, 0x206f6c, 0x206f74, 0x207061, 0x207065, 0x207265, 0x207579, + 0x20f6f0, 0x2c2065, 0x2c2070, 0x43756d, 0x44696a, 0x4bfce7, 0x4f6b75, + 0x536573, 0x612074, 0x612075, 0x612c20, 0x616665, 0x61680a, 0x616cfd, + 0x616d61, 0x616e64, 0x616e6c, 0x616efd, 0x61706c, 0x61722c, 0x617261, + 0x617264, 0x617269, 0x61726b, 0x617274, 0x617461, 0x6174f6, 0x617661, + 0x617a65, 0x617afd, 0x61fee7, 0x626168, 0x62656b, 0x626572, 0x62696c, + 0x626972, 0x62fc73, 0x636920, 0x63696c, 0x63fd20, 0x646120, 0x64616e, + 0x646520, 0x64656e, 0x647520, 0x64fd6d, 0x652061, 0x652065, 0x652069, + 0x652072, 0x652073, + ], + }, + { + encoding: 'windows-1255', + language: 'he', + documents: 4, + trigrams: [ + 0xe9ed20, 0xe5f8e9, 0xed20ec, 0xed20ee, 0xf1f4f8, 0xf8e9ed, 0x20e4f7, + 0x20ece0, 0x20eee7, 0x2c20e1, 0xe0e9ed, 0xe4f1f4, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed2e, 0xed2e0a, 0xef20ee, 0xf0e420, 0xf2e9f8, + 0xf4f8e9, 0xf7e5f8, 0xf820ee, 0xf8e5fa, 0xf8fa20, 0xf9e9ed, 0xfa0a0a, + 0xfa20ec, 0x0a0ae4, 0x0a0ae7, 0x0a0af6, 0x0a0af7, 0x0ae4f2, 0x0ae7f7, + 0x0af6e5, 0x0af7e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, + 0x20e1fa, 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e8f8, 0x20ece1, + 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, 0x20eeeb, 0x20eef7, + 0x20eefa, 0x20f0f4, 0x20f1f4, 0x20f2e5, 0x20f4e9, 0x20f7e8, 0x20f9e1, + 0x20fae9, 0x2c20e3, 0x2c20f4, 0xe0e5e8, 0xe0e5ef, 0xe0e5f8, 0xe0e82c, + 0xe0f0f9, 0xe0f9e5, 0xe0fa20, 0xe120fa, 0xe1e5e3, 0xe1e5f1, 0xe1e5f7, + 0xe1e7e5, 0xe1e9f0, 0xe1ebe9, 0xe1f2e9, 0xe1f8e5, 0xe1f92c, 0xe1fa0a, + 0xe1fae9, 0xe2e1e9, 0xe2e9e8, 0xe320e5, 0xe3e1f9, 0xe3e9e2, 0xe3e9ed, + 0xe3f0e4, 0xe3f220, 0xe3f720, 0xe3f8e9, 0xe40a0a, 0xe420e1, 0xe420e3, + 0xe420e4, 0xe420e5, 0xe420f0, 0xe42e0a, 0xe4e9f1, 0xe4f2e9, 0xe4f2ee, + 0xe4f7e5, 0xe4f7f4, 0xe4f8e0, 0xe4f9f7, 0xe5e0f0, 0xe5e1e5, 0xe5e2e1, + 0xe5e320, 0xe5e3f7, 0xe5e4f1, 0xe5e5fa, 0xe5e6e9, 0xe5e6f8, 0xe5e8e5, + 0xe5ebe5, 0xe5ebf8, + ], + }, + { + encoding: 'windows-1256', + language: 'ar', + documents: 4, + trigrams: [ + 0x20c7e1, 0xc7e1e3, 0x20e6c7, 0x20e6ed, 0xdfcac8, 0xe420c7, 0xe6c7e1, + 0xe6e420, 0x0a0aed, 0xc90a0a, 0xd120c7, 0xc7ca20, 0xc7ccda, 0xc7d120, + 0xc7e1c3, 0xc7e1da, 0xc7e1e6, 0xc820c7, 0xc8c920, 0xc920c7, 0xc920e6, + 0xc92e0a, 0xca20e6, 0xcac820, 0xcac8c9, 0xcb20c7, 0xccda20, 0xcd20c7, + 0xcfede4, 0xd1c7cc, 0xd5ddc7, 0xda20c7, 0xe1e3cf, 0xe1e3df, 0xe3cfed, + 0xe3dfca, 0xe4c7d3, 0xe6d5dd, 0xede4c9, 0x0a0aca, 0x0acad3, 0x0aedc8, + 0x0aedd5, 0x0aedda, 0x20c3c8, 0x20c3e3, 0x20c7ce, 0x20c8e5, 0x20c8ed, + 0x20cfe1, 0x20d1de, 0x20d5db, 0x20dae4, 0x20ddd1, 0x20dded, 0x20dfca, + 0x20e1e1, 0x20e6ca, 0x20e6d5, 0x20ede4, 0xc120da, 0xc120e6, 0xc3c8e6, + 0xc3cdc7, 0xc3cec8, 0xc3e3ed, 0xc4e6e4, 0xc6c90a, 0xc720c8, 0xc720e1, + 0xc720e6, 0xc720ed, 0xc72e0a, 0xc7c120, 0xc7c820, 0xc7c8e5, 0xc7cd20, + 0xc7ceca, 0xc7cfc6, 0xc7cfe1, 0xc7cfed, 0xc7d12e, 0xc7d1da, 0xc7d1ed, + 0xc7d320, 0xc7d3c8, 0xc7d6cd, 0xc7dacf, 0xc7dde1, 0xc7dfe5, 0xc7e1ca, + 0xc7e1cd, 0xc7e1ce, 0xc7e1d2, 0xc7e1d3, 0xc7e1d4, 0xc7e1d8, 0xc7e1dd, + 0xc7e1de, 0xc7e1e4, 0xc7e1e5, 0xc7e5ed, 0xc820dd, 0xc8c7cd, 0xc8c7cf, + 0xc8c7d1, 0xc8c92e, 0xc8ca0a, 0xc8cdcb, 0xc8d1e3, 0xc8d220, 0xc8e5c7, + 0xc8e5cf, 0xc8e6c7, 0xc8ede4, 0xc920c8, 0xc920d1, 0xca0a0a, 0xcac7d1, + 0xcac8c7, 0xcacd20, 0xcacddd, 0xcad3ca, 0xcad9d1, 0xcaddca, 0xcaedc7, + 0xcaedde, 0xcce6e4, + ], + }, + { + encoding: 'windows-1257', + language: 'et', + documents: 4, + trigrams: [ + 0x616420, 0x6a6120, 0x206a61, 0x206b6f, 0x616220, 0x766164, 0x207261, + 0x61616d, 0x616d61, 0x617475, 0x617661, 0x6d6174, 0x6e6520, 0x726161, + 0x746162, 0x0a0a4c, 0x206b69, 0x206d65, 0x2076e4, 0x2c206b, 0x642061, + 0x64206b, 0x64206f, 0x647573, 0x652074, 0x656420, 0x657369, 0x692e0a, + 0x696420, 0x696b75, 0x696d65, 0x696e6e, 0x69726a, 0x697461, 0x6b6972, + 0x6b6f67, 0x6b6f6e, 0x6b6f6f, 0x6b7564, 0x6c6173, 0x6d6573, 0x6f6775, + 0x736564, 0x73656c, 0x73692e, 0x74756b, 0x756420, 0x756b6f, 0x757573, + 0x0a0a54, 0x0a0a56, 0x0a4c69, 0x0a4c75, 0x0a5461, 0x0a56e4, 0x206165, + 0x206169, 0x20616a, 0x206176, 0x206275, 0x206573, 0x20696e, 0x206a75, + 0x206c65, 0x206c69, 0x206d75, 0x206dfc, 0x206e69, 0x206f6f, 0x206f74, + 0x207365, 0x207465, 0x207475, 0x2074f6, 0x20756b, 0x20e472, 0x20f570, + 0x20f575, 0x2c206a, 0x2c20f5, 0x2d206a, 0x446967, 0x486f6d, 0x4c6175, + 0x4c696e, 0x4c7567, 0x54616c, 0x566169, 0x56e469, 0x610a0a, 0x612061, + 0x612069, 0x61206b, 0x61206d, 0x612074, 0x612c20, 0x61616c, 0x616475, + 0x616567, 0x616876, 0x61696b, 0x616974, 0x616a61, 0x616b75, 0x616c6e, + 0x616c6f, 0x616c75, 0x616e65, 0x61726f, 0x61730a, 0x617365, 0x617369, + 0x617570, 0x622061, 0x62206b, 0x62206d, 0x622073, 0x6220f5, 0x62610a, + 0x62612c, 0x627573, 0x642065, 0x64206a, 0x64206d, 0x64206e, 0x642075, + 0x642076, 0x646920, + ], + }, + { + encoding: 'windows-1257', + language: 'lt', + documents: 4, + trigrams: [ + 0x616920, 0x6b6169, 0x206972, 0x207265, 0x612070, 0x616974, 0x646120, + 0x656e69, 0x696169, 0x696e69, 0x697220, 0x6c696f, 0x6e696e, 0x726961, + 0x732e0a, 0x757320, 0x206461, 0x207072, 0x617573, 0x626c69, 0x656461, + 0x692072, 0x696120, 0x69626c, 0x696573, 0x696f74, 0x6a6169, 0x6b696e, + 0x6d656e, 0x6eeb73, 0x6f7465, 0x732061, 0x732c20, 0x732e20, 0x74656b, + 0xeb7320, 0x206169, 0x206170, 0x206269, 0x206475, 0x206b6f, 0x206d65, + 0x206d6f, 0x206f20, 0x207061, 0x20736b, 0x20f076, 0x2c206d, 0x2c206f, + 0x536b61, 0x612064, 0x61206b, 0x61692e, 0x617069, 0x617320, 0x617369, + 0x626962, 0x657269, 0x657374, 0x677573, 0x67f820, 0x692074, 0x692e20, + 0x696e6b, 0x696eeb, 0x696f20, 0x69746d, 0x697573, 0x6b6f20, 0x6d6f6b, + 0x6d756f, 0x6e6461, 0x6e6961, 0x6e6b61, 0x6f20fe, 0x6f6275, 0x6f64e0, + 0x6f6a61, 0x6f6e6f, 0x6f7269, 0x6f7320, 0x707261, 0x72206b, 0x7261f0, + 0x72656e, 0x730a0a, 0x732064, 0x732069, 0x73206d, 0x736920, 0x736b61, + 0x73746f, 0x746173, 0x746d65, 0x746f6a, 0x746f72, 0x757267, 0x757269, + 0x75732c, 0x75732e, 0x766965, 0x796c69, 0xe02c20, 0xeb6a61, 0xf07669, + 0x0a0a4d, 0x0a0a4e, 0x0a0a52, 0x0a0a54, 0x0a4d69, 0x0a4e65, 0x0a5261, + 0x0a5475, 0x204269, 0x204469, 0x204b61, 0x205069, 0x20536b, 0x205669, + 0x206174, 0x206175, 0x206275, 0x206469, 0x206965, 0x206973, 0x206b65, + 0x206b6e, 0x206b75, + ], + }, + { + encoding: 'windows-1257', + language: 'lv', + documents: 4, + trigrams: [ + 0x20756e, 0x617320, 0x756e20, 0x207069, 0x61732e, 0x757320, 0x20697a, + 0x206c61, 0x61206b, 0x626c69, 0x646120, 0x692073, 0x692e20, 0x69626c, + 0x69656d, 0x696f74, 0x6a6920, 0x6b7374, 0x6c696f, 0x6c7573, 0x732c20, + 0x732e0a, 0x732e20, 0x206269, 0x206365, 0x206461, 0x206761, 0x206b6f, + 0x206d61, 0x206d65, 0x2070e2, 0x207265, 0x20736b, 0x207a69, 0x610a0a, + 0x616b73, 0x617262, 0x61756b, 0x626962, 0x627573, 0x6365ef, 0x646172, + 0x64752c, 0x656d2e, 0x676974, 0x677573, 0x692070, 0x692c20, 0x69656e, + 0x696769, 0x696c73, 0x697267, 0x6974e2, 0x697a76, 0x6a756d, 0x6b6920, + 0x6be272, 0x6c6173, 0x6c73e7, 0x6ce76e, 0x6d6172, 0x6d6f73, 0x6e2063, + 0x6e2073, 0x6e6965, 0x6eee63, 0x6f2069, 0x6f74e7, 0x706965, 0x70696c, + 0x70e272, 0x72616b, 0x726775, 0x726920, 0x727669, 0x732061, 0x73206c, + 0x73206d, 0x732070, 0x732075, 0x736920, 0x737420, 0x737461, 0x737475, + 0x73e774, 0x746120, 0x747573, 0x74e26a, 0x74e26c, 0x74e76b, 0x752c20, + 0x75732c, 0x757369, 0x76e76c, 0x7a76e7, 0xe26a69, 0xe76e69, 0xe774e2, + 0xee6361, 0x0a0a4e, 0x0a0a50, 0x0a0a52, 0x0a0a54, 0x0a4e65, 0x0a5069, + 0x0a5261, 0x0a5469, 0x204269, 0x204469, 0x204b61, 0x204c61, 0x205069, + 0x205669, 0x205a65, 0x206170, 0x206172, 0x206174, 0x206175, 0x206265, + 0x206369, 0x206465, 0x206469, 0x206475, 0x2064e2, 0x206772, 0x206a75, + 0x206b6c, 0x206b72, + ], + }, + { + encoding: 'windows-1258', + language: 'vi', + documents: 4, + trigrams: [ + 0x6e6720, 0x207468, 0x206368, 0x207068, 0x2076e0, 0x672074, 0x6e6820, + 0x76e020, 0x2064e2, 0x207469, 0x207472, 0x686ff2, 0x6e2062, 0x6e2074, + 0x7068f4, 0x7468fd, 0xf27420, 0x206261, 0x2062e1, 0x206769, 0x20686f, + 0x206c69, 0x206d61, 0x206e68, 0x207175, 0x2073e1, 0x207461, 0x2074e0, + 0x207669, 0x61d220, 0x6261d2, 0x62e16e, 0x636820, 0x672064, 0x6720f0, + 0x682070, 0x68e06e, 0x68f4ec, 0x68fd20, 0x68fdec, 0x69eaf2, 0x6f6e67, + 0x6ff263, 0x746875, 0x7669ea, 0xd22074, 0xd26e67, 0xe0206d, 0xe06e67, + 0xe06e68, 0xe16e20, 0xe2f274, 0xec0a0a, 0xec7420, 0xf26320, 0xf26920, + 0xf26e20, 0xf46e67, 0xf4ec0a, 0xfdf5cc, 0x0a0a4d, 0x0a0a4e, 0x0a0a54, + 0x0a0ad0, 0x0a4df4, 0x0a4ef4, 0x0a5468, 0x0ad0f4, 0x206275, 0x20636f, + 0x2063e0, 0x2063fd, 0x2068fd, 0x206b68, 0x206b69, 0x206b79, 0x206ce0, + 0x206d69, 0x206d6f, 0x206de2, 0x206df4, 0x206df5, 0x206e67, 0x206f6e, + 0x207261, 0x20726f, 0x2072e0, 0x207369, 0x2073f4, 0x2073fd, 0x2074e1, + 0x207865, 0x2079ea, 0x20f0e2, 0x20f0f4, 0x20f0fa, 0x2c2063, 0x2c206b, + 0x2c2070, 0x2c2071, 0x2c2074, 0x4275f4, 0x4368f5, 0x4df4f2, 0x4ef46e, + 0x5468e0, 0x5468fd, 0x58fdf5, 0x612068, 0x61206d, 0x612076, 0x616920, + 0x61d26e, 0x61d26f, 0x61d279, 0x61de20, 0x61de69, 0x61f269, 0x627579, + 0x632064, 0x632067, 0x632073, 0x632c20, 0x632e0a, 0x63686f, 0x636875, + 0x6368e2, 0x6368f5, + ], + }, + { + encoding: 'windows-874', + language: 'th', + documents: 4, + trigrams: [ + 0xe1c5d0, 0xb7d5e8, 0xc1d7cd, 0x20e1c5, 0xb5c3ec, 0xbbc3d0, 0xbcd9e9, + 0xc8d2ca, 0xca20e1, 0xcab5c3, 0xcad2c3, 0xcbe9cd, 0xd1a7ca, 0xd2cab5, + 0xd5c2b9, 0xd7cda7, 0xe0c1d7, 0xe9cda7, 0x0a0aaa, 0x0a0ab7, 0x0a0abc, + 0x0a0ae0, 0x0aaad2, 0x0ab7d5, 0x0abcd9, 0x0ae0c1, 0x20aad5, 0x20b5c3, + 0x20b5d7, 0x20bac3, 0x20bcc5, 0x20c3e9, 0x20e620, 0xa1a2e9, 0xa1a4d1, + 0xa1a8d1, 0xa1c9ec, 0xa1d2e1, 0xa1e0c3, 0xa2b9c1, 0xa2d2c2, 0xa2d5c2, + 0xa2e9cd, 0xa4b9c3, 0xa4d1a1, 0xa4d3cd, 0xa4d6a1, 0xa4d9e8, 0xa4e8cd, + 0xa4e9b9, 0xa70a0a, 0xa7a4e8, 0xa7b7d3, 0xa7ba0a, 0xa7cab4, 0xa7cac1, + 0xa7cad7, 0xa7d2b9, 0xa7d5c2, 0xa7e3b9, 0xa8b90a, 0xa8c3cb, 0xa8d1b4, + 0xa8d4b7, 0xaad1b4, 0xaad2c7, 0xaad5ca, 0xaae8c7, 0xaae9d2, 0xb1ec20, + 0xb3b1ec, 0xb3d2c3, 0xb420bc, 0xb4b7d3, 0xb4bbc3, 0xb4c2ca, 0xb4c7d1, + 0xb4d4a8, 0xb4e0a7, 0xb4e0a8, 0xb5c3c7, 0xb5c5d2, 0xb5d4c8, 0xb5d7e8, + 0xb5d9e1, 0xb6e2b4, 0xb7c2d2, 0xb7d1c5, 0xb7d3a4, 0xb7d3a7, 0xb7d5c1, + 0xb8c0d1, 0xb8d4ba, 0xb920c3, 0xb9a1d2, 0xb9a4d3, 0xb9a4e9, 0xb9b4d4, + 0xb9b7d5, 0xb9c1bb, 0xb9c3cd, 0xb9c5d2, 0xb9cbd2, 0xb9d1a1, 0xb9d1a7, + 0xb9d2a2, 0xb9e0c1, 0xb9e0c5, 0xb9e0ca, 0xb9e9d3, 0xba0a0a, 0xbac3c3, + 0xbacaa7, 0xbad2c2, 0xbbd1a7, 0xbbd4b4, 0xbcc5e4, 0xbcd6e9, 0xbed4b8, + 0xbed4be, 0xbfe0bb, 0xc0d1b3, 0xc1bbd1, 0xc1d8b4, 0xc1d9c5, 0xc1e0c5, + 0xc1e920, 0xc220e6, + ], + }, { encoding: 'x-mac-cyrillic', language: 'ru', diff --git a/corpus/generated/windows-1250/cs/test/community-garden.bin b/corpus/generated/windows-1250/cs/test/community-garden.bin new file mode 100644 index 0000000..61bfe36 --- /dev/null +++ b/corpus/generated/windows-1250/cs/test/community-garden.bin @@ -0,0 +1,3 @@ +Sousedská zahrada + +Sousedé promìnili prázdný dvùr v zelenou zahradu, dìti namalovaly barevné znaèky a rodiny zalévají rostliny. diff --git a/corpus/generated/windows-1250/cs/train/city-morning.bin b/corpus/generated/windows-1250/cs/train/city-morning.bin new file mode 100644 index 0000000..6efcbfe --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/city-morning.bin @@ -0,0 +1,3 @@ +Ráno ve mìstì + +Mìsto se pomalu probouzí, kavárny otevírají dveøe a lidé èekají na první autobusy. diff --git a/corpus/generated/windows-1250/cs/train/library.bin b/corpus/generated/windows-1250/cs/train/library.bin new file mode 100644 index 0000000..2d9442e --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/library.bin @@ -0,0 +1,3 @@ +Tichá knihovna + +Ètenáøi hledají knihy o historii a vìdì, zatímco knihovnice radí žákùm. diff --git a/corpus/generated/windows-1250/cs/train/market.bin b/corpus/generated/windows-1250/cs/train/market.bin new file mode 100644 index 0000000..f274a31 --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/market.bin @@ -0,0 +1,3 @@ +Sobotní trh + +Farmáøi prodávají èerstvý chléb, med, ovoce a voòavé byliny. diff --git a/corpus/generated/windows-1250/cs/train/workshop.bin b/corpus/generated/windows-1250/cs/train/workshop.bin new file mode 100644 index 0000000..78a9f82 --- /dev/null +++ b/corpus/generated/windows-1250/cs/train/workshop.bin @@ -0,0 +1,3 @@ +Digitální dílna + +Malý tým pøipravuje prùvodce muzeem, kontroluje kód a píše jasné popisy. diff --git a/corpus/generated/windows-1250/cs/validation/river-trip.bin b/corpus/generated/windows-1250/cs/validation/river-trip.bin new file mode 100644 index 0000000..dc3d158 --- /dev/null +++ b/corpus/generated/windows-1250/cs/validation/river-trip.bin @@ -0,0 +1,3 @@ +Výlet k øece + +Pøátelé vzali mapu a teplý èaj, prošli lesem a odpoèívali u øeky. diff --git a/corpus/generated/windows-1250/hu/test/community-garden.bin b/corpus/generated/windows-1250/hu/test/community-garden.bin new file mode 100644 index 0000000..251d1d3 --- /dev/null +++ b/corpus/generated/windows-1250/hu/test/community-garden.bin @@ -0,0 +1,3 @@ +Közösségi kert + +A szomszédok zöld kertté alakították az üres udvart, a gyerekek táblákat festettek, a családok pedig öntözik a növényeket. diff --git a/corpus/generated/windows-1250/hu/train/city-morning.bin b/corpus/generated/windows-1250/hu/train/city-morning.bin new file mode 100644 index 0000000..74c3adc --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/city-morning.bin @@ -0,0 +1,3 @@ +Reggel a városban + +A város lassan ébred, a kávézók kinyitnak, az emberek pedig buszra várnak. diff --git a/corpus/generated/windows-1250/hu/train/library.bin b/corpus/generated/windows-1250/hu/train/library.bin new file mode 100644 index 0000000..4bbeea1 --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/library.bin @@ -0,0 +1,3 @@ +A csendes könyvtár + +Az olvasók történelmi és tudományos könyveket keresnek, a könyvtáros segít a diákoknak. diff --git a/corpus/generated/windows-1250/hu/train/market.bin b/corpus/generated/windows-1250/hu/train/market.bin new file mode 100644 index 0000000..b95088b --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/market.bin @@ -0,0 +1,3 @@ +Szombati piac + +A gazdák friss kenyeret, almát, sajtot és mézet árulnak a téren. diff --git a/corpus/generated/windows-1250/hu/train/workshop.bin b/corpus/generated/windows-1250/hu/train/workshop.bin new file mode 100644 index 0000000..819901d --- /dev/null +++ b/corpus/generated/windows-1250/hu/train/workshop.bin @@ -0,0 +1,3 @@ +Digitális mûhely + +Egy kis csapat múzeumi útmutatót készít, kódot ellenõriz és világos leírásokat ír. diff --git a/corpus/generated/windows-1250/hu/validation/river-trip.bin b/corpus/generated/windows-1250/hu/validation/river-trip.bin new file mode 100644 index 0000000..f807c95 --- /dev/null +++ b/corpus/generated/windows-1250/hu/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kirándulás a folyóhoz + +A barátok térképet és meleg teát vittek, majd megpihentek a folyó partján. diff --git a/corpus/generated/windows-1250/pl/test/community-garden.bin b/corpus/generated/windows-1250/pl/test/community-garden.bin new file mode 100644 index 0000000..0e547a6 --- /dev/null +++ b/corpus/generated/windows-1250/pl/test/community-garden.bin @@ -0,0 +1,3 @@ +Ogród s¹siedzki + +Mieszkañcy zmienili pusty dziedziniec w zielony ogród z kwiatami, warzywami i m³odymi drzewami. Dzieci zrobi³y kolorowe tabliczki, a rodziny co tydzieñ podlewaj¹ roœliny. diff --git a/corpus/generated/windows-1250/pl/train/city-morning.bin b/corpus/generated/windows-1250/pl/train/city-morning.bin new file mode 100644 index 0000000..9ff76bb --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/city-morning.bin @@ -0,0 +1,3 @@ +Poranek w mieœcie + +Miasto budzi siê powoli. Kawiarnie otwieraj¹ drzwi, autobusy rozpoczynaj¹ pierwsze kursy, a ludzie czytaj¹ wiadomoœci. Nad dachami robi siê coraz jaœniej. diff --git a/corpus/generated/windows-1250/pl/train/library.bin b/corpus/generated/windows-1250/pl/train/library.bin new file mode 100644 index 0000000..a649424 --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/library.bin @@ -0,0 +1,3 @@ +Cicha biblioteka + +W osiedlowej bibliotece s³ychaæ szelest kartek. Czytelnicy szukaj¹ ksi¹¿ek o historii, nauce i podró¿ach. Bibliotekarka przygotowuje wystawê dla uczniów. diff --git a/corpus/generated/windows-1250/pl/train/market.bin b/corpus/generated/windows-1250/pl/train/market.bin new file mode 100644 index 0000000..1171993 --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/market.bin @@ -0,0 +1,3 @@ +Sobotni targ + +Targ wype³nia rynek kolorami i zapachami. Rolnicy sprzedaj¹ jab³ka, miód, ser i œwie¿y chleb. Klienci wybieraj¹ warzywa i wymieniaj¹ rodzinne przepisy. diff --git a/corpus/generated/windows-1250/pl/train/workshop.bin b/corpus/generated/windows-1250/pl/train/workshop.bin new file mode 100644 index 0000000..c1eb29a --- /dev/null +++ b/corpus/generated/windows-1250/pl/train/workshop.bin @@ -0,0 +1,3 @@ +Cyfrowa pracownia + +Ma³y zespó³ tworzy cyfrowy przewodnik po muzeum. Projektanci uk³adaj¹ menu, programiœci sprawdzaj¹ kod, a redaktorzy pisz¹ jasne opisy dla goœci. diff --git a/corpus/generated/windows-1250/pl/validation/river-trip.bin b/corpus/generated/windows-1250/pl/validation/river-trip.bin new file mode 100644 index 0000000..e817765 --- /dev/null +++ b/corpus/generated/windows-1250/pl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Spacer nad rzek¹ + +W niedzielê przyjaciele poszli nad spokojn¹ rzekê za miastem. Zabrali mapê, ciep³¹ herbatê i aparat. Po d³ugim spacerze odpoczywali na brzegu. diff --git a/corpus/generated/windows-1250/ro/test/community-garden.bin b/corpus/generated/windows-1250/ro/test/community-garden.bin new file mode 100644 index 0000000..54f8405 --- /dev/null +++ b/corpus/generated/windows-1250/ro/test/community-garden.bin @@ -0,0 +1,3 @@ +Grãdina comunitãþii + +Vecinii au transformat o curte goalã într-o grãdinã cu flori, legume ºi pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se întâlnesc sãptãmânal pentru a uda plantele. diff --git a/corpus/generated/windows-1250/ro/train/city-morning.bin b/corpus/generated/windows-1250/ro/train/city-morning.bin new file mode 100644 index 0000000..08d7469 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/city-morning.bin @@ -0,0 +1,3 @@ +Dimineaþa în oraº + +Oraºul se trezeºte încet. Cafenelele deschid uºile, autobuzele pornesc pe traseu, iar oamenii citesc ºtirile. Cerul devine mai luminos deasupra clãdirilor. diff --git a/corpus/generated/windows-1250/ro/train/library.bin b/corpus/generated/windows-1250/ro/train/library.bin new file mode 100644 index 0000000..1afc415 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/library.bin @@ -0,0 +1,3 @@ +Biblioteca liniºtitã + +În biblioteca de cartier se aud pagini întoarse încet. Cititorii cautã cãrþi despre istorie, ºtiinþã ºi cãlãtorii. Bibliotecara pregãteºte o expoziþie pentru elevi. diff --git a/corpus/generated/windows-1250/ro/train/market.bin b/corpus/generated/windows-1250/ro/train/market.bin new file mode 100644 index 0000000..15a0338 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/market.bin @@ -0,0 +1,3 @@ +Piaþa de sâmbãtã + +Piaþa umple centrul cu mirosuri ºi culori. Fermierii aduc mere, miere, brânzã ºi pâine proaspãtã. Cumpãrãtorii aleg legume ºi schimbã reþete de familie. diff --git a/corpus/generated/windows-1250/ro/train/workshop.bin b/corpus/generated/windows-1250/ro/train/workshop.bin new file mode 100644 index 0000000..132f010 --- /dev/null +++ b/corpus/generated/windows-1250/ro/train/workshop.bin @@ -0,0 +1,3 @@ +Atelier digital + +O echipã micã realizeazã ghidul digital al muzeului. Designerii organizeazã meniul, programatorii verificã sursa, iar redactorii scriu explicaþii clare. diff --git a/corpus/generated/windows-1250/ro/validation/river-trip.bin b/corpus/generated/windows-1250/ro/validation/river-trip.bin new file mode 100644 index 0000000..c125be0 --- /dev/null +++ b/corpus/generated/windows-1250/ro/validation/river-trip.bin @@ -0,0 +1,3 @@ +Plimbare lângã râu + +Duminicã, prietenii au mers spre un râu liniºtit din afara oraºului. Au luat o hartã, ceai cald ºi un aparat foto. Dupã plimbare s-au odihnit pe mal. diff --git a/corpus/generated/windows-1251/ru/test/community-garden.bin b/corpus/generated/windows-1251/ru/test/community-garden.bin new file mode 100644 index 0000000..2f6119a --- /dev/null +++ b/corpus/generated/windows-1251/ru/test/community-garden.bin @@ -0,0 +1,3 @@ +Îáùèé ñàä + +Æèòåëè âìåñòå ïðèâåëè â ïîðÿäîê ñòàðûé äâîð è ïîñàäèëè öâåòû, çåëåíü è ìîëîäûå äåðåâüÿ. Äåòè ñäåëàëè ÿðêèå òàáëè÷êè, à ñîñåäè äîãîâîðèëèñü êàæäóþ íåäåëþ ïîëèâàòü ðàñòåíèÿ. diff --git a/corpus/generated/windows-1251/ru/train/city-morning.bin b/corpus/generated/windows-1251/ru/train/city-morning.bin new file mode 100644 index 0000000..5d58e7b --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/city-morning.bin @@ -0,0 +1,3 @@ +Óòðî â ãîðîäå + +Ãîðîä ïðîñûïàåòñÿ ïîñòåïåííî. Íà îñòàíîâêàõ ïîÿâëÿþòñÿ ïàññàæèðû, áóëî÷íàÿ îòêðûâàåò äâåðè, à ïåðâûå àâòîáóñû âûõîäÿò íà ìàðøðóò. Ëþäè ÷èòàþò ñîîáùåíèÿ è îáñóæäàþò ïëàíû íà íîâûé äåíü. diff --git a/corpus/generated/windows-1251/ru/train/library.bin b/corpus/generated/windows-1251/ru/train/library.bin new file mode 100644 index 0000000..3c62d20 --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/library.bin @@ -0,0 +1,3 @@ +Ðàéîííàÿ áèáëèîòåêà + + áèáëèîòåêå òèõî øåëåñòÿò ñòðàíèöû. ×èòàòåëè âûáèðàþò êíèãè îá èñòîðèè, íàóêå è ïóòåøåñòâèÿõ. Ñîòðóäíèêè ãîòîâÿò âûñòàâêó, îòâå÷àþò íà âîïðîñû è ïîìîãàþò øêîëüíèêàì íàéòè íóæíûå ìàòåðèàëû. diff --git a/corpus/generated/windows-1251/ru/train/market.bin b/corpus/generated/windows-1251/ru/train/market.bin new file mode 100644 index 0000000..668f4f3 --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/market.bin @@ -0,0 +1,3 @@ +Îñåííÿÿ ÿðìàðêà + +Íà ïëîùàäè ïðîõîäèò îñåííÿÿ ÿðìàðêà. Ôåðìåðû ïðèâåçëè ÿáëîêè, ì¸ä, ñûð è ñâåæèé õëåá. Ïîêóïàòåëè ñðàâíèâàþò ïðîäóêòû, áåñåäóþò ñ ïðîäàâöàìè è çàïèñûâàþò èíòåðåñíûå äîìàøíèå ðåöåïòû. diff --git a/corpus/generated/windows-1251/ru/train/workshop.bin b/corpus/generated/windows-1251/ru/train/workshop.bin new file mode 100644 index 0000000..53d315f --- /dev/null +++ b/corpus/generated/windows-1251/ru/train/workshop.bin @@ -0,0 +1,3 @@ +Ðàáî÷àÿ ìàñòåðñêàÿ + +Íåáîëüøàÿ êîìàíäà ñîçäà¸ò ýëåêòðîííûé êàòàëîã ìóçåÿ. Äèçàéíåðû îáñóæäàþò ìåíþ, ïðîãðàììèñòû ïðîâåðÿþò êîä, à ðåäàêòîðû ãîòîâÿò ÿñíûå îïèñàíèÿ äëÿ ïîñåòèòåëåé ðàçíûõ âîçðàñòîâ. diff --git a/corpus/generated/windows-1251/ru/validation/river-trip.bin b/corpus/generated/windows-1251/ru/validation/river-trip.bin new file mode 100644 index 0000000..afd48bf --- /dev/null +++ b/corpus/generated/windows-1251/ru/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ïîåçäêà ê ðåêå + + âîñêðåñåíüå äðóçüÿ îòïðàâèëèñü ê ñïîêîéíîé ðåêå çà ãîðîäîì. Îíè âçÿëè êàðòó, ãîðÿ÷èé ÷àé è ôîòîàïïàðàò. Ïîñëå äîëãîé ïðîãóëêè ïóòåøåñòâåííèêè îòäûõàëè íà áåðåãó è ñëóøàëè âå÷åðíèõ ïòèö. diff --git a/corpus/generated/windows-1252/da/test/community-garden.bin b/corpus/generated/windows-1252/da/test/community-garden.bin new file mode 100644 index 0000000..bbb5980 --- /dev/null +++ b/corpus/generated/windows-1252/da/test/community-garden.bin @@ -0,0 +1,3 @@ +Fælleshaven + +Naboerne gjorde en tom gård til en grøn have, børnene malede skilte, og familierne vander planterne hver uge. diff --git a/corpus/generated/windows-1252/da/train/city-morning.bin b/corpus/generated/windows-1252/da/train/city-morning.bin new file mode 100644 index 0000000..aafe557 --- /dev/null +++ b/corpus/generated/windows-1252/da/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen vågner langsomt, caféerne åbner, og folk venter på de første busser. diff --git a/corpus/generated/windows-1252/da/train/library.bin b/corpus/generated/windows-1252/da/train/library.bin new file mode 100644 index 0000000..dfcd24b --- /dev/null +++ b/corpus/generated/windows-1252/da/train/library.bin @@ -0,0 +1,3 @@ +Det stille bibliotek + +Læsere finder bøger om historie og videnskab, mens bibliotekaren hjælper eleverne. diff --git a/corpus/generated/windows-1252/da/train/market.bin b/corpus/generated/windows-1252/da/train/market.bin new file mode 100644 index 0000000..e156a58 --- /dev/null +++ b/corpus/generated/windows-1252/da/train/market.bin @@ -0,0 +1,3 @@ +Lørdagsmarked + +Bønder sælger frisk brød, æbler, ost og honning på den travle plads. diff --git a/corpus/generated/windows-1252/da/train/workshop.bin b/corpus/generated/windows-1252/da/train/workshop.bin new file mode 100644 index 0000000..04a0b0c --- /dev/null +++ b/corpus/generated/windows-1252/da/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt værksted + +Et lille hold bygger en museumsguide, undersøger koden og skriver tydelige tekster. diff --git a/corpus/generated/windows-1252/da/validation/river-trip.bin b/corpus/generated/windows-1252/da/validation/river-trip.bin new file mode 100644 index 0000000..386c495 --- /dev/null +++ b/corpus/generated/windows-1252/da/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til floden + +Vennerne tog et kort og varm te med på en søndag og hvilede bagefter ved floden. diff --git a/corpus/generated/windows-1252/de/test/community-garden.bin b/corpus/generated/windows-1252/de/test/community-garden.bin new file mode 100644 index 0000000..df4bbec --- /dev/null +++ b/corpus/generated/windows-1252/de/test/community-garden.bin @@ -0,0 +1,3 @@ +Der Gemeinschaftsgarten + +Die Nachbarn haben einen leeren Hof begrünt und dort Kräuter, Gemüse und junge Bäume gepflanzt. Die Schüler malten bunte Schilder, während ältere Bewohner regelmäßig gießen und Geräte aufräumen. diff --git a/corpus/generated/windows-1252/de/train/city-morning.bin b/corpus/generated/windows-1252/de/train/city-morning.bin new file mode 100644 index 0000000..3ccc201 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen in der Stadt + +Die Stadt erwacht langsam. Bäckereien öffnen ihre Türen, Busse beginnen ihre Fahrt und viele Menschen lesen unterwegs die Nachrichten. Über den Dächern wird der Himmel allmählich heller. diff --git a/corpus/generated/windows-1252/de/train/library.bin b/corpus/generated/windows-1252/de/train/library.bin new file mode 100644 index 0000000..992cca1 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/library.bin @@ -0,0 +1,3 @@ +Die ruhige Bibliothek + +In der kleinen Bibliothek rascheln leise die Seiten. Besucher suchen Bücher über Geschichte, Technik und ferne Länder. Eine Mitarbeiterin bereitet eine Ausstellung für Schüler vor. diff --git a/corpus/generated/windows-1252/de/train/market.bin b/corpus/generated/windows-1252/de/train/market.bin new file mode 100644 index 0000000..db29233 --- /dev/null +++ b/corpus/generated/windows-1252/de/train/market.bin @@ -0,0 +1,3 @@ +Markt am Samstag + +Auf dem Platz bieten Bauern Äpfel, Käse, Honig und frisches Brot an. Die Kunden vergleichen Gemüse, sprechen über die Ernte und tauschen einfache Rezepte aus. diff --git a/corpus/generated/windows-1252/de/train/workshop.bin b/corpus/generated/windows-1252/de/train/workshop.bin new file mode 100644 index 0000000..17ea61c --- /dev/null +++ b/corpus/generated/windows-1252/de/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale Werkstatt + +Ein kleines Team entwickelt einen digitalen Museumsführer. Designer ordnen das Menü, Entwickler prüfen den Code und Redakteure schreiben verständliche Texte für Gäste. diff --git a/corpus/generated/windows-1252/de/validation/river-trip.bin b/corpus/generated/windows-1252/de/validation/river-trip.bin new file mode 100644 index 0000000..09bc118 --- /dev/null +++ b/corpus/generated/windows-1252/de/validation/river-trip.bin @@ -0,0 +1,3 @@ +Ausflug an den Fluss + +Am Sonntag wandern Freunde zu einem ruhigen Fluss. Sie nehmen eine Karte, heißen Tee und eine Kamera mit. Nach dem langen Weg sitzen sie am Ufer und hören den Vögeln zu. diff --git a/corpus/generated/windows-1252/en/test/community-garden.bin b/corpus/generated/windows-1252/en/test/community-garden.bin new file mode 100644 index 0000000..549b374 --- /dev/null +++ b/corpus/generated/windows-1252/en/test/community-garden.bin @@ -0,0 +1,3 @@ +The community garden + +Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small café table. diff --git a/corpus/generated/windows-1252/en/train/city-morning.bin b/corpus/generated/windows-1252/en/train/city-morning.bin new file mode 100644 index 0000000..f7199ae --- /dev/null +++ b/corpus/generated/windows-1252/en/train/city-morning.bin @@ -0,0 +1,3 @@ +Morning in the city + +The city wakes slowly; cafés open their doors and commuters read news beside a façade. diff --git a/corpus/generated/windows-1252/en/train/library.bin b/corpus/generated/windows-1252/en/train/library.bin new file mode 100644 index 0000000..20400f3 --- /dev/null +++ b/corpus/generated/windows-1252/en/train/library.bin @@ -0,0 +1,3 @@ +The quiet library + +Readers browse books about history, science, and naïve art while the librarian prepares a résumé. diff --git a/corpus/generated/windows-1252/en/train/market.bin b/corpus/generated/windows-1252/en/train/market.bin new file mode 100644 index 0000000..c760513 --- /dev/null +++ b/corpus/generated/windows-1252/en/train/market.bin @@ -0,0 +1,3 @@ +Saturday market + +Farmers sell bread, apples, jalapeños, and pâté while neighbours exchange recipes. diff --git a/corpus/generated/windows-1252/en/train/workshop.bin b/corpus/generated/windows-1252/en/train/workshop.bin new file mode 100644 index 0000000..788536f --- /dev/null +++ b/corpus/generated/windows-1252/en/train/workshop.bin @@ -0,0 +1,3 @@ +Digital workshop + +A small team designs the museum guide and discusses its rôle, code, and clear descriptions. diff --git a/corpus/generated/windows-1252/en/validation/river-trip.bin b/corpus/generated/windows-1252/en/validation/river-trip.bin new file mode 100644 index 0000000..b686334 --- /dev/null +++ b/corpus/generated/windows-1252/en/validation/river-trip.bin @@ -0,0 +1,3 @@ +Journey to the river + +Friends carried a map and a thermos, walked past the café, and rested by the river. diff --git a/corpus/generated/windows-1252/es/test/community-garden.bin b/corpus/generated/windows-1252/es/test/community-garden.bin new file mode 100644 index 0000000..3c82a6f --- /dev/null +++ b/corpus/generated/windows-1252/es/test/community-garden.bin @@ -0,0 +1,3 @@ +El jardín comunitario + +Los vecinos convirtieron un patio vacío en un jardín lleno de árboles, hierbas y verduras. Los niños pintaron carteles de colores y cada sábado varias familias riegan las plantas. diff --git a/corpus/generated/windows-1252/es/train/city-morning.bin b/corpus/generated/windows-1252/es/train/city-morning.bin new file mode 100644 index 0000000..518bc97 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/city-morning.bin @@ -0,0 +1,3 @@ +La mañana en la ciudad + +La ciudad despierta poco a poco. Las cafeterías abren sus puertas, los autobuses inician la ruta y los vecinos leen las noticias. El cielo se vuelve más claro sobre los tejados. diff --git a/corpus/generated/windows-1252/es/train/library.bin b/corpus/generated/windows-1252/es/train/library.bin new file mode 100644 index 0000000..4d6010f --- /dev/null +++ b/corpus/generated/windows-1252/es/train/library.bin @@ -0,0 +1,3 @@ +Una biblioteca tranquila + +En la biblioteca del barrio se oyen páginas y pasos suaves. Los lectores buscan libros de historia, ciencia y viajes. La bibliotecaria prepara una exposición para los alumnos. diff --git a/corpus/generated/windows-1252/es/train/market.bin b/corpus/generated/windows-1252/es/train/market.bin new file mode 100644 index 0000000..31af767 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/market.bin @@ -0,0 +1,3 @@ +El mercado del sábado + +El mercado llena la plaza de colores. Los agricultores venden manzanas, miel, queso y pan recién hecho. Los clientes comparan las verduras y comparten recetas de sus familias. diff --git a/corpus/generated/windows-1252/es/train/workshop.bin b/corpus/generated/windows-1252/es/train/workshop.bin new file mode 100644 index 0000000..92a2f60 --- /dev/null +++ b/corpus/generated/windows-1252/es/train/workshop.bin @@ -0,0 +1,3 @@ +Taller digital + +Un equipo pequeño crea el catálogo digital del museo. Los diseñadores ordenan el menú, los programadores revisan el código y los editores escriben textos claros para el público. diff --git a/corpus/generated/windows-1252/es/validation/river-trip.bin b/corpus/generated/windows-1252/es/validation/river-trip.bin new file mode 100644 index 0000000..022d6f6 --- /dev/null +++ b/corpus/generated/windows-1252/es/validation/river-trip.bin @@ -0,0 +1,3 @@ +Paseo junto al río + +El domingo unos amigos caminan hacia un río tranquilo. Llevan un mapa, té caliente y una cámara. Después del paseo descansan en la orilla y escuchan a los pájaros. diff --git a/corpus/generated/windows-1252/fr/test/community-garden.bin b/corpus/generated/windows-1252/fr/test/community-garden.bin new file mode 100644 index 0000000..2dce62d --- /dev/null +++ b/corpus/generated/windows-1252/fr/test/community-garden.bin @@ -0,0 +1,3 @@ +Le jardin partagé + +Les habitants ont transformé une cour oubliée en jardin fleuri. Les enfants ont préparé des étiquettes colorées, tandis que les voisins se réunissent chaque semaine pour arroser les légumes et les jeunes arbres. diff --git a/corpus/generated/windows-1252/fr/train/city-morning.bin b/corpus/generated/windows-1252/fr/train/city-morning.bin new file mode 100644 index 0000000..411550d --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/city-morning.bin @@ -0,0 +1,3 @@ +Le matin en ville + +La ville se réveille lentement. Les cafés ouvrent leurs portes, les autobus commencent leur trajet et les passants consultent les nouvelles. Chacun prépare sa journée sous un ciel encore pâle. diff --git a/corpus/generated/windows-1252/fr/train/library.bin b/corpus/generated/windows-1252/fr/train/library.bin new file mode 100644 index 0000000..e06a9ef --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/library.bin @@ -0,0 +1,3 @@ +La bibliothèque calme + +Dans la bibliothèque du quartier, les pages tournent en silence. Les lecteurs cherchent des livres sur l'histoire, les sciences et les voyages. Une bibliothécaire prépare une exposition pour les élèves. diff --git a/corpus/generated/windows-1252/fr/train/market.bin b/corpus/generated/windows-1252/fr/train/market.bin new file mode 100644 index 0000000..9717faa --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/market.bin @@ -0,0 +1,3 @@ +Le marché du samedi + +Le marché remplit la place de couleurs et de parfums. Les producteurs proposent des pommes, du miel, du fromage et du pain frais. Les clients discutent des saisons et échangent des recettes familiales. diff --git a/corpus/generated/windows-1252/fr/train/workshop.bin b/corpus/generated/windows-1252/fr/train/workshop.bin new file mode 100644 index 0000000..4dea00e --- /dev/null +++ b/corpus/generated/windows-1252/fr/train/workshop.bin @@ -0,0 +1,3 @@ +Un atelier numérique + +Une petite équipe construit le catalogue numérique du musée. Les graphistes organisent le menu, les développeurs vérifient le code et les rédacteurs préparent des textes clairs pour le public. diff --git a/corpus/generated/windows-1252/fr/validation/river-trip.bin b/corpus/generated/windows-1252/fr/validation/river-trip.bin new file mode 100644 index 0000000..48f3a71 --- /dev/null +++ b/corpus/generated/windows-1252/fr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Une promenade près de la rivière + +Dimanche, des amis partent vers une rivière tranquille. Ils emportent une carte, du thé chaud et un appareil photo. Après une longue marche, ils se reposent sur la rive et écoutent les oiseaux. diff --git a/corpus/generated/windows-1252/it/test/community-garden.bin b/corpus/generated/windows-1252/it/test/community-garden.bin new file mode 100644 index 0000000..76a54ea --- /dev/null +++ b/corpus/generated/windows-1252/it/test/community-garden.bin @@ -0,0 +1,3 @@ +Il giardino comune + +I vicini hanno trasformato un cortile vuoto in un giardino della città, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante. diff --git a/corpus/generated/windows-1252/it/train/city-morning.bin b/corpus/generated/windows-1252/it/train/city-morning.bin new file mode 100644 index 0000000..a8faca7 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/city-morning.bin @@ -0,0 +1,3 @@ +Mattina in città + +La città si sveglia lentamente, i caffè aprono e la gente aspetta i primi autobus. diff --git a/corpus/generated/windows-1252/it/train/library.bin b/corpus/generated/windows-1252/it/train/library.bin new file mode 100644 index 0000000..e9b65ab --- /dev/null +++ b/corpus/generated/windows-1252/it/train/library.bin @@ -0,0 +1,3 @@ +La biblioteca tranquilla + +I lettori cercano più libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti più giovani. diff --git a/corpus/generated/windows-1252/it/train/market.bin b/corpus/generated/windows-1252/it/train/market.bin new file mode 100644 index 0000000..780f149 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/market.bin @@ -0,0 +1,3 @@ +Mercato del sabato + +Gli agricoltori vendono pane fresco, caffè, mele, formaggio e miele nella piazza della città. diff --git a/corpus/generated/windows-1252/it/train/workshop.bin b/corpus/generated/windows-1252/it/train/workshop.bin new file mode 100644 index 0000000..21734d6 --- /dev/null +++ b/corpus/generated/windows-1252/it/train/workshop.bin @@ -0,0 +1,3 @@ +Laboratorio digitale + +Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perché siano utili a più persone. diff --git a/corpus/generated/windows-1252/it/validation/river-trip.bin b/corpus/generated/windows-1252/it/validation/river-trip.bin new file mode 100644 index 0000000..0ff2d62 --- /dev/null +++ b/corpus/generated/windows-1252/it/validation/river-trip.bin @@ -0,0 +1,3 @@ +Gita al fiume + +Gli amici portano una mappa e tè caldo, camminano nel bosco e riposano sulla riva. diff --git a/corpus/generated/windows-1252/nl/test/community-garden.bin b/corpus/generated/windows-1252/nl/test/community-garden.bin new file mode 100644 index 0000000..89526b9 --- /dev/null +++ b/corpus/generated/windows-1252/nl/test/community-garden.bin @@ -0,0 +1,3 @@ +De buurttuin + +Buren veranderden een lege binnenplaats naast een café in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water. diff --git a/corpus/generated/windows-1252/nl/train/city-morning.bin b/corpus/generated/windows-1252/nl/train/city-morning.bin new file mode 100644 index 0000000..13f5ea7 --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/city-morning.bin @@ -0,0 +1,3 @@ +Ochtend in de stad + +De stad ontwaakt langzaam, cafés openen hun deuren en mensen wachten op de eerste bussen. diff --git a/corpus/generated/windows-1252/nl/train/library.bin b/corpus/generated/windows-1252/nl/train/library.bin new file mode 100644 index 0000000..cdf34ac --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/library.bin @@ -0,0 +1,3 @@ +De stille bibliotheek + +Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris geïnteresseerde leerlingen en één leraar helpt. diff --git a/corpus/generated/windows-1252/nl/train/market.bin b/corpus/generated/windows-1252/nl/train/market.bin new file mode 100644 index 0000000..9c584bb --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/market.bin @@ -0,0 +1,3 @@ +Zaterdagmarkt + +Boeren verkopen vers brood, appels, kaas en honing naast twee cafés op het drukke plein. diff --git a/corpus/generated/windows-1252/nl/train/workshop.bin b/corpus/generated/windows-1252/nl/train/workshop.bin new file mode 100644 index 0000000..53ff667 --- /dev/null +++ b/corpus/generated/windows-1252/nl/train/workshop.bin @@ -0,0 +1,3 @@ +Digitale werkplaats + +Een klein team maakt een museumgids, controleert de code en schrijft duidelijke ideeën over poëzie en teksten. diff --git a/corpus/generated/windows-1252/nl/validation/river-trip.bin b/corpus/generated/windows-1252/nl/validation/river-trip.bin new file mode 100644 index 0000000..27db29e --- /dev/null +++ b/corpus/generated/windows-1252/nl/validation/river-trip.bin @@ -0,0 +1,3 @@ +Reis naar de rivier + +Vrienden namen een kaart en warme thee mee en rustten na één lange wandeling bij de rivier. diff --git a/corpus/generated/windows-1252/no/test/community-garden.bin b/corpus/generated/windows-1252/no/test/community-garden.bin new file mode 100644 index 0000000..b82814c --- /dev/null +++ b/corpus/generated/windows-1252/no/test/community-garden.bin @@ -0,0 +1,3 @@ +Felleshagen + +Naboene gjorde en tom gård til en grønn hage, barna malte skilt og familiene vanner plantene hver uke. diff --git a/corpus/generated/windows-1252/no/train/city-morning.bin b/corpus/generated/windows-1252/no/train/city-morning.bin new file mode 100644 index 0000000..972847d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgen i byen + +Byen våkner langsomt, kafeene åpner dørene og folk venter på de første bussene nær bryggen og sjøen. diff --git a/corpus/generated/windows-1252/no/train/library.bin b/corpus/generated/windows-1252/no/train/library.bin new file mode 100644 index 0000000..f972c3d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/library.bin @@ -0,0 +1,3 @@ +Det stille biblioteket + +Lesere finner bøker om historie, ærlige fortellinger og vitenskap mens bibliotekaren hjelper elevene. diff --git a/corpus/generated/windows-1252/no/train/market.bin b/corpus/generated/windows-1252/no/train/market.bin new file mode 100644 index 0000000..c020f8d --- /dev/null +++ b/corpus/generated/windows-1252/no/train/market.bin @@ -0,0 +1,3 @@ +Lørdagsmarked + +Bønder selger ferskt brød, epler, ost og honning på det travle torget. diff --git a/corpus/generated/windows-1252/no/train/workshop.bin b/corpus/generated/windows-1252/no/train/workshop.bin new file mode 100644 index 0000000..954e228 --- /dev/null +++ b/corpus/generated/windows-1252/no/train/workshop.bin @@ -0,0 +1,3 @@ +Digitalt verksted + +Et lite lag lager en museumsguide, undersøker koden og skriver tydelige tekster. diff --git a/corpus/generated/windows-1252/no/validation/river-trip.bin b/corpus/generated/windows-1252/no/validation/river-trip.bin new file mode 100644 index 0000000..1489859 --- /dev/null +++ b/corpus/generated/windows-1252/no/validation/river-trip.bin @@ -0,0 +1,3 @@ +Tur til elven + +Vennene tok med kart og varm te på søndag og hvilte senere ved elven. diff --git a/corpus/generated/windows-1252/pt/test/community-garden.bin b/corpus/generated/windows-1252/pt/test/community-garden.bin new file mode 100644 index 0000000..65848b6 --- /dev/null +++ b/corpus/generated/windows-1252/pt/test/community-garden.bin @@ -0,0 +1,3 @@ +A horta comunitária + +Os vizinhos transformaram um pátio vazio em jardim, as crianças pintaram placas e as famílias regam as plantas toda semana. diff --git a/corpus/generated/windows-1252/pt/train/city-morning.bin b/corpus/generated/windows-1252/pt/train/city-morning.bin new file mode 100644 index 0000000..18b861f --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/city-morning.bin @@ -0,0 +1,3 @@ +Manhã na cidade + +A cidade acorda devagar, os cafés abrem e as pessoas esperam os primeiros ônibus. diff --git a/corpus/generated/windows-1252/pt/train/library.bin b/corpus/generated/windows-1252/pt/train/library.bin new file mode 100644 index 0000000..50b8034 --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/library.bin @@ -0,0 +1,3 @@ +A biblioteca tranquila + +Os leitores procuram livros de história e ciência enquanto a bibliotecária ajuda os alunos. diff --git a/corpus/generated/windows-1252/pt/train/market.bin b/corpus/generated/windows-1252/pt/train/market.bin new file mode 100644 index 0000000..3810190 --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/market.bin @@ -0,0 +1,3 @@ +Mercado de sábado + +Os agricultores vendem pão fresco, maçãs, queijo e mel na praça movimentada. diff --git a/corpus/generated/windows-1252/pt/train/workshop.bin b/corpus/generated/windows-1252/pt/train/workshop.bin new file mode 100644 index 0000000..ac78c4e --- /dev/null +++ b/corpus/generated/windows-1252/pt/train/workshop.bin @@ -0,0 +1,3 @@ +Oficina digital + +Uma pequena equipe prepara o guia do museu, verifica o código e escreve descrições claras. diff --git a/corpus/generated/windows-1252/pt/validation/river-trip.bin b/corpus/generated/windows-1252/pt/validation/river-trip.bin new file mode 100644 index 0000000..de2590a --- /dev/null +++ b/corpus/generated/windows-1252/pt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Viagem ao rio + +Os amigos levaram um mapa e chá quente, caminharam pela mata e descansaram junto ao rio. diff --git a/corpus/generated/windows-1252/sv/test/community-garden.bin b/corpus/generated/windows-1252/sv/test/community-garden.bin new file mode 100644 index 0000000..0acb233 --- /dev/null +++ b/corpus/generated/windows-1252/sv/test/community-garden.bin @@ -0,0 +1,3 @@ +Gemensam trädgård + +Grannarna gjorde en tom gård till en grön trädgård, barnen målade skyltar och familjerna vattnar växterna varje vecka. diff --git a/corpus/generated/windows-1252/sv/train/city-morning.bin b/corpus/generated/windows-1252/sv/train/city-morning.bin new file mode 100644 index 0000000..f0ba24b --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/city-morning.bin @@ -0,0 +1,3 @@ +Morgon i staden + +Staden vaknar långsamt, kaféerna öppnar och människor väntar på de första bussarna. diff --git a/corpus/generated/windows-1252/sv/train/library.bin b/corpus/generated/windows-1252/sv/train/library.bin new file mode 100644 index 0000000..a7319ac --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/library.bin @@ -0,0 +1,3 @@ +Det tysta biblioteket + +Läsare söker böcker om historia och vetenskap medan bibliotekarien hjälper eleverna. diff --git a/corpus/generated/windows-1252/sv/train/market.bin b/corpus/generated/windows-1252/sv/train/market.bin new file mode 100644 index 0000000..0bd6cfb --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/market.bin @@ -0,0 +1,3 @@ +Lördagsmarknad + +Bönder säljer färskt bröd, äpplen, ost och honung på det livliga torget. diff --git a/corpus/generated/windows-1252/sv/train/workshop.bin b/corpus/generated/windows-1252/sv/train/workshop.bin new file mode 100644 index 0000000..81db320 --- /dev/null +++ b/corpus/generated/windows-1252/sv/train/workshop.bin @@ -0,0 +1,3 @@ +Digital verkstad + +Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter för besökare. diff --git a/corpus/generated/windows-1252/sv/validation/river-trip.bin b/corpus/generated/windows-1252/sv/validation/river-trip.bin new file mode 100644 index 0000000..4704973 --- /dev/null +++ b/corpus/generated/windows-1252/sv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Utflykt till floden + +Vännerna tog med karta och varmt te och vilade sedan vid floden. diff --git a/corpus/generated/windows-1253/el/test/community-garden.bin b/corpus/generated/windows-1253/el/test/community-garden.bin new file mode 100644 index 0000000..2e8196c --- /dev/null +++ b/corpus/generated/windows-1253/el/test/community-garden.bin @@ -0,0 +1,3 @@ +Ï êïéíïôéêüò êÞðïò + +Ïé ãåßôïíåò Ýêáíáí ìéá Üäåéá áõëÞ ðñÜóéíï êÞðï, ôá ðáéäéÜ æùãñÜöéóáí ðéíáêßäåò êáé ïé ïéêïãÝíåéåò ðïôßæïõí ôá öõôÜ. diff --git a/corpus/generated/windows-1253/el/train/city-morning.bin b/corpus/generated/windows-1253/el/train/city-morning.bin new file mode 100644 index 0000000..f58abba --- /dev/null +++ b/corpus/generated/windows-1253/el/train/city-morning.bin @@ -0,0 +1,3 @@ +Ðñùß óôçí ðüëç + +Ç ðüëç îõðíÜ áñãÜ, ôá êáöÝ áíïßãïõí êáé ïé Üíèñùðïé ðåñéìÝíïõí ôá ëåùöïñåßá. diff --git a/corpus/generated/windows-1253/el/train/library.bin b/corpus/generated/windows-1253/el/train/library.bin new file mode 100644 index 0000000..6724218 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/library.bin @@ -0,0 +1,3 @@ +¹óõ÷ç âéâëéïèÞêç + +Ïé áíáãíþóôåò øÜ÷íïõí âéâëßá éóôïñßáò êáé åðéóôÞìçò êáé ç âéâëéïèçêÜñéïò âïçèÜ ôïõò ìáèçôÝò. diff --git a/corpus/generated/windows-1253/el/train/market.bin b/corpus/generated/windows-1253/el/train/market.bin new file mode 100644 index 0000000..40995f7 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/market.bin @@ -0,0 +1,3 @@ +ÓáââáôéÜôéêç áãïñÜ + +Ïé áãñüôåò ðïõëïýí øùìß, ìÝëé, öñïýôá êáé ìõñùäéêÜ óôçí ðëáôåßá. diff --git a/corpus/generated/windows-1253/el/train/workshop.bin b/corpus/generated/windows-1253/el/train/workshop.bin new file mode 100644 index 0000000..4f44381 --- /dev/null +++ b/corpus/generated/windows-1253/el/train/workshop.bin @@ -0,0 +1,3 @@ +Øçöéáêü åñãáóôÞñéï + +Ìéá ìéêñÞ ïìÜäá åôïéìÜæåé ïäçãü ìïõóåßïõ, åëÝã÷åé ôïí êþäéêá êáé ãñÜöåé êáèáñÜ êåßìåíá. diff --git a/corpus/generated/windows-1253/el/validation/river-trip.bin b/corpus/generated/windows-1253/el/validation/river-trip.bin new file mode 100644 index 0000000..dcea70f --- /dev/null +++ b/corpus/generated/windows-1253/el/validation/river-trip.bin @@ -0,0 +1,3 @@ +ÅêäñïìÞ óôï ðïôÜìé + +Ïé ößëïé ðÞñáí ÷Üñôç êáé æåóôü ôóÜé êáé îåêïõñÜóôçêáí äßðëá óôï ðïôÜìé. diff --git a/corpus/generated/windows-1254/tr/test/community-garden.bin b/corpus/generated/windows-1254/tr/test/community-garden.bin new file mode 100644 index 0000000..831e0ea --- /dev/null +++ b/corpus/generated/windows-1254/tr/test/community-garden.bin @@ -0,0 +1,3 @@ +Mahalle bahçesi + +Komþular boþ avluyu yeþil bir bahçeye çevirdi, çocuklar renkli tabelalar yaptý ve aileler bitkileri her hafta suluyor. diff --git a/corpus/generated/windows-1254/tr/train/city-morning.bin b/corpus/generated/windows-1254/tr/train/city-morning.bin new file mode 100644 index 0000000..e8458f6 --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/city-morning.bin @@ -0,0 +1,3 @@ +Þehirde sabah + +Þehir yavaþça uyanýr, kafeler açýlýr ve insanlar ilk otobüsleri bekler. diff --git a/corpus/generated/windows-1254/tr/train/library.bin b/corpus/generated/windows-1254/tr/train/library.bin new file mode 100644 index 0000000..20c465c --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/library.bin @@ -0,0 +1,3 @@ +Sessiz kütüphane + +Okurlar tarih ve bilim kitaplarý ararken kütüphaneci öðrencilere yardýmcý olur. diff --git a/corpus/generated/windows-1254/tr/train/market.bin b/corpus/generated/windows-1254/tr/train/market.bin new file mode 100644 index 0000000..0856260 --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/market.bin @@ -0,0 +1,3 @@ +Cumartesi pazarý + +Çiftçiler kalabalýk meydanda taze ekmek, elma, peynir ve bal satar. diff --git a/corpus/generated/windows-1254/tr/train/workshop.bin b/corpus/generated/windows-1254/tr/train/workshop.bin new file mode 100644 index 0000000..844ca1e --- /dev/null +++ b/corpus/generated/windows-1254/tr/train/workshop.bin @@ -0,0 +1,3 @@ +Dijital atölye + +Küçük bir ekip müze rehberi hazýrlar, kodu denetler ve açýk açýklamalar yazar. diff --git a/corpus/generated/windows-1254/tr/validation/river-trip.bin b/corpus/generated/windows-1254/tr/validation/river-trip.bin new file mode 100644 index 0000000..2568000 --- /dev/null +++ b/corpus/generated/windows-1254/tr/validation/river-trip.bin @@ -0,0 +1,3 @@ +Nehir gezisi + +Arkadaþlar harita ve sýcak çay alýp ormanda yürüdü, sonra nehir kýyýsýnda dinlendi. diff --git a/corpus/generated/windows-1255/he/test/community-garden.bin b/corpus/generated/windows-1255/he/test/community-garden.bin new file mode 100644 index 0000000..28d54b2 --- /dev/null +++ b/corpus/generated/windows-1255/he/test/community-garden.bin @@ -0,0 +1,3 @@ +äâéðä ä÷äéìúéú + +äùëðéí äôëå çöø øé÷ä ìâéðä éøå÷ä, äéìãéí öééøå ùìèéí åäîùôçåú îù÷åú àú äöîçéí áëì ùáåò. diff --git a/corpus/generated/windows-1255/he/train/city-morning.bin b/corpus/generated/windows-1255/he/train/city-morning.bin new file mode 100644 index 0000000..90ce4a0 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/city-morning.bin @@ -0,0 +1,3 @@ +áå÷ø áòéø + +äòéø îúòåøøú ìàè, áúé ä÷ôä ðôúçéí åàðùéí îçëéí ìàåèåáåñéí äøàùåðéí. diff --git a/corpus/generated/windows-1255/he/train/library.bin b/corpus/generated/windows-1255/he/train/library.bin new file mode 100644 index 0000000..d4abd33 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/library.bin @@ -0,0 +1,3 @@ +äñôøééä äù÷èä + +÷åøàéí îçôùéí ñôøé äéñèåøéä åîãò åäñôøðéú òåæøú ìúìîéãéí ìáçåø î÷åøåú. diff --git a/corpus/generated/windows-1255/he/train/market.bin b/corpus/generated/windows-1255/he/train/market.bin new file mode 100644 index 0000000..d4e1672 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/market.bin @@ -0,0 +1,3 @@ +ùå÷ ùáú + +ç÷ìàéí îåëøéí ìçí èøé, ãáù, ôéøåú åâáéðä áëéëø äòîåñä. diff --git a/corpus/generated/windows-1255/he/train/workshop.bin b/corpus/generated/windows-1255/he/train/workshop.bin new file mode 100644 index 0000000..9120483 --- /dev/null +++ b/corpus/generated/windows-1255/he/train/workshop.bin @@ -0,0 +1,3 @@ +ñãðä ãéâéèìéú + +öååú ÷èï îëéï îãøéê ìîåæéàåï, áåã÷ àú ä÷åã åëåúá úéàåøéí áøåøéí. diff --git a/corpus/generated/windows-1255/he/validation/river-trip.bin b/corpus/generated/windows-1255/he/validation/river-trip.bin new file mode 100644 index 0000000..f2a8f51 --- /dev/null +++ b/corpus/generated/windows-1255/he/validation/river-trip.bin @@ -0,0 +1,3 @@ +èéåì àì äðäø + +äçáøéí ì÷çå îôä åúä çí, äìëå áéòø åðçå ìéã äðäø. diff --git a/corpus/generated/windows-1256/ar/test/community-garden.bin b/corpus/generated/windows-1256/ar/test/community-garden.bin new file mode 100644 index 0000000..613b5d4 --- /dev/null +++ b/corpus/generated/windows-1256/ar/test/community-garden.bin @@ -0,0 +1,3 @@ +ÍÏíÞÉ ÇáÍí + +ÒÑÚ ÇáÌíÑÇä ÇáÒåæÑ æÇáÃÔÌÇÑ Ýí ÓÇÍÉ ÞÏíãÉ¡ æÕäÚ ÇáÃØÝÇá áÇÝÊÇÊ ãáæäÉ æÇÊÝÞ ÇáÌãíÚ Úáì ÓÞí ÇáäÈÇÊÇÊ. diff --git a/corpus/generated/windows-1256/ar/train/city-morning.bin b/corpus/generated/windows-1256/ar/train/city-morning.bin new file mode 100644 index 0000000..59e7a5d --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/city-morning.bin @@ -0,0 +1,3 @@ +ÕÈÇÍ ÇáãÏíäÉ + +ÊÓÊíÞÙ ÇáãÏíäÉ ÈåÏæÁ æÊÝÊÍ ÇáãÞÇåí ÃÈæÇÈåÇ ÈíäãÇ íäÊÙÑ ÇáäÇÓ ÇáÍÇÝáÇÊ æíÞÑÄæä ÇáÃÎÈÇÑ. diff --git a/corpus/generated/windows-1256/ar/train/library.bin b/corpus/generated/windows-1256/ar/train/library.bin new file mode 100644 index 0000000..394f9ea --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/library.bin @@ -0,0 +1,3 @@ +ÇáãßÊÈÉ ÇáåÇÏÆÉ + +íÈÍË ÇáÞÑÇÁ Úä ßÊÈ ÇáÊÇÑíÎ æÇáÚáæã æíÓÇÚÏ Ããíä ÇáãßÊÈÉ ÇáØáÇÈ Ýí ÇÎÊíÇÑ ÇáãÑÇÌÚ ÇáãäÇÓÈÉ. diff --git a/corpus/generated/windows-1256/ar/train/market.bin b/corpus/generated/windows-1256/ar/train/market.bin new file mode 100644 index 0000000..0ef6121 --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/market.bin @@ -0,0 +1,3 @@ +ÓæÞ ÇáÓÈÊ + +íÚÑÖ ÇáãÒÇÑÚæä ÇáÎÈÒ æÇáÚÓá æÇáÝÇßåÉ æíÊÈÇÏá ÇáÒæÇÑ ÇáæÕÝÇÊ æÇáÃÍÇÏíË ÇáæÏíÉ. diff --git a/corpus/generated/windows-1256/ar/train/workshop.bin b/corpus/generated/windows-1256/ar/train/workshop.bin new file mode 100644 index 0000000..b9579e6 --- /dev/null +++ b/corpus/generated/windows-1256/ar/train/workshop.bin @@ -0,0 +1,3 @@ +æÑÔÉ ÑÞãíÉ + +íÕãã ÝÑíÞ ÕÛíÑ ÏáíáÇ ááãÊÍÝ æíÑÇÌÚ ÇáãÈÑãÌæä ÇáÔÝÑÉ æíßÊÈ ÇáãÍÑÑæä æÕÝÇ æÇÖÍÇ. diff --git a/corpus/generated/windows-1256/ar/validation/river-trip.bin b/corpus/generated/windows-1256/ar/validation/river-trip.bin new file mode 100644 index 0000000..fee14e0 --- /dev/null +++ b/corpus/generated/windows-1256/ar/validation/river-trip.bin @@ -0,0 +1,3 @@ +ÑÍáÉ Åáì ÇáäåÑ + +Íãá ÇáÃÕÏÞÇÁ ÎÑíØÉ æÔÇíÇ ÓÇÎäÇ æãÔæÇ ÍÊì ÇáäåÑ Ëã ÇÓÊãÚæÇ Åáì ØíæÑ ÇáãÓÇÁ. diff --git a/corpus/generated/windows-1257/et/test/community-garden.bin b/corpus/generated/windows-1257/et/test/community-garden.bin new file mode 100644 index 0000000..46ad8a9 --- /dev/null +++ b/corpus/generated/windows-1257/et/test/community-garden.bin @@ -0,0 +1,3 @@ +Kogukonna aed + +Naabrid muutsid tühja hoovi roheliseks aiaks, lapsed maalisid sildid ja pered kastavad taimi igal nädalal. diff --git a/corpus/generated/windows-1257/et/train/city-morning.bin b/corpus/generated/windows-1257/et/train/city-morning.bin new file mode 100644 index 0000000..f24f4c7 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/city-morning.bin @@ -0,0 +1,3 @@ +Hommik linnas + +Linn ärkab aeglaselt, kohvikud avavad uksed ja inimesed ootavad esimesi busse. diff --git a/corpus/generated/windows-1257/et/train/library.bin b/corpus/generated/windows-1257/et/train/library.bin new file mode 100644 index 0000000..1791206 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/library.bin @@ -0,0 +1,3 @@ +Vaikne raamatukogu + +Lugejad otsivad ajaloo- ja teadusraamatuid ning raamatukoguhoidja aitab õpilasi. diff --git a/corpus/generated/windows-1257/et/train/market.bin b/corpus/generated/windows-1257/et/train/market.bin new file mode 100644 index 0000000..f7f3563 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/market.bin @@ -0,0 +1,3 @@ +Laupäevane turg + +Talunikud müüvad värsket leiba, õunu, juustu ja mett rahvarohkel väljakul. diff --git a/corpus/generated/windows-1257/et/train/workshop.bin b/corpus/generated/windows-1257/et/train/workshop.bin new file mode 100644 index 0000000..a61fe78 --- /dev/null +++ b/corpus/generated/windows-1257/et/train/workshop.bin @@ -0,0 +1,3 @@ +Digitaalne töötuba + +Väike meeskond koostab muuseumijuhti, kontrollib koodi ja kirjutab selgeid kirjeldusi. diff --git a/corpus/generated/windows-1257/et/validation/river-trip.bin b/corpus/generated/windows-1257/et/validation/river-trip.bin new file mode 100644 index 0000000..9b2f7eb --- /dev/null +++ b/corpus/generated/windows-1257/et/validation/river-trip.bin @@ -0,0 +1,3 @@ +Retk jõe äärde + +Sõbrad võtsid kaardi ja sooja tee ning puhkasid pärast jalutuskäiku jõe ääres. diff --git a/corpus/generated/windows-1257/lt/test/community-garden.bin b/corpus/generated/windows-1257/lt/test/community-garden.bin new file mode 100644 index 0000000..5d1e77b --- /dev/null +++ b/corpus/generated/windows-1257/lt/test/community-garden.bin @@ -0,0 +1,3 @@ +Bendruomenës sodas + +Kaimynai tuðèià kiemà pavertë þaliu sodu su gëlëmis, darþovëmis ir jaunais medþiais. Vaikai nupieðë spalvingus þenklus, o ðeimos kas savaitæ laisto augalus. diff --git a/corpus/generated/windows-1257/lt/train/city-morning.bin b/corpus/generated/windows-1257/lt/train/city-morning.bin new file mode 100644 index 0000000..4ab6bae --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/city-morning.bin @@ -0,0 +1,3 @@ +Rytas mieste + +Miestas bunda pamaþu. Kavinës atveria duris, autobusai pradeda pirmuosius marðrutus, o þmonës skaito þinias. Virð senø stogø ðviesëja dangus. diff --git a/corpus/generated/windows-1257/lt/train/library.bin b/corpus/generated/windows-1257/lt/train/library.bin new file mode 100644 index 0000000..3731c52 --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/library.bin @@ -0,0 +1,3 @@ +Tyli biblioteka + +Rajono bibliotekoje tyliai ðiugþda puslapiai. Skaitytojai ieðko knygø apie istorijà, mokslà ir keliones. Bibliotekininkë rengia parodà mokiniams. diff --git a/corpus/generated/windows-1257/lt/train/market.bin b/corpus/generated/windows-1257/lt/train/market.bin new file mode 100644 index 0000000..d2203ae --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/market.bin @@ -0,0 +1,3 @@ +Ðeðtadienio turgus + +Turgaus aikðtæ pripildo spalvos ir kvapai. Ûkininkai siûlo obuoliø, medaus, sûrio ir ðvieþios duonos. Pirkëjai renkasi darþoves ir dalijasi receptais. diff --git a/corpus/generated/windows-1257/lt/train/workshop.bin b/corpus/generated/windows-1257/lt/train/workshop.bin new file mode 100644 index 0000000..3a5f804 --- /dev/null +++ b/corpus/generated/windows-1257/lt/train/workshop.bin @@ -0,0 +1,3 @@ +Skaitmeninës dirbtuvës + +Nedidelë komanda kuria skaitmeniná muziejaus vadovà. Dizaineriai tvarko meniu, programuotojai tikrina kodà, o redaktoriai raðo aiðkius apraðymus. diff --git a/corpus/generated/windows-1257/lt/validation/river-trip.bin b/corpus/generated/windows-1257/lt/validation/river-trip.bin new file mode 100644 index 0000000..77189e1 --- /dev/null +++ b/corpus/generated/windows-1257/lt/validation/river-trip.bin @@ -0,0 +1,3 @@ +Kelionë prie upës + +Sekmadiená draugai iðëjo prie ramios upës uþ miesto. Jie pasiëmë þemëlapá, ðiltos arbatos ir fotoaparatà. Po ilgo þygio visi ilsëjosi ant kranto. diff --git a/corpus/generated/windows-1257/lv/test/community-garden.bin b/corpus/generated/windows-1257/lv/test/community-garden.bin new file mode 100644 index 0000000..9caae38 --- /dev/null +++ b/corpus/generated/windows-1257/lv/test/community-garden.bin @@ -0,0 +1,3 @@ +Kopienas dârzs + +Kaimiòi pârvçrta tukðu pagalmu par zaïu dârzu ar puíçm, dârzeòiem un jauniem kokiem. Bçrni izgatavoja krâsainas zîmes, bet ìimenes katru nedçïu laista augus. diff --git a/corpus/generated/windows-1257/lv/train/city-morning.bin b/corpus/generated/windows-1257/lv/train/city-morning.bin new file mode 100644 index 0000000..c32767d --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/city-morning.bin @@ -0,0 +1,3 @@ +Rîts pilsçtâ + +Pilsçta mostas lçni. Kafejnîcas atver durvis, autobusi sâk pirmos marðrutus, un cilvçki lasa ziòas. Virs mâju jumtiem debesis kïûst arvien gaiðâkas. diff --git a/corpus/generated/windows-1257/lv/train/library.bin b/corpus/generated/windows-1257/lv/train/library.bin new file mode 100644 index 0000000..c112840 --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/library.bin @@ -0,0 +1,3 @@ +Klusâ bibliotçka + +Rajona bibliotçkâ klusi èaukst grâmatu lapas. Lasîtâji meklç darbus par vçsturi, zinâtni un ceïojumiem. Bibliotekâre gatavo izstâdi skolçniem. diff --git a/corpus/generated/windows-1257/lv/train/market.bin b/corpus/generated/windows-1257/lv/train/market.bin new file mode 100644 index 0000000..5d940ef --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/market.bin @@ -0,0 +1,3 @@ +Sestdienas tirgus + +Tirgus laukumu piepilda krâsas un smarþas. Zemnieki piedâvâ âbolus, medu, sieru un svaigu maizi. Pircçji izvçlas dârzeòus un pârrunâ ìimenes receptes. diff --git a/corpus/generated/windows-1257/lv/train/workshop.bin b/corpus/generated/windows-1257/lv/train/workshop.bin new file mode 100644 index 0000000..0fbfb56 --- /dev/null +++ b/corpus/generated/windows-1257/lv/train/workshop.bin @@ -0,0 +1,3 @@ +Digitâlâ darbnîca + +Neliela komanda veido muzeja digitâlo ceïvedi. Dizaineri sakârto izvçlni, programmçtâji pârbauda kodu, bet redaktori raksta skaidrus aprakstus. diff --git a/corpus/generated/windows-1257/lv/validation/river-trip.bin b/corpus/generated/windows-1257/lv/validation/river-trip.bin new file mode 100644 index 0000000..5ba00ad --- /dev/null +++ b/corpus/generated/windows-1257/lv/validation/river-trip.bin @@ -0,0 +1,3 @@ +Pastaiga gar upi + +Svçtdien draugi devâs uz klusu upi ârpus pilsçtas. Viòi paòçma karti, siltu tçju un fotoaparâtu. Pçc garâs pastaigas visi atpûtâs krastâ. diff --git a/corpus/generated/windows-1258/vi/test/community-garden.bin b/corpus/generated/windows-1258/vi/test/community-garden.bin new file mode 100644 index 0000000..be14fff --- /dev/null +++ b/corpus/generated/windows-1258/vi/test/community-garden.bin @@ -0,0 +1,3 @@ +VýõÌn côòng ðôÌng + +Hàng xóm biêìn sân trôìng thành khu výõÌn xanh, treÒ em veÞ biêÒn màu và các gia ðiÌnh týõìi cây môÞi tuâÌn. diff --git a/corpus/generated/windows-1258/vi/train/city-morning.bin b/corpus/generated/windows-1258/vi/train/city-morning.bin new file mode 100644 index 0000000..321ee45 --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/city-morning.bin @@ -0,0 +1,3 @@ +BuôÒi sáng trong thành phôì + +Thành phôì thýìc dâòy châòm raÞi, quán cà phê mõÒ cýÒa và moòi ngýõÌi chõÌ chuyêìn xe buyìt ðâÌu tiên. diff --git a/corpus/generated/windows-1258/vi/train/library.bin b/corpus/generated/windows-1258/vi/train/library.bin new file mode 100644 index 0000000..89d20aa --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/library.bin @@ -0,0 +1,3 @@ +Thý viêòn yên tiÞnh + +Ðôòc giaÒ tiÌm sách liòch sýÒ và khoa hoòc, coÌn thuÒ thý giúp hoòc sinh choòn tài liêòu. diff --git a/corpus/generated/windows-1258/vi/train/market.bin b/corpus/generated/windows-1258/vi/train/market.bin new file mode 100644 index 0000000..deb93fb --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/market.bin @@ -0,0 +1,3 @@ +Chõò thýì baÒy + +Nông dân bán bánh miÌ, táo, phô mai và mâòt ong taòi quaÒng trýõÌng ðông ðúc. diff --git a/corpus/generated/windows-1258/vi/train/workshop.bin b/corpus/generated/windows-1258/vi/train/workshop.bin new file mode 100644 index 0000000..6f4c26b --- /dev/null +++ b/corpus/generated/windows-1258/vi/train/workshop.bin @@ -0,0 +1,3 @@ +XýõÒng kyÞ thuâòt sôì + +Môòt nhóm nhoÒ làm hýõìng dâÞn baÒo tàng, kiêÒm tra maÞ và viêìt mô taÒ roÞ ràng. diff --git a/corpus/generated/windows-1258/vi/validation/river-trip.bin b/corpus/generated/windows-1258/vi/validation/river-trip.bin new file mode 100644 index 0000000..aca81b2 --- /dev/null +++ b/corpus/generated/windows-1258/vi/validation/river-trip.bin @@ -0,0 +1,3 @@ +Chuyêìn ði ra sông + +Baòn bè mang baÒn ðôÌ và trà nóng, ði qua rýÌng rôÌi nghiÒ bên bõÌ sông. diff --git a/corpus/generated/windows-874/th/test/community-garden.bin b/corpus/generated/windows-874/th/test/community-garden.bin new file mode 100644 index 0000000..8f1080c --- /dev/null +++ b/corpus/generated/windows-874/th/test/community-garden.bin @@ -0,0 +1,3 @@ +ÊǹªØÁª¹ + +à¾×è͹ºéÒ¹à»ÅÕè¹ÅÒ¹ÇèÒ§à»ç¹ÊǹÊÕà¢ÕÂÇ à´ç¡ æ ÇÒ´»éÒÂáÅФÃͺ¤ÃÑǪèÇ¡ѹô¹éÓµé¹äÁé·Ø¡ÊÑ»´ÒËì diff --git a/corpus/generated/windows-874/th/train/city-morning.bin b/corpus/generated/windows-874/th/train/city-morning.bin new file mode 100644 index 0000000..3325a1b --- /dev/null +++ b/corpus/generated/windows-874/th/train/city-morning.bin @@ -0,0 +1,3 @@ +àªéÒã¹àÁ×ͧ + +àÁ×ͧ¤èÍÂ æ µ×è¹ ÃéÒ¹¡Òá¿à»Ô´»ÃеÙáÅмÙ餹ÃÍöâ´ÂÊÒÃà·ÕèÂÇáá diff --git a/corpus/generated/windows-874/th/train/library.bin b/corpus/generated/windows-874/th/train/library.bin new file mode 100644 index 0000000..8574c4f --- /dev/null +++ b/corpus/generated/windows-874/th/train/library.bin @@ -0,0 +1,3 @@ +ËéͧÊÁØ´à§Õºʧº + +¼ÙéÍèÒ¹¤é¹ËÒ˹ѧÊ×Í»ÃÐÇѵÔÈÒʵÃìáÅÐÇÔ·ÂÒÈÒʵÃì ºÃóÒÃÑ¡ÉìªèǹѡàÃÕ¹àÅ×Í¡¢éÍÁÙÅ diff --git a/corpus/generated/windows-874/th/train/market.bin b/corpus/generated/windows-874/th/train/market.bin new file mode 100644 index 0000000..e2a1717 --- /dev/null +++ b/corpus/generated/windows-874/th/train/market.bin @@ -0,0 +1,3 @@ +µÅÒ´ÇѹàÊÒÃì + +ªÒǹҢÒ¢¹Á»Ñ§Ê´ ¼ÅäÁé ªÕÊ áÅйéÓ¼Öé§ã¹ÅÒ¹·Õè¤Ö¡¤Ñ¡ diff --git a/corpus/generated/windows-874/th/train/workshop.bin b/corpus/generated/windows-874/th/train/workshop.bin new file mode 100644 index 0000000..c571435 --- /dev/null +++ b/corpus/generated/windows-874/th/train/workshop.bin @@ -0,0 +1,3 @@ +Ëéͧ·Ó§Ò¹´Ô¨Ô·ÑÅ + +·ÕÁàÅ硨Ѵ·Ó¤ÙèÁ×;ԾԸÀѳ±ì µÃǨÃËÑÊ áÅÐà¢Õ¹¤Ó͸ԺÒ·ÕèªÑ´à¨¹ diff --git a/corpus/generated/windows-874/th/validation/river-trip.bin b/corpus/generated/windows-874/th/validation/river-trip.bin new file mode 100644 index 0000000..0e376c8 --- /dev/null +++ b/corpus/generated/windows-874/th/validation/river-trip.bin @@ -0,0 +1,3 @@ +à·ÕèÂÇÃÔÁáÁè¹éÓ + +à¾×è͹¹Óá¼¹·Õè¡ÑºªÒÃé͹ä»à´Ô¹ã¹»èÒáÅéǾѡ¼è͹ÃÔÁáÁè¹éÓ diff --git a/corpus/manifest.json b/corpus/manifest.json index 5121f91..56efa23 100644 --- a/corpus/manifest.json +++ b/corpus/manifest.json @@ -1,5 +1,45 @@ { "encodings": [ + { + "name": "ISO-8859-1", + "iconv": "ISO-8859-1", + "languages": ["da", "de", "en", "es", "fr", "it", "nl", "no", "pt", "sv"] + }, + { + "name": "ISO-8859-2", + "iconv": "ISO-8859-2", + "languages": ["cs", "hu", "pl", "ro"], + "sourceReplacements": { "Ș": "Åž", "È™": "ÅŸ", "Èš": "Å¢", "È›": "Å£" } + }, + { "name": "ISO-8859-5", "iconv": "ISO-8859-5", "languages": ["ru"] }, + { "name": "ISO-8859-6", "iconv": "ISO-8859-6", "languages": ["ar"] }, + { "name": "ISO-8859-7", "iconv": "ISO-8859-7", "languages": ["el"] }, + { "name": "ISO-8859-8", "iconv": "ISO-8859-8", "languages": ["he"] }, + { "name": "ISO-8859-9", "iconv": "ISO-8859-9", "languages": ["tr"] }, + { + "name": "windows-1250", + "iconv": "WINDOWS-1250", + "languages": ["cs", "hu", "pl", "ro"], + "sourceReplacements": { "Ș": "Åž", "È™": "ÅŸ", "Èš": "Å¢", "È›": "Å£" } + }, + { "name": "windows-1251", "iconv": "WINDOWS-1251", "languages": ["ru"] }, + { + "name": "windows-1252", + "iconv": "WINDOWS-1252", + "languages": ["da", "de", "en", "es", "fr", "it", "nl", "no", "pt", "sv"] + }, + { "name": "windows-1253", "iconv": "WINDOWS-1253", "languages": ["el"] }, + { "name": "windows-1254", "iconv": "WINDOWS-1254", "languages": ["tr"] }, + { "name": "windows-1255", "iconv": "WINDOWS-1255", "languages": ["he"] }, + { "name": "windows-1256", "iconv": "WINDOWS-1256", "languages": ["ar"] }, + { + "name": "windows-1257", + "iconv": "WINDOWS-1257", + "languages": ["et", "lv", "lt"] + }, + { "name": "windows-1258", "iconv": "WINDOWS-1258", "languages": ["vi"] }, + { "name": "windows-874", "iconv": "WINDOWS-874", "languages": ["th"] }, + { "name": "KOI8-R", "iconv": "KOI8-R", "languages": ["ru"] }, { "name": "KOI8-U", "iconv": "KOI8-U", "languages": ["uk"] }, { "name": "IBM866", "iconv": "CP866", "languages": ["ru"] }, { "name": "IBM855", "iconv": "CP855", "languages": ["ru", "sr"] }, diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 31a0927..12b72dd 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -1,9 +1,9 @@ { "summary": { - "tests": 22, - "encodingCorrect": 22, - "languageCorrect": 22, - "topScoreTies": 0 + "tests": 66, + "encodingCorrect": 43, + "languageCorrect": 42, + "topScoreTies": 35 }, "confusion": { "CP850": { @@ -18,20 +18,27 @@ "IBM866": { "IBM866": 1 }, + "ISO-8859-1": { + "ISO-8859-1": 8, + "ISO-8859-15": 2 + }, "ISO-8859-10": { "ISO-8859-10": 1 }, "ISO-8859-13": { - "ISO-8859-13": 1 + "windows-1257": 1 }, "ISO-8859-14": { "ISO-8859-14": 1 }, "ISO-8859-15": { - "ISO-8859-15": 3 + "ISO-8859-1": 3 }, "ISO-8859-16": { - "ISO-8859-16": 1 + "ISO-8859-2": 1 + }, + "ISO-8859-2": { + "ISO-8859-2": 4 }, "ISO-8859-3": { "ISO-8859-3": 1 @@ -39,12 +46,62 @@ "ISO-8859-4": { "ISO-8859-4": 1 }, + "ISO-8859-5": { + "ISO-8859-5": 1 + }, + "ISO-8859-6": { + "ISO-8859-6": 1 + }, + "ISO-8859-7": { + "ISO-8859-7": 1 + }, + "ISO-8859-8": { + "ISO-8859-8": 1 + }, + "ISO-8859-9": { + "ISO-8859-9": 1 + }, + "KOI8-R": { + "KOI8-R": 1 + }, "KOI8-U": { "KOI8-U": 1 }, "macintosh": { "macintosh": 3 }, + "windows-1250": { + "ISO-8859-2": 3, + "windows-1250": 1 + }, + "windows-1251": { + "windows-1251": 1 + }, + "windows-1252": { + "ISO-8859-1": 8, + "ISO-8859-15": 2 + }, + "windows-1253": { + "ISO-8859-7": 1 + }, + "windows-1254": { + "ISO-8859-9": 1 + }, + "windows-1255": { + "ISO-8859-8": 1 + }, + "windows-1256": { + "windows-1256": 1 + }, + "windows-1257": { + "windows-1257": 3 + }, + "windows-1258": { + "windows-1258": 1 + }, + "windows-874": { + "windows-874": 1 + }, "x-mac-cyrillic": { "x-mac-cyrillic": 2 } @@ -77,6 +134,24 @@ "total": 216, "hitRate": 0.24537037037037038 }, + { + "encoding": "ISO-8859-1", + "confidence": 72, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "windows-1252", + "confidence": 72, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, { "encoding": "ISO-8859-15", "confidence": 72, @@ -104,6 +179,42 @@ "total": 215, "hitRate": 0.07441860465116279 }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, + { + "encoding": "ISO-8859-9", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, + { + "encoding": "windows-1254", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 215, + "hitRate": 0.046511627906976744 + }, { "encoding": "ISO-8859-16", "confidence": 13, @@ -113,6 +224,15 @@ "total": 215, "hitRate": 0.046511627906976744 }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 215, + "hitRate": 0.037209302325581395 + }, { "encoding": "ISO-8859-4", "confidence": 11, @@ -122,6 +242,15 @@ "total": 215, "hitRate": 0.037209302325581395 }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.172849867117, + "hits": 5, + "total": 215, + "hitRate": 0.023255813953488372 + }, { "encoding": "ISO-8859-3", "confidence": 5, @@ -158,6 +287,96 @@ "total": 215, "hitRate": 0.009302325581395349 }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.099525732258, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.729917997667, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.542674768484, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.504753627871, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.219881518497, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.542674768484, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.504753627871, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.511862527725, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.737138560752, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 215, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -223,6 +442,24 @@ "total": 200, "hitRate": 0.205 }, + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, { "encoding": "macintosh", "confidence": 58, @@ -241,6 +478,15 @@ "total": 199, "hitRate": 0.19597989949748743 }, + { + "encoding": "windows-1257", + "confidence": 25, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, { "encoding": "ISO-8859-4", "confidence": 25, @@ -268,6 +514,24 @@ "total": 199, "hitRate": 0.05527638190954774 }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, { "encoding": "ISO-8859-16", "confidence": 15, @@ -277,6 +541,33 @@ "total": 199, "hitRate": 0.05025125628140704 }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, { "encoding": "CP852", "confidence": 7, @@ -295,6 +586,96 @@ "total": 199, "hitRate": 0.01507537688442211 }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.61627103998, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -370,9 +751,27 @@ "hitRate": 0.25877192982456143 }, { - "encoding": "ISO-8859-15", + "encoding": "ISO-8859-1", "confidence": 75, - "language": "es", + "language": "nl", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "windows-1252", + "confidence": 75, + "language": "nl", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "ISO-8859-15", + "confidence": 75, + "language": "es", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -387,6 +786,24 @@ "total": 228, "hitRate": 0.23684210526315788 }, + { + "encoding": "ISO-8859-2", + "confidence": 12, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "windows-1250", + "confidence": 12, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, { "encoding": "ISO-8859-10", "confidence": 12, @@ -405,6 +822,15 @@ "total": 224, "hitRate": 0.04017857142857143 }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 224, + "hitRate": 0.03125 + }, { "encoding": "ISO-8859-4", "confidence": 9, @@ -432,6 +858,33 @@ "total": 228, "hitRate": 0.02631578947368421 }, + { + "encoding": "ISO-8859-9", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1254", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, { "encoding": "ISO-8859-13", "confidence": 5, @@ -450,6 +903,96 @@ "total": 224, "hitRate": 0.013392857142857142 }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -504,7 +1047,7 @@ }, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { "encoding": "CP852", @@ -515,6 +1058,51 @@ "total": 184, "hitRate": 0.1358695652173913 }, + { + "encoding": "windows-1250", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "ISO-8859-2", + "confidence": 37, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 23, + "total": 184, + "hitRate": 0.125 + }, + { + "encoding": "ISO-8859-1", + "confidence": 8, + "language": "sv", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "confidence": 8, + "language": "sv", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.536305162191, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, { "encoding": "macintosh", "confidence": 6, @@ -542,6 +1130,15 @@ "total": 183, "hitRate": 0.02185792349726776 }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 183, + "hitRate": 0.01639344262295082 + }, { "encoding": "ISO-8859-3", "confidence": 4, @@ -596,6 +1193,114 @@ "total": 183, "hitRate": 0.01092896174863388 }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.311255977022, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.762644549521, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.302642222102, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.005185419538, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.762644549521, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.302642222102, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.54369343373, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.549771342707, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.538051035716, + "hits": 0, + "total": 183, + "hitRate": 0 + }, { "encoding": "KOI8-U", "confidence": 0, @@ -679,6 +1384,42 @@ "total": 130, "hitRate": 0.038461538461538464 }, + { + "encoding": "windows-874", + "confidence": 10, + "language": "th", + "byteLogLikelihood": -5.027512765832, + "hits": 5, + "total": 146, + "hitRate": 0.03424657534246575 + }, + { + "encoding": "KOI8-R", + "confidence": 10, + "language": "ru", + "byteLogLikelihood": -6.090200292669, + "hits": 5, + "total": 140, + "hitRate": 0.03571428571428571 + }, + { + "encoding": "ISO-8859-7", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.19500629466, + "hits": 3, + "total": 138, + "hitRate": 0.021739130434782608 + }, + { + "encoding": "windows-1253", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.19500629466, + "hits": 3, + "total": 139, + "hitRate": 0.02158273381294964 + }, { "encoding": "x-mac-cyrillic", "confidence": 4, @@ -689,13 +1430,22 @@ "hitRate": 0.014598540145985401 }, { - "encoding": "ISO-8859-10", + "encoding": "ISO-8859-5", "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.06395526639, + "language": "ru", + "byteLogLikelihood": -5.217274879676, "hits": 1, - "total": 160, - "hitRate": 0.00625 + "total": 175, + "hitRate": 0.005714285714285714 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.06395526639, + "hits": 1, + "total": 160, + "hitRate": 0.00625 }, { "encoding": "ISO-8859-14", @@ -706,6 +1456,123 @@ "total": 174, "hitRate": 0.005747126436781609 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.789358524023, + "hits": 0, + "total": 139, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.901389021393, + "hits": 0, + "total": 165, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.523443710193, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.21454182173, + "hits": 0, + "total": 84, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.033184680772, + "hits": 0, + "total": 139, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.901389021393, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.690578567068, + "hits": 0, + "total": 156, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.789358524023, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.033184680772, + "hits": 0, + "total": 141, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.21454182173, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.354655838496, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.913061957297, + "hits": 0, + "total": 149, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.237936257135, + "hits": 0, + "total": 134, + "hitRate": 0 + }, { "encoding": "macintosh", "confidence": 0, @@ -816,6 +1683,24 @@ "total": 137, "hitRate": 0.08759124087591241 }, + { + "encoding": "windows-874", + "confidence": 35, + "language": "th", + "byteLogLikelihood": -5.092901405888, + "hits": 15, + "total": 128, + "hitRate": 0.1171875 + }, + { + "encoding": "KOI8-R", + "confidence": 11, + "language": "ru", + "byteLogLikelihood": -6.207100144751, + "hits": 4, + "total": 104, + "hitRate": 0.038461538461538464 + }, { "encoding": "KOI8-U", "confidence": 9, @@ -834,6 +1719,150 @@ "total": 112, "hitRate": 0.026785714285714284 }, + { + "encoding": "ISO-8859-8", + "confidence": 4, + "language": "he", + "byteLogLikelihood": -5.427107080966, + "hits": 1, + "total": 64, + "hitRate": 0.015625 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -5.427107080966, + "hits": 1, + "total": 80, + "hitRate": 0.0125 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.764166439133, + "hits": 0, + "total": 104, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.902465386166, + "hits": 0, + "total": 128, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.640443286036, + "hits": 0, + "total": 157, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.592489718341, + "hits": 0, + "total": 84, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.3747848425, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.028189084099, + "hits": 0, + "total": 104, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.902465386166, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.910594216272, + "hits": 0, + "total": 144, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.764166439133, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.3747848425, + "hits": 0, + "total": 107, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.028189084099, + "hits": 0, + "total": 104, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.564612100894, + "hits": 0, + "total": 108, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.902859191011, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.23021689626, + "hits": 0, + "total": 103, + "hitRate": 0 + }, { "encoding": "macintosh", "confidence": 0, @@ -953,6 +1982,15 @@ "total": 179, "hitRate": 0.13966480446927373 }, + { + "encoding": "KOI8-R", + "confidence": 23, + "language": "ru", + "byteLogLikelihood": -6.620542862927, + "hits": 6, + "total": 77, + "hitRate": 0.07792207792207792 + }, { "encoding": "x-mac-cyrillic", "confidence": 15, @@ -971,6 +2009,15 @@ "total": 103, "hitRate": 0.038834951456310676 }, + { + "encoding": "windows-874", + "confidence": 9, + "language": "th", + "byteLogLikelihood": -5.160775609519, + "hits": 5, + "total": 165, + "hitRate": 0.030303030303030304 + }, { "encoding": "IBM855", "confidence": 8, @@ -980,6 +2027,42 @@ "total": 169, "hitRate": 0.029585798816568046 }, + { + "encoding": "windows-1251", + "confidence": 7, + "language": "ru", + "byteLogLikelihood": -5.801879921101, + "hits": 4, + "total": 153, + "hitRate": 0.026143790849673203 + }, + { + "encoding": "windows-1253", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.523099191635, + "hits": 2, + "total": 89, + "hitRate": 0.02247191011235955 + }, + { + "encoding": "ISO-8859-5", + "confidence": 3, + "language": "ru", + "byteLogLikelihood": -5.798412758856, + "hits": 2, + "total": 174, + "hitRate": 0.011494252873563218 + }, + { + "encoding": "ISO-8859-7", + "confidence": 3, + "language": "el", + "byteLogLikelihood": -5.523099191635, + "hits": 1, + "total": 81, + "hitRate": 0.012345679012345678 + }, { "encoding": "ISO-8859-10", "confidence": 1, @@ -999,524 +2082,502 @@ "hitRate": 0.005917159763313609 }, { - "encoding": "macintosh", + "encoding": "ISO-8859-1", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944169511699, + "language": "it", + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 67, + "total": 81, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.096764716542, + "language": "hu", + "byteLogLikelihood": -4.941337938888, "hits": 0, - "total": 116, + "total": 145, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -4.979786266462, + "language": "ar", + "byteLogLikelihood": -5.82555458734, "hits": 0, - "total": 148, + "total": 57, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -4.939479070188, + "language": "he", + "byteLogLikelihood": -5.371812787625, "hits": 0, - "total": 116, + "total": 75, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "ISO-8859-9", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.848242748706, + "language": "tr", + "byteLogLikelihood": -5.062286993055, "hits": 0, - "total": 111, + "total": 81, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1250", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.06091613544, + "language": "hu", + "byteLogLikelihood": -4.941337938888, "hits": 0, - "total": 144, + "total": 117, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "windows-1252", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.677183836449, + "language": "it", + "byteLogLikelihood": -4.841944488282, "hits": 0, - "total": 130, + "total": 82, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "windows-1254", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -4.699618505771, + "language": "tr", + "byteLogLikelihood": -5.062286993055, "hits": 0, - "total": 166, + "total": 81, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-10", - "language": "is" - }, - "predicted": { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-10", - "confidence": 38, - "language": "is", - "byteLogLikelihood": -3.025836278694, - "hits": 22, - "total": 171, - "hitRate": 0.1286549707602339 - }, - { - "encoding": "ISO-8859-3", - "confidence": 12, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 }, { - "encoding": "ISO-8859-13", - "confidence": 7, - "language": "lt", - "byteLogLikelihood": -3.989583472462, - "hits": 4, - "total": 171, - "hitRate": 0.023391812865497075 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 75, + "hitRate": 0 }, { - "encoding": "macintosh", - "confidence": 5, - "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.857645252562, + "hits": 0, + "total": 82, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 5, + "encoding": "windows-1257", + "confidence": 0, "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 + "byteLogLikelihood": -4.928541579896, + "hits": 0, + "total": 117, + "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 5, - "language": "es", - "byteLogLikelihood": -4.894502014278, - "hits": 3, - "total": 171, - "hitRate": 0.017543859649122806 + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.078371422345, + "hits": 0, + "total": 76, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 5, + "encoding": "macintosh", + "confidence": 0, "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "byteLogLikelihood": -4.944169511699, + "hits": 0, + "total": 67, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 5, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 3, - "total": 169, - "hitRate": 0.01775147928994083 + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.096764716542, + "hits": 0, + "total": 116, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.979786266462, + "hits": 0, + "total": 148, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-13", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.026638569428, + "language": "lt", + "byteLogLikelihood": -4.939479070188, "hits": 0, - "total": 171, + "total": 116, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-15", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.838861551509, + "language": "es", + "byteLogLikelihood": -4.848242748706, "hits": 0, - "total": 171, + "total": 111, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-16", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.795369889152, + "language": "ro", + "byteLogLikelihood": -5.06091613544, "hits": 0, - "total": 169, + "total": 144, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "CP850", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.733983711968, + "language": "es", + "byteLogLikelihood": -4.677183836449, "hits": 0, - "total": 171, + "total": 130, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "CP852", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.657284871846, + "language": "pl", + "byteLogLikelihood": -4.699618505771, "hits": 0, - "total": 171, + "total": 166, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-13", - "language": "lt" + "encoding": "ISO-8859-1", + "language": "da" }, "predicted": { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 2, "candidates": [ { - "encoding": "ISO-8859-13", - "confidence": 34, - "language": "lt", - "byteLogLikelihood": -3.200530207132, - "hits": 20, - "total": 173, - "hitRate": 0.11560693641618497 + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { - "encoding": "ISO-8859-4", - "confidence": 22, - "language": "lv", - "byteLogLikelihood": -4.610398759565, - "hits": 13, - "total": 173, - "hitRate": 0.07514450867052024 + "encoding": "windows-1252", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 }, { "encoding": "macintosh", - "confidence": 12, + "confidence": 30, "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 168, - "hitRate": 0.041666666666666664 + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 }, { "encoding": "ISO-8859-15", - "confidence": 12, - "language": "fr", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 173, - "hitRate": 0.04046242774566474 + "confidence": 30, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 }, { "encoding": "CP850", - "confidence": 12, + "confidence": 30, "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 170, - "hitRate": 0.041176470588235294 + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "encoding": "ISO-8859-14", + "confidence": 15, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "cs", + "byteLogLikelihood": -5.176149732574, "hits": 4, - "total": 170, - "hitRate": 0.023529411764705882 + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 172, - "hitRate": 0.01744186046511628 + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.6272413408, - "hits": 2, - "total": 173, - "hitRate": 0.011560693641618497 + "encoding": "windows-1250", + "confidence": 10, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "IBM855", - "confidence": 1, - "language": "sr", - "byteLogLikelihood": -5.488073838002, - "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -4.416382325607, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "encoding": "windows-1257", + "confidence": 10, + "language": "lt", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { "encoding": "ISO-8859-16", - "confidence": 1, + "confidence": 10, "language": "ro", - "byteLogLikelihood": -4.659155714377, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.252539473952, - "hits": 0, - "total": 173, - "hitRate": 0 + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 120, + "hitRate": 0.025 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -4.717529149813, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 120, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.658005606594, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.145958228046, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.893109334109, + "byteLogLikelihood": -5.825649766807, "hits": 0, - "total": 171, + "total": 120, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.441328843196, + "language": "ar", + "byteLogLikelihood": -4.736335095675, "hits": 0, - "total": 173, + "total": 120, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-14", - "language": "cy" - }, - "predicted": { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-14", - "confidence": 50, - "language": "cy", - "byteLogLikelihood": -3.988458644989, - "hits": 32, - "total": 189, - "hitRate": 0.1693121693121693 }, { - "encoding": "macintosh", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "fr", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 189, - "hitRate": 0.037037037037037035 + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 188, - "hitRate": 0.03723404255319149 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-3", - "confidence": 4, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -4.009500374258, - "hits": 3, - "total": 189, - "hitRate": 0.015873015873015872 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 3, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 2, - "total": 188, - "hitRate": 0.010638297872340425 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.861023356308, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 1, - "language": "lv", - "byteLogLikelihood": -4.961907790704, - "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.888170123788, + "hits": 0, + "total": 120, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 1, - "language": "lt", - "byteLogLikelihood": -3.647025046455, - "hits": 1, - "total": 189, - "hitRate": 0.005291005291005291 + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 120, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -5.821976831991, + "byteLogLikelihood": -6.476992253447, "hits": 0, - "total": 189, + "total": 120, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.350299845595, + "byteLogLikelihood": -5.825649766807, "hits": 0, - "total": 188, + "total": 120, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.325879646284, + "byteLogLikelihood": -5.685745819823, "hits": 0, - "total": 188, + "total": 120, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.255815902698, - "hits": 0, - "total": 189, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-16", - "confidence": 0, - "language": "ro", - "byteLogLikelihood": -3.981643419755, + "language": "ru", + "byteLogLikelihood": -4.288232392491, "hits": 0, - "total": 189, + "total": 120, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", + "encoding": "ISO-8859-1", "language": "de" }, "predicted": { - "encoding": "ISO-8859-15", + "encoding": "ISO-8859-1", "confidence": 73, "language": "de", "byteLogLikelihood": -3.169712077519, @@ -1526,8 +2587,26 @@ }, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 3, "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, { "encoding": "ISO-8859-15", "confidence": 73, @@ -1564,6 +2643,24 @@ "total": 216, "hitRate": 0.06944444444444445 }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, { "encoding": "ISO-8859-16", "confidence": 13, @@ -1573,6 +2670,33 @@ "total": 216, "hitRate": 0.046296296296296294 }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, { "encoding": "ISO-8859-4", "confidence": 9, @@ -1591,6 +2715,15 @@ "total": 216, "hitRate": 0.023148148148148147 }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, { "encoding": "ISO-8859-3", "confidence": 5, @@ -1619,634 +2752,16156 @@ "hitRate": 0.004629629629629629 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-5", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.245943193261, + "language": "ru", + "byteLogLikelihood": -6.162408104115, "hits": 0, "total": 216, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.692083742507, + "language": "ar", + "byteLogLikelihood": -4.007313397456, "hits": 0, "total": 216, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.942998477092, + "language": "el", + "byteLogLikelihood": -4.349754587225, "hits": 0, "total": 216, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -3.629332579687, + "language": "he", + "byteLogLikelihood": -4.279537403415, "hits": 0, "total": 216, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-15", - "language": "es" - }, - "predicted": { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-15", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 - }, - { - "encoding": "macintosh", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 - }, - { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 }, { - "encoding": "ISO-8859-4", - "confidence": 24, - "language": "lv", - "byteLogLikelihood": -5.009854802779, - "hits": 16, - "total": 200, - "hitRate": 0.08 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.542402054434, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -4.892914919217, - "hits": 13, - "total": 200, - "hitRate": 0.065 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 200, - "hitRate": 0.055 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 200, - "hitRate": 0.05 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.657121063601, + "hits": 0, + "total": 216, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -3.846028832089, - "hits": 3, - "total": 200, - "hitRate": 0.015 + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.28774788718, + "hits": 0, + "total": 216, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.303705458307, + "byteLogLikelihood": -6.245943193261, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.977292083619, + "byteLogLikelihood": -6.692083742507, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.203740544852, + "language": "sr", + "byteLogLikelihood": -5.942998477092, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -3.336328204376, - "hits": 0, - "total": 200, - "hitRate": 0 - }, - { - "encoding": "ISO-8859-3", - "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "uk", + "byteLogLikelihood": -3.629332579687, "hits": 0, - "total": 200, + "total": 216, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-15", - "language": "fr" + "encoding": "ISO-8859-1", + "language": "en" }, "predicted": { "encoding": "ISO-8859-15", - "confidence": 77, + "confidence": 29, "language": "fr", "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 }, - "encodingCorrect": true, - "languageCorrect": true, + "encodingCorrect": false, + "languageCorrect": false, "tiedAtTop": 1, "candidates": [ { "encoding": "ISO-8859-15", - "confidence": 77, + "confidence": 29, "language": "fr", "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "fr", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "fr", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 }, { "encoding": "macintosh", - "confidence": 71, + "confidence": 27, "language": "es", "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 }, { "encoding": "CP850", - "confidence": 71, + "confidence": 27, "language": "es", "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 }, { - "encoding": "ISO-8859-16", + "encoding": "windows-1257", "confidence": 11, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 }, { - "encoding": "ISO-8859-10", - "confidence": 10, - "language": "is", - "byteLogLikelihood": -4.510859506517, - "hits": 8, - "total": 228, - "hitRate": 0.03508771929824561 + "encoding": "ISO-8859-14", + "confidence": 11, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 }, { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "hu", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "windows-1250", + "confidence": 9, + "language": "hu", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-3", - "confidence": 5, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "encoding": "windows-1258", + "confidence": 9, + "language": "vi", + "byteLogLikelihood": -5.497168225293, "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 + "total": 132, + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-13", - "confidence": 5, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 228, - "hitRate": 0.017543859649122806 + "encoding": "ISO-8859-4", + "confidence": 6, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 3, + "total": 132, + "hitRate": 0.022727272727272728 }, { - "encoding": "ISO-8859-14", - "confidence": 3, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "encoding": "ISO-8859-16", + "confidence": 6, + "language": "ro", + "byteLogLikelihood": -5.176149732574, "hits": 3, - "total": 228, - "hitRate": 0.013157894736842105 + "total": 132, + "hitRate": 0.022727272727272728 }, { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, - "hits": 0, - "total": 228, - "hitRate": 0 + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 }, { - "encoding": "IBM866", + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 1, + "total": 132, + "hitRate": 0.007575757575757576 + }, + { + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", "byteLogLikelihood": -5.593471453839, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -4.728861658331, + "language": "ar", + "byteLogLikelihood": -6.113682179832, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.252383775789, + "language": "el", + "byteLogLikelihood": -2.654561516151, "hits": 0, - "total": 228, + "total": 132, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "ISO-8859-16", - "language": "ro" - }, - "predicted": { - "encoding": "ISO-8859-16", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "ISO-8859-16", - "confidence": 41, - "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 - }, - { - "encoding": "macintosh", - "confidence": 19, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 198, - "hitRate": 0.06565656565656566 - }, - { - "encoding": "CP850", - "confidence": 15, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 }, { - "encoding": "ISO-8859-15", - "confidence": 14, - "language": "fr", - "byteLogLikelihood": -4.903997266318, - "hits": 10, - "total": 202, - "hitRate": 0.04950495049504951 + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-3", - "confidence": 13, - "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 9, - "total": 200, - "hitRate": 0.045 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 8, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 201, - "hitRate": 0.029850746268656716 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 7, - "language": "lv", - "byteLogLikelihood": -4.936633427031, - "hits": 5, - "total": 202, - "hitRate": 0.024752475247524754 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 5, - "language": "is", - "byteLogLikelihood": -5.150687673188, - "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.974278583322, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 131, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 4, - "language": "cy", - "byteLogLikelihood": -4.539421788511, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 132, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.32130473174, + "byteLogLikelihood": -6.650279048587, "hits": 0, - "total": 201, + "total": 132, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.208609169552, + "byteLogLikelihood": -5.593471453839, "hits": 0, - "total": 201, + "total": 132, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.608648280371, + "language": "sr", + "byteLogLikelihood": -4.728861658331, "hits": 0, - "total": 201, + "total": 132, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -4.229886082374, + "language": "uk", + "byteLogLikelihood": -4.252383775789, "hits": 0, - "total": 202, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 0, + "total": 132, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-3", - "language": "mt" + "encoding": "ISO-8859-1", + "language": "es" }, "predicted": { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 1, + "tiedAtTop": 3, "candidates": [ { - "encoding": "ISO-8859-3", - "confidence": 52, - "language": "mt", - "byteLogLikelihood": -2.738213481781, - "hits": 32, - "total": 183, - "hitRate": 0.17486338797814208 + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { - "encoding": "CP852", - "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 8, - "total": 177, - "hitRate": 0.04519774011299435 + "encoding": "windows-1252", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 }, { "encoding": "macintosh", - "confidence": 11, + "confidence": 58, "language": "es", "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 24, + "language": "lt", + "byteLogLikelihood": -5.19295685089, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-4", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.009854802779, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -4.892914919217, + "hits": 13, + "total": 200, + "hitRate": 0.065 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 200, + "hitRate": 0.055 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -4.848531508941, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -3.846028832089, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.346153815501, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.412820202576, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.203740544852, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "macintosh", + "confidence": 71, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 11, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 228, + "hitRate": 0.013157894736842105 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "it" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "macintosh", + "confidence": 24, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 158, + "hitRate": 0.08227848101265822 + }, + { + "encoding": "ISO-8859-15", + "confidence": 24, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "CP850", + "confidence": 24, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "windows-1258", + "confidence": 16, + "language": "vi", + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1257", + "confidence": 15, + "language": "lt", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-4", + "confidence": 15, + "language": "lv", + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-14", + "confidence": 9, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 159, + "hitRate": 0.031446540880503145 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 159, + "hitRate": 0.025157232704402517 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -6.520621127559, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "fr", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingCorrect": true, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "fr", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "windows-1252", + "confidence": 83, + "language": "fr", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "macintosh", + "confidence": 63, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-15", + "confidence": 63, + "language": "fr", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "CP850", + "confidence": 63, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-14", + "confidence": 33, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 17, + "total": 151, + "hitRate": 0.11258278145695365 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-9", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1254", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-16", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-10", + "confidence": 11, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 151, + "hitRate": 0.033112582781456956 + }, + { + "encoding": "windows-1258", + "confidence": 3, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-3", + "confidence": 1, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-4", + "confidence": 1, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "CP852", + "confidence": 1, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 151, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "no" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "macintosh", + "confidence": 18, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-14", + "confidence": 18, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-15", + "confidence": 18, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "CP850", + "confidence": 18, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-10", + "confidence": 13, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 5, + "total": 114, + "hitRate": 0.043859649122807015 + }, + { + "encoding": "windows-1258", + "confidence": 10, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-2", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1250", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-16", + "confidence": 5, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 114, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.319245023186, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.641462690249, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.679407897139, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "es", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "es", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "es", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "es", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1257", + "confidence": 29, + "language": "lv", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "ISO-8859-13", + "confidence": 29, + "language": "lt", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "macintosh", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "CP850", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "ISO-8859-4", + "confidence": 23, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 11, + "total": 143, + "hitRate": 0.07692307692307693 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "ISO-8859-14", + "confidence": 10, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -4.524213150766, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-10", + "confidence": 6, + "language": "is", + "byteLogLikelihood": -4.06291963921, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.514449766028, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.846220145127, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.503504267293, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.91521504926, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.476992253447, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -3.774998078834, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.766179593299, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 143, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-1", + "language": "sv" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1258", + "confidence": 19, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "ISO-8859-10", + "confidence": 19, + "language": "is", + "byteLogLikelihood": -4.980036753423, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "macintosh", + "confidence": 17, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP850", + "confidence": 17, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "ISO-8859-9", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "windows-1254", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "windows-1257", + "confidence": 4, + "language": "et", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.287368473007, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.25594643617, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.801526726218, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.432153549797, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.390348855877, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.419477704429, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-10", + "language": "is" + }, + "predicted": { + "encoding": "ISO-8859-10", + "confidence": 38, + "language": "is", + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-10", + "confidence": 38, + "language": "is", + "byteLogLikelihood": -3.025836278694, + "hits": 22, + "total": 171, + "hitRate": 0.1286549707602339 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.633646206449, + "hits": 8, + "total": 171, + "hitRate": 0.04678362573099415 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.633646206449, + "hits": 8, + "total": 171, + "hitRate": 0.04678362573099415 + }, + { + "encoding": "ISO-8859-3", + "confidence": 12, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 7, + "total": 169, + "hitRate": 0.04142011834319527 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.642697796053, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "ISO-8859-13", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -3.989583472462, + "hits": 4, + "total": 171, + "hitRate": 0.023391812865497075 + }, + { + "encoding": "macintosh", + "confidence": 5, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "ISO-8859-4", + "confidence": 5, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 3, + "total": 171, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-15", + "confidence": 5, + "language": "es", + "byteLogLikelihood": -4.894502014278, + "hits": 3, + "total": 171, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "CP850", + "confidence": 5, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "ISO-8859-2", + "confidence": 3, + "language": "hu", + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 + }, + { + "encoding": "windows-1250", + "confidence": 3, + "language": "hu", + "byteLogLikelihood": -4.497848163857, + "hits": 2, + "total": 171, + "hitRate": 0.011695906432748537 + }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -4.646338703133, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -4.646338703133, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "ISO-8859-16", + "confidence": 1, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.838861551509, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.345124108708, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.963723644165, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.249634019938, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.84046735452, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.963723644165, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.249634019938, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.696464660765, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.644763656357, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.290788006393, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.026638569428, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.838861551509, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.795369889152, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.733983711968, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.657284871846, + "hits": 0, + "total": 171, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-13", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-13", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-4", + "confidence": 22, + "language": "lv", + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "ISO-8859-1", + "confidence": 12, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "confidence": 12, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "macintosh", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 168, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "confidence": 12, + "language": "fr", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "CP850", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 170, + "hitRate": 0.041176470588235294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "windows-1254", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 172, + "hitRate": 0.01744186046511628 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.6272413408, + "hits": 2, + "total": 173, + "hitRate": 0.011560693641618497 + }, + { + "encoding": "windows-1258", + "confidence": 1, + "language": "vi", + "byteLogLikelihood": -4.960186085121, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "IBM855", + "confidence": 1, + "language": "sr", + "byteLogLikelihood": -5.488073838002, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-16", + "confidence": 1, + "language": "ro", + "byteLogLikelihood": -4.659155714377, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.893109334109, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.773457169107, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.531827362419, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.924336543075, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.58967630288, + "hits": 0, + "total": 166, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.365896834008, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.252539473952, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.893109334109, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.441328843196, + "hits": 0, + "total": 173, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-14", + "language": "cy" + }, + "predicted": { + "encoding": "ISO-8859-14", + "confidence": 50, + "language": "cy", + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-14", + "confidence": 50, + "language": "cy", + "byteLogLikelihood": -3.988458644989, + "hits": 32, + "total": 189, + "hitRate": 0.1693121693121693 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "en", + "byteLogLikelihood": -4.955827057601, + "hits": 9, + "total": 189, + "hitRate": 0.047619047619047616 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "en", + "byteLogLikelihood": -4.955827057601, + "hits": 9, + "total": 189, + "hitRate": 0.047619047619047616 + }, + { + "encoding": "windows-1258", + "confidence": 14, + "language": "vi", + "byteLogLikelihood": -4.34192292436, + "hits": 9, + "total": 188, + "hitRate": 0.047872340425531915 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 188, + "hitRate": 0.03723404255319149 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "fr", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 189, + "hitRate": 0.037037037037037035 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 188, + "hitRate": 0.03723404255319149 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1250", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.159055299215, + "hits": 6, + "total": 189, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "lv", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 189, + "hitRate": 0.021164021164021163 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 189, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -4.009500374258, + "hits": 3, + "total": 189, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-9", + "confidence": 3, + "language": "tr", + "byteLogLikelihood": -4.613075694861, + "hits": 2, + "total": 189, + "hitRate": 0.010582010582010581 + }, + { + "encoding": "windows-1254", + "confidence": 3, + "language": "tr", + "byteLogLikelihood": -4.613075694861, + "hits": 2, + "total": 189, + "hitRate": 0.010582010582010581 + }, + { + "encoding": "CP852", + "confidence": 3, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 188, + "hitRate": 0.010638297872340425 + }, + { + "encoding": "ISO-8859-4", + "confidence": 1, + "language": "lv", + "byteLogLikelihood": -4.961907790704, + "hits": 1, + "total": 189, + "hitRate": 0.005291005291005291 + }, + { + "encoding": "ISO-8859-13", + "confidence": 1, + "language": "lt", + "byteLogLikelihood": -3.647025046455, + "hits": 1, + "total": 189, + "hitRate": 0.005291005291005291 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.350299845595, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.420534999272, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.254156765951, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.917404374874, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.467895804539, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.254156765951, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.917404374874, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.830407858741, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.229985622133, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -5.821976831991, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.350299845595, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.325879646284, + "hits": 0, + "total": 188, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.255815902698, + "hits": 0, + "total": 189, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -3.981643419755, + "hits": 0, + "total": 189, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "macintosh", + "confidence": 72, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "CP850", + "confidence": 70, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "ISO-8859-14", + "confidence": 20, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 15, + "total": 216, + "hitRate": 0.06944444444444445 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "ISO-8859-16", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 216, + "hitRate": 0.023148148148148147 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 216, + "hitRate": 0.009259259259259259 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 1, + "total": 216, + "hitRate": 0.004629629629629629 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.162408104115, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.542402054434, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.657121063601, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.28774788718, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.245943193261, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.942998477092, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "macintosh", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 24, + "language": "lt", + "byteLogLikelihood": -5.19295685089, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-4", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.009854802779, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -4.892914919217, + "hits": 13, + "total": 200, + "hitRate": 0.065 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 200, + "hitRate": 0.055 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -4.848531508941, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -3.846028832089, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.346153815501, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.412820202576, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.203740544852, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-15", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "macintosh", + "confidence": 71, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 11, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 228, + "hitRate": 0.013157894736842105 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-16", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-1", + "confidence": 26, + "language": "pt", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "confidence": 26, + "language": "pt", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "macintosh", + "confidence": 19, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 198, + "hitRate": 0.06565656565656566 + }, + { + "encoding": "CP850", + "confidence": 15, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.903997266318, + "hits": 10, + "total": 202, + "hitRate": 0.04950495049504951 + }, + { + "encoding": "ISO-8859-3", + "confidence": 13, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 9, + "total": 200, + "hitRate": 0.045 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.396820338954, + "hits": 6, + "total": 202, + "hitRate": 0.0297029702970297 + }, + { + "encoding": "CP852", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 201, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -4.936633427031, + "hits": 5, + "total": 202, + "hitRate": 0.024752475247524754 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.177254141958, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-10", + "confidence": 5, + "language": "is", + "byteLogLikelihood": -5.150687673188, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.974278583322, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.539421788511, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.828974536143, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.528215037204, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.239455511679, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.32130473174, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.608648280371, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "windows-1250", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "confidence": 11, + "language": "fr", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "confidence": 11, + "language": "fr", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "CP852", + "confidence": 11, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.506724065575, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "macintosh", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 124, + "hitRate": 0.024193548387096774 + }, + { + "encoding": "ISO-8859-13", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-15", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-16", + "confidence": 7, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "CP850", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 125, + "hitRate": 0.024 + }, + { + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -5.129943470839, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.402405269754, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.249199684222, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.164526575273, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.504431122267, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.566056982405, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.524252288486, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.108905030169, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.226561011787, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "sv", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "sv", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP852", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 135, + "hitRate": 0.05185185185185185 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 135, + "hitRate": 0.022222222222222223 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 134, + "hitRate": 0.022388059701492536 + }, + { + "encoding": "ISO-8859-15", + "confidence": 4, + "language": "fr", + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1257", + "confidence": 2, + "language": "et", + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "confidence": 2, + "language": "is", + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.950677184263, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.538035906584, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.6300671031, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.009772962617, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.010690688029, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.953752719217, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.606957349802, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.100966544791, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "pl" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "windows-1250", + "confidence": 37, + "language": "pl", + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 183, + "hitRate": 0.12568306010928962 + }, + { + "encoding": "CP852", + "confidence": 41, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 25, + "total": 182, + "hitRate": 0.13736263736263737 + }, + { + "encoding": "ISO-8859-1", + "confidence": 8, + "language": "es", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "confidence": 8, + "language": "es", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1258", + "confidence": 8, + "language": "vi", + "byteLogLikelihood": -5.343135518502, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-13", + "confidence": 6, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -4.794788079951, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-16", + "confidence": 3, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 183, + "hitRate": 0.01092896174863388 + }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "ISO-8859-3", + "confidence": 1, + "language": "mt", + "byteLogLikelihood": -4.476342508555, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -4.71573455878, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-14", + "confidence": 1, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.293914971567, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.002422325308, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.300217461931, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.002422325308, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.586271794593, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.538051035716, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.496246341796, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.447947678358, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.497577429688, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.300217461931, + "hits": 0, + "total": 183, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-2", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-1", + "confidence": 26, + "language": "pt", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "confidence": 26, + "language": "pt", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "macintosh", + "confidence": 19, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 198, + "hitRate": 0.06565656565656566 + }, + { + "encoding": "CP850", + "confidence": 15, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.903997266318, + "hits": 10, + "total": 202, + "hitRate": 0.04950495049504951 + }, + { + "encoding": "ISO-8859-3", + "confidence": 13, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 9, + "total": 200, + "hitRate": 0.045 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.396820338954, + "hits": 6, + "total": 202, + "hitRate": 0.0297029702970297 + }, + { + "encoding": "CP852", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 201, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -4.936633427031, + "hits": 5, + "total": 202, + "hitRate": 0.024752475247524754 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.177254141958, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-10", + "confidence": 5, + "language": "is", + "byteLogLikelihood": -5.150687673188, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.974278583322, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.539421788511, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.828974536143, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.528215037204, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.239455511679, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.32130473174, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.608648280371, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-3", + "language": "mt" + }, + "predicted": { + "encoding": "ISO-8859-3", + "confidence": 52, + "language": "mt", + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-3", + "confidence": 52, + "language": "mt", + "byteLogLikelihood": -2.738213481781, + "hits": 32, + "total": 183, + "hitRate": 0.17486338797814208 + }, + { + "encoding": "ISO-8859-9", + "confidence": 16, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 181, + "hitRate": 0.055248618784530384 + }, + { + "encoding": "windows-1254", + "confidence": 16, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 181, + "hitRate": 0.055248618784530384 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "pt", + "byteLogLikelihood": -4.81334289805, + "hits": 9, + "total": 181, + "hitRate": 0.049723756906077346 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "pt", + "byteLogLikelihood": -4.81334289805, + "hits": 9, + "total": 181, + "hitRate": 0.049723756906077346 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -4.980958307454, + "hits": 8, + "total": 182, + "hitRate": 0.04395604395604396 + }, + { + "encoding": "windows-1258", + "confidence": 13, + "language": "vi", + "byteLogLikelihood": -4.506236351586, + "hits": 8, + "total": 181, + "hitRate": 0.04419889502762431 + }, + { + "encoding": "CP852", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 8, + "total": 177, + "hitRate": 0.04519774011299435 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -4.945351018208, + "hits": 7, + "total": 183, + "hitRate": 0.03825136612021858 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 181, + "hitRate": 0.03867403314917127 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 177, + "hitRate": 0.03954802259887006 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "pl", + "byteLogLikelihood": -3.834078315244, + "hits": 6, + "total": 183, + "hitRate": 0.03278688524590164 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 182, + "hitRate": 0.03296703296703297 + }, + { + "encoding": "ISO-8859-16", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 6, + "total": 182, + "hitRate": 0.03296703296703297 + }, + { + "encoding": "ISO-8859-14", + "confidence": 6, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.986136327924, + "hits": 3, + "total": 182, + "hitRate": 0.016483516483516484 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 2, + "total": 183, + "hitRate": 0.01092896174863388 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.351251681057, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.198030006336, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.755123257218, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.572384450833, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.198030006336, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.755123257218, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.490920024976, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.383683979141, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.404570694946, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.943656762101, + "hits": 0, + "total": 181, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.572384450833, + "hits": 0, + "total": 182, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-4", + "language": "lv" + }, + "predicted": { + "encoding": "ISO-8859-4", + "confidence": 40, + "language": "lv", + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-4", + "confidence": 40, + "language": "lv", + "byteLogLikelihood": -3.1328616083, + "hits": 23, + "total": 170, + "hitRate": 0.13529411764705881 + }, + { + "encoding": "windows-1257", + "confidence": 31, + "language": "lt", + "byteLogLikelihood": -5.111410123766, + "hits": 18, + "total": 169, + "hitRate": 0.10650887573964497 + }, + { + "encoding": "CP850", + "confidence": 16, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 9, + "total": 168, + "hitRate": 0.05357142857142857 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "it", + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "it", + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "macintosh", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "es", + "byteLogLikelihood": -4.721042183854, + "hits": 8, + "total": 169, + "hitRate": 0.047337278106508875 + }, + { + "encoding": "CP852", + "confidence": 12, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 169, + "hitRate": 0.04142011834319527 + }, + { + "encoding": "ISO-8859-13", + "confidence": 10, + "language": "lt", + "byteLogLikelihood": -4.596975900219, + "hits": 6, + "total": 169, + "hitRate": 0.03550295857988166 + }, + { + "encoding": "ISO-8859-2", + "confidence": 8, + "language": "ro", + "byteLogLikelihood": -4.763992911083, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "windows-1250", + "confidence": 8, + "language": "ro", + "byteLogLikelihood": -4.846752537455, + "hits": 5, + "total": 169, + "hitRate": 0.029585798816568046 + }, + { + "encoding": "ISO-8859-3", + "confidence": 8, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 169, + "hitRate": 0.023668639053254437 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 169, + "hitRate": 0.023668639053254437 + }, + { + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -5.074758182528, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -4.725539964734, + "hits": 3, + "total": 169, + "hitRate": 0.01775147928994083 + }, + { + "encoding": "ISO-8859-16", + "confidence": 1, + "language": "ro", + "byteLogLikelihood": -4.572632236701, + "hits": 1, + "total": 169, + "hitRate": 0.005917159763313609 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.226735250772, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.427610697135, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.928827369245, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.704365179493, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.427610697135, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.928827369245, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.578946937177, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.610537015382, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.609505685025, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.267508614334, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.182018138172, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.65535318117, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 170, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-5", + "language": "ru" + }, + "predicted": { + "encoding": "ISO-8859-5", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-5", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + { + "encoding": "IBM866", + "confidence": 20, + "language": "ru", + "byteLogLikelihood": -5.798412758856, + "hits": 5, + "total": 75, + "hitRate": 0.06666666666666667 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 12, + "language": "ru", + "byteLogLikelihood": -5.66890403251, + "hits": 6, + "total": 147, + "hitRate": 0.04081632653061224 + }, + { + "encoding": "IBM855", + "confidence": 9, + "language": "ru", + "byteLogLikelihood": -4.826799236728, + "hits": 5, + "total": 166, + "hitRate": 0.030120481927710843 + }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -4.986561094704, + "hits": 1, + "total": 176, + "hitRate": 0.005681818181818182 + }, + { + "encoding": "KOI8-U", + "confidence": 1, + "language": "uk", + "byteLogLikelihood": -4.816522715875, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.219340428659, + "hits": 0, + "total": 152, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.141186484933, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 75, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.982255634278, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.941337938888, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.667544038277, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.841944488282, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -5.141186484933, + "hits": 0, + "total": 174, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.982255634278, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.371812787625, + "hits": 0, + "total": 113, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.847671496604, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.755485363394, + "hits": 0, + "total": 166, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.810914106722, + "hits": 0, + "total": 84, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977706457443, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 98, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.966015130424, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.103902669993, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.847671496604, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.83002806951, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.848242748706, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.06091613544, + "hits": 0, + "total": 178, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 162, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.078415577874, + "hits": 0, + "total": 166, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-6", + "language": "ar" + }, + "predicted": { + "encoding": "ISO-8859-6", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-6", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + { + "encoding": "windows-1256", + "confidence": 10, + "language": "ar", + "byteLogLikelihood": -3.968382111451, + "hits": 4, + "total": 110, + "hitRate": 0.03636363636363636 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 10, + "language": "uk", + "byteLogLikelihood": -5.359123221706, + "hits": 3, + "total": 82, + "hitRate": 0.036585365853658534 + }, + { + "encoding": "ISO-8859-8", + "confidence": 8, + "language": "he", + "byteLogLikelihood": -5.049228744152, + "hits": 2, + "total": 70, + "hitRate": 0.02857142857142857 + }, + { + "encoding": "windows-1255", + "confidence": 8, + "language": "he", + "byteLogLikelihood": -5.049228744152, + "hits": 2, + "total": 72, + "hitRate": 0.027777777777777776 + }, + { + "encoding": "ISO-8859-7", + "confidence": 5, + "language": "el", + "byteLogLikelihood": -4.984009642853, + "hits": 2, + "total": 109, + "hitRate": 0.01834862385321101 + }, + { + "encoding": "windows-1253", + "confidence": 5, + "language": "el", + "byteLogLikelihood": -4.984009642853, + "hits": 2, + "total": 109, + "hitRate": 0.01834862385321101 + }, + { + "encoding": "KOI8-R", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.040654549299, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "KOI8-U", + "confidence": 2, + "language": "uk", + "byteLogLikelihood": -5.174167309306, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.69584393439, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -4.951980582514, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.534213772279, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.933475282212, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -4.951980582514, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.223507468988, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.69584393439, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.933475282212, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.72024182268, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029794115635, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.643135408712, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.068419445019, + "hits": 0, + "total": 72, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.811360688713, + "hits": 0, + "total": 100, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 55, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.047518534687, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.155285808468, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.051594021628, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.063690311528, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.799641758471, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "de", + "byteLogLikelihood": -4.755317425962, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.115876064699, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 98, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 100, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-7", + "language": "el" + }, + "predicted": { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "windows-1253", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "KOI8-U", + "confidence": 8, + "language": "uk", + "byteLogLikelihood": -6.254780178237, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "IBM855", + "confidence": 7, + "language": "ru", + "byteLogLikelihood": -5.358823187498, + "hits": 3, + "total": 118, + "hitRate": 0.025423728813559324 + }, + { + "encoding": "KOI8-R", + "confidence": 4, + "language": "ru", + "byteLogLikelihood": -6.302671202821, + "hits": 2, + "total": 134, + "hitRate": 0.014925373134328358 + }, + { + "encoding": "ISO-8859-6", + "confidence": 3, + "language": "ar", + "byteLogLikelihood": -5.268243094623, + "hits": 1, + "total": 81, + "hitRate": 0.012345679012345678 + }, + { + "encoding": "ISO-8859-8", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "windows-1255", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -4.203232324163, + "hits": 1, + "total": 131, + "hitRate": 0.007633587786259542 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.165809265202, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288697198304, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.013410783916, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.827626146169, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.681848366371, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.483358374511, + "hits": 0, + "total": 52, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.55248803416, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 116, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.945729021813, + "hits": 0, + "total": 129, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.015268979336, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.576407109739, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.890771320266, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.788878603493, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.603974037796, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.093353952573, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 96, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-8", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "windows-1255", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "ISO-8859-7", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "IBM855", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.394495827364, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.124491051534, + "hits": 0, + "total": 73, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.199203901043, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.32364067176, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.639272141503, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029681805165, + "hits": 0, + "total": 99, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.31385343816, + "hits": 0, + "total": 48, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.40669720385, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.393692981541, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 72, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.134202940863, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.956632184798, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.146385148445, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.745792460039, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.842271132243, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.813633859942, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "fr", + "byteLogLikelihood": -4.521941300627, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.048864674087, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 85, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 89, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "ISO-8859-9", + "language": "tr" + }, + "predicted": { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "ISO-8859-3", + "confidence": 22, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 10, + "total": 134, + "hitRate": 0.07462686567164178 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-10", + "confidence": 17, + "language": "is", + "byteLogLikelihood": -4.720106560713, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "ISO-8859-16", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -4.93302665818, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-13", + "confidence": 11, + "language": "lt", + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.652724693026, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.197431316193, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.576073075491, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.714883328648, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.085579959517, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.742947096814, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.22610845571, + "hits": 0, + "total": 134, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-R", + "language": "ru" + }, + "predicted": { + "encoding": "KOI8-R", + "confidence": 43, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-R", + "confidence": 43, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 26, + "total": 179, + "hitRate": 0.1452513966480447 + }, + { + "encoding": "KOI8-U", + "confidence": 28, + "language": "uk", + "byteLogLikelihood": -3.443215105753, + "hits": 17, + "total": 179, + "hitRate": 0.09497206703910614 + }, + { + "encoding": "windows-874", + "confidence": 12, + "language": "th", + "byteLogLikelihood": -4.705552828452, + "hits": 7, + "total": 162, + "hitRate": 0.043209876543209874 + }, + { + "encoding": "ISO-8859-7", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.840779048091, + "hits": 4, + "total": 177, + "hitRate": 0.022598870056497175 + }, + { + "encoding": "windows-1253", + "confidence": 6, + "language": "el", + "byteLogLikelihood": -5.840779048091, + "hits": 4, + "total": 177, + "hitRate": 0.022598870056497175 + }, + { + "encoding": "ISO-8859-5", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.543217452567, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.150827322214, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "nl", + "byteLogLikelihood": -4.900684399759, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.759099059366, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.899711049132, + "hits": 0, + "total": 7, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.056032280413, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.313908864785, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "nl", + "byteLogLikelihood": -4.900684399759, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.056032280413, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.899711049132, + "hits": 0, + "total": 27, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.796335743252, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.91533239831, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.174619785116, + "hits": 0, + "total": 174, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.670016162969, + "hits": 0, + "total": 7, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.983474020054, + "hits": 0, + "total": 96, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.595875032476, + "hits": 0, + "total": 116, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.182298321669, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.105602464686, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "de", + "byteLogLikelihood": -4.935090625757, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.162378596536, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 94, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "KOI8-U", + "language": "uk" + }, + "predicted": { + "encoding": "KOI8-U", + "confidence": 32, + "language": "uk", + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "KOI8-U", + "confidence": 32, + "language": "uk", + "byteLogLikelihood": -3.34369756932, + "hits": 19, + "total": 177, + "hitRate": 0.10734463276836158 + }, + { + "encoding": "KOI8-R", + "confidence": 17, + "language": "ru", + "byteLogLikelihood": -3.510012695801, + "hits": 10, + "total": 172, + "hitRate": 0.05813953488372093 + }, + { + "encoding": "windows-874", + "confidence": 11, + "language": "th", + "byteLogLikelihood": -4.843281209678, + "hits": 6, + "total": 163, + "hitRate": 0.03680981595092025 + }, + { + "encoding": "ISO-8859-7", + "confidence": 8, + "language": "el", + "byteLogLikelihood": -5.876007286158, + "hits": 5, + "total": 168, + "hitRate": 0.02976190476190476 + }, + { + "encoding": "windows-1253", + "confidence": 8, + "language": "el", + "byteLogLikelihood": -5.876007286158, + "hits": 5, + "total": 168, + "hitRate": 0.02976190476190476 + }, + { + "encoding": "ISO-8859-1", + "confidence": 1, + "language": "nl", + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "ISO-8859-5", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -5.626926512473, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 + }, + { + "encoding": "ISO-8859-6", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -4.726706027838, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "windows-1252", + "confidence": 1, + "language": "nl", + "byteLogLikelihood": -4.915420747009, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -4.693044688745, + "hits": 1, + "total": 168, + "hitRate": 0.005952380952380952 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.128376292359, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.883441366246, + "hits": 0, + "total": 7, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.055144062354, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.295907791508, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -5.055144062354, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.883441366246, + "hits": 0, + "total": 27, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.922686041886, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.213223244884, + "hits": 0, + "total": 164, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.582632026409, + "hits": 0, + "total": 22, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.922515418323, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 89, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.589678128375, + "hits": 0, + "total": 117, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.110283442945, + "hits": 0, + "total": 168, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.944199711559, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.148788659657, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 92, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 94, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "de" + }, + "predicted": { + "encoding": "macintosh", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-1", + "confidence": 74, + "language": "nl", + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "ISO-8859-15", + "confidence": 74, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 53, + "total": 214, + "hitRate": 0.24766355140186916 + }, + { + "encoding": "windows-1252", + "confidence": 70, + "language": "nl", + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "CP850", + "confidence": 69, + "language": "fr", + "byteLogLikelihood": -5.003946305945, + "hits": 50, + "total": 216, + "hitRate": 0.23148148148148148 + }, + { + "encoding": "ISO-8859-14", + "confidence": 22, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 16, + "total": 214, + "hitRate": 0.07476635514018691 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 214, + "hitRate": 0.0514018691588785 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 216, + "hitRate": 0.05092592592592592 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 11, + "total": 214, + "hitRate": 0.0514018691588785 + }, + { + "encoding": "ISO-8859-9", + "confidence": 14, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 10, + "total": 214, + "hitRate": 0.04672897196261682 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 214, + "hitRate": 0.037383177570093455 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 214, + "hitRate": 0.037383177570093455 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 3, + "total": 214, + "hitRate": 0.014018691588785047 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 214, + "hitRate": 0.009345794392523364 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.661807706795, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 214, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.534754518494, + "hits": 0, + "total": 215, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.345510152227, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -6.337519079447, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "es" + }, + "predicted": { + "encoding": "macintosh", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "ISO-8859-15", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 25, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, + { + "encoding": "ISO-8859-4", + "confidence": 25, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 17, + "total": 199, + "hitRate": 0.08542713567839195 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 13, + "total": 199, + "hitRate": 0.06532663316582915 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 199, + "hitRate": 0.05527638190954774 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 199, + "hitRate": 0.02512562814070352 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 3, + "total": 199, + "hitRate": 0.01507537688442211 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.624741392945, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 199, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "macintosh", + "language": "fr" + }, + "predicted": { + "encoding": "macintosh", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "macintosh", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-1", + "confidence": 75, + "language": "nl", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "ISO-8859-15", + "confidence": 75, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 56, + "total": 224, + "hitRate": 0.25 + }, + { + "encoding": "windows-1252", + "confidence": 71, + "language": "nl", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "de", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 12, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "ISO-8859-10", + "confidence": 12, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "ISO-8859-16", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 224, + "hitRate": 0.04017857142857143 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 224, + "hitRate": 0.03125 + }, + { + "encoding": "ISO-8859-3", + "confidence": 8, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 6, + "total": 224, + "hitRate": 0.026785714285714284 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1254", + "confidence": 6, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 5, + "total": 224, + "hitRate": 0.022321428571428572 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 224, + "hitRate": 0.017857142857142856 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 224, + "hitRate": 0.013392857142857142 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.932245187448, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.998936561947, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -3.576182148392, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.998936561947, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "cs" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "windows-1250", + "confidence": 30, + "language": "cs", + "byteLogLikelihood": -3.429159108442, + "hits": 13, + "total": 126, + "hitRate": 0.10317460317460317 + }, + { + "encoding": "ISO-8859-1", + "confidence": 11, + "language": "fr", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1252", + "confidence": 11, + "language": "fr", + "byteLogLikelihood": -4.5670392286, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "CP852", + "confidence": 11, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 126, + "hitRate": 0.03968253968253968 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.506724065575, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "macintosh", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 124, + "hitRate": 0.024193548387096774 + }, + { + "encoding": "ISO-8859-13", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.997937219301, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-15", + "confidence": 7, + "language": "fr", + "byteLogLikelihood": -4.081579721495, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-16", + "confidence": 7, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "CP850", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 3, + "total": 125, + "hitRate": 0.024 + }, + { + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -4.849273210708, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -5.129943470839, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 2, + "total": 126, + "hitRate": 0.015873015873015872 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.402405269754, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.249199684222, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.401446196881, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.467509652199, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.164526575273, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.504431122267, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.566056982405, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.524252288486, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.254067466067, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.108905030169, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.226561011787, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.43713857777, + "hits": 0, + "total": 126, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "hu" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "windows-1250", + "confidence": 46, + "language": "hu", + "byteLogLikelihood": -3.161205902203, + "hits": 21, + "total": 136, + "hitRate": 0.15441176470588236 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "sv", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "sv", + "byteLogLikelihood": -4.507887190294, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP852", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 135, + "hitRate": 0.05185185185185185 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "macintosh", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 135, + "hitRate": 0.022222222222222223 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 134, + "hitRate": 0.022388059701492536 + }, + { + "encoding": "ISO-8859-15", + "confidence": 4, + "language": "fr", + "byteLogLikelihood": -4.274525549305, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "windows-1257", + "confidence": 2, + "language": "et", + "byteLogLikelihood": -4.453831056838, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-10", + "confidence": 2, + "language": "is", + "byteLogLikelihood": -3.90025153387, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.950677184263, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 1, + "total": 136, + "hitRate": 0.007352941176470588 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.538035906584, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.6300671031, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.430247904624, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.984546441118, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.009772962617, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.010690688029, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.953752719217, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.606957349802, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.485271956182, + "hits": 0, + "total": 135, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.100966544791, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.187765411941, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "pl" + }, + "predicted": { + "encoding": "windows-1250", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1250", + "confidence": 40, + "language": "pl", + "byteLogLikelihood": -3.52910527172, + "hits": 25, + "total": 184, + "hitRate": 0.1358695652173913 + }, + { + "encoding": "ISO-8859-2", + "confidence": 37, + "language": "pl", + "byteLogLikelihood": -4.297518988235, + "hits": 23, + "total": 184, + "hitRate": 0.125 + }, + { + "encoding": "CP852", + "confidence": 41, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 25, + "total": 182, + "hitRate": 0.13736263736263737 + }, + { + "encoding": "ISO-8859-1", + "confidence": 8, + "language": "es", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1252", + "confidence": 8, + "language": "es", + "byteLogLikelihood": -4.919980925828, + "hits": 5, + "total": 183, + "hitRate": 0.0273224043715847 + }, + { + "encoding": "windows-1257", + "confidence": 6, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.343135518502, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-13", + "confidence": 6, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "ISO-8859-15", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.396558412648, + "hits": 4, + "total": 183, + "hitRate": 0.02185792349726776 + }, + { + "encoding": "CP850", + "confidence": 6, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 182, + "hitRate": 0.02197802197802198 + }, + { + "encoding": "macintosh", + "confidence": 4, + "language": "es", + "byteLogLikelihood": -4.826691858304, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-4", + "confidence": 4, + "language": "lv", + "byteLogLikelihood": -4.704684722593, + "hits": 3, + "total": 184, + "hitRate": 0.016304347826086956 + }, + { + "encoding": "ISO-8859-3", + "confidence": 3, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -4.71573455878, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-16", + "confidence": 3, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 184, + "hitRate": 0.010869565217391304 + }, + { + "encoding": "ISO-8859-9", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "windows-1254", + "confidence": 1, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 1, + "total": 183, + "hitRate": 0.00546448087431694 + }, + { + "encoding": "ISO-8859-14", + "confidence": 1, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 184, + "hitRate": 0.005434782608695652 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.293914971567, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.848389618517, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.300217461931, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.848389618517, + "hits": 0, + "total": 184, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -5.470147067075, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 182, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.163001081631, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.538051035716, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.496246341796, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.447947678358, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.204348837508, + "hits": 0, + "total": 183, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.955485669412, + "hits": 0, + "total": 184, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1250", + "language": "ro" + }, + "predicted": { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-2", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "windows-1250", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-16", + "confidence": 41, + "language": "ro", + "byteLogLikelihood": -2.684539297542, + "hits": 28, + "total": 202, + "hitRate": 0.13861386138613863 + }, + { + "encoding": "ISO-8859-1", + "confidence": 26, + "language": "pt", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "windows-1252", + "confidence": 26, + "language": "pt", + "byteLogLikelihood": -4.919980925828, + "hits": 18, + "total": 202, + "hitRate": 0.0891089108910891 + }, + { + "encoding": "macintosh", + "confidence": 19, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 198, + "hitRate": 0.06565656565656566 + }, + { + "encoding": "CP850", + "confidence": 15, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 10, + "total": 199, + "hitRate": 0.05025125628140704 + }, + { + "encoding": "ISO-8859-15", + "confidence": 14, + "language": "fr", + "byteLogLikelihood": -4.903997266318, + "hits": 10, + "total": 202, + "hitRate": 0.04950495049504951 + }, + { + "encoding": "ISO-8859-3", + "confidence": 13, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 9, + "total": 200, + "hitRate": 0.045 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.021854801345, + "hits": 7, + "total": 202, + "hitRate": 0.034653465346534656 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.396820338954, + "hits": 6, + "total": 202, + "hitRate": 0.0297029702970297 + }, + { + "encoding": "CP852", + "confidence": 8, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 201, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -4.936633427031, + "hits": 5, + "total": 202, + "hitRate": 0.024752475247524754 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.177254141958, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-10", + "confidence": 5, + "language": "is", + "byteLogLikelihood": -5.150687673188, + "hits": 4, + "total": 202, + "hitRate": 0.019801980198019802 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -4.974278583322, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-14", + "confidence": 4, + "language": "cy", + "byteLogLikelihood": -4.539421788511, + "hits": 3, + "total": 202, + "hitRate": 0.01485148514851485 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.828974536143, + "hits": 0, + "total": 199, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.33505338151, + "hits": 0, + "total": 202, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.197813655274, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.528215037204, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.239455511679, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.42548867306, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.32130473174, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.208609169552, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.608648280371, + "hits": 0, + "total": 201, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.229886082374, + "hits": 0, + "total": 202, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1251", + "language": "ru" + }, + "predicted": { + "encoding": "windows-1251", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1251", + "confidence": 41, + "language": "ru", + "byteLogLikelihood": -3.313760777855, + "hits": 25, + "total": 179, + "hitRate": 0.13966480446927373 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 34, + "language": "ru", + "byteLogLikelihood": -3.411211853364, + "hits": 20, + "total": 175, + "hitRate": 0.11428571428571428 + }, + { + "encoding": "IBM855", + "confidence": 9, + "language": "ru", + "byteLogLikelihood": -5.604243216214, + "hits": 5, + "total": 164, + "hitRate": 0.03048780487804878 + }, + { + "encoding": "KOI8-U", + "confidence": 5, + "language": "uk", + "byteLogLikelihood": -6.290472763796, + "hits": 3, + "total": 179, + "hitRate": 0.01675977653631285 + }, + { + "encoding": "ISO-8859-8", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -5.552676456081, + "hits": 1, + "total": 165, + "hitRate": 0.006060606060606061 + }, + { + "encoding": "KOI8-R", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -6.346028593552, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-14", + "confidence": 1, + "language": "cy", + "byteLogLikelihood": -4.724449359889, + "hits": 1, + "total": 179, + "hitRate": 0.00558659217877095 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.054854043559, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.827625181516, + "hits": 0, + "total": 129, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.209243802684, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.61521723683, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.899487478125, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.446015416921, + "hits": 0, + "total": 88, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.059444422239, + "hits": 0, + "total": 167, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.904224209153, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.79659105312, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.939249573759, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.66408545306, + "hits": 0, + "total": 176, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "fr", + "byteLogLikelihood": -4.760478683076, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.931296681486, + "hits": 0, + "total": 179, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 155, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "da" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + { + "encoding": "windows-1252", + "confidence": 52, + "language": "da", + "byteLogLikelihood": -3.146558879633, + "hits": 21, + "total": 120, + "hitRate": 0.175 + }, + { + "encoding": "macintosh", + "confidence": 30, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "ISO-8859-15", + "confidence": 30, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "CP850", + "confidence": 30, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 12, + "total": 120, + "hitRate": 0.1 + }, + { + "encoding": "ISO-8859-14", + "confidence": 15, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-9", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1254", + "confidence": 10, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1257", + "confidence": 10, + "language": "lt", + "byteLogLikelihood": -4.934473933131, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-10", + "confidence": 7, + "language": "is", + "byteLogLikelihood": -4.717529149813, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 5, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 120, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.658005606594, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.145958228046, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.825649766807, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.736335095675, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.689401143463, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.231348206195, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.861023356308, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.888170123788, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.476992253447, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.825649766807, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.685745819823, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288232392491, + "hits": 0, + "total": 120, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "de" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "windows-1252", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "ISO-8859-15", + "confidence": 73, + "language": "de", + "byteLogLikelihood": -3.169712077519, + "hits": 53, + "total": 216, + "hitRate": 0.24537037037037038 + }, + { + "encoding": "macintosh", + "confidence": 72, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 52, + "total": 215, + "hitRate": 0.24186046511627907 + }, + { + "encoding": "CP850", + "confidence": 70, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 51, + "total": 216, + "hitRate": 0.2361111111111111 + }, + { + "encoding": "ISO-8859-14", + "confidence": 20, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 15, + "total": 216, + "hitRate": 0.06944444444444445 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "ISO-8859-16", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 216, + "hitRate": 0.046296296296296294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -4.5258676709, + "hits": 9, + "total": 216, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 7, + "total": 216, + "hitRate": 0.032407407407407406 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 216, + "hitRate": 0.023148148148148147 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 216, + "hitRate": 0.018518518518518517 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 216, + "hitRate": 0.009259259259259259 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 1, + "total": 216, + "hitRate": 0.004629629629629629 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.162408104115, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.007313397456, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.542402054434, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.349754587225, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.279537403415, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.318835143108, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.657121063601, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.28774788718, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.245943193261, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.942998477092, + "hits": 0, + "total": 216, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.629332579687, + "hits": 0, + "total": 216, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "en" + }, + "predicted": { + "encoding": "ISO-8859-15", + "confidence": 29, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-15", + "confidence": 29, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 13, + "total": 132, + "hitRate": 0.09848484848484848 + }, + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "fr", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "fr", + "byteLogLikelihood": -3.310543013394, + "hits": 21, + "total": 132, + "hitRate": 0.1590909090909091 + }, + { + "encoding": "macintosh", + "confidence": 27, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 + }, + { + "encoding": "CP850", + "confidence": 27, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 12, + "total": 132, + "hitRate": 0.09090909090909091 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-14", + "confidence": 11, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 9, + "language": "hu", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1250", + "confidence": 9, + "language": "hu", + "byteLogLikelihood": -3.54961738678, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "windows-1258", + "confidence": 9, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 132, + "hitRate": 0.030303030303030304 + }, + { + "encoding": "ISO-8859-4", + "confidence": 6, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 3, + "total": 132, + "hitRate": 0.022727272727272728 + }, + { + "encoding": "ISO-8859-16", + "confidence": 6, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 3, + "total": 132, + "hitRate": 0.022727272727272728 + }, + { + "encoding": "ISO-8859-9", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "windows-1254", + "confidence": 4, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 132, + "hitRate": 0.015151515151515152 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 1, + "total": 132, + "hitRate": 0.007575757575757576 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 131, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 0, + "total": 132, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "es" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "windows-1252", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "ISO-8859-15", + "confidence": 61, + "language": "es", + "byteLogLikelihood": -3.834657935767, + "hits": 41, + "total": 200, + "hitRate": 0.205 + }, + { + "encoding": "macintosh", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 199, + "hitRate": 0.19597989949748743 + }, + { + "encoding": "CP850", + "confidence": 58, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 39, + "total": 200, + "hitRate": 0.195 + }, + { + "encoding": "windows-1257", + "confidence": 24, + "language": "lt", + "byteLogLikelihood": -5.19295685089, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-4", + "confidence": 24, + "language": "lv", + "byteLogLikelihood": -5.009854802779, + "hits": 16, + "total": 200, + "hitRate": 0.08 + }, + { + "encoding": "ISO-8859-13", + "confidence": 19, + "language": "lt", + "byteLogLikelihood": -4.892914919217, + "hits": 13, + "total": 200, + "hitRate": 0.065 + }, + { + "encoding": "ISO-8859-14", + "confidence": 16, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 11, + "total": 200, + "hitRate": 0.055 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "cs", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 10, + "total": 200, + "hitRate": 0.05 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 200, + "hitRate": 0.025 + }, + { + "encoding": "windows-1258", + "confidence": 4, + "language": "vi", + "byteLogLikelihood": -4.848531508941, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-10", + "confidence": 4, + "language": "is", + "byteLogLikelihood": -3.846028832089, + "hits": 3, + "total": 200, + "hitRate": 0.015 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.346153815501, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.106555532888, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.411159813207, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.300604963183, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.412820202576, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.977292083619, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.203740544852, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.336328204376, + "hits": 0, + "total": 200, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 200, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "fr" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 3, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "windows-1252", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "ISO-8859-15", + "confidence": 77, + "language": "fr", + "byteLogLikelihood": -2.06619631493, + "hits": 59, + "total": 228, + "hitRate": 0.25877192982456143 + }, + { + "encoding": "macintosh", + "confidence": 71, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "CP850", + "confidence": 71, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 54, + "total": 228, + "hitRate": 0.23684210526315788 + }, + { + "encoding": "ISO-8859-2", + "confidence": 11, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "windows-1250", + "confidence": 11, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 9, + "total": 228, + "hitRate": 0.039473684210526314 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "et", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-4", + "confidence": 7, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 6, + "total": 228, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "windows-1258", + "confidence": 5, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-13", + "confidence": 5, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 228, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 3, + "total": 228, + "hitRate": 0.013157894736842105 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 224, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 228, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 228, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "it" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "windows-1252", + "confidence": 58, + "language": "it", + "byteLogLikelihood": -3.533686564708, + "hits": 31, + "total": 159, + "hitRate": 0.1949685534591195 + }, + { + "encoding": "macintosh", + "confidence": 24, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 158, + "hitRate": 0.08227848101265822 + }, + { + "encoding": "ISO-8859-15", + "confidence": 24, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "CP850", + "confidence": 24, + "language": "es", + "byteLogLikelihood": -5.010635294096, + "hits": 13, + "total": 159, + "hitRate": 0.08176100628930817 + }, + { + "encoding": "windows-1258", + "confidence": 16, + "language": "vi", + "byteLogLikelihood": -3.012261575505, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "pl", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "windows-1257", + "confidence": 15, + "language": "lt", + "byteLogLikelihood": -5.19295685089, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-4", + "confidence": 15, + "language": "lv", + "byteLogLikelihood": -1.974081026022, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 8, + "total": 159, + "hitRate": 0.050314465408805034 + }, + { + "encoding": "ISO-8859-14", + "confidence": 9, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 159, + "hitRate": 0.031446540880503145 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 159, + "hitRate": 0.025157232704402517 + }, + { + "encoding": "ISO-8859-9", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "windows-1254", + "confidence": 5, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 159, + "hitRate": 0.018867924528301886 + }, + { + "encoding": "ISO-8859-10", + "confidence": 3, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -3.332204510175, + "hits": 2, + "total": 159, + "hitRate": 0.012578616352201259 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -6.120297418951, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.629660094454, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.496507561466, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.954414124223, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -6.520621127559, + "hits": 0, + "total": 159, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -2.533200659147, + "hits": 0, + "total": 159, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "nl" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "fr", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 83, + "language": "fr", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "windows-1252", + "confidence": 83, + "language": "fr", + "byteLogLikelihood": -3.295836866004, + "hits": 42, + "total": 151, + "hitRate": 0.2781456953642384 + }, + { + "encoding": "macintosh", + "confidence": 63, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-15", + "confidence": 63, + "language": "fr", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "CP850", + "confidence": 63, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 32, + "total": 151, + "hitRate": 0.2119205298013245 + }, + { + "encoding": "ISO-8859-14", + "confidence": 33, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 17, + "total": 151, + "hitRate": 0.11258278145695365 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-9", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "hu", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "windows-1254", + "confidence": 13, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-16", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 7, + "total": 151, + "hitRate": 0.046357615894039736 + }, + { + "encoding": "ISO-8859-10", + "confidence": 11, + "language": "is", + "byteLogLikelihood": -4.510859506517, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "et", + "byteLogLikelihood": -4.934473933131, + "hits": 5, + "total": 151, + "hitRate": 0.033112582781456956 + }, + { + "encoding": "windows-1258", + "confidence": 3, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-13", + "confidence": 3, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 151, + "hitRate": 0.013245033112582781 + }, + { + "encoding": "ISO-8859-3", + "confidence": 1, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-4", + "confidence": 1, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "CP852", + "confidence": 1, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 151, + "hitRate": 0.006622516556291391 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.612642200827, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -2.654561516151, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.405884662832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -3.57655026914, + "hits": 0, + "total": 150, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.593471453839, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.728861658331, + "hits": 0, + "total": 151, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.252383775789, + "hits": 0, + "total": 151, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "no" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "windows-1252", + "confidence": 47, + "language": "no", + "byteLogLikelihood": -3.052495812716, + "hits": 18, + "total": 114, + "hitRate": 0.15789473684210525 + }, + { + "encoding": "macintosh", + "confidence": 18, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-14", + "confidence": 18, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-15", + "confidence": 18, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "CP850", + "confidence": 18, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 7, + "total": 114, + "hitRate": 0.06140350877192982 + }, + { + "encoding": "ISO-8859-10", + "confidence": 13, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 5, + "total": 114, + "hitRate": 0.043859649122807015 + }, + { + "encoding": "windows-1258", + "confidence": 10, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, + { + "encoding": "ISO-8859-2", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1250", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -4.354336342997, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -5.075173815234, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "windows-1257", + "confidence": 7, + "language": "lt", + "byteLogLikelihood": -4.934473933131, + "hits": 3, + "total": 114, + "hitRate": 0.02631578947368421 + }, + { + "encoding": "ISO-8859-16", + "confidence": 5, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 114, + "hitRate": 0.017543859649122806 + }, + { + "encoding": "ISO-8859-3", + "confidence": 2, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-4", + "confidence": 2, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-13", + "confidence": 2, + "language": "lt", + "byteLogLikelihood": -4.319245023186, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 114, + "hitRate": 0.008771929824561403 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.641462690249, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.154384602589, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -2.647504477241, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -6.061456918928, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.650279048587, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.679407897139, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.753215851617, + "hits": 0, + "total": 114, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "pt" + }, + "predicted": { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "es", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "ISO-8859-15", + "confidence": 33, + "language": "es", + "byteLogLikelihood": -3.970754139021, + "hits": 16, + "total": 143, + "hitRate": 0.11188811188811189 + }, + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "es", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "es", + "byteLogLikelihood": -4.059947322987, + "hits": 24, + "total": 143, + "hitRate": 0.16783216783216784 + }, + { + "encoding": "windows-1257", + "confidence": 29, + "language": "lv", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "ISO-8859-13", + "confidence": 29, + "language": "lt", + "byteLogLikelihood": -4.777390389123, + "hits": 14, + "total": 143, + "hitRate": 0.0979020979020979 + }, + { + "encoding": "macintosh", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "CP850", + "confidence": 31, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 15, + "total": 143, + "hitRate": 0.1048951048951049 + }, + { + "encoding": "ISO-8859-4", + "confidence": 23, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 11, + "total": 143, + "hitRate": 0.07692307692307693 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "hu", + "byteLogLikelihood": -3.361609233757, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "ISO-8859-14", + "confidence": 10, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 5, + "total": 143, + "hitRate": 0.03496503496503497 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -4.524213150766, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-10", + "confidence": 6, + "language": "is", + "byteLogLikelihood": -4.06291963921, + "hits": 3, + "total": 143, + "hitRate": 0.02097902097902098 + }, + { + "encoding": "ISO-8859-9", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "windows-1254", + "confidence": 2, + "language": "tr", + "byteLogLikelihood": -4.58869627797, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "CP852", + "confidence": 2, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 1, + "total": 143, + "hitRate": 0.006993006993006993 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.514449766028, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.846220145127, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.017269736716, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.50352863905, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.503504267293, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.91521504926, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.518796947367, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.476992253447, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.338763438513, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -3.774998078834, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.766179593299, + "hits": 0, + "total": 143, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 0, + "total": 143, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1252", + "language": "sv" + }, + "predicted": { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-1", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1252", + "confidence": 50, + "language": "sv", + "byteLogLikelihood": -3.23645137845, + "hits": 23, + "total": 136, + "hitRate": 0.16911764705882354 + }, + { + "encoding": "windows-1258", + "confidence": 19, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "ISO-8859-10", + "confidence": 19, + "language": "is", + "byteLogLikelihood": -4.980036753423, + "hits": 9, + "total": 136, + "hitRate": 0.0661764705882353 + }, + { + "encoding": "macintosh", + "confidence": 17, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-15", + "confidence": 17, + "language": "de", + "byteLogLikelihood": -4.137512330246, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "CP850", + "confidence": 17, + "language": "es", + "byteLogLikelihood": -5.003946305945, + "hits": 8, + "total": 136, + "hitRate": 0.058823529411764705 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "hu", + "byteLogLikelihood": -5.159055299215, + "hits": 7, + "total": 136, + "hitRate": 0.051470588235294115 + }, + { + "encoding": "ISO-8859-9", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "windows-1254", + "confidence": 11, + "language": "tr", + "byteLogLikelihood": -4.93784727915, + "hits": 5, + "total": 136, + "hitRate": 0.03676470588235294 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 4, + "total": 136, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "windows-1257", + "confidence": 4, + "language": "et", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-3", + "confidence": 4, + "language": "mt", + "byteLogLikelihood": -4.210065739287, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-13", + "confidence": 4, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "CP852", + "confidence": 4, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -3.287368473007, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -3.876181596107, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -3.066153183614, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.25594643617, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.801526726218, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.432153549797, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.390348855877, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.652362971667, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.419477704429, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.393743210995, + "hits": 0, + "total": 136, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 136, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1253", + "language": "el" + }, + "predicted": { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-7", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "windows-1253", + "confidence": 51, + "language": "el", + "byteLogLikelihood": -3.435199502, + "hits": 23, + "total": 134, + "hitRate": 0.17164179104477612 + }, + { + "encoding": "KOI8-U", + "confidence": 8, + "language": "uk", + "byteLogLikelihood": -6.254780178237, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "IBM855", + "confidence": 7, + "language": "ru", + "byteLogLikelihood": -5.358823187498, + "hits": 3, + "total": 118, + "hitRate": 0.025423728813559324 + }, + { + "encoding": "KOI8-R", + "confidence": 4, + "language": "ru", + "byteLogLikelihood": -6.302671202821, + "hits": 2, + "total": 134, + "hitRate": 0.014925373134328358 + }, + { + "encoding": "ISO-8859-6", + "confidence": 3, + "language": "ar", + "byteLogLikelihood": -5.268243094623, + "hits": 1, + "total": 81, + "hitRate": 0.012345679012345678 + }, + { + "encoding": "ISO-8859-8", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "windows-1255", + "confidence": 2, + "language": "he", + "byteLogLikelihood": -4.09495359507, + "hits": 1, + "total": 127, + "hitRate": 0.007874015748031496 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -4.203232324163, + "hits": 1, + "total": 131, + "hitRate": 0.007633587786259542 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.165809265202, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "hu", + "byteLogLikelihood": -4.358825259583, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.288697198304, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "pt", + "byteLogLikelihood": -4.59932343766, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.969377662519, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.013410783916, + "hits": 0, + "total": 122, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.827626146169, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.681848366371, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.483358374511, + "hits": 0, + "total": 52, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.55248803416, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 116, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.945729021813, + "hits": 0, + "total": 129, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.015268979336, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.576407109739, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.890771320266, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.788878603493, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.603974037796, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.093353952573, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 96, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1254", + "language": "tr" + }, + "predicted": { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-9", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "windows-1254", + "confidence": 55, + "language": "tr", + "byteLogLikelihood": -3.531675216773, + "hits": 25, + "total": 134, + "hitRate": 0.1865671641791045 + }, + { + "encoding": "ISO-8859-3", + "confidence": 22, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 10, + "total": 134, + "hitRate": 0.07462686567164178 + }, + { + "encoding": "ISO-8859-1", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "windows-1252", + "confidence": 17, + "language": "pt", + "byteLogLikelihood": -4.969813299576, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-10", + "confidence": 17, + "language": "is", + "byteLogLikelihood": -4.720106560713, + "hits": 8, + "total": 134, + "hitRate": 0.05970149253731343 + }, + { + "encoding": "ISO-8859-2", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1250", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "ISO-8859-16", + "confidence": 13, + "language": "ro", + "byteLogLikelihood": -4.352190516073, + "hits": 6, + "total": 134, + "hitRate": 0.04477611940298507 + }, + { + "encoding": "windows-1257", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -3.613497993373, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "macintosh", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-4", + "confidence": 11, + "language": "lv", + "byteLogLikelihood": -4.93302665818, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-13", + "confidence": 11, + "language": "lt", + "byteLogLikelihood": -4.394247673508, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "ISO-8859-15", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, + { + "encoding": "CP850", + "confidence": 11, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 132, + "hitRate": 0.03787878787878788 + }, + { + "encoding": "ISO-8859-14", + "confidence": 8, + "language": "cy", + "byteLogLikelihood": -4.652724693026, + "hits": 4, + "total": 134, + "hitRate": 0.029850746268656716 + }, + { + "encoding": "windows-1258", + "confidence": 6, + "language": "vi", + "byteLogLikelihood": -5.197431316193, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "CP852", + "confidence": 6, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 3, + "total": 133, + "hitRate": 0.022556390977443608 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 133, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.217802445218, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.576073075491, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.216966174066, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.95929011292, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -6.113682179832, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.714883328648, + "hits": 0, + "total": 130, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.085579959517, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 134, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.88736478629, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -4.742947096814, + "hits": 0, + "total": 132, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.22610845571, + "hits": 0, + "total": 134, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1255", + "language": "he" + }, + "predicted": { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + "encodingCorrect": false, + "languageCorrect": false, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "ISO-8859-8", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "windows-1255", + "confidence": 47, + "language": "he", + "byteLogLikelihood": -3.320333973076, + "hits": 16, + "total": 102, + "hitRate": 0.1568627450980392 + }, + { + "encoding": "ISO-8859-7", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "windows-1253", + "confidence": 2, + "language": "el", + "byteLogLikelihood": -3.981926233577, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "IBM855", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.394495827364, + "hits": 1, + "total": 102, + "hitRate": 0.00980392156862745 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.124491051534, + "hits": 0, + "total": 73, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.461975046319, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.199203901043, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.395116706452, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.941685614581, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.32364067176, + "hits": 0, + "total": 93, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.639272141503, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029681805165, + "hits": 0, + "total": 99, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.31385343816, + "hits": 0, + "total": 48, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.40669720385, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.393692981541, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.683415446443, + "hits": 0, + "total": 94, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 72, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -4.134202940863, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -4.956632184798, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.146385148445, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.745792460039, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -4.842271132243, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.813633859942, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "fr", + "byteLogLikelihood": -4.521941300627, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.048864674087, + "hits": 0, + "total": 102, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 85, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 89, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1256", + "language": "ar" + }, + "predicted": { + "encoding": "windows-1256", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1256", + "confidence": 62, + "language": "ar", + "byteLogLikelihood": -3.306009422775, + "hits": 23, + "total": 110, + "hitRate": 0.20909090909090908 + }, + { + "encoding": "IBM855", + "confidence": 21, + "language": "ru", + "byteLogLikelihood": -5.640676681359, "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 + "total": 100, + "hitRate": 0.07 + }, + { + "encoding": "ISO-8859-6", + "confidence": 15, + "language": "ar", + "byteLogLikelihood": -3.973986161297, + "hits": 5, + "total": 99, + "hitRate": 0.050505050505050504 + }, + { + "encoding": "ISO-8859-8", + "confidence": 5, + "language": "he", + "byteLogLikelihood": -5.107875566922, + "hits": 1, + "total": 58, + "hitRate": 0.017241379310344827 + }, + { + "encoding": "windows-1255", + "confidence": 5, + "language": "he", + "byteLogLikelihood": -5.107875566922, + "hits": 1, + "total": 60, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-5", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.047579876205, + "hits": 1, + "total": 111, + "hitRate": 0.009009009009009009 + }, + { + "encoding": "KOI8-R", + "confidence": 2, + "language": "ru", + "byteLogLikelihood": -5.041761137138, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "KOI8-U", + "confidence": 2, + "language": "uk", + "byteLogLikelihood": -4.982175743167, + "hits": 1, + "total": 110, + "hitRate": 0.00909090909090909 + }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.916907952775, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.668535969782, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.575363278011, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.916907952775, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.906860727803, + "hits": 0, + "total": 109, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.856571347512, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.029693479499, + "hits": 0, + "total": 106, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -4.736714293626, + "hits": 0, + "total": 90, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.982133409941, + "hits": 0, + "total": 60, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 46, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.655679070563, + "hits": 0, + "total": 81, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-3", + "confidence": 0, + "language": "mt", + "byteLogLikelihood": -5.078935619012, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -4.796168589227, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-13", + "confidence": 0, + "language": "lt", + "byteLogLikelihood": -5.003416643654, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-14", + "confidence": 0, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-15", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.70358763711, + "hits": 0, + "total": 110, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.075354869964, + "hits": 0, + "total": 111, + "hitRate": 0 + }, + { + "encoding": "CP850", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.941225681894, + "hits": 0, + "total": 97, + "hitRate": 0 + }, + { + "encoding": "CP852", + "confidence": 0, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 0, + "total": 100, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1257", + "language": "et" + }, + "predicted": { + "encoding": "windows-1257", + "confidence": 40, + "language": "et", + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, + "candidates": [ + { + "encoding": "windows-1257", + "confidence": 40, + "language": "et", + "byteLogLikelihood": -3.489288054183, + "hits": 16, + "total": 120, + "hitRate": 0.13333333333333333 + }, + { + "encoding": "ISO-8859-1", + "confidence": 12, + "language": "de", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-9", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -3.976561526566, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1252", + "confidence": 12, + "language": "de", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "windows-1254", + "confidence": 12, + "language": "tr", + "byteLogLikelihood": -3.976561526566, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "macintosh", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-3", + "confidence": 12, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "confidence": 12, + "language": "de", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "CP850", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 5, + "total": 120, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-2", + "confidence": 10, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1250", + "confidence": 10, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-4", + "confidence": 10, + "language": "lv", + "byteLogLikelihood": -5.19295685089, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-10", + "confidence": 10, + "language": "is", + "byteLogLikelihood": -5.204006687077, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "ISO-8859-13", + "confidence": 10, + "language": "lt", + "byteLogLikelihood": -5.123963979403, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "CP852", + "confidence": 10, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 120, + "hitRate": 0.03333333333333333 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -5.497168225293, + "hits": 3, + "total": 120, + "hitRate": 0.025 + }, + { + "encoding": "ISO-8859-16", + "confidence": 5, + "language": "ro", + "byteLogLikelihood": -5.176149732574, + "hits": 2, + "total": 120, + "hitRate": 0.016666666666666666 + }, + { + "encoding": "ISO-8859-14", + "confidence": 2, + "language": "cy", + "byteLogLikelihood": -4.912654885736, + "hits": 1, + "total": 120, + "hitRate": 0.008333333333333333 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.30822322351, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.41969872812, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.51563851542, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.453415335268, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.41969872812, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.51563851542, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.831207501101, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.714883328648, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.345510152227, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.303705458307, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -6.692083742507, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "sr", + "byteLogLikelihood": -5.827473946999, + "hits": 0, + "total": 120, + "hitRate": 0 + }, + { + "encoding": "x-mac-cyrillic", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -3.632538332325, + "hits": 0, + "total": 120, + "hitRate": 0 + } + ] + }, + { + "expected": { + "encoding": "windows-1257", + "language": "lt" + }, + "predicted": { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, + "candidates": [ + { + "encoding": "windows-1257", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 + }, + { + "encoding": "ISO-8859-13", + "confidence": 34, + "language": "lt", + "byteLogLikelihood": -3.200530207132, + "hits": 20, + "total": 173, + "hitRate": 0.11560693641618497 }, { "encoding": "ISO-8859-4", - "confidence": 11, + "confidence": 22, "language": "lv", - "byteLogLikelihood": -4.945351018208, + "byteLogLikelihood": -4.610398759565, + "hits": 13, + "total": 173, + "hitRate": 0.07514450867052024 + }, + { + "encoding": "ISO-8859-2", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "windows-1250", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.117993812417, + "hits": 9, + "total": 173, + "hitRate": 0.05202312138728324 + }, + { + "encoding": "ISO-8859-1", + "confidence": 12, + "language": "nl", + "byteLogLikelihood": -4.948759890378, "hits": 7, - "total": 183, - "hitRate": 0.03825136612021858 + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "windows-1252", + "confidence": 12, + "language": "nl", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "macintosh", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 168, + "hitRate": 0.041666666666666664 + }, + { + "encoding": "ISO-8859-15", + "confidence": 12, + "language": "fr", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 173, + "hitRate": 0.04046242774566474 + }, + { + "encoding": "CP850", + "confidence": 12, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 7, + "total": 170, + "hitRate": 0.041176470588235294 + }, + { + "encoding": "ISO-8859-9", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "windows-1254", + "confidence": 8, + "language": "tr", + "byteLogLikelihood": -4.789760270297, + "hits": 5, + "total": 173, + "hitRate": 0.028901734104046242 + }, + { + "encoding": "CP852", + "confidence": 7, + "language": "pl", + "byteLogLikelihood": -5.087596335232, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "ISO-8859-3", + "confidence": 5, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 3, + "total": 172, + "hitRate": 0.01744186046511628 + }, + { + "encoding": "ISO-8859-14", + "confidence": 3, + "language": "cy", + "byteLogLikelihood": -4.6272413408, + "hits": 2, + "total": 173, + "hitRate": 0.011560693641618497 + }, + { + "encoding": "windows-1258", + "confidence": 1, + "language": "vi", + "byteLogLikelihood": -4.960186085121, + "hits": 1, + "total": 171, + "hitRate": 0.005847953216374269 + }, + { + "encoding": "IBM855", + "confidence": 1, + "language": "sr", + "byteLogLikelihood": -5.488073838002, + "hits": 1, + "total": 170, + "hitRate": 0.0058823529411764705 + }, + { + "encoding": "ISO-8859-10", + "confidence": 1, + "language": "is", + "byteLogLikelihood": -4.416382325607, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-16", + "confidence": 1, + "language": "ro", + "byteLogLikelihood": -4.659155714377, + "hits": 1, + "total": 173, + "hitRate": 0.005780346820809248 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.893109334109, + "hits": 0, + "total": 172, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.773457169107, + "hits": 0, + "total": 167, + "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 181, - "hitRate": 0.03867403314917127 + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 11, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 7, - "total": 177, - "hitRate": 0.03954802259887006 + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 9, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 6, - "total": 182, - "hitRate": 0.03296703296703297 + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -3.531827362419, + "hits": 0, + "total": 173, + "hitRate": 0 }, { - "encoding": "ISO-8859-14", - "confidence": 6, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 4, - "total": 183, - "hitRate": 0.02185792349726776 + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.472500251122, + "hits": 0, + "total": 173, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 4, - "language": "lt", - "byteLogLikelihood": -4.986136327924, - "hits": 3, - "total": 182, - "hitRate": 0.016483516483516484 + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.444220754634, + "hits": 0, + "total": 171, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 3, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 2, - "total": 183, - "hitRate": 0.01092896174863388 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.924336543075, + "hits": 0, + "total": 170, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "windows-874", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.383683979141, + "language": "th", + "byteLogLikelihood": -5.58967630288, "hits": 0, - "total": 181, + "total": 166, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "KOI8-R", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.404570694946, + "byteLogLikelihood": -6.365896834008, "hits": 0, - "total": 181, + "total": 173, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -6.252539473952, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.943656762101, + "byteLogLikelihood": -4.893109334109, "hits": 0, - "total": 181, + "total": 171, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.572384450833, + "language": "uk", + "byteLogLikelihood": -3.441328843196, "hits": 0, - "total": 182, + "total": 173, "hitRate": 0 } ] }, { "expected": { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", "language": "lv" }, "predicted": { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", "confidence": 40, "language": "lv", "byteLogLikelihood": -3.1328616083, @@ -2259,7 +18914,7 @@ "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-4", + "encoding": "windows-1257", "confidence": 40, "language": "lv", "byteLogLikelihood": -3.1328616083, @@ -2267,14 +18922,41 @@ "total": 170, "hitRate": 0.13529411764705881 }, + { + "encoding": "ISO-8859-4", + "confidence": 31, + "language": "lv", + "byteLogLikelihood": -5.003611214133, + "hits": 18, + "total": 170, + "hitRate": 0.10588235294117647 + }, { "encoding": "CP850", - "confidence": 16, + "confidence": 15, "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 9, - "total": 168, - "hitRate": 0.05357142857142857 + "total": 170, + "hitRate": 0.052941176470588235 + }, + { + "encoding": "ISO-8859-1", + "confidence": 14, + "language": "en", + "byteLogLikelihood": -4.806768476285, + "hits": 8, + "total": 170, + "hitRate": 0.047058823529411764 + }, + { + "encoding": "windows-1252", + "confidence": 14, + "language": "en", + "byteLogLikelihood": -4.806768476285, + "hits": 8, + "total": 170, + "hitRate": 0.047058823529411764 }, { "encoding": "macintosh", @@ -2282,17 +18964,17 @@ "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 + "total": 170, + "hitRate": 0.047058823529411764 }, { "encoding": "ISO-8859-15", "confidence": 14, - "language": "es", - "byteLogLikelihood": -4.721042183854, + "language": "fr", + "byteLogLikelihood": -4.806768476285, "hits": 8, - "total": 169, - "hitRate": 0.047337278106508875 + "total": 170, + "hitRate": 0.047058823529411764 }, { "encoding": "CP852", @@ -2300,32 +18982,77 @@ "language": "pl", "byteLogLikelihood": -5.087596335232, "hits": 7, - "total": 169, - "hitRate": 0.04142011834319527 + "total": 170, + "hitRate": 0.041176470588235294 + }, + { + "encoding": "ISO-8859-3", + "confidence": 10, + "language": "mt", + "byteLogLikelihood": -5.105945473901, + "hits": 6, + "total": 170, + "hitRate": 0.03529411764705882 }, { "encoding": "ISO-8859-13", "confidence": 10, "language": "lt", - "byteLogLikelihood": -4.596975900219, + "byteLogLikelihood": -4.977793000004, "hits": 6, - "total": 169, - "hitRate": 0.03550295857988166 + "total": 170, + "hitRate": 0.03529411764705882 }, { - "encoding": "ISO-8859-3", + "encoding": "ISO-8859-2", "confidence": 8, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "ro", + "byteLogLikelihood": -5.087596335232, + "hits": 5, + "total": 170, + "hitRate": 0.029411764705882353 + }, + { + "encoding": "windows-1250", + "confidence": 8, + "language": "ro", + "byteLogLikelihood": -5.087596335232, "hits": 5, "total": 170, "hitRate": 0.029411764705882353 }, + { + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.576539240129, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.576539240129, + "hits": 4, + "total": 170, + "hitRate": 0.023529411764705882 + }, + { + "encoding": "windows-1258", + "confidence": 7, + "language": "vi", + "byteLogLikelihood": -4.342387954294, + "hits": 4, + "total": 169, + "hitRate": 0.023668639053254437 + }, { "encoding": "ISO-8859-10", "confidence": 7, "language": "is", - "byteLogLikelihood": -5.074758182528, + "byteLogLikelihood": -4.928587203128, "hits": 4, "total": 170, "hitRate": 0.023529411764705882 @@ -2334,638 +19061,760 @@ "encoding": "ISO-8859-16", "confidence": 1, "language": "ro", - "byteLogLikelihood": -4.572632236701, + "byteLogLikelihood": -4.673743278572, "hits": 1, - "total": 169, - "hitRate": 0.005917159763313609 - }, - { - "encoding": "KOI8-U", - "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.609505685025, - "hits": 0, - "total": 169, - "hitRate": 0 + "total": 170, + "hitRate": 0.0058823529411764705 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -5.267508614334, + "byteLogLikelihood": -4.459882683974, "hits": 0, - "total": 168, + "total": 170, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-6", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -6.182018138172, + "language": "ar", + "byteLogLikelihood": -5.080491263049, "hits": 0, "total": 169, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-7", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -4.65535318117, + "language": "el", + "byteLogLikelihood": -3.703260469986, "hits": 0, "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "ISO-8859-8", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "language": "he", + "byteLogLikelihood": -4.207917523838, "hits": 0, "total": 170, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "KOI8-U", - "language": "uk" - }, - "predicted": { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ - { - "encoding": "KOI8-U", - "confidence": 32, - "language": "uk", - "byteLogLikelihood": -3.34369756932, - "hits": 19, - "total": 177, - "hitRate": 0.10734463276836158 - }, - { - "encoding": "ISO-8859-10", - "confidence": 1, - "language": "is", - "byteLogLikelihood": -5.128376292359, - "hits": 1, - "total": 177, - "hitRate": 0.005649717514124294 }, { - "encoding": "IBM866", + "encoding": "windows-1251", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.582632026409, - "hits": 0, - "total": 22, - "hitRate": 0 - }, - { - "encoding": "IBM855", - "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.922515418323, + "byteLogLikelihood": -3.543665829263, "hits": 0, - "total": 94, + "total": 170, "hitRate": 0 }, { - "encoding": "macintosh", + "encoding": "windows-1253", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "language": "el", + "byteLogLikelihood": -3.703260469986, "hits": 0, - "total": 89, + "total": 170, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1255", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.589678128375, + "language": "he", + "byteLogLikelihood": -4.207917523838, "hits": 0, - "total": 117, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1256", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "ar", + "byteLogLikelihood": -5.924336543075, "hits": 0, - "total": 175, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-4", + "encoding": "windows-874", "confidence": 0, - "language": "lv", - "byteLogLikelihood": -5.19295685089, + "language": "th", + "byteLogLikelihood": -5.58909903103, "hits": 0, - "total": 176, + "total": 167, "hitRate": 0 }, { - "encoding": "ISO-8859-13", + "encoding": "KOI8-R", "confidence": 0, - "language": "lt", - "byteLogLikelihood": -5.110283442945, + "language": "ru", + "byteLogLikelihood": -6.155101602334, "hits": 0, - "total": 168, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-14", + "encoding": "KOI8-U", "confidence": 0, - "language": "cy", - "byteLogLikelihood": -4.912654885736, + "language": "uk", + "byteLogLikelihood": -6.147141858115, "hits": 0, - "total": 177, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-15", + "encoding": "IBM866", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.944199711559, + "language": "ru", + "byteLogLikelihood": -4.459882683974, "hits": 0, - "total": 176, + "total": 170, "hitRate": 0 }, { - "encoding": "ISO-8859-16", + "encoding": "IBM855", "confidence": 0, - "language": "ro", - "byteLogLikelihood": -5.148788659657, + "language": "ru", + "byteLogLikelihood": -5.609454107834, "hits": 0, - "total": 177, + "total": 170, "hitRate": 0 }, { - "encoding": "CP850", + "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "es", - "byteLogLikelihood": -4.948759890378, + "language": "uk", + "byteLogLikelihood": -3.416476215238, "hits": 0, - "total": 92, + "total": 170, "hitRate": 0 }, { - "encoding": "CP852", + "encoding": "ISO-8859-14", "confidence": 0, - "language": "pl", - "byteLogLikelihood": -5.087596335232, + "language": "cy", + "byteLogLikelihood": -4.423374522988, "hits": 0, - "total": 94, + "total": 170, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "de" + "encoding": "windows-1258", + "language": "vi" }, "predicted": { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "windows-1258", + "confidence": 98, + "language": "vi", + "byteLogLikelihood": -3.323414763285, + "hits": 42, + "total": 124, + "hitRate": 0.3387096774193548 }, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 73, - "language": "de", - "byteLogLikelihood": -3.169712077519, - "hits": 53, - "total": 216, - "hitRate": 0.24537037037037038 + "encoding": "windows-1258", + "confidence": 98, + "language": "vi", + "byteLogLikelihood": -3.323414763285, + "hits": 42, + "total": 124, + "hitRate": 0.3387096774193548 }, { - "encoding": "ISO-8859-15", - "confidence": 74, + "encoding": "ISO-8859-1", + "confidence": 9, + "language": "pt", + "byteLogLikelihood": -4.721506246424, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1252", + "confidence": 9, + "language": "pt", + "byteLogLikelihood": -4.721506246424, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "windows-1257", + "confidence": 9, + "language": "lt", + "byteLogLikelihood": -4.767575136152, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "macintosh", + "confidence": 9, "language": "es", - "byteLogLikelihood": -5.003946305945, - "hits": 53, - "total": 214, - "hitRate": 0.24766355140186916 + "byteLogLikelihood": -4.948759890378, + "hits": 4, + "total": 122, + "hitRate": 0.03278688524590164 }, { - "encoding": "CP850", - "confidence": 69, - "language": "fr", - "byteLogLikelihood": -5.003946305945, - "hits": 50, - "total": 216, - "hitRate": 0.23148148148148148 + "encoding": "ISO-8859-4", + "confidence": 9, + "language": "lv", + "byteLogLikelihood": -4.83530398146, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 + }, + { + "encoding": "ISO-8859-13", + "confidence": 9, + "language": "lt", + "byteLogLikelihood": -4.767575136152, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { "encoding": "ISO-8859-14", - "confidence": 22, + "confidence": 9, "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 16, - "total": 214, - "hitRate": 0.07476635514018691 + "byteLogLikelihood": -4.643097648852, + "hits": 4, + "total": 126, + "hitRate": 0.031746031746031744 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 11, - "total": 214, - "hitRate": 0.0514018691588785 + "encoding": "ISO-8859-2", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -5.060625202481, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 }, { - "encoding": "ISO-8859-4", - "confidence": 11, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 8, - "total": 214, - "hitRate": 0.037383177570093455 + "encoding": "ISO-8859-9", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.76855606835, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1250", + "confidence": 7, + "language": "cs", + "byteLogLikelihood": -5.060625202481, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "windows-1254", + "confidence": 7, + "language": "tr", + "byteLogLikelihood": -4.76855606835, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 }, { "encoding": "ISO-8859-3", - "confidence": 5, + "confidence": 7, "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 4, - "total": 214, - "hitRate": 0.018691588785046728 + "byteLogLikelihood": -4.906120867834, + "hits": 3, + "total": 124, + "hitRate": 0.024193548387096774 + }, + { + "encoding": "ISO-8859-15", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -4.873536273681, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "ISO-8859-16", + "confidence": 7, + "language": "ro", + "byteLogLikelihood": -5.060625202481, + "hits": 3, + "total": 126, + "hitRate": 0.023809523809523808 + }, + { + "encoding": "CP850", + "confidence": 7, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 3, + "total": 116, + "hitRate": 0.02586206896551724 }, { "encoding": "CP852", - "confidence": 5, + "confidence": 7, "language": "pl", "byteLogLikelihood": -5.087596335232, - "hits": 4, - "total": 216, - "hitRate": 0.018518518518518517 + "hits": 3, + "total": 119, + "hitRate": 0.025210084033613446 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 214, - "hitRate": 0.014018691588785047 + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.315573016103, + "hits": 0, + "total": 124, + "hitRate": 0 }, { - "encoding": "ISO-8859-13", - "confidence": 2, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 2, - "total": 214, - "hitRate": 0.009345794392523364 + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.428051016065, + "hits": 0, + "total": 115, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.428592570712, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-8", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.890004274964, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "windows-1251", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.843070172798, + "hits": 0, + "total": 126, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.428592570712, + "hits": 0, + "total": 125, + "hitRate": 0 + }, + { + "encoding": "windows-1255", + "confidence": 0, + "language": "he", + "byteLogLikelihood": -4.890004274964, + "hits": 0, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.628297137503, + "hits": 0, + "total": 121, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.351753988745, + "hits": 0, + "total": 114, + "hitRate": 0 + }, + { + "encoding": "KOI8-R", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.775144482981, + "hits": 0, + "total": 126, + "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.534754518494, + "byteLogLikelihood": -5.565906355903, "hits": 0, - "total": 215, + "total": 126, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.345510152227, + "byteLogLikelihood": -5.727056945557, "hits": 0, - "total": 216, + "total": 124, + "hitRate": 0 + }, + { + "encoding": "IBM855", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.876501639082, + "hits": 0, + "total": 124, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "x-mac-cyrillic", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -6.337519079447, + "language": "ru", + "byteLogLikelihood": -4.782036156761, "hits": 0, - "total": 216, + "total": 125, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "ISO-8859-10", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "is", + "byteLogLikelihood": -4.794574996468, "hits": 0, - "total": 216, + "total": 126, "hitRate": 0 } ] }, { "expected": { - "encoding": "macintosh", - "language": "es" + "encoding": "windows-874", + "language": "th" }, "predicted": { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "windows-874", + "confidence": 21, + "language": "th", + "byteLogLikelihood": -3.827006747769, + "hits": 7, + "total": 100, + "hitRate": 0.07 }, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { - "encoding": "macintosh", - "confidence": 61, - "language": "es", - "byteLogLikelihood": -3.834657935767, - "hits": 41, - "total": 200, - "hitRate": 0.205 + "encoding": "windows-874", + "confidence": 21, + "language": "th", + "byteLogLikelihood": -3.827006747769, + "hits": 7, + "total": 100, + "hitRate": 0.07 }, { - "encoding": "ISO-8859-15", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 199, - "hitRate": 0.19597989949748743 + "encoding": "ISO-8859-6", + "confidence": 6, + "language": "ar", + "byteLogLikelihood": -5.033884846794, + "hits": 2, + "total": 93, + "hitRate": 0.021505376344086023 }, { - "encoding": "CP850", - "confidence": 58, - "language": "es", - "byteLogLikelihood": -4.948759890378, - "hits": 39, - "total": 200, - "hitRate": 0.195 + "encoding": "ISO-8859-8", + "confidence": 6, + "language": "he", + "byteLogLikelihood": -5.415196995324, + "hits": 1, + "total": 43, + "hitRate": 0.023255813953488372 }, { - "encoding": "ISO-8859-4", - "confidence": 25, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 17, - "total": 199, - "hitRate": 0.08542713567839195 + "encoding": "windows-1255", + "confidence": 6, + "language": "he", + "byteLogLikelihood": -5.415196995324, + "hits": 1, + "total": 47, + "hitRate": 0.02127659574468085 }, { - "encoding": "ISO-8859-13", - "confidence": 19, - "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 13, - "total": 199, - "hitRate": 0.06532663316582915 + "encoding": "ISO-8859-7", + "confidence": 12, + "language": "el", + "byteLogLikelihood": -5.563277974308, + "hits": 4, + "total": 97, + "hitRate": 0.041237113402061855 }, { - "encoding": "ISO-8859-14", - "confidence": 16, - "language": "cy", - "byteLogLikelihood": -4.912654885736, - "hits": 11, - "total": 199, - "hitRate": 0.05527638190954774 + "encoding": "windows-1253", + "confidence": 12, + "language": "el", + "byteLogLikelihood": -5.563277974308, + "hits": 4, + "total": 97, + "hitRate": 0.041237113402061855 }, { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 + "encoding": "windows-1251", + "confidence": 5, + "language": "ru", + "byteLogLikelihood": -5.926448142289, + "hits": 2, + "total": 102, + "hitRate": 0.0196078431372549 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 5, - "total": 200, - "hitRate": 0.025 + "encoding": "x-mac-cyrillic", + "confidence": 11, + "language": "uk", + "byteLogLikelihood": -6.059253444951, + "hits": 3, + "total": 79, + "hitRate": 0.0379746835443038 }, { - "encoding": "ISO-8859-10", - "confidence": 4, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 3, - "total": 199, - "hitRate": 0.01507537688442211 + "encoding": "KOI8-R", + "confidence": 3, + "language": "ru", + "byteLogLikelihood": -5.413211053323, + "hits": 1, + "total": 94, + "hitRate": 0.010638297872340425 }, { - "encoding": "KOI8-U", + "encoding": "IBM855", + "confidence": 3, + "language": "sr", + "byteLogLikelihood": -5.781484174666, + "hits": 1, + "total": 88, + "hitRate": 0.011363636363636364 + }, + { + "encoding": "ISO-8859-1", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "it", + "byteLogLikelihood": -4.767489617961, "hits": 0, - "total": 199, + "total": 97, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.903917269007, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.692083742507, + "byteLogLikelihood": -5.535400651308, "hits": 0, - "total": 200, + "total": 103, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "ISO-8859-9", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -5.624741392945, + "language": "tr", + "byteLogLikelihood": -4.985031025935, "hits": 0, - "total": 200, + "total": 97, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "windows-1250", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.303705458307, + "language": "pl", + "byteLogLikelihood": -4.798656950405, "hits": 0, - "total": 200, + "total": 101, "hitRate": 0 }, { - "encoding": "ISO-8859-3", + "encoding": "windows-1252", "confidence": 0, - "language": "mt", - "byteLogLikelihood": -5.105945473901, + "language": "it", + "byteLogLikelihood": -4.767489617961, "hits": 0, - "total": 199, + "total": 97, "hitRate": 0 - } - ] - }, - { - "expected": { - "encoding": "macintosh", - "language": "fr" - }, - "predicted": { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 - }, - "encodingCorrect": true, - "languageCorrect": true, - "tiedAtTop": 1, - "candidates": [ + }, { - "encoding": "macintosh", - "confidence": 77, - "language": "fr", - "byteLogLikelihood": -2.06619631493, - "hits": 59, - "total": 228, - "hitRate": 0.25877192982456143 + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.985031025935, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-15", - "confidence": 75, - "language": "es", - "byteLogLikelihood": -5.010635294096, - "hits": 56, - "total": 224, - "hitRate": 0.25 + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.209625144761, + "hits": 0, + "total": 95, + "hitRate": 0 }, { - "encoding": "CP850", - "confidence": 71, - "language": "de", - "byteLogLikelihood": -5.010635294096, - "hits": 54, - "total": 228, - "hitRate": 0.23684210526315788 + "encoding": "windows-1257", + "confidence": 0, + "language": "et", + "byteLogLikelihood": -4.916002186025, + "hits": 0, + "total": 97, + "hitRate": 0 }, { - "encoding": "ISO-8859-10", - "confidence": 12, - "language": "is", - "byteLogLikelihood": -5.204006687077, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -5.154894826847, + "hits": 0, + "total": 91, + "hitRate": 0 }, { - "encoding": "ISO-8859-16", - "confidence": 12, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 224, - "hitRate": 0.04017857142857143 + "encoding": "KOI8-U", + "confidence": 0, + "language": "uk", + "byteLogLikelihood": -5.334497573029, + "hits": 0, + "total": 98, + "hitRate": 0 }, { - "encoding": "ISO-8859-4", - "confidence": 9, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 7, - "total": 224, - "hitRate": 0.03125 + "encoding": "IBM866", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -5.961753913164, + "hits": 0, + "total": 58, + "hitRate": 0 + }, + { + "encoding": "macintosh", + "confidence": 0, + "language": "es", + "byteLogLikelihood": -4.948759890378, + "hits": 0, + "total": 65, + "hitRate": 0 }, { "encoding": "ISO-8859-3", - "confidence": 8, + "confidence": 0, "language": "mt", - "byteLogLikelihood": -5.105945473901, - "hits": 6, - "total": 224, - "hitRate": 0.026785714285714284 + "byteLogLikelihood": -5.084507932234, + "hits": 0, + "total": 98, + "hitRate": 0 }, { - "encoding": "CP852", - "confidence": 7, - "language": "pl", - "byteLogLikelihood": -5.087596335232, - "hits": 6, - "total": 228, - "hitRate": 0.02631578947368421 + "encoding": "ISO-8859-4", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.783724339715, + "hits": 0, + "total": 103, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-10", + "confidence": 0, + "language": "is", + "byteLogLikelihood": -5.119633132705, + "hits": 0, + "total": 103, + "hitRate": 0 }, { "encoding": "ISO-8859-13", - "confidence": 5, + "confidence": 0, "language": "lt", - "byteLogLikelihood": -5.123963979403, - "hits": 4, - "total": 224, - "hitRate": 0.017857142857142856 + "byteLogLikelihood": -5.010167702209, + "hits": 0, + "total": 97, + "hitRate": 0 }, { "encoding": "ISO-8859-14", - "confidence": 4, + "confidence": 0, "language": "cy", "byteLogLikelihood": -4.912654885736, - "hits": 3, - "total": 224, - "hitRate": 0.013392857142857142 + "hits": 0, + "total": 103, + "hitRate": 0 }, { - "encoding": "KOI8-U", + "encoding": "ISO-8859-15", "confidence": 0, - "language": "uk", - "byteLogLikelihood": -6.650279048587, + "language": "fr", + "byteLogLikelihood": -4.801693311206, "hits": 0, - "total": 224, + "total": 101, "hitRate": 0 }, { - "encoding": "IBM866", + "encoding": "ISO-8859-16", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, + "language": "ro", + "byteLogLikelihood": -5.123264178812, "hits": 0, - "total": 228, + "total": 103, "hitRate": 0 }, { - "encoding": "IBM855", + "encoding": "CP850", "confidence": 0, - "language": "sr", - "byteLogLikelihood": -3.576182148392, + "language": "es", + "byteLogLikelihood": -4.887413059939, "hits": 0, - "total": 228, + "total": 85, "hitRate": 0 }, { - "encoding": "x-mac-cyrillic", + "encoding": "CP852", "confidence": 0, - "language": "ru", - "byteLogLikelihood": -5.998936561947, + "language": "pl", + "byteLogLikelihood": -5.056712497361, "hits": 0, - "total": 228, + "total": 85, "hitRate": 0 } ] @@ -2997,6 +19846,15 @@ "total": 179, "hitRate": 0.12290502793296089 }, + { + "encoding": "windows-1251", + "confidence": 42, + "language": "ru", + "byteLogLikelihood": -3.411211853364, + "hits": 25, + "total": 177, + "hitRate": 0.14124293785310735 + }, { "encoding": "IBM855", "confidence": 9, @@ -3006,6 +19864,24 @@ "total": 165, "hitRate": 0.030303030303030304 }, + { + "encoding": "ISO-8859-8", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -3.941595102268, + "hits": 2, + "total": 167, + "hitRate": 0.011976047904191617 + }, { "encoding": "KOI8-U", "confidence": 3, @@ -3015,6 +19891,24 @@ "total": 177, "hitRate": 0.011299435028248588 }, + { + "encoding": "windows-1256", + "confidence": 1, + "language": "ar", + "byteLogLikelihood": -5.525051915661, + "hits": 1, + "total": 163, + "hitRate": 0.006134969325153374 + }, + { + "encoding": "KOI8-R", + "confidence": 1, + "language": "ru", + "byteLogLikelihood": -6.390589047944, + "hits": 1, + "total": 177, + "hitRate": 0.005649717514124294 + }, { "encoding": "ISO-8859-14", "confidence": 1, @@ -3024,6 +19918,123 @@ "total": 177, "hitRate": 0.005649717514124294 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.975257565806, + "hits": 0, + "total": 175, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-6", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -4.847464468096, + "hits": 0, + "total": 127, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.148248171081, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.845647442888, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "sv", + "byteLogLikelihood": -4.665227236998, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.148248171081, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.950903003627, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.61521723683, + "hits": 0, + "total": 177, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.899487478125, + "hits": 0, + "total": 173, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.446015416921, + "hits": 0, + "total": 85, + "hitRate": 0 + }, { "encoding": "IBM866", "confidence": 0, @@ -3143,6 +20154,15 @@ "total": 177, "hitRate": 0.10734463276836158 }, + { + "encoding": "windows-1251", + "confidence": 17, + "language": "ru", + "byteLogLikelihood": -3.606822645813, + "hits": 10, + "total": 176, + "hitRate": 0.056818181818181816 + }, { "encoding": "IBM855", "confidence": 5, @@ -3152,6 +20172,42 @@ "total": 163, "hitRate": 0.018404907975460124 }, + { + "encoding": "ISO-8859-8", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -4.062591421695, + "hits": 2, + "total": 165, + "hitRate": 0.012121212121212121 + }, + { + "encoding": "windows-1255", + "confidence": 3, + "language": "he", + "byteLogLikelihood": -4.062591421695, + "hits": 2, + "total": 165, + "hitRate": 0.012121212121212121 + }, + { + "encoding": "KOI8-R", + "confidence": 3, + "language": "ru", + "byteLogLikelihood": -6.397907632883, + "hits": 2, + "total": 169, + "hitRate": 0.011834319526627219 + }, + { + "encoding": "ISO-8859-6", + "confidence": 2, + "language": "ar", + "byteLogLikelihood": -4.990380473945, + "hits": 1, + "total": 119, + "hitRate": 0.008403361344537815 + }, { "encoding": "KOI8-U", "confidence": 1, @@ -3161,6 +20217,123 @@ "total": 175, "hitRate": 0.005714285714285714 }, + { + "encoding": "ISO-8859-1", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.629500955096, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-2", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.739775351774, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-5", + "confidence": 0, + "language": "ru", + "byteLogLikelihood": -4.989983227843, + "hits": 0, + "total": 171, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-7", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.300043808736, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "ISO-8859-9", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.951345660734, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1250", + "confidence": 0, + "language": "cs", + "byteLogLikelihood": -4.739775351774, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-1252", + "confidence": 0, + "language": "it", + "byteLogLikelihood": -4.629500955096, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-1253", + "confidence": 0, + "language": "el", + "byteLogLikelihood": -4.300043808736, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1254", + "confidence": 0, + "language": "tr", + "byteLogLikelihood": -4.951345660734, + "hits": 0, + "total": 170, + "hitRate": 0 + }, + { + "encoding": "windows-1256", + "confidence": 0, + "language": "ar", + "byteLogLikelihood": -5.607550779971, + "hits": 0, + "total": 155, + "hitRate": 0 + }, + { + "encoding": "windows-1257", + "confidence": 0, + "language": "lv", + "byteLogLikelihood": -4.60874799529, + "hits": 0, + "total": 169, + "hitRate": 0 + }, + { + "encoding": "windows-1258", + "confidence": 0, + "language": "vi", + "byteLogLikelihood": -4.777030248466, + "hits": 0, + "total": 164, + "hitRate": 0 + }, + { + "encoding": "windows-874", + "confidence": 0, + "language": "th", + "byteLogLikelihood": -5.197321748228, + "hits": 0, + "total": 83, + "hitRate": 0 + }, { "encoding": "IBM866", "confidence": 0, diff --git a/corpus/sources.json b/corpus/sources.json index 3b05948..e65eab5 100644 --- a/corpus/sources.json +++ b/corpus/sources.json @@ -505,6 +505,582 @@ "text": "W niedzielÄ™ przyjaciele poszli nad spokojnÄ… rzekÄ™ za miastem. Zabrali mapÄ™, ciepÅ‚Ä… herbatÄ™ i aparat. Po dÅ‚ugim spacerze odpoczywali na brzegu." } ] + }, + { + "code": "ar", + "name": "Arabic", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "صباح المدينة", + "text": "تستيقظ المدينة بهدوء وتÙتح المقاهي أبوابها بينما ينتظر الناس الحاÙلات ويقرؤون الأخبار." + }, + { + "id": "library", + "split": "train", + "title": "المكتبة الهادئة", + "text": "يبحث القراء عن كتب التاريخ والعلوم ويساعد أمين المكتبة الطلاب ÙÙŠ اختيار المراجع المناسبة." + }, + { + "id": "market", + "split": "train", + "title": "سوق السبت", + "text": "يعرض المزارعون الخبز والعسل والÙاكهة ويتبادل الزوار الوصÙات والأحاديث الودية." + }, + { + "id": "workshop", + "split": "train", + "title": "ورشة رقمية", + "text": "يصمم Ùريق صغير دليلا للمتح٠ويراجع المبرمجون الشÙرة ويكتب المحررون وصÙا واضحا." + }, + { + "id": "river-trip", + "split": "validation", + "title": "رحلة إلى النهر", + "text": "حمل الأصدقاء خريطة وشايا ساخنا ومشوا حتى النهر ثم استمعوا إلى طيور المساء." + } + ] + }, + { + "code": "cs", + "name": "Czech", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Ráno ve mÄ›stÄ›", + "text": "MÄ›sto se pomalu probouzí, kavárny otevírají dveÅ™e a lidé Äekají na první autobusy." + }, + { + "id": "library", + "split": "train", + "title": "Tichá knihovna", + "text": "ÄŒtenáři hledají knihy o historii a vÄ›dÄ›, zatímco knihovnice radí žákům." + }, + { + "id": "market", + "split": "train", + "title": "Sobotní trh", + "text": "Farmáři prodávají Äerstvý chléb, med, ovoce a voňavé byliny." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitální dílna", + "text": "Malý tým pÅ™ipravuje průvodce muzeem, kontroluje kód a píše jasné popisy." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Výlet k Å™ece", + "text": "Přátelé vzali mapu a teplý Äaj, proÅ¡li lesem a odpoÄívali u Å™eky." + } + ] + }, + { + "code": "da", + "name": "Danish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgen i byen", + "text": "Byen vÃ¥gner langsomt, caféerne Ã¥bner, og folk venter pÃ¥ de første busser." + }, + { + "id": "library", + "split": "train", + "title": "Det stille bibliotek", + "text": "Læsere finder bøger om historie og videnskab, mens bibliotekaren hjælper eleverne." + }, + { + "id": "market", + "split": "train", + "title": "Lørdagsmarked", + "text": "Bønder sælger frisk brød, æbler, ost og honning pÃ¥ den travle plads." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitalt værksted", + "text": "Et lille hold bygger en museumsguide, undersøger koden og skriver tydelige tekster." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Tur til floden", + "text": "Vennerne tog et kort og varm te med pÃ¥ en søndag og hvilede bagefter ved floden." + } + ] + }, + { + "code": "en", + "name": "English", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morning in the city", + "text": "The city wakes slowly; cafés open their doors and commuters read news beside a façade." + }, + { + "id": "library", + "split": "train", + "title": "The quiet library", + "text": "Readers browse books about history, science, and naïve art while the librarian prepares a résumé." + }, + { + "id": "market", + "split": "train", + "title": "Saturday market", + "text": "Farmers sell bread, apples, jalapeños, and pâté while neighbours exchange recipes." + }, + { + "id": "workshop", + "split": "train", + "title": "Digital workshop", + "text": "A small team designs the museum guide and discusses its rôle, code, and clear descriptions." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Journey to the river", + "text": "Friends carried a map and a thermos, walked past the café, and rested by the river." + } + ] + }, + { + "code": "el", + "name": "Greek", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "ΠÏωί στην πόλη", + "text": "Η πόλη ξυπνά αÏγά, τα καφέ ανοίγουν και οι άνθÏωποι πεÏιμένουν τα λεωφοÏεία." + }, + { + "id": "library", + "split": "train", + "title": "Ήσυχη βιβλιοθήκη", + "text": "Οι αναγνώστες ψάχνουν βιβλία ιστοÏίας και επιστήμης και η βιβλιοθηκάÏιος βοηθά τους μαθητές." + }, + { + "id": "market", + "split": "train", + "title": "Σαββατιάτικη αγοÏά", + "text": "Οι αγÏότες πουλοÏν ψωμί, μέλι, φÏοÏτα και μυÏωδικά στην πλατεία." + }, + { + "id": "workshop", + "split": "train", + "title": "Ψηφιακό εÏγαστήÏιο", + "text": "Μια μικÏή ομάδα ετοιμάζει οδηγό μουσείου, ελέγχει τον κώδικα και γÏάφει καθαÏά κείμενα." + }, + { + "id": "river-trip", + "split": "validation", + "title": "ΕκδÏομή στο ποτάμι", + "text": "Οι φίλοι πήÏαν χάÏτη και ζεστό τσάι και ξεκουÏάστηκαν δίπλα στο ποτάμι." + } + ] + }, + { + "code": "et", + "name": "Estonian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Hommik linnas", + "text": "Linn ärkab aeglaselt, kohvikud avavad uksed ja inimesed ootavad esimesi busse." + }, + { + "id": "library", + "split": "train", + "title": "Vaikne raamatukogu", + "text": "Lugejad otsivad ajaloo- ja teadusraamatuid ning raamatukoguhoidja aitab õpilasi." + }, + { + "id": "market", + "split": "train", + "title": "Laupäevane turg", + "text": "Talunikud müüvad värsket leiba, õunu, juustu ja mett rahvarohkel väljakul." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitaalne töötuba", + "text": "Väike meeskond koostab muuseumijuhti, kontrollib koodi ja kirjutab selgeid kirjeldusi." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Retk jõe äärde", + "text": "Sõbrad võtsid kaardi ja sooja tee ning puhkasid pärast jalutuskäiku jõe ääres." + } + ] + }, + { + "code": "he", + "name": "Hebrew", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "בוקר בעיר", + "text": "העיר מתעוררת ל×ט, בתי הקפה × ×¤×ª×—×™× ×•×× ×©×™× ×ž×—×›×™× ×œ××•×˜×•×‘×•×¡×™× ×”×¨×שוני×." + }, + { + "id": "library", + "split": "train", + "title": "הספרייה השקטה", + "text": "קור××™× ×ž×—×¤×©×™× ×¡×¤×¨×™ היסטוריה ומדע והספרנית עוזרת ×œ×ª×œ×ž×™×“×™× ×œ×‘×—×•×¨ מקורות." + }, + { + "id": "market", + "split": "train", + "title": "שוק שבת", + "text": "חקל××™× ×ž×•×›×¨×™× ×œ×—× ×˜×¨×™, דבש, פירות וגבינה בכיכר העמוסה." + }, + { + "id": "workshop", + "split": "train", + "title": "סדנה דיגיטלית", + "text": "צוות קטן מכין מדריך למוזי×ון, בודק ×ת הקוד וכותב תי××•×¨×™× ×‘×¨×•×¨×™×." + }, + { + "id": "river-trip", + "split": "validation", + "title": "טיול ×ל הנהר", + "text": "×”×—×‘×¨×™× ×œ×§×—×• מפה ותה ×—×, הלכו ביער ונחו ליד הנהר." + } + ] + }, + { + "code": "hu", + "name": "Hungarian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Reggel a városban", + "text": "A város lassan ébred, a kávézók kinyitnak, az emberek pedig buszra várnak." + }, + { + "id": "library", + "split": "train", + "title": "A csendes könyvtár", + "text": "Az olvasók történelmi és tudományos könyveket keresnek, a könyvtáros segít a diákoknak." + }, + { + "id": "market", + "split": "train", + "title": "Szombati piac", + "text": "A gazdák friss kenyeret, almát, sajtot és mézet árulnak a téren." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitális műhely", + "text": "Egy kis csapat múzeumi útmutatót készít, kódot ellenÅ‘riz és világos leírásokat ír." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Kirándulás a folyóhoz", + "text": "A barátok térképet és meleg teát vittek, majd megpihentek a folyó partján." + } + ] + }, + { + "code": "it", + "name": "Italian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Mattina in città", + "text": "La città si sveglia lentamente, i caffè aprono e la gente aspetta i primi autobus." + }, + { + "id": "library", + "split": "train", + "title": "La biblioteca tranquilla", + "text": "I lettori cercano più libri di storia, geografia e scienza mentre la bibliotecaria aiuta gli studenti più giovani." + }, + { + "id": "market", + "split": "train", + "title": "Mercato del sabato", + "text": "Gli agricoltori vendono pane fresco, caffè, mele, formaggio e miele nella piazza della città." + }, + { + "id": "workshop", + "split": "train", + "title": "Laboratorio digitale", + "text": "Una piccola squadra prepara la guida del museo, controlla il codice e scrive descrizioni chiare perché siano utili a più persone." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Gita al fiume", + "text": "Gli amici portano una mappa e tè caldo, camminano nel bosco e riposano sulla riva." + } + ] + }, + { + "code": "nl", + "name": "Dutch", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Ochtend in de stad", + "text": "De stad ontwaakt langzaam, cafés openen hun deuren en mensen wachten op de eerste bussen." + }, + { + "id": "library", + "split": "train", + "title": "De stille bibliotheek", + "text": "Lezers zoeken boeken over geschiedenis en wetenschap terwijl de bibliothecaris geïnteresseerde leerlingen en één leraar helpt." + }, + { + "id": "market", + "split": "train", + "title": "Zaterdagmarkt", + "text": "Boeren verkopen vers brood, appels, kaas en honing naast twee cafés op het drukke plein." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitale werkplaats", + "text": "Een klein team maakt een museumgids, controleert de code en schrijft duidelijke ideeën over poëzie en teksten." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Reis naar de rivier", + "text": "Vrienden namen een kaart en warme thee mee en rustten na één lange wandeling bij de rivier." + } + ] + }, + { + "code": "no", + "name": "Norwegian", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgen i byen", + "text": "Byen vÃ¥kner langsomt, kafeene Ã¥pner dørene og folk venter pÃ¥ de første bussene nær bryggen og sjøen." + }, + { + "id": "library", + "split": "train", + "title": "Det stille biblioteket", + "text": "Lesere finner bøker om historie, ærlige fortellinger og vitenskap mens bibliotekaren hjelper elevene." + }, + { + "id": "market", + "split": "train", + "title": "Lørdagsmarked", + "text": "Bønder selger ferskt brød, epler, ost og honning pÃ¥ det travle torget." + }, + { + "id": "workshop", + "split": "train", + "title": "Digitalt verksted", + "text": "Et lite lag lager en museumsguide, undersøker koden og skriver tydelige tekster." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Tur til elven", + "text": "Vennene tok med kart og varm te pÃ¥ søndag og hvilte senere ved elven." + } + ] + }, + { + "code": "pt", + "name": "Portuguese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Manhã na cidade", + "text": "A cidade acorda devagar, os cafés abrem e as pessoas esperam os primeiros ônibus." + }, + { + "id": "library", + "split": "train", + "title": "A biblioteca tranquila", + "text": "Os leitores procuram livros de história e ciência enquanto a bibliotecária ajuda os alunos." + }, + { + "id": "market", + "split": "train", + "title": "Mercado de sábado", + "text": "Os agricultores vendem pão fresco, maçãs, queijo e mel na praça movimentada." + }, + { + "id": "workshop", + "split": "train", + "title": "Oficina digital", + "text": "Uma pequena equipe prepara o guia do museu, verifica o código e escreve descrições claras." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Viagem ao rio", + "text": "Os amigos levaram um mapa e chá quente, caminharam pela mata e descansaram junto ao rio." + } + ] + }, + { + "code": "sv", + "name": "Swedish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Morgon i staden", + "text": "Staden vaknar lÃ¥ngsamt, kaféerna öppnar och människor väntar pÃ¥ de första bussarna." + }, + { + "id": "library", + "split": "train", + "title": "Det tysta biblioteket", + "text": "Läsare söker böcker om historia och vetenskap medan bibliotekarien hjälper eleverna." + }, + { + "id": "market", + "split": "train", + "title": "Lördagsmarknad", + "text": "Bönder säljer färskt bröd, äpplen, ost och honung pÃ¥ det livliga torget." + }, + { + "id": "workshop", + "split": "train", + "title": "Digital verkstad", + "text": "Ett litet lag skapar en museiguide, granskar koden och skriver tydliga texter för besökare." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Utflykt till floden", + "text": "Vännerna tog med karta och varmt te och vilade sedan vid floden." + } + ] + }, + { + "code": "th", + "name": "Thai", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "เช้าในเมือง", + "text": "เมืองค่อย ๆ ตื่น ร้านà¸à¸²à¹à¸Ÿà¹€à¸›à¸´à¸”ประตูà¹à¸¥à¸°à¸œà¸¹à¹‰à¸„นรอรถโดยสารเที่ยวà¹à¸£à¸" + }, + { + "id": "library", + "split": "train", + "title": "ห้องสมุดเงียบสงบ", + "text": "ผู้อ่านค้นหาหนังสือประวัติศาสตร์à¹à¸¥à¸°à¸§à¸´à¸—ยาศาสตร์ บรรณารัà¸à¸©à¹Œà¸Šà¹ˆà¸§à¸¢à¸™à¸±à¸à¹€à¸£à¸µà¸¢à¸™à¹€à¸¥à¸·à¸­à¸à¸‚้อมูล" + }, + { + "id": "market", + "split": "train", + "title": "ตลาดวันเสาร์", + "text": "ชาวนาขายขนมปังสด ผลไม้ ชีส à¹à¸¥à¸°à¸™à¹‰à¸³à¸œà¸¶à¹‰à¸‡à¹ƒà¸™à¸¥à¸²à¸™à¸—ี่คึà¸à¸„ัà¸" + }, + { + "id": "workshop", + "split": "train", + "title": "ห้องทำงานดิจิทัล", + "text": "ทีมเล็à¸à¸ˆà¸±à¸”ทำคู่มือพิพิธภัณฑ์ ตรวจรหัส à¹à¸¥à¸°à¹€à¸‚ียนคำอธิบายที่ชัดเจน" + }, + { + "id": "river-trip", + "split": "validation", + "title": "เที่ยวริมà¹à¸¡à¹ˆà¸™à¹‰à¸³", + "text": "เพื่อนนำà¹à¸œà¸™à¸—ี่à¸à¸±à¸šà¸Šà¸²à¸£à¹‰à¸­à¸™à¹„ปเดินในป่าà¹à¸¥à¹‰à¸§à¸žà¸±à¸à¸œà¹ˆà¸­à¸™à¸£à¸´à¸¡à¹à¸¡à¹ˆà¸™à¹‰à¸³" + } + ] + }, + { + "code": "tr", + "name": "Turkish", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Åžehirde sabah", + "text": "Åžehir yavaşça uyanır, kafeler açılır ve insanlar ilk otobüsleri bekler." + }, + { + "id": "library", + "split": "train", + "title": "Sessiz kütüphane", + "text": "Okurlar tarih ve bilim kitapları ararken kütüphaneci öğrencilere yardımcı olur." + }, + { + "id": "market", + "split": "train", + "title": "Cumartesi pazarı", + "text": "Çiftçiler kalabalık meydanda taze ekmek, elma, peynir ve bal satar." + }, + { + "id": "workshop", + "split": "train", + "title": "Dijital atölye", + "text": "Küçük bir ekip müze rehberi hazırlar, kodu denetler ve açık açıklamalar yazar." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Nehir gezisi", + "text": "ArkadaÅŸlar harita ve sıcak çay alıp ormanda yürüdü, sonra nehir kıyısında dinlendi." + } + ] + }, + { + "code": "vi", + "name": "Vietnamese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "Buổi sáng trong thành phố", + "text": "Thành phố thức dậy chậm rãi, quán cà phê mở cá»­a và má»i ngÆ°á»i chá» chuyến xe buýt đầu tiên." + }, + { + "id": "library", + "split": "train", + "title": "ThÆ° viện yên tÄ©nh", + "text": "Äá»™c giả tìm sách lịch sá»­ và khoa há»c, còn thủ thÆ° giúp há»c sinh chá»n tài liệu." + }, + { + "id": "market", + "split": "train", + "title": "Chợ thứ bảy", + "text": "Nông dân bán bánh mì, táo, phô mai và mật ong tại quảng trÆ°á»ng đông đúc." + }, + { + "id": "workshop", + "split": "train", + "title": "Xưởng kỹ thuật số", + "text": "Má»™t nhóm nhá» làm hÆ°á»›ng dẫn bảo tàng, kiểm tra mã và viết mô tả rõ ràng." + }, + { + "id": "river-trip", + "split": "validation", + "title": "Chuyến Ä‘i ra sông", + "text": "Bạn bè mang bản đồ và trà nóng, Ä‘i qua rừng rồi nghỉ bên bá» sông." + } + ] } ] } diff --git a/corpus/test-sources.json b/corpus/test-sources.json index 9f75718..e138203 100644 --- a/corpus/test-sources.json +++ b/corpus/test-sources.json @@ -85,6 +85,102 @@ "id": "community-garden", "title": "Ogród sÄ…siedzki", "text": "MieszkaÅ„cy zmienili pusty dziedziniec w zielony ogród z kwiatami, warzywami i mÅ‚odymi drzewami. Dzieci zrobiÅ‚y kolorowe tabliczki, a rodziny co tydzieÅ„ podlewajÄ… roÅ›liny." + }, + { + "language": "ar", + "id": "community-garden", + "title": "حديقة الحي", + "text": "زرع الجيران الزهور والأشجار ÙÙŠ ساحة قديمة، وصنع الأطÙال لاÙتات ملونة واتÙÙ‚ الجميع على سقي النباتات." + }, + { + "language": "cs", + "id": "community-garden", + "title": "Sousedská zahrada", + "text": "Sousedé promÄ›nili prázdný dvůr v zelenou zahradu, dÄ›ti namalovaly barevné znaÄky a rodiny zalévají rostliny." + }, + { + "language": "da", + "id": "community-garden", + "title": "Fælleshaven", + "text": "Naboerne gjorde en tom gÃ¥rd til en grøn have, børnene malede skilte, og familierne vander planterne hver uge." + }, + { + "language": "en", + "id": "community-garden", + "title": "The community garden", + "text": "Neighbours turned an empty courtyard into a garden with flowers, vegetables, young trees, and a small café table." + }, + { + "language": "el", + "id": "community-garden", + "title": "Ο κοινοτικός κήπος", + "text": "Οι γείτονες έκαναν μια άδεια αυλή Ï€Ïάσινο κήπο, τα παιδιά ζωγÏάφισαν πινακίδες και οι οικογένειες ποτίζουν τα φυτά." + }, + { + "language": "et", + "id": "community-garden", + "title": "Kogukonna aed", + "text": "Naabrid muutsid tühja hoovi roheliseks aiaks, lapsed maalisid sildid ja pered kastavad taimi igal nädalal." + }, + { + "language": "he", + "id": "community-garden", + "title": "×”×’×™× ×” הקהילתית", + "text": "×”×©×›× ×™× ×”×¤×›×• חצר ריקה לגינה ירוקה, ×”×™×œ×“×™× ×¦×™×™×¨×• ×©×œ×˜×™× ×•×”×ž×©×¤×—×•×ª משקות ×ת ×”×¦×ž×—×™× ×‘×›×œ שבוע." + }, + { + "language": "hu", + "id": "community-garden", + "title": "Közösségi kert", + "text": "A szomszédok zöld kertté alakították az üres udvart, a gyerekek táblákat festettek, a családok pedig öntözik a növényeket." + }, + { + "language": "it", + "id": "community-garden", + "title": "Il giardino comune", + "text": "I vicini hanno trasformato un cortile vuoto in un giardino della città, i bambini hanno dipinto cartelli e le famiglie annaffiano le piante." + }, + { + "language": "nl", + "id": "community-garden", + "title": "De buurttuin", + "text": "Buren veranderden een lege binnenplaats naast een café in een groene tuin, kinderen schilderden borden en gezinnen geven de planten water." + }, + { + "language": "no", + "id": "community-garden", + "title": "Felleshagen", + "text": "Naboene gjorde en tom gÃ¥rd til en grønn hage, barna malte skilt og familiene vanner plantene hver uke." + }, + { + "language": "pt", + "id": "community-garden", + "title": "A horta comunitária", + "text": "Os vizinhos transformaram um pátio vazio em jardim, as crianças pintaram placas e as famílias regam as plantas toda semana." + }, + { + "language": "sv", + "id": "community-garden", + "title": "Gemensam trädgÃ¥rd", + "text": "Grannarna gjorde en tom gÃ¥rd till en grön trädgÃ¥rd, barnen mÃ¥lade skyltar och familjerna vattnar växterna varje vecka." + }, + { + "language": "th", + "id": "community-garden", + "title": "สวนชุมชน", + "text": "เพื่อนบ้านเปลี่ยนลานว่างเป็นสวนสีเขียว เด็ภๆ วาดป้ายà¹à¸¥à¸°à¸„รอบครัวช่วยà¸à¸±à¸™à¸£à¸”น้ำต้นไม้ทุà¸à¸ªà¸±à¸›à¸”าห์" + }, + { + "language": "tr", + "id": "community-garden", + "title": "Mahalle bahçesi", + "text": "KomÅŸular boÅŸ avluyu yeÅŸil bir bahçeye çevirdi, çocuklar renkli tabelalar yaptı ve aileler bitkileri her hafta suluyor." + }, + { + "language": "vi", + "id": "community-garden", + "title": "VÆ°á»n cá»™ng đồng", + "text": "Hàng xóm biến sân trống thành khu vÆ°á»n xanh, trẻ em vẽ biển màu và các gia đình tÆ°á»›i cây má»—i tuần." } ] } diff --git a/scripts/corpus-utils.mjs b/scripts/corpus-utils.mjs index 0d6b230..d12c846 100644 --- a/scripts/corpus-utils.mjs +++ b/scripts/corpus-utils.mjs @@ -137,13 +137,19 @@ export function buildCorpus(destination) { let modelHighByteCount = 0; const modelHighByteValues = new Set(); for (const document of language.documents) { - const utf8 = Buffer.from( - `${document.title}\n\n${document.text}\n`, - 'utf8', - ); + let sourceText = `${document.title}\n\n${document.text}\n`; + for (const [from, to] of Object.entries( + encoding.sourceReplacements ?? {}, + )) { + sourceText = sourceText.replaceAll(from, to); + } + const utf8 = Buffer.from(sourceText, 'utf8'); const encoded = iconv(utf8, 'UTF-8', encoding.iconv); const decoded = iconv(encoded, encoding.iconv, 'UTF-8'); - if (!decoded.equals(utf8)) { + if ( + decoded.toString('utf8').normalize('NFC') !== + utf8.toString('utf8').normalize('NFC') + ) { throw new Error( `${encoding.name}/${languageCode}/${document.id} did not round trip`, ); diff --git a/src/encoding/models/generated.ts b/src/encoding/models/generated.ts index 0f95a79..cd4eddb 100644 --- a/src/encoding/models/generated.ts +++ b/src/encoding/models/generated.ts @@ -1,6 +1,1995 @@ // Generated by npm run models:build. Do not edit manually. export const generatedSBCSModels = [ + { + encoding: 'ISO-8859-1', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'da', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206279, 0x2062f8, 0x206465, 0x2066f8, + 0x20686f, 0x206ce6, 0x206f67, 0x2070e5, 0x2073e6, 0x2076e5, 0x2076e6, + 0x20e562, 0x20e662, 0x6166e9, 0x626962, 0x626c69, 0x6272f8, 0x627965, + 0x62f867, 0x62f86e, 0x642062, 0x6420e6, 0x646520, 0x64656e, 0x646572, + 0x652062, 0x652066, 0x6520e5, 0x656420, 0x656e20, 0x656e73, 0x657220, + 0x65726e, 0x657420, 0x66e965, 0x66f872, 0x676572, 0x686ae6, 0x69626c, + 0x696465, 0x696c6c, 0x696f74, 0x6ae66c, 0x6b7374, 0x6c6520, 0x6c696f, + 0x6c6c65, 0x6ce673, 0x6e6465, 0x6e6520, 0x6e6572, 0x6f6720, 0x6f7465, + 0x70e520, 0x722020, 0x722065, 0x72206f, 0x737465, 0x74656b, 0xe52064, + 0xf86765, + ], + highBytes: { + total: 16, + counts: [ + [0xe5, 4], + [0xe6, 5], + [0xe9, 1], + [0xf8, 6], + ], + }, + }, + { + language: 'de', + ngrams: [ + 0x206175, 0x206265, 0x206269, 0x2062e4, 0x2062fc, 0x206465, 0x206469, + 0x2064e4, 0x206569, 0x2066fc, 0x2067e4, 0x206be4, 0x206ce4, 0x207465, + 0x2074fc, 0x20756e, 0x20e470, 0x20f666, 0x20fc62, 0x616368, 0x616d20, + 0x617573, 0x626572, 0x62e463, 0x62fc63, 0x636865, 0x636874, 0x6368fc, + 0x64656e, 0x646572, 0x646965, 0x64e463, 0x652073, 0x652074, 0x65696e, + 0x656dfc, 0x656e20, 0x656efc, 0x657220, 0x65726e, 0x66fc68, 0x66fc72, + 0x67e473, 0x68656e, 0x687265, 0x696368, 0x696520, 0x696e20, 0x696e65, + 0x6c6569, 0x6e2062, 0x6e2064, 0x6e20fc, 0x6e6420, 0x6e656e, 0x736368, + 0x736520, 0x737461, 0x742065, 0x746520, 0x756e64, 0xe46e64, 0xfc6265, + 0xfc7220, + ], + highBytes: { + total: 21, + counts: [ + [0xc4, 1], + [0xdc, 1], + [0xe4, 7], + [0xf6, 1], + [0xfc, 11], + ], + }, + }, + { + language: 'en', + ngrams: [ + 0x206120, 0x20616e, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206661, + 0x206c69, 0x206e65, 0x2070e2, 0x207265, 0x2072e9, 0x2072f4, 0x207468, + 0x207768, 0x616420, 0x616465, 0x6166e9, 0x616e64, 0x61e761, 0x61ef76, + 0x626f75, 0x627261, 0x636974, 0x642063, 0x64206e, 0x646520, 0x646573, + 0x652061, 0x652063, 0x656164, 0x657273, 0x657320, 0x657369, 0x657420, + 0x65f16f, 0x6661e7, 0x66e973, 0x686520, 0x68696c, 0x696272, 0x696465, + 0x696c65, 0x6c6520, 0x6de920, 0x6e61ef, 0x6e6420, 0x7065f1, 0x70e274, + 0x726561, 0x727320, 0x72e973, 0x72f46c, 0x732061, 0x746865, 0x74e920, + 0x756de9, 0xe274e9, 0xe76164, 0xe92020, 0xe92077, 0xe97320, 0xe97375, + 0xef7665, + ], + highBytes: { + total: 9, + counts: [ + [0xe2, 1], + [0xe7, 1], + [0xe9, 4], + [0xef, 1], + [0xf1, 1], + [0xf4, 1], + ], + }, + }, + { + language: 'es', + ngrams: [ + 0x206269, 0x206369, 0x20636c, 0x20636f, 0x2063f3, 0x206465, 0x206469, + 0x20656c, 0x206c61, 0x206c6f, 0x206d65, 0x206de1, 0x207061, 0x2070e1, + 0x2070fa, 0x207265, 0x207375, 0x2073e1, 0x207665, 0x207920, 0x612062, + 0x612065, 0x612070, 0x61646f, 0x616e20, 0x617261, 0x617320, 0x6174e1, + 0x61f161, 0x626c69, 0x6369e9, 0x6369f3, 0x63f364, 0x656c20, 0x656e20, + 0x656efa, 0x6572ed, 0x657320, 0x65f161, 0x65f16f, 0x69e96e, 0x69f36e, + 0x6c206d, 0x6c6120, 0x6c6f73, 0x6d61f1, 0x6de173, 0x6e206c, 0x6e6120, + 0x6efa20, 0x6f206c, 0x6f7265, 0x6f7320, 0x706172, 0x70e167, 0x70fa62, + 0x726573, 0x72ed61, 0x732061, 0x732063, 0x73206c, 0x7365f1, 0x73e162, + 0x746f72, + ], + highBytes: { + total: 13, + counts: [ + [0xe1, 4], + [0xe9, 1], + [0xed, 1], + [0xf1, 3], + [0xf3, 2], + [0xfa, 2], + ], + }, + }, + { + language: 'fr', + ngrams: [ + 0x20636f, 0x206465, 0x206475, 0x2064e9, 0x206574, 0x206c65, 0x207061, + 0x20706f, 0x207072, 0x2070e2, 0x2072e9, 0x20756e, 0x2076e9, 0x20e963, + 0x20e96c, 0x20e971, 0x6166e9, 0x626c69, 0x6368e9, 0x646573, 0x647520, + 0x64e976, 0x652063, 0x652064, 0x65206c, 0x65206d, 0x652070, 0x6520e9, + 0x656e63, 0x656e74, 0x657320, 0x657420, 0x657572, 0x66e973, 0x68e871, + 0x68e920, 0x6c6120, 0x6c6520, 0x6c6573, 0x6c6c65, 0x6de972, 0x6e7420, + 0x6f7572, 0x706172, 0x7072e9, 0x717565, 0x72206c, 0x726520, 0x727320, + 0x72e970, 0x732063, 0x73206c, 0x732070, 0x742064, 0x74206c, 0x7468e8, + 0x756520, 0x756de9, 0x757273, 0xe87175, 0xe96520, 0xe97061, 0xe97269, + 0xe97665, + ], + highBytes: { + total: 22, + counts: [ + [0xe2, 1], + [0xe8, 3], + [0xe9, 18], + ], + }, + }, + { + language: 'it', + ngrams: [ + 0x00006c, 0x00006d, 0x006c61, 0x206269, 0x206361, 0x206369, 0x20636f, + 0x206465, 0x206469, 0x206520, 0x206765, 0x20676c, 0x206920, 0x206c61, + 0x206c65, 0x206d65, 0x207065, 0x207069, 0x207072, 0x207363, 0x207369, + 0x207374, 0x612062, 0x612063, 0x612064, 0x612067, 0x612069, 0x612070, + 0x61746f, 0x6368e9, 0x636974, 0x64656c, 0x656e74, 0x657263, 0x6666e8, + 0x66e820, 0x676c69, 0x68e920, 0x692061, 0x692063, 0x692073, 0x696120, + 0x697474, 0x69f920, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6f20, + 0x6f7269, 0x7069f9, 0x726920, 0x746f72, 0x7474e0, 0x74e020, 0xe02020, + 0xe0206c, 0xe02073, 0xe82061, 0xe8206d, 0xe92073, 0xf92067, 0xf9206c, + 0xf92070, + ], + highBytes: { + total: 9, + counts: [ + [0xe0, 3], + [0xe8, 2], + [0xe9, 1], + [0xf9, 3], + ], + }, + }, + { + language: 'nl', + ngrams: [ + 0x000064, 0x206269, 0x20626f, 0x206361, 0x20636f, 0x206465, 0x206565, + 0x20656e, 0x206765, 0x206865, 0x206c65, 0x206f70, 0x206f76, 0x207374, + 0x207465, 0x207665, 0x207765, 0x20e9e9, 0x61616b, 0x616173, 0x616420, + 0x6166e9, 0x616b74, 0x616d20, 0x626962, 0x626c69, 0x626f65, 0x636166, + 0x636874, 0x646520, 0x652062, 0x652063, 0x652065, 0x652073, 0x65656e, + 0x656572, 0x6565eb, 0x65696e, 0x656b65, 0x656e20, 0x657273, 0x65eb6e, + 0x65ef6e, 0x66e973, 0x6765ef, 0x696e20, 0x6b7420, 0x6e2020, 0x6e206f, + 0x6e20e9, 0x6feb7a, 0x706feb, 0x732065, 0x736368, 0x742064, 0x74656e, + 0x746572, 0x766572, 0xe96e20, 0xe97320, 0xe9e96e, 0xeb6e20, 0xeb7a69, + 0xef6e74, + ], + highBytes: { + total: 7, + counts: [ + [0xe9, 4], + [0xeb, 2], + [0xef, 1], + ], + }, + }, + { + language: 'no', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206272, 0x206279, 0x2062f8, 0x206465, + 0x2064f8, 0x20666f, 0x2066f8, 0x206c61, 0x206ee6, 0x206f67, 0x2070e5, + 0x207665, 0x2076e5, 0x20e570, 0x20e672, 0x626962, 0x626c69, 0x6272f8, + 0x627965, 0x62f86b, 0x62f86e, 0x642065, 0x646520, 0x646572, 0x646574, + 0x64f872, 0x652062, 0x652066, 0x652074, 0x6520e5, 0x6520e6, 0x656420, + 0x656e20, 0x656e65, 0x656e73, 0x657220, 0x657273, 0x657420, 0x66f872, + 0x672073, 0x676520, 0x67656e, 0x676572, 0x69626c, 0x696765, 0x696e67, + 0x696f74, 0x6af865, 0x6cf872, 0x6e6520, 0x6e6572, 0x6ee672, 0x6f6720, + 0x70e520, 0x72206f, 0x7273f8, 0x72f864, 0x737465, 0x74656b, 0xe52064, + 0xf86b65, + ], + highBytes: { + total: 14, + counts: [ + [0xe5, 4], + [0xe6, 2], + [0xf8, 8], + ], + }, + }, + { + language: 'pt', + ngrams: [ + 0x00006d, 0x206120, 0x206269, 0x206369, 0x2063f3, 0x206465, 0x206520, + 0x206573, 0x206e61, 0x206f20, 0x206f73, 0x207065, 0x207072, 0x2070e3, + 0x2073e1, 0x207665, 0x20f46e, 0x612062, 0x612063, 0x612064, 0x612065, + 0x61206f, 0x612070, 0x616465, 0x61646f, 0x6166e9, 0x616d20, 0x617261, + 0x617320, 0x61e761, 0x61e7e3, 0x626962, 0x626c69, 0x636120, 0x6369ea, + 0x63e172, 0x63f364, 0x646120, 0x646520, 0x646f20, 0x652061, 0x6563e1, + 0x657320, 0x657363, 0x66e973, 0x68e320, 0x696120, 0x69e7f5, 0x69ea6e, + 0x6d61e7, 0x6e6120, 0x6e68e3, 0x6f7320, 0x70e36f, 0x7261e7, 0x726573, + 0x7269e7, 0x732020, 0x732061, 0x732070, 0x7320f4, 0x7374f3, 0x73e162, + 0x74f372, + ], + highBytes: { + total: 14, + counts: [ + [0xe1, 2], + [0xe3, 3], + [0xe7, 3], + [0xe9, 1], + [0xea, 1], + [0xf3, 2], + [0xf4, 1], + [0xf5, 1], + ], + }, + }, + { + language: 'sv', + ngrams: [ + 0x000064, 0x006cf6, 0x206269, 0x2062f6, 0x206465, 0x2066e4, 0x2066f6, + 0x206c69, 0x206ce4, 0x206ce5, 0x206de4, 0x206f63, 0x2070e5, 0x20736b, + 0x207374, 0x2073e4, 0x2073f6, 0x207479, 0x207665, 0x2076e4, 0x20e470, + 0x20f670, 0x612020, 0x612062, 0x612074, 0x6120f6, 0x616420, 0x616465, + 0x6166e9, 0x617220, 0x617265, 0x626962, 0x626c69, 0x6272f6, 0x62f663, + 0x62f66e, 0x636820, 0x6420e4, 0x646520, 0x64656e, 0x646574, 0x656e20, + 0x657220, 0x65726e, 0x6573f6, 0x657420, 0x66f672, 0x676120, 0x69626c, + 0x696761, 0x696f74, 0x6b6170, 0x6b6172, 0x6e6120, 0x6f6368, 0x70e520, + 0x726e61, 0x736b61, 0x737461, 0x73f66b, 0x74206c, 0x746164, 0x766572, + 0xe52064, + ], + highBytes: { + total: 20, + counts: [ + [0xe4, 7], + [0xe5, 3], + [0xe9, 1], + [0xf6, 9], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-2', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0x20, + 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0x20, 0xb1, 0x20, 0xb3, + 0x20, 0xb5, 0xb6, 0xb7, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'cs', + ngrams: [ + 0x000064, 0x000072, 0x000073, 0x000074, 0x006469, 0x0072e1, 0x00736f, + 0x007469, 0x206120, 0x206175, 0x206279, 0x206368, 0x206476, 0x2064ed, + 0x206661, 0x206869, 0x20686c, 0x206b6e, 0x206bf3, 0x206dec, 0x20706f, + 0x207072, 0x2070ed, 0x2070f8, 0x2074fd, 0x2076ec, 0x20bee1, 0x20e865, + 0x20e874, 0x612070, 0x612076, 0x6120e8, 0x6164ed, 0x616aed, 0x616cfd, + 0x6174ed, 0x636520, 0x652061, 0x65206d, 0x652070, 0x686f76, 0x69686f, + 0x6a6520, 0x6aed20, 0x6b6e69, 0x6d616c, 0x6dec73, 0x6e6120, 0x6e6968, + 0x6e7920, 0x6eed20, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, 0x737920, + 0x756a65, 0x792020, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xed206b, + 0xf86920, + ], + highBytes: { + total: 46, + counts: [ + [0xb9, 1], + [0xbe, 1], + [0xc8, 1], + [0xe1, 8], + [0xe8, 2], + [0xe9, 4], + [0xec, 5], + [0xed, 13], + [0xf2, 1], + [0xf3, 1], + [0xf8, 4], + [0xf9, 2], + [0xfd, 3], + ], + }, + }, + { + language: 'hu', + ngrams: [ + 0x000061, 0x000064, 0x000072, 0x000073, 0x006120, 0x006469, 0x007265, + 0x00737a, 0x206120, 0x20616c, 0x20617a, 0x206275, 0x206373, 0x206b65, + 0x206b69, 0x206be1, 0x206be9, 0x206bf3, 0x206bf6, 0x206de9, 0x206dfa, + 0x206dfb, 0x2074e9, 0x2074f6, 0x2076e1, 0x20e172, 0x20e962, 0x20e973, + 0x20ed72, 0x20fa74, 0x61206b, 0x612076, 0x616b20, 0x616e20, 0x617420, + 0x617a20, 0x646967, 0x656b20, 0x657265, 0x657420, 0x697320, 0x6b2020, + 0x6b2061, 0x6bf66e, 0x6d6920, 0x6e616b, 0x6e7976, 0x6f7320, 0x6f7420, + 0x726f73, 0x73206b, 0x73206c, 0x73206d, 0x742061, 0x74206b, 0x74e172, + 0x7674e1, 0x76e172, 0x797674, 0xe1726f, 0xe97320, 0xed7420, 0xf36b20, + 0xf66e79, + ], + highBytes: { + total: 39, + counts: [ + [0xe1, 14], + [0xe9, 9], + [0xed, 4], + [0xf3, 4], + [0xf5, 1], + [0xf6, 4], + [0xfa, 2], + [0xfb, 1], + ], + }, + }, + { + language: 'pl', + ngrams: [ + 0x000063, 0x206120, 0x206269, 0x20637a, 0x20646c, 0x206920, 0x206a61, + 0x206b61, 0x206b6f, 0x206d69, 0x20706f, 0x207072, 0x20726f, 0x2073b3, + 0x207779, 0x20b677, 0x6162b3, 0x616368, 0x616ab1, 0x616d69, 0x6177ea, + 0x61b379, 0x61b66e, 0x61e620, 0x626962, 0x626c69, 0x62b36b, 0x636861, + 0x636920, 0x637a79, 0x6472f3, 0x647a69, 0x656b20, 0x65b36e, 0x65b663, + 0x65bf79, 0x676fb6, 0x6861e6, 0x692073, 0x6920b6, 0x69626c, 0x696520, + 0x696572, 0x696f74, 0x69ea20, 0x6ab120, 0x6b6120, 0x6c696f, 0x6d6920, + 0x6e6961, 0x6f7261, 0x6f7465, 0x6fb663, 0x706973, 0x70727a, 0x727a65, + 0x727a79, 0x7369ea, 0x737920, 0x74656b, 0xb1206a, 0xb1206b, 0xb12077, + 0xb66369, + ], + highBytes: { + total: 34, + counts: [ + [0xb1, 11], + [0xb3, 6], + [0xb6, 6], + [0xbf, 3], + [0xe6, 1], + [0xea, 3], + [0xf3, 4], + ], + }, + }, + { + language: 'ro', + ngrams: [ + 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, + 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, + 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, + 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, + 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, + 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, + 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, + 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, + 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, + 0xfe6120, + ], + highBytes: { + total: 49, + counts: [ + [0xba, 12], + [0xce, 1], + [0xe2, 3], + [0xe3, 21], + [0xee, 4], + [0xfe, 8], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-5', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff, 0xd0, 0xd1, 0xd2, 0xd3, + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0x20, 0xfe, 0xff, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000e0, 0x00e0d0, 0x20d020, 0x20d1d8, 0x20d220, 0x20d2de, 0x20d2eb, + 0x20d3de, 0x20d820, 0x20dade, 0x20dcd0, 0x20ddd0, 0x20ded1, 0x20dee1, + 0x20dee2, 0x20dfdb, 0x20dfde, 0x20dfe0, 0x20e0d5, 0x20e1de, 0x20e7d8, + 0x20efe0, 0xd020d2, 0xd020dd, 0xd020df, 0xd0d5e2, 0xd0ddd8, 0xd0e0da, + 0xd0e1e2, 0xd0e2d5, 0xd0eee2, 0xd0ef20, 0xd1d8d1, 0xd1dbd8, 0xd1e1e3, + 0xd2d0ee, 0xd2d5e0, 0xd2efe2, 0xd3dee0, 0xd3dee2, 0xd4d0ee, 0xd4d820, + 0xd5dbd5, 0xd5dbd8, 0xd5dddd, 0xd820df, 0xdad020, 0xdbd820, 0xdcd0e0, + 0xddd020, 0xddebd5, 0xdee1d5, 0xdfe0de, 0xe0ded4, 0xe0eb20, 0xe220e1, + 0xe2d5db, 0xe2d5e0, 0xe2ded2, 0xe2eb20, 0xebd520, 0xeee220, 0xef20df, + 0xefe220, + ], + highBytes: { + total: 678, + counts: [ + [0xb2, 1], + [0xb3, 1], + [0xb4, 1], + [0xbb, 1], + [0xbd, 3], + [0xbe, 1], + [0xbf, 1], + [0xc0, 2], + [0xc1, 1], + [0xc3, 1], + [0xc4, 1], + [0xc7, 1], + [0xd0, 63], + [0xd1, 16], + [0xd2, 26], + [0xd3, 7], + [0xd4, 21], + [0xd5, 50], + [0xd6, 5], + [0xd7, 7], + [0xd8, 47], + [0xd9, 7], + [0xda, 21], + [0xdb, 19], + [0xdc, 16], + [0xdd, 37], + [0xde, 60], + [0xdf, 23], + [0xe0, 41], + [0xe1, 34], + [0xe2, 55], + [0xe3, 14], + [0xe5, 7], + [0xe6, 3], + [0xe7, 4], + [0xe8, 6], + [0xe9, 2], + [0xeb, 26], + [0xec, 3], + [0xed, 1], + [0xee, 13], + [0xef, 27], + [0xf1, 2], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-6', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'ar', + ngrams: [ + 0x0000c7, 0x0000d3, 0x0000d5, 0x0000e8, 0x00c7e4, 0x00d3e8, 0x00d5c8, + 0x00e8d1, 0x20c3c8, 0x20c3e5, 0x20c7ce, 0x20c7e4, 0x20c8e7, 0x20c8ea, + 0x20cad3, 0x20cfe4, 0x20d1e2, 0x20d5da, 0x20d9e6, 0x20e1d1, 0x20e1ea, + 0x20e3ca, 0x20e4e4, 0x20e8c7, 0x20e8ca, 0x20e8d5, 0x20e8ea, 0xc7ca20, + 0xc7ccd9, 0xc7d120, 0xc7e4c3, 0xc7e4d9, 0xc7e4e5, 0xc7e4e8, 0xc820c7, + 0xc8c920, 0xc92020, 0xc920c7, 0xc920e8, 0xc920ea, 0xca20e8, 0xcac820, + 0xcac8c9, 0xcb20c7, 0xccd920, 0xcd20c7, 0xcfeae6, 0xd120c7, 0xd1c7cc, + 0xd5e1c7, 0xd920c7, 0xe3cac8, 0xe4e5cf, 0xe4e5e3, 0xe5cfea, 0xe5e3ca, + 0xe620c7, 0xe6c7d3, 0xe6c920, 0xe8c7e4, 0xe8d5e1, 0xe8e620, 0xeac920, + 0xeae6c9, + ], + highBytes: { + total: 324, + counts: [ + [0xc1, 2], + [0xc3, 4], + [0xc4, 1], + [0xc6, 1], + [0xc7, 56], + [0xc8, 17], + [0xc9, 11], + [0xca, 16], + [0xcb, 2], + [0xcc, 3], + [0xcd, 8], + [0xce, 4], + [0xcf, 9], + [0xd1, 19], + [0xd2, 3], + [0xd3, 7], + [0xd4, 2], + [0xd5, 5], + [0xd6, 2], + [0xd7, 1], + [0xd8, 2], + [0xd9, 8], + [0xda, 1], + [0xe1, 9], + [0xe2, 7], + [0xe3, 5], + [0xe4, 36], + [0xe5, 18], + [0xe6, 12], + [0xe7, 5], + [0xe8, 24], + [0xea, 24], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-7', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xdc, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'el', + ngrams: [ + 0x0000de, 0x0000f0, 0x0000f3, 0x0000f8, 0x00def3, 0x00f0f1, 0x00f3e1, + 0x00f8e7, 0x20dced, 0x20e1e3, 0x20e1ed, 0x20e1f1, 0x20e2e9, 0x20e2ef, + 0x20e3f1, 0x20e5eb, 0x20e5f0, 0x20e5f1, 0x20e720, 0x20eae1, 0x20ece9, + 0x20efe9, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xdc20f4, 0xdfe120, + 0xe12020, 0xe120ea, 0xe1e920, 0xe2e9e2, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, + 0xe5e920, 0xe5f220, 0xe720e2, 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9e2eb, + 0xe9efe8, 0xe9f3f4, 0xeae1e9, 0xeae720, 0xebe720, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xeff5ed, 0xf0fceb, 0xf1dc20, 0xf1e9ef, + 0xf220ea, 0xf3f4de, 0xf3f4e7, 0xf4e120, 0xf4e5f2, 0xf4e7ed, 0xf5ed20, + 0xfcebe7, + ], + highBytes: { + total: 327, + counts: [ + [0xb9, 1], + [0xc7, 1], + [0xcc, 1], + [0xcf, 2], + [0xd0, 1], + [0xd3, 1], + [0xd8, 1], + [0xdc, 13], + [0xdd, 5], + [0xde, 4], + [0xdf, 9], + [0xe1, 31], + [0xe2, 9], + [0xe3, 9], + [0xe4, 4], + [0xe5, 16], + [0xe6, 1], + [0xe7, 14], + [0xe8, 6], + [0xe9, 31], + [0xea, 16], + [0xeb, 10], + [0xec, 11], + [0xed, 15], + [0xee, 1], + [0xef, 24], + [0xf0, 8], + [0xf1, 16], + [0xf2, 7], + [0xf3, 8], + [0xf4, 17], + [0xf5, 10], + [0xf6, 5], + [0xf7, 3], + [0xf8, 2], + [0xf9, 5], + [0xfc, 5], + [0xfd, 2], + [0xfe, 2], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-8', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'he', + ngrams: [ + 0x0000e1, 0x0000e4, 0x0000f1, 0x0000f9, 0x00e1e5, 0x00e4f1, 0x00f1e3, + 0x00f9e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, 0x20e1fa, + 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f7, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e7f7, 0x20e8f8, + 0x20ece0, 0x20ece1, 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, + 0x20eee7, 0x20eeeb, 0xe0e9ed, 0xe4f1f4, 0xe5f8e9, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed20, 0xe9f820, 0xe9fa20, 0xed2020, 0xed20ec, + 0xed20ee, 0xef20ee, 0xf0e420, 0xf1f4f8, 0xf2e9f8, 0xf4f8e9, 0xf7e5f8, + 0xf820e4, 0xf820ee, 0xf8e5fa, 0xf8e920, 0xf8e9ed, 0xf8fa20, 0xf9e9ed, + 0xfa20ec, + ], + highBytes: { + total: 249, + counts: [ + [0xe0, 9], + [0xe1, 12], + [0xe2, 2], + [0xe3, 8], + [0xe4, 16], + [0xe5, 30], + [0xe6, 2], + [0xe7, 6], + [0xe8, 7], + [0xe9, 33], + [0xea, 1], + [0xeb, 6], + [0xec, 9], + [0xed, 13], + [0xee, 11], + [0xef, 3], + [0xf0, 6], + [0xf1, 7], + [0xf2, 6], + [0xf4, 7], + [0xf6, 1], + [0xf7, 10], + [0xf8, 22], + [0xf9, 7], + [0xfa, 15], + ], + }, + }, + ], + }, + { + encoding: 'ISO-8859-9', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'tr', + ngrams: [ + 0x000063, 0x000064, 0x000073, 0x0000fe, 0x006375, 0x00fe65, 0x2061e7, + 0x206269, 0x20656b, 0x206b61, 0x206bfc, 0x206dfc, 0x207361, 0x207461, + 0x207665, 0x207961, 0x20e769, 0x20f6f0, 0x20fe65, 0x616261, 0x616c20, + 0x616c61, 0x616cfd, 0x616e65, 0x616efd, 0x617220, 0x6172fd, 0x6174f6, + 0x617a61, 0x617afd, 0x61e7fd, 0x61fee7, 0x62616c, 0x652062, 0x656869, + 0x657220, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697220, 0x697461, + 0x6bfc74, 0x6c6172, 0x6c6572, 0x706861, 0x722020, 0x72206b, 0x722076, + 0x722079, 0x726920, 0x726c61, 0x72fd20, 0x746172, 0x74fc70, 0x766520, + 0x7a6172, 0x7a6520, 0xe7fd6b, 0xfc7068, 0xfc74fc, 0xfd6b20, 0xfd7220, + 0xfe6568, + ], + highBytes: { + total: 32, + counts: [ + [0xc7, 1], + [0xde, 2], + [0xe7, 6], + [0xf0, 1], + [0xf6, 2], + [0xfc, 8], + [0xfd, 11], + [0xfe, 1], + ], + }, + }, + ], + }, + { + encoding: 'windows-1250', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa1, 0x20, 0xb3, 0x20, 0xb9, 0x20, 0x20, + 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0x20, 0xb3, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0xb9, 0xba, 0x20, 0xbe, 0x20, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'cs', + ngrams: [ + 0x000064, 0x000072, 0x000073, 0x000074, 0x006469, 0x0072e1, 0x00736f, + 0x007469, 0x206120, 0x206175, 0x206279, 0x206368, 0x206476, 0x2064ed, + 0x206661, 0x206869, 0x20686c, 0x206b6e, 0x206bf3, 0x206dec, 0x20706f, + 0x207072, 0x2070ed, 0x2070f8, 0x2074fd, 0x2076ec, 0x209ee1, 0x20e865, + 0x20e874, 0x612070, 0x612076, 0x6120e8, 0x6164ed, 0x616aed, 0x616cfd, + 0x6174ed, 0x636520, 0x652061, 0x65206d, 0x652070, 0x686f76, 0x69686f, + 0x6a6520, 0x6aed20, 0x6b6e69, 0x6d616c, 0x6dec73, 0x6e6120, 0x6e6968, + 0x6e7920, 0x6eed20, 0x6f626f, 0x6f766e, 0x70726f, 0x73746f, 0x737920, + 0x756a65, 0x792020, 0x79206f, 0xe1f869, 0xec7374, 0xed2064, 0xed206b, + 0xf86920, + ], + highBytes: { + total: 46, + counts: [ + [0x9a, 1], + [0x9e, 1], + [0xc8, 1], + [0xe1, 8], + [0xe8, 2], + [0xe9, 4], + [0xec, 5], + [0xed, 13], + [0xf2, 1], + [0xf3, 1], + [0xf8, 4], + [0xf9, 2], + [0xfd, 3], + ], + }, + }, + { + language: 'hu', + ngrams: [ + 0x000061, 0x000064, 0x000072, 0x000073, 0x006120, 0x006469, 0x007265, + 0x00737a, 0x206120, 0x20616c, 0x20617a, 0x206275, 0x206373, 0x206b65, + 0x206b69, 0x206be1, 0x206be9, 0x206bf3, 0x206bf6, 0x206de9, 0x206dfa, + 0x206dfb, 0x2074e9, 0x2074f6, 0x2076e1, 0x20e172, 0x20e962, 0x20e973, + 0x20ed72, 0x20fa74, 0x61206b, 0x612076, 0x616b20, 0x616e20, 0x617420, + 0x617a20, 0x646967, 0x656b20, 0x657265, 0x657420, 0x697320, 0x6b2020, + 0x6b2061, 0x6bf66e, 0x6d6920, 0x6e616b, 0x6e7976, 0x6f7320, 0x6f7420, + 0x726f73, 0x73206b, 0x73206c, 0x73206d, 0x742061, 0x74206b, 0x74e172, + 0x7674e1, 0x76e172, 0x797674, 0xe1726f, 0xe97320, 0xed7420, 0xf36b20, + 0xf66e79, + ], + highBytes: { + total: 39, + counts: [ + [0xe1, 14], + [0xe9, 9], + [0xed, 4], + [0xf3, 4], + [0xf5, 1], + [0xf6, 4], + [0xfa, 2], + [0xfb, 1], + ], + }, + }, + { + language: 'pl', + ngrams: [ + 0x000063, 0x206120, 0x206269, 0x20637a, 0x20646c, 0x206920, 0x206a61, + 0x206b61, 0x206b6f, 0x206d69, 0x20706f, 0x207072, 0x20726f, 0x2073b3, + 0x207779, 0x209c77, 0x6162b3, 0x616368, 0x616ab9, 0x616d69, 0x6177ea, + 0x619c6e, 0x61b379, 0x61e620, 0x626962, 0x626c69, 0x62b36b, 0x636861, + 0x636920, 0x637a79, 0x6472f3, 0x647a69, 0x656b20, 0x659c63, 0x65b36e, + 0x65bf79, 0x676f9c, 0x6861e6, 0x692073, 0x69209c, 0x69626c, 0x696520, + 0x696572, 0x696f74, 0x69ea20, 0x6ab920, 0x6b6120, 0x6c696f, 0x6d6920, + 0x6e6961, 0x6f7261, 0x6f7465, 0x6f9c63, 0x706973, 0x70727a, 0x727a65, + 0x727a79, 0x7369ea, 0x737920, 0x74656b, 0x9c6369, 0xb9206a, 0xb9206b, + 0xb92077, + ], + highBytes: { + total: 34, + counts: [ + [0x9c, 6], + [0xb3, 6], + [0xb9, 11], + [0xbf, 3], + [0xe6, 1], + [0xea, 3], + [0xf3, 4], + ], + }, + }, + { + language: 'ro', + ngrams: [ + 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, + 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, + 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, + 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, + 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, + 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, + 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, + 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, + 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, + 0xfe6120, + ], + highBytes: { + total: 49, + counts: [ + [0xba, 12], + [0xce, 1], + [0xe2, 3], + [0xe3, 21], + [0xee, 4], + [0xfe, 8], + ], + }, + }, + ], + }, + { + encoding: 'windows-1251', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x90, 0x83, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, 0x9c, 0x9d, 0x9e, 0x9f, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x9d, 0x9e, 0x9f, 0x20, 0xa2, 0xa2, 0xbc, 0x20, 0xb4, 0x20, 0x20, + 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0xb3, 0xb3, + 0xb4, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0xbc, 0xbe, 0xbe, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000f0, 0x00f0e0, 0x20e020, 0x20e1e8, 0x20e220, 0x20e2ee, 0x20e2fb, + 0x20e3ee, 0x20e820, 0x20eaee, 0x20ece0, 0x20ede0, 0x20eee1, 0x20eef1, + 0x20eef2, 0x20efeb, 0x20efee, 0x20eff0, 0x20f0e5, 0x20f1ee, 0x20f7e8, + 0x20fff0, 0xe020e2, 0xe020ed, 0xe020ef, 0xe0e5f2, 0xe0ede8, 0xe0f0ea, + 0xe0f1f2, 0xe0f2e5, 0xe0fef2, 0xe0ff20, 0xe1e8e1, 0xe1ebe8, 0xe1f1f3, + 0xe2e0fe, 0xe2e5f0, 0xe2fff2, 0xe3eef0, 0xe3eef2, 0xe4e0fe, 0xe4e820, + 0xe5ebe5, 0xe5ebe8, 0xe5eded, 0xe820ef, 0xeae020, 0xebe820, 0xece0f0, + 0xede020, 0xedfbe5, 0xeef1e5, 0xeff0ee, 0xf0eee4, 0xf0fb20, 0xf220f1, + 0xf2e5eb, 0xf2e5f0, 0xf2eee2, 0xf2fb20, 0xfbe520, 0xfef220, 0xff20ef, + 0xfff220, + ], + highBytes: { + total: 678, + counts: [ + [0xb8, 2], + [0xc2, 1], + [0xc3, 1], + [0xc4, 1], + [0xcb, 1], + [0xcd, 3], + [0xce, 1], + [0xcf, 1], + [0xd0, 2], + [0xd1, 1], + [0xd3, 1], + [0xd4, 1], + [0xd7, 1], + [0xe0, 63], + [0xe1, 16], + [0xe2, 26], + [0xe3, 7], + [0xe4, 21], + [0xe5, 50], + [0xe6, 5], + [0xe7, 7], + [0xe8, 47], + [0xe9, 7], + [0xea, 21], + [0xeb, 19], + [0xec, 16], + [0xed, 37], + [0xee, 60], + [0xef, 23], + [0xf0, 41], + [0xf1, 34], + [0xf2, 55], + [0xf3, 14], + [0xf5, 7], + [0xf6, 3], + [0xf7, 4], + [0xf8, 6], + [0xf9, 2], + [0xfb, 26], + [0xfc, 3], + [0xfd, 1], + [0xfe, 13], + [0xff, 27], + ], + }, + }, + ], + }, + { + encoding: 'windows-1252', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x9a, 0x20, 0x9c, 0x20, 0x9e, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x9e, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'da', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206279, 0x2062f8, 0x206465, 0x2066f8, + 0x20686f, 0x206ce6, 0x206f67, 0x2070e5, 0x2073e6, 0x2076e5, 0x2076e6, + 0x20e562, 0x20e662, 0x6166e9, 0x626962, 0x626c69, 0x6272f8, 0x627965, + 0x62f867, 0x62f86e, 0x642062, 0x6420e6, 0x646520, 0x64656e, 0x646572, + 0x652062, 0x652066, 0x6520e5, 0x656420, 0x656e20, 0x656e73, 0x657220, + 0x65726e, 0x657420, 0x66e965, 0x66f872, 0x676572, 0x686ae6, 0x69626c, + 0x696465, 0x696c6c, 0x696f74, 0x6ae66c, 0x6b7374, 0x6c6520, 0x6c696f, + 0x6c6c65, 0x6ce673, 0x6e6465, 0x6e6520, 0x6e6572, 0x6f6720, 0x6f7465, + 0x70e520, 0x722020, 0x722065, 0x72206f, 0x737465, 0x74656b, 0xe52064, + 0xf86765, + ], + highBytes: { + total: 16, + counts: [ + [0xe5, 4], + [0xe6, 5], + [0xe9, 1], + [0xf8, 6], + ], + }, + }, + { + language: 'de', + ngrams: [ + 0x206175, 0x206265, 0x206269, 0x2062e4, 0x2062fc, 0x206465, 0x206469, + 0x2064e4, 0x206569, 0x2066fc, 0x2067e4, 0x206be4, 0x206ce4, 0x207465, + 0x2074fc, 0x20756e, 0x20e470, 0x20f666, 0x20fc62, 0x616368, 0x616d20, + 0x617573, 0x626572, 0x62e463, 0x62fc63, 0x636865, 0x636874, 0x6368fc, + 0x64656e, 0x646572, 0x646965, 0x64e463, 0x652073, 0x652074, 0x65696e, + 0x656dfc, 0x656e20, 0x656efc, 0x657220, 0x65726e, 0x66fc68, 0x66fc72, + 0x67e473, 0x68656e, 0x687265, 0x696368, 0x696520, 0x696e20, 0x696e65, + 0x6c6569, 0x6e2062, 0x6e2064, 0x6e20fc, 0x6e6420, 0x6e656e, 0x736368, + 0x736520, 0x737461, 0x742065, 0x746520, 0x756e64, 0xe46e64, 0xfc6265, + 0xfc7220, + ], + highBytes: { + total: 21, + counts: [ + [0xc4, 1], + [0xdc, 1], + [0xe4, 7], + [0xf6, 1], + [0xfc, 11], + ], + }, + }, + { + language: 'en', + ngrams: [ + 0x206120, 0x20616e, 0x206272, 0x206369, 0x20636f, 0x206465, 0x206661, + 0x206c69, 0x206e65, 0x2070e2, 0x207265, 0x2072e9, 0x2072f4, 0x207468, + 0x207768, 0x616420, 0x616465, 0x6166e9, 0x616e64, 0x61e761, 0x61ef76, + 0x626f75, 0x627261, 0x636974, 0x642063, 0x64206e, 0x646520, 0x646573, + 0x652061, 0x652063, 0x656164, 0x657273, 0x657320, 0x657369, 0x657420, + 0x65f16f, 0x6661e7, 0x66e973, 0x686520, 0x68696c, 0x696272, 0x696465, + 0x696c65, 0x6c6520, 0x6de920, 0x6e61ef, 0x6e6420, 0x7065f1, 0x70e274, + 0x726561, 0x727320, 0x72e973, 0x72f46c, 0x732061, 0x746865, 0x74e920, + 0x756de9, 0xe274e9, 0xe76164, 0xe92020, 0xe92077, 0xe97320, 0xe97375, + 0xef7665, + ], + highBytes: { + total: 9, + counts: [ + [0xe2, 1], + [0xe7, 1], + [0xe9, 4], + [0xef, 1], + [0xf1, 1], + [0xf4, 1], + ], + }, + }, + { + language: 'es', + ngrams: [ + 0x206269, 0x206369, 0x20636c, 0x20636f, 0x2063f3, 0x206465, 0x206469, + 0x20656c, 0x206c61, 0x206c6f, 0x206d65, 0x206de1, 0x207061, 0x2070e1, + 0x2070fa, 0x207265, 0x207375, 0x2073e1, 0x207665, 0x207920, 0x612062, + 0x612065, 0x612070, 0x61646f, 0x616e20, 0x617261, 0x617320, 0x6174e1, + 0x61f161, 0x626c69, 0x6369e9, 0x6369f3, 0x63f364, 0x656c20, 0x656e20, + 0x656efa, 0x6572ed, 0x657320, 0x65f161, 0x65f16f, 0x69e96e, 0x69f36e, + 0x6c206d, 0x6c6120, 0x6c6f73, 0x6d61f1, 0x6de173, 0x6e206c, 0x6e6120, + 0x6efa20, 0x6f206c, 0x6f7265, 0x6f7320, 0x706172, 0x70e167, 0x70fa62, + 0x726573, 0x72ed61, 0x732061, 0x732063, 0x73206c, 0x7365f1, 0x73e162, + 0x746f72, + ], + highBytes: { + total: 13, + counts: [ + [0xe1, 4], + [0xe9, 1], + [0xed, 1], + [0xf1, 3], + [0xf3, 2], + [0xfa, 2], + ], + }, + }, + { + language: 'fr', + ngrams: [ + 0x20636f, 0x206465, 0x206475, 0x2064e9, 0x206574, 0x206c65, 0x207061, + 0x20706f, 0x207072, 0x2070e2, 0x2072e9, 0x20756e, 0x2076e9, 0x20e963, + 0x20e96c, 0x20e971, 0x6166e9, 0x626c69, 0x6368e9, 0x646573, 0x647520, + 0x64e976, 0x652063, 0x652064, 0x65206c, 0x65206d, 0x652070, 0x6520e9, + 0x656e63, 0x656e74, 0x657320, 0x657420, 0x657572, 0x66e973, 0x68e871, + 0x68e920, 0x6c6120, 0x6c6520, 0x6c6573, 0x6c6c65, 0x6de972, 0x6e7420, + 0x6f7572, 0x706172, 0x7072e9, 0x717565, 0x72206c, 0x726520, 0x727320, + 0x72e970, 0x732063, 0x73206c, 0x732070, 0x742064, 0x74206c, 0x7468e8, + 0x756520, 0x756de9, 0x757273, 0xe87175, 0xe96520, 0xe97061, 0xe97269, + 0xe97665, + ], + highBytes: { + total: 22, + counts: [ + [0xe2, 1], + [0xe8, 3], + [0xe9, 18], + ], + }, + }, + { + language: 'it', + ngrams: [ + 0x00006c, 0x00006d, 0x006c61, 0x206269, 0x206361, 0x206369, 0x20636f, + 0x206465, 0x206469, 0x206520, 0x206765, 0x20676c, 0x206920, 0x206c61, + 0x206c65, 0x206d65, 0x207065, 0x207069, 0x207072, 0x207363, 0x207369, + 0x207374, 0x612062, 0x612063, 0x612064, 0x612067, 0x612069, 0x612070, + 0x61746f, 0x6368e9, 0x636974, 0x64656c, 0x656e74, 0x657263, 0x6666e8, + 0x66e820, 0x676c69, 0x68e920, 0x692061, 0x692063, 0x692073, 0x696120, + 0x697474, 0x69f920, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6f20, + 0x6f7269, 0x7069f9, 0x726920, 0x746f72, 0x7474e0, 0x74e020, 0xe02020, + 0xe0206c, 0xe02073, 0xe82061, 0xe8206d, 0xe92073, 0xf92067, 0xf9206c, + 0xf92070, + ], + highBytes: { + total: 9, + counts: [ + [0xe0, 3], + [0xe8, 2], + [0xe9, 1], + [0xf9, 3], + ], + }, + }, + { + language: 'nl', + ngrams: [ + 0x000064, 0x206269, 0x20626f, 0x206361, 0x20636f, 0x206465, 0x206565, + 0x20656e, 0x206765, 0x206865, 0x206c65, 0x206f70, 0x206f76, 0x207374, + 0x207465, 0x207665, 0x207765, 0x20e9e9, 0x61616b, 0x616173, 0x616420, + 0x6166e9, 0x616b74, 0x616d20, 0x626962, 0x626c69, 0x626f65, 0x636166, + 0x636874, 0x646520, 0x652062, 0x652063, 0x652065, 0x652073, 0x65656e, + 0x656572, 0x6565eb, 0x65696e, 0x656b65, 0x656e20, 0x657273, 0x65eb6e, + 0x65ef6e, 0x66e973, 0x6765ef, 0x696e20, 0x6b7420, 0x6e2020, 0x6e206f, + 0x6e20e9, 0x6feb7a, 0x706feb, 0x732065, 0x736368, 0x742064, 0x74656e, + 0x746572, 0x766572, 0xe96e20, 0xe97320, 0xe9e96e, 0xeb6e20, 0xeb7a69, + 0xef6e74, + ], + highBytes: { + total: 7, + counts: [ + [0xe9, 4], + [0xeb, 2], + [0xef, 1], + ], + }, + }, + { + language: 'no', + ngrams: [ + 0x000064, 0x006cf8, 0x206269, 0x206272, 0x206279, 0x2062f8, 0x206465, + 0x2064f8, 0x20666f, 0x2066f8, 0x206c61, 0x206ee6, 0x206f67, 0x2070e5, + 0x207665, 0x2076e5, 0x20e570, 0x20e672, 0x626962, 0x626c69, 0x6272f8, + 0x627965, 0x62f86b, 0x62f86e, 0x642065, 0x646520, 0x646572, 0x646574, + 0x64f872, 0x652062, 0x652066, 0x652074, 0x6520e5, 0x6520e6, 0x656420, + 0x656e20, 0x656e65, 0x656e73, 0x657220, 0x657273, 0x657420, 0x66f872, + 0x672073, 0x676520, 0x67656e, 0x676572, 0x69626c, 0x696765, 0x696e67, + 0x696f74, 0x6af865, 0x6cf872, 0x6e6520, 0x6e6572, 0x6ee672, 0x6f6720, + 0x70e520, 0x72206f, 0x7273f8, 0x72f864, 0x737465, 0x74656b, 0xe52064, + 0xf86b65, + ], + highBytes: { + total: 14, + counts: [ + [0xe5, 4], + [0xe6, 2], + [0xf8, 8], + ], + }, + }, + { + language: 'pt', + ngrams: [ + 0x00006d, 0x206120, 0x206269, 0x206369, 0x2063f3, 0x206465, 0x206520, + 0x206573, 0x206e61, 0x206f20, 0x206f73, 0x207065, 0x207072, 0x2070e3, + 0x2073e1, 0x207665, 0x20f46e, 0x612062, 0x612063, 0x612064, 0x612065, + 0x61206f, 0x612070, 0x616465, 0x61646f, 0x6166e9, 0x616d20, 0x617261, + 0x617320, 0x61e761, 0x61e7e3, 0x626962, 0x626c69, 0x636120, 0x6369ea, + 0x63e172, 0x63f364, 0x646120, 0x646520, 0x646f20, 0x652061, 0x6563e1, + 0x657320, 0x657363, 0x66e973, 0x68e320, 0x696120, 0x69e7f5, 0x69ea6e, + 0x6d61e7, 0x6e6120, 0x6e68e3, 0x6f7320, 0x70e36f, 0x7261e7, 0x726573, + 0x7269e7, 0x732020, 0x732061, 0x732070, 0x7320f4, 0x7374f3, 0x73e162, + 0x74f372, + ], + highBytes: { + total: 14, + counts: [ + [0xe1, 2], + [0xe3, 3], + [0xe7, 3], + [0xe9, 1], + [0xea, 1], + [0xf3, 2], + [0xf4, 1], + [0xf5, 1], + ], + }, + }, + { + language: 'sv', + ngrams: [ + 0x000064, 0x006cf6, 0x206269, 0x2062f6, 0x206465, 0x2066e4, 0x2066f6, + 0x206c69, 0x206ce4, 0x206ce5, 0x206de4, 0x206f63, 0x2070e5, 0x20736b, + 0x207374, 0x2073e4, 0x2073f6, 0x207479, 0x207665, 0x2076e4, 0x20e470, + 0x20f670, 0x612020, 0x612062, 0x612074, 0x6120f6, 0x616420, 0x616465, + 0x6166e9, 0x617220, 0x617265, 0x626962, 0x626c69, 0x6272f6, 0x62f663, + 0x62f66e, 0x636820, 0x6420e4, 0x646520, 0x64656e, 0x646574, 0x656e20, + 0x657220, 0x65726e, 0x6573f6, 0x657420, 0x66f672, 0x676120, 0x69626c, + 0x696761, 0x696f74, 0x6b6170, 0x6b6172, 0x6e6120, 0x6f6368, 0x70e520, + 0x726e61, 0x736b61, 0x737461, 0x73f66b, 0x74206c, 0x746164, 0x766572, + 0xe52064, + ], + highBytes: { + total: 20, + counts: [ + [0xe4, 7], + [0xe5, 3], + [0xe9, 1], + [0xf6, 9], + ], + }, + }, + ], + }, + { + encoding: 'windows-1253', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xdc, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0xdd, 0xde, 0xdf, 0x20, 0xfc, 0x20, 0xfd, 0xfe, + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'el', + ngrams: [ + 0x0000de, 0x0000f0, 0x0000f3, 0x0000f8, 0x00def3, 0x00f0f1, 0x00f3e1, + 0x00f8e7, 0x20dced, 0x20e1e3, 0x20e1ed, 0x20e1f1, 0x20e2e9, 0x20e2ef, + 0x20e3f1, 0x20e5eb, 0x20e5f0, 0x20e5f1, 0x20e720, 0x20eae1, 0x20ece9, + 0x20efe9, 0x20f0fc, 0x20f3f4, 0x20f4e1, 0x20f4ef, 0xdc20f4, 0xdfe120, + 0xe12020, 0xe120ea, 0xe1e920, 0xe2e9e2, 0xe2ebe9, 0xe4e9ea, 0xe5dfe1, + 0xe5e920, 0xe5f220, 0xe720e2, 0xe7ed20, 0xe920e1, 0xe920ef, 0xe9e2eb, + 0xe9efe8, 0xe9f3f4, 0xeae1e9, 0xeae720, 0xebe720, 0xebe9ef, 0xed20ea, + 0xed20f0, 0xedeff5, 0xefe920, 0xeff5ed, 0xf0fceb, 0xf1dc20, 0xf1e9ef, + 0xf220ea, 0xf3f4de, 0xf3f4e7, 0xf4e120, 0xf4e5f2, 0xf4e7ed, 0xf5ed20, + 0xfcebe7, + ], + highBytes: { + total: 327, + counts: [ + [0xb9, 1], + [0xc7, 1], + [0xcc, 1], + [0xcf, 2], + [0xd0, 1], + [0xd3, 1], + [0xd8, 1], + [0xdc, 13], + [0xdd, 5], + [0xde, 4], + [0xdf, 9], + [0xe1, 31], + [0xe2, 9], + [0xe3, 9], + [0xe4, 4], + [0xe5, 16], + [0xe6, 1], + [0xe7, 14], + [0xe8, 6], + [0xe9, 31], + [0xea, 16], + [0xeb, 10], + [0xec, 11], + [0xed, 15], + [0xee, 1], + [0xef, 24], + [0xf0, 8], + [0xf1, 16], + [0xf2, 7], + [0xf3, 8], + [0xf4, 17], + [0xf5, 10], + [0xf6, 5], + [0xf7, 3], + [0xf8, 2], + [0xf9, 5], + [0xfc, 5], + [0xfd, 2], + [0xfe, 2], + ], + }, + }, + ], + }, + { + encoding: 'windows-1254', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x9a, 0x20, 0x9c, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, + ], + languages: [ + { + language: 'tr', + ngrams: [ + 0x000063, 0x000064, 0x000073, 0x0000fe, 0x006375, 0x00fe65, 0x2061e7, + 0x206269, 0x20656b, 0x206b61, 0x206bfc, 0x206dfc, 0x207361, 0x207461, + 0x207665, 0x207961, 0x20e769, 0x20f6f0, 0x20fe65, 0x616261, 0x616c20, + 0x616c61, 0x616cfd, 0x616e65, 0x616efd, 0x617220, 0x6172fd, 0x6174f6, + 0x617a61, 0x617afd, 0x61e7fd, 0x61fee7, 0x62616c, 0x652062, 0x656869, + 0x657220, 0x657269, 0x68616e, 0x686972, 0x696c65, 0x697220, 0x697461, + 0x6bfc74, 0x6c6172, 0x6c6572, 0x706861, 0x722020, 0x72206b, 0x722076, + 0x722079, 0x726920, 0x726c61, 0x72fd20, 0x746172, 0x74fc70, 0x766520, + 0x7a6172, 0x7a6520, 0xe7fd6b, 0xfc7068, 0xfc74fc, 0xfd6b20, 0xfd7220, + 0xfe6568, + ], + highBytes: { + total: 32, + counts: [ + [0xc7, 1], + [0xde, 2], + [0xe7, 6], + [0xf0, 1], + [0xf6, 2], + [0xfc, 8], + [0xfd, 11], + [0xfe, 1], + ], + }, + }, + ], + }, + { + encoding: 'windows-1255', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xd4, 0xd5, 0xd6, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'he', + ngrams: [ + 0x0000e1, 0x0000e4, 0x0000f1, 0x0000f9, 0x00e1e5, 0x00e4f1, 0x00f1e3, + 0x00f9e5, 0x20e0fa, 0x20e1e5, 0x20e1eb, 0x20e1f2, 0x20e1f8, 0x20e1fa, + 0x20e3e1, 0x20e3e9, 0x20e4e9, 0x20e4f2, 0x20e4f7, 0x20e4f8, 0x20e4f9, + 0x20e5e0, 0x20e5e2, 0x20e5e4, 0x20e5eb, 0x20e5ee, 0x20e7f7, 0x20e8f8, + 0x20ece0, 0x20ece1, 0x20ece7, 0x20ecee, 0x20ecfa, 0x20eee3, 0x20eee5, + 0x20eee7, 0x20eeeb, 0xe0e9ed, 0xe4f1f4, 0xe5f8e9, 0xe5fa20, 0xe920e4, + 0xe9e0e5, 0xe9e420, 0xe9ed20, 0xe9f820, 0xe9fa20, 0xed2020, 0xed20ec, + 0xed20ee, 0xef20ee, 0xf0e420, 0xf1f4f8, 0xf2e9f8, 0xf4f8e9, 0xf7e5f8, + 0xf820e4, 0xf820ee, 0xf8e5fa, 0xf8e920, 0xf8e9ed, 0xf8fa20, 0xf9e9ed, + 0xfa20ec, + ], + highBytes: { + total: 249, + counts: [ + [0xe0, 9], + [0xe1, 12], + [0xe2, 2], + [0xe3, 8], + [0xe4, 16], + [0xe5, 30], + [0xe6, 2], + [0xe7, 6], + [0xe8, 7], + [0xe9, 33], + [0xea, 1], + [0xeb, 6], + [0xec, 9], + [0xed, 13], + [0xee, 11], + [0xef, 3], + [0xf0, 6], + [0xf1, 7], + [0xf2, 6], + [0xf4, 7], + [0xf6, 1], + [0xf7, 10], + [0xf8, 22], + [0xf9, 7], + [0xfa, 15], + ], + }, + }, + ], + }, + { + encoding: 'windows-1256', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x81, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x8a, 0x20, 0x9c, 0x8d, 0x8e, 0x8f, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x98, 0x20, 0x9a, 0x20, + 0x9c, 0x20, 0x20, 0x9f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0x20, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0x20, 0x20, 0x20, 0x20, 0xf4, 0x20, 0x20, 0x20, 0x20, 0xf9, 0x20, 0xfb, + 0xfc, 0x20, 0x20, 0xff, + ], + languages: [ + { + language: 'ar', + ngrams: [ + 0x0000c7, 0x0000d3, 0x0000d5, 0x0000e6, 0x00c7e1, 0x00d3e6, 0x00d5c8, + 0x00e6d1, 0x20c3c8, 0x20c3e3, 0x20c7ce, 0x20c7e1, 0x20c8e5, 0x20c8ed, + 0x20cad3, 0x20cfe1, 0x20d1de, 0x20d5db, 0x20dae4, 0x20ddd1, 0x20dded, + 0x20dfca, 0x20e1e1, 0x20e6c7, 0x20e6ca, 0x20e6d5, 0x20e6ed, 0xc7ca20, + 0xc7ccda, 0xc7d120, 0xc7e1c3, 0xc7e1da, 0xc7e1e3, 0xc7e1e6, 0xc820c7, + 0xc8c920, 0xc92020, 0xc920c7, 0xc920e6, 0xc920ed, 0xca20e6, 0xcac820, + 0xcac8c9, 0xcb20c7, 0xccda20, 0xcd20c7, 0xcfede4, 0xd120c7, 0xd1c7cc, + 0xd5ddc7, 0xda20c7, 0xdfcac8, 0xe1e3cf, 0xe1e3df, 0xe3cfed, 0xe3dfca, + 0xe420c7, 0xe4c7d3, 0xe4c920, 0xe6c7e1, 0xe6d5dd, 0xe6e420, 0xedc920, + 0xede4c9, + ], + highBytes: { + total: 324, + counts: [ + [0xc1, 2], + [0xc3, 4], + [0xc4, 1], + [0xc6, 1], + [0xc7, 56], + [0xc8, 17], + [0xc9, 11], + [0xca, 16], + [0xcb, 2], + [0xcc, 3], + [0xcd, 8], + [0xce, 4], + [0xcf, 9], + [0xd1, 19], + [0xd2, 3], + [0xd3, 7], + [0xd4, 2], + [0xd5, 5], + [0xd6, 2], + [0xd8, 1], + [0xd9, 2], + [0xda, 8], + [0xdb, 1], + [0xdd, 9], + [0xde, 7], + [0xdf, 5], + [0xe1, 36], + [0xe3, 18], + [0xe4, 12], + [0xe5, 5], + [0xe6, 24], + [0xed, 24], + ], + }, + }, + ], + }, + { + encoding: 'windows-1257', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x8e, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0xb8, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0xbf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0x20, + ], + languages: [ + { + language: 'et', + ngrams: [ + 0x206a61, 0x206b69, 0x206b6f, 0x206c69, 0x206d65, 0x206dfc, 0x207261, + 0x2074f6, 0x2076e4, 0x20e472, 0x20f570, 0x20f575, 0x6120f5, 0x61616d, + 0x616220, 0x616420, 0x616d61, 0x617475, 0x617661, 0x6220f5, 0x626120, + 0x642061, 0x64206b, 0x64206f, 0x647573, 0x652074, 0x656420, 0x657369, + 0x692020, 0x696420, 0x696b75, 0x696d65, 0x696e6e, 0x69726a, 0x697461, + 0x6a6120, 0x6b6972, 0x6b6f67, 0x6b6f6e, 0x6b6f6f, 0x6b7564, 0x6c6173, + 0x6d6174, 0x6dfcfc, 0x6e20e4, 0x6e6520, 0x70e465, 0x726161, 0x736920, + 0x746162, 0x74f6f6, 0x7570e4, 0x766164, 0x76e469, 0x76e46c, 0x76e472, + 0xe46576, 0xe4696b, 0xe46c6a, 0xe4726b, 0xe47273, 0xf57069, 0xf5756e, + 0xf67475, + ], + highBytes: { + total: 11, + counts: [ + [0xe4, 5], + [0xf5, 2], + [0xf6, 2], + [0xfc, 2], + ], + }, + }, + { + language: 'lv', + ngrams: [ + 0x0072ee, 0x206269, 0x206365, 0x206461, 0x206469, 0x2064e2, 0x206761, + 0x20697a, 0x206b6f, 0x206bef, 0x206c61, 0x206ce7, 0x206d61, 0x206d65, + 0x206de2, 0x207069, 0x2070e2, 0x207261, 0x207265, 0x20756e, 0x61206b, + 0x617320, 0x626962, 0x626c69, 0x6365ef, 0x646120, 0x656d20, 0x692070, + 0x692073, 0x69626c, 0x69656d, 0x696f74, 0x6974e2, 0x6a6920, 0x6b7374, + 0x6be272, 0x6c6173, 0x6c696f, 0x6c73e7, 0x6c7573, 0x6ce76e, 0x6e6920, + 0x6eee63, 0x6f74e7, 0x70696c, 0x70e272, 0x726920, 0x732020, 0x732061, + 0x73206c, 0x73206d, 0x732075, 0x73e774, 0x74e26a, 0x74e26c, 0x74e76b, + 0x756e20, 0x757320, 0x76e76c, 0x7a76e7, 0xe26a69, 0xe76e69, 0xe774e2, + 0xee6361, + ], + highBytes: { + total: 52, + counts: [ + [0xe2, 24], + [0xe7, 13], + [0xe8, 1], + [0xec, 1], + [0xee, 4], + [0xef, 3], + [0xf0, 2], + [0xf2, 2], + [0xfb, 1], + [0xfe, 1], + ], + }, + }, + { + language: 'lt', + ngrams: [ + 0x0000f0, 0x00f065, 0x206169, 0x206170, 0x206269, 0x206461, 0x206972, + 0x207072, 0x207265, 0x20736b, 0x2073fb, 0x20f069, 0x20f076, 0x20fb6b, + 0x20fe69, 0x20fe6d, 0x612070, 0x616920, 0x616974, 0x6169f0, 0x6172f0, + 0x6172fe, 0x617320, 0x617573, 0x61f06f, 0x61f079, 0x626962, 0x626c69, + 0x646120, 0x64e020, 0x656461, 0x656e69, 0x67f820, 0x692072, 0x696120, + 0x696169, 0x69626c, 0x696573, 0x696e69, 0x696eeb, 0x696f74, 0x697220, + 0x6a6169, 0x6b6169, 0x6b696e, 0x6c696f, 0x6d656e, 0x6e696e, 0x6eeb73, + 0x6f20fe, 0x6f64e0, 0x6f7320, 0x6f7465, 0x7261f0, 0x726961, 0x732020, + 0x732061, 0x736b61, 0x74656b, 0x757320, 0xe0206d, 0xeb6a61, 0xeb7320, + 0xf07669, + ], + highBytes: { + total: 40, + counts: [ + [0xd0, 1], + [0xdb, 1], + [0xe0, 5], + [0xe1, 1], + [0xe6, 1], + [0xeb, 8], + [0xf0, 11], + [0xf8, 4], + [0xfb, 2], + [0xfe, 6], + ], + }, + }, + ], + }, + { + encoding: 'windows-1258', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x83, + 0x20, 0x20, 0x20, 0x20, 0x88, 0x20, 0x20, 0x20, 0x9c, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x9c, 0x20, 0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0x20, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0x20, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0x20, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0x20, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0x20, 0xff, + ], + languages: [ + { + language: 'vi', + ngrams: [ + 0x206261, 0x2062e1, 0x206320, 0x206368, 0x2064e2, 0x206769, 0x20686f, + 0x206920, 0x206c69, 0x206d20, 0x206d61, 0x206df4, 0x206e20, 0x206e67, + 0x206e68, 0x207068, 0x207175, 0x2073e1, 0x207420, 0x207461, 0x207468, + 0x207469, 0x207472, 0x2074e0, 0x207520, 0x207669, 0x2076e0, 0x207920, + 0x20f0f4, 0x612069, 0x612076, 0x626120, 0x62e16e, 0x636820, 0x6368f5, + 0x64e220, 0x672064, 0x67206b, 0x672074, 0x6720f0, 0x682070, 0x686f20, + 0x68e06e, 0x68f420, 0x68f520, 0x68fd20, 0x692063, 0x69206e, 0x692071, + 0x69ea20, 0x6df420, 0x6e2062, 0x6e2074, 0x6e6720, 0x6e6820, 0x6f2063, + 0x6f206e, 0x7068f4, 0x7468e0, 0x7468fd, 0x76e020, 0xf42074, 0xf5206e, + 0xfdf520, + ], + highBytes: { + total: 116, + counts: [ + [0xcc, 7], + [0xd0, 1], + [0xd2, 13], + [0xde, 6], + [0xe0, 11], + [0xe1, 6], + [0xe2, 7], + [0xea, 8], + [0xec, 9], + [0xf0, 3], + [0xf2, 15], + [0xf3, 1], + [0xf4, 10], + [0xf5, 7], + [0xfa, 2], + [0xfd, 10], + ], + }, + }, + ], + }, + { + encoding: 'windows-874', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0x20, 0xd2, 0xd3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, + ], + languages: [ + { + language: 'th', + ngrams: [ + 0x0000b5, 0x0000cb, 0x0000e0, 0x00b5c5, 0x00cb20, 0x00e0aa, 0x20a120, + 0x20a1a4, 0x20a1a8, 0x20a1c9, 0x20a1e0, 0x20a420, 0x20a4b9, 0x20a7ca, + 0x20a7e3, 0x20a820, 0x20aa20, 0x20aad2, 0x20b3b1, 0x20b4b7, 0x20b4bb, + 0x20b4e0, 0x20b520, 0x20b5c3, 0x20b720, 0x20b7c2, 0x20b8c0, 0x20b920, + 0x20b9cb, 0x20b9e0, 0x20bac3, 0x20bad2, 0x20bc20, 0x20bcc5, 0x20be20, + 0x20c120, 0x20c1e0, 0x20c2b9, 0x20c2ba, 0x20c2c7, 0x20c320, 0x20c520, + 0x20c7c2, 0x20c8d2, 0x20ca20, 0x20cda7, 0x20d2b9, 0x20e1c5, 0xa12020, + 0xa420a1, 0xb5c320, 0xbbc3d0, 0xc120cd, 0xc8d2ca, 0xca20e1, 0xcab5c3, + 0xcad2c3, 0xcb20cd, 0xd0c720, 0xd2c320, 0xd2cab5, 0xe0c120, 0xe0c520, + 0xe1c5d0, + ], + highBytes: { + total: 301, + counts: [ + [0xa1, 8], + [0xa2, 4], + [0xa4, 7], + [0xa7, 10], + [0xa8, 4], + [0xaa, 5], + [0xb1, 1], + [0xb3, 2], + [0xb4, 8], + [0xb5, 7], + [0xb6, 1], + [0xb7, 8], + [0xb8, 2], + [0xb9, 18], + [0xba, 4], + [0xbb, 4], + [0xbc, 4], + [0xbe, 2], + [0xbf, 1], + [0xc0, 1], + [0xc1, 8], + [0xc2, 10], + [0xc3, 16], + [0xc5, 11], + [0xc7, 7], + [0xc8, 2], + [0xc9, 1], + [0xca, 10], + [0xcb, 5], + [0xcd, 12], + [0xd0, 6], + [0xd1, 12], + [0xd2, 18], + [0xd3, 4], + [0xd4, 8], + [0xd5, 8], + [0xd6, 2], + [0xd7, 6], + [0xd8, 1], + [0xd9, 5], + [0xe0, 12], + [0xe1, 6], + [0xe2, 1], + [0xe3, 2], + [0xe4, 1], + [0xe6, 1], + [0xe7, 1], + [0xe8, 8], + [0xe9, 11], + [0xec, 5], + ], + }, + }, + ], + }, + { + encoding: 'KOI8-R', + byteMap: [ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xa3, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, + 0xdc, 0xdd, 0xde, 0xdf, + ], + languages: [ + { + language: 'ru', + ngrams: [ + 0x0000d2, 0x00d2c1, 0x20c120, 0x20c2c9, 0x20c7cf, 0x20c920, 0x20cbcf, + 0x20cdc1, 0x20cec1, 0x20cfc2, 0x20cfd3, 0x20cfd4, 0x20d0cc, 0x20d0cf, + 0x20d0d2, 0x20d1d2, 0x20d2c5, 0x20d3cf, 0x20d720, 0x20d7cf, 0x20d7d9, + 0x20dec9, 0xc0d420, 0xc120ce, 0xc120d0, 0xc120d7, 0xc1c0d4, 0xc1c5d4, + 0xc1cec9, 0xc1d120, 0xc1d2cb, 0xc1d3d4, 0xc1d4c5, 0xc2c9c2, 0xc2ccc9, + 0xc2d3d5, 0xc4c1c0, 0xc4c920, 0xc5ccc5, 0xc5ccc9, 0xc5cece, 0xc5d2c9, + 0xc5d2d9, 0xc5d3d4, 0xc7cfd2, 0xc7cfd4, 0xc920d0, 0xcbc120, 0xccc920, + 0xcdc1d2, 0xcec120, 0xced9c5, 0xcfd3c5, 0xd0d2cf, 0xd120d0, 0xd1d420, + 0xd2cfc4, 0xd2d920, 0xd420d3, 0xd4c5cc, 0xd4c5d2, 0xd4cfd7, 0xd4d920, + 0xd9c520, + ], + highBytes: { + total: 678, + counts: [ + [0xa3, 2], + [0xc0, 13], + [0xc1, 63], + [0xc2, 16], + [0xc3, 3], + [0xc4, 21], + [0xc5, 50], + [0xc7, 7], + [0xc8, 7], + [0xc9, 47], + [0xca, 7], + [0xcb, 21], + [0xcc, 19], + [0xcd, 16], + [0xce, 37], + [0xcf, 60], + [0xd0, 23], + [0xd1, 27], + [0xd2, 41], + [0xd3, 34], + [0xd4, 55], + [0xd5, 14], + [0xd6, 5], + [0xd7, 26], + [0xd8, 3], + [0xd9, 26], + [0xda, 7], + [0xdb, 6], + [0xdc, 1], + [0xdd, 2], + [0xde, 4], + [0xe4, 1], + [0xe6, 1], + [0xe7, 1], + [0xec, 1], + [0xee, 3], + [0xef, 1], + [0xf0, 1], + [0xf2, 2], + [0xf3, 1], + [0xf5, 1], + [0xf7, 1], + [0xfe, 1], + ], + }, + }, + ], + }, { encoding: 'KOI8-U', byteMap: [ From 92f5d105e0a71214085e1b859cab73e06c3b3dde Mon Sep 17 00:00:00 2001 From: Dmitry Shirokov Date: Sun, 12 Jul 2026 10:30:32 +1000 Subject: [PATCH 4/4] handle equivalent pairs which are byte-exact --- corpus/README.md | 5 +- .../ISO-8859-16/ro/test/community-garden.bin | 2 +- .../ISO-8859-16/ro/train/city-morning.bin | 2 +- .../ISO-8859-16/ro/train/library.bin | 2 +- .../generated/ISO-8859-16/ro/train/market.bin | 2 +- .../ISO-8859-16/ro/train/workshop.bin | 2 +- .../ISO-8859-16/ro/validation/river-trip.bin | 2 +- corpus/generated/index.json | 48 +- corpus/generated/ngrams.mjs | 38 +- corpus/manifest.json | 7 +- corpus/model-evaluation.json | 1612 +++++++++-------- scripts/corpus-utils.mjs | 3 + scripts/evaluate-models.mjs | 8 +- scripts/model-utils.mjs | 81 +- src/encoding/models/generated.ts | 25 +- 15 files changed, 1025 insertions(+), 814 deletions(-) diff --git a/corpus/README.md b/corpus/README.md index f2a6b52..b1cb4fb 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -39,6 +39,9 @@ learns Laplace-smoothed high-byte distributions from the training split. The evaluator uses byte likelihood to rank candidates whose trigram hit rates are statistically competitive at a 95% margin. `models:evaluate` reports every held-out test document, while `models:verify` checks reproducibility and fails if -any encoding or language result is incorrect. CP949 is excluded because its +any encoding or language result is incorrect. A different encoding label is +accepted only when both encodings decode the actual held-out bytes identically; +the report distinguishes these byte-equivalent results from exact matches. +CP949 is excluded because its multibyte recogniser requires character frequency data rather than byte trigrams. diff --git a/corpus/generated/ISO-8859-16/ro/test/community-garden.bin b/corpus/generated/ISO-8859-16/ro/test/community-garden.bin index 54f8405..c763657 100644 --- a/corpus/generated/ISO-8859-16/ro/test/community-garden.bin +++ b/corpus/generated/ISO-8859-16/ro/test/community-garden.bin @@ -1,3 +1,3 @@ Grãdina comunitãþii -Vecinii au transformat o curte goalã într-o grãdinã cu flori, legume ºi pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se întâlnesc sãptãmânal pentru a uda plantele. +Vecinii au transformat o curte goalã într-o grãdinã cu flori, legume ºi pomi tineri. Copiii au pictat indicatoare colorate, iar familiile se întâlnesc sãptãmânal pentru a uda plantele. Preþul este 10 ¤. diff --git a/corpus/generated/ISO-8859-16/ro/train/city-morning.bin b/corpus/generated/ISO-8859-16/ro/train/city-morning.bin index 08d7469..2986bf8 100644 --- a/corpus/generated/ISO-8859-16/ro/train/city-morning.bin +++ b/corpus/generated/ISO-8859-16/ro/train/city-morning.bin @@ -1,3 +1,3 @@ Dimineaþa în oraº -Oraºul se trezeºte încet. Cafenelele deschid uºile, autobuzele pornesc pe traseu, iar oamenii citesc ºtirile. Cerul devine mai luminos deasupra clãdirilor. +Oraºul se trezeºte încet. Cafenelele deschid uºile, autobuzele pornesc pe traseu, iar oamenii citesc ºtirile. Cerul devine mai luminos deasupra clãdirilor. Preþul este 10 ¤. diff --git a/corpus/generated/ISO-8859-16/ro/train/library.bin b/corpus/generated/ISO-8859-16/ro/train/library.bin index 1afc415..6517980 100644 --- a/corpus/generated/ISO-8859-16/ro/train/library.bin +++ b/corpus/generated/ISO-8859-16/ro/train/library.bin @@ -1,3 +1,3 @@ Biblioteca liniºtitã -În biblioteca de cartier se aud pagini întoarse încet. Cititorii cautã cãrþi despre istorie, ºtiinþã ºi cãlãtorii. Bibliotecara pregãteºte o expoziþie pentru elevi. +În biblioteca de cartier se aud pagini întoarse încet. Cititorii cautã cãrþi despre istorie, ºtiinþã ºi cãlãtorii. Bibliotecara pregãteºte o expoziþie pentru elevi. Preþul este 10 ¤. diff --git a/corpus/generated/ISO-8859-16/ro/train/market.bin b/corpus/generated/ISO-8859-16/ro/train/market.bin index 15a0338..7178006 100644 --- a/corpus/generated/ISO-8859-16/ro/train/market.bin +++ b/corpus/generated/ISO-8859-16/ro/train/market.bin @@ -1,3 +1,3 @@ Piaþa de sâmbãtã -Piaþa umple centrul cu mirosuri ºi culori. Fermierii aduc mere, miere, brânzã ºi pâine proaspãtã. Cumpãrãtorii aleg legume ºi schimbã reþete de familie. +Piaþa umple centrul cu mirosuri ºi culori. Fermierii aduc mere, miere, brânzã ºi pâine proaspãtã. Cumpãrãtorii aleg legume ºi schimbã reþete de familie. Preþul este 10 ¤. diff --git a/corpus/generated/ISO-8859-16/ro/train/workshop.bin b/corpus/generated/ISO-8859-16/ro/train/workshop.bin index 132f010..9fdeac1 100644 --- a/corpus/generated/ISO-8859-16/ro/train/workshop.bin +++ b/corpus/generated/ISO-8859-16/ro/train/workshop.bin @@ -1,3 +1,3 @@ Atelier digital -O echipã micã realizeazã ghidul digital al muzeului. Designerii organizeazã meniul, programatorii verificã sursa, iar redactorii scriu explicaþii clare. +O echipã micã realizeazã ghidul digital al muzeului. Designerii organizeazã meniul, programatorii verificã sursa, iar redactorii scriu explicaþii clare. Preþul este 10 ¤. diff --git a/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin b/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin index c125be0..60dfedf 100644 --- a/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin +++ b/corpus/generated/ISO-8859-16/ro/validation/river-trip.bin @@ -1,3 +1,3 @@ Plimbare lângã râu -Duminicã, prietenii au mers spre un râu liniºtit din afara oraºului. Au luat o hartã, ceai cald ºi un aparat foto. Dupã plimbare s-au odihnit pe mal. +Duminicã, prietenii au mers spre un râu liniºtit din afara oraºului. Au luat o hartã, ceai cald ºi un aparat foto. Dupã plimbare s-au odihnit pe mal. Preþul este 10 ¤. diff --git a/corpus/generated/index.json b/corpus/generated/index.json index 3467f44..136b1b1 100644 --- a/corpus/generated/index.json +++ b/corpus/generated/index.json @@ -1734,10 +1734,10 @@ "split": "test", "document": "community-garden", "path": "ISO-8859-16/ro/test/community-garden.bin", - "bytes": 206, - "highBytes": 13, - "uniqueHighBytes": 5, - "sha256": "e4d43eaeb3f3bce0f6df9d5686ab19d34d2a691464c0b3372e95e5c12fd577a8" + "bytes": 224, + "highBytes": 15, + "uniqueHighBytes": 6, + "sha256": "cbb06341ae7b36086fa1cc8e9ce8df8a66cf37ac24a4558200e71f037eaf6bc5" }, { "encoding": "ISO-8859-16", @@ -1746,10 +1746,10 @@ "split": "train", "document": "city-morning", "path": "ISO-8859-16/ro/train/city-morning.bin", - "bytes": 175, - "highBytes": 9, - "uniqueHighBytes": 4, - "sha256": "43a5a1c9acb7d2f14701b3b5572198acbce41ccfc9d90a00a95ca2489147830e" + "bytes": 193, + "highBytes": 11, + "uniqueHighBytes": 5, + "sha256": "47d8c1bf653e17a058fd3c8a3141eaa08652db1c8e27a46be5b177a8770b1006" }, { "encoding": "ISO-8859-16", @@ -1758,10 +1758,10 @@ "split": "train", "document": "library", "path": "ISO-8859-16/ro/train/library.bin", - "bytes": 187, - "highBytes": 17, - "uniqueHighBytes": 5, - "sha256": "817a8d7982dedb766196da67179b581a6940e424e6ae2861577aeeaaf10e114b" + "bytes": 205, + "highBytes": 19, + "uniqueHighBytes": 6, + "sha256": "fbd051c8abf29897208b0f3199e4504b8d40677a58edea5d541b37bfea98b459" }, { "encoding": "ISO-8859-16", @@ -1770,10 +1770,10 @@ "split": "train", "document": "market", "path": "ISO-8859-16/ro/train/market.bin", - "bytes": 171, - "highBytes": 17, - "uniqueHighBytes": 4, - "sha256": "ffff0df00228d68d1960c1f38b157084fce5c73e17e7a732a49d86251c834437" + "bytes": 189, + "highBytes": 19, + "uniqueHighBytes": 5, + "sha256": "8584a99429d049f70c73ea6c50208b23faef446f1c50b9833b7e31e2ef3adbc5" }, { "encoding": "ISO-8859-16", @@ -1782,10 +1782,10 @@ "split": "train", "document": "workshop", "path": "ISO-8859-16/ro/train/workshop.bin", - "bytes": 170, - "highBytes": 6, - "uniqueHighBytes": 2, - "sha256": "46e9670deecb580c1738a31a3aab804b6d54ffd3dcfee66eb0adf988e60d7c5f" + "bytes": 188, + "highBytes": 8, + "uniqueHighBytes": 3, + "sha256": "d3dbc5ecf59a9705bc6dd1d7ed97222304426d439ee87084d3729d221a00145c" }, { "encoding": "ISO-8859-16", @@ -1794,10 +1794,10 @@ "split": "validation", "document": "river-trip", "path": "ISO-8859-16/ro/validation/river-trip.bin", - "bytes": 170, - "highBytes": 10, - "uniqueHighBytes": 3, - "sha256": "a815d6b249d38b71948eac38e0de8ff5dc4b1be0e04b1c108d75a0d9bba37a99" + "bytes": 188, + "highBytes": 12, + "uniqueHighBytes": 5, + "sha256": "d130ff9bdb9431000be39f0d1f6cd5d7a0aa805e8bd4eca9723c71761b110020" }, { "encoding": "ISO-8859-2", diff --git a/corpus/generated/ngrams.mjs b/corpus/generated/ngrams.mjs index ab72c86..5b27b94 100644 --- a/corpus/generated/ngrams.mjs +++ b/corpus/generated/ngrams.mjs @@ -628,25 +628,25 @@ export default [ language: 'ro', documents: 4, trigrams: [ - 0x696920, 0x206465, 0x6f7269, 0x726969, 0x746f72, 0x692063, 0x20ba69, - 0x20ee6e, 0x2e2043, 0x652c20, 0x656c65, 0x696572, 0x756c20, 0xba6920, - 0x206d69, 0x207072, 0x207265, 0x61fe61, 0x626c69, 0x636869, 0x646520, - 0x652070, 0x656361, 0x657269, 0x657363, 0x692e20, 0x69626c, 0x696e65, - 0x696f74, 0x6c6520, 0x6c696f, 0x6f7465, 0x736520, 0x746520, 0x746563, - 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, 0x20616c, 0x206175, 0x206361, - 0x20636c, 0x206375, 0x2063e3, 0x206469, 0x206578, 0x206961, 0x206d65, - 0x206f72, 0x207065, 0x207363, 0x207365, 0x207472, 0x20ba74, 0x2c2069, - 0x426962, 0x506961, 0x612064, 0x616c20, 0x617220, 0x617574, 0x617ae3, - 0x636120, 0x636172, 0x636574, 0x63e320, 0x646573, 0x646967, 0x652063, - 0x652064, 0x652074, 0x6520ee, 0x652e0a, 0x65617a, 0x656e69, 0x656e74, - 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, 0x65ba74, 0x676974, - 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, 0x6963e3, 0x696769, - 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, 0x6c2064, 0x6c6567, - 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, 0x6d696e, 0x6e6365, - 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, 0x7261ba, 0x72652c, - 0x72696c, 0x72756c, 0x736320, 0x736368, 0x737572, 0x742e20, 0x74616c, - 0x746974, 0x747275, 0x74e30a, 0x752065, 0x756d70, 0x757a65, 0x7a6561, - 0xba7465, 0xe30a0a, + 0x696920, 0x756c20, 0x206465, 0x6f7269, 0x726969, 0x746520, 0x746f72, + 0x692063, 0x7265fe, 0x203130, 0x205072, 0x206573, 0x20a42e, 0x20ba69, + 0x20ee6e, 0x2e2043, 0x2e2050, 0x3020a4, 0x313020, 0x507265, 0x652031, + 0x652c20, 0x656c65, 0x657374, 0x65fe75, 0x692e20, 0x696572, 0x6c2065, + 0x737465, 0xa42e0a, 0xba6920, 0xfe756c, 0x206d69, 0x207072, 0x207265, + 0x61fe61, 0x626c69, 0x636869, 0x646520, 0x652070, 0x652e20, 0x656361, + 0x657269, 0x657363, 0x69626c, 0x696e65, 0x696f74, 0x6c6520, 0x6c696f, + 0x6f7465, 0x736520, 0x746563, 0x7ae320, 0xba7469, 0xfe6120, 0x0a0a4f, + 0x20616c, 0x206175, 0x206361, 0x20636c, 0x206375, 0x2063e3, 0x206469, + 0x206578, 0x206961, 0x206d65, 0x206f72, 0x207065, 0x207363, 0x207365, + 0x207472, 0x20ba74, 0x2c2069, 0x426962, 0x506961, 0x612064, 0x616c20, + 0x617220, 0x617574, 0x617ae3, 0x636120, 0x636172, 0x636574, 0x63e320, + 0x646573, 0x646967, 0x652063, 0x652064, 0x652074, 0x6520ee, 0x65617a, + 0x656e69, 0x656e74, 0x657220, 0x657265, 0x65742e, 0x657669, 0x657870, + 0x65ba74, 0x676974, 0x686964, 0x692061, 0x692073, 0x696172, 0x6961fe, + 0x6963e3, 0x696769, 0x696c65, 0x696e69, 0x697269, 0x697461, 0x697a65, + 0x6c2064, 0x6c6567, 0x6c6965, 0x6c6f72, 0x6d62e3, 0x6d656e, 0x6d6965, + 0x6d696e, 0x6e6365, 0x6e6520, 0x6e7472, 0x707265, 0x70726f, 0x726120, + 0x7261ba, 0x72652c, ], }, { diff --git a/corpus/manifest.json b/corpus/manifest.json index 56efa23..c747e1e 100644 --- a/corpus/manifest.json +++ b/corpus/manifest.json @@ -64,7 +64,12 @@ "iconv": "ISO-8859-15", "languages": ["fr", "de", "es"] }, - { "name": "ISO-8859-16", "iconv": "ISO-8859-16", "languages": ["ro"] }, + { + "name": "ISO-8859-16", + "iconv": "ISO-8859-16", + "languages": ["ro"], + "sourceSuffix": " PreÈ›ul este 10 €." + }, { "name": "CP850", "iconv": "CP850", "languages": ["fr", "de", "es"] }, { "name": "CP852", "iconv": "CP852", "languages": ["pl"] } ] diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 12b72dd..013725c 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -1,9 +1,11 @@ { "summary": { "tests": 66, - "encodingCorrect": 43, - "languageCorrect": 42, - "topScoreTies": 35 + "encodingExact": 44, + "encodingEquivalent": 22, + "encodingCorrect": 66, + "languageCorrect": 66, + "topScoreTies": 34 }, "confusion": { "CP850": { @@ -35,7 +37,7 @@ "ISO-8859-1": 3 }, "ISO-8859-16": { - "ISO-8859-2": 1 + "ISO-8859-16": 1 }, "ISO-8859-2": { "ISO-8859-2": 4 @@ -121,6 +123,8 @@ "total": 216, "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -137,7 +141,7 @@ { "encoding": "ISO-8859-1", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -146,7 +150,7 @@ { "encoding": "windows-1252", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -155,7 +159,7 @@ { "encoding": "ISO-8859-15", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -164,7 +168,7 @@ { "encoding": "macintosh", "confidence": 70, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -182,7 +186,7 @@ { "encoding": "ISO-8859-2", "confidence": 13, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 215, @@ -200,7 +204,7 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 215, @@ -215,19 +219,10 @@ "total": 215, "hitRate": 0.046511627906976744 }, - { - "encoding": "ISO-8859-16", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 215, - "hitRate": 0.046511627906976744 - }, { "encoding": "windows-1257", "confidence": 11, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 8, "total": 215, @@ -242,6 +237,15 @@ "total": 215, "hitRate": 0.037209302325581395 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 215, + "hitRate": 0.037209302325581395 + }, { "encoding": "windows-1258", "confidence": 6, @@ -429,6 +433,8 @@ "total": 200, "hitRate": 0.205 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -445,7 +451,7 @@ { "encoding": "ISO-8859-1", "confidence": 58, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -454,7 +460,7 @@ { "encoding": "windows-1252", "confidence": 58, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -481,7 +487,7 @@ { "encoding": "windows-1257", "confidence": 25, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 17, "total": 199, @@ -517,7 +523,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 199, @@ -526,7 +532,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 199, @@ -536,7 +542,7 @@ "encoding": "ISO-8859-16", "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -4.952116173006, "hits": 10, "total": 199, "hitRate": 0.05025125628140704 @@ -737,6 +743,8 @@ "total": 228, "hitRate": 0.25877192982456143 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -753,7 +761,7 @@ { "encoding": "ISO-8859-1", "confidence": 75, - "language": "nl", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -762,7 +770,7 @@ { "encoding": "windows-1252", "confidence": 75, - "language": "nl", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -771,7 +779,7 @@ { "encoding": "ISO-8859-15", "confidence": 75, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -780,7 +788,7 @@ { "encoding": "macintosh", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -789,7 +797,7 @@ { "encoding": "ISO-8859-2", "confidence": 12, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, @@ -798,7 +806,7 @@ { "encoding": "windows-1250", "confidence": 12, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, @@ -817,7 +825,7 @@ "encoding": "ISO-8859-16", "confidence": 12, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 9, "total": 224, "hitRate": 0.04017857142857143 @@ -825,7 +833,7 @@ { "encoding": "windows-1257", "confidence": 9, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 7, "total": 224, @@ -1045,6 +1053,8 @@ "total": 184, "hitRate": 0.1358695652173913 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -1079,7 +1089,7 @@ { "encoding": "ISO-8859-1", "confidence": 8, - "language": "sv", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -1088,7 +1098,7 @@ { "encoding": "windows-1252", "confidence": 8, - "language": "sv", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -1106,7 +1116,7 @@ { "encoding": "macintosh", "confidence": 6, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 4, "total": 182, @@ -1124,7 +1134,7 @@ { "encoding": "ISO-8859-15", "confidence": 6, - "language": "de", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 4, "total": 183, @@ -1160,7 +1170,7 @@ { "encoding": "CP850", "confidence": 4, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 184, @@ -1188,7 +1198,7 @@ "encoding": "ISO-8859-16", "confidence": 3, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 2, "total": 183, "hitRate": 0.01092896174863388 @@ -1353,6 +1363,8 @@ "total": 179, "hitRate": 0.1452513966480447 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -1423,7 +1435,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 4, - "language": "ru", + "language": "uk", "byteLogLikelihood": -5.827543836069, "hits": 2, "total": 137, @@ -1622,7 +1634,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.991915414162, + "byteLogLikelihood": -5.025462977445, "hits": 0, "total": 162, "hitRate": 0 @@ -1661,6 +1673,8 @@ "total": 166, "hitRate": 0.0963855421686747 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -1713,7 +1727,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 8, - "language": "ru", + "language": "uk", "byteLogLikelihood": -5.930887805719, "hits": 3, "total": 112, @@ -1930,7 +1944,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.064961155115, + "byteLogLikelihood": -5.086009867584, "hits": 0, "total": 142, "hitRate": 0 @@ -1969,6 +1983,8 @@ "total": 179, "hitRate": 0.13966480446927373 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -1994,7 +2010,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 15, - "language": "uk", + "language": "ru", "byteLogLikelihood": -5.792699163742, "hits": 7, "total": 133, @@ -2238,7 +2254,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.06091613544, + "byteLogLikelihood": -4.987878406509, "hits": 0, "total": 144, "hitRate": 0 @@ -2277,6 +2293,8 @@ "total": 120, "hitRate": 0.175 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -2302,7 +2320,7 @@ { "encoding": "macintosh", "confidence": 30, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 12, "total": 120, @@ -2311,7 +2329,7 @@ { "encoding": "ISO-8859-15", "confidence": 30, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 12, "total": 120, @@ -2320,7 +2338,7 @@ { "encoding": "CP850", "confidence": 30, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 12, "total": 120, @@ -2335,10 +2353,19 @@ "total": 120, "hitRate": 0.05 }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, { "encoding": "ISO-8859-2", "confidence": 10, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 4, "total": 120, @@ -2356,7 +2383,7 @@ { "encoding": "windows-1250", "confidence": 10, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 4, "total": 120, @@ -2374,21 +2401,12 @@ { "encoding": "windows-1257", "confidence": 10, - "language": "lt", + "language": "et", "byteLogLikelihood": -4.934473933131, "hits": 4, "total": 120, "hitRate": 0.03333333333333333 }, - { - "encoding": "ISO-8859-16", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, { "encoding": "windows-1258", "confidence": 7, @@ -2585,6 +2603,8 @@ "total": 216, "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 3, @@ -2619,7 +2639,7 @@ { "encoding": "macintosh", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -2628,7 +2648,7 @@ { "encoding": "CP850", "confidence": 70, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -2646,7 +2666,7 @@ { "encoding": "ISO-8859-2", "confidence": 13, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 216, @@ -2655,15 +2675,6 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "ISO-8859-16", - "confidence": 13, "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, @@ -2688,10 +2699,19 @@ "total": 216, "hitRate": 0.041666666666666664 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 216, + "hitRate": 0.037037037037037035 + }, { "encoding": "windows-1257", "confidence": 9, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 7, "total": 216, @@ -2887,14 +2907,16 @@ "predicted": { "encoding": "ISO-8859-15", "confidence": 29, - "language": "fr", + "language": "en", "byteLogLikelihood": -2.06619631493, "hits": 13, "total": 132, "hitRate": 0.09848484848484848 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { @@ -2909,7 +2931,7 @@ { "encoding": "ISO-8859-1", "confidence": 47, - "language": "fr", + "language": "en", "byteLogLikelihood": -3.310543013394, "hits": 21, "total": 132, @@ -2918,7 +2940,7 @@ { "encoding": "windows-1252", "confidence": 47, - "language": "fr", + "language": "en", "byteLogLikelihood": -3.310543013394, "hits": 21, "total": 132, @@ -2927,7 +2949,7 @@ { "encoding": "macintosh", "confidence": 27, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 12, "total": 132, @@ -2936,7 +2958,7 @@ { "encoding": "CP850", "confidence": 27, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 12, "total": 132, @@ -2963,7 +2985,7 @@ { "encoding": "ISO-8859-2", "confidence": 9, - "language": "hu", + "language": "cs", "byteLogLikelihood": -3.54961738678, "hits": 4, "total": 132, @@ -2972,7 +2994,7 @@ { "encoding": "windows-1250", "confidence": 9, - "language": "hu", + "language": "cs", "byteLogLikelihood": -3.54961738678, "hits": 4, "total": 132, @@ -2988,19 +3010,19 @@ "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-4", - "confidence": 6, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, + "encoding": "ISO-8859-16", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, "total": 132, - "hitRate": 0.022727272727272728 + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-4", "confidence": 6, - "language": "ro", - "byteLogLikelihood": -5.176149732574, + "language": "lv", + "byteLogLikelihood": -5.19295685089, "hits": 3, "total": 132, "hitRate": 0.022727272727272728 @@ -3201,6 +3223,8 @@ "total": 200, "hitRate": 0.205 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 3, @@ -3253,7 +3277,7 @@ { "encoding": "windows-1257", "confidence": 24, - "language": "lt", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 16, "total": 200, @@ -3289,7 +3313,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 200, @@ -3298,7 +3322,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 200, @@ -3308,7 +3332,7 @@ "encoding": "ISO-8859-16", "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 10, "total": 200, "hitRate": 0.05 @@ -3509,6 +3533,8 @@ "total": 228, "hitRate": 0.25877192982456143 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 3, @@ -3543,7 +3569,7 @@ { "encoding": "macintosh", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -3552,7 +3578,7 @@ { "encoding": "CP850", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -3561,7 +3587,7 @@ { "encoding": "ISO-8859-2", "confidence": 11, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 228, @@ -3570,15 +3596,6 @@ { "encoding": "windows-1250", "confidence": 11, - "language": "hu", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, @@ -3594,10 +3611,19 @@ "total": 228, "hitRate": 0.03508771929824561 }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, { "encoding": "windows-1257", "confidence": 7, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 6, "total": 228, @@ -3817,6 +3843,8 @@ "total": 159, "hitRate": 0.1949685534591195 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -3842,7 +3870,7 @@ { "encoding": "macintosh", "confidence": 24, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 158, @@ -3851,7 +3879,7 @@ { "encoding": "ISO-8859-15", "confidence": 24, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 159, @@ -3860,7 +3888,7 @@ { "encoding": "CP850", "confidence": 24, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 159, @@ -3875,10 +3903,19 @@ "total": 159, "hitRate": 0.05660377358490566 }, + { + "encoding": "ISO-8859-16", + "confidence": 16, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, { "encoding": "ISO-8859-2", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 8, "total": 159, @@ -3887,7 +3924,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 8, "total": 159, @@ -3896,7 +3933,7 @@ { "encoding": "windows-1257", "confidence": 15, - "language": "lt", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 8, "total": 159, @@ -3911,15 +3948,6 @@ "total": 159, "hitRate": 0.050314465408805034 }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, { "encoding": "ISO-8859-14", "confidence": 9, @@ -4119,20 +4147,22 @@ "predicted": { "encoding": "ISO-8859-1", "confidence": 83, - "language": "fr", + "language": "nl", "byteLogLikelihood": -3.295836866004, "hits": 42, "total": 151, "hitRate": 0.2781456953642384 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, - "languageCorrect": false, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { "encoding": "ISO-8859-1", "confidence": 83, - "language": "fr", + "language": "nl", "byteLogLikelihood": -3.295836866004, "hits": 42, "total": 151, @@ -4141,7 +4171,7 @@ { "encoding": "windows-1252", "confidence": 83, - "language": "fr", + "language": "nl", "byteLogLikelihood": -3.295836866004, "hits": 42, "total": 151, @@ -4150,7 +4180,7 @@ { "encoding": "macintosh", "confidence": 63, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 32, "total": 151, @@ -4159,7 +4189,7 @@ { "encoding": "ISO-8859-15", "confidence": 63, - "language": "fr", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 32, "total": 151, @@ -4168,7 +4198,7 @@ { "encoding": "CP850", "confidence": 63, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 32, "total": 151, @@ -4186,7 +4216,7 @@ { "encoding": "ISO-8859-2", "confidence": 13, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 7, "total": 151, @@ -4204,7 +4234,7 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 7, "total": 151, @@ -4219,15 +4249,6 @@ "total": 151, "hitRate": 0.046357615894039736 }, - { - "encoding": "ISO-8859-16", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, { "encoding": "ISO-8859-10", "confidence": 11, @@ -4237,6 +4258,15 @@ "total": 151, "hitRate": 0.039735099337748346 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, { "encoding": "windows-1257", "confidence": 9, @@ -4433,6 +4463,8 @@ "total": 114, "hitRate": 0.15789473684210525 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -4458,7 +4490,7 @@ { "encoding": "macintosh", "confidence": 18, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 7, "total": 114, @@ -4476,7 +4508,7 @@ { "encoding": "ISO-8859-15", "confidence": 18, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 7, "total": 114, @@ -4485,7 +4517,7 @@ { "encoding": "CP850", "confidence": 18, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 7, "total": 114, @@ -4509,6 +4541,15 @@ "total": 114, "hitRate": 0.03508771929824561 }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, { "encoding": "ISO-8859-2", "confidence": 7, @@ -4548,21 +4589,12 @@ { "encoding": "windows-1257", "confidence": 7, - "language": "lt", + "language": "et", "byteLogLikelihood": -4.934473933131, "hits": 3, "total": 114, "hitRate": 0.02631578947368421 }, - { - "encoding": "ISO-8859-16", - "confidence": 5, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 2, - "total": 114, - "hitRate": 0.017543859649122806 - }, { "encoding": "ISO-8859-3", "confidence": 2, @@ -4735,14 +4767,16 @@ "predicted": { "encoding": "ISO-8859-15", "confidence": 33, - "language": "es", + "language": "pt", "byteLogLikelihood": -3.970754139021, "hits": 16, "total": 143, "hitRate": 0.11188811188811189 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { @@ -4757,7 +4791,7 @@ { "encoding": "ISO-8859-1", "confidence": 50, - "language": "es", + "language": "pt", "byteLogLikelihood": -4.059947322987, "hits": 24, "total": 143, @@ -4766,7 +4800,7 @@ { "encoding": "windows-1252", "confidence": 50, - "language": "es", + "language": "pt", "byteLogLikelihood": -4.059947322987, "hits": 24, "total": 143, @@ -4775,7 +4809,7 @@ { "encoding": "windows-1257", "confidence": 29, - "language": "lv", + "language": "lt", "byteLogLikelihood": -4.777390389123, "hits": 14, "total": 143, @@ -4884,7 +4918,7 @@ "encoding": "ISO-8859-16", "confidence": 2, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 1, "total": 143, "hitRate": 0.006993006993006993 @@ -5049,6 +5083,8 @@ "total": 136, "hitRate": 0.16911764705882354 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -5092,7 +5128,7 @@ { "encoding": "macintosh", "confidence": 17, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 8, "total": 136, @@ -5110,7 +5146,7 @@ { "encoding": "CP850", "confidence": 17, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 8, "total": 136, @@ -5119,7 +5155,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "hu", + "language": "cs", "byteLogLikelihood": -5.159055299215, "hits": 7, "total": 136, @@ -5128,7 +5164,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "hu", + "language": "cs", "byteLogLikelihood": -5.159055299215, "hits": 7, "total": 136, @@ -5161,10 +5197,19 @@ "total": 136, "hitRate": 0.029411764705882353 }, + { + "encoding": "ISO-8859-16", + "confidence": 6, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 136, + "hitRate": 0.022058823529411766 + }, { "encoding": "windows-1257", "confidence": 4, - "language": "et", + "language": "lt", "byteLogLikelihood": -5.123963979403, "hits": 2, "total": 136, @@ -5188,15 +5233,6 @@ "total": 136, "hitRate": 0.014705882352941176 }, - { - "encoding": "ISO-8859-16", - "confidence": 4, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, { "encoding": "CP852", "confidence": 4, @@ -5357,6 +5393,8 @@ "total": 171, "hitRate": 0.1286549707602339 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -5373,7 +5411,7 @@ { "encoding": "ISO-8859-1", "confidence": 14, - "language": "es", + "language": "sv", "byteLogLikelihood": -4.633646206449, "hits": 8, "total": 171, @@ -5382,7 +5420,7 @@ { "encoding": "windows-1252", "confidence": 14, - "language": "es", + "language": "sv", "byteLogLikelihood": -4.633646206449, "hits": 8, "total": 171, @@ -5427,7 +5465,7 @@ { "encoding": "macintosh", "confidence": 5, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 3, "total": 169, @@ -5445,7 +5483,7 @@ { "encoding": "ISO-8859-15", "confidence": 5, - "language": "es", + "language": "de", "byteLogLikelihood": -4.894502014278, "hits": 3, "total": 171, @@ -5454,7 +5492,7 @@ { "encoding": "CP850", "confidence": 5, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 3, "total": 169, @@ -5472,7 +5510,7 @@ { "encoding": "ISO-8859-2", "confidence": 3, - "language": "hu", + "language": "cs", "byteLogLikelihood": -4.497848163857, "hits": 2, "total": 171, @@ -5481,7 +5519,7 @@ { "encoding": "windows-1250", "confidence": 3, - "language": "hu", + "language": "cs", "byteLogLikelihood": -4.497848163857, "hits": 2, "total": 171, @@ -5505,15 +5543,6 @@ "total": 171, "hitRate": 0.005847953216374269 }, - { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 1, - "total": 171, - "hitRate": 0.005847953216374269 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -5648,6 +5677,15 @@ "hits": 0, "total": 171, "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 0, + "total": 171, + "hitRate": 0 } ] }, @@ -5665,8 +5703,10 @@ "total": 173, "hitRate": 0.11560693641618497 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -5699,7 +5739,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "ro", + "language": "hu", "byteLogLikelihood": -5.117993812417, "hits": 9, "total": 173, @@ -5708,7 +5748,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "ro", + "language": "hu", "byteLogLikelihood": -5.117993812417, "hits": 9, "total": 173, @@ -5717,7 +5757,7 @@ { "encoding": "ISO-8859-1", "confidence": 12, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, @@ -5726,7 +5766,7 @@ { "encoding": "windows-1252", "confidence": 12, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, @@ -5744,7 +5784,7 @@ { "encoding": "ISO-8859-15", "confidence": 12, - "language": "fr", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, @@ -5816,7 +5856,7 @@ { "encoding": "IBM855", "confidence": 1, - "language": "sr", + "language": "ru", "byteLogLikelihood": -5.488073838002, "hits": 1, "total": 170, @@ -5831,15 +5871,6 @@ "total": 173, "hitRate": 0.005780346820809248 }, - { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.659155714377, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -5956,6 +5987,15 @@ "hits": 0, "total": 173, "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.616838329205, + "hits": 0, + "total": 173, + "hitRate": 0 } ] }, @@ -5973,6 +6013,8 @@ "total": 189, "hitRate": 0.1693121693121693 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -5989,7 +6031,7 @@ { "encoding": "ISO-8859-1", "confidence": 14, - "language": "en", + "language": "pt", "byteLogLikelihood": -4.955827057601, "hits": 9, "total": 189, @@ -5998,7 +6040,7 @@ { "encoding": "windows-1252", "confidence": 14, - "language": "en", + "language": "pt", "byteLogLikelihood": -4.955827057601, "hits": 9, "total": 189, @@ -6025,7 +6067,7 @@ { "encoding": "ISO-8859-15", "confidence": 11, - "language": "fr", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 189, @@ -6043,7 +6085,7 @@ { "encoding": "ISO-8859-2", "confidence": 9, - "language": "ro", + "language": "cs", "byteLogLikelihood": -5.159055299215, "hits": 6, "total": 189, @@ -6052,7 +6094,7 @@ { "encoding": "windows-1250", "confidence": 9, - "language": "ro", + "language": "cs", "byteLogLikelihood": -5.159055299215, "hits": 6, "total": 189, @@ -6061,7 +6103,7 @@ { "encoding": "windows-1257", "confidence": 6, - "language": "lv", + "language": "et", "byteLogLikelihood": -4.934473933131, "hits": 4, "total": 189, @@ -6260,7 +6302,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -3.981643419755, + "byteLogLikelihood": -3.903274585551, "hits": 0, "total": 189, "hitRate": 0 @@ -6281,8 +6323,10 @@ "total": 216, "hitRate": 0.24537037037037038 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 3, "candidates": [ { @@ -6315,7 +6359,7 @@ { "encoding": "macintosh", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -6324,7 +6368,7 @@ { "encoding": "CP850", "confidence": 70, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -6342,7 +6386,7 @@ { "encoding": "ISO-8859-2", "confidence": 13, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 216, @@ -6351,15 +6395,6 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "ISO-8859-16", - "confidence": 13, "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, @@ -6384,10 +6419,19 @@ "total": 216, "hitRate": 0.041666666666666664 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 216, + "hitRate": 0.037037037037037035 + }, { "encoding": "windows-1257", "confidence": 9, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 7, "total": 216, @@ -6589,8 +6633,10 @@ "total": 200, "hitRate": 0.205 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 3, "candidates": [ { @@ -6641,7 +6687,7 @@ { "encoding": "windows-1257", "confidence": 24, - "language": "lt", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 16, "total": 200, @@ -6677,7 +6723,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 200, @@ -6686,7 +6732,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 200, @@ -6696,7 +6742,7 @@ "encoding": "ISO-8859-16", "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 10, "total": 200, "hitRate": 0.05 @@ -6897,8 +6943,10 @@ "total": 228, "hitRate": 0.25877192982456143 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 3, "candidates": [ { @@ -6931,7 +6979,7 @@ { "encoding": "macintosh", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -6940,7 +6988,7 @@ { "encoding": "CP850", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -6949,7 +6997,7 @@ { "encoding": "ISO-8859-2", "confidence": 11, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 228, @@ -6958,15 +7006,6 @@ { "encoding": "windows-1250", "confidence": 11, - "language": "hu", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, @@ -6982,10 +7021,19 @@ "total": 228, "hitRate": 0.03508771929824561 }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, { "encoding": "windows-1257", "confidence": 7, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 6, "total": 228, @@ -7197,304 +7245,306 @@ "language": "ro" }, "predicted": { - "encoding": "ISO-8859-2", - "confidence": 41, + "encoding": "ISO-8859-16", + "confidence": 53, "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 }, - "encodingCorrect": false, - "languageCorrect": false, - "tiedAtTop": 3, + "encodingExact": true, + "encodingEquivalent": false, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 1, "candidates": [ { - "encoding": "ISO-8859-2", - "confidence": 41, + "encoding": "ISO-8859-16", + "confidence": 53, "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 + "byteLogLikelihood": -2.758152644716, + "hits": 38, + "total": 214, + "hitRate": 0.17757009345794392 }, { - "encoding": "windows-1250", - "confidence": 41, + "encoding": "ISO-8859-2", + "confidence": 44, "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 }, { - "encoding": "ISO-8859-16", - "confidence": 41, + "encoding": "windows-1250", + "confidence": 44, "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, - "total": 202, - "hitRate": 0.13861386138613863 + "byteLogLikelihood": -2.870272383724, + "hits": 32, + "total": 214, + "hitRate": 0.14953271028037382 }, { "encoding": "ISO-8859-1", "confidence": 26, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.919980925828, - "hits": 18, - "total": 202, - "hitRate": 0.0891089108910891 + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 }, { "encoding": "windows-1252", "confidence": 26, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.919980925828, - "hits": 18, - "total": 202, - "hitRate": 0.0891089108910891 + "hits": 19, + "total": 214, + "hitRate": 0.08878504672897196 }, { "encoding": "macintosh", - "confidence": 19, - "language": "es", + "confidence": 22, + "language": "fr", "byteLogLikelihood": -5.010635294096, - "hits": 13, - "total": 198, - "hitRate": 0.06565656565656566 + "hits": 16, + "total": 210, + "hitRate": 0.0761904761904762 }, { "encoding": "CP850", - "confidence": 15, - "language": "es", + "confidence": 18, + "language": "fr", "byteLogLikelihood": -5.010635294096, - "hits": 10, - "total": 199, - "hitRate": 0.05025125628140704 + "hits": 13, + "total": 213, + "hitRate": 0.06103286384976526 }, { "encoding": "ISO-8859-15", - "confidence": 14, + "confidence": 16, "language": "fr", - "byteLogLikelihood": -4.903997266318, - "hits": 10, - "total": 202, - "hitRate": 0.04950495049504951 + "byteLogLikelihood": -4.918215670022, + "hits": 12, + "total": 214, + "hitRate": 0.056074766355140186 }, { "encoding": "ISO-8859-3", - "confidence": 13, + "confidence": 12, "language": "mt", "byteLogLikelihood": -5.105945473901, "hits": 9, - "total": 200, - "hitRate": 0.045 + "total": 212, + "hitRate": 0.04245283018867924 }, { "encoding": "ISO-8859-9", - "confidence": 10, + "confidence": 9, "language": "tr", - "byteLogLikelihood": -5.021854801345, + "byteLogLikelihood": -4.982754191159, "hits": 7, - "total": 202, - "hitRate": 0.034653465346534656 + "total": 214, + "hitRate": 0.03271028037383177 }, { "encoding": "windows-1254", - "confidence": 10, + "confidence": 9, "language": "tr", - "byteLogLikelihood": -5.021854801345, + "byteLogLikelihood": -4.982754191159, "hits": 7, - "total": 202, - "hitRate": 0.034653465346534656 - }, - { - "encoding": "windows-1257", - "confidence": 8, - "language": "lv", - "byteLogLikelihood": -4.396820338954, - "hits": 6, - "total": 202, - "hitRate": 0.0297029702970297 + "total": 214, + "hitRate": 0.03271028037383177 }, { "encoding": "CP852", - "confidence": 8, + "confidence": 9, "language": "pl", "byteLogLikelihood": -5.087596335232, + "hits": 7, + "total": 215, + "hitRate": 0.03255813953488372 + }, + { + "encoding": "windows-1257", + "confidence": 8, + "language": "lv", + "byteLogLikelihood": -4.456762061842, "hits": 6, - "total": 201, - "hitRate": 0.029850746268656716 + "total": 214, + "hitRate": 0.028037383177570093 }, { "encoding": "ISO-8859-4", "confidence": 7, "language": "lv", - "byteLogLikelihood": -4.936633427031, + "byteLogLikelihood": -4.924600071508, "hits": 5, - "total": 202, - "hitRate": 0.024752475247524754 + "total": 214, + "hitRate": 0.02336448598130841 }, { "encoding": "windows-1258", "confidence": 5, "language": "vi", - "byteLogLikelihood": -5.177254141958, + "byteLogLikelihood": -5.219909353069, "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 + "total": 214, + "hitRate": 0.018691588785046728 }, { "encoding": "ISO-8859-10", "confidence": 5, "language": "is", - "byteLogLikelihood": -5.150687673188, + "byteLogLikelihood": -5.111587063002, "hits": 4, - "total": 202, - "hitRate": 0.019801980198019802 + "total": 216, + "hitRate": 0.018518518518518517 }, { "encoding": "ISO-8859-13", - "confidence": 4, + "confidence": 5, "language": "lt", - "byteLogLikelihood": -4.974278583322, - "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "byteLogLikelihood": -4.864509292863, + "hits": 4, + "total": 214, + "hitRate": 0.018691588785046728 }, { "encoding": "ISO-8859-14", "confidence": 4, "language": "cy", - "byteLogLikelihood": -4.539421788511, + "byteLogLikelihood": -4.542976389437, "hits": 3, - "total": 202, - "hitRate": 0.01485148514851485 + "total": 216, + "hitRate": 0.013888888888888888 + }, + { + "encoding": "windows-874", + "confidence": 1, + "language": "th", + "byteLogLikelihood": -5.210426263201, + "hits": 1, + "total": 214, + "hitRate": 0.004672897196261682 }, { "encoding": "ISO-8859-5", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.208609169552, + "byteLogLikelihood": -4.539739112612, "hits": 0, - "total": 202, + "total": 216, "hitRate": 0 }, { "encoding": "ISO-8859-6", "confidence": 0, "language": "ar", - "byteLogLikelihood": -4.828974536143, + "byteLogLikelihood": -5.000268888635, "hits": 0, - "total": 199, + "total": 211, "hitRate": 0 }, { "encoding": "ISO-8859-7", "confidence": 0, "language": "el", - "byteLogLikelihood": -4.33505338151, + "byteLogLikelihood": -4.499845100591, "hits": 0, - "total": 202, + "total": 214, "hitRate": 0 }, { "encoding": "ISO-8859-8", "confidence": 0, "language": "he", - "byteLogLikelihood": -4.197813655274, + "byteLogLikelihood": -4.429071192897, "hits": 0, - "total": 201, + "total": 213, "hitRate": 0 }, { "encoding": "windows-1251", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.229886082374, + "byteLogLikelihood": -4.382241948417, "hits": 0, - "total": 202, + "total": 214, "hitRate": 0 }, { "encoding": "windows-1253", "confidence": 0, "language": "el", - "byteLogLikelihood": -4.33505338151, + "byteLogLikelihood": -4.499845100591, "hits": 0, - "total": 202, + "total": 214, "hitRate": 0 }, { "encoding": "windows-1255", "confidence": 0, "language": "he", - "byteLogLikelihood": -4.197813655274, + "byteLogLikelihood": -4.429071192897, "hits": 0, - "total": 201, + "total": 213, "hitRate": 0 }, { "encoding": "windows-1256", "confidence": 0, "language": "ar", - "byteLogLikelihood": -4.528215037204, + "byteLogLikelihood": -4.739610656221, "hits": 0, - "total": 201, - "hitRate": 0 - }, - { - "encoding": "windows-874", - "confidence": 0, - "language": "th", - "byteLogLikelihood": -5.239455511679, - "hits": 0, - "total": 200, + "total": 213, "hitRate": 0 }, { "encoding": "KOI8-R", "confidence": 0, "language": "ru", - "byteLogLikelihood": -6.42548867306, + "byteLogLikelihood": -6.414824870283, "hits": 0, - "total": 201, + "total": 213, "hitRate": 0 }, { "encoding": "KOI8-U", "confidence": 0, "language": "uk", - "byteLogLikelihood": -6.32130473174, + "byteLogLikelihood": -6.199507530667, "hits": 0, - "total": 201, + "total": 215, "hitRate": 0 }, { "encoding": "IBM866", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.208609169552, + "byteLogLikelihood": -4.333669615722, "hits": 0, - "total": 201, + "total": 215, "hitRate": 0 }, { "encoding": "IBM855", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.608648280371, + "byteLogLikelihood": -4.794020051248, "hits": 0, - "total": 201, + "total": 215, "hitRate": 0 }, { "encoding": "x-mac-cyrillic", "confidence": 0, "language": "ru", - "byteLogLikelihood": -4.229886082374, + "byteLogLikelihood": -4.382241948417, "hits": 0, - "total": 202, + "total": 214, "hitRate": 0 } ] @@ -7513,6 +7563,8 @@ "total": 126, "hitRate": 0.10317460317460317 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -7538,7 +7590,7 @@ { "encoding": "ISO-8859-1", "confidence": 11, - "language": "fr", + "language": "pt", "byteLogLikelihood": -4.5670392286, "hits": 5, "total": 126, @@ -7547,7 +7599,7 @@ { "encoding": "windows-1252", "confidence": 11, - "language": "fr", + "language": "pt", "byteLogLikelihood": -4.5670392286, "hits": 5, "total": 126, @@ -7565,7 +7617,7 @@ { "encoding": "windows-1257", "confidence": 7, - "language": "et", + "language": "lt", "byteLogLikelihood": -4.997937219301, "hits": 3, "total": 126, @@ -7583,7 +7635,7 @@ { "encoding": "macintosh", "confidence": 7, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 124, @@ -7611,7 +7663,7 @@ "encoding": "ISO-8859-16", "confidence": 7, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 3, "total": 126, "hitRate": 0.023809523809523808 @@ -7619,7 +7671,7 @@ { "encoding": "CP850", "confidence": 7, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 125, @@ -7821,6 +7873,8 @@ "total": 136, "hitRate": 0.15441176470588236 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -7846,7 +7900,7 @@ { "encoding": "ISO-8859-1", "confidence": 17, - "language": "sv", + "language": "pt", "byteLogLikelihood": -4.507887190294, "hits": 8, "total": 136, @@ -7855,7 +7909,7 @@ { "encoding": "windows-1252", "confidence": 17, - "language": "sv", + "language": "pt", "byteLogLikelihood": -4.507887190294, "hits": 8, "total": 136, @@ -7906,6 +7960,15 @@ "total": 136, "hitRate": 0.014705882352941176 }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, { "encoding": "ISO-8859-9", "confidence": 2, @@ -7960,15 +8023,6 @@ "total": 136, "hitRate": 0.007352941176470588 }, - { - "encoding": "ISO-8859-16", - "confidence": 2, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -8129,6 +8183,8 @@ "total": 184, "hitRate": 0.1358695652173913 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -8163,7 +8219,7 @@ { "encoding": "ISO-8859-1", "confidence": 8, - "language": "es", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -8172,7 +8228,7 @@ { "encoding": "windows-1252", "confidence": 8, - "language": "es", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -8226,7 +8282,7 @@ { "encoding": "CP850", "confidence": 6, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 4, "total": 182, @@ -8245,7 +8301,7 @@ "encoding": "ISO-8859-16", "confidence": 3, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 2, "total": 183, "hitRate": 0.01092896174863388 @@ -8437,9 +8493,11 @@ "total": 202, "hitRate": 0.13861386138613863 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, - "tiedAtTop": 3, + "tiedAtTop": 2, "candidates": [ { "encoding": "ISO-8859-2", @@ -8461,17 +8519,17 @@ }, { "encoding": "ISO-8859-16", - "confidence": 41, + "confidence": 38, "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, + "byteLogLikelihood": -2.700458868498, + "hits": 26, "total": 202, - "hitRate": 0.13861386138613863 + "hitRate": 0.12871287128712872 }, { "encoding": "ISO-8859-1", "confidence": 26, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 18, "total": 202, @@ -8480,7 +8538,7 @@ { "encoding": "windows-1252", "confidence": 26, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 18, "total": 202, @@ -8489,7 +8547,7 @@ { "encoding": "macintosh", "confidence": 19, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 198, @@ -8498,7 +8556,7 @@ { "encoding": "CP850", "confidence": 15, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 10, "total": 199, @@ -8745,6 +8803,8 @@ "total": 183, "hitRate": 0.17486338797814208 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -8779,7 +8839,7 @@ { "encoding": "ISO-8859-1", "confidence": 14, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.81334289805, "hits": 9, "total": 181, @@ -8788,7 +8848,7 @@ { "encoding": "windows-1252", "confidence": 14, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.81334289805, "hits": 9, "total": 181, @@ -8797,7 +8857,7 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "hu", + "language": "pl", "byteLogLikelihood": -4.980958307454, "hits": 8, "total": 182, @@ -8869,21 +8929,12 @@ { "encoding": "windows-1257", "confidence": 9, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 6, "total": 182, "hitRate": 0.03296703296703297 }, - { - "encoding": "ISO-8859-16", - "confidence": 9, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 6, - "total": 182, - "hitRate": 0.03296703296703297 - }, { "encoding": "ISO-8859-14", "confidence": 6, @@ -8902,6 +8953,15 @@ "total": 182, "hitRate": 0.016483516483516484 }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 182, + "hitRate": 0.016483516483516484 + }, { "encoding": "ISO-8859-10", "confidence": 3, @@ -9053,6 +9113,8 @@ "total": 170, "hitRate": 0.13529411764705881 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -9069,7 +9131,7 @@ { "encoding": "windows-1257", "confidence": 31, - "language": "lt", + "language": "lv", "byteLogLikelihood": -5.111410123766, "hits": 18, "total": 169, @@ -9087,7 +9149,7 @@ { "encoding": "ISO-8859-1", "confidence": 14, - "language": "it", + "language": "es", "byteLogLikelihood": -4.721042183854, "hits": 8, "total": 169, @@ -9096,7 +9158,7 @@ { "encoding": "windows-1252", "confidence": 14, - "language": "it", + "language": "es", "byteLogLikelihood": -4.721042183854, "hits": 8, "total": 169, @@ -9141,7 +9203,7 @@ { "encoding": "ISO-8859-2", "confidence": 8, - "language": "ro", + "language": "pl", "byteLogLikelihood": -4.763992911083, "hits": 5, "total": 170, @@ -9150,7 +9212,7 @@ { "encoding": "windows-1250", "confidence": 8, - "language": "ro", + "language": "pl", "byteLogLikelihood": -4.846752537455, "hits": 5, "total": 169, @@ -9201,15 +9263,6 @@ "total": 169, "hitRate": 0.01775147928994083 }, - { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.572632236701, - "hits": 1, - "total": 169, - "hitRate": 0.005917159763313609 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -9344,6 +9397,15 @@ "hits": 0, "total": 170, "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.616838329205, + "hits": 0, + "total": 169, + "hitRate": 0 } ] }, @@ -9361,6 +9423,8 @@ "total": 179, "hitRate": 0.13966480446927373 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -9386,7 +9450,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 12, - "language": "ru", + "language": "uk", "byteLogLikelihood": -5.66890403251, "hits": 6, "total": 147, @@ -9630,7 +9694,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.06091613544, + "byteLogLikelihood": -5.105122227945, "hits": 0, "total": 178, "hitRate": 0 @@ -9669,6 +9733,8 @@ "total": 110, "hitRate": 0.20909090909090908 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -9694,7 +9760,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 10, - "language": "uk", + "language": "ru", "byteLogLikelihood": -5.359123221706, "hits": 3, "total": 82, @@ -9938,7 +10004,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.115876064699, + "byteLogLikelihood": -5.160082157204, "hits": 0, "total": 111, "hitRate": 0 @@ -9977,6 +10043,8 @@ "total": 134, "hitRate": 0.17164179104477612 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -10056,7 +10124,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 2, - "language": "ru", + "language": "uk", "byteLogLikelihood": -4.203232324163, "hits": 1, "total": 131, @@ -10246,7 +10314,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.093353952573, + "byteLogLikelihood": -5.137560045078, "hits": 0, "total": 134, "hitRate": 0 @@ -10285,6 +10353,8 @@ "total": 102, "hitRate": 0.1568627450980392 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -10554,7 +10624,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.048864674087, + "byteLogLikelihood": -5.093070766591, "hits": 0, "total": 102, "hitRate": 0 @@ -10593,6 +10663,8 @@ "total": 134, "hitRate": 0.1865671641791045 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -10627,7 +10699,7 @@ { "encoding": "ISO-8859-1", "confidence": 17, - "language": "pt", + "language": "da", "byteLogLikelihood": -4.969813299576, "hits": 8, "total": 134, @@ -10636,7 +10708,7 @@ { "encoding": "windows-1252", "confidence": 17, - "language": "pt", + "language": "da", "byteLogLikelihood": -4.969813299576, "hits": 8, "total": 134, @@ -10669,15 +10741,6 @@ "total": 134, "hitRate": 0.04477611940298507 }, - { - "encoding": "ISO-8859-16", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -4.352190516073, - "hits": 6, - "total": 134, - "hitRate": 0.04477611940298507 - }, { "encoding": "windows-1257", "confidence": 11, @@ -10723,6 +10786,15 @@ "total": 134, "hitRate": 0.03731343283582089 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, { "encoding": "CP850", "confidence": 11, @@ -10901,6 +10973,8 @@ "total": 179, "hitRate": 0.1452513966480447 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -11170,7 +11244,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.162378596536, + "byteLogLikelihood": -5.206584689041, "hits": 0, "total": 179, "hitRate": 0 @@ -11209,6 +11283,8 @@ "total": 177, "hitRate": 0.10734463276836158 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -11261,7 +11337,7 @@ { "encoding": "ISO-8859-1", "confidence": 1, - "language": "nl", + "language": "en", "byteLogLikelihood": -4.915420747009, "hits": 1, "total": 168, @@ -11288,7 +11364,7 @@ { "encoding": "windows-1252", "confidence": 1, - "language": "nl", + "language": "en", "byteLogLikelihood": -4.915420747009, "hits": 1, "total": 168, @@ -11478,7 +11554,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.148788659657, + "byteLogLikelihood": -5.192994752161, "hits": 0, "total": 177, "hitRate": 0 @@ -11517,6 +11593,8 @@ "total": 216, "hitRate": 0.24537037037037038 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -11533,7 +11611,7 @@ { "encoding": "ISO-8859-1", "confidence": 74, - "language": "nl", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 53, "total": 214, @@ -11542,7 +11620,7 @@ { "encoding": "ISO-8859-15", "confidence": 74, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 53, "total": 214, @@ -11551,7 +11629,7 @@ { "encoding": "windows-1252", "confidence": 70, - "language": "nl", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -11560,7 +11638,7 @@ { "encoding": "CP850", "confidence": 69, - "language": "fr", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 50, "total": 216, @@ -11578,7 +11656,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 11, "total": 214, @@ -11587,21 +11665,12 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 11, "total": 216, "hitRate": 0.05092592592592592 }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 11, - "total": 214, - "hitRate": 0.0514018691588785 - }, { "encoding": "ISO-8859-9", "confidence": 14, @@ -11620,10 +11689,19 @@ "total": 216, "hitRate": 0.041666666666666664 }, + { + "encoding": "ISO-8859-16", + "confidence": 12, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 214, + "hitRate": 0.04205607476635514 + }, { "encoding": "windows-1257", "confidence": 11, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 8, "total": 214, @@ -11825,6 +11903,8 @@ "total": 200, "hitRate": 0.205 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -11841,7 +11921,7 @@ { "encoding": "ISO-8859-1", "confidence": 58, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -11850,7 +11930,7 @@ { "encoding": "windows-1252", "confidence": 58, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 39, "total": 199, @@ -11877,7 +11957,7 @@ { "encoding": "windows-1257", "confidence": 25, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 17, "total": 199, @@ -11913,7 +11993,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 199, @@ -11922,7 +12002,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 199, @@ -11932,7 +12012,7 @@ "encoding": "ISO-8859-16", "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 10, "total": 199, "hitRate": 0.05025125628140704 @@ -12133,6 +12213,8 @@ "total": 228, "hitRate": 0.25877192982456143 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -12149,7 +12231,7 @@ { "encoding": "ISO-8859-1", "confidence": 75, - "language": "nl", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -12158,7 +12240,7 @@ { "encoding": "ISO-8859-15", "confidence": 75, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 56, "total": 224, @@ -12167,7 +12249,7 @@ { "encoding": "windows-1252", "confidence": 71, - "language": "nl", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -12176,7 +12258,7 @@ { "encoding": "CP850", "confidence": 71, - "language": "de", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -12185,7 +12267,7 @@ { "encoding": "ISO-8859-2", "confidence": 12, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 224, @@ -12204,7 +12286,7 @@ "encoding": "ISO-8859-16", "confidence": 12, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 9, "total": 224, "hitRate": 0.04017857142857143 @@ -12212,7 +12294,7 @@ { "encoding": "windows-1250", "confidence": 11, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 228, @@ -12239,7 +12321,7 @@ { "encoding": "windows-1257", "confidence": 7, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 6, "total": 228, @@ -12441,8 +12523,10 @@ "total": 126, "hitRate": 0.10317460317460317 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -12466,7 +12550,7 @@ { "encoding": "ISO-8859-1", "confidence": 11, - "language": "fr", + "language": "pt", "byteLogLikelihood": -4.5670392286, "hits": 5, "total": 126, @@ -12475,7 +12559,7 @@ { "encoding": "windows-1252", "confidence": 11, - "language": "fr", + "language": "pt", "byteLogLikelihood": -4.5670392286, "hits": 5, "total": 126, @@ -12493,7 +12577,7 @@ { "encoding": "windows-1257", "confidence": 7, - "language": "et", + "language": "lt", "byteLogLikelihood": -4.997937219301, "hits": 3, "total": 126, @@ -12511,7 +12595,7 @@ { "encoding": "macintosh", "confidence": 7, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 124, @@ -12539,7 +12623,7 @@ "encoding": "ISO-8859-16", "confidence": 7, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 3, "total": 126, "hitRate": 0.023809523809523808 @@ -12547,7 +12631,7 @@ { "encoding": "CP850", "confidence": 7, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 3, "total": 125, @@ -12749,8 +12833,10 @@ "total": 136, "hitRate": 0.15441176470588236 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -12774,7 +12860,7 @@ { "encoding": "ISO-8859-1", "confidence": 17, - "language": "sv", + "language": "pt", "byteLogLikelihood": -4.507887190294, "hits": 8, "total": 136, @@ -12783,7 +12869,7 @@ { "encoding": "windows-1252", "confidence": 17, - "language": "sv", + "language": "pt", "byteLogLikelihood": -4.507887190294, "hits": 8, "total": 136, @@ -12834,6 +12920,15 @@ "total": 136, "hitRate": 0.014705882352941176 }, + { + "encoding": "ISO-8859-16", + "confidence": 4, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 2, + "total": 136, + "hitRate": 0.014705882352941176 + }, { "encoding": "ISO-8859-9", "confidence": 2, @@ -12888,15 +12983,6 @@ "total": 136, "hitRate": 0.007352941176470588 }, - { - "encoding": "ISO-8859-16", - "confidence": 2, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 1, - "total": 136, - "hitRate": 0.007352941176470588 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -13057,6 +13143,8 @@ "total": 184, "hitRate": 0.1358695652173913 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -13091,7 +13179,7 @@ { "encoding": "ISO-8859-1", "confidence": 8, - "language": "es", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -13100,7 +13188,7 @@ { "encoding": "windows-1252", "confidence": 8, - "language": "es", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 5, "total": 183, @@ -13191,7 +13279,7 @@ "encoding": "ISO-8859-16", "confidence": 3, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 2, "total": 184, "hitRate": 0.010869565217391304 @@ -13365,9 +13453,11 @@ "total": 202, "hitRate": 0.13861386138613863 }, - "encodingCorrect": false, - "languageCorrect": false, - "tiedAtTop": 3, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, + "tiedAtTop": 2, "candidates": [ { "encoding": "ISO-8859-2", @@ -13389,17 +13479,17 @@ }, { "encoding": "ISO-8859-16", - "confidence": 41, + "confidence": 38, "language": "ro", - "byteLogLikelihood": -2.684539297542, - "hits": 28, + "byteLogLikelihood": -2.700458868498, + "hits": 26, "total": 202, - "hitRate": 0.13861386138613863 + "hitRate": 0.12871287128712872 }, { "encoding": "ISO-8859-1", "confidence": 26, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 18, "total": 202, @@ -13408,7 +13498,7 @@ { "encoding": "windows-1252", "confidence": 26, - "language": "pt", + "language": "it", "byteLogLikelihood": -4.919980925828, "hits": 18, "total": 202, @@ -13417,7 +13507,7 @@ { "encoding": "macintosh", "confidence": 19, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 198, @@ -13426,7 +13516,7 @@ { "encoding": "CP850", "confidence": 15, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 10, "total": 199, @@ -13673,6 +13763,8 @@ "total": 179, "hitRate": 0.13966480446927373 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -13698,7 +13790,7 @@ { "encoding": "IBM855", "confidence": 9, - "language": "ru", + "language": "sr", "byteLogLikelihood": -5.604243216214, "hits": 5, "total": 164, @@ -13942,7 +14034,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.931296681486, + "byteLogLikelihood": -4.970632247101, "hits": 0, "total": 179, "hitRate": 0 @@ -13981,8 +14073,10 @@ "total": 120, "hitRate": 0.175 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -14006,7 +14100,7 @@ { "encoding": "macintosh", "confidence": 30, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 12, "total": 120, @@ -14015,7 +14109,7 @@ { "encoding": "ISO-8859-15", "confidence": 30, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 12, "total": 120, @@ -14024,7 +14118,7 @@ { "encoding": "CP850", "confidence": 30, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 12, "total": 120, @@ -14039,10 +14133,19 @@ "total": 120, "hitRate": 0.05 }, + { + "encoding": "ISO-8859-16", + "confidence": 15, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 120, + "hitRate": 0.05 + }, { "encoding": "ISO-8859-2", "confidence": 10, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 4, "total": 120, @@ -14060,7 +14163,7 @@ { "encoding": "windows-1250", "confidence": 10, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 4, "total": 120, @@ -14078,21 +14181,12 @@ { "encoding": "windows-1257", "confidence": 10, - "language": "lt", + "language": "et", "byteLogLikelihood": -4.934473933131, "hits": 4, "total": 120, "hitRate": 0.03333333333333333 }, - { - "encoding": "ISO-8859-16", - "confidence": 10, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 4, - "total": 120, - "hitRate": 0.03333333333333333 - }, { "encoding": "windows-1258", "confidence": 7, @@ -14289,8 +14383,10 @@ "total": 216, "hitRate": 0.24537037037037038 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 3, "candidates": [ { @@ -14323,7 +14419,7 @@ { "encoding": "macintosh", "confidence": 72, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 52, "total": 215, @@ -14332,7 +14428,7 @@ { "encoding": "CP850", "confidence": 70, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 51, "total": 216, @@ -14350,7 +14446,7 @@ { "encoding": "ISO-8859-2", "confidence": 13, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 216, @@ -14359,15 +14455,6 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "pl", - "byteLogLikelihood": -5.176149732574, - "hits": 10, - "total": 216, - "hitRate": 0.046296296296296294 - }, - { - "encoding": "ISO-8859-16", - "confidence": 13, "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, @@ -14392,10 +14479,19 @@ "total": 216, "hitRate": 0.041666666666666664 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 216, + "hitRate": 0.037037037037037035 + }, { "encoding": "windows-1257", "confidence": 9, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 7, "total": 216, @@ -14591,14 +14687,16 @@ "predicted": { "encoding": "ISO-8859-15", "confidence": 29, - "language": "fr", + "language": "en", "byteLogLikelihood": -2.06619631493, "hits": 13, "total": 132, "hitRate": 0.09848484848484848 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { @@ -14613,7 +14711,7 @@ { "encoding": "ISO-8859-1", "confidence": 47, - "language": "fr", + "language": "en", "byteLogLikelihood": -3.310543013394, "hits": 21, "total": 132, @@ -14622,7 +14720,7 @@ { "encoding": "windows-1252", "confidence": 47, - "language": "fr", + "language": "en", "byteLogLikelihood": -3.310543013394, "hits": 21, "total": 132, @@ -14631,7 +14729,7 @@ { "encoding": "macintosh", "confidence": 27, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 12, "total": 132, @@ -14640,7 +14738,7 @@ { "encoding": "CP850", "confidence": 27, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 12, "total": 132, @@ -14667,7 +14765,7 @@ { "encoding": "ISO-8859-2", "confidence": 9, - "language": "hu", + "language": "cs", "byteLogLikelihood": -3.54961738678, "hits": 4, "total": 132, @@ -14676,7 +14774,7 @@ { "encoding": "windows-1250", "confidence": 9, - "language": "hu", + "language": "cs", "byteLogLikelihood": -3.54961738678, "hits": 4, "total": 132, @@ -14692,19 +14790,19 @@ "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-4", - "confidence": 6, - "language": "lv", - "byteLogLikelihood": -5.19295685089, - "hits": 3, + "encoding": "ISO-8859-16", + "confidence": 9, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, "total": 132, - "hitRate": 0.022727272727272728 + "hitRate": 0.030303030303030304 }, { - "encoding": "ISO-8859-16", + "encoding": "ISO-8859-4", "confidence": 6, - "language": "ro", - "byteLogLikelihood": -5.176149732574, + "language": "lv", + "byteLogLikelihood": -5.19295685089, "hits": 3, "total": 132, "hitRate": 0.022727272727272728 @@ -14905,8 +15003,10 @@ "total": 200, "hitRate": 0.205 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 3, "candidates": [ { @@ -14957,7 +15057,7 @@ { "encoding": "windows-1257", "confidence": 24, - "language": "lt", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 16, "total": 200, @@ -14993,7 +15093,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 200, @@ -15002,7 +15102,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 10, "total": 200, @@ -15012,7 +15112,7 @@ "encoding": "ISO-8859-16", "confidence": 15, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 10, "total": 200, "hitRate": 0.05 @@ -15213,8 +15313,10 @@ "total": 228, "hitRate": 0.25877192982456143 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 3, "candidates": [ { @@ -15247,7 +15349,7 @@ { "encoding": "macintosh", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -15256,7 +15358,7 @@ { "encoding": "CP850", "confidence": 71, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 54, "total": 228, @@ -15265,7 +15367,7 @@ { "encoding": "ISO-8859-2", "confidence": 11, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, "total": 228, @@ -15274,15 +15376,6 @@ { "encoding": "windows-1250", "confidence": 11, - "language": "hu", - "byteLogLikelihood": -5.176149732574, - "hits": 9, - "total": 228, - "hitRate": 0.039473684210526314 - }, - { - "encoding": "ISO-8859-16", - "confidence": 11, "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 9, @@ -15298,10 +15391,19 @@ "total": 228, "hitRate": 0.03508771929824561 }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 8, + "total": 228, + "hitRate": 0.03508771929824561 + }, { "encoding": "windows-1257", "confidence": 7, - "language": "et", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 6, "total": 228, @@ -15521,8 +15623,10 @@ "total": 159, "hitRate": 0.1949685534591195 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -15546,7 +15650,7 @@ { "encoding": "macintosh", "confidence": 24, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 158, @@ -15555,7 +15659,7 @@ { "encoding": "ISO-8859-15", "confidence": 24, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 159, @@ -15564,7 +15668,7 @@ { "encoding": "CP850", "confidence": 24, - "language": "es", + "language": "fr", "byteLogLikelihood": -5.010635294096, "hits": 13, "total": 159, @@ -15579,10 +15683,19 @@ "total": 159, "hitRate": 0.05660377358490566 }, + { + "encoding": "ISO-8859-16", + "confidence": 16, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 9, + "total": 159, + "hitRate": 0.05660377358490566 + }, { "encoding": "ISO-8859-2", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 8, "total": 159, @@ -15591,7 +15704,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "pl", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 8, "total": 159, @@ -15600,7 +15713,7 @@ { "encoding": "windows-1257", "confidence": 15, - "language": "lt", + "language": "lv", "byteLogLikelihood": -5.19295685089, "hits": 8, "total": 159, @@ -15615,15 +15728,6 @@ "total": 159, "hitRate": 0.050314465408805034 }, - { - "encoding": "ISO-8859-16", - "confidence": 15, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 8, - "total": 159, - "hitRate": 0.050314465408805034 - }, { "encoding": "ISO-8859-14", "confidence": 9, @@ -15823,20 +15927,22 @@ "predicted": { "encoding": "ISO-8859-1", "confidence": 83, - "language": "fr", + "language": "nl", "byteLogLikelihood": -3.295836866004, "hits": 42, "total": 151, "hitRate": 0.2781456953642384 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { "encoding": "ISO-8859-1", "confidence": 83, - "language": "fr", + "language": "nl", "byteLogLikelihood": -3.295836866004, "hits": 42, "total": 151, @@ -15845,7 +15951,7 @@ { "encoding": "windows-1252", "confidence": 83, - "language": "fr", + "language": "nl", "byteLogLikelihood": -3.295836866004, "hits": 42, "total": 151, @@ -15854,7 +15960,7 @@ { "encoding": "macintosh", "confidence": 63, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 32, "total": 151, @@ -15863,7 +15969,7 @@ { "encoding": "ISO-8859-15", "confidence": 63, - "language": "fr", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 32, "total": 151, @@ -15872,7 +15978,7 @@ { "encoding": "CP850", "confidence": 63, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 32, "total": 151, @@ -15890,7 +15996,7 @@ { "encoding": "ISO-8859-2", "confidence": 13, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 7, "total": 151, @@ -15908,7 +16014,7 @@ { "encoding": "windows-1250", "confidence": 13, - "language": "hu", + "language": "ro", "byteLogLikelihood": -5.176149732574, "hits": 7, "total": 151, @@ -15923,15 +16029,6 @@ "total": 151, "hitRate": 0.046357615894039736 }, - { - "encoding": "ISO-8859-16", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 7, - "total": 151, - "hitRate": 0.046357615894039736 - }, { "encoding": "ISO-8859-10", "confidence": 11, @@ -15941,6 +16038,15 @@ "total": 151, "hitRate": 0.039735099337748346 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 6, + "total": 151, + "hitRate": 0.039735099337748346 + }, { "encoding": "windows-1257", "confidence": 9, @@ -16137,8 +16243,10 @@ "total": 114, "hitRate": 0.15789473684210525 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -16162,7 +16270,7 @@ { "encoding": "macintosh", "confidence": 18, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 7, "total": 114, @@ -16180,7 +16288,7 @@ { "encoding": "ISO-8859-15", "confidence": 18, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 7, "total": 114, @@ -16189,7 +16297,7 @@ { "encoding": "CP850", "confidence": 18, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 7, "total": 114, @@ -16213,6 +16321,15 @@ "total": 114, "hitRate": 0.03508771929824561 }, + { + "encoding": "ISO-8859-16", + "confidence": 10, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 4, + "total": 114, + "hitRate": 0.03508771929824561 + }, { "encoding": "ISO-8859-2", "confidence": 7, @@ -16252,21 +16369,12 @@ { "encoding": "windows-1257", "confidence": 7, - "language": "lt", + "language": "et", "byteLogLikelihood": -4.934473933131, "hits": 3, "total": 114, "hitRate": 0.02631578947368421 }, - { - "encoding": "ISO-8859-16", - "confidence": 5, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 2, - "total": 114, - "hitRate": 0.017543859649122806 - }, { "encoding": "ISO-8859-3", "confidence": 2, @@ -16439,14 +16547,16 @@ "predicted": { "encoding": "ISO-8859-15", "confidence": 33, - "language": "es", + "language": "pt", "byteLogLikelihood": -3.970754139021, "hits": 16, "total": 143, "hitRate": 0.11188811188811189 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 1, "candidates": [ { @@ -16461,7 +16571,7 @@ { "encoding": "ISO-8859-1", "confidence": 50, - "language": "es", + "language": "pt", "byteLogLikelihood": -4.059947322987, "hits": 24, "total": 143, @@ -16470,7 +16580,7 @@ { "encoding": "windows-1252", "confidence": 50, - "language": "es", + "language": "pt", "byteLogLikelihood": -4.059947322987, "hits": 24, "total": 143, @@ -16479,7 +16589,7 @@ { "encoding": "windows-1257", "confidence": 29, - "language": "lv", + "language": "lt", "byteLogLikelihood": -4.777390389123, "hits": 14, "total": 143, @@ -16588,7 +16698,7 @@ "encoding": "ISO-8859-16", "confidence": 2, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 1, "total": 143, "hitRate": 0.006993006993006993 @@ -16753,8 +16863,10 @@ "total": 136, "hitRate": 0.16911764705882354 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -16796,7 +16908,7 @@ { "encoding": "macintosh", "confidence": 17, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 8, "total": 136, @@ -16814,7 +16926,7 @@ { "encoding": "CP850", "confidence": 17, - "language": "es", + "language": "de", "byteLogLikelihood": -5.003946305945, "hits": 8, "total": 136, @@ -16823,7 +16935,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "hu", + "language": "cs", "byteLogLikelihood": -5.159055299215, "hits": 7, "total": 136, @@ -16832,7 +16944,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "hu", + "language": "cs", "byteLogLikelihood": -5.159055299215, "hits": 7, "total": 136, @@ -16865,10 +16977,19 @@ "total": 136, "hitRate": 0.029411764705882353 }, + { + "encoding": "ISO-8859-16", + "confidence": 6, + "language": "ro", + "byteLogLikelihood": -5.220355825078, + "hits": 3, + "total": 136, + "hitRate": 0.022058823529411766 + }, { "encoding": "windows-1257", "confidence": 4, - "language": "et", + "language": "lt", "byteLogLikelihood": -5.123963979403, "hits": 2, "total": 136, @@ -16892,15 +17013,6 @@ "total": 136, "hitRate": 0.014705882352941176 }, - { - "encoding": "ISO-8859-16", - "confidence": 4, - "language": "ro", - "byteLogLikelihood": -5.176149732574, - "hits": 2, - "total": 136, - "hitRate": 0.014705882352941176 - }, { "encoding": "CP852", "confidence": 4, @@ -17061,8 +17173,10 @@ "total": 134, "hitRate": 0.17164179104477612 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -17140,7 +17254,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 2, - "language": "ru", + "language": "uk", "byteLogLikelihood": -4.203232324163, "hits": 1, "total": 131, @@ -17330,7 +17444,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.093353952573, + "byteLogLikelihood": -5.137560045078, "hits": 0, "total": 134, "hitRate": 0 @@ -17369,8 +17483,10 @@ "total": 134, "hitRate": 0.1865671641791045 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -17403,7 +17519,7 @@ { "encoding": "ISO-8859-1", "confidence": 17, - "language": "pt", + "language": "da", "byteLogLikelihood": -4.969813299576, "hits": 8, "total": 134, @@ -17412,7 +17528,7 @@ { "encoding": "windows-1252", "confidence": 17, - "language": "pt", + "language": "da", "byteLogLikelihood": -4.969813299576, "hits": 8, "total": 134, @@ -17445,15 +17561,6 @@ "total": 134, "hitRate": 0.04477611940298507 }, - { - "encoding": "ISO-8859-16", - "confidence": 13, - "language": "ro", - "byteLogLikelihood": -4.352190516073, - "hits": 6, - "total": 134, - "hitRate": 0.04477611940298507 - }, { "encoding": "windows-1257", "confidence": 11, @@ -17499,6 +17606,15 @@ "total": 134, "hitRate": 0.03731343283582089 }, + { + "encoding": "ISO-8859-16", + "confidence": 11, + "language": "ro", + "byteLogLikelihood": -4.25849981603, + "hits": 5, + "total": 134, + "hitRate": 0.03731343283582089 + }, { "encoding": "CP850", "confidence": 11, @@ -17677,8 +17793,10 @@ "total": 102, "hitRate": 0.1568627450980392 }, - "encodingCorrect": false, - "languageCorrect": false, + "encodingExact": false, + "encodingEquivalent": true, + "encodingCorrect": true, + "languageCorrect": true, "tiedAtTop": 2, "candidates": [ { @@ -17946,7 +18064,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.048864674087, + "byteLogLikelihood": -5.093070766591, "hits": 0, "total": 102, "hitRate": 0 @@ -17985,6 +18103,8 @@ "total": 110, "hitRate": 0.20909090909090908 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -18001,7 +18121,7 @@ { "encoding": "IBM855", "confidence": 21, - "language": "ru", + "language": "sr", "byteLogLikelihood": -5.640676681359, "hits": 7, "total": 100, @@ -18254,7 +18374,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.075354869964, + "byteLogLikelihood": -5.119560962469, "hits": 0, "total": 111, "hitRate": 0 @@ -18293,6 +18413,8 @@ "total": 120, "hitRate": 0.13333333333333333 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -18309,7 +18431,7 @@ { "encoding": "ISO-8859-1", "confidence": 12, - "language": "de", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 5, "total": 120, @@ -18327,7 +18449,7 @@ { "encoding": "windows-1252", "confidence": 12, - "language": "de", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 5, "total": 120, @@ -18363,7 +18485,7 @@ { "encoding": "ISO-8859-15", "confidence": 12, - "language": "de", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 5, "total": 120, @@ -18445,7 +18567,7 @@ "encoding": "ISO-8859-16", "confidence": 5, "language": "ro", - "byteLogLikelihood": -5.176149732574, + "byteLogLikelihood": -5.220355825078, "hits": 2, "total": 120, "hitRate": 0.016666666666666666 @@ -18601,6 +18723,8 @@ "total": 173, "hitRate": 0.11560693641618497 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 2, @@ -18635,7 +18759,7 @@ { "encoding": "ISO-8859-2", "confidence": 15, - "language": "ro", + "language": "hu", "byteLogLikelihood": -5.117993812417, "hits": 9, "total": 173, @@ -18644,7 +18768,7 @@ { "encoding": "windows-1250", "confidence": 15, - "language": "ro", + "language": "hu", "byteLogLikelihood": -5.117993812417, "hits": 9, "total": 173, @@ -18653,7 +18777,7 @@ { "encoding": "ISO-8859-1", "confidence": 12, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, @@ -18662,7 +18786,7 @@ { "encoding": "windows-1252", "confidence": 12, - "language": "nl", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, @@ -18680,7 +18804,7 @@ { "encoding": "ISO-8859-15", "confidence": 12, - "language": "fr", + "language": "es", "byteLogLikelihood": -4.948759890378, "hits": 7, "total": 173, @@ -18752,7 +18876,7 @@ { "encoding": "IBM855", "confidence": 1, - "language": "sr", + "language": "ru", "byteLogLikelihood": -5.488073838002, "hits": 1, "total": 170, @@ -18767,15 +18891,6 @@ "total": 173, "hitRate": 0.005780346820809248 }, - { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.659155714377, - "hits": 1, - "total": 173, - "hitRate": 0.005780346820809248 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -18892,6 +19007,15 @@ "hits": 0, "total": 173, "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.616838329205, + "hits": 0, + "total": 173, + "hitRate": 0 } ] }, @@ -18909,6 +19033,8 @@ "total": 170, "hitRate": 0.13529411764705881 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -18943,7 +19069,7 @@ { "encoding": "ISO-8859-1", "confidence": 14, - "language": "en", + "language": "fr", "byteLogLikelihood": -4.806768476285, "hits": 8, "total": 170, @@ -18952,7 +19078,7 @@ { "encoding": "windows-1252", "confidence": 14, - "language": "en", + "language": "fr", "byteLogLikelihood": -4.806768476285, "hits": 8, "total": 170, @@ -19006,7 +19132,7 @@ { "encoding": "ISO-8859-2", "confidence": 8, - "language": "ro", + "language": "pl", "byteLogLikelihood": -5.087596335232, "hits": 5, "total": 170, @@ -19015,7 +19141,7 @@ { "encoding": "windows-1250", "confidence": 8, - "language": "ro", + "language": "pl", "byteLogLikelihood": -5.087596335232, "hits": 5, "total": 170, @@ -19057,15 +19183,6 @@ "total": 170, "hitRate": 0.023529411764705882 }, - { - "encoding": "ISO-8859-16", - "confidence": 1, - "language": "ro", - "byteLogLikelihood": -4.673743278572, - "hits": 1, - "total": 170, - "hitRate": 0.0058823529411764705 - }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -19200,6 +19317,15 @@ "hits": 0, "total": 170, "hitRate": 0 + }, + { + "encoding": "ISO-8859-16", + "confidence": 0, + "language": "ro", + "byteLogLikelihood": -4.717949371076, + "hits": 0, + "total": 170, + "hitRate": 0 } ] }, @@ -19217,6 +19343,8 @@ "total": 124, "hitRate": 0.3387096774193548 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -19296,7 +19424,7 @@ { "encoding": "ISO-8859-2", "confidence": 7, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.060625202481, "hits": 3, "total": 126, @@ -19314,7 +19442,7 @@ { "encoding": "windows-1250", "confidence": 7, - "language": "cs", + "language": "ro", "byteLogLikelihood": -5.060625202481, "hits": 3, "total": 126, @@ -19347,15 +19475,6 @@ "total": 126, "hitRate": 0.023809523809523808 }, - { - "encoding": "ISO-8859-16", - "confidence": 7, - "language": "ro", - "byteLogLikelihood": -5.060625202481, - "hits": 3, - "total": 126, - "hitRate": 0.023809523809523808 - }, { "encoding": "CP850", "confidence": 7, @@ -19374,6 +19493,15 @@ "total": 119, "hitRate": 0.025210084033613446 }, + { + "encoding": "ISO-8859-16", + "confidence": 2, + "language": "ro", + "byteLogLikelihood": -5.104831294985, + "hits": 1, + "total": 126, + "hitRate": 0.007936507936507936 + }, { "encoding": "ISO-8859-5", "confidence": 0, @@ -19525,6 +19653,8 @@ "total": 100, "hitRate": 0.07 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -19595,7 +19725,7 @@ { "encoding": "x-mac-cyrillic", "confidence": 11, - "language": "uk", + "language": "ru", "byteLogLikelihood": -6.059253444951, "hits": 3, "total": 79, @@ -19794,7 +19924,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -5.123264178812, + "byteLogLikelihood": -5.134285984462, "hits": 0, "total": 103, "hitRate": 0 @@ -19833,6 +19963,8 @@ "total": 179, "hitRate": 0.12290502793296089 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -19858,7 +19990,7 @@ { "encoding": "IBM855", "confidence": 9, - "language": "ru", + "language": "sr", "byteLogLikelihood": -5.603105135042, "hits": 5, "total": 165, @@ -20102,7 +20234,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.935887060165, + "byteLogLikelihood": -4.97522262578, "hits": 0, "total": 177, "hitRate": 0 @@ -20141,6 +20273,8 @@ "total": 177, "hitRate": 0.10734463276836158 }, + "encodingExact": true, + "encodingEquivalent": false, "encodingCorrect": true, "languageCorrect": true, "tiedAtTop": 1, @@ -20166,7 +20300,7 @@ { "encoding": "IBM855", "confidence": 5, - "language": "ru", + "language": "sr", "byteLogLikelihood": -5.830774318921, "hits": 3, "total": 163, @@ -20410,7 +20544,7 @@ "encoding": "ISO-8859-16", "confidence": 0, "language": "ro", - "byteLogLikelihood": -4.986714390957, + "byteLogLikelihood": -5.030920483462, "hits": 0, "total": 175, "hitRate": 0 diff --git a/scripts/corpus-utils.mjs b/scripts/corpus-utils.mjs index d12c846..bdb185b 100644 --- a/scripts/corpus-utils.mjs +++ b/scripts/corpus-utils.mjs @@ -143,6 +143,9 @@ export function buildCorpus(destination) { )) { sourceText = sourceText.replaceAll(from, to); } + if (encoding.sourceSuffix) { + sourceText = `${sourceText.trimEnd()}${encoding.sourceSuffix}\n`; + } const utf8 = Buffer.from(sourceText, 'utf8'); const encoded = iconv(utf8, 'UTF-8', encoding.iconv); const decoded = iconv(encoded, encoding.iconv, 'UTF-8'); diff --git a/scripts/evaluate-models.mjs b/scripts/evaluate-models.mjs index f80c70d..2b557e0 100644 --- a/scripts/evaluate-models.mjs +++ b/scripts/evaluate-models.mjs @@ -4,7 +4,11 @@ const { report } = compileModels(); for (const result of report.results) { const expected = `${result.expected.encoding}/${result.expected.language}`; const predicted = `${result.predicted.encoding}/${result.predicted.language}`; - const marker = result.encodingCorrect ? 'PASS' : 'FAIL'; + const marker = result.encodingExact + ? 'PASS' + : result.encodingEquivalent + ? 'EQUIV' + : 'FAIL'; console.log( `${marker} ${expected.padEnd(24)} -> ${predicted.padEnd(24)} ${result.predicted.confidence}%`, ); @@ -12,6 +16,8 @@ for (const result of report.results) { console.log( `\nEncoding: ${report.summary.encodingCorrect}/${report.summary.tests}; ` + + `exact: ${report.summary.encodingExact}/${report.summary.tests}; ` + + `equivalent: ${report.summary.encodingEquivalent}; ` + `language: ${report.summary.languageCorrect}/${report.summary.tests}; ` + `top-score ties: ${report.summary.topScoreTies}`, ); diff --git a/scripts/model-utils.mjs b/scripts/model-utils.mjs index 7c66d93..f9e9f15 100644 --- a/scripts/model-utils.mjs +++ b/scripts/model-utils.mjs @@ -197,6 +197,39 @@ function statisticallyCompetitive(best, candidate) { return best.hitRate - candidate.hitRate <= margin; } +function decodedText(buffer, encodingName) { + const encoding = manifest.encodings.find( + (candidate) => candidate.name === encodingName, + ); + if (!encoding) throw new Error(`Missing manifest encoding: ${encodingName}`); + return iconv(buffer, encoding.iconv, 'UTF-8'); +} + +function byteEquivalent(buffer, leftEncoding, rightEncoding) { + try { + return decodedText(buffer, leftEncoding).equals( + decodedText(buffer, rightEncoding), + ); + } catch { + return false; + } +} + +const equivalentEncodingFamilies = [ + ['ISO-8859-1', 'ISO-8859-15', 'windows-1252'], + ['ISO-8859-2', 'windows-1250'], + ['ISO-8859-7', 'windows-1253'], + ['ISO-8859-8', 'windows-1255'], + ['ISO-8859-9', 'windows-1254'], + ['ISO-8859-13', 'windows-1257'], +]; + +function encodingFamily(name) { + return ( + equivalentEncodingFamilies.find((family) => family.includes(name)) ?? [name] + ); +} + function compileSingleByteModels() { const models = []; for (const encoding of manifest.encodings) { @@ -250,15 +283,10 @@ function evaluate(models) { (left, right) => right.byteLogLikelihood - left.byteLogLikelihood, ); const confidenceLanguage = confidenceLanguages[0]; - languageScores.sort( - (left, right) => - right.byteLogLikelihood - left.byteLogLikelihood || - right.confidence - left.confidence, - ); return { encoding: model.encoding, confidence: confidenceScore, - language: languageScores[0].language, + language: confidenceLanguage.language, byteLogLikelihood: confidenceLanguage.byteLogLikelihood, hits: confidenceLanguage.hits, total: confidenceLanguage.total, @@ -282,14 +310,42 @@ function evaluate(models) { (candidate) => !competitiveEncodings.has(candidate.encoding), ); candidates.splice(0, candidates.length, ...competitive, ...remaining); - const predicted = candidates[0]; + const strongestPrediction = candidates[0]; + const strongestFamily = encodingFamily(strongestPrediction.encoding); + const equivalentLanguageCandidates = candidates.filter( + (candidate) => + strongestFamily.includes(candidate.encoding) && + byteEquivalent( + buffer, + strongestPrediction.encoding, + candidate.encoding, + ), + ); + equivalentLanguageCandidates.sort( + (left, right) => + manifest.encodings.findIndex( + (encoding) => encoding.name === left.encoding, + ) - + manifest.encodings.findIndex( + (encoding) => encoding.name === right.encoding, + ), + ); + const predicted = { + ...strongestPrediction, + language: equivalentLanguageCandidates[0].language, + }; + const encodingExact = predicted.encoding === test.encoding; + const encodingEquivalent = + !encodingExact && + byteEquivalent(buffer, test.encoding, predicted.encoding); + const encodingCorrect = encodingExact || encodingEquivalent; return { expected: { encoding: test.encoding, language: test.language }, predicted, - encodingCorrect: predicted.encoding === test.encoding, - languageCorrect: - predicted.encoding === test.encoding && - predicted.language === test.language, + encodingExact, + encodingEquivalent, + encodingCorrect, + languageCorrect: encodingCorrect && predicted.language === test.language, tiedAtTop: candidates.filter( (candidate) => candidate.confidence === predicted.confidence, ).length, @@ -308,6 +364,9 @@ function evaluate(models) { return { summary: { tests: results.length, + encodingExact: results.filter((result) => result.encodingExact).length, + encodingEquivalent: results.filter((result) => result.encodingEquivalent) + .length, encodingCorrect: results.filter((result) => result.encodingCorrect) .length, languageCorrect: results.filter((result) => result.languageCorrect) diff --git a/src/encoding/models/generated.ts b/src/encoding/models/generated.ts index cd4eddb..4f81c87 100644 --- a/src/encoding/models/generated.ts +++ b/src/encoding/models/generated.ts @@ -3003,26 +3003,27 @@ export const generatedSBCSModels = [ { language: 'ro', ngrams: [ - 0x20616c, 0x206175, 0x206269, 0x206361, 0x206375, 0x2063e3, 0x206465, - 0x206d69, 0x206f72, 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, - 0x616c20, 0x617ae3, 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, - 0x646520, 0x646573, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, - 0x656c65, 0x657269, 0x657363, 0x65ba74, 0x692063, 0x6961fe, 0x69626c, - 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, 0x6c6520, - 0x6c696f, 0x6d62e3, 0x6f7269, 0x6f7465, 0x7261ba, 0x726520, 0x726969, - 0x736520, 0x746520, 0x746563, 0x746f72, 0x74e320, 0x756c20, 0x7ae320, - 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xe3206d, 0xe32072, 0xe320ba, - 0xfe6120, + 0x206361, 0x206375, 0x2063e3, 0x206465, 0x206573, 0x206d69, 0x206f72, + 0x207072, 0x207265, 0x20ba69, 0x20ba74, 0x20ee6e, 0x616c20, 0x617ae3, + 0x61fe61, 0x626962, 0x626c69, 0x636869, 0x63e320, 0x646520, 0x646573, + 0x652020, 0x652063, 0x652070, 0x6520ba, 0x6520ee, 0x656361, 0x656c65, + 0x657269, 0x657363, 0x657374, 0x65ba74, 0x65fe75, 0x692063, 0x6961fe, + 0x69626c, 0x6963e3, 0x696520, 0x696572, 0x696920, 0x696e65, 0x696f74, + 0x6c2065, 0x6c6520, 0x6c696f, 0x6d62e3, 0x6f7269, 0x707265, 0x7261ba, + 0x726520, 0x7265fe, 0x726969, 0x737465, 0x746520, 0x746f72, 0x74e320, + 0x756c20, 0x7ae320, 0xba6920, 0xba7465, 0xba7469, 0xe32063, 0xfe6120, + 0xfe756c, ], highBytes: { - total: 49, + total: 57, counts: [ + [0xa4, 4], [0xba, 12], [0xce, 1], [0xe2, 3], [0xe3, 21], [0xee, 4], - [0xfe, 8], + [0xfe, 12], ], }, },