v3 branch

.github/workflows/test-build.js

@@ -8,15 +8,28 @@ assert(typeof chardet.detectFile, 'function');
 assert(typeof chardet.detectFileSync, 'function');
 
 assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [
-  { confidence: 100, name: 'ASCII', lang: undefined },
-  { confidence: 98, name: 'ISO-8859-1', lang: 'en' },
-  { confidence: 98, name: 'ISO-8859-2', lang: 'hu' },
-  { confidence: 10, name: 'UTF-8', lang: undefined },
-  { confidence: 10, name: 'Shift_JIS', lang: 'ja' },
-  { confidence: 10, name: 'Big5', lang: 'zh' },
-  { confidence: 10, name: 'EUC-JP', lang: 'ja' },
-  { confidence: 10, name: 'EUC-KR', lang: 'ko' },
-  { confidence: 10, name: 'GB18030', lang: 'zh' },
+  { confidence: 1, name: 'ASCII', lang: undefined },
+  { name: 'ISO-8859-2', lang: 'hu', confidence: 0.6 },
+  { name: 'windows-1250', lang: 'hu', confidence: 0.6 },
+  { name: 'ISO-8859-1', lang: 'en', confidence: 0.4 },
+  { name: 'windows-1252', lang: 'en', confidence: 0.4 },
+  { name: 'ISO-8859-10', lang: 'is', confidence: 0.4 },
+  { name: 'windows-1257', lang: 'lv', confidence: 0.2 },
+  { name: 'macintosh', lang: 'de', confidence: 0.2 },
+  { name: 'ISO-8859-3', lang: 'mt', confidence: 0.2 },
+  { name: 'ISO-8859-4', lang: 'lv', confidence: 0.2 },
+  { name: 'ISO-8859-13', lang: 'lt', confidence: 0.2 },
+  { name: 'ISO-8859-14', lang: 'cy', confidence: 0.2 },
+  { name: 'ISO-8859-15', lang: 'de', confidence: 0.2 },
+  { name: 'ISO-8859-16', lang: 'ro', confidence: 0.2 },
+  { name: 'CP850', lang: 'de', confidence: 0.2 },
+  { name: 'CP852', lang: 'pl', confidence: 0.2 },
+  { confidence: 0.1, name: 'UTF-8', lang: undefined },
+  { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' },
+  { confidence: 0.1, name: 'Big5', lang: 'zh' },
+  { confidence: 0.1, name: 'EUC-JP', lang: 'ja' },
+  { confidence: 0.1, name: 'EUC-KR', lang: 'ko' },
+  { confidence: 0.1, name: 'GB18030', lang: 'zh' },
 ]);
 
 console.log(' > test-build.js OK');

.github/workflows/test-build.ts

@@ -11,15 +11,28 @@ const main = async () => {
   assert(typeof chardet.detectFileSync, 'function');
 
   assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [
-    { confidence: 100, name: 'ASCII', lang: undefined },
-    { confidence: 98, name: 'ISO-8859-1', lang: 'en' },
-    { confidence: 98, name: 'ISO-8859-2', lang: 'hu' },
-    { confidence: 10, name: 'UTF-8', lang: undefined },
-    { confidence: 10, name: 'Shift_JIS', lang: 'ja' },
-    { confidence: 10, name: 'Big5', lang: 'zh' },
-    { confidence: 10, name: 'EUC-JP', lang: 'ja' },
-    { confidence: 10, name: 'EUC-KR', lang: 'ko' },
-    { confidence: 10, name: 'GB18030', lang: 'zh' },
+    { confidence: 1, name: 'ASCII', lang: undefined },
+    { name: 'ISO-8859-2', lang: 'hu', confidence: 0.6 },
+    { name: 'windows-1250', lang: 'hu', confidence: 0.6 },
+    { name: 'ISO-8859-1', lang: 'en', confidence: 0.4 },
+    { name: 'windows-1252', lang: 'en', confidence: 0.4 },
+    { name: 'ISO-8859-10', lang: 'is', confidence: 0.4 },
+    { name: 'windows-1257', lang: 'lv', confidence: 0.2 },
+    { name: 'macintosh', lang: 'de', confidence: 0.2 },
+    { name: 'ISO-8859-3', lang: 'mt', confidence: 0.2 },
+    { name: 'ISO-8859-4', lang: 'lv', confidence: 0.2 },
+    { name: 'ISO-8859-13', lang: 'lt', confidence: 0.2 },
+    { name: 'ISO-8859-14', lang: 'cy', confidence: 0.2 },
+    { name: 'ISO-8859-15', lang: 'de', confidence: 0.2 },
+    { name: 'ISO-8859-16', lang: 'ro', confidence: 0.2 },
+    { name: 'CP850', lang: 'de', confidence: 0.2 },
+    { name: 'CP852', lang: 'pl', confidence: 0.2 },
+    { confidence: 0.1, name: 'UTF-8', lang: undefined },
+    { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' },
+    { confidence: 0.1, name: 'Big5', lang: 'zh' },
+    { confidence: 0.1, name: 'EUC-JP', lang: 'ja' },
+    { confidence: 0.1, name: 'EUC-KR', lang: 'ko' },
+    { confidence: 0.1, name: 'GB18030', lang: 'zh' },
   ]);
 };
 

.gitignore

@@ -7,3 +7,4 @@ lib
 TODO.md
 package-lock.json
 .vscode
+BASELINE.md

BASELINE.md

@@ -0,0 +1,115 @@
+# Generated model migration baseline
+
+Recorded on 2026-07-12 at commit
+`df1e8dd0088f5d4b71790a54ee2fc1f2afe6cb0a` (`v3`), using Node.js v24.10.0
+and npm 11.6.1.
+
+`TODO.md` is locally present but ignored by the repository. There were no
+tracked or unignored changes when this baseline was recorded. Generated `lib/`
+and coverage output are also ignored.
+
+## Checks
+
+| Check                             | Result                                                             |
+| --------------------------------- | ------------------------------------------------------------------ |
+| `npm test`                        | Pass: 8 files, 48 tests passed, 8 todo                             |
+| TypeScript                        | Pass                                                               |
+| Coverage                          | 93.04% statements, 85.44% branches, 95.86% functions, 93.13% lines |
+| `npm run models:verify`           | Pass; models and evaluation are reproducible                       |
+| `npm run validate -- --split=all` | Completes; 0/138 encoding and language predictions correct         |
+| `npm run benchmark`               | 7,848.13 files/sec; 1.42 MiB/sec                                   |
+| `npm run build`                   | Pass                                                               |
+| Built `lib/`                      | 341,661 bytes across files; 448 KiB allocated on disk              |
+| `npm pack --dry-run`              | 42,359 bytes packed; 348,964 bytes unpacked; 42 entries            |
+
+The benchmark processed 13,800 calls and 2,622,400 bytes in 1,758.38 ms,
+producing 100,200 matches. These figures are a local performance reference, not
+a hard CI threshold; comparisons should use the same machine and runtime.
+
+## Runtime corpus validation
+
+The validation corpus currently contains only newly generated models that are
+not registered with the runtime. The 0% result is therefore the expected
+pre-migration baseline, not a model-evaluator failure. Every input still
+received at least one prediction.
+
+### By split
+
+| Split      |   Files | Encoding correct | Language correct |
+| ---------- | ------: | ---------------: | ---------------: |
+| train      |      92 |        0 (0.00%) |        0 (0.00%) |
+| validation |      23 |        0 (0.00%) |        0 (0.00%) |
+| test       |      23 |        0 (0.00%) |        0 (0.00%) |
+| **Total**  | **138** |    **0 (0.00%)** |    **0 (0.00%)** |
+
+### By expected encoding
+
+| Expected encoding | Files | Encoding correct | Most common current prediction |
+| ----------------- | ----: | ---------------: | ------------------------------ |
+| CP850             |    18 |                0 | windows-1252                   |
+| CP852             |     6 |                0 | windows-1250                   |
+| CP949             |     6 |                0 | EUC-KR                         |
+| IBM855            |    12 |                0 | windows-1252                   |
+| IBM866            |     6 |                0 | windows-1252                   |
+| ISO-8859-10       |     6 |                0 | ISO-8859-1                     |
+| ISO-8859-13       |     6 |                0 | windows-1257                   |
+| ISO-8859-14       |     6 |                0 | ISO-8859-1                     |
+| ISO-8859-15       |    18 |                0 | ISO-8859-1                     |
+| ISO-8859-16       |     6 |                0 | ISO-8859-2                     |
+| ISO-8859-3        |     6 |                0 | ISO-8859-1                     |
+| ISO-8859-4        |     6 |                0 | windows-1257                   |
+| KOI8-U            |     6 |                0 | KOI8-R                         |
+| macintosh         |    18 |                0 | windows-1252                   |
+| x-mac-cyrillic    |    12 |                0 | windows-1251                   |
+
+The most common predictions above are useful collision targets, but do not
+describe every document. Notably, weak IBM855 input can also be classified as
+Shift_JIS or windows-1255.
+
+## Public naming decisions
+
+Existing names remain exactly as declared by `EncodingName`; the migration must
+not silently recase or rename them. The new SBCS canonical result names are:
+
+```text
+KOI8-U
+IBM866
+IBM855
+macintosh
+x-mac-cyrillic
+ISO-8859-3
+ISO-8859-4
+ISO-8859-10
+ISO-8859-13
+ISO-8859-14
+ISO-8859-15
+ISO-8859-16
+CP850
+CP852
+```
+
+CP949 is reserved as the canonical future result name for the separate
+multibyte follow-up.
+
+The current public API does not accept encoding-name filters and does not expose
+alias normalization. Consequently, the initial migration adds only these
+canonical `EncodingName` values. Iconv/compiler input labels such as `CP866`,
+`CP855`, `MACINTOSH`, and `MAC-CYRILLIC` are implementation aliases and must not
+be returned by `detect` or `analyse`.
+
+## Migration scope decision
+
+All existing SBCS recognizers will move to generated models. The legacy byte
+maps and n-gram tables in `src/encoding/sbcs.ts` will not remain as a parallel
+model system. Existing encodings must first be added to the corpus/model inputs,
+generated, and validated before their hard-coded tables are removed.
+
+## Known pre-migration limitations
+
+- Generated SBCS models are not consumed or registered by the detector.
+- New encodings therefore lose to the closest existing recognizer.
+- Confidence values are not yet calibrated between generated SBCS and existing
+  UTF, ISO-2022, or MBCS recognizers.
+- The existing test suite has eight explicitly pending tests.
+- CP949 is represented in the corpus but intentionally excluded from the SBCS
+  model compiler and requires separate MBCS work.

README.md

@@ -41,8 +41,8 @@ Returned value is an array of objects sorted by confidence value in descending o
 
 ```javascript
 [
-  { confidence: 90, name: 'UTF-8' },
-  { confidence: 20, name: 'windows-1252', lang: 'fr' },
+  { confidence: 0.9, name: 'UTF-8' },
+  { confidence: 0.2, name: 'windows-1252', lang: 'fr' },
 ];
 ```
 
@@ -89,16 +89,29 @@ Note on [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncod
 - ISO-2022-CN
 - ISO-8859-1
 - ISO-8859-2
+- ISO-8859-3
+- ISO-8859-4
 - ISO-8859-5
 - ISO-8859-6
 - ISO-8859-7
 - ISO-8859-8
 - ISO-8859-9
+- ISO-8859-10
+- ISO-8859-13
+- ISO-8859-14
+- ISO-8859-15
+- ISO-8859-16
 - Big5
+- CP850
+- CP852
 - EUC-JP
 - EUC-KR
 - GB18030
+- IBM855
+- IBM866
 - KOI8-R
+- KOI8-U
+- macintosh
 - Shift_JIS
 - UTF-8
 - UTF-16BE
@@ -115,6 +128,7 @@ Note on [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncod
 - windows-1256
 - windows-1257
 - windows-1258
+- x-mac-cyrillic
 
 Currently only these encodings are supported.
 
@@ -129,11 +143,4 @@ Yes. Type definitions are included.
 
 ### TODO
 
-- [ ] KOI8-U for Ukrainian
-- [ ] IBM866 for DOS Cyrillic
-- [ ] macintosh and x-mac-cyrillic
 - [ ] CP949 / UHC support within the Korean recognizer
-- [ ] ISO-8859-3, ISO-8859-4
-- [ ] ISO-8859-10, ISO-8859-13
-- [ ] ISO-8859-14, ISO-8859-15, ISO-8859-16
-- [ ] DOS CP850, CP852, CP855

corpus/README.md

@@ -4,8 +4,9 @@ This directory contains a deterministic seed corpus for developing additional
 encoding recognisers. The prose is synthetic and should not be treated as a
 replacement for a large, independently sourced natural-language corpus.
 
-`sources.json` stores UTF-8 originals. `manifest.json` maps languages to iconv
-encodings. Generated byte samples and statistics live under `generated/`.
+`sources.json` stores UTF-8 originals. Held-out material is kept separately in
+`test-sources.json`. `manifest.json` maps languages to iconv encodings.
+Generated byte samples and statistics live under `generated/`.
 
 Build and verify the corpus with:
 
@@ -17,8 +18,21 @@ npm run models:evaluate
 npm run models:verify
 ```
 
-The build script generates the corpus and performs an exact iconv round trip for
-every document. The check script independently rebuilds into a temporary
+Benchmark all inputs or isolate a stable recognizer family workload with:
+
+```sh
+npm run benchmark
+npm run benchmark -- --family=singlebyte
+npm run benchmark -- --family=multibyte
+```
+
+Benchmark output includes the input bytes per iteration and a workload SHA-256.
+Performance results are comparable only when the family, split, and workload
+hash match.
+
+The build script generates the corpus and performs an iconv round trip for every
+document, allowing canonically equivalent Unicode composition where an encoding
+such as windows-1258 stores combining marks. The check script independently rebuilds into a temporary
 directory and byte-compares the result with `generated/`. Four documents per
 language are assigned to training, one to validation, and one independent
 document to testing. Statistics are calculated from training documents only.
@@ -34,9 +48,20 @@ The model compiler derives each single-byte encoding's case-folding byte map
 through iconv, applies the detector's whitespace normalization, and emits 64
 sorted detector-ready trigrams into `src/encoding/models/generated.ts`. It also
 learns Laplace-smoothed high-byte distributions from the training split. The
-evaluator uses byte likelihood to rank candidates whose trigram hit rates are
-statistically competitive at a 95% margin. `models:evaluate` reports every
-held-out test document, while `models:verify` checks reproducibility and fails if
-any encoding or language result is incorrect. CP949 is excluded because its
-multibyte recogniser requires character frequency data rather than byte
-trigrams.
+compiler evaluator and library runtime share the normalization, trigram,
+byte-likelihood, confidence, candidate-prefiltering, and tie-breaking
+implementation in `src/encoding/sbcs-scoring.ts`. Candidates whose trigram hit
+rates are statistically competitive at a 95% margin are ranked by byte
+likelihood and deterministic encoding-family preferences. `models:evaluate`
+reports every held-out test document, while `models:verify` checks
+reproducibility and fails if any encoding or language result is incorrect. A
+different encoding label is accepted only when both encodings decode the actual
+held-out bytes identically; the report distinguishes these byte-equivalent
+results from exact matches.
+
+For multibyte encodings, the structural parsers identify valid raw code units
+and the compiler selects the 128 most frequent units from the training split.
+Runtime confidence is the structurally valid proportion of input code units
+covered by that generated model. Shift_JIS, EUC-JP, Big5, EUC-KR, and GB18030
+use this path. CP949 remains in the corpus but is excluded from the runtime
+models until its extended byte sequences have a dedicated parser.

corpus/generated/Big5/zh/test/community-garden.bin

@@ -0,0 +1,3 @@
+���Ϫ��
+
+�F�~�̧�Ű|�l��y�������A�ؤU�A��B����M�p��C�Ĥl�̵e�F�m��еP�A�U�a�C�g�@�_���Ӫ�����C

corpus/generated/Big5/zh/train/city-morning.bin

@@ -0,0 +1,3 @@
+����������
+
+�����C�C���ӡA�����E���F�W�Z���H�A�@�ة����}�j���A�Ĥ@�Z���@�T���}�l��L�w�R����D�C

corpus/generated/Big5/zh/train/library.bin

@@ -0,0 +1,3 @@
+�w�R���Ϯ��]
+
+���ϹϮ��]�̥u��½�Ѫ��n���CŪ�̴M����v�B��ǩM�Ȧ�譱���ѡA�޲z�����U�ǥͬd���ơC

corpus/generated/Big5/zh/train/market.bin

@@ -0,0 +1,3 @@
+�P����������
+
+�s���W�������\���s�A����B���G�B���e�M�ѥ]�C�U�ȦV�A���߰ݦ����A�å�y�a�`�檺���k�C

corpus/generated/Big5/zh/train/workshop.bin

@@ -0,0 +1,3 @@
+�Ʀ�u�@��
+
+�@�Ӥp�ζ����b�s�@�ժ��]���n�C�]�p�v��z�����A�}�o���ˬd�{���A�s�謰���P�~�֪����[�̼��g�M���������C

corpus/generated/Big5/zh/validation/river-trip.bin

@@ -0,0 +1,3 @@
+�e��Ȧ�
+
+�B�̱ͭa�ۦa�ϩM������L��L�C���F�ܤ[�H��A�L�̧��b�w�R���e��𮧡Ať�ıߪ�����C

corpus/generated/EUC-JP/ja/test/community-garden.bin

@@ -0,0 +1,3 @@
+�ϰ����
+
+���οͤ����϶����Ƥ��������������֤���ڤ�㤤�ڤ򿢤������Ҥɤ⤿���Ͽ����䤫�ʻ����ꡢ��²���轵��ʪ�˿���롣

corpus/generated/EUC-JP/ja/train/city-morning.bin

@@ -0,0 +1,3 @@
+����ī
+
+ī�γ��Ϥ�ä����ܤ�Фޤ����ؤˤ��̶Ф���ͤ����ޤꡢ����Ź����򳫤����ǽ�ΥХ����Ť����̤������Ϥ�롣

corpus/generated/EUC-JP/ja/train/library.bin

@@ -0,0 +1,3 @@
+�Ť��ʿ޽��
+
+�ϰ�ο޽�ۤǤ��ܤΥڡ�����᤯�벻��ʹ�����롣�ɼԤ���ˤ�ʳؤ�ι�Ԥ��ܤ�õ�����ʽ�ϳ�����Ĵ��ʪ���������

corpus/generated/EUC-JP/ja/train/market.bin

@@ -0,0 +1,3 @@
+�������λԾ�
+
+����λԾ�ˤϿ�������ڡ���ʪ��˪̪���Ƥ����ƤΥѥ��¤֡��㤤ʪ�Ҥ����Ȥȵ���������ˤĤ����ä���

corpus/generated/EUC-JP/ja/train/workshop.bin

@@ -0,0 +1,3 @@
+�ǥ����빩˼
+
+�����ʥ����ब��ʪ�ۤΰ�����äƤ��롣�߷׼Ԥ����̤���������ȯ�Ԥ������ɤ��ǧ�����Խ��Ԥ�ʬ����䤹��������񤯡�

corpus/generated/EUC-JP/ja/validation/river-trip.bin

@@ -0,0 +1,3 @@
+��ؤξ�ι��
+
+ͧ�ͤ������ϿޤȲ������������äƿ����⤤����Ĺ������θ塢�Ť������դǵ٤ߤʤ���ͼ����Ļ������ʹ������

corpus/generated/EUC-KR/ko/test/community-garden.bin

@@ -0,0 +1,3 @@
+���� ���� ����
+
+�ֹε��� ��� �ִ� ������ �����ϰ� �ɰ� ä��, � ������ �ɾ���. ���̵��� �˷ϴ޷��� �̸�ǥ�� ������� �̿����� ���� �� ���� �ְ� ������ �����Ѵ�.

corpus/generated/EUC-KR/ko/train/city-morning.bin

@@ -0,0 +1,3 @@
+������ ��ħ
+
+���ô� õõ�� ��ħ�� �´´�. ī�䰡 ���� ���� ������ ù ������ �����ϸ� ������� �޴���ȭ�� ������ �д´�. �ǹ� ���̷� ���� �޺��� ������.

corpus/generated/EUC-KR/ko/train/library.bin

@@ -0,0 +1,3 @@
+������ ������
+
+���� ���������� å�� �ѱ�� �Ҹ��� �����ϰ� �鸰��. ���ڵ��� ����� ����, ���࿡ ���� å�� ã�´�. �缭�� �л����� ���� ���� ���ø� �غ��Ѵ�.

corpus/generated/EUC-KR/ko/train/market.bin

@@ -0,0 +1,3 @@
+����� ����
+
+���忡 ���� ���忡�� ����� ��, ġ��� �� ���� ���� ���δ�. �մԵ��� ä�Ҹ� ������ ��ο� ���� �̾߱⸦ ������ ������ ���� �丮���� �Ұ��Ѵ�.

corpus/generated/EUC-KR/ko/train/workshop.bin

@@ -0,0 +1,3 @@
+������ �۾���
+
+���� ���� �ڹ��� �ȳ� ���α׷��� �����. �����̳ʴ� �޴��� �����ϰ� �����ڴ� �ڵ带 Ȯ���ϸ� �����ڴ� �湮������ ���� ������ ����. Ȯ�� ���� �۰� �ڵ� �˻��Ѵ�.

corpus/generated/EUC-KR/ko/validation/river-trip.bin

@@ -0,0 +1,3 @@
+������ ���� ����
+
+�Ͽ��Ͽ� ģ������ ���� ���� ������ ������ �ɾ��. ������ ������ ��, �����⸦ ì���. �� ��å �ڿ��� ������ �ɾ� ���� ���Ҹ��� �����.

corpus/generated/GB18030/zh/test/community-garden.bin

@@ -0,0 +1,3 @@
+��^���@
+
+���ӂ��ѿ�Ժ�Ӹ���ɾGɫ���@���N���r�����߲˺�С�䡣���ӂ����˲�ɫ���ƣ�����ÿ�Lһ��oֲ�ˮ��