diff --git a/corpus/README.md b/corpus/README.md index 48dce88..d1a1fba 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -18,6 +18,18 @@ npm run models:evaluate npm run models:verify ``` +Benchmark all inputs or isolate a stable recognizer family workload with: + +```sh +npm run benchmark +npm run benchmark -- --family=singlebyte +npm run benchmark -- --family=multibyte +``` + +Benchmark output includes the input bytes per iteration and a workload SHA-256. +Performance results are comparable only when the family, split, and workload +hash match. + The build script generates the corpus and performs an iconv round trip for every document, allowing canonically equivalent Unicode composition where an encoding such as windows-1258 stores combining marks. The check script independently rebuilds into a temporary @@ -45,5 +57,11 @@ reports every held-out test document, while `models:verify` checks reproducibility and fails if any encoding or language result is incorrect. A different encoding label is accepted only when both encodings decode the actual held-out bytes identically; the report distinguishes these byte-equivalent -results from exact matches. CP949 is excluded because its multibyte recogniser -requires character frequency data rather than byte trigrams. +results from exact matches. + +For multibyte encodings, the structural parsers identify valid raw code units +and the compiler selects the 128 most frequent units from the training split. +Runtime confidence is the structurally valid proportion of input code units +covered by that generated model. Shift_JIS, EUC-JP, Big5, EUC-KR, and GB18030 +use this path. CP949 remains in the corpus but is excluded from the runtime +models until its extended byte sequences have a dedicated parser. diff --git a/corpus/generated/Big5/zh/test/community-garden.bin b/corpus/generated/Big5/zh/test/community-garden.bin new file mode 100644 index 0000000..594af6e --- /dev/null +++ b/corpus/generated/Big5/zh/test/community-garden.bin @@ -0,0 +1,3 @@ +社區花園 + +鄰居們把空院子改造成綠色花園，種下鮮花、蔬菜和小樹。孩子們畫了彩色標牌，各家每週一起給植物澆水。 diff --git a/corpus/generated/Big5/zh/train/city-morning.bin b/corpus/generated/Big5/zh/train/city-morning.bin new file mode 100644 index 0000000..71e4cc7 --- /dev/null +++ b/corpus/generated/Big5/zh/train/city-morning.bin @@ -0,0 +1,3 @@ +城市的早晨 + +城市慢慢醒來，車站聚集了上班的人，咖啡店打開大門，第一班公共汽車開始穿過安靜的街道。 diff --git a/corpus/generated/Big5/zh/train/library.bin b/corpus/generated/Big5/zh/train/library.bin new file mode 100644 index 0000000..82396af --- /dev/null +++ b/corpus/generated/Big5/zh/train/library.bin @@ -0,0 +1,3 @@ +安靜的圖書館 + +社區圖書館裡只有翻書的聲音。讀者尋找歷史、科學和旅行方面的書，管理員幫助學生查找資料。 diff --git a/corpus/generated/Big5/zh/train/market.bin b/corpus/generated/Big5/zh/train/market.bin new file mode 100644 index 0000000..97ee3e1 --- /dev/null +++ b/corpus/generated/Big5/zh/train/market.bin @@ -0,0 +1,3 @@ +星期六的市場 + +廣場上的市場擺滿新鮮蔬菜、水果、蜂蜜和麵包。顧客向農民詢問收成，並交流家常菜的做法。 diff --git a/corpus/generated/Big5/zh/train/workshop.bin b/corpus/generated/Big5/zh/train/workshop.bin new file mode 100644 index 0000000..a6e6c69 --- /dev/null +++ b/corpus/generated/Big5/zh/train/workshop.bin @@ -0,0 +1,3 @@ +數位工作室 + +一個小團隊正在製作博物館指南。設計師整理頁面，開發者檢查程式，編輯為不同年齡的參觀者撰寫清楚的說明。 diff --git a/corpus/generated/Big5/zh/validation/river-trip.bin b/corpus/generated/Big5/zh/validation/river-trip.bin new file mode 100644 index 0000000..c52a8b5 --- /dev/null +++ b/corpus/generated/Big5/zh/validation/river-trip.bin @@ -0,0 +1,3 @@ +河邊旅行 + +朋友們帶著地圖和熱茶穿過樹林。走了很久以後，他們坐在安靜的河邊休息，聽傍晚的鳥鳴。 diff --git a/corpus/generated/EUC-JP/ja/test/community-garden.bin b/corpus/generated/EUC-JP/ja/test/community-garden.bin new file mode 100644 index 0000000..d9ad4af --- /dev/null +++ b/corpus/generated/EUC-JP/ja/test/community-garden.bin @@ -0,0 +1,3 @@ +譁啗及瓔 + +嗤赭及諦凶切反塢中化中凶瓔毛幫尹﹜累支沓窪支潦中枆毛縈尹凶﹝閡升手凶切反縉蟒支井卅誑毛綜曰﹜笮簡反咍蔥縈坁卞踹毛支月﹝ diff --git a/corpus/generated/EUC-JP/ja/train/city-morning.bin b/corpus/generated/EUC-JP/ja/train/city-morning.bin new file mode 100644 index 0000000..72a9c8a --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/city-morning.bin @@ -0,0 +1,3 @@ +章及蘋 + +蘋及章反斗勻仁曰杴毛創引允﹝斛卞反騷剷允月諦互螂引曰﹜脾辭饕互滮繷咫情ⅡХ擗峊苭馱珊聾咫岉怳磥臗鬗篕洃嶀諢 diff --git a/corpus/generated/EUC-JP/ja/train/library.bin b/corpus/generated/EUC-JP/ja/train/library.bin new file mode 100644 index 0000000..f15e8f5 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/library.bin @@ -0,0 +1,3 @@ +斃井卅豭踏欺 + +譁啗及豭踏欺匹反呿及矢□斥毛戶仁月祥互妏仇尹月﹝づ樊反昳銖支粗喀支庣墊及呿毛繭仄﹜銘踏反喀戲及譬屯坁毛澎鰻丹﹝ diff --git a/corpus/generated/EUC-JP/ja/train/market.bin b/corpus/generated/EUC-JP/ja/train/market.bin new file mode 100644 index 0000000..4169791 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/market.bin @@ -0,0 +1,3 @@ +蘿豖ゥ及閩樺 + +僮樺及閩樺卞反蕙蟒卅沓窪﹜絆坁﹜邞怐﹜嬴五凶化及由件互岈少﹝ъ中坁菸反Ш笮午筐濡及恔咥卞勾中化狪允﹝ diff --git a/corpus/generated/EUC-JP/ja/train/workshop.bin b/corpus/generated/EUC-JP/ja/train/workshop.bin new file mode 100644 index 0000000..20c2244 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/train/workshop.bin @@ -0,0 +1,3 @@ +犯斥正伙鼎佷 + +凝今卅民□丞互⑥坁欺及副ま毛綜勻化中月﹝澀煌樊互脰昍毛幫尹﹜釩紡啎洛部憧氻繷恉坐楚〨埣蜈啎畎洶咫磥鉹馱分獑壑羷韙砥 diff --git a/corpus/generated/EUC-JP/ja/validation/river-trip.bin b/corpus/generated/EUC-JP/ja/validation/river-trip.bin new file mode 100644 index 0000000..a0faed9 --- /dev/null +++ b/corpus/generated/EUC-JP/ja/validation/river-trip.bin @@ -0,0 +1,3 @@ +燠尺及凝庣墊 + +竻諦凶切反譁豭午硃井中云辭毛儅勻化蕨毛忷中凶﹝贏中辣忷及詨﹜斃井卅燠彶匹菱心卅互日芞杅及躉及擘毛妏中凶﹝ diff --git a/corpus/generated/EUC-KR/ko/test/community-garden.bin b/corpus/generated/EUC-KR/ko/test/community-garden.bin new file mode 100644 index 0000000..d5f8130 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/test/community-garden.bin @@ -0,0 +1,3 @@ +葆擊 奢翕 薑錳 + +輿團菟擎 綠橫 氈湍 葆渡擊 薑葬ж堅 粒婁 瓣模, 橫萼 釭鼠蒂 褕歷棻. 嬴檜菟擎 憲煙殖煙и 檜葷ル蒂 虜菟歷堅 檜醒菟擎 衙輿 賅罹 僭擊 輿堅 紫掘蒂 薑絲и棻. diff --git a/corpus/generated/EUC-KR/ko/train/city-morning.bin b/corpus/generated/EUC-KR/ko/train/city-morning.bin new file mode 100644 index 0000000..d2976f1 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/city-morning.bin @@ -0,0 +1,3 @@ +紫衛曖 嬴藹 + +紫衛朝 繭繭 嬴藹擊 蜃朝棻. 蘋む陛 僥擊 翮堅 幗蝶陛 羅 遴ч擊 衛濛ж貊 餌塋菟擎 瑒蹀風 景蝶蒂 檗朝棻. 勒僭 餌檜煎 嫩擎 х維檜 廓霞棻. diff --git a/corpus/generated/EUC-KR/ko/train/library.bin b/corpus/generated/EUC-KR/ko/train/library.bin new file mode 100644 index 0000000..6c5b886 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/library.bin @@ -0,0 +1,3 @@ +褻辨и 紫憮婦 + +翕啻 紫憮婦縑朝 疇濰 剩晦朝 模葬陛 濤濤ж啪 菟萼棻. 絮濠菟擎 羲餌諦 婁з, 罹ч縑 婦и 疇擊 瓊朝棻. 餌憮朝 з儅菟擊 嬪и 濛擎 瞪衛蒂 遽綠и棻. diff --git a/corpus/generated/EUC-KR/ko/train/market.bin b/corpus/generated/EUC-KR/ko/train/market.bin new file mode 100644 index 0000000..3789568 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/market.bin @@ -0,0 +1,3 @@ +饜蹂橾 衛濰 + +惜濰縑 翮萼 衛濰縑朝 餌婁諦 羚, 纂鍔諦 鬥 掘遴 誧檜 場檣棻. 槳椒菟擎 瓣模蒂 堅腦堅 堯睡諦 啗瞰 檜撿晦蒂 釭援貊 餵縑憮 虜萇 蹂葬徹擊 模偃и棻. diff --git a/corpus/generated/EUC-KR/ko/train/workshop.bin b/corpus/generated/EUC-KR/ko/train/workshop.bin new file mode 100644 index 0000000..c1ed449 --- /dev/null +++ b/corpus/generated/EUC-KR/ko/train/workshop.bin @@ -0,0 +1,3 @@ +蛤雖瓚 濛機褒 + +濛擎 ぜ檜 夢僭婦 寰頂 Щ煎斜極擊 虜萇棻. 蛤濠檜傘朝 詭景蒂 薑葬ж堅 偃嫦濠朝 囀萄蒂 挫恉牉 ら餵濠朝 寞僥偌縑啪 蓮遴 撲貲擊 噹棻. 挫 旋濠 旋婁 濠紫 匐餌и棻. diff --git a/corpus/generated/EUC-KR/ko/validation/river-trip.bin b/corpus/generated/EUC-KR/ko/validation/river-trip.bin new file mode 100644 index 0000000..c8a9dce --- /dev/null +++ b/corpus/generated/EUC-KR/ko/validation/river-trip.bin @@ -0,0 +1,3 @@ +鬼陛煎 集陪 罹ч + +橾蹂橾縑 耀掘菟擎 紫衛 夤曖 褻辨и 鬼戲煎 勘橫鬲棻. 雖紫諦 評嗆и 離, 餌霞晦蒂 穫啣棻. 曹 骯疇 菴縑朝 鬼陛縑 導嬴 盪喔 億模葬蒂 菟歷棻. diff --git a/corpus/generated/GB18030/zh/test/community-garden.bin b/corpus/generated/GB18030/zh/test/community-garden.bin new file mode 100644 index 0000000..ab73969 --- /dev/null +++ b/corpus/generated/GB18030/zh/test/community-garden.bin @@ -0,0 +1,3 @@ +扦^豪@ + +懈參諾埏赽蜊婖傖鄭伎豪@ㄛ意狟鰎豪﹜忣粕睿苤銦ㄩＩ賸粗伎侜ゞ爰鷐疰貓L珨れ緻眵昜粕恣 diff --git a/corpus/generated/GB18030/zh/train/city-morning.bin b/corpus/generated/GB18030/zh/train/city-morning.bin new file mode 100644 index 0000000..9759d58 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/city-morning.bin @@ -0,0 +1,3 @@ +傑庈腔婌鹵 + +傑庈鞣鞣倳瞨眉桴擄摩賸奻啤腔芄玷孚虛湖嶱湮嬡ㄛ菴珨啤鼠僕イ嶱宎援綎假髺腔誰耋﹝ diff --git a/corpus/generated/GB18030/zh/train/library.bin b/corpus/generated/GB18030/zh/train/library.bin new file mode 100644 index 0000000..01c0725 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/library.bin @@ -0,0 +1,3 @@ +假髺腔D斸^ + +扦^D斸^孬硐衄楹礸鐃秞﹝莮氪孔v妢﹜褪W睿藏俴源醱腔齮牲傱T邳W汜脤梑揃蹋﹝ diff --git a/corpus/generated/GB18030/zh/train/market.bin b/corpus/generated/GB18030/zh/train/market.bin new file mode 100644 index 0000000..a2c8003 --- /dev/null +++ b/corpus/generated/GB18030/zh/train/market.bin @@ -0,0 +1,3 @@ +陎ぶ鞠腔庈 + +V鹺炵騫[M陔鰎忣粕﹜阨彆﹜瑚蹲睿I婦﹝諦砃煢鏍彶傖ㄛK蝠霜模都粕腔酕楊﹝ diff --git a/corpus/generated/GB18030/zh/train/workshop.bin b/corpus/generated/GB18030/zh/train/workshop.bin new file mode 100644 index 0000000..819d65e --- /dev/null +++ b/corpus/generated/GB18030/zh/train/workshop.bin @@ -0,0 +1,3 @@ +舒遜尤鼳 + +珨苤F淏婓庨釬痔昜禭硌鰍﹝偞淕燴醱ㄛ嶱追氪z脤最宒ㄛ斢銓珀蕡g腔Ｅ^氪蚴ь奠腔挲隴﹝ diff --git a/corpus/generated/GB18030/zh/validation/river-trip.bin b/corpus/generated/GB18030/zh/validation/river-trip.bin new file mode 100644 index 0000000..fe7365d --- /dev/null +++ b/corpus/generated/GB18030/zh/validation/river-trip.bin @@ -0,0 +1,3 @@ +碩藏俴 + +攬衭快礸D睿幘韐拋^靾痋軗賸竭壅眕摽ㄛ坻釴婓假髺腔碩倎洘ㄛ奢俀腔讋頀﹝ diff --git a/corpus/generated/Shift_JIS/ja/test/community-garden.bin b/corpus/generated/Shift_JIS/ja/test/community-garden.bin new file mode 100644 index 0000000..e110411 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/test/community-garden.bin @@ -0,0 +1,3 @@ +n + +lAABqFNDATAB diff --git a/corpus/generated/Shift_JIS/ja/train/city-morning.bin b/corpus/generated/Shift_JIS/ja/train/city-morning.bin new file mode 100644 index 0000000..f31111e --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/city-morning.bin @@ -0,0 +1,3 @@ +X + +XoBwlWAiXJAoXnB diff --git a/corpus/generated/Shift_JIS/ja/train/library.bin b/corpus/generated/Shift_JIS/ja/train/library.bin new file mode 100644 index 0000000..258556e --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/library.bin @@ -0,0 +1,3 @@ +} + +n}{y[WBjws{TAiw`B diff --git a/corpus/generated/Shift_JIS/ja/train/market.bin b/corpus/generated/Shift_JIS/ja/train/market.bin new file mode 100644 index 0000000..a71af01 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/market.bin @@ -0,0 +1,3 @@ +yjs + +LsVNAAIApBq_GbB diff --git a/corpus/generated/Shift_JIS/ja/train/workshop.bin b/corpus/generated/Shift_JIS/ja/train/workshop.bin new file mode 100644 index 0000000..8f8126c --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/train/workshop.bin @@ -0,0 +1,3 @@ +fW^H[ + +`[BvAJR[hmFAWB diff --git a/corpus/generated/Shift_JIS/ja/validation/river-trip.bin b/corpus/generated/Shift_JIS/ja/validation/river-trip.bin new file mode 100644 index 0000000..2724ab8 --- /dev/null +++ b/corpus/generated/Shift_JIS/ja/validation/river-trip.bin @@ -0,0 +1,3 @@ +s + +Fln}XBUAx[B diff --git a/corpus/generated/index.json b/corpus/generated/index.json index 136b1b1..7885b13 100644 --- a/corpus/generated/index.json +++ b/corpus/generated/index.json @@ -1,4 +1,76 @@ [ + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "test", + "document": "community-garden", + "path": "Big5/zh/test/community-garden.bin", + "bytes": 107, + "highBytes": 79, + "uniqueHighBytes": 39, + "sha256": "4602b4204cff37f57af8cc21fa4751c6e33e56c4078927b211a14e00657fc8bb" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "city-morning", + "path": "Big5/zh/train/city-morning.bin", + "bytes": 97, + "highBytes": 70, + "uniqueHighBytes": 33, + "sha256": "f141f927f2a8b8e6540a117ccee6678e8f4db6f81cd5c0ebad699de8e9d05b7c" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "library", + "path": "Big5/zh/train/library.bin", + "bytes": 101, + "highBytes": 82, + "uniqueHighBytes": 40, + "sha256": "b20e84ac072e488aa6114e2ad862b39d127774244f26304e9fdf9716c9937ac9" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "market", + "path": "Big5/zh/train/market.bin", + "bytes": 99, + "highBytes": 74, + "uniqueHighBytes": 35, + "sha256": "3fa6ec765b1c5ca517ec35b276defa25d9505cfb06d89f5b8bc09079165ac75b" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "train", + "document": "workshop", + "path": "Big5/zh/train/workshop.bin", + "bytes": 113, + "highBytes": 83, + "uniqueHighBytes": 42, + "sha256": "0673960c008a34a445c94870c643cf40e55beaec03c290bab0dcbaa86c52b128" + }, + { + "encoding": "Big5", + "iconv": "BIG5", + "language": "zh", + "split": "validation", + "document": "river-trip", + "path": "Big5/zh/validation/river-trip.bin", + "bytes": 93, + "highBytes": 71, + "uniqueHighBytes": 39, + "sha256": "d23ba0d2a924cfc8881a734d32b20c591791610270bb5fe0c5b44d2aa8257827" + }, { "encoding": "CP850", "iconv": "CP850", @@ -359,6 +431,222 @@ "uniqueHighBytes": 56, "sha256": "f6399d3705fc3e6c2a6427516ce6c8b9585a0a1e035b53a9efb7ef0811dd6a00" }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "test", + "document": "community-garden", + "path": "EUC-JP/ja/test/community-garden.bin", + "bytes": 129, + "highBytes": 126, + "uniqueHighBytes": 48, + "sha256": "6654d80ff3f372a76c64cff40e76361bd50fddfad6745d9a39f0b265bebbec27" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "city-morning", + "path": "EUC-JP/ja/train/city-morning.bin", + "bytes": 117, + "highBytes": 114, + "uniqueHighBytes": 44, + "sha256": "5c1bc169f52cbbc621feafdd6956fa4be9ed66211a385da85e1364ad0cb9c0fe" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "library", + "path": "EUC-JP/ja/train/library.bin", + "bytes": 125, + "highBytes": 122, + "uniqueHighBytes": 49, + "sha256": "9b8da9e80124276f5d3a46c82fac0bb35489164b972e529f0ee03df6568e1549" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "market", + "path": "EUC-JP/ja/train/market.bin", + "bytes": 115, + "highBytes": 112, + "uniqueHighBytes": 45, + "sha256": "861116732268050cb5c7bdc19c0ceff1aa1fb80a99f172a95f0926a5aaafd9a2" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "train", + "document": "workshop", + "path": "EUC-JP/ja/train/workshop.bin", + "bytes": 131, + "highBytes": 128, + "uniqueHighBytes": 50, + "sha256": "1e952f775f77207dbf5faed31865416068b03f82009e6e86b0517cf564fc1d68" + }, + { + "encoding": "EUC-JP", + "iconv": "EUC-JP", + "language": "ja", + "split": "validation", + "document": "river-trip", + "path": "EUC-JP/ja/validation/river-trip.bin", + "bytes": 119, + "highBytes": 116, + "uniqueHighBytes": 43, + "sha256": "b09785c8f0852a1239d2283d7fc8d531a85fa57443894bd9ec14a8f9cd364103" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "test", + "document": "community-garden", + "path": "EUC-KR/ko/test/community-garden.bin", + "bytes": 166, + "highBytes": 138, + "uniqueHighBytes": 49, + "sha256": "15d58dc224354498565689b014ef393a0c6d9bcf0d536235997785c0c864e336" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "city-morning", + "path": "EUC-KR/ko/train/city-morning.bin", + "bytes": 152, + "highBytes": 126, + "uniqueHighBytes": 45, + "sha256": "54eadc94abafcb5446f54f61029b48a6f7d080fbd3f80219416537fbcf469ee9" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "library", + "path": "EUC-KR/ko/train/library.bin", + "bytes": 157, + "highBytes": 130, + "uniqueHighBytes": 48, + "sha256": "1aca0feb504c558b985ec91d05a54eb6c61f06399cb276e82e08551d4dffbeac" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "market", + "path": "EUC-KR/ko/train/market.bin", + "bytes": 155, + "highBytes": 128, + "uniqueHighBytes": 54, + "sha256": "2de80f2cde96c8a97d8cca91d0d85e47ec37d8362458829797a2576c9f11a880" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "train", + "document": "workshop", + "path": "EUC-KR/ko/train/workshop.bin", + "bytes": 176, + "highBytes": 148, + "uniqueHighBytes": 53, + "sha256": "e9f1924b0b9e410049817e0355f167d012b3a1b24ee91074acd08306102a65a6" + }, + { + "encoding": "EUC-KR", + "iconv": "EUC-KR", + "language": "ko", + "split": "validation", + "document": "river-trip", + "path": "EUC-KR/ko/validation/river-trip.bin", + "bytes": 154, + "highBytes": 126, + "uniqueHighBytes": 56, + "sha256": "f6399d3705fc3e6c2a6427516ce6c8b9585a0a1e035b53a9efb7ef0811dd6a00" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "test", + "document": "community-garden", + "path": "GB18030/zh/test/community-garden.bin", + "bytes": 107, + "highBytes": 96, + "uniqueHighBytes": 55, + "sha256": "0b2447d40ee94fc0f54193e4a0657fb7d524db9a1456dfbf161db4e2d66a238f" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "city-morning", + "path": "GB18030/zh/train/city-morning.bin", + "bytes": 97, + "highBytes": 89, + "uniqueHighBytes": 52, + "sha256": "f831e80d10c0cc072648d19ea815faa0ece4ed763cea6925b18f38e487d8b2ea" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "library", + "path": "GB18030/zh/train/library.bin", + "bytes": 101, + "highBytes": 85, + "uniqueHighBytes": 51, + "sha256": "1e0c934cfe5346ae0e1dd54f7355e8de90fcb47013d9edbd842db64f510abad8" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "market", + "path": "GB18030/zh/train/market.bin", + "bytes": 99, + "highBytes": 89, + "uniqueHighBytes": 56, + "sha256": "4b7df06fad0d5ad72c7c3aea20f843409d5176839813ade77ed773537863918f" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "train", + "document": "workshop", + "path": "GB18030/zh/train/workshop.bin", + "bytes": 113, + "highBytes": 100, + "uniqueHighBytes": 61, + "sha256": "3209cbd7a69500b9825d7226b0065c9a7ae6206b113608c94b7108a89dca4a11" + }, + { + "encoding": "GB18030", + "iconv": "GB18030", + "language": "zh", + "split": "validation", + "document": "river-trip", + "path": "GB18030/zh/validation/river-trip.bin", + "bytes": 93, + "highBytes": 85, + "uniqueHighBytes": 48, + "sha256": "2d4ab1b56be3899eb86a28c8f46ccecb392e4e39ecdba918ae0e4cc2ed4f94ab" + }, { "encoding": "IBM855", "iconv": "CP855", @@ -2951,6 +3239,78 @@ "uniqueHighBytes": 2, "sha256": "f1de58ca10bb5f70ac7da15bf7b8f21d388143fa5cb161d4a76de84cc15939f8" }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "test", + "document": "community-garden", + "path": "Shift_JIS/ja/test/community-garden.bin", + "bytes": 129, + "highBytes": 113, + "uniqueHighBytes": 43, + "sha256": "a58ecbafb06d5f88d10f2d056a96eb0e826d2dfe50360c9da6f591ece6ec45e0" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "city-morning", + "path": "Shift_JIS/ja/train/city-morning.bin", + "bytes": 117, + "highBytes": 98, + "uniqueHighBytes": 37, + "sha256": "7848fc97e017d4eebb1dcc1ad7755a035330bb9fd5cc01fbe3df7db505745e34" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "library", + "path": "Shift_JIS/ja/train/library.bin", + "bytes": 125, + "highBytes": 104, + "uniqueHighBytes": 43, + "sha256": "e0d3b6fec3a86651a59ca899d3f9df16529da913ffdd4d4e1b3da9cc0fce6517" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "market", + "path": "Shift_JIS/ja/train/market.bin", + "bytes": 115, + "highBytes": 94, + "uniqueHighBytes": 39, + "sha256": "2c442382b1bc33652f62e4086143582114db0e2adde38f7cda22cf94e29c9f8b" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "train", + "document": "workshop", + "path": "Shift_JIS/ja/train/workshop.bin", + "bytes": 131, + "highBytes": 109, + "uniqueHighBytes": 45, + "sha256": "e78003a71014de82d26e954ca441ce084fd1f07dc39f5f045596d91e776116a2" + }, + { + "encoding": "Shift_JIS", + "iconv": "SHIFT_JIS", + "language": "ja", + "split": "validation", + "document": "river-trip", + "path": "Shift_JIS/ja/validation/river-trip.bin", + "bytes": 119, + "highBytes": 104, + "uniqueHighBytes": 41, + "sha256": "9d7d47ab703aab26e1ecb264cb5cb838224a32f7992bb54af5fe10d27f954809" + }, { "encoding": "windows-1250", "iconv": "WINDOWS-1250", diff --git a/corpus/generated/ngrams.mjs b/corpus/generated/ngrams.mjs index 5b27b94..d6be5ed 100644 --- a/corpus/generated/ngrams.mjs +++ b/corpus/generated/ngrams.mjs @@ -1,4 +1,30 @@ export default [ + { + encoding: 'Big5', + language: 'zh', + documents: 4, + trigrams: [ + 0xa1430a, 0x52aaba, 0x77c052, 0xa141a8, 0xa143c5, 0xa5abb3, 0xa677c0, + 0xaabaa5, 0xaabab0, 0xabb0a5, 0xabb3f5, 0xaed1c0, 0xb0a5ab, 0xb27aad, + 0xb9cfae, 0xbaa5ab, 0xc052aa, 0xcfaed1, 0xd1c05d, 0x0a0aa4, 0x0a0aaa, + 0x0a0aab, 0x0a0abc, 0x0aa440, 0x0aaac0, 0x0aabb0, 0x0abc73, 0x40a854, + 0x40abc7, 0x40add3, 0x40af5a, 0x40b0d8, 0x40b3d5, 0x41a5c1, 0x41a8ae, + 0x41a8c3, 0x41a940, 0x41b2c4, 0x41b67d, 0x41bade, 0x41bd73, 0x41bdad, + 0x42a4f4, 0x42acec, 0x42b8c1, 0x43b35d, 0x43ba43, 0x43bff4, 0x43c555, + 0x43c5aa, 0x44a143, 0x45b6b0, 0x46a457, 0x47a142, 0x48a141, 0x4ca677, + 0x4da7e4, 0x4daec8, 0x4db7a1, 0x4dc4d1, 0x50a67e, 0x50b4c1, 0x54a8ae, + 0x55abc8, 0x55bec7, 0x56b941, 0x57aaba, 0x57af5a, 0x5aa4bd, 0x5aaaba, + 0x5baacc, 0x5cbaa1, 0x5d0a0a, 0x5da143, 0x5dabfc, 0x5dad70, 0x5db8cc, + 0x60b5e6, 0x61b160, 0x62bb73, 0x64a7e4, 0x64b57b, 0x65a94d, 0x67b24d, + 0x6aaaf9, 0x6ba143, 0x6cacef, 0x6ea143, 0x6eadb5, 0x6faacc, 0x70ae76, + 0x70b9ce, 0x73a740, 0x73b3f5, 0x73bfe8, 0x73c241, 0x75a6b3, 0x75a740, + 0x76a142, 0x76bee3, 0x79ae61, 0x7aadb6, 0x7aadfb, 0x7ba6a1, 0x7da46a, + 0x7da96c, 0x7db56f, 0x7ec4d6, 0xa141a9, 0xa141b2, 0xa141b6, 0xa141ba, + 0xa141bd, 0xa142a4, 0xa142ac, 0xa142b8, 0xa143b3, 0xa1a141, 0xa1a9fa, + 0xa1aaba, 0xa1b773, 0xa3a650, 0xa440ad, 0xa440af, 0xa446a4, 0xa448a1, + 0xa457aa, 0xa457af, + ], + }, { encoding: 'CP850', language: 'de', @@ -129,6 +155,84 @@ export default [ 0x20c4a1, 0x20c4ab, ], }, + { + encoding: 'EUC-JP', + language: 'ja', + documents: 4, + trigrams: [ + 0xa1a30a, 0xbcd4a4, 0xa1bca5, 0xa4aba4, 0xa4acca, 0xa4cba4, 0xa4cea5, + 0xa4eba1, 0xa4f2b3, 0xd4a4ac, 0xeba1a3, 0xa4a4a4, 0xa4afa4, 0xa4b7a1, + 0xa4b9a1, 0xa4b9a4, 0xa4c3a4, 0xa4c6a4, 0xa4cebb, 0xa4cec4, 0xa4dea4, + 0xa4e1a4, 0xa4eaa4, 0xaaa1a2, 0xaba4ca, 0xb3b9a4, 0xb4dba4, 0xb7a1a2, + 0xb9a1a3, 0xb9a4ac, 0xbbd4be, 0xbdf1a4, 0xbdf1b4, 0xbeeca4, 0xbfdebd, + 0xc0c5a4, 0xc5a4ab, 0xcba4cf, 0xcbdca4, 0xcebbd4, 0xd4beec, 0xdca4f2, + 0xdebdf1, 0xe2a4f2, 0xf1b4db, 0x0a0ab9, 0x0a0abe, 0x0a0ac3, 0x0a0ac4, + 0x0ab9ad, 0x0abeae, 0x0ac3cf, 0x0ac4ab, 0xa1a2b2, 0xa1a2b3, 0xa1a2b5, + 0xa1a2ba, 0xa1a2bb, 0xa1a2be, 0xa1a2ca, 0xa1a2cb, 0xa1a3b1, 0xa1a3c0, + 0xa1a3c6, 0xa1a3c7, 0xa2b2cc, 0xa2b3ab, 0xa2b5ca, 0xa2bac7, 0xa2bbca, + 0xa2bec6, 0xa2cad4, 0xa2cbaa, 0xa3b1d8, 0xa3c0df, 0xa3c6c9, 0xa3c7e3, + 0xa4a4c0, 0xa4a4c6, 0xa4a4ca, 0xa4a4eb, 0xa4a6a1, 0xa4a8a1, 0xa4a8a4, + 0xa4aca5, 0xa4acb2, 0xa4acbd, 0xa4acc0, 0xa4acc7, 0xa4acc8, 0xa4ada4, + 0xa4afa1, 0xa4b1a1, 0xa4b3a4, 0xa4b5a4, 0xa4bfa4, 0xa4c0e2, 0xa4c4a4, + 0xa4c6cf, 0xa4c7a4, 0xa4c8b5, 0xa4caa5, 0xa4caaa, 0xa4cabf, 0xa4cac4, + 0xa4cacc, 0xa4ceb0, 0xa4ceb3, 0xa4cebf, 0xa4cecb, 0xa4cece, 0xa4cfa4, + 0xa4cfb3, 0xa4cfbf, 0xa4cfc4, 0xa4cfc7, 0xa4cfcb, 0xa4cfce, 0xa4d6a1, + 0xa4d9ca, 0xa4e4a4, 0xa4e4b2, 0xa4e4ce, 0xa4e6a4, 0xa4eaa1, 0xa4eabb, + 0xa4eacc, 0xa4ebb2, + ], + }, + { + encoding: 'EUC-KR', + language: 'ko', + documents: 4, + trigrams: [ + 0xb4d92e, 0xb4c220, 0xc0bb20, 0xd92e20, 0xb8a620, 0xc0ba20, 0x20bbe7, + 0xb5e9c0, 0xbdc3c0, 0xbfcd20, 0xd92e0a, 0x20bdc3, 0xb0a120, 0xb0ed20, + 0xb4c2b4, 0xb8e720, 0xc0cc20, 0xc2b4d9, 0xc7d120, 0xc7d1b4, 0xcd20b0, + 0xd1b4d9, 0xe9c0ba, 0x0a0ab5, 0x20b1db, 0x20b5b5, 0x20b8b8, 0x20bcd2, + 0x20bec6, 0x20bfad, 0x20c0db, 0x20c3a5, 0x20c8ae, 0x2e20b5, 0xa1b4c2, + 0xa5c0bb, 0xa620c1, 0xadb0fc, 0xb0d420, 0xb5b5bc, 0xb5b5bd, 0xb5bcad, + 0xb5bdc3, 0xb7ce20, 0xb8b5e7, 0xb8b8b5, 0xbab8a6, 0xbb20b8, 0xbcadb0, + 0xbcd2b8, 0xbec6c4, 0xbfa120, 0xbfa1b4, 0xbfee20, 0xc0dab4, 0xc0dab5, + 0xc0dbc0, 0xc0e520, 0xc0e5bf, 0xc220c3, 0xc3a5c0, 0xc3c0e5, 0xc6c4a7, + 0xc7cfb0, 0xc7cfb8, 0xc8aec0, 0xcc20b9, 0xcfb8e7, 0xdab4c2, 0xdbc0ba, + 0xe5bfa1, 0x0a0ab1, 0x0a0ac0, 0x0ab1a4, 0x0ab5b5, 0x0ab5bf, 0x0ac0db, + 0x20b0ab, 0x20b0b3, 0x20b0c7, 0x20b0cb, 0x20b0e8, 0x20b0ed, 0x20b0fa, + 0x20b0fc, 0x20b1b8, 0x20b2dc, 0x20b3aa, 0x20b3d1, 0x20b3f3, 0x20b3f5, + 0x20b4ba, 0x20b5b6, 0x20b5e9, 0x20b5f0, 0x20b8c2, 0x20b8de, 0x20b9ae, + 0x20b9da, 0x20b9e0, 0x20b9e6, 0x20b9f6, 0x20b9f8, 0x20bba7, 0x20bcb3, + 0x20bcd5, 0x20bdac, 0x20beb4, 0x20bec8, 0x20bfa9, 0x20bfaa, 0x20bfe4, + 0x20bfee, 0x20c0a7, 0x20c0cc, 0x20c0d0, 0x20c0da, 0x20c0dc, 0x20c0fc, + 0x20c1a4, 0x20c1d8, 0x20c1fd, 0x20c3a3, 0x20c3a4, 0x20c3b5, 0x20c3b9, + 0x20c4a1, 0x20c4ab, + ], + }, + { + encoding: 'GB18030', + language: 'zh', + documents: 4, + trigrams: [ + 0xa1a30a, 0x4495f8, 0x6fb5c4, 0x884495, 0x95f8f0, 0xb0b2ec, 0xb2ec6f, + 0xb3c7ca, 0xb5c4ca, 0xc4cad0, 0xc7cad0, 0xcad088, 0xcfa1a3, 0xd088f6, + 0xec6fb5, 0xf8f05e, 0x0a0a8f, 0x0a0ab3, 0x0a0ac9, 0x0a0ad2, 0x0a8f56, + 0x0ab3c7, 0x0ac9e7, 0x0ad2bb, 0x46eaa0, 0x49b0fc, 0x4bbdbb, 0x4dd0c2, + 0x4fd38b, 0x548ecd, 0x54a3ac, 0x5688f6, 0x57bacd, 0x57c9fa, 0x59c1cf, + 0x5b9d4d, 0x5e0a0a, 0x5e8844, 0x5eb0b2, 0x5ed165, 0x5ed5df, 0x5ed6b8, + 0x5fb06c, 0x5fb4f3, 0x5fcabc, 0x65d6bb, 0x66c3f7, 0x67b5c4, 0x6cd5df, + 0x72c3f1, 0x72cadf, 0x75d7f7, 0x76cab7, 0x78d5df, 0x7ab2e9, 0x80d0a1, + 0x814bbd, 0x81eda3, 0x8280d0, 0x838696, 0x855e88, 0x85a2d3, 0x86548e, + 0x8696ca, 0x87d5be, 0x87e95f, 0x8846ea, 0x88f60a, 0x88f694, 0x88f6c9, + 0x8b8e9f, 0x8b9ee9, 0x8c57ba, 0x8c57c9, 0x8c91c7, 0x8ca4d5, 0x8e9fd5, + 0x8ecdd6, 0x8edd8b, 0x8f5688, 0x91c7e5, 0x93c3e6, 0x945b9d, 0x94b5ce, + 0x95d2f4, 0x95f8a3, 0x95f8b5, 0x96cad5, 0x997ab2, 0x99bfcd, 0x9a76ca, + 0x9d4dd0, 0x9ee9b2, 0x9fd5fb, 0xa0d5fd, 0xa18846, 0xa1a2b7, 0xa1a2bf, + 0xa1a2cb, 0xa1a3d4, 0xa1a3d7, 0xa1a3ee, 0xa2b7e4, 0xa2bfc6, 0xa2cbae, + 0xa2d35e, 0xa3ac81, 0xa3acb5, 0xa3acb9, 0xa3acbe, 0xa3acbf, 0xa3acdc, + 0xa3ace9, 0xa3b2cb, 0xa3d44f, 0xa3d778, 0xa3ee99, 0xa4d5d2, 0xa4d7f7, + 0xa7b7c8, 0xa8a1a3, 0xa9ceef, 0xa9df5e, 0xab8c91, 0xabb9b2, 0xac814b, + 0xacb5da, 0xacb9dc, + ], + }, { encoding: 'IBM855', language: 'ru', @@ -1065,6 +1169,32 @@ export default [ 0x70726f, 0x72656e, ], }, + { + encoding: 'Shift_JIS', + language: 'ja', + documents: 4, + trigrams: [ + 0x81420a, 0x8ed282, 0x815b83, 0x82a982, 0x82aa95, 0x82c982, 0x82cc83, + 0x82e981, 0x82f08a, 0xd282aa, 0xe98142, 0x0a0a92, 0x5882aa, 0x738fea, + 0x7d8f91, 0x82a282, 0x82aa94, 0x82ad82, 0x82b581, 0x82b781, 0x82b782, + 0x82c182, 0x82c482, 0x82cc8e, 0x82cc92, 0x82dc82, 0x82df82, 0x82e882, + 0x8a5882, 0x8ad982, 0x8e738f, 0x8f9182, 0x8f918a, 0x8fea82, 0x907d8f, + 0x90c382, 0x918ad9, 0x967b82, 0xa88141, 0xa982c8, 0xb58141, 0xb78142, + 0xc382a9, 0xc982cd, 0xcc8e73, 0xe082f0, 0x0a0a8d, 0x0a0a8f, 0x0a8d4c, + 0x0a8fac, 0x0a926e, 0x0a92a9, 0x4189ca, 0x418a4a, 0x418b69, 0x418dc5, + 0x418e69, 0x418fc4, 0x4195d2, 0x419649, 0x428977, 0x4290dd, 0x4293c7, + 0x429483, 0x4682b5, 0x4790df, 0x48965b, 0x4996a8, 0x4a82af, 0x4a94ad, + 0x4c8fea, 0x4e82c8, 0x52815b, 0x5482b5, 0x56914e, 0x5782dc, 0x5782f0, + 0x57835e, 0x578ed2, 0x5882cc, 0x5882cd, 0x5b0a0a, 0x5b8357, 0x5b8368, + 0x5b8380, 0x5e838b, 0x5f89c6, 0x60815b, 0x6082a4, 0x6282b7, 0x668357, + 0x6882f0, 0x698f91, 0x699283, 0x6a82e2, 0x6a93fa, 0x6c82aa, 0x6d9446, + 0x6e82df, 0x6e88e6, 0x6f82dc, 0x6f8358, 0x708393, 0x7182cd, 0x7382cc, + 0x768ed2, 0x7782c9, 0x7782e2, 0x7790b6, 0x79815b, 0x79976a, 0x7b82cc, + 0x7b82f0, 0x8082aa, 0x814189, 0x81418a, 0x81418b, 0x81418d, 0x81418e, + 0x81418f, 0x814195, 0x814196, 0x814289, 0x814290, 0x814293, 0x814294, + 0x82a290, 0x82a295, + ], + }, { encoding: 'windows-1250', language: 'cs', diff --git a/corpus/manifest.json b/corpus/manifest.json index c747e1e..776c6c4 100644 --- a/corpus/manifest.json +++ b/corpus/manifest.json @@ -53,7 +53,44 @@ "iconv": "MAC-CYRILLIC", "languages": ["ru", "uk"] }, - { "name": "CP949", "iconv": "CP949", "languages": ["ko"] }, + { + "name": "Shift_JIS", + "iconv": "SHIFT_JIS", + "languages": ["ja"], + "family": "multibyte" + }, + { + "name": "EUC-JP", + "iconv": "EUC-JP", + "languages": ["ja"], + "family": "multibyte" + }, + { + "name": "Big5", + "iconv": "BIG5", + "languages": ["zh"], + "family": "multibyte" + }, + { + "name": "GB18030", + "iconv": "GB18030", + "languages": ["zh"], + "family": "multibyte" + }, + { + "name": "EUC-KR", + "iconv": "EUC-KR", + "languages": ["ko"], + "family": "multibyte", + "sourceReplacements": { "": "篣", "賰": "" } + }, + { + "name": "CP949", + "iconv": "CP949", + "languages": ["ko"], + "family": "multibyte", + "runtime": false + }, { "name": "ISO-8859-3", "iconv": "ISO-8859-3", "languages": ["mt"] }, { "name": "ISO-8859-4", "iconv": "ISO-8859-4", "languages": ["lv"] }, { "name": "ISO-8859-10", "iconv": "ISO-8859-10", "languages": ["is"] }, diff --git a/corpus/model-evaluation.json b/corpus/model-evaluation.json index 3021440..dde38c1 100644 --- a/corpus/model-evaluation.json +++ b/corpus/model-evaluation.json @@ -8687,5 +8687,304 @@ } ] } - ] + ], + "multibyte": { + "summary": { + "tests": 5, + "encodingCorrect": 5, + "languageCorrect": 5 + }, + "results": [ + { + "expected": { + "encoding": "Big5", + "language": "zh" + }, + "predicted": { + "encoding": "Big5", + "language": "zh", + "confidence": 0.31, + "hits": 16, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.31, + "hits": 16, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 27, + "invalidCharacters": 25 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 1, + "multibyteCharacters": 27, + "invalidCharacters": 25 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 13, + "invalidCharacters": 1 + } + ] + }, + { + "expected": { + "encoding": "EUC-JP", + "language": "ja" + }, + "predicted": { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.1, + "hits": 6, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.03, + "hits": 2, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 21, + "invalidCharacters": 1 + } + ] + }, + { + "expected": { + "encoding": "EUC-KR", + "language": "ko" + }, + "predicted": { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.59, + "hits": 41, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0.59, + "hits": 41, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.03, + "hits": 2, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.01, + "hits": 1, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0.01, + "hits": 1, + "multibyteCharacters": 69, + "invalidCharacters": 0 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 12, + "invalidCharacters": 7 + } + ] + }, + { + "expected": { + "encoding": "GB18030", + "language": "zh" + }, + "predicted": { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.33, + "hits": 17, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0.33, + "hits": 17, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0.02, + "hits": 1, + "multibyteCharacters": 52, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 4, + "multibyteCharacters": 40, + "invalidCharacters": 7 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 1, + "multibyteCharacters": 40, + "invalidCharacters": 7 + }, + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 14, + "invalidCharacters": 0 + } + ] + }, + { + "expected": { + "encoding": "Shift_JIS", + "language": "ja" + }, + "predicted": { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + "encodingCorrect": true, + "languageCorrect": true, + "candidates": [ + { + "encoding": "Shift_JIS", + "language": "ja", + "confidence": 0.67, + "hits": 42, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "Big5", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + }, + { + "encoding": "EUC-JP", + "language": "ja", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 3, + "invalidCharacters": 48 + }, + { + "encoding": "EUC-KR", + "language": "ko", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 3, + "invalidCharacters": 48 + }, + { + "encoding": "GB18030", + "language": "zh", + "confidence": 0, + "hits": 0, + "multibyteCharacters": 63, + "invalidCharacters": 0 + } + ] + } + ] + } } diff --git a/corpus/sources.json b/corpus/sources.json index e65eab5..83a82ae 100644 --- a/corpus/sources.json +++ b/corpus/sources.json @@ -1081,6 +1081,78 @@ "text": "B廕》 b癡 mang b廕τ 廙 v tr n籀ng, i qua r廙南g r廙i ngh廙 b礙n b廙 s繫ng." } ] + }, + { + "code": "ja", + "name": "Japanese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "銵格", + "text": "株胯柴閬整擏怒舫扎鈭箝整怨嗅柴嫘芷韏啜憪" + }, + { + "id": "library", + "split": "train", + "title": "芸單賊尹", + "text": "啣桀單賊尹扼舀研柴潦詻喋隤剛舀韭脯蝘摮艾銵格研Ｕ豢詻臬郎株矽寧押隡" + }, + { + "id": "market", + "split": "train", + "title": "乓桀", + "text": "摨氬桀氬怒舀圈悅芷押潦艾柴喋銝艾嗚鞎瑯拙恥航噙摰嗚典迤蝭格怒扎西店" + }, + { + "id": "workshop", + "split": "train", + "title": "詻踴怠極", + "text": "撠芥潦拚尹格雿艾閮剛駁Ｕ氬箄喋潦蝣箄蝺券隤祆詻" + }, + { + "id": "river-trip", + "split": "validation", + "title": "撌詻桀銵", + "text": "鈭箝∼臬啣喋冽萱嗚行ㄝ甇押瑯甇押桀芸颲箝找踴芥憭嫘桅野桀ㄟ" + } + ] + }, + { + "code": "zh", + "name": "Chinese", + "documents": [ + { + "id": "city-morning", + "split": "train", + "title": "撣拇", + "text": "撣Ｘａ靘嚗頠蝡鈭銝剔鈭綽∪憭折嚗蝚砌剖砍望掃頠憪蝛輸摰銵" + }, + { + "id": "library", + "split": "train", + "title": "摰賊尹", + "text": "蝷曉賊尹鋆∪芣蝧餅貊脤喋霈撠暹風脯蝘摮詨銵寥Ｙ賂蝞∠∪鼠拙飛交曇" + }, + { + "id": "market", + "split": "train", + "title": "剔撣", + "text": "撱港撣湔箸遛圈悅祈瘞湔暻萄憿批恥颲脫閰Ｗ嗆嚗銝虫漱瘚摰嗅虜瘜" + }, + { + "id": "workshop", + "split": "train", + "title": "訾撌乩摰", + "text": "銝撠甇刻ˊ雿拚尹閮剛撣急渡ｇ潸瑼Ｘ亦撘嚗蝺刻摩箔撟湧翩閫啣神皜璆隤芣" + }, + { + "id": "river-trip", + "split": "validation", + "title": "瘝喲銵", + "text": "撣嗉啣梯嗥忽璅寞韏唬敺銋隞亙嚗隞典瘝喲隡荔賢曈仿陷" + } + ] } ] } diff --git a/corpus/test-sources.json b/corpus/test-sources.json index e138203..fb6a3c8 100644 --- a/corpus/test-sources.json +++ b/corpus/test-sources.json @@ -181,6 +181,18 @@ "id": "community-garden", "title": "Vが廙n c廙ng 廙ng", "text": "Hng x籀m bi廕積 s璽n tr廙ng thnh khu vが廙n xanh, tr廕 em v廕 bi廙n mu v c獺c gia 穫nh tが廙i c璽y m廙i tu廕吵." + }, + { + "language": "ja", + "id": "community-garden", + "title": "啣桀滬", + "text": "餈桐犖∼舐征艾摨准氬晞乓具璊摮押∼航脤悅芣准雿摰嗆舀望押急偌" + }, + { + "language": "zh", + "id": "community-garden", + "title": "蝷曉勗", + "text": "啣蝛粹Ｗ寥蝬脰勗嚗蝔桐擙株晞祈撠璅嫘摮拙思敶抵脫嚗摰嗆曹韏瑞策璊拇瘞氬" } ] } diff --git a/scripts/benchmark.mjs b/scripts/benchmark.mjs index f5be321..d5bb06e 100644 --- a/scripts/benchmark.mjs +++ b/scripts/benchmark.mjs @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -8,6 +9,15 @@ const generatedCorpus = join(root, 'corpus', 'generated'); const corpusIndex = JSON.parse( readFileSync(join(generatedCorpus, 'index.json'), 'utf8'), ); +const manifest = JSON.parse( + readFileSync(join(root, 'corpus', 'manifest.json'), 'utf8'), +); +const familyByEncoding = new Map( + manifest.encodings.map((encoding) => [ + encoding.name, + encoding.family === 'multibyte' ? 'multibyte' : 'singlebyte', + ]), +); function option(name, fallback) { const prefix = `--${name}=`; @@ -17,13 +27,19 @@ function option(name, fallback) { const iterations = Number(option('iterations', '100')); const split = option('split', 'all'); +const family = option('family', 'all'); if (!Number.isInteger(iterations) || iterations < 1) { throw new Error('--iterations must be a positive integer'); } +if (!['all', 'singlebyte', 'multibyte'].includes(family)) { + throw new Error('--family must be all, singlebyte, or multibyte'); +} const rows = corpusIndex.filter( - (row) => split === 'all' || row.split === split, + (row) => + (split === 'all' || row.split === split) && + (family === 'all' || familyByEncoding.get(row.encoding) === family), ); if (rows.length === 0) { throw new Error(`No corpus files matched split: ${split}`); @@ -33,6 +49,9 @@ const inputs = rows.map((row) => ({ ...row, buffer: readFileSync(join(generatedCorpus, row.path)), })); +const workloadHash = createHash('sha256') + .update(rows.map((row) => `${row.path}\0${row.sha256}\n`).join('')) + .digest('hex'); for (const input of inputs) analyse(input.buffer); @@ -54,7 +73,15 @@ const bytesPerSecond = bytes / (elapsedMs / 1000); console.log('Library corpus benchmark'); console.log('========================'); console.log(`Split: ${split}`); +console.log(`Family: ${family}`); console.log(`Files: ${inputs.length}`); +console.log( + `Input bytes per iteration: ${inputs.reduce( + (total, input) => total + input.buffer.length, + 0, + )}`, +); +console.log(`Workload SHA-256: ${workloadHash}`); console.log(`Iterations: ${iterations}`); console.log(`Calls: ${calls}`); console.log(`Bytes processed: ${bytes}`); diff --git a/scripts/model-utils.mjs b/scripts/model-utils.mjs index e068aa9..604db30 100644 --- a/scripts/model-utils.mjs +++ b/scripts/model-utils.mjs @@ -7,6 +7,8 @@ import { scoreSBCS, trigrams, } from '../src/encoding/sbcs-scoring.ts'; +import { big5, euc_jp, euc_kr, gb_18030, sjis } from '../src/encoding/mbcs.ts'; +import { prepareMBCSModel, scoreMBCS } from '../src/encoding/mbcs-scoring.ts'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const corpus = join(root, 'corpus'); @@ -162,7 +164,7 @@ function byteEquivalent(buffer, leftEncoding, rightEncoding) { function compileSingleByteModels() { const models = []; for (const encoding of manifest.encodings) { - if (encoding.name === 'CP949') continue; + if (encoding.family === 'multibyte') continue; const byteMap = byteMapFor(encoding.iconv); const languages = encoding.languages.map((language) => { @@ -190,10 +192,88 @@ function compileSingleByteModels() { return models; } +const multibyteRecognisers = new Map([ + ['Shift_JIS', new sjis()], + ['Big5', new big5()], + ['EUC-JP', new euc_jp()], + ['EUC-KR', new euc_kr()], + ['GB18030', new gb_18030()], +]); + +function detectorContext(buffer) { + const byteStats = Array(256).fill(0); + for (const byte of buffer) byteStats[byte] += 1; + return { + byteStats, + c1Bytes: buffer.some((byte) => byte >= 0x80 && byte <= 0x9f), + rawInput: buffer, + rawLen: buffer.length, + inputBytes: buffer, + inputLen: buffer.length, + }; +} + +function compileMultibyteModels() { + return manifest.encodings + .filter( + (encoding) => + encoding.family === 'multibyte' && encoding.runtime !== false, + ) + .map((encoding) => { + const recogniser = multibyteRecognisers.get(encoding.name); + if (!recogniser) { + throw new Error(`Missing multibyte recogniser: ${encoding.name}`); + } + if (encoding.languages.length !== 1) { + throw new Error( + `Expected one language for multibyte model: ${encoding.name}`, + ); + } + const language = encoding.languages[0]; + const frequencies = new Map(); + let total = 0; + for (const row of corpusIndex.filter( + (candidate) => + candidate.encoding === encoding.name && + candidate.language === language && + candidate.split === 'train', + )) { + const buffer = readFileSync(join(generatedCorpus, row.path)); + const statistics = recogniser.statistics(detectorContext(buffer)); + if (statistics.invalidCharacters !== 0) { + throw new Error( + `${encoding.name}/${language}/${row.document} has invalid multibyte characters`, + ); + } + for (const value of statistics.multibyteCharacters) { + frequencies.set(value, (frequencies.get(value) ?? 0) + 1); + total += 1; + } + } + if (total === 0) { + throw new Error(`Empty multibyte model: ${encoding.name}/${language}`); + } + return { + encoding: encoding.name, + language, + total, + commonCharacters: [...frequencies] + .sort( + ([left, leftCount], [right, rightCount]) => + rightCount - leftCount || left - right, + ) + .slice(0, 128) + .map(([value]) => value) + .sort((left, right) => left - right), + }; + }); +} + function evaluate(models) { const preparedModels = prepareSBCSModels(models); + const modelNames = new Set(models.map((model) => model.encoding)); const tests = corpusIndex.filter( - (row) => row.split === 'test' && row.encoding !== 'CP949', + (row) => row.split === 'test' && modelNames.has(row.encoding), ); const results = tests.map((test) => { const buffer = readFileSync(join(generatedCorpus, test.path)); @@ -243,6 +323,56 @@ function evaluate(models) { }; } +function evaluateMultibyte(models) { + const preparedModels = models.map((model) => ({ + model, + prepared: prepareMBCSModel(model), + recogniser: multibyteRecognisers.get(model.encoding), + })); + const tests = corpusIndex.filter( + (row) => + row.split === 'test' && + models.some((model) => model.encoding === row.encoding), + ); + const results = tests.map((test) => { + const buffer = readFileSync(join(generatedCorpus, test.path)); + const candidates = preparedModels + .map(({ model, prepared, recogniser }) => { + const score = scoreMBCS( + recogniser.statistics(detectorContext(buffer)), + prepared, + ); + return { + encoding: model.encoding, + language: model.language, + ...score, + }; + }) + .sort( + (left, right) => + right.confidence - left.confidence || + left.encoding.localeCompare(right.encoding), + ); + return { + expected: { encoding: test.encoding, language: test.language }, + predicted: candidates[0], + encodingCorrect: candidates[0].encoding === test.encoding, + languageCorrect: candidates[0].language === test.language, + candidates, + }; + }); + return { + summary: { + tests: results.length, + encodingCorrect: results.filter((result) => result.encodingCorrect) + .length, + languageCorrect: results.filter((result) => result.languageCorrect) + .length, + }, + results, + }; +} + function javascriptString(value) { return `'${value.replaceAll('\\', '\\\\').replaceAll("'", "\\'")}'`; } @@ -251,9 +381,10 @@ function hex(value, width) { return `0x${value.toString(16).padStart(width, '0')}`; } -function serializeModels(models) { +function serializeModels(models, multibyteModels) { const lines = [ '// Generated by npm run models:build. Do not edit manually.', + "import type { GeneratedMBCSModel } from './types';", '', 'export const generatedSBCSModels = [', ]; @@ -307,18 +438,47 @@ function serializeModels(models) { lines.push(' },'); } lines.push('] as const;', ''); + lines.push( + 'export const generatedMBCSModels: readonly GeneratedMBCSModel[] = [', + ); + for (const model of multibyteModels) { + lines.push(' {'); + lines.push(` encoding: ${javascriptString(model.encoding)},`); + lines.push(` language: ${javascriptString(model.language)},`); + lines.push(` total: ${model.total},`); + lines.push(' commonCharacters: ['); + for (let index = 0; index < model.commonCharacters.length; index += 6) { + lines.push( + ` ${model.commonCharacters + .slice(index, index + 6) + .map((value) => hex(value, 8)) + .join(', ')},`, + ); + } + lines.push(' ],'); + lines.push(' },'); + } + lines.push('];', ''); return lines.join('\n'); } export function compileModels() { const models = compileSingleByteModels(); - return { models, report: evaluate(models) }; + const multibyteModels = compileMultibyteModels(); + return { + models, + multibyteModels, + report: { + ...evaluate(models), + multibyte: evaluateMultibyte(multibyteModels), + }, + }; } export function buildModels(modelPath, evaluationPath) { - const { models, report } = compileModels(); + const { models, multibyteModels, report } = compileModels(); mkdirSync(dirname(modelPath), { recursive: true }); mkdirSync(dirname(evaluationPath), { recursive: true }); - writeFileSync(modelPath, serializeModels(models)); + writeFileSync(modelPath, serializeModels(models, multibyteModels)); writeFileSync(evaluationPath, `${JSON.stringify(report, null, 2)}\n`); } diff --git a/scripts/verify-models.mjs b/scripts/verify-models.mjs index 56442e5..a964fce 100644 --- a/scripts/verify-models.mjs +++ b/scripts/verify-models.mjs @@ -15,12 +15,24 @@ try { if (!readFileSync(rebuiltModel).equals(readFileSync(generatedModel))) { throw new Error('Generated detector models are out of date'); } + const evaluation = JSON.parse(readFileSync(rebuiltEvaluation, 'utf8')); if ( !readFileSync(rebuiltEvaluation).equals(readFileSync(generatedEvaluation)) ) { throw new Error('Generated model evaluation is out of date'); } - const evaluation = JSON.parse(readFileSync(rebuiltEvaluation, 'utf8')); + if ( + evaluation.multibyte.summary.encodingCorrect !== + evaluation.multibyte.summary.tests || + evaluation.multibyte.summary.languageCorrect !== + evaluation.multibyte.summary.tests + ) { + throw new Error( + `Multibyte model evaluation failed: ${evaluation.multibyte.summary.encodingCorrect}/` + + `${evaluation.multibyte.summary.tests} encodings, ${evaluation.multibyte.summary.languageCorrect}/` + + `${evaluation.multibyte.summary.tests} languages`, + ); + } if ( evaluation.summary.encodingCorrect !== evaluation.summary.tests || evaluation.summary.languageCorrect !== evaluation.summary.tests diff --git a/src/encoding/mbcs-scoring.test.ts b/src/encoding/mbcs-scoring.test.ts new file mode 100644 index 0000000..79354e4 --- /dev/null +++ b/src/encoding/mbcs-scoring.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import type { GeneratedMBCSModel } from './models/types'; +import { + prepareMBCSModel, + scoreMBCS, + type MBCSStatistics, +} from './mbcs-scoring'; + +const model: GeneratedMBCSModel = { + encoding: 'EUC-JP', + language: 'ja', + total: 100, + commonCharacters: [0xa1a1, 0xa4a2], +}; +const prepared = prepareMBCSModel(model); + +function statistics( + multibyteCharacters: number[], + invalidCharacters = 0, +): MBCSStatistics { + return { + totalCharacters: multibyteCharacters.length + invalidCharacters, + multibyteCharacters, + invalidCharacters, + }; +} + +describe('generated MBCS scoring', () => { + it('scores model coverage rather than structural compatibility alone', () => { + expect( + scoreMBCS(statistics(Array(20).fill(0xa4a2)), prepared), + ).toMatchObject({ confidence: 1, hits: 20 }); + expect( + scoreMBCS(statistics(Array(20).fill(0xb0b0)), prepared), + ).toMatchObject({ confidence: 0, hits: 0 }); + }); + + it('reduces model confidence when invalid sequences are present', () => { + expect( + scoreMBCS(statistics(Array(20).fill(0xa4a2), 1), prepared).confidence, + ).toBe(0.5); + }); + + it('returns finite boundary scores for short and empty input', () => { + expect(scoreMBCS(statistics([]), prepared).confidence).toBe(0); + expect(scoreMBCS(statistics([0xa4a2]), prepared).confidence).toBe(0.1); + }); +}); diff --git a/src/encoding/mbcs-scoring.ts b/src/encoding/mbcs-scoring.ts new file mode 100644 index 0000000..b47b32a --- /dev/null +++ b/src/encoding/mbcs-scoring.ts @@ -0,0 +1,55 @@ +import type { GeneratedMBCSModel } from './models/types'; + +export interface MBCSStatistics { + readonly totalCharacters: number; + readonly multibyteCharacters: readonly number[]; + readonly invalidCharacters: number; +} + +export interface MBCSScore { + readonly confidence: number; + readonly hits: number; + readonly multibyteCharacters: number; + readonly invalidCharacters: number; +} + +export interface PreparedMBCSModel { + readonly model: GeneratedMBCSModel; + readonly commonCharacters: ReadonlySet; +} + +export function prepareMBCSModel(model: GeneratedMBCSModel): PreparedMBCSModel { + return { + model, + commonCharacters: new Set(model.commonCharacters), + }; +} + +export function scoreMBCS( + statistics: MBCSStatistics, + prepared: PreparedMBCSModel, +): MBCSScore { + const multibyteCharacters = statistics.multibyteCharacters.length; + const invalidCharacters = statistics.invalidCharacters; + const hits = statistics.multibyteCharacters.reduce( + (count, value) => count + (prepared.commonCharacters.has(value) ? 1 : 0), + 0, + ); + + let confidence = 0; + if (multibyteCharacters <= 10 && invalidCharacters === 0) { + confidence = + multibyteCharacters === 0 && statistics.totalCharacters < 10 ? 0 : 0.1; + } else if (multibyteCharacters >= 20 * invalidCharacters) { + const structuralValidity = + multibyteCharacters / (multibyteCharacters + invalidCharacters * 20); + confidence = (hits / multibyteCharacters) * structuralValidity; + } + + return { + confidence: Number(Math.min(confidence, 1).toFixed(2)), + hits, + multibyteCharacters, + invalidCharacters, + }; +} diff --git a/src/encoding/mbcs.test.ts b/src/encoding/mbcs.test.ts index 4a510bc..5a0b418 100644 --- a/src/encoding/mbcs.test.ts +++ b/src/encoding/mbcs.test.ts @@ -1,8 +1,11 @@ import * as chardet from '..'; +import fs from 'fs'; +import path from 'path'; import { describe, expect, it } from 'vitest'; describe('Multibyte Character Sets', () => { const base = __dirname + '/../test/data/encodings'; + const corpus = path.join(__dirname, '../../corpus/generated'); it('should return Shift_JIS', () => { expect(chardet.detectFileSync(base + '/shiftjis')).toBe('Shift_JIS'); @@ -23,4 +26,23 @@ describe('Multibyte Character Sets', () => { it('should return EUC-KR', () => { expect(chardet.detectFileSync(base + '/euc_kr')).toBe('EUC-KR'); }); + + it.each([ + ['Shift_JIS', 'ja'], + ['EUC-JP', 'ja'], + ['Big5', 'zh'], + ['GB18030', 'zh'], + ['EUC-KR', 'ko'], + ])('uses the generated %s model for held-out %s text', (encoding, lang) => { + const fixture = path.join( + corpus, + encoding, + lang, + 'test/community-garden.bin', + ); + expect(chardet.analyse(fs.readFileSync(fixture))[0]).toMatchObject({ + name: encoding, + lang, + }); + }); }); diff --git a/src/encoding/mbcs.ts b/src/encoding/mbcs.ts index 1334a53..0dff1c0 100644 --- a/src/encoding/mbcs.ts +++ b/src/encoding/mbcs.ts @@ -1,59 +1,26 @@ import type { Context, Recogniser } from '.'; -import match, { type Match, type EncodingName } from '../match'; +import match, { type EncodingName, type Match } from '../match'; +import { + prepareMBCSModel, + scoreMBCS, + type MBCSStatistics, +} from './mbcs-scoring'; +import { generatedMBCSModels } from './models/generated'; +import type { GeneratedMBCSModel } from './models/types'; + +const models = new Map( + (generatedMBCSModels as readonly GeneratedMBCSModel[]).map((model) => [ + model.encoding, + prepareMBCSModel(model), + ]), +); -/** - * Binary search implementation (recursive) - */ -function binarySearch(arr: number[], searchValue: number) { - const find = ( - arr: number[], - searchValue: number, - left: number, - right: number, - ): number => { - if (right < left) return -1; - - /* - int mid = mid = (left + right) / 2; - There is a bug in the above line; - Joshua Bloch suggests the following replacement: - */ - const mid = Math.floor((left + right) >>> 1); - if (searchValue > arr[mid]) return find(arr, searchValue, mid + 1, right); - - if (searchValue < arr[mid]) return find(arr, searchValue, left, mid - 1); - - return mid; - }; - - return find(arr, searchValue, 0, arr.length - 1); -} - -// 'Character' iterated character class. -// Recognizers for specific mbcs encodings make their 'characters' available -// by providing a nextChar() function that fills in an instance of iteratedChar -// with the next char from the input. -// The returned characters are not converted to Unicode, but remain as the raw -// bytes (concatenated into an int) from the codepage data. -// -// For Asian charsets, use the raw input rather than the input that has been -// stripped of markup. Detection only considers multi-byte chars, effectively -// stripping markup anyway, and double byte chars do occur in markup too. -// class IteratedChar { - charValue: number; // 1-4 bytes from the raw input data - index: number; - nextIndex: number; - error: boolean; - done: boolean; - - constructor() { - this.charValue = 0; // 1-4 bytes from the raw input data - this.index = 0; - this.nextIndex = 0; - this.error = false; - this.done = false; - } + charValue = 0; + index = 0; + nextIndex = 0; + error = false; + done = false; reset() { this.charValue = 0; @@ -68,128 +35,43 @@ class IteratedChar { this.done = true; return -1; } - const byteValue = det.rawInput[this.nextIndex++] & 0x00ff; - return byteValue; + return det.rawInput[this.nextIndex++] & 0xff; } } /** - * Asian double or multi-byte - charsets. - * Match is determined mostly by the input data adhering to the - * encoding scheme for the charset, and, optionally, - * frequency-of-occurrence of characters. + * Base recogniser for encodings whose structural validity is determined by + * parsing variable-width byte sequences. Character likelihood comes from the + * generated corpus model for the concrete encoding. */ - -abstract class mbcs implements Recogniser { - commonChars: number[] = []; - +export abstract class mbcs implements Recogniser { abstract name(): EncodingName; + abstract nextChar(iter: IteratedChar, det: Context): boolean; - /** - * Test the match of this charset with the input text data - * which is obtained via the CharsetDetector object. - * - * @param det The CharsetDetector, which contains the input text - * to be checked for being in this charset. - * @return A match with confidence ranging from 0 to 1. - */ - match(det: Context): Match | null { - let doubleByteCharCount = 0, - commonCharCount = 0, - badCharCount = 0, - totalCharCount = 0, - confidence = 0; - + statistics(det: Context): MBCSStatistics { + let invalidCharacters = 0; + let totalCharacters = 0; + const multibyteCharacters: number[] = []; const iter = new IteratedChar(); - detectBlock: { - for (iter.reset(); this.nextChar(iter, det); ) { - totalCharCount++; - if (iter.error) { - badCharCount++; - } else { - const cv = iter.charValue & 0xffffffff; - - if (cv > 0xff) { - doubleByteCharCount++; - if (this.commonChars != null) { - // NOTE: This assumes that there are no 4-byte common chars. - if (binarySearch(this.commonChars, cv) >= 0) { - commonCharCount++; - } - } - } - } - if (badCharCount >= 2 && badCharCount * 5 >= doubleByteCharCount) { - // console.log('its here!') - // Bail out early if the byte data is not matching the encoding scheme. - break detectBlock; - } + for (iter.reset(); this.nextChar(iter, det); ) { + totalCharacters += 1; + if (iter.error) invalidCharacters += 1; + else if (iter.charValue > 0xff) { + multibyteCharacters.push(iter.charValue >>> 0); } - - if (doubleByteCharCount <= 10 && badCharCount == 0) { - // Not many multi-byte chars. - if (doubleByteCharCount == 0 && totalCharCount < 10) { - // There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes. - // We don't have enough data to have any confidence. - // Statistical analysis of single byte non-ASCII characters would probably help here. - confidence = 0; - } else { - // ASCII or ISO file? It's probably not our encoding, - // but is not incompatible with our encoding, so don't give it a zero. - confidence = 0.1; - } - break detectBlock; - } - - // - // No match if there are too many characters that don't fit the encoding scheme. - // (should we have zero tolerance for these?) - // - if (doubleByteCharCount < 20 * badCharCount) { - confidence = 0; - break detectBlock; - } - - if (this.commonChars == null) { - // We have no statistics on frequently occurring characters. - // Assess confidence purely on having a reasonable number of - // multi-byte characters (the more the better - confidence = (30 + doubleByteCharCount - 20 * badCharCount) / 100; - if (confidence > 1) { - confidence = 1; - } - } else { - // Frequency of occurrence statistics exist. - const maxVal = Math.log(doubleByteCharCount / 4); - const scaleFactor = 90.0 / maxVal; - confidence = - Math.floor(Math.log(commonCharCount + 1) * scaleFactor + 10) / 100; - confidence = Math.min(confidence, 1); - } - } // end of detectBlock: - - return confidence == 0 ? null : match(det, this, confidence); + } + return { totalCharacters, multibyteCharacters, invalidCharacters }; } - /** - * Get the next character (however many bytes it is) from the input data - * Subclasses for specific charset encodings must implement this function - * to get characters according to the rules of their encoding scheme. - * - * This function is not a method of class iteratedChar only because - * that would require a lot of extra derived classes, which is awkward. - * @param it The iteratedChar 'struct' into which the returned char is placed. - * @param det The charset detector, which is needed to get at the input byte data - * being iterated over. - * @return True if a character was returned, false at end of input. - */ - abstract nextChar(iter: IteratedChar, det: Context): boolean; + match(det: Context): Match | null { + const prepared = models.get(this.name()); + if (!prepared) return null; + const { confidence } = scoreMBCS(this.statistics(det), prepared); + return confidence === 0 ? null : match(det, this, confidence); + } } -/** - * Shift_JIS charset recognizer. - */ export class sjis extends mbcs { name(): EncodingName { return 'Shift_JIS'; @@ -199,29 +81,17 @@ export class sjis extends mbcs { return 'ja'; } - commonChars = [ - 0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, - 0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, - 0x82b3, 0x82b5, 0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, - 0x82c8, 0x82c9, 0x82cc, 0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, - 0x82ea, 0x82f0, 0x82f1, 0x8341, 0x8343, 0x834e, 0x834f, 0x8358, 0x835e, - 0x8362, 0x8367, 0x8375, 0x8376, 0x8389, 0x838a, 0x838b, 0x838d, 0x8393, - 0x8e96, 0x93fa, 0x95aa, - ]; - nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - const firstByte = (iter.charValue = iter.nextByte(det)); if (firstByte < 0) return false; - - if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) + if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) { return true; + } const secondByte = iter.nextByte(det); if (secondByte < 0) return false; - iter.charValue = (firstByte << 8) | secondByte; if ( !( @@ -229,16 +99,12 @@ export class sjis extends mbcs { (secondByte >= 0x80 && secondByte <= 0xff) ) ) { - // Illegal second byte value. iter.error = true; } return true; } } -/** - * Big5 charset recognizer. - */ export class big5 extends mbcs { name(): EncodingName { return 'Big5'; @@ -247,109 +113,53 @@ export class big5 extends mbcs { language() { return 'zh'; } - commonChars = [ - 0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, - 0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, - 0xa477, 0xa4a3, 0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, - 0xa4fd, 0xa540, 0xa548, 0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, - 0xa662, 0xa668, 0xa670, 0xa6a8, 0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, - 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da, 0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, - 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, 0xaa6b, 0xaaba, 0xaabe, - 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, 0xaec9, 0xafe0, - 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, 0xb5a5, - 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, - 0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f, - ]; nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - const firstByte = (iter.charValue = iter.nextByte(det)); - if (firstByte < 0) return false; - - // single byte character. - if (firstByte <= 0x7f || firstByte == 0xff) return true; + if (firstByte <= 0x7f || firstByte === 0xff) return true; const secondByte = iter.nextByte(det); - if (secondByte < 0) return false; - - iter.charValue = (iter.charValue << 8) | secondByte; - - if (secondByte < 0x40 || secondByte == 0x7f || secondByte == 0xff) + iter.charValue = (firstByte << 8) | secondByte; + if (secondByte < 0x40 || secondByte === 0x7f || secondByte === 0xff) { iter.error = true; - + } return true; } } -/** - * EUC charset recognizers. One abstract class that provides the common function - * for getting the next character according to the EUC encoding scheme, - * and nested derived classes for EUC_KR, EUC_JP, EUC_CN. - * - * Get the next character value for EUC based encodings. - * Character 'value' is simply the raw bytes that make up the character - * packed into an int. - */ function eucNextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - let firstByte = 0; - let secondByte = 0; - let thirdByte = 0; - //int fourthByte = 0; - buildChar: { - firstByte = iter.charValue = iter.nextByte(det); - if (firstByte < 0) { - // Ran off the end of the input data - iter.done = true; - break buildChar; - } - if (firstByte <= 0x8d) { - // single byte char - break buildChar; - } - secondByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | secondByte; - if (firstByte >= 0xa1 && firstByte <= 0xfe) { - // Two byte Char - if (secondByte < 0xa1) { - iter.error = true; - } - break buildChar; - } - if (firstByte == 0x8e) { - // Code Set 2. - // In EUC-JP, total char size is 2 bytes, only one byte of actual char value. - // In EUC-TW, total char size is 4 bytes, three bytes contribute to char value. - // We don't know which we've got. - // Treat it like EUC-JP. If the data really was EUC-TW, the following two - // bytes will look like a well formed 2 byte char. - if (secondByte < 0xa1) { - iter.error = true; - } - break buildChar; - } - if (firstByte == 0x8f) { - // Code set 3. - // Three byte total char size, two bytes of actual char value. - thirdByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | thirdByte; - if (thirdByte < 0xa1) { - iter.error = true; - } - } + const firstByte = (iter.charValue = iter.nextByte(det)); + if (firstByte < 0) { + iter.done = true; + return false; + } + if (firstByte <= 0x8d) return true; + + const secondByte = iter.nextByte(det); + if (secondByte < 0) return false; + iter.charValue = (firstByte << 8) | secondByte; + if (firstByte >= 0xa1 && firstByte <= 0xfe) { + if (secondByte < 0xa1) iter.error = true; + return true; + } + if (firstByte === 0x8e) { + if (secondByte < 0xa1) iter.error = true; + return true; } - return iter.done == false; + if (firstByte === 0x8f) { + const thirdByte = iter.nextByte(det); + iter.charValue = (iter.charValue << 8) | thirdByte; + if (thirdByte < 0xa1) iter.error = true; + } + return !iter.done; } -/** - * The charset recognize for EUC-JP. A singleton instance of this class - * is created and kept by the public CharsetDetector class - */ export class euc_jp extends mbcs { name(): EncodingName { return 'EUC-JP'; @@ -359,28 +169,9 @@ export class euc_jp extends mbcs { return 'ja'; } - commonChars = [ - 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, - 0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, - 0xa4b1, 0xa4b3, 0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, - 0xa4c1, 0xa4c3, 0xa4c4, 0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, - 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de, 0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, - 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, 0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, - 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, 0xa5b0, 0xa5b3, 0xa5b5, - 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, 0xa5c8, 0xa5c9, - 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, 0xa5e5, - 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, - 0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, - 0xcdd1, - ]; - nextChar = eucNextChar; } -/** - * The charset recognize for EUC-KR. A singleton instance of this class - * is created and kept by the public CharsetDetector class - */ export class euc_kr extends mbcs { name(): EncodingName { return 'EUC-KR'; @@ -390,27 +181,9 @@ export class euc_kr extends mbcs { return 'ko'; } - commonChars = [ - 0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, - 0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, - 0xb4cf, 0xb4d9, 0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, - 0xb7af, 0xb7c2, 0xb7ce, 0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, - 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce, 0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, - 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, 0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, - 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, 0xbef8, 0xbefa, 0xbfa1, - 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, 0xc0af, 0xc0b8, - 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, 0xc0da, - 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, - 0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, - 0xc8ad, - ]; - nextChar = eucNextChar; } -/** - * GB-18030 recognizer. Uses simplified Chinese statistics. - */ export class gb_18030 extends mbcs { name(): EncodingName { return 'GB18030'; @@ -420,71 +193,39 @@ export class gb_18030 extends mbcs { return 'zh'; } - /* - * Get the next character value for EUC based encodings. - * Character 'value' is simply the raw bytes that make up the character - * packed into an int. - */ - nextChar(iter: IteratedChar, det: Context) { iter.index = iter.nextIndex; iter.error = false; - let firstByte = 0; - let secondByte = 0; - let thirdByte = 0; - let fourthByte = 0; - buildChar: { - firstByte = iter.charValue = iter.nextByte(det); - if (firstByte < 0) { - // Ran off the end of the input data - iter.done = true; - break buildChar; - } - if (firstByte <= 0x80) { - // single byte char - break buildChar; + const firstByte = (iter.charValue = iter.nextByte(det)); + if (firstByte < 0) { + iter.done = true; + return false; + } + if (firstByte <= 0x80) return true; + + const secondByte = iter.nextByte(det); + if (secondByte < 0) return false; + iter.charValue = (firstByte << 8) | secondByte; + if (firstByte >= 0x81 && firstByte <= 0xfe) { + if ( + (secondByte >= 0x40 && secondByte <= 0x7e) || + (secondByte >= 0x80 && secondByte <= 0xfe) + ) { + return true; } - secondByte = iter.nextByte(det); - iter.charValue = (iter.charValue << 8) | secondByte; - if (firstByte >= 0x81 && firstByte <= 0xfe) { - // Two byte Char - if ( - (secondByte >= 0x40 && secondByte <= 0x7e) || - (secondByte >= 80 && secondByte <= 0xfe) - ) { - break buildChar; - } - // Four byte char - if (secondByte >= 0x30 && secondByte <= 0x39) { - thirdByte = iter.nextByte(det); - if (thirdByte >= 0x81 && thirdByte <= 0xfe) { - fourthByte = iter.nextByte(det); - if (fourthByte >= 0x30 && fourthByte <= 0x39) { - iter.charValue = - (iter.charValue << 16) | (thirdByte << 8) | fourthByte; - break buildChar; - } + if (secondByte >= 0x30 && secondByte <= 0x39) { + const thirdByte = iter.nextByte(det); + if (thirdByte >= 0x81 && thirdByte <= 0xfe) { + const fourthByte = iter.nextByte(det); + if (fourthByte >= 0x30 && fourthByte <= 0x39) { + iter.charValue = + (iter.charValue << 16) | (thirdByte << 8) | fourthByte; + return true; } } - iter.error = true; - break buildChar; } + iter.error = true; } - return iter.done == false; + return !iter.done; } - - commonChars = [ - 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, - 0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, - 0xb5bd, 0xb5c4, 0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, - 0xb7d6, 0xb7dd, 0xb8b4, 0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, - 0xb9fd, 0xbacd, 0xbba7, 0xbbd6, 0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, - 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, 0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, - 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, 0xc7f8, 0xc8ab, 0xc8cb, - 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, 0xcad0, 0xcad6, - 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, 0xcfb5, - 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, - 0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, - 0xd6d0, - ]; } diff --git a/src/encoding/models/generated.ts b/src/encoding/models/generated.ts index 4f81c87..5aad487 100644 --- a/src/encoding/models/generated.ts +++ b/src/encoding/models/generated.ts @@ -1,4 +1,5 @@ // Generated by npm run models:build. Do not edit manually. +import type { GeneratedMBCSModel } from './types'; export const generatedSBCSModels = [ { @@ -3189,3 +3190,151 @@ export const generatedSBCSModels = [ ], }, ] as const; + +export const generatedMBCSModels: readonly GeneratedMBCSModel[] = [ + { + encoding: 'Shift_JIS', + language: 'ja', + total: 238, + commonCharacters: [ + 0x00008141, 0x00008142, 0x0000815b, 0x000082a2, 0x000082a4, 0x000082a6, + 0x000082a9, 0x000082aa, 0x000082ab, 0x000082ad, 0x000082af, 0x000082b1, + 0x000082b3, 0x000082b5, 0x000082b7, 0x000082bd, 0x000082c1, 0x000082c2, + 0x000082c4, 0x000082c5, 0x000082c6, 0x000082c8, 0x000082c9, 0x000082cc, + 0x000082cd, 0x000082d4, 0x000082d7, 0x000082dc, 0x000082df, 0x000082e2, + 0x000082e4, 0x000082e8, 0x000082e9, 0x000082f0, 0x00008352, 0x00008357, + 0x00008358, 0x0000835e, 0x00008360, 0x00008366, 0x00008368, 0x0000836f, + 0x00008370, 0x00008379, 0x00008380, 0x0000838b, 0x00008393, 0x000088c4, + 0x000088e6, 0x00008977, 0x000089b9, 0x000089c6, 0x000089c8, 0x000089ca, + 0x000089e6, 0x00008a4a, 0x00008a58, 0x00008a6d, 0x00008a6f, 0x00008a77, + 0x00008ad9, 0x00008b47, 0x00008b69, 0x00008b71, 0x00008bce, 0x00008c76, + 0x00008d48, 0x00008d4c, 0x00008d73, 0x00008dc5, 0x00008dd8, 0x00008dec, + 0x00008e69, 0x00008e6a, 0x00008e6e, 0x00008e73, 0x00008ed2, 0x00008ee8, + 0x00008f57, 0x00008f89, 0x00008f91, 0x00008fac, 0x00008fc4, 0x00008fea, + 0x00009056, 0x0000906c, 0x0000907d, 0x000090ae, 0x000090b6, 0x000090c3, + 0x000090dd, 0x000090df, 0x000090e0, 0x0000914e, 0x00009196, 0x00009254, + 0x0000926e, 0x00009283, 0x000092a9, 0x000092b2, 0x000092ca, 0x00009358, + 0x00009360, 0x00009379, 0x000093c7, 0x000093e0, 0x000093fa, 0x00009446, + 0x0000945f, 0x00009483, 0x0000948e, 0x000094ad, 0x000094e0, 0x000095a8, + 0x000095aa, 0x000095b7, 0x000095c0, 0x000095d2, 0x00009649, 0x0000965b, + 0x0000967b, 0x000096a8, 0x000096be, 0x000096ca, 0x000096da, 0x000096ec, + 0x0000976a, 0x0000979d, + ], + }, + { + encoding: 'EUC-JP', + language: 'ja', + total: 238, + commonCharacters: [ + 0x0000a1a2, 0x0000a1a3, 0x0000a1bc, 0x0000a4a4, 0x0000a4a6, 0x0000a4a8, + 0x0000a4ab, 0x0000a4ac, 0x0000a4ad, 0x0000a4af, 0x0000a4b1, 0x0000a4b3, + 0x0000a4b5, 0x0000a4b7, 0x0000a4b9, 0x0000a4bf, 0x0000a4c3, 0x0000a4c4, + 0x0000a4c6, 0x0000a4c7, 0x0000a4c8, 0x0000a4ca, 0x0000a4cb, 0x0000a4ce, + 0x0000a4cf, 0x0000a4d6, 0x0000a4d9, 0x0000a4de, 0x0000a4e1, 0x0000a4e4, + 0x0000a4e6, 0x0000a4ea, 0x0000a4eb, 0x0000a4f2, 0x0000a5b3, 0x0000a5b8, + 0x0000a5b9, 0x0000a5bf, 0x0000a5c1, 0x0000a5c7, 0x0000a5c9, 0x0000a5d0, + 0x0000a5d1, 0x0000a5da, 0x0000a5e0, 0x0000a5eb, 0x0000a5f3, 0x0000b0c6, + 0x0000b0e8, 0x0000b1d8, 0x0000b2bb, 0x0000b2c8, 0x0000b2ca, 0x0000b2cc, + 0x0000b2e8, 0x0000b3ab, 0x0000b3b9, 0x0000b3ce, 0x0000b3d0, 0x0000b3d8, + 0x0000b4db, 0x0000b5a8, 0x0000b5ca, 0x0000b5d2, 0x0000b6d0, 0x0000b7d7, + 0x0000b9a9, 0x0000b9ad, 0x0000b9d4, 0x0000bac7, 0x0000bada, 0x0000baee, + 0x0000bbca, 0x0000bbcb, 0x0000bbcf, 0x0000bbd4, 0x0000bcd4, 0x0000bcea, + 0x0000bdb8, 0x0000bde9, 0x0000bdf1, 0x0000beae, 0x0000bec6, 0x0000beec, + 0x0000bfb7, 0x0000bfcd, 0x0000bfde, 0x0000c0b0, 0x0000c0b8, 0x0000c0c5, + 0x0000c0df, 0x0000c0e1, 0x0000c0e2, 0x0000c1af, 0x0000c1f6, 0x0000c3b5, + 0x0000c3cf, 0x0000c3e3, 0x0000c4ab, 0x0000c4b4, 0x0000c4cc, 0x0000c5b9, + 0x0000c5c1, 0x0000c5da, 0x0000c6c9, 0x0000c6e2, 0x0000c6fc, 0x0000c7a7, + 0x0000c7c0, 0x0000c7e3, 0x0000c7ee, 0x0000c8af, 0x0000c8e2, 0x0000caaa, + 0x0000caac, 0x0000cab9, 0x0000cac2, 0x0000cad4, 0x0000cbaa, 0x0000cbbc, + 0x0000cbdc, 0x0000ccaa, 0x0000ccc0, 0x0000cccc, 0x0000ccdc, 0x0000ccee, + 0x0000cdcb, 0x0000cdfd, + ], + }, + { + encoding: 'Big5', + language: 'zh', + total: 199, + commonCharacters: [ + 0x0000a141, 0x0000a142, 0x0000a143, 0x0000a440, 0x0000a446, 0x0000a448, + 0x0000a457, 0x0000a46a, 0x0000a470, 0x0000a475, 0x0000a4a3, 0x0000a4bb, + 0x0000a4bd, 0x0000a4e8, 0x0000a4f4, 0x0000a55d, 0x0000a575, 0x0000a576, + 0x0000a5ab, 0x0000a5b4, 0x0000a5bf, 0x0000a5c1, 0x0000a5cd, 0x0000a5e6, + 0x0000a640, 0x0000a650, 0x0000a656, 0x0000a662, 0x0000a677, 0x0000a67e, + 0x0000a6a1, 0x0000a6a8, 0x0000a6ac, 0x0000a6ad, 0x0000a6b3, 0x0000a6e6, + 0x0000a6ec, 0x0000a740, 0x0000a755, 0x0000a7e4, 0x0000a854, 0x0000a8ae, + 0x0000a8c3, 0x0000a8d3, 0x0000a940, 0x0000a94d, 0x0000a96c, 0x0000a9b1, + 0x0000a9fa, 0x0000aa47, 0x0000aa6b, 0x0000aaab, 0x0000aaba, 0x0000aac0, + 0x0000aacc, 0x0000aaf9, 0x0000ab6e, 0x0000abb0, 0x0000abc7, 0x0000abc8, + 0x0000abfc, 0x0000ac50, 0x0000ac64, 0x0000ac79, 0x0000acb0, 0x0000acec, + 0x0000acef, 0x0000ad70, 0x0000adb1, 0x0000adb5, 0x0000adb6, 0x0000add3, + 0x0000adfb, 0x0000ae61, 0x0000ae76, 0x0000aec6, 0x0000aec8, 0x0000aed1, + 0x0000af5a, 0x0000afb8, 0x0000b0b5, 0x0000b0cf, 0x0000b0d1, 0x0000b0d8, + 0x0000b0dd, 0x0000b160, 0x0000b1e1, 0x0000b24d, 0x0000b27a, 0x0000b2c4, + 0x0000b35d, 0x0000b3d5, 0x0000b3f5, 0x0000b44d, 0x0000b4c1, 0x0000b56f, + 0x0000b57b, 0x0000b5e6, 0x0000b5f3, 0x0000b67d, 0x0000b6a4, 0x0000b6b0, + 0x0000b773, 0x0000b7a1, 0x0000b8c1, 0x0000b8cc, 0x0000b8df, 0x0000b8ea, + 0x0000b941, 0x0000b944, 0x0000b94c, 0x0000b9ce, 0x0000b9cf, 0x0000ba43, + 0x0000baa1, 0x0000bade, 0x0000bb45, 0x0000bb65, 0x0000bb73, 0x0000bba1, + 0x0000bc67, 0x0000bc73, 0x0000bcb6, 0x0000bcc6, 0x0000bd73, 0x0000bec7, + 0x0000c052, 0x0000c05d, + ], + }, + { + encoding: 'GB18030', + language: 'zh', + total: 199, + commonCharacters: [ + 0x0000814b, 0x000081ed, 0x00008280, 0x0000855e, 0x000085a2, 0x00008654, + 0x00008696, 0x00008844, 0x00008846, 0x000088f6, 0x00008c57, 0x00008c91, + 0x00008ca4, 0x00008e9f, 0x00008ecd, 0x00008f56, 0x0000945b, 0x000094b5, + 0x000095f8, 0x0000997a, 0x00009a76, 0x00009d4d, 0x00009ee9, 0x0000a1a2, + 0x0000a1a3, 0x0000a3ac, 0x0000b06c, 0x0000b0b2, 0x0000b0e0, 0x0000b0fc, + 0x0000b2a9, 0x0000b2bb, 0x0000b2cb, 0x0000b2e9, 0x0000b3a3, 0x0000b3bf, + 0x0000b3c7, 0x0000b3c9, 0x0000b3cc, 0x0000b3fe, 0x0000b4a9, 0x0000b4f2, + 0x0000b4f3, 0x0000b5c0, 0x0000b5c4, 0x0000b5da, 0x0000b5ea, 0x0000b7a8, + 0x0000b7ad, 0x0000b7bd, 0x0000b7c8, 0x0000b7e4, 0x0000b9a4, 0x0000b9ab, + 0x0000b9b2, 0x0000b9dc, 0x0000b9fb, 0x0000bacd, 0x0000bcaf, 0x0000bcd2, + 0x0000bdbb, 0x0000bdd6, 0x0000be8e, 0x0000bedb, 0x0000bfa7, 0x0000bfc6, + 0x0000bfcd, 0x0000c0ed, 0x0000c1cb, 0x0000c1cf, 0x0000c1f7, 0x0000c1f9, + 0x0000c295, 0x0000c2c3, 0x0000c2fd, 0x0000c3db, 0x0000c3e6, 0x0000c3f1, + 0x0000c3f7, 0x0000c4cf, 0x0000c4ea, 0x0000c6da, 0x0000c6fb, 0x0000c7e5, + 0x0000c8cb, 0x0000c9cf, 0x0000c9e7, 0x0000c9fa, 0x0000cab7, 0x0000cabc, + 0x0000cabd, 0x0000cad0, 0x0000cad2, 0x0000cad5, 0x0000cadf, 0x0000cbae, + 0x0000cdac, 0x0000cebb, 0x0000ceef, 0x0000cff2, 0x0000d0a1, 0x0000d0c2, + 0x0000d0c7, 0x0000d0d0, 0x0000d0d1, 0x0000d165, 0x0000d175, 0x0000d2bb, + 0x0000d2f4, 0x0000d35e, 0x0000d38b, 0x0000d3d0, 0x0000d44f, 0x0000d483, + 0x0000d4da, 0x0000d4e7, 0x0000d566, 0x0000d5be, 0x0000d5d2, 0x0000d5df, + 0x0000d5fb, 0x0000d5fd, 0x0000d6b8, 0x0000d7f7, 0x0000dc87, 0x0000e95f, + 0x0000ec6f, 0x0000f05e, + ], + }, + { + encoding: 'EUC-KR', + language: 'ko', + total: 266, + commonCharacters: [ + 0x0000b0a1, 0x0000b0ab, 0x0000b0b3, 0x0000b0b4, 0x0000b0c7, 0x0000b0cb, + 0x0000b0d4, 0x0000b0e8, 0x0000b0ed, 0x0000b0fa, 0x0000b0fc, 0x0000b1a4, + 0x0000b1b8, 0x0000b1d7, 0x0000b1db, 0x0000b1e2, 0x0000b2dc, 0x0000b3aa, + 0x0000b3bb, 0x0000b3ca, 0x0000b3d1, 0x0000b3d7, 0x0000b3f3, 0x0000b3f5, + 0x0000b4a9, 0x0000b4ba, 0x0000b4c2, 0x0000b4d4, 0x0000b4d9, 0x0000b4eb, + 0x0000b5b5, 0x0000b5b6, 0x0000b5bf, 0x0000b5e5, 0x0000b5e7, 0x0000b5e9, + 0x0000b5f0, 0x0000b6f7, 0x0000b7a5, 0x0000b7ce, 0x0000b8a3, 0x0000b8a6, + 0x0000b8ae, 0x0000b8b0, 0x0000b8b8, 0x0000b8c2, 0x0000b8de, 0x0000b8e7, + 0x0000b8ed, 0x0000b9ae, 0x0000b9b0, 0x0000b9da, 0x0000b9df, 0x0000b9e0, + 0x0000b9e6, 0x0000b9f6, 0x0000b9f8, 0x0000b9fd, 0x0000bace, 0x0000baf1, + 0x0000bafb, 0x0000bba7, 0x0000bbe7, 0x0000bbfd, 0x0000bcad, 0x0000bcb3, + 0x0000bcd2, 0x0000bcd5, 0x0000bdac, 0x0000bdba, 0x0000bdc3, 0x0000bdc7, + 0x0000beb4, 0x0000bec6, 0x0000bec8, 0x0000bedf, 0x0000bef7, 0x0000bfa1, + 0x0000bfa9, 0x0000bfaa, 0x0000bfad, 0x0000bfcd, 0x0000bfe4, 0x0000bfeb, + 0x0000bfee, 0x0000c0a7, 0x0000c0ba, 0x0000c0bb, 0x0000c0c7, 0x0000c0cc, + 0x0000c0ce, 0x0000c0cf, 0x0000c0d0, 0x0000c0da, 0x0000c0db, 0x0000c0dc, + 0x0000c0e5, 0x0000c0fc, 0x0000c0fd, 0x0000c1a4, 0x0000c1b6, 0x0000c1d8, + 0x0000c1ee, 0x0000c1f6, 0x0000c1f8, 0x0000c1fd, 0x0000c3a3, 0x0000c3a4, + 0x0000c3a5, 0x0000c3b5, 0x0000c3b9, 0x0000c4a1, 0x0000c4a7, 0x0000c4ab, + 0x0000c4da, 0x0000c5d0, 0x0000c5e4, 0x0000c6c0, 0x0000c6e4, 0x0000c6ed, + 0x0000c7c1, 0x0000c7cf, 0x0000c7d0, 0x0000c7d1, 0x0000c7de, 0x0000c7e0, + 0x0000c8ad, 0x0000c8ae, + ], + }, +]; diff --git a/src/encoding/models/types.ts b/src/encoding/models/types.ts index a592c76..93b4fbd 100644 --- a/src/encoding/models/types.ts +++ b/src/encoding/models/types.ts @@ -16,3 +16,10 @@ export interface GeneratedSBCSModel { readonly byteMap: readonly number[]; readonly languages: readonly GeneratedSBCSLanguage[]; } + +export interface GeneratedMBCSModel { + readonly encoding: EncodingName; + readonly language: string; + readonly total: number; + readonly commonCharacters: readonly number[]; +} diff --git a/src/encoding/sbcs-scoring.test.ts b/src/encoding/sbcs-scoring.test.ts index a509527..5f6aee1 100644 --- a/src/encoding/sbcs-scoring.test.ts +++ b/src/encoding/sbcs-scoring.test.ts @@ -59,7 +59,10 @@ describe('shared SBCS scoring', () => { const corpusIndex = JSON.parse( fs.readFileSync(path.join(corpusRoot, 'index.json'), 'utf8'), ) as { encoding: string; path: string }[]; - const rows = corpusIndex.filter(({ encoding }) => encoding !== 'CP949'); + const modelNames = new Set( + generatedSBCSModels.map(({ encoding }) => encoding as string), + ); + const rows = corpusIndex.filter(({ encoding }) => modelNames.has(encoding)); for (const row of rows) { const input = fs.readFileSync(path.join(corpusRoot, row.path)); diff --git a/src/encoding/sbcs.test.ts b/src/encoding/sbcs.test.ts index 9614e2f..3beb615 100644 --- a/src/encoding/sbcs.test.ts +++ b/src/encoding/sbcs.test.ts @@ -76,6 +76,22 @@ describe('Singlebyte Character Sets', () => { expect(detect('windows_874')).toBe('windows-874'); }); + it.each(['validation/river-trip.bin', 'test/community-garden.bin'])( + 'should prefer windows-874 to weak multibyte matches in %s', + (filename) => { + const matches = chardet.analyse( + fs.readFileSync(corpusFixture('windows-874', 'th', filename)), + ); + const windows874 = matches.find((match) => match.name === 'windows-874'); + const eucJp = matches.find((match) => match.name === 'EUC-JP'); + + expect(matches[0]).toMatchObject({ name: 'windows-874', lang: 'th' }); + expect(eucJp?.confidence).toBeLessThanOrEqual( + windows874?.confidence ?? 0, + ); + }, + ); + // iso-8859-12 is abandoned it.each([ ['ISO-8859-13', 'lt'],