diff --git a/.github/workflows/test-build.js b/.github/workflows/test-build.js index febbf05..85dde71 100644 --- a/.github/workflows/test-build.js +++ b/.github/workflows/test-build.js @@ -8,15 +8,15 @@ assert(typeof chardet.detectFile, 'function'); assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ - { confidence: 100, name: 'ASCII', lang: undefined }, - { confidence: 98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 98, name: 'ISO-8859-2', lang: 'hu' }, - { confidence: 10, name: 'UTF-8', lang: undefined }, - { confidence: 10, name: 'Shift_JIS', lang: 'ja' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'EUC-JP', lang: 'ja' }, - { confidence: 10, name: 'EUC-KR', lang: 'ko' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, + { confidence: 1, name: 'ASCII', lang: undefined }, + { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' }, + { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' }, + { confidence: 0.1, name: 'UTF-8', lang: undefined }, + { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'EUC-JP', lang: 'ja' }, + { confidence: 0.1, name: 'EUC-KR', lang: 'ko' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, ]); console.log(' > test-build.js OK'); diff --git a/.github/workflows/test-build.ts b/.github/workflows/test-build.ts index ae04c30..cdc31d8 100644 --- a/.github/workflows/test-build.ts +++ b/.github/workflows/test-build.ts @@ -11,15 +11,15 @@ const main = async () => { assert(typeof chardet.detectFileSync, 'function'); assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [ - { confidence: 100, name: 'ASCII', lang: undefined }, - { confidence: 98, name: 'ISO-8859-1', lang: 'en' }, - { confidence: 98, name: 'ISO-8859-2', lang: 'hu' }, - { confidence: 10, name: 'UTF-8', lang: undefined }, - { confidence: 10, name: 'Shift_JIS', lang: 'ja' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'EUC-JP', lang: 'ja' }, - { confidence: 10, name: 'EUC-KR', lang: 'ko' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, + { confidence: 1, name: 'ASCII', lang: undefined }, + { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' }, + { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' }, + { confidence: 0.1, name: 'UTF-8', lang: undefined }, + { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'EUC-JP', lang: 'ja' }, + { confidence: 0.1, name: 'EUC-KR', lang: 'ko' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, ]); }; diff --git a/README.md b/README.md index 6c10081..2a797b8 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ Returned value is an array of objects sorted by confidence value in descending o ```javascript [ - { confidence: 90, name: 'UTF-8' }, - { confidence: 20, name: 'windows-1252', lang: 'fr' }, + { confidence: 0.9, name: 'UTF-8' }, + { confidence: 0.2, name: 'windows-1252', lang: 'fr' }, ]; ``` diff --git a/src/encoding/ascii.ts b/src/encoding/ascii.ts index 013c286..d65535e 100644 --- a/src/encoding/ascii.ts +++ b/src/encoding/ascii.ts @@ -16,6 +16,6 @@ export default class Ascii implements Recogniser { } } - return match(det, this, 100); + return match(det, this, 1); } } diff --git a/src/encoding/iso2022.test.ts b/src/encoding/iso2022.test.ts index 15b611b..c7d7b7e 100644 --- a/src/encoding/iso2022.test.ts +++ b/src/encoding/iso2022.test.ts @@ -11,7 +11,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-JP', () => { expect(analyse('iso2022jp')).toEqual({ - confidence: 100, + confidence: 1, lang: 'ja', name: 'ISO-2022-JP', }); @@ -19,7 +19,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-KR', () => { expect(analyse('iso2022kr')).toEqual({ - confidence: 100, + confidence: 1, lang: 'kr', name: 'ISO-2022-KR', }); @@ -27,7 +27,7 @@ describe('ISO-2022', () => { it('should return ISO-2022-CN', () => { expect(analyse('iso2022cn')).toEqual({ - confidence: 100, + confidence: 1, lang: 'zh', name: 'ISO-2022-CN', }); diff --git a/src/encoding/iso2022.ts b/src/encoding/iso2022.ts index 6252639..5cc1d80 100644 --- a/src/encoding/iso2022.ts +++ b/src/encoding/iso2022.ts @@ -7,12 +7,10 @@ import match, { type Match, type EncodingName } from '../match'; * of encodings. */ -class ISO_2022 implements Recogniser { +abstract class ISO_2022 implements Recogniser { escapeSequences: number[][] = []; - name(): EncodingName { - return 'ISO_2022'; - } + abstract name(): EncodingName; match(det: Context): Match | null { /** @@ -70,15 +68,15 @@ class ISO_2022 implements Recogniser { // // Initial quality is based on relative proportion of recognized vs. // unrecognized escape sequences. - // All good: quality = 100; + // All good: quality = 1; // half or less good: quality = 0; // linear in between. - confidence = (100 * hits - 100 * misses) / (hits + misses); + confidence = (hits - misses) / (hits + misses); // Back off quality if there were too few escape sequences seen. // Include shifts in this computation, so that KR does not get penalized // for having only a single Escape sequence, but many shifts. - if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 10; + if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 0.1; return confidence <= 0 ? null : match(det, this, confidence); } diff --git a/src/encoding/mbcs.ts b/src/encoding/mbcs.ts index 4eb2b97..1334a53 100644 --- a/src/encoding/mbcs.ts +++ b/src/encoding/mbcs.ts @@ -80,12 +80,10 @@ class IteratedChar { * frequency-of-occurrence of characters. */ -class mbcs implements Recogniser { +abstract class mbcs implements Recogniser { commonChars: number[] = []; - name(): EncodingName { - return 'mbcs'; - } + abstract name(): EncodingName; /** * Test the match of this charset with the input text data @@ -93,9 +91,7 @@ class mbcs implements Recogniser { * * @param det The CharsetDetector, which contains the input text * to be checked for being in this charset. - * @return Two values packed into one int (Damn java, anyhow) - * bits 0-7: the match confidence, ranging from 0-100 - * bits 8-15: The match reason, an enum-like value. + * @return A match with confidence ranging from 0 to 1. */ match(det: Context): Match | null { let doubleByteCharCount = 0, @@ -141,7 +137,7 @@ class mbcs implements Recogniser { } else { // ASCII or ISO file? It's probably not our encoding, // but is not incompatible with our encoding, so don't give it a zero. - confidence = 10; + confidence = 0.1; } break detectBlock; } @@ -159,18 +155,17 @@ class mbcs implements Recogniser { // We have no statistics on frequently occurring characters. // Assess confidence purely on having a reasonable number of // multi-byte characters (the more the better - confidence = 30 + doubleByteCharCount - 20 * badCharCount; - if (confidence > 100) { - confidence = 100; + confidence = (30 + doubleByteCharCount - 20 * badCharCount) / 100; + if (confidence > 1) { + confidence = 1; } } else { // Frequency of occurrence statistics exist. const maxVal = Math.log(doubleByteCharCount / 4); const scaleFactor = 90.0 / maxVal; - confidence = Math.floor( - Math.log(commonCharCount + 1) * scaleFactor + 10, - ); - confidence = Math.min(confidence, 100); + confidence = + Math.floor(Math.log(commonCharCount + 1) * scaleFactor + 10) / 100; + confidence = Math.min(confidence, 1); } } // end of detectBlock: @@ -189,9 +184,7 @@ class mbcs implements Recogniser { * being iterated over. * @return True if a character was returned, false at end of input. */ - nextChar(_iter: IteratedChar, _det: Context): boolean { - return true; - } + abstract nextChar(iter: IteratedChar, det: Context): boolean; } /** @@ -206,9 +199,6 @@ export class sjis extends mbcs { return 'ja'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, @@ -257,9 +247,6 @@ export class big5 extends mbcs { language() { return 'zh'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, @@ -372,9 +359,6 @@ export class euc_jp extends mbcs { return 'ja'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, @@ -406,9 +390,6 @@ export class euc_kr extends mbcs { return 'ko'; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, @@ -492,9 +473,6 @@ export class gb_18030 extends mbcs { return iter.done == false; } - // TODO: This set of data comes from the character frequency- - // of-occurrence analysis tool. The data needs to be moved - // into a resource and loaded from there. commonChars = [ 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, diff --git a/src/encoding/sbcs.ts b/src/encoding/sbcs.ts index 279531f..4dbabb1 100644 --- a/src/encoding/sbcs.ts +++ b/src/encoding/sbcs.ts @@ -18,7 +18,6 @@ class NGramParser { ngramList: number[]; byteMap: number[]; - // TODO: is it safe to set it like this? spaceChar: number = 0x20; constructor(theNgramList: number[], theByteMap: number[]) { @@ -71,7 +70,6 @@ class NGramParser { while ((b = this.nextByte(det)) >= 0) { const mb = this.byteMap[b]; - // TODO: 0x20 might not be a space in all character sets... if (mb != 0) { if (!(mb == this.spaceChar && ignoreSpace)) { this.addByte(mb); @@ -81,16 +79,13 @@ class NGramParser { } } - // TODO: Is this OK? The buffer could have ended in the middle of a word... this.addByte(this.spaceChar); const rawPercent = this.hitCount / this.ngramCount; - // TODO - This is a bit of a hack to take care of a case - // were we were getting a confidence of 135... - if (rawPercent > 0.33) return 98; + if (rawPercent > 0.33) return 0.98; - return Math.floor(rawPercent * 300.0); + return Math.floor(rawPercent * 300.0) / 100; } } @@ -107,7 +102,7 @@ class NGramsPlusLang { const isFlatNgrams = (val: NGramsPlusLang[] | number[]): val is number[] => Array.isArray(val) && isFinite(val[0] as number); -class sbcs implements Recogniser { +abstract class sbcs implements Recogniser { spaceChar = 0x20; private nGramLang?: string = undefined; @@ -120,9 +115,7 @@ class sbcs implements Recogniser { return []; } - name(_input: Context): EncodingName { - return 'sbcs'; - } + abstract name(input: Context): EncodingName; language(): string | undefined { return this.nGramLang; @@ -1064,30 +1057,3 @@ export class KOI8_R extends sbcs { return 'ru'; } } - -/* -module.exports.ISO_8859_7 = function() { - this.byteMap = function() { - return [ - - ]; - }; - - this.ngrams = function() { - return [ - - ]; - }; - - this.name = function(det) { - if (typeof det == 'undefined') - return 'ISO-8859-7'; - return det.c1Bytes ? 'windows-1253' : 'ISO-8859-7'; - }; - - language() { - return 'el'; - }; -}; -util.inherits(module.exports.ISO_8859_7, sbcs); -*/ diff --git a/src/encoding/unicode.ts b/src/encoding/unicode.ts index 7ca1a47..098b200 100644 --- a/src/encoding/unicode.ts +++ b/src/encoding/unicode.ts @@ -18,7 +18,7 @@ export class UTF_16BE implements Recogniser { (input[0] & 0xff) == 0xfe && (input[1] & 0xff) == 0xff ) { - return match(det, this, 100); // confidence = 100 + return match(det, this, 1); } // TODO: Do some statistics to check for unsigned UTF-16BE @@ -44,7 +44,7 @@ export class UTF_16LE implements Recogniser { // It is probably UTF-32 LE, not UTF-16 return null; } - return match(det, this, 100); // confidence = 100 + return match(det, this, 1); } // TODO: Do some statistics to check for unsigned UTF-16LE @@ -94,19 +94,18 @@ class UTF_32 implements Recogniser, WithGetChar { // Cook up some sort of confidence score, based on presence of a BOM // and the existence of valid and/or invalid multi-byte sequences. if (hasBOM && numInvalid == 0) { - confidence = 100; + confidence = 1; } else if (hasBOM && numValid > numInvalid * 10) { - confidence = 80; + confidence = 0.8; } else if (numValid > 3 && numInvalid == 0) { - confidence = 100; + confidence = 1; } else if (numValid > 0 && numInvalid == 0) { - confidence = 80; + confidence = 0.8; } else if (numValid > numInvalid * 10) { // Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance. - confidence = 25; + confidence = 0.25; } - // return confidence == 0 ? null : new CharsetMatch(det, this, confidence); return confidence == 0 ? null : match(det, this, confidence); } } diff --git a/src/encoding/utf8.ts b/src/encoding/utf8.ts index 6e5c4d5..6c8c087 100644 --- a/src/encoding/utf8.ts +++ b/src/encoding/utf8.ts @@ -60,16 +60,16 @@ export default class Utf8 implements Recogniser { // Cook up some sort of confidence score, based on presence of a BOM // and the existence of valid and/or invalid multi-byte sequences. confidence = 0; - if (hasBOM && numInvalid == 0) confidence = 100; - else if (hasBOM && numValid > numInvalid * 10) confidence = 80; - else if (numValid > 3 && numInvalid == 0) confidence = 100; - else if (numValid > 0 && numInvalid == 0) confidence = 80; + if (hasBOM && numInvalid == 0) confidence = 1; + else if (hasBOM && numValid > numInvalid * 10) confidence = 0.8; + else if (numValid > 3 && numInvalid == 0) confidence = 1; + else if (numValid > 0 && numInvalid == 0) confidence = 0.8; else if (numValid == 0 && numInvalid == 0) // Plain ASCII. - confidence = 10; + confidence = 0.1; else if (numValid > numInvalid * 10) // Probably corrupt utf-8 data. Valid sequences aren't likely by chance. - confidence = 25; + confidence = 0.25; else return null; return match(det, this, confidence); diff --git a/src/index.test.ts b/src/index.test.ts index 6111043..697241a 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -6,17 +6,17 @@ import { describe, expect, it } from 'vitest'; describe('chardet', () => { const path = __dirname + '/test/data/encodings/utf8'; const expectedEncodingsFromPath = [ - { confidence: 100, name: 'UTF-8', lang: undefined }, - { confidence: 32, name: 'windows-1252', lang: 'fr' }, - { confidence: 19, name: 'KOI8-R', lang: 'ru' }, - { confidence: 10, name: 'Big5', lang: 'zh' }, - { confidence: 10, name: 'GB18030', lang: 'zh' }, // Mandarin - { confidence: 10, name: 'windows-1253', lang: 'el' }, // Greek - { confidence: 6, name: 'windows-1250', lang: 'pl' }, - { confidence: 4, name: 'windows-1254', lang: 'tr' }, - { confidence: 3, name: 'windows-1257', lang: 'et' }, - { confidence: 2, name: 'windows-1251', lang: 'ru' }, - { confidence: 1, name: 'windows-1258', lang: 'vi' }, + { confidence: 1, name: 'UTF-8', lang: undefined }, + { confidence: 0.32, name: 'windows-1252', lang: 'fr' }, + { confidence: 0.19, name: 'KOI8-R', lang: 'ru' }, + { confidence: 0.1, name: 'Big5', lang: 'zh' }, + { confidence: 0.1, name: 'GB18030', lang: 'zh' }, // Mandarin + { confidence: 0.1, name: 'windows-1253', lang: 'el' }, // Greek + { confidence: 0.06, name: 'windows-1250', lang: 'pl' }, + { confidence: 0.04, name: 'windows-1254', lang: 'tr' }, + { confidence: 0.03, name: 'windows-1257', lang: 'et' }, + { confidence: 0.02, name: 'windows-1251', lang: 'ru' }, + { confidence: 0.01, name: 'windows-1258', lang: 'vi' }, { confidence: 0, name: 'ASCII', lang: undefined }, ]; diff --git a/src/match.ts b/src/match.ts index a9bc9a2..434956d 100644 --- a/src/match.ts +++ b/src/match.ts @@ -6,7 +6,6 @@ export type EncodingName = | 'EUC-JP' | 'EUC-KR' | 'GB18030' - | 'ISO_2022' // TODO: Use hyphen | 'ISO-2022-CN' | 'ISO-2022-JP' | 'ISO-2022-KR' @@ -17,11 +16,8 @@ export type EncodingName = | 'ISO-8859-7' | 'ISO-8859-8' | 'ISO-8859-9' - | 'ISO-8859-9' | 'KOI8-R' - | 'mbcs' - | 'sbcs' - | 'Shift_JIS' // TODO: Use hyphen + | 'Shift_JIS' | 'UTF-16BE' | 'UTF-16LE' | 'UTF-32' @@ -33,7 +29,6 @@ export type EncodingName = | 'windows-1252' | 'windows-1253' | 'windows-1254' - | 'windows-1254' | 'windows-1255' | 'windows-1256' | 'windows-1257'