feat!: normalize encoding names and confidence scores

.github/workflows/test-build.js

@@ -8,15 +8,15 @@ assert(typeof chardet.detectFile, 'function');
 assert(typeof chardet.detectFileSync, 'function');
 
 assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [
-  { confidence: 100, name: 'ASCII', lang: undefined },
-  { confidence: 98, name: 'ISO-8859-1', lang: 'en' },
-  { confidence: 98, name: 'ISO-8859-2', lang: 'hu' },
-  { confidence: 10, name: 'UTF-8', lang: undefined },
-  { confidence: 10, name: 'Shift_JIS', lang: 'ja' },
-  { confidence: 10, name: 'Big5', lang: 'zh' },
-  { confidence: 10, name: 'EUC-JP', lang: 'ja' },
-  { confidence: 10, name: 'EUC-KR', lang: 'ko' },
-  { confidence: 10, name: 'GB18030', lang: 'zh' },
+  { confidence: 1, name: 'ASCII', lang: undefined },
+  { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' },
+  { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' },
+  { confidence: 0.1, name: 'UTF-8', lang: undefined },
+  { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' },
+  { confidence: 0.1, name: 'Big5', lang: 'zh' },
+  { confidence: 0.1, name: 'EUC-JP', lang: 'ja' },
+  { confidence: 0.1, name: 'EUC-KR', lang: 'ko' },
+  { confidence: 0.1, name: 'GB18030', lang: 'zh' },
 ]);
 
 console.log(' > test-build.js OK');

.github/workflows/test-build.ts

@@ -11,15 +11,15 @@ const main = async () => {
   assert(typeof chardet.detectFileSync, 'function');
 
   assert.deepStrictEqual(chardet.analyse(Buffer.from('This is a test')), [
-    { confidence: 100, name: 'ASCII', lang: undefined },
-    { confidence: 98, name: 'ISO-8859-1', lang: 'en' },
-    { confidence: 98, name: 'ISO-8859-2', lang: 'hu' },
-    { confidence: 10, name: 'UTF-8', lang: undefined },
-    { confidence: 10, name: 'Shift_JIS', lang: 'ja' },
-    { confidence: 10, name: 'Big5', lang: 'zh' },
-    { confidence: 10, name: 'EUC-JP', lang: 'ja' },
-    { confidence: 10, name: 'EUC-KR', lang: 'ko' },
-    { confidence: 10, name: 'GB18030', lang: 'zh' },
+    { confidence: 1, name: 'ASCII', lang: undefined },
+    { confidence: 0.98, name: 'ISO-8859-1', lang: 'en' },
+    { confidence: 0.98, name: 'ISO-8859-2', lang: 'hu' },
+    { confidence: 0.1, name: 'UTF-8', lang: undefined },
+    { confidence: 0.1, name: 'Shift_JIS', lang: 'ja' },
+    { confidence: 0.1, name: 'Big5', lang: 'zh' },
+    { confidence: 0.1, name: 'EUC-JP', lang: 'ja' },
+    { confidence: 0.1, name: 'EUC-KR', lang: 'ko' },
+    { confidence: 0.1, name: 'GB18030', lang: 'zh' },
   ]);
 };
 

README.md

@@ -41,8 +41,8 @@ Returned value is an array of objects sorted by confidence value in descending o
 
 ```javascript
 [
-  { confidence: 90, name: 'UTF-8' },
-  { confidence: 20, name: 'windows-1252', lang: 'fr' },
+  { confidence: 0.9, name: 'UTF-8' },
+  { confidence: 0.2, name: 'windows-1252', lang: 'fr' },
 ];
 ```
 

src/encoding/ascii.ts

@@ -16,6 +16,6 @@ export default class Ascii implements Recogniser {
       }
     }
 
-    return match(det, this, 100);
+    return match(det, this, 1);
   }
 }

src/encoding/iso2022.test.ts

@@ -11,23 +11,23 @@ describe('ISO-2022', () => {
 
   it('should return ISO-2022-JP', () => {
     expect(analyse('iso2022jp')).toEqual({
-      confidence: 100,
+      confidence: 1,
       lang: 'ja',
       name: 'ISO-2022-JP',
     });
   });
 
   it('should return ISO-2022-KR', () => {
     expect(analyse('iso2022kr')).toEqual({
-      confidence: 100,
+      confidence: 1,
       lang: 'kr',
       name: 'ISO-2022-KR',
     });
   });
 
   it('should return ISO-2022-CN', () => {
     expect(analyse('iso2022cn')).toEqual({
-      confidence: 100,
+      confidence: 1,
       lang: 'zh',
       name: 'ISO-2022-CN',
     });

src/encoding/iso2022.ts

@@ -7,12 +7,10 @@ import match, { type Match, type EncodingName } from '../match';
  * of encodings.
  */
 
-class ISO_2022 implements Recogniser {
+abstract class ISO_2022 implements Recogniser {
   escapeSequences: number[][] = [];
 
-  name(): EncodingName {
-    return 'ISO_2022';
-  }
+  abstract name(): EncodingName;
 
   match(det: Context): Match | null {
     /**
@@ -70,15 +68,15 @@ class ISO_2022 implements Recogniser {
     //
     // Initial quality is based on relative proportion of recognized vs.
     //   unrecognized escape sequences.
-    //   All good:  quality = 100;
+    //   All good:  quality = 1;
     //   half or less good: quality = 0;
     //   linear in between.
-    confidence = (100 * hits - 100 * misses) / (hits + misses);
+    confidence = (hits - misses) / (hits + misses);
 
     // Back off quality if there were too few escape sequences seen.
     //   Include shifts in this computation, so that KR does not get penalized
     //   for having only a single Escape sequence, but many shifts.
-    if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 10;
+    if (hits + shifts < 5) confidence -= (5 - (hits + shifts)) * 0.1;
 
     return confidence <= 0 ? null : match(det, this, confidence);
   }

src/encoding/mbcs.ts

@@ -80,22 +80,18 @@ class IteratedChar {
  * frequency-of-occurrence of characters.
  */
 
-class mbcs implements Recogniser {
+abstract class mbcs implements Recogniser {
   commonChars: number[] = [];
 
-  name(): EncodingName {
-    return 'mbcs';
-  }
+  abstract name(): EncodingName;
 
   /**
    * Test the match of this charset with the input text data
    *      which is obtained via the CharsetDetector object.
    *
    * @param det  The CharsetDetector, which contains the input text
    *             to be checked for being in this charset.
-   * @return     Two values packed into one int  (Damn java, anyhow)
-   *             bits 0-7:  the match confidence, ranging from 0-100
-   *             bits 8-15: The match reason, an enum-like value.
+   * @return     A match with confidence ranging from 0 to 1.
    */
   match(det: Context): Match | null {
     let doubleByteCharCount = 0,
@@ -141,7 +137,7 @@ class mbcs implements Recogniser {
         } else {
           //   ASCII or ISO file?  It's probably not our encoding,
           //   but is not incompatible with our encoding, so don't give it a zero.
-          confidence = 10;
+          confidence = 0.1;
         }
         break detectBlock;
       }
@@ -159,18 +155,17 @@ class mbcs implements Recogniser {
         // We have no statistics on frequently occurring characters.
         //  Assess confidence purely on having a reasonable number of
         //  multi-byte characters (the more the better
-        confidence = 30 + doubleByteCharCount - 20 * badCharCount;
-        if (confidence > 100) {
-          confidence = 100;
+        confidence = (30 + doubleByteCharCount - 20 * badCharCount) / 100;
+        if (confidence > 1) {
+          confidence = 1;
         }
       } else {
         // Frequency of occurrence statistics exist.
         const maxVal = Math.log(doubleByteCharCount / 4);
         const scaleFactor = 90.0 / maxVal;
-        confidence = Math.floor(
-          Math.log(commonCharCount + 1) * scaleFactor + 10,
-        );
-        confidence = Math.min(confidence, 100);
+        confidence =
+          Math.floor(Math.log(commonCharCount + 1) * scaleFactor + 10) / 100;
+        confidence = Math.min(confidence, 1);
       }
     } // end of detectBlock:
 
@@ -189,9 +184,7 @@ class mbcs implements Recogniser {
    *            being iterated over.
    * @return    True if a character was returned, false at end of input.
    */
-  nextChar(_iter: IteratedChar, _det: Context): boolean {
-    return true;
-  }
+  abstract nextChar(iter: IteratedChar, det: Context): boolean;
 }
 
 /**
@@ -206,9 +199,6 @@ export class sjis extends mbcs {
     return 'ja';
   }
 
-  // TODO:  This set of data comes from the character frequency-
-  //        of-occurrence analysis tool.  The data needs to be moved
-  //        into a resource and loaded from there.
   commonChars = [
     0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176,
     0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1,
@@ -257,9 +247,6 @@ export class big5 extends mbcs {
   language() {
     return 'zh';
   }
-  // TODO:  This set of data comes from the character frequency-
-  //        of-occurrence analysis tool.  The data needs to be moved
-  //        into a resource and loaded from there.
   commonChars = [
     0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440,
     0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c,
@@ -372,9 +359,6 @@ export class euc_jp extends mbcs {
     return 'ja';
   }
 
-  // TODO:  This set of data comes from the character frequency-
-  //        of-occurrence analysis tool.  The data needs to be moved
-  //        into a resource and loaded from there.
   commonChars = [
     0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7,
     0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af,
@@ -406,9 +390,6 @@ export class euc_kr extends mbcs {
     return 'ko';
   }
 
-  // TODO:  This set of data comes from the character frequency-
-  //        of-occurrence analysis tool.  The data needs to be moved
-  //        into a resource and loaded from there.
   commonChars = [
     0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa,
     0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2,
@@ -492,9 +473,6 @@ export class gb_18030 extends mbcs {
     return iter.done == false;
   }
 
-  // TODO:  This set of data comes from the character frequency-
-  //        of-occurrence analysis tool.  The data needs to be moved
-  //        into a resource and loaded from there.
   commonChars = [
     0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1,
     0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3,

src/encoding/sbcs.ts

@@ -18,7 +18,6 @@ class NGramParser {
   ngramList: number[];
   byteMap: number[];
 
-  // TODO: is it safe to set it like this?
   spaceChar: number = 0x20;
 
   constructor(theNgramList: number[], theByteMap: number[]) {
@@ -71,7 +70,6 @@ class NGramParser {
     while ((b = this.nextByte(det)) >= 0) {
       const mb = this.byteMap[b];
 
-      // TODO: 0x20 might not be a space in all character sets...
       if (mb != 0) {
         if (!(mb == this.spaceChar && ignoreSpace)) {
           this.addByte(mb);
@@ -81,16 +79,13 @@ class NGramParser {
       }
     }
 
-    // TODO: Is this OK? The buffer could have ended in the middle of a word...
     this.addByte(this.spaceChar);
 
     const rawPercent = this.hitCount / this.ngramCount;
 
-    // TODO - This is a bit of a hack to take care of a case
-    // were we were getting a confidence of 135...
-    if (rawPercent > 0.33) return 98;
+    if (rawPercent > 0.33) return 0.98;
 
-    return Math.floor(rawPercent * 300.0);
+    return Math.floor(rawPercent * 300.0) / 100;
   }
 }
 
@@ -107,7 +102,7 @@ class NGramsPlusLang {
 const isFlatNgrams = (val: NGramsPlusLang[] | number[]): val is number[] =>
   Array.isArray(val) && isFinite(val[0] as number);
 
-class sbcs implements Recogniser {
+abstract class sbcs implements Recogniser {
   spaceChar = 0x20;
 
   private nGramLang?: string = undefined;
@@ -120,9 +115,7 @@ class sbcs implements Recogniser {
     return [];
   }
 
-  name(_input: Context): EncodingName {
-    return 'sbcs';
-  }
+  abstract name(input: Context): EncodingName;
 
   language(): string | undefined {
     return this.nGramLang;
@@ -1064,30 +1057,3 @@ export class KOI8_R extends sbcs {
     return 'ru';
   }
 }
-
-/*
-module.exports.ISO_8859_7 = function() {
-  this.byteMap = function() {
-    return [
-
-    ];
-  };
-
-  this.ngrams = function() {
-    return [
-
-    ];
-  };
-
-  this.name = function(det) {
-    if (typeof det == 'undefined')
-      return 'ISO-8859-7';
-    return det.c1Bytes ? 'windows-1253' : 'ISO-8859-7';
-  };
-
-  language() {
-    return 'el';
-  };
-};
-util.inherits(module.exports.ISO_8859_7, sbcs);
-*/

src/encoding/unicode.ts

@@ -18,7 +18,7 @@ export class UTF_16BE implements Recogniser {
       (input[0] & 0xff) == 0xfe &&
       (input[1] & 0xff) == 0xff
     ) {
-      return match(det, this, 100); // confidence = 100
+      return match(det, this, 1);
     }
 
     // TODO: Do some statistics to check for unsigned UTF-16BE
@@ -44,7 +44,7 @@ export class UTF_16LE implements Recogniser {
         // It is probably UTF-32 LE, not UTF-16
         return null;
       }
-      return match(det, this, 100); // confidence = 100
+      return match(det, this, 1);
     }
 
     // TODO: Do some statistics to check for unsigned UTF-16LE
@@ -94,19 +94,18 @@ class UTF_32 implements Recogniser, WithGetChar {
     // Cook up some sort of confidence score, based on presence of a BOM
     //    and the existence of valid and/or invalid multi-byte sequences.
     if (hasBOM && numInvalid == 0) {
-      confidence = 100;
+      confidence = 1;
     } else if (hasBOM && numValid > numInvalid * 10) {
-      confidence = 80;
+      confidence = 0.8;
     } else if (numValid > 3 && numInvalid == 0) {
-      confidence = 100;
+      confidence = 1;
     } else if (numValid > 0 && numInvalid == 0) {
-      confidence = 80;
+      confidence = 0.8;
     } else if (numValid > numInvalid * 10) {
       // Probably corrupt UTF-32BE data.  Valid sequences aren't likely by chance.
-      confidence = 25;
+      confidence = 0.25;
     }
 
-    // return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
     return confidence == 0 ? null : match(det, this, confidence);
   }
 }