From d84e4ffe8dd45fc1f499cc49fb7eb9916c2a96e2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Sep 2026 15:21:39 -0400 Subject: [PATCH 01/13] Add a Location class --- javascript/src/index.js | 1 + javascript/src/location.js | 40 +++++++++++++++++++++ javascript/test.js | 11 ++++-- templates/javascript/src/deserialize.js.erb | 13 ++++--- templates/javascript/src/nodes.js.erb | 7 +--- 5 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 javascript/src/location.js diff --git a/javascript/src/index.js b/javascript/src/index.js index 3c40c4c16b..a0945e7d2a 100644 --- a/javascript/src/index.js +++ b/javascript/src/index.js @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { ParseResult } from "./deserialize.js"; import { parsePrism } from "./parsePrism.js"; +export * from "./location.js"; export * from "./visitor.js"; export * from "./nodes.js"; diff --git a/javascript/src/location.js b/javascript/src/location.js new file mode 100644 index 0000000000..0d1d1633ad --- /dev/null +++ b/javascript/src/location.js @@ -0,0 +1,40 @@ +/** + * A location in the source code. Locations are stored as byte offsets into the + * source, because bytes are the unit that the parser itself works in. + */ +export class Location { + /** + * The byte offset from the beginning of the source where this location + * starts. + * + * @type {number} + */ + startOffset; + + /** + * The length of this location in bytes. + * + * @type {number} + */ + length; + + /** + * Construct a new Location. + * + * @param {number} startOffset + * @param {number} length + */ + constructor(startOffset, length) { + this.startOffset = startOffset; + this.length = length; + } + + /** + * The byte offset from the beginning of the source where this location ends. + * + * @returns {number} + */ + endOffset() { + return this.startOffset + this.length; + } +} diff --git a/javascript/test.js b/javascript/test.js index 26dca10b74..934592cc9e 100644 --- a/javascript/test.js +++ b/javascript/test.js @@ -2,6 +2,7 @@ import test from "node:test"; import assert from "node:assert"; import { loadPrism } from "./src/index.js"; import * as nodes from "./src/nodes.js"; +import { Location } from "./src/location.js"; import { Visitor } from "./src/visitor.js"; const parse = await loadPrism(); @@ -142,12 +143,18 @@ test("constant[]", async() => { test("location", () => { const result = parse("foo = 1"); - assert(typeof result.value.location.startOffset === "number"); + const location = result.value.location; + + assert(location instanceof Location); + assert(location.startOffset === 0); + assert(location.length === 7); + assert(location.endOffset() === 7); }); test("location? present", () => { const result = parse("def foo = bar"); - assert(statement(result).equalLoc !== null); + + assert(statement(result).equalLoc instanceof Location); }); test("location? absent", () => { diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index 49499c43ea..839a0b3468 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -1,4 +1,5 @@ import * as nodes from "./nodes.js"; +import { Location } from "./location.js"; const MAJOR_VERSION = 1; const MINOR_VERSION = 9; @@ -82,7 +83,9 @@ class SerializationBuffer { } readLocation() { - return { startOffset: this.readVarInt(), length: this.readVarInt() }; + const startOffset = this.readVarInt(); + const length = this.readVarInt(); + return new Location(startOffset, length); } readOptionalLocation() { @@ -171,12 +174,6 @@ class SerializationBuffer { } } -/** - * A location in the source code. - * - * @typedef {{ startOffset: number, length: number }} Location - */ - /** * A comment in the source code. * @@ -229,6 +226,7 @@ export class ParseResult { /** * @type {Location | null} */ + dataLoc; /** * @type {ParseError[]} @@ -244,6 +242,7 @@ export class ParseResult { * @param {nodes.ProgramNode} value * @param {Comment[]} comments * @param {MagicComment[]} magicComments + * @param {Location | null} dataLoc * @param {ParseError[]} errors * @param {ParseWarning[]} warnings */ diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb index f31c2ad216..23ea1de0f1 100644 --- a/templates/javascript/src/nodes.js.erb +++ b/templates/javascript/src/nodes.js.erb @@ -24,6 +24,7 @@ def jstype(field) end end -%> +import { Location } from "./location.js" import * as visitors from "./visitor.js" <%- flags.each do |flag| -%> @@ -38,12 +39,6 @@ const <%= flag.name %> = { }; <%- end -%> -/** - * A location in the source code. - * - * @typedef {{ startOffset: number, length: number }} Location - */ - /** * An encoded Ruby string. * From d12f46e608bbf266f4eea0d4bfc9c68170f218d5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Sep 2026 18:00:29 -0400 Subject: [PATCH 02/13] Add a Source class --- javascript/src/decoding.js | 56 +++ javascript/src/index.js | 1 + javascript/src/location.js | 64 +++- javascript/src/parsePrism.js | 4 +- javascript/src/source.js | 148 +++++++ javascript/test.js | 404 ++++++++++++-------- templates/javascript/src/deserialize.js.erb | 132 +++---- templates/javascript/src/nodes.js.erb | 4 +- 8 files changed, 565 insertions(+), 248 deletions(-) create mode 100644 javascript/src/decoding.js create mode 100644 javascript/src/source.js diff --git a/javascript/src/decoding.js b/javascript/src/decoding.js new file mode 100644 index 0000000000..31e2baa3d0 --- /dev/null +++ b/javascript/src/decoding.js @@ -0,0 +1,56 @@ +/** + * An object that can be used to decode a byte array into a string. + * + * @typedef {{ decode: (bytes: Uint8Array) => string }} Decoder + */ + +/** + * The decoder used both as the decoder for binary and the fallback in case the + * encoding is not supported. + */ +const binaryTextDecoder = { + /** + * Decodes a byte array into a string by treating each byte as a single + * character. This is used for the ASCII-8BIT encoding from Ruby. + * + * @param {Uint8Array} bytes + * @returns {string} + */ + decode(bytes) { + let result = ""; + for (const byte of bytes) { + result += String.fromCharCode(byte); + } + + return result; + } +}; + +/** + * Get a Decoder from the encoding name. If the encoding is not supported, a + * decoder that treats each byte as a single character is returned. + * + * @param {string} name + * @param {{ fatal: boolean }} options + * @returns {Decoder} + */ +export function getDecoder(name, options = { fatal: false }) { + const lower = name.toLowerCase(); + let decoder = null; + + if (lower === "ascii-8bit" || lower === "binary") { + decoder = binaryTextDecoder; + } else { + try { + decoder = new TextDecoder(lower, options); + } catch (error) { + if (error instanceof RangeError) { + decoder = binaryTextDecoder; + } else { + throw error; + } + } + } + + return decoder; +} diff --git a/javascript/src/index.js b/javascript/src/index.js index a0945e7d2a..9562df8ff3 100644 --- a/javascript/src/index.js +++ b/javascript/src/index.js @@ -6,6 +6,7 @@ import { ParseResult } from "./deserialize.js"; import { parsePrism } from "./parsePrism.js"; export * from "./location.js"; +export * from "./source.js"; export * from "./visitor.js"; export * from "./nodes.js"; diff --git a/javascript/src/location.js b/javascript/src/location.js index 0d1d1633ad..00f410b5a1 100644 --- a/javascript/src/location.js +++ b/javascript/src/location.js @@ -1,8 +1,21 @@ +import { Source } from "./source.js"; + /** * A location in the source code. Locations are stored as byte offsets into the - * source, because bytes are the unit that the parser itself works in. + * source, because bytes are the unit that the parser itself works in. They hold + * a pointer to the source they came from so that they can resolve line numbers, + * columns, and the source code that they represent. */ export class Location { + /** + * The source that this location points into. It is private so that it stays + * off of the instance itself, which keeps locations cheap to serialize with + * JSON.stringify. + * + * @type {Source} + */ + #source; + /** * The byte offset from the beginning of the source where this location * starts. @@ -21,10 +34,12 @@ export class Location { /** * Construct a new Location. * + * @param {Source} source * @param {number} startOffset * @param {number} length */ - constructor(startOffset, length) { + constructor(source, startOffset, length) { + this.#source = source; this.startOffset = startOffset; this.length = length; } @@ -37,4 +52,49 @@ export class Location { endOffset() { return this.startOffset + this.length; } + + /** + * The line number where this location starts. + * + * @returns {number} + */ + startLine() { + return this.#source.line(this.startOffset); + } + + /** + * The line number where this location ends. + * + * @returns {number} + */ + endLine() { + return this.#source.line(this.endOffset()); + } + + /** + * The column in bytes where this location starts from the start of its line. + * + * @returns {number} + */ + startColumn() { + return this.#source.column(this.startOffset); + } + + /** + * The column in bytes where this location ends from the start of its line. + * + * @returns {number} + */ + endColumn() { + return this.#source.column(this.endOffset()); + } + + /** + * The source code that this location represents. + * + * @returns {string} + */ + slice() { + return this.#source.slice(this.startOffset, this.length); + } } diff --git a/javascript/src/parsePrism.js b/javascript/src/parsePrism.js index 615cc83d13..b85047be3f 100644 --- a/javascript/src/parsePrism.js +++ b/javascript/src/parsePrism.js @@ -41,7 +41,7 @@ export function parsePrism(prism, source, options = {}) { prism.pm_serialize_parse(bufferPointer, sourcePointer, sourceArray.length, optionsPointer); const serializedView = new Uint8Array(prism.memory.buffer, prism.pm_buffer_value(bufferPointer), prism.pm_buffer_length(bufferPointer)); - const result = deserialize(serializedView); + const result = deserialize(sourceArray, serializedView); prism.pm_buffer_free(bufferPointer); prism.free(sourcePointer); @@ -118,7 +118,7 @@ function dumpOptions(options) { } template.push("l"); - values.push(options.line || 1); + values.push(options.line === undefined ? 1 : options.line); template.push("L"); if (options.encoding) { diff --git a/javascript/src/source.js b/javascript/src/source.js new file mode 100644 index 0000000000..6020e97974 --- /dev/null +++ b/javascript/src/source.js @@ -0,0 +1,148 @@ +import { getDecoder } from "./decoding.js"; + +/** + * A source of Ruby code that has been parsed. Locations hold a pointer to the + * source they came from, which is what allows them to resolve line numbers, + * columns, and the source code that they represent. + */ +export class Source { + /** + * The bytes of the source code, in the encoding that it was parsed in. + * + * @type {Uint8Array} + */ + bytes; + + /** + * The name of the encoding that the source code is in, as determined by the + * parser options or by the encoding magic comment. + * + * @type {string} + */ + encoding; + + /** + * The line number that the source starts on. + * + * @type {number} + */ + startLine; + + /** + * The byte offset of the start of each line in the source code. The first + * element is always 0 to mark the first line. + * + * @type {number[]} + */ + offsets; + + /** + * The decoder used to convert this source's bytes into strings. It is created + * on first use because not every encoding that the parser accepts has a + * TextDecoder equivalent, and parsing should not fail on that basis. + * + * @type {TextDecoder | null} + */ + #decoder; + + /** + * Construct a new Source. + * + * @param {Uint8Array} bytes + * @param {string} encoding + * @param {number} startLine + * @param {number[]} offsets + */ + constructor(bytes, encoding, startLine, offsets) { + this.bytes = bytes; + this.encoding = encoding; + this.startLine = startLine; + this.offsets = offsets; + this.#decoder = null; + } + + /** + * Decode the given byte range of the source code into a string. Because a + * byte range can begin or end in the middle of a multi-byte character, bytes + * that do not form a whole character are decoded into replacement + * characters. + * + * @param {number} byteOffset + * @param {number} length + * @returns {string} + */ + slice(byteOffset, length) { + if (this.#decoder === null) { + this.#decoder = getDecoder(this.encoding); + } + + return this.#decoder.decode(this.bytes.subarray(byteOffset, byteOffset + length)); + } + + /** + * The line number that the given byte offset is on. + * + * @param {number} byteOffset + * @returns {number} + */ + line(byteOffset) { + return this.startLine + this.findLine(byteOffset); + } + + /** + * The byte offset of the start of the line that the given byte offset is on. + * + * @param {number} byteOffset + * @returns {number} + */ + lineStart(byteOffset) { + return this.offsets[this.findLine(byteOffset)]; + } + + /** + * The byte offset of the end of the line that the given byte offset is on. + * + * @param {number} byteOffset + * @returns {number} + */ + lineEnd(byteOffset) { + const offset = this.offsets[this.findLine(byteOffset) + 1]; + return offset === undefined ? this.bytes.length : offset; + } + + /** + * The column in bytes of the given byte offset from the start of its line. + * + * @param {number} byteOffset + * @returns {number} + */ + column(byteOffset) { + return byteOffset - this.lineStart(byteOffset); + } + + /** + * Binary search through the offsets to find the index of the line that the + * given byte offset is on. + * + * @param {number} byteOffset + * @returns {number} + */ + findLine(byteOffset) { + let low = 0; + let high = this.offsets.length; + + /* Find the first line that starts after the given byte offset; the line + * that contains the offset is the one before it. */ + while (low < high) { + const middle = (low + high) >>> 1; + + if (this.offsets[middle] > byteOffset) { + high = middle; + } else { + low = middle + 1; + } + } + + return low - 1; + } +} diff --git a/javascript/test.js b/javascript/test.js index 934592cc9e..e4369e25c9 100644 --- a/javascript/test.js +++ b/javascript/test.js @@ -1,8 +1,9 @@ -import test from "node:test"; +import test, { suite } from "node:test"; import assert from "node:assert"; import { loadPrism } from "./src/index.js"; import * as nodes from "./src/nodes.js"; import { Location } from "./src/location.js"; +import { Source } from "./src/source.js"; import { Visitor } from "./src/visitor.js"; const parse = await loadPrism(); @@ -19,191 +20,197 @@ function eachNode(node, callback) { } } -test("node", () => { - const result = parse("foo"); - assert(result.value instanceof nodes.ProgramNode); -}); - -test("node? present", () => { - const result = parse("foo.bar"); - assert(statement(result).receiver instanceof nodes.CallNode); -}); - -test("node? absent", () => { - const result = parse("foo"); - assert(statement(result).receiver === null); -}); - -test("node[]", () => { - const result = parse("foo.bar"); - assert(result.value.statements.body instanceof Array); -}); - -test("string", () => { - const result = parse('"foo"'); - const node = statement(result); - - assert(!node.isForcedUtf8Encoding()) - assert(!node.isForcedBinaryEncoding()) - - assert(node.unescaped.value === "foo"); - assert(node.unescaped.encoding === "utf-8"); - assert(node.unescaped.validEncoding); -}); - -test("forced utf-8 string using \\u syntax", () => { - const result = parse('# encoding: utf-8\n"\\u{9E7F}"'); - const node = statement(result); - const str = node.unescaped; - - assert(node.isForcedUtf8Encoding()); - assert(!node.isForcedBinaryEncoding()); - - assert(str.value === "鹿"); - assert(str.encoding === "utf-8"); - assert(str.validEncoding); -}); - -test("forced utf-8 string with invalid byte sequence", () => { - const result = parse('# encoding: utf-8\n"\\xFF\\xFF\\xFF"'); - const node = statement(result); - const str = node.unescaped; - - assert(node.isForcedUtf8Encoding()); - assert(!node.isForcedBinaryEncoding()); - - assert(str.value === "ÿÿÿ"); - assert(str.encoding === "utf-8"); - assert(!str.validEncoding); -}); - -test("ascii string with embedded utf-8 character", () => { - // # encoding: ascii\n"鹿"' - // # encoding: ascii\n"鹿"' - const ascii_str = new Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 233, 185, 191, 34]); - const result = parse(ascii_str); - const node = statement(result); - const str = node.unescaped; - - assert(!node.isForcedUtf8Encoding()); - assert(node.isForcedBinaryEncoding()); - - assert(str.value === "鹿"); - assert(str.encoding === "ascii"); - assert(str.validEncoding); -}); - -test("forced binary string", () => { - const result = parse('# encoding: ascii\n"\\xFF\\xFF\\xFF"'); - const node = statement(result); - const str = node.unescaped; - - assert(!node.isForcedUtf8Encoding()); - assert(node.isForcedBinaryEncoding()); - - assert(str.value === "ÿÿÿ"); - assert(str.encoding === "ascii"); - assert(str.validEncoding); -}); - -test("forced binary string with Unicode character", () => { - // # encoding: us-ascii\n"\\xFF鹿\\xFF" - const ascii_str = Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 92, 120, 70, 70, 233, 185, 191, 92, 120, 70, 70, 34]); - const result = parse(ascii_str); - const node = statement(result); - const str = node.unescaped; - - assert(!node.isForcedUtf8Encoding()); - assert(node.isForcedBinaryEncoding()); +suite("fields", () => { + test("node", () => { + const result = parse("foo"); + assert(result.value instanceof nodes.ProgramNode); + }); + + test("node? present", () => { + const result = parse("foo.bar"); + assert(statement(result).receiver instanceof nodes.CallNode); + }); + + test("node? absent", () => { + const result = parse("foo"); + assert(statement(result).receiver === null); + }); + + test("node[]", () => { + const result = parse("foo.bar"); + assert(result.value.statements.body instanceof Array); + }); + + suite("string", () => { + test("basic", () => { + const result = parse('"foo"'); + const node = statement(result); + + assert(!node.isForcedUtf8Encoding()) + assert(!node.isForcedBinaryEncoding()) + + assert(node.unescaped.value === "foo"); + assert(node.unescaped.encoding === "utf-8"); + assert(node.unescaped.validEncoding); + }); - assert(str.value === "ÿ鹿ÿ"); - assert(str.encoding === "ascii"); - assert(str.validEncoding); -}); + test("forced utf-8 using \\u syntax", () => { + const result = parse('# encoding: utf-8\n"\\u{9E7F}"'); + const node = statement(result); + const str = node.unescaped; -test("constant", () => { - const result = parse("foo = 1"); - assert(result.value.locals[0] === "foo"); -}); + assert(node.isForcedUtf8Encoding()); + assert(!node.isForcedBinaryEncoding()); -test("constant? present", () => { - const result = parse("def foo(*bar); end"); - assert(statement(result).parameters.rest.name === "bar"); -}); + assert(str.value === "鹿"); + assert(str.encoding === "utf-8"); + assert(str.validEncoding); + }); -test("constant? absent", () => { - const result = parse("def foo(*); end"); - assert(statement(result).parameters.rest.name === null); -}); + test("forced utf-8 string with invalid byte sequence", () => { + const result = parse('# encoding: utf-8\n"\\xFF\\xFF\\xFF"'); + const node = statement(result); + const str = node.unescaped; -test("constant[]", async() => { - const result = parse("foo = 1"); - assert(result.value.locals instanceof Array); -}); + assert(node.isForcedUtf8Encoding()); + assert(!node.isForcedBinaryEncoding()); -test("location", () => { - const result = parse("foo = 1"); - const location = result.value.location; + assert(str.value === "ÿÿÿ"); + assert(str.encoding === "utf-8"); + assert(!str.validEncoding); + }); - assert(location instanceof Location); - assert(location.startOffset === 0); - assert(location.length === 7); - assert(location.endOffset() === 7); -}); + test("ascii with embedded utf-8 character", () => { + // # encoding: ascii\n"鹿"' + // # encoding: ascii\n"鹿"' + const ascii_str = new Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 233, 185, 191, 34]); + const result = parse(ascii_str); + const node = statement(result); + const str = node.unescaped; -test("location? present", () => { - const result = parse("def foo = bar"); + assert(!node.isForcedUtf8Encoding()); + assert(node.isForcedBinaryEncoding()); - assert(statement(result).equalLoc instanceof Location); -}); + assert(str.value === "鹿"); + assert(str.encoding === "ascii-8bit"); + assert(str.validEncoding); + }); -test("location? absent", () => { - const result = parse("def foo; bar; end"); - assert(statement(result).equalLoc === null); -}); + test("forced binary", () => { + const result = parse('# encoding: ascii\n"\\xFF\\xFF\\xFF"'); + const node = statement(result); + const str = node.unescaped; -test("uint8", () => { - const result = parse("-> { _3 }"); - assert(statement(result).parameters.maximum === 3); -}); + assert(!node.isForcedUtf8Encoding()); + assert(node.isForcedBinaryEncoding()); -test("uint32", () => { - const result = parse("foo = 1"); - assert(statement(result).depth === 0); -}); + assert(str.value === "ÿÿÿ"); + assert(str.encoding === "ascii-8bit"); + assert(str.validEncoding); + }); -test("flags", () => { - const result = parse("/foo/mi"); - const regexp = statement(result); + test("forced binary with Unicode character", () => { + // # encoding: us-ascii\n"\\xFF鹿\\xFF" + const ascii_str = Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 92, 120, 70, 70, 233, 185, 191, 92, 120, 70, 70, 34]); + const result = parse(ascii_str); + const node = statement(result); + const str = node.unescaped; - assert(regexp.isIgnoreCase()); - assert(regexp.isMultiLine()); - assert(!regexp.isExtended()); -}); + assert(!node.isForcedUtf8Encoding()); + assert(node.isForcedBinaryEncoding()); -test("integer (decimal)", () => { - const result = parse("10"); - assert(statement(result).value === 10); -}); + assert(str.value === "ÿ鹿ÿ"); + assert(str.encoding === "ascii-8bit"); + assert(str.validEncoding); + }); + }); + + test("constant", () => { + const result = parse("foo = 1"); + assert(result.value.locals[0] === "foo"); + }); + + test("constant? present", () => { + const result = parse("def foo(*bar); end"); + assert(statement(result).parameters.rest.name === "bar"); + }); + + test("constant? absent", () => { + const result = parse("def foo(*); end"); + assert(statement(result).parameters.rest.name === null); + }); + + test("constant[]", async() => { + const result = parse("foo = 1"); + assert(result.value.locals instanceof Array); + }); + + test("location", () => { + const result = parse("foo = 1"); + const location = result.value.location; + + assert(location instanceof Location); + assert(location.startOffset === 0); + assert(location.length === 7); + assert(location.endOffset() === 7); + }); + + test("location? present", () => { + const result = parse("def foo = bar"); + + assert(statement(result).equalLoc instanceof Location); + }); + + test("location? absent", () => { + const result = parse("def foo; bar; end"); + assert(statement(result).equalLoc === null); + }); + + test("uint8", () => { + const result = parse("-> { _3 }"); + assert(statement(result).parameters.maximum === 3); + }); + + test("uint32", () => { + const result = parse("foo = 1"); + assert(statement(result).depth === 0); + }); + + test("flags", () => { + const result = parse("/foo/mi"); + const regexp = statement(result); + + assert(regexp.isIgnoreCase()); + assert(regexp.isMultiLine()); + assert(!regexp.isExtended()); + }); + + suite("integer", () => { + test("decimal", () => { + const result = parse("10"); + assert(statement(result).value === 10); + }); -test("integer (hex)", () => { - const result = parse("0xA"); - assert(statement(result).value === 10); -}); + test("hex", () => { + const result = parse("0xA"); + assert(statement(result).value === 10); + }); -test("integer (2 nodes)", () => { - const result = parse("4294967296"); - assert(statement(result).value === 4294967296n); -}); + test("2 nodes", () => { + const result = parse("4294967296"); + assert(statement(result).value === 4294967296n); + }); -test("integer (3 nodes)", () => { - const result = parse("18446744073709552000"); - assert(statement(result).value === 18446744073709552000n); -}); + test("3 nodes", () => { + const result = parse("18446744073709552000"); + assert(statement(result).value === 18446744073709552000n); + }); + }); -test("double", () => { - const result = parse("1.0"); - assert(statement(result).value === 1.0); + test("double", () => { + const result = parse("1.0"); + assert(statement(result).value === 1.0); + }); }); test("scopes", () => { @@ -266,3 +273,66 @@ test("visitor visits nodes inside node list fields", () => { assert.deepStrictEqual(collect("begin\n a\nrescue B\n c\nend"), ["a", "c"]); assert.deepStrictEqual(collect("def foo(a = b); end"), ["b"]); }); + +test("source", () => { + const source = new Source(new TextEncoder().encode("foo\nbar\nbaz"), "utf-8", 1, [0, 4, 8]); + + assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.line(offset)), [1, 2, 3, 3]); + assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.lineStart(offset)), [0, 4, 8, 8]); + assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.lineEnd(offset)), [4, 8, 11, 11]); + assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.column(offset)), [0, 0, 0, 3]); + + assert(source.slice(4, 3) === "bar"); + assert(source.slice(0, 11) === "foo\nbar\nbaz"); +}); + +suite("location", () => { + test("lines and columns", () => { + const result = parse("foo\nbar\nbaz"); + const body = result.value.statements.body; + + assert.deepStrictEqual(body.map((node) => node.location.startLine()), [1, 2, 3]); + assert.deepStrictEqual(body.map((node) => node.location.endLine()), [1, 2, 3]); + assert.deepStrictEqual(body.map((node) => node.location.startColumn()), [0, 0, 0]); + assert.deepStrictEqual(body.map((node) => node.location.endColumn()), [3, 3, 3]); + }); + + test("lines and columns spanning multiple lines", () => { + const result = parse("foo(\n bar\n)"); + const location = statement(result).location; + + assert(location.startLine() === 1); + assert(location.startColumn() === 0); + assert(location.endLine() === 3); + assert(location.endColumn() === 1); + }); + + test("lines respect the line option", () => { + for (const line of [-2147483648, -1073741824, -5, -1, 0, 1, 10, 1073741824, 2147483646]) { + const result = parse("foo\nbar", { line }); + const body = result.value.statements.body; + + assert.deepStrictEqual(body.map((node) => node.location.startLine()), [line, line + 1]); + } + }); + + test("slice", () => { + const result = parse("foo(bar)"); + const node = statement(result); + + assert(node.location.slice() === "foo(bar)"); + assert(node.messageLoc.slice() === "foo"); + assert(node.arguments_.arguments_[0].location.slice() === "bar"); + }); + + test("slice with multibyte characters", () => { + const result = parse('"鹿" + "foo"'); + const node = statement(result); + + assert(node.receiver.location.slice() === '"鹿"'); + assert(node.receiver.location.startColumn() === 0); + + assert(node.receiver.location.endColumn() === 5); + assert(node.arguments_.arguments_[0].location.slice() === '"foo"'); + }); +}); diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index 839a0b3468..1f01599054 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -1,14 +1,16 @@ import * as nodes from "./nodes.js"; +import { getDecoder } from "./encoding.js"; import { Location } from "./location.js"; +import { Source } from "./source.js"; const MAJOR_VERSION = 1; const MINOR_VERSION = 9; const PATCH_VERSION = 0; -// The DataView getFloat64 function takes an optional second argument that -// specifies whether the number is little-endian or big-endian. It does not -// appear to have a native endian mode, so we need to determine the endianness -// of the system at runtime. +/* The DataView getFloat64 function takes an optional second argument that + * specifies whether the number is little-endian or big-endian. It does not + * appear to have a native endian mode, so we need to determine the endianness + * of the system at runtime. */ const LITTLE_ENDIAN = (() => { let uint32 = new Uint32Array([0x11223344]); let uint8 = new Uint8Array(uint32.buffer); @@ -23,18 +25,11 @@ const LITTLE_ENDIAN = (() => { })(); class SerializationBuffer { - FORCED_UTF8_ENCODING_FLAG = 1 << 2; - FORCED_BINARY_ENCODING_FLAG = 1 << 3; - - DECODER_MAP = new Map([ - ["ascii-8bit", "ascii"] - ]); - constructor(array) { this.array = array; this.index = 0; - this.fileEncoding = "utf-8"; - this.decoders = new Map(); + this.encoding = "utf-8"; + this.source = null; } readByte() { @@ -53,7 +48,7 @@ class SerializationBuffer { return this.decodeString(this.readBytes(length), flags).value; } - // Read a 32-bit unsigned integer in little-endian format. + /* Read a 32-bit unsigned integer in little-endian format. */ readUint32() { const result = this.scanUint32(this.index); this.index += 4; @@ -65,6 +60,17 @@ class SerializationBuffer { return bytes[0] | (bytes[1] << 8) | (bytes[2] << 16) | (bytes[3] << 24); } + /** + * Read a signed integer that was written in zigzag encoding, where the low + * bit carries the sign. Both operators here are 32-bit, which is what makes + * this correct for zigzag values above 2^31, where readVarInt overflows and + * returns the bit pattern as a negative number. + */ + readVarSInt() { + const value = this.readVarInt(); + return (value >>> 1) ^ -(value & 1); + } + readVarInt() { let result = 0; let shift = 0; @@ -85,7 +91,7 @@ class SerializationBuffer { readLocation() { const startOffset = this.readVarInt(); const length = this.readVarInt(); - return new Location(startOffset, length); + return new Location(this.source, startOffset, length); } readOptionalLocation() { @@ -105,8 +111,7 @@ class SerializationBuffer { const offset = constantPoolOffset + constantIndex * 8; const startOffset = this.scanUint32(offset); const length = this.scanUint32(offset + 4); - - return this.getDecoder(this.fileEncoding).decode(this.array.slice(startOffset, startOffset + length)); + return getDecoder(this.encoding, { fatal: true }).decode(this.array.slice(startOffset, startOffset + length)); } readDouble() { @@ -119,58 +124,31 @@ class SerializationBuffer { } decodeString(bytes, flags) { - const forcedBin = (flags & this.FORCED_BINARY_ENCODING_FLAG) !== 0; - const forcedUtf8 = (flags & this.FORCED_UTF8_ENCODING_FLAG) !== 0; - - if (forcedBin) { - // just use raw bytes - return { - encoding: "ascii", - validEncoding: true, - value: this.asciiDecoder.decode(bytes) - }; - } else { - const encoding = forcedUtf8 ? "utf-8" : this.fileEncoding.toLowerCase(); - const decoder = this.getDecoder(encoding); - - try { - // decode with encoding - return { - encoding, - validEncoding: true, - value: decoder.decode(bytes) - }; - } catch(e) { - // just use raw bytes, capture what the encoding should be, set flag saying encoding is invalid - if (e.code === "ERR_ENCODING_INVALID_ENCODED_DATA") { - return { - encoding, - validEncoding: false, - value: this.asciiDecoder.decode(bytes) - }; - } - - throw e; - } - } - } - - getDecoder(encoding) { - encoding = this.DECODER_MAP.get(encoding) || encoding; + let value, encoding, validEncoding; - if (!this.decoders.has(encoding)) { - this.decoders.set(encoding, new TextDecoder(encoding, {fatal: true})); + if ((flags & nodes.StringFlags.FORCED_BINARY_ENCODING) !== 0) { + encoding = "ascii-8bit"; + } else if ((flags & nodes.StringFlags.FORCED_UTF8_ENCODING) !== 0) { + encoding = "utf-8"; + } else { + encoding = this.encoding.toLowerCase(); } - return this.decoders.get(encoding); - } + const decoder = getDecoder(encoding, { fatal: true }); - get asciiDecoder() { - if (!this._asciiDecoder) { - this._asciiDecoder = new TextDecoder("ascii"); + try { + value = decoder.decode(bytes); + validEncoding = true; + } catch (error) { + if (error instanceof TypeError && error.code === "ERR_ENCODING_INVALID_ENCODED_DATA") { + value = getDecoder("ascii-8bit").decode(bytes); + validEncoding = false; + } else { + throw error; + } } - return this._asciiDecoder; + return { value, encoding, validEncoding }; } } @@ -238,6 +216,11 @@ export class ParseResult { */ warnings; + /** + * @type {boolean} + */ + continuable; + /** * @param {nodes.ProgramNode} value * @param {Comment[]} comments @@ -245,14 +228,16 @@ export class ParseResult { * @param {Location | null} dataLoc * @param {ParseError[]} errors * @param {ParseWarning[]} warnings + * @param {boolean} continuable */ - constructor(value, comments, magicComments, dataLoc, errors, warnings) { + constructor(value, comments, magicComments, dataLoc, errors, warnings, continuable) { this.value = value; this.comments = comments; this.magicComments = magicComments; this.dataLoc = dataLoc; this.errors = errors; this.warnings = warnings; + this.continuable = continuable; } } @@ -274,11 +259,12 @@ const warningTypes = [ * Accept two Uint8Arrays, one for the source and one for the serialized format. * Return the AST corresponding to the serialized form. * + * @param {Uint8Array} source * @param {Uint8Array} array * @returns {ParseResult} * @throws {Error} */ -export function deserialize(array) { +export function deserialize(source, array) { const buffer = new SerializationBuffer(array); if (buffer.readString(5) !== "PRISM") { @@ -294,18 +280,14 @@ export function deserialize(array) { } // Read the file's encoding. - buffer.fileEncoding = buffer.readString(buffer.readVarInt()); + buffer.encoding = buffer.readString(buffer.readVarInt()); - // Skip past the start line, as we don't support that option yet in - // JavaScript. - buffer.readVarInt(); + // Read the line that the source starts on along with the byte offset of the + // start of each line, which together let locations resolve lines and columns. + const startLine = buffer.readVarSInt(); + const lineOffsets = Array.from({ length: buffer.readVarInt() }, () => buffer.readVarInt()); - // Skip past the line offsets, as there is no Source object yet in JavaScript. - // const lineOffsets = Array.from({ length: buffer.readVarInt() }, () => buffer.readVarInt()); - const lineOffsetsCount = buffer.readVarInt(); - for (let i = 0; i < lineOffsetsCount; i ++) { - buffer.readVarInt(); - } + buffer.source = new Source(source, buffer.encoding, startLine, lineOffsets); const comments = Array.from({ length: buffer.readVarInt() }, () => ({ type: buffer.readVarInt(), diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb index 23ea1de0f1..fcab285461 100644 --- a/templates/javascript/src/nodes.js.erb +++ b/templates/javascript/src/nodes.js.erb @@ -32,7 +32,7 @@ import * as visitors from "./visitor.js" /** * <%= flag.comment %> */ -const <%= flag.name %> = { +export const <%= flag.name %> = { <%- flag.values.each_with_index do |value, index| -%> <%= value.name %>: 1 << <%= index + Prism::Template::COMMON_FLAGS_COUNT %>, <%- end -%> @@ -132,7 +132,7 @@ export class <%= node.name -%> { when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "this.#{prop(field)}" when Prism::Template::NodeListField then "...this.#{prop(field)}" end - }.compact.join(", ") %>] + }.compact.join(", ") %>]; } /** From cef8af6ea876e489c5f1e71a454fc678c0162546 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Sep 2026 22:32:04 -0400 Subject: [PATCH 03/13] JavaScript: make parsePrism accept a Uint8Array --- doc/playground.js | 18 ++++++++---------- docs/javascript.md | 2 +- javascript/example.html | 2 +- javascript/src/parsePrism.js | 18 ++++++++---------- javascript/test.js | 6 +++++- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/doc/playground.js b/doc/playground.js index ee33a0f965..d6e827598b 100644 --- a/doc/playground.js +++ b/doc/playground.js @@ -117,8 +117,7 @@ end }; // URL-safe base64 encode/decode (RFC 4648 §5) -function encodeSource(str) { - const bytes = encoder.encode(str); +function encodeSource(bytes) { let binary = ""; for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]); return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); @@ -159,7 +158,7 @@ const monacoEditor = monaco.editor.create(document.getElementById("monaco-contai let currentTab = "ast"; let lastResult = null; -let lastSource = ""; +let lastBytes = new Uint8Array(); let currentDecorations = []; // Tab switching @@ -481,10 +480,9 @@ function render() { output.setAttribute("aria-labelledby", currentTab === "ast" ? "tab-ast" : "tab-diagnostics"); - const utf8Bytes = encoder.encode(lastSource); switch (currentTab) { case "ast": - const tree = renderNode(lastResult.value, utf8Bytes, "", true, true); + const tree = renderNode(lastResult.value, lastBytes, "", true, true); output.innerHTML = tree ? `
${tree}
` : `
${escapeHtml(lastResult.error || "Failed to parse.")}
`; @@ -497,8 +495,8 @@ function render() { output.innerHTML = `
No errors or warnings.
`; } else { let html = ""; - for (const err of errors) html += renderDiagnostic(utf8Bytes, err, "Error"); - for (const warn of warnings) html += renderDiagnostic(utf8Bytes, warn, "Warning"); + for (const err of errors) html += renderDiagnostic(lastBytes, err, "Error"); + for (const warn of warnings) html += renderDiagnostic(lastBytes, warn, "Warning"); output.innerHTML = html; } break; @@ -509,10 +507,10 @@ let timeout = null; function parse() { if (timeout) clearTimeout(timeout); timeout = setTimeout(() => { - lastSource = monacoEditor.getValue(); - history.replaceState(null, "", `#${encodeSource(lastSource)}`); + lastBytes = encoder.encode(monacoEditor.getValue()); + history.replaceState(null, "", `#${encodeSource(lastBytes)}`); try { - lastResult = parsePrism(instance.exports, lastSource); + lastResult = parsePrism(instance.exports, lastBytes); } catch (e) { lastResult = { value: null, error: e.message, errors: [], warnings: [] }; } diff --git a/docs/javascript.md b/docs/javascript.md index 2ccd5f8231..ce6e1ba6d9 100644 --- a/docs/javascript.md +++ b/docs/javascript.md @@ -56,7 +56,7 @@ Finally, you can create a function that will parse a string of Ruby code. ```js function parse(source) { - return parsePrism(instance.exports, source); + return parsePrism(instance.exports, new TextEncoder().encode(source)); } ``` diff --git a/javascript/example.html b/javascript/example.html index d5ce3bbba6..99021614ab 100644 --- a/javascript/example.html +++ b/javascript/example.html @@ -30,7 +30,7 @@ if (timeout) clearTimeout(timeout); timeout = setTimeout(function () { - const result = parsePrism(instance.exports, event.target.value); + const result = parsePrism(instance.exports, new TextEncoder().encode(event.target.value)); output.textContent = JSON.stringify(result, null, 2); }, 250); }); diff --git a/javascript/src/parsePrism.js b/javascript/src/parsePrism.js index b85047be3f..ebe9785eb2 100644 --- a/javascript/src/parsePrism.js +++ b/javascript/src/parsePrism.js @@ -1,7 +1,7 @@ import { ParseResult, deserialize } from "./deserialize.js"; /** - * Parse the given source code. + * Parse the given source code represented as a Uint8Array. * * @typedef {{ * locals?: string[], @@ -18,30 +18,28 @@ import { ParseResult, deserialize } from "./deserialize.js"; * main_script?: boolean, * partial_script?: boolean, * scopes?: (string[] | Scope)[] - * }} Options + * }} Options * * @param {WebAssembly.Exports} prism - * @param {string} source + * @param {Uint8Array} source * @param {Options} options * @returns {ParseResult} */ export function parsePrism(prism, source, options = {}) { - const sourceArray = new TextEncoder().encode(source); - const sourcePointer = prism.calloc(1, sourceArray.length); - const packedOptions = dumpOptions(options); const optionsPointer = prism.calloc(1, packedOptions.length); const bufferPointer = prism.pm_buffer_new(); - const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, sourceArray.length); - sourceView.set(sourceArray); + const sourcePointer = prism.calloc(1, source.length); + const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, source.length); + sourceView.set(source); const optionsView = new Uint8Array(prism.memory.buffer, optionsPointer, packedOptions.length); optionsView.set(packedOptions); - prism.pm_serialize_parse(bufferPointer, sourcePointer, sourceArray.length, optionsPointer); + prism.pm_serialize_parse(bufferPointer, sourcePointer, source.length, optionsPointer); const serializedView = new Uint8Array(prism.memory.buffer, prism.pm_buffer_value(bufferPointer), prism.pm_buffer_length(bufferPointer)); - const result = deserialize(sourceArray, serializedView); + const result = deserialize(source, serializedView); prism.pm_buffer_free(bufferPointer); prism.free(sourcePointer); diff --git a/javascript/test.js b/javascript/test.js index e4369e25c9..bb0e86f4e0 100644 --- a/javascript/test.js +++ b/javascript/test.js @@ -6,7 +6,11 @@ import { Location } from "./src/location.js"; import { Source } from "./src/source.js"; import { Visitor } from "./src/visitor.js"; -const parse = await loadPrism(); +const parseArray = await loadPrism(); + +function parse(source, options = {}) { + return parseArray(new TextEncoder().encode(source), options); +} function statement(result) { return result.value.statements.body[0]; From b15f5c6e96410bf0b412393af87089be05baccdb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Sep 2026 10:13:50 -0400 Subject: [PATCH 04/13] Allow passing decoder through slice functions --- javascript/src/location.js | 5 +++-- javascript/src/source.js | 14 +++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/javascript/src/location.js b/javascript/src/location.js index 00f410b5a1..3a64f6baaa 100644 --- a/javascript/src/location.js +++ b/javascript/src/location.js @@ -92,9 +92,10 @@ export class Location { /** * The source code that this location represents. * + * @param {TextDecoder | null} decoder * @returns {string} */ - slice() { - return this.#source.slice(this.startOffset, this.length); + slice(decoder = null) { + return this.#source.slice(this.startOffset, this.length, decoder); } } diff --git a/javascript/src/source.js b/javascript/src/source.js index 6020e97974..a03a9f454e 100644 --- a/javascript/src/source.js +++ b/javascript/src/source.js @@ -41,7 +41,7 @@ export class Source { * on first use because not every encoding that the parser accepts has a * TextDecoder equivalent, and parsing should not fail on that basis. * - * @type {TextDecoder | null} + * @type {Decoder | null} */ #decoder; @@ -69,14 +69,18 @@ export class Source { * * @param {number} byteOffset * @param {number} length + * @param {TextDecoder | null} decoder * @returns {string} */ - slice(byteOffset, length) { - if (this.#decoder === null) { - this.#decoder = getDecoder(this.encoding); + slice(byteOffset, length, decoder = null) { + if (decoder === null) { + if (this.#decoder === null) { + this.#decoder = getDecoder(this.encoding); + } + decoder = this.#decoder; } - return this.#decoder.decode(this.bytes.subarray(byteOffset, byteOffset + length)); + return decoder.decode(this.bytes.subarray(byteOffset, byteOffset + length)); } /** From 491098cffccd886c5b938d2a3a984c7208dc239e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Sep 2026 16:24:46 -0400 Subject: [PATCH 05/13] Add JS code units counting --- javascript/src/location.js | 22 ++++++++++ javascript/src/source.js | 84 ++++++++++++++++++++++++++++++++++-- javascript/test.js | 87 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 4 deletions(-) diff --git a/javascript/src/location.js b/javascript/src/location.js index 3a64f6baaa..26fa507f47 100644 --- a/javascript/src/location.js +++ b/javascript/src/location.js @@ -89,6 +89,28 @@ export class Location { return this.#source.column(this.endOffset()); } + /** + * The column in code units where this location starts from the start of its + * line, counted in the given position encoding. + * + * @param {"utf-8" | "utf-16" | "utf-32"} encoding + * @returns {number} + */ + startCodeUnitsColumn(encoding = "utf-16") { + return this.#source.codeUnitsColumn(this.startOffset, encoding); + } + + /** + * The column in code units where this location ends from the start of its + * line, counted in the given position encoding. + * + * @param {"utf-8" | "utf-16" | "utf-32"} encoding + * @returns {number} + */ + endCodeUnitsColumn(encoding = "utf-16") { + return this.#source.codeUnitsColumn(this.endOffset(), encoding); + } + /** * The source code that this location represents. * diff --git a/javascript/src/source.js b/javascript/src/source.js index a03a9f454e..e69610cf2c 100644 --- a/javascript/src/source.js +++ b/javascript/src/source.js @@ -1,5 +1,12 @@ import { getDecoder } from "./decoding.js"; +/** + * The encoder used to count code units for the utf-8 position encoding. The + * TextEncoder interface only ever emits utf-8, which is exactly what is needed + * here. + */ +const encoder = new TextEncoder(); + /** * A source of Ruby code that has been parsed. Locations hold a pointer to the * source they came from, which is what allows them to resolve line numbers, @@ -74,15 +81,27 @@ export class Source { */ slice(byteOffset, length, decoder = null) { if (decoder === null) { - if (this.#decoder === null) { - this.#decoder = getDecoder(this.encoding); - } - decoder = this.#decoder; + decoder = this.#defaultDecoder(); } return decoder.decode(this.bytes.subarray(byteOffset, byteOffset + length)); } + /** + * The decoder for this source's own encoding. It is created on first use + * because not every encoding that the parser accepts has a TextDecoder + * equivalent, and parsing should not fail on that basis. + * + * @returns {Decoder} + */ + #defaultDecoder() { + if (this.#decoder === null) { + this.#decoder = getDecoder(this.encoding); + } + + return this.#decoder; + } + /** * The line number that the given byte offset is on. * @@ -124,6 +143,63 @@ export class Source { return byteOffset - this.lineStart(byteOffset); } + /** + * The column in code units of the given byte offset from the start of its + * line, counted in the given position encoding. A code unit is the smallest + * unit of an encoding form, so utf-8 counts bytes, utf-16 counts sixteen bit + * units where characters outside the basic multilingual plane take two, and + * utf-32 counts whole codepoints. + * + * These are the three position encodings of the language server protocol, + * spelled the way the protocol spells them, so a negotiated value can be + * passed straight through. It defaults to utf-16 because that is what the + * protocol defaults to. + * + * The prefix of the line is decoded through this source's own encoding + * first, so a given character resolves to the same column no matter which + * encoding the source was parsed in. + * + * @param {number} byteOffset + * @param {"utf-8" | "utf-16" | "utf-32"} encoding + * @returns {number} + */ + codeUnitsColumn(byteOffset, encoding = "utf-16") { + const lineStart = this.lineStart(byteOffset); + + /* Byte offsets are themselves utf-8 code units when the source is utf-8, + * and counting them directly keeps bytes that do not decode from inflating + * the column into the width of the replacement character. */ + if (encoding === "utf-8" && this.encoding.toLowerCase() === "utf-8") { + return byteOffset - lineStart; + } + + const prefix = this.#defaultDecoder().decode(this.bytes.subarray(lineStart, byteOffset)); + + switch (encoding) { + case "utf-8": + return encoder.encode(prefix).length; + case "utf-16": + return prefix.length; + case "utf-32": { + /* Every codepoint is one utf-16 code unit except those outside the + * basic multilingual plane, which are a surrogate pair. Decoders only + * ever produce well-formed utf-16, substituting the replacement + * character for anything they cannot pair up, so every low surrogate + * here closes a pair and dropping them leaves the codepoint count. */ + let count = prefix.length; + + for (let index = 0; index < prefix.length; index++) { + const unit = prefix.charCodeAt(index); + if (unit >= 0xdc00 && unit <= 0xdfff) count--; + } + + return count; + } + default: + throw new Error(`Unsupported position encoding '${encoding}'`); + } + } + /** * Binary search through the offsets to find the index of the line that the * given byte offset is on. diff --git a/javascript/test.js b/javascript/test.js index bb0e86f4e0..3131ec30dd 100644 --- a/javascript/test.js +++ b/javascript/test.js @@ -285,6 +285,13 @@ test("source", () => { assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.lineStart(offset)), [0, 4, 8, 8]); assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.lineEnd(offset)), [4, 8, 11, 11]); assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.column(offset)), [0, 0, 0, 3]); + assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.codeUnitsColumn(offset)), [0, 0, 0, 3]); + + for (const encoding of ["utf-8", "utf-16", "utf-32"]) { + assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.codeUnitsColumn(offset, encoding)), [0, 0, 0, 3]); + } + + assert.throws(() => source.codeUnitsColumn(0, "utf-64")); assert(source.slice(4, 3) === "bar"); assert(source.slice(0, 11) === "foo\nbar\nbaz"); @@ -339,4 +346,84 @@ suite("location", () => { assert(node.receiver.location.endColumn() === 5); assert(node.arguments_.arguments_[0].location.slice() === '"foo"'); }); + + test("code units columns default to utf-16 and count code units, not bytes", () => { + const result = parse('"鹿" + "foo"'); + const node = statement(result); + + assert(node.receiver.location.startCodeUnitsColumn() === 0); + assert(node.receiver.location.endCodeUnitsColumn() === 3); + + assert(node.arguments_.arguments_[0].location.startCodeUnitsColumn() === 6); + assert(node.arguments_.arguments_[0].location.endCodeUnitsColumn() === 11); + }); + + test("code units columns count in the requested position encoding", () => { + const result = parse('"鹿" + "foo"'); + const node = statement(result); + const argument = node.arguments_.arguments_[0]; + + // 鹿 is three bytes and one code unit in every form, so only utf-8 differs. + assert.deepStrictEqual( + ["utf-8", "utf-16", "utf-32"].map((encoding) => node.receiver.location.endCodeUnitsColumn(encoding)), + [5, 3, 3] + ); + + assert.deepStrictEqual( + ["utf-8", "utf-16", "utf-32"].map((encoding) => argument.location.startCodeUnitsColumn(encoding)), + [8, 6, 6] + ); + }); + + test("code units columns count characters outside the BMP per encoding", () => { + const result = parse('"\u{1F600}" + "foo"'); + const node = statement(result); + const argument = node.arguments_.arguments_[0]; + + // The emoji is four bytes, two utf-16 code units, and one codepoint. + assert(node.receiver.location.endColumn() === 6); + assert.deepStrictEqual( + ["utf-8", "utf-16", "utf-32"].map((encoding) => node.receiver.location.endCodeUnitsColumn(encoding)), + [6, 4, 3] + ); + + assert.deepStrictEqual( + ["utf-8", "utf-16", "utf-32"].map((encoding) => argument.location.endCodeUnitsColumn(encoding)), + [14, 12, 11] + ); + }); + + test("code units columns count bytes that do not decode as themselves in utf-8", () => { + // 0xff is not valid utf-8, and the parser tolerates it in the source. The + // editor still holds that one byte, so the utf-8 column stays 7 rather than + // widening to the three bytes a replacement character would encode to. + const bytes = new Uint8Array([0x78, 0x20, 0x3d, 0x20, 0x22, 0xff, 0x22]); + const source = new Source(bytes, "UTF-8", 1, [0]); + + assert(source.codeUnitsColumn(7, "utf-8") === 7); + assert(source.codeUnitsColumn(7, "utf-16") === 7); + }); + + test("code units columns in a source that is not utf-8", () => { + // x = "ソ" in Shift_JIS, where ソ is 0x83 0x5c and that trailing byte is an + // ASCII backslash. + const bytes = new Uint8Array([0x78, 0x20, 0x3d, 0x20, 0x22, 0x83, 0x5c, 0x22]); + const source = new Source(bytes, "Shift_JIS", 1, [0]); + + assert(source.column(8) === 8); + assert.deepStrictEqual( + ["utf-8", "utf-16", "utf-32"].map((encoding) => source.codeUnitsColumn(8, encoding)), + [9, 7, 7] + ); + }); + + test("code units columns are relative to the start of the line", () => { + const result = parse('x = 1\n"鹿" + "foo"'); + const node = result.value.statements.body[1]; + + assert(node.location.startLine() === 2); + assert(node.receiver.location.startCodeUnitsColumn() === 0); + assert(node.receiver.location.endCodeUnitsColumn() === 3); + assert(node.receiver.location.endCodeUnitsColumn("utf-8") === 5); + }); }); From 2a874304e6c2efb79c4b025f2e07346483052149 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Sep 2026 16:34:27 -0400 Subject: [PATCH 06/13] Use new APIs in playground --- doc/playground.js | 50 ++++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/doc/playground.js b/doc/playground.js index d6e827598b..fcc2cca4ae 100644 --- a/doc/playground.js +++ b/doc/playground.js @@ -158,7 +158,6 @@ const monacoEditor = monaco.editor.create(document.getElementById("monaco-contai let currentTab = "ast"; let lastResult = null; -let lastBytes = new Uint8Array(); let currentDecorations = []; // Tab switching @@ -243,28 +242,17 @@ document.getElementById("expand-all").addEventListener("click", () => { output.querySelectorAll(".tree-toggle").forEach(toggle => setToggleState(toggle, false)); }); -// Convert byte offset to line:column using the utf8 bytes -function offsetToLineCol(utf8Bytes, offset) { - let line = 1, col = 0; - for (let i = 0; i < offset && i < utf8Bytes.length; i++) { - // Check for newline - if (utf8Bytes[i] === 10) { line++; col = 0; } - else { col++; } - } - return { line, col }; -} - - -function formatLoc(utf8Bytes, loc, includeSlice) { +// Monaco counts columns in utf-16 code units, which is what the code units +// columns report, so the two line up without any conversion here. +function formatLoc(loc, includeSlice) { if (!loc || loc.startOffset === undefined) return null; - const start = offsetToLineCol(utf8Bytes, loc.startOffset); - const end = offsetToLineCol(utf8Bytes, loc.startOffset + loc.length); + const start = { line: loc.startLine(), col: loc.startCodeUnitsColumn() }; + const end = { line: loc.endLine(), col: loc.endCodeUnitsColumn() }; let text = `${start.line}:${start.col}-${end.line}:${end.col}`; if (includeSlice) { - const slice = decoder.decode(utf8Bytes.slice(loc.startOffset, loc.startOffset + loc.length)); - text = `${text} = ${escapeHtml(JSON.stringify(slice))}` + text = `${text} = ${escapeHtml(JSON.stringify(loc.slice()))}` } return { start, end, text }; } @@ -351,7 +339,7 @@ function hasChildNodes(fields, node) { const CONNECTOR = { last: "└── ", mid: "├── ", lastPad: " ", midPad: "│ " }; // Build the AST tree as interactive HTML -function renderNode(node, utf8Bytes, prefix, isLast, isRoot) { +function renderNode(node, prefix, isLast, isRoot) { if (!isNode(node)) return ""; const type = nodeType(node); @@ -364,7 +352,7 @@ function renderNode(node, utf8Bytes, prefix, isLast, isRoot) { if (!isRoot) html += ``; if (foldable) html += ``; - const loc = formatLoc(utf8Bytes, node.location, false); + const loc = formatLoc(node.location, false); const locAttrs = locDataAttrs(loc); html += `@ ${escapedType}`; @@ -393,7 +381,7 @@ function renderNode(node, utf8Bytes, prefix, isLast, isRoot) { html += `
${escapeHtml(field)}: (${value.length} item${value.length === 1 ? "" : "s"})
`; value.forEach((item, i) => { if (isNode(item)) { - html += renderNode(item, utf8Bytes, fieldChildPrefix, i === value.length - 1); + html += renderNode(item, fieldChildPrefix, i === value.length - 1); } else { const itemConnector = i === value.length - 1 ? CONNECTOR.last : CONNECTOR.mid; if (isConstant(item)) { @@ -406,9 +394,9 @@ function renderNode(node, utf8Bytes, prefix, isLast, isRoot) { } } else if (isNode(value)) { html += `
${escapeHtml(field)}:
`; - html += renderNode(value, utf8Bytes, fieldChildPrefix, true); + html += renderNode(value, fieldChildPrefix, true); } else if (typeof value === "object" && value.startOffset !== undefined) { - const fieldLoc = formatLoc(utf8Bytes, value, true); + const fieldLoc = formatLoc(value, true); if (fieldLoc) { html += `
${escapeHtml(field)}: ${fieldLoc.text}
`; } @@ -434,8 +422,8 @@ function escapeHtml(str) { } // Render a single diagnostic line -function renderDiagnostic(utf8Bytes, item, kind) { - const loc = formatLoc(utf8Bytes, item.location, false); +function renderDiagnostic(item, kind) { + const loc = formatLoc(item.location, false); const cssClass = kind === "Error" ? "error-text" : "warning-text"; return `
${kind}: ${escapeHtml(item.message)}${loc ? ` (${loc.text})` : ""}
`; } @@ -482,7 +470,7 @@ function render() { switch (currentTab) { case "ast": - const tree = renderNode(lastResult.value, lastBytes, "", true, true); + const tree = renderNode(lastResult.value, "", true, true); output.innerHTML = tree ? `
${tree}
` : `
${escapeHtml(lastResult.error || "Failed to parse.")}
`; @@ -495,8 +483,8 @@ function render() { output.innerHTML = `
No errors or warnings.
`; } else { let html = ""; - for (const err of errors) html += renderDiagnostic(lastBytes, err, "Error"); - for (const warn of warnings) html += renderDiagnostic(lastBytes, warn, "Warning"); + for (const err of errors) html += renderDiagnostic(err, "Error"); + for (const warn of warnings) html += renderDiagnostic(warn, "Warning"); output.innerHTML = html; } break; @@ -507,10 +495,10 @@ let timeout = null; function parse() { if (timeout) clearTimeout(timeout); timeout = setTimeout(() => { - lastBytes = encoder.encode(monacoEditor.getValue()); - history.replaceState(null, "", `#${encodeSource(lastBytes)}`); + const bytes = encoder.encode(monacoEditor.getValue()); + history.replaceState(null, "", `#${encodeSource(bytes)}`); try { - lastResult = parsePrism(instance.exports, lastBytes); + lastResult = parsePrism(instance.exports, bytes); } catch (e) { lastResult = { value: null, error: e.message, errors: [], warnings: [] }; } From 0285c9915015932f0115f21025e389bf33f1d959 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 11:28:55 -0400 Subject: [PATCH 07/13] Fix up playground encoding --- doc/playground.js | 126 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 6 deletions(-) diff --git a/doc/playground.js b/doc/playground.js index fcc2cca4ae..cca3724ee5 100644 --- a/doc/playground.js +++ b/doc/playground.js @@ -128,6 +128,110 @@ function decodeSource(str) { return decoder.decode(Uint8Array.from(atob(padded), ch => ch.codePointAt(0))); } +/* Ruby takes the source encoding from a magic comment on the first line, or the + * second when the first is a shebang. Every encoding the parser accepts is + * ascii compatible, which is what lets the comment be read before the encoding + * it names is known. */ +function declaredEncoding(source) { + const lines = source.split("\n", 2); + const line = lines[0].startsWith("#!") ? lines[1] : lines[0]; + const match = line && line.match(/^[ \t]*#.*?coding\s*[:=]\s*([\w-]+)/i); + return match ? match[1].toLowerCase() : "utf-8"; +} + +/* TextEncoder only ever emits utf-8, so an encoder for any other encoding is + * built by inverting the decoder the browser already ships. Sweeping the byte + * space costs enough that each one is built once and kept. + * + * The sweep reaches sequences of up to three bytes. Four byte sequences, which + * gb18030 uses for everything outside its two byte range, would take a million + * and a half more probes, so they are left out and a character that needs one + * is reported when the source actually uses it. Everything shorter, which is + * all of an encoding's common range, encodes exactly. */ +const encoders = new Map(); + +/* The character a byte sequence decodes to, or null when the sequence is not a + * single character. Sequences that decode to more than one character are + * rejected so that a run of single byte characters cannot shadow a real + * multi byte mapping. */ +function decodeSingle(decoder, bytes) { + try { + const character = decoder.decode(new Uint8Array(bytes)); + return [...character].length === 1 ? character : null; + } catch { + return null; + } +} + +function getEncoder(canonical) { + if (encoders.has(canonical)) return encoders.get(canonical); + + const decoder = new TextDecoder(canonical, { fatal: true }); + const map = new Map(); + + for (let byte = 0; byte < 0x100; byte++) { + const character = decodeSingle(decoder, [byte]); + if (character && !map.has(character)) map.set(character, [byte]); + } + + for (let lead = 0x80; lead < 0x100; lead++) { + for (let trail = 0; trail < 0x100; trail++) { + const character = decodeSingle(decoder, [lead, trail]); + if (character && !map.has(character)) map.set(character, [lead, trail]); + } + } + + /* euc-jp is the only encoding the browser decodes that reaches a third byte, + * which it spends on JIS X 0212 behind a 0x8f lead with both continuation + * bytes in 0xa1 to 0xfe. */ + if (canonical === "euc-jp") { + for (let mid = 0xa1; mid <= 0xfe; mid++) { + for (let trail = 0xa1; trail <= 0xfe; trail++) { + const character = decodeSingle(decoder, [0x8f, mid, trail]); + if (character && !map.has(character)) map.set(character, [0x8f, mid, trail]); + } + } + } + + const encode = (source) => { + const bytes = []; + + for (const character of source) { + const encoded = map.get(character); + if (!encoded) throw new Error(`${JSON.stringify(character)} could not be encoded as ${canonical}`); + bytes.push(...encoded); + } + + return new Uint8Array(bytes); + }; + + encoders.set(canonical, encode); + return encode; +} + +/* The bytes to hand the parser, in the encoding the source declares, so that it + * sees what it would see reading the same file from disk. */ +function sourceBytes(source) { + const name = declaredEncoding(source); + + /* A comment can spell an encoding several ways, so the choice is made on the + * canonical name the browser resolves it to rather than on what the comment + * says. That keeps utf-8 on the encoder the platform already has, whichever + * of its spellings was used. */ + let canonical; + try { + canonical = new TextDecoder(name).encoding; + } catch { + return { + bytes: encoder.encode(source), + notice: `This browser cannot encode ${name}, so the source was sent as utf-8. Results may differ from Ruby.` + }; + } + + if (canonical === "utf-8") return { bytes: encoder.encode(source) }; + return { bytes: getEncoder(canonical)(source) }; +} + // Read initial source from URL hash or use default function sourceFromHash() { const hash = location.hash.slice(1); @@ -158,6 +262,7 @@ const monacoEditor = monaco.editor.create(document.getElementById("monaco-contai let currentTab = "ast"; let lastResult = null; +let lastNotice = null; let currentDecorations = []; // Tab switching @@ -468,21 +573,23 @@ function render() { output.setAttribute("aria-labelledby", currentTab === "ast" ? "tab-ast" : "tab-diagnostics"); + const notice = lastNotice ? `
${escapeHtml(lastNotice)}
` : ""; + switch (currentTab) { case "ast": const tree = renderNode(lastResult.value, "", true, true); - output.innerHTML = tree + output.innerHTML = notice + (tree ? `
${tree}
` - : `
${escapeHtml(lastResult.error || "Failed to parse.")}
`; + : `
${escapeHtml(lastResult.error || "Failed to parse.")}
`); break; case "diagnostics": const errors = lastResult.errors || []; const warnings = lastResult.warnings || []; if (errors.length === 0 && warnings.length === 0) { - output.innerHTML = `
No errors or warnings.
`; + output.innerHTML = notice || `
No errors or warnings.
`; } else { - let html = ""; + let html = notice; for (const err of errors) html += renderDiagnostic(err, "Error"); for (const warn of warnings) html += renderDiagnostic(warn, "Warning"); output.innerHTML = html; @@ -495,11 +602,18 @@ let timeout = null; function parse() { if (timeout) clearTimeout(timeout); timeout = setTimeout(() => { - const bytes = encoder.encode(monacoEditor.getValue()); - history.replaceState(null, "", `#${encodeSource(bytes)}`); + const source = monacoEditor.getValue(); + + /* The hash carries the editor's text, which is independent of the encoding + * the source declares, so it stays utf-8. */ + history.replaceState(null, "", `#${encodeSource(encoder.encode(source))}`); + try { + const { bytes, notice } = sourceBytes(source); + lastNotice = notice || null; lastResult = parsePrism(instance.exports, bytes); } catch (e) { + lastNotice = null; lastResult = { value: null, error: e.message, errors: [], warnings: [] }; } render(); From 84467224753155bd15bcbca97098c8cca217a1ce Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 11:41:18 -0400 Subject: [PATCH 08/13] JS: Better documentation and types --- docs/javascript.md | 21 ++++++++++++++++++--- javascript/src/index.js | 6 ++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/javascript.md b/docs/javascript.md index ce6e1ba6d9..63ae2bae89 100644 --- a/docs/javascript.md +++ b/docs/javascript.md @@ -16,10 +16,25 @@ Then import the package: import { loadPrism } from "@ruby/prism"; ``` -Then call the load function to get a parse function: +Then call the load function to get a parse function. It takes the source as +bytes, in the encoding the source is written in, because that is what the parser +reads and what the locations in the result are offsets into. If you are starting +from a string, encode it first: ```js -const parse = await loadPrism(); +const parseArray = await loadPrism(); + +function parse(source) { + return parseArray(new TextEncoder().encode(source)); +} +``` + +Reading a file gives you those bytes already, so there is nothing to encode: + +```js +import { readFile } from "node:fs/promises"; + +const parseResult = parseArray(await readFile("example.rb")); ``` ## Browser @@ -84,7 +99,7 @@ Here's an example of a custom `FooCalls` visitor: import { loadPrism, Visitor } from "@ruby/prism" const parse = await loadPrism(); -const parseResult = parse("foo()"); +const parseResult = parse(new TextEncoder().encode("foo()")); class FooCalls extends Visitor { visitCallNode(node) { diff --git a/javascript/src/index.js b/javascript/src/index.js index 9562df8ff3..47b6a594ad 100644 --- a/javascript/src/index.js +++ b/javascript/src/index.js @@ -11,11 +11,13 @@ export * from "./visitor.js"; export * from "./nodes.js"; /** - * Load the prism wasm module and return a parse function. + * Load the prism wasm module and return a parse function. The function takes + * the source as bytes in the encoding it is written in, which is what the + * parser reads and what the locations in the result are offsets into. * * @typedef {import("./parsePrism.js").Options} Options * - * @returns {Promise<(source: string, options?: Options) => ParseResult>} + * @returns {Promise<(source: Uint8Array, options?: Options) => ParseResult>} */ export async function loadPrism() { const wasm = await WebAssembly.compile(await readFile(fileURLToPath(new URL("prism.wasm", import.meta.url)))); From 3b2be24c77fbadd0ecb77f1e067225b6c40e90c1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 13:34:42 -0400 Subject: [PATCH 09/13] Restrict JS exports for WASM --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0f6f5264d1..efc20f3ba9 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,11 @@ AR ?= ar ARFLAGS ?= -r$(V0:1=v) WASI_SDK_PATH := /opt/wasi-sdk +# Exporting a symbol makes it a GC root, so the JavaScript module exports only +# the symbols javascript/src/parsePrism.js calls. That lets --gc-sections drop +# the rest of the C API along with the wasi-libc code reachable from it. +WASM_EXPORTS := -Wl,--export=calloc,--export=free,--export=pm_buffer_new,--export=pm_buffer_value,--export=pm_buffer_length,--export=pm_buffer_free,--export=pm_serialize_parse + MAKEDIRS ?= mkdir -p RMALL ?= rm -f -r @@ -47,7 +52,7 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS) $(DEBUG_FLAGS) \ -DPRISM_EXPORT_SYMBOLS -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_JSON \ -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) \ - -Wl,--export-all -Wl,--gc-sections -Wl,--strip-all -Wl,--lto-O3 -Wl,--no-entry -mexec-model=reactor \ + $(WASM_EXPORTS) -Wl,--gc-sections -Wl,--strip-all -Wl,--lto-O3 -Wl,--no-entry -mexec-model=reactor \ -Oz -g0 -flto -fdata-sections -ffunction-sections \ -o $@ $(SOURCES) From 56b3ba6c024dcaf3a75dd6132e03259a1fadcdd7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 14:11:23 -0400 Subject: [PATCH 10/13] JS: use malloc for WASM allocations --- Makefile | 2 +- javascript/src/parsePrism.js | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index efc20f3ba9..2d67a3f14d 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ WASI_SDK_PATH := /opt/wasi-sdk # Exporting a symbol makes it a GC root, so the JavaScript module exports only # the symbols javascript/src/parsePrism.js calls. That lets --gc-sections drop # the rest of the C API along with the wasi-libc code reachable from it. -WASM_EXPORTS := -Wl,--export=calloc,--export=free,--export=pm_buffer_new,--export=pm_buffer_value,--export=pm_buffer_length,--export=pm_buffer_free,--export=pm_serialize_parse +WASM_EXPORTS := -Wl,--export=malloc,--export=free,--export=pm_buffer_new,--export=pm_buffer_value,--export=pm_buffer_length,--export=pm_buffer_free,--export=pm_serialize_parse MAKEDIRS ?= mkdir -p RMALL ?= rm -f -r diff --git a/javascript/src/parsePrism.js b/javascript/src/parsePrism.js index ebe9785eb2..1b350aa212 100644 --- a/javascript/src/parsePrism.js +++ b/javascript/src/parsePrism.js @@ -26,25 +26,25 @@ import { ParseResult, deserialize } from "./deserialize.js"; * @returns {ParseResult} */ export function parsePrism(prism, source, options = {}) { - const packedOptions = dumpOptions(options); - const optionsPointer = prism.calloc(1, packedOptions.length); + const dumpedOptions = dumpOptions(options); const bufferPointer = prism.pm_buffer_new(); + const sourcePointer = prism.malloc(source.length); + const optionsPointer = prism.malloc(dumpedOptions.length); - const sourcePointer = prism.calloc(1, source.length); const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, source.length); sourceView.set(source); - const optionsView = new Uint8Array(prism.memory.buffer, optionsPointer, packedOptions.length); - optionsView.set(packedOptions); + const optionsView = new Uint8Array(prism.memory.buffer, optionsPointer, dumpedOptions.length); + optionsView.set(dumpedOptions); prism.pm_serialize_parse(bufferPointer, sourcePointer, source.length, optionsPointer); const serializedView = new Uint8Array(prism.memory.buffer, prism.pm_buffer_value(bufferPointer), prism.pm_buffer_length(bufferPointer)); - const result = deserialize(source, serializedView); + const deserialized = deserialize(source, serializedView); prism.pm_buffer_free(bufferPointer); prism.free(sourcePointer); prism.free(optionsPointer); - return result; + return deserialized; } /** From 49d5fd6e6c51357ce1ba45cd3731e3c10188b8f3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 14:20:11 -0400 Subject: [PATCH 11/13] JS: Fix import for decoder --- templates/javascript/src/deserialize.js.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index 1f01599054..168ae71660 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -1,5 +1,5 @@ import * as nodes from "./nodes.js"; -import { getDecoder } from "./encoding.js"; +import { getDecoder } from "./decoding.js"; import { Location } from "./location.js"; import { Source } from "./source.js"; From 11a3f0b3c2689dfebde80af1a65acccf4ff85564 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 15:14:38 -0400 Subject: [PATCH 12/13] JS: Use the built package for the playground --- .github/workflows/github-pages.yml | 8 ++++++++ .gitignore | 1 + Makefile | 11 ++++++++++- doc/playground.html | 2 +- doc/playground.js | 6 +++--- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml index 46b8ac4f28..4e706d7431 100644 --- a/.github/workflows/github-pages.yml +++ b/.github/workflows/github-pages.yml @@ -55,6 +55,14 @@ jobs: bundle exec rake cargo:build cargo doc --no-deps --target-dir ../doc/rust working-directory: rust + - name: Set up WASI-SDK + run: | + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-33/wasi-sdk-33.0-x86_64-linux.tar.gz + tar xvf wasi-sdk-33.0-x86_64-linux.tar.gz + - name: Build the playground parser + run: | + bundle exec rake templates + make playground WASI_SDK_PATH=$(pwd)/wasi-sdk-33.0-x86_64-linux - name: Upload artifact uses: actions/upload-pages-artifact@v5 with: diff --git a/.gitignore b/.gitignore index e8dfeddbe6..9635725edb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /coverage/ /doc/c/ /doc/java/ +/doc/playground/ /doc/rb/ /doc/rust/ /grammars/ diff --git a/Makefile b/Makefile index 2d67a3f14d..1273f3b1ef 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ shared: build/libprism.$(SOEXT) static: build/libprism.a wasm: javascript/src/prism.wasm java-wasm: java/wasm/src/main/wasm/prism.wasm +playground: doc/playground build/libprism.$(SOEXT): $(SHARED_OBJECTS) $(ECHO) "linking $@ with $(CC)" @@ -56,6 +57,14 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS) -Oz -g0 -flto -fdata-sections -ffunction-sections \ -o $@ $(SOURCES) +# The playground parses with the JavaScript package built from this checkout, so +# the page is served the parser the rest of the site was generated from. The +# JavaScript half of the package is generated by `rake templates`. +doc/playground: javascript/src/prism.wasm + $(ECHO) "building $@" + $(Q) $(MAKEDIRS) $@ + $(Q) cp javascript/src/*.js javascript/src/prism.wasm javascript/package.json $@ + java/wasm/src/main/wasm/prism.wasm: Makefile $(SOURCES) $(HEADERS) $(ECHO) "building $@" $(Q) $(MAKEDIRS) $(@D) @@ -111,7 +120,7 @@ fuzz-clean: clean: $(Q) $(RMALL) build -.PHONY: clean fuzz-clean +.PHONY: clean doc/playground fuzz-clean all-no-debug: DEBUG_FLAGS := -DNDEBUG=1 all-no-debug: OPTFLAGS := -O3 diff --git a/doc/playground.html b/doc/playground.html index cc1e17367f..8171460070 100644 --- a/doc/playground.html +++ b/doc/playground.html @@ -4,7 +4,7 @@ - + Prism - Playground diff --git a/doc/playground.js b/doc/playground.js index cca3724ee5..b2ea23878a 100644 --- a/doc/playground.js +++ b/doc/playground.js @@ -1,5 +1,5 @@ import { WASI } from "https://unpkg.com/@bjorn3/browser_wasi_shim@latest/dist/index.js"; -import { parsePrism } from "https://unpkg.com/@ruby/prism@latest/src/parsePrism.js"; +import { parsePrism } from "./playground/parsePrism.js"; const output = document.getElementById("output"); const editorDiv = document.getElementById("editor"); @@ -13,13 +13,13 @@ const decoder = new TextDecoder(); let instance, monaco; try { const [wasmResult] = await Promise.all([ - WebAssembly.compileStreaming(fetch("https://unpkg.com/@ruby/prism@latest/src/prism.wasm")) + WebAssembly.compileStreaming(fetch("./playground/prism.wasm")) .then(wasm => { const wasi = new WASI([], [], []); return WebAssembly.instantiate(wasm, { wasi_snapshot_preview1: wasi.wasiImport }) .then(inst => { wasi.initialize(inst); return inst; }); }), - fetch("https://unpkg.com/@ruby/prism@latest/package.json") + fetch("./playground/package.json") .then(r => r.json()) .then(pkg => { document.getElementById("version").textContent = `v${pkg.version}`; }) .catch(() => {}) From 4bf9bf3d6cdf09e91458c917ec19d6f2d39d57d8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Sep 2026 20:04:56 -0400 Subject: [PATCH 13/13] Cache decoders per encoding --- javascript/src/decoding.js | 18 +++++++++++++++++- templates/javascript/src/deserialize.js.erb | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/javascript/src/decoding.js b/javascript/src/decoding.js index 31e2baa3d0..42d3f81ad2 100644 --- a/javascript/src/decoding.js +++ b/javascript/src/decoding.js @@ -26,6 +26,16 @@ const binaryTextDecoder = { } }; +/** + * Decoders built so far, keyed by the encoding and fatal setting they were + * built for. Nothing here decodes in streaming mode, so a decoder carries no + * state between calls and one instance serves every caller that wants the same + * pair. + * + * @type {Map} + */ +const decoders = new Map(); + /** * Get a Decoder from the encoding name. If the encoding is not supported, a * decoder that treats each byte as a single character is returned. @@ -36,7 +46,12 @@ const binaryTextDecoder = { */ export function getDecoder(name, options = { fatal: false }) { const lower = name.toLowerCase(); - let decoder = null; + const key = `${lower}:${options.fatal ? "fatal" : "replacement"}`; + + let decoder = decoders.get(key); + if (decoder !== undefined) { + return decoder; + } if (lower === "ascii-8bit" || lower === "binary") { decoder = binaryTextDecoder; @@ -52,5 +67,6 @@ export function getDecoder(name, options = { fatal: false }) { } } + decoders.set(key, decoder); return decoder; } diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index 168ae71660..5673edb181 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -140,7 +140,10 @@ class SerializationBuffer { value = decoder.decode(bytes); validEncoding = true; } catch (error) { - if (error instanceof TypeError && error.code === "ERR_ENCODING_INVALID_ENCODED_DATA") { + /* A fatal decoder reports bytes that are not valid in its encoding by + * throwing a TypeError, and that is the whole of what it throws, so the + * string is read back as binary and flagged. */ + if (error instanceof TypeError) { value = getDecoder("ascii-8bit").decode(bytes); validEncoding = false; } else {