diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml
index 46b8ac4f28..4e706d7431 100644
--- a/.github/workflows/github-pages.yml
+++ b/.github/workflows/github-pages.yml
@@ -55,6 +55,14 @@ jobs:
bundle exec rake cargo:build
cargo doc --no-deps --target-dir ../doc/rust
working-directory: rust
+ - name: Set up WASI-SDK
+ run: |
+ wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-33/wasi-sdk-33.0-x86_64-linux.tar.gz
+ tar xvf wasi-sdk-33.0-x86_64-linux.tar.gz
+ - name: Build the playground parser
+ run: |
+ bundle exec rake templates
+ make playground WASI_SDK_PATH=$(pwd)/wasi-sdk-33.0-x86_64-linux
- name: Upload artifact
uses: actions/upload-pages-artifact@v5
with:
diff --git a/.gitignore b/.gitignore
index e8dfeddbe6..9635725edb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
/coverage/
/doc/c/
/doc/java/
+/doc/playground/
/doc/rb/
/doc/rust/
/grammars/
diff --git a/Makefile b/Makefile
index 0f6f5264d1..1273f3b1ef 100644
--- a/Makefile
+++ b/Makefile
@@ -18,6 +18,11 @@ AR ?= ar
ARFLAGS ?= -r$(V0:1=v)
WASI_SDK_PATH := /opt/wasi-sdk
+# Exporting a symbol makes it a GC root, so the JavaScript module exports only
+# the symbols javascript/src/parsePrism.js calls. That lets --gc-sections drop
+# the rest of the C API along with the wasi-libc code reachable from it.
+WASM_EXPORTS := -Wl,--export=malloc,--export=free,--export=pm_buffer_new,--export=pm_buffer_value,--export=pm_buffer_length,--export=pm_buffer_free,--export=pm_serialize_parse
+
MAKEDIRS ?= mkdir -p
RMALL ?= rm -f -r
@@ -32,6 +37,7 @@ shared: build/libprism.$(SOEXT)
static: build/libprism.a
wasm: javascript/src/prism.wasm
java-wasm: java/wasm/src/main/wasm/prism.wasm
+playground: doc/playground
build/libprism.$(SOEXT): $(SHARED_OBJECTS)
$(ECHO) "linking $@ with $(CC)"
@@ -47,10 +53,18 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
$(DEBUG_FLAGS) \
-DPRISM_EXPORT_SYMBOLS -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_JSON \
-D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) \
- -Wl,--export-all -Wl,--gc-sections -Wl,--strip-all -Wl,--lto-O3 -Wl,--no-entry -mexec-model=reactor \
+ $(WASM_EXPORTS) -Wl,--gc-sections -Wl,--strip-all -Wl,--lto-O3 -Wl,--no-entry -mexec-model=reactor \
-Oz -g0 -flto -fdata-sections -ffunction-sections \
-o $@ $(SOURCES)
+# The playground parses with the JavaScript package built from this checkout, so
+# the page is served the parser the rest of the site was generated from. The
+# JavaScript half of the package is generated by `rake templates`.
+doc/playground: javascript/src/prism.wasm
+ $(ECHO) "building $@"
+ $(Q) $(MAKEDIRS) $@
+ $(Q) cp javascript/src/*.js javascript/src/prism.wasm javascript/package.json $@
+
java/wasm/src/main/wasm/prism.wasm: Makefile $(SOURCES) $(HEADERS)
$(ECHO) "building $@"
$(Q) $(MAKEDIRS) $(@D)
@@ -106,7 +120,7 @@ fuzz-clean:
clean:
$(Q) $(RMALL) build
-.PHONY: clean fuzz-clean
+.PHONY: clean doc/playground fuzz-clean
all-no-debug: DEBUG_FLAGS := -DNDEBUG=1
all-no-debug: OPTFLAGS := -O3
diff --git a/doc/playground.html b/doc/playground.html
index cc1e17367f..8171460070 100644
--- a/doc/playground.html
+++ b/doc/playground.html
@@ -4,7 +4,7 @@
Prism - Playground
${escapeHtml(JSON.stringify(slice))} `
+ text = `${text} = ${escapeHtml(JSON.stringify(loc.slice()))} `
}
return { start, end, text };
}
@@ -352,7 +444,7 @@ function hasChildNodes(fields, node) {
const CONNECTOR = { last: "└── ", mid: "├── ", lastPad: " ", midPad: "│ " };
// Build the AST tree as interactive HTML
-function renderNode(node, utf8Bytes, prefix, isLast, isRoot) {
+function renderNode(node, prefix, isLast, isRoot) {
if (!isNode(node)) return "";
const type = nodeType(node);
@@ -365,7 +457,7 @@ function renderNode(node, utf8Bytes, prefix, isLast, isRoot) {
if (!isRoot) html += `${prefix}${isLast ? CONNECTOR.last : CONNECTOR.mid} `;
if (foldable) html += `▼ `;
- const loc = formatLoc(utf8Bytes, node.location, false);
+ const loc = formatLoc(node.location, false);
const locAttrs = locDataAttrs(loc);
html += `@ ${escapedType} `;
@@ -394,7 +486,7 @@ function renderNode(node, utf8Bytes, prefix, isLast, isRoot) {
html += `${childPrefix}${fieldConnector} ${escapeHtml(field)} : (${value.length} item${value.length === 1 ? "" : "s"})
`;
value.forEach((item, i) => {
if (isNode(item)) {
- html += renderNode(item, utf8Bytes, fieldChildPrefix, i === value.length - 1);
+ html += renderNode(item, fieldChildPrefix, i === value.length - 1);
} else {
const itemConnector = i === value.length - 1 ? CONNECTOR.last : CONNECTOR.mid;
if (isConstant(item)) {
@@ -407,9 +499,9 @@ function renderNode(node, utf8Bytes, prefix, isLast, isRoot) {
}
} else if (isNode(value)) {
html += `${childPrefix}${fieldConnector} ${escapeHtml(field)} :
`;
- html += renderNode(value, utf8Bytes, fieldChildPrefix, true);
+ html += renderNode(value, fieldChildPrefix, true);
} else if (typeof value === "object" && value.startOffset !== undefined) {
- const fieldLoc = formatLoc(utf8Bytes, value, true);
+ const fieldLoc = formatLoc(value, true);
if (fieldLoc) {
html += `${childPrefix}${fieldConnector} ${escapeHtml(field)} : ${fieldLoc.text}
`;
}
@@ -435,8 +527,8 @@ function escapeHtml(str) {
}
// Render a single diagnostic line
-function renderDiagnostic(utf8Bytes, item, kind) {
- const loc = formatLoc(utf8Bytes, item.location, false);
+function renderDiagnostic(item, kind) {
+ const loc = formatLoc(item.location, false);
const cssClass = kind === "Error" ? "error-text" : "warning-text";
return `${kind}: ${escapeHtml(item.message)}${loc ? ` (${loc.text}) ` : ""}
`;
}
@@ -481,24 +573,25 @@ function render() {
output.setAttribute("aria-labelledby", currentTab === "ast" ? "tab-ast" : "tab-diagnostics");
- const utf8Bytes = encoder.encode(lastSource);
+ const notice = lastNotice ? `${escapeHtml(lastNotice)}
` : "";
+
switch (currentTab) {
case "ast":
- const tree = renderNode(lastResult.value, utf8Bytes, "", true, true);
- output.innerHTML = tree
+ const tree = renderNode(lastResult.value, "", true, true);
+ output.innerHTML = notice + (tree
? `${tree}
`
- : `${escapeHtml(lastResult.error || "Failed to parse.")}
`;
+ : `${escapeHtml(lastResult.error || "Failed to parse.")}
`);
break;
case "diagnostics":
const errors = lastResult.errors || [];
const warnings = lastResult.warnings || [];
if (errors.length === 0 && warnings.length === 0) {
- output.innerHTML = `No errors or warnings.
`;
+ output.innerHTML = notice || `No errors or warnings.
`;
} else {
- let html = "";
- for (const err of errors) html += renderDiagnostic(utf8Bytes, err, "Error");
- for (const warn of warnings) html += renderDiagnostic(utf8Bytes, warn, "Warning");
+ let html = notice;
+ for (const err of errors) html += renderDiagnostic(err, "Error");
+ for (const warn of warnings) html += renderDiagnostic(warn, "Warning");
output.innerHTML = html;
}
break;
@@ -509,11 +602,18 @@ let timeout = null;
function parse() {
if (timeout) clearTimeout(timeout);
timeout = setTimeout(() => {
- lastSource = monacoEditor.getValue();
- history.replaceState(null, "", `#${encodeSource(lastSource)}`);
+ const source = monacoEditor.getValue();
+
+ /* The hash carries the editor's text, which is independent of the encoding
+ * the source declares, so it stays utf-8. */
+ history.replaceState(null, "", `#${encodeSource(encoder.encode(source))}`);
+
try {
- lastResult = parsePrism(instance.exports, lastSource);
+ const { bytes, notice } = sourceBytes(source);
+ lastNotice = notice || null;
+ lastResult = parsePrism(instance.exports, bytes);
} catch (e) {
+ lastNotice = null;
lastResult = { value: null, error: e.message, errors: [], warnings: [] };
}
render();
diff --git a/docs/javascript.md b/docs/javascript.md
index 2ccd5f8231..63ae2bae89 100644
--- a/docs/javascript.md
+++ b/docs/javascript.md
@@ -16,10 +16,25 @@ Then import the package:
import { loadPrism } from "@ruby/prism";
```
-Then call the load function to get a parse function:
+Then call the load function to get a parse function. It takes the source as
+bytes, in the encoding the source is written in, because that is what the parser
+reads and what the locations in the result are offsets into. If you are starting
+from a string, encode it first:
```js
-const parse = await loadPrism();
+const parseArray = await loadPrism();
+
+function parse(source) {
+ return parseArray(new TextEncoder().encode(source));
+}
+```
+
+Reading a file gives you those bytes already, so there is nothing to encode:
+
+```js
+import { readFile } from "node:fs/promises";
+
+const parseResult = parseArray(await readFile("example.rb"));
```
## Browser
@@ -56,7 +71,7 @@ Finally, you can create a function that will parse a string of Ruby code.
```js
function parse(source) {
- return parsePrism(instance.exports, source);
+ return parsePrism(instance.exports, new TextEncoder().encode(source));
}
```
@@ -84,7 +99,7 @@ Here's an example of a custom `FooCalls` visitor:
import { loadPrism, Visitor } from "@ruby/prism"
const parse = await loadPrism();
-const parseResult = parse("foo()");
+const parseResult = parse(new TextEncoder().encode("foo()"));
class FooCalls extends Visitor {
visitCallNode(node) {
diff --git a/javascript/example.html b/javascript/example.html
index d5ce3bbba6..99021614ab 100644
--- a/javascript/example.html
+++ b/javascript/example.html
@@ -30,7 +30,7 @@
if (timeout) clearTimeout(timeout);
timeout = setTimeout(function () {
- const result = parsePrism(instance.exports, event.target.value);
+ const result = parsePrism(instance.exports, new TextEncoder().encode(event.target.value));
output.textContent = JSON.stringify(result, null, 2);
}, 250);
});
diff --git a/javascript/src/decoding.js b/javascript/src/decoding.js
new file mode 100644
index 0000000000..42d3f81ad2
--- /dev/null
+++ b/javascript/src/decoding.js
@@ -0,0 +1,72 @@
+/**
+ * An object that can be used to decode a byte array into a string.
+ *
+ * @typedef {{ decode: (bytes: Uint8Array) => string }} Decoder
+ */
+
+/**
+ * The decoder used both as the decoder for binary and the fallback in case the
+ * encoding is not supported.
+ */
+const binaryTextDecoder = {
+ /**
+ * Decodes a byte array into a string by treating each byte as a single
+ * character. This is used for the ASCII-8BIT encoding from Ruby.
+ *
+ * @param {Uint8Array} bytes
+ * @returns {string}
+ */
+ decode(bytes) {
+ let result = "";
+ for (const byte of bytes) {
+ result += String.fromCharCode(byte);
+ }
+
+ return result;
+ }
+};
+
+/**
+ * Decoders built so far, keyed by the encoding and fatal setting they were
+ * built for. Nothing here decodes in streaming mode, so a decoder carries no
+ * state between calls and one instance serves every caller that wants the same
+ * pair.
+ *
+ * @type {Map}
+ */
+const decoders = new Map();
+
+/**
+ * Get a Decoder from the encoding name. If the encoding is not supported, a
+ * decoder that treats each byte as a single character is returned.
+ *
+ * @param {string} name
+ * @param {{ fatal: boolean }} options
+ * @returns {Decoder}
+ */
+export function getDecoder(name, options = { fatal: false }) {
+ const lower = name.toLowerCase();
+ const key = `${lower}:${options.fatal ? "fatal" : "replacement"}`;
+
+ let decoder = decoders.get(key);
+ if (decoder !== undefined) {
+ return decoder;
+ }
+
+ if (lower === "ascii-8bit" || lower === "binary") {
+ decoder = binaryTextDecoder;
+ } else {
+ try {
+ decoder = new TextDecoder(lower, options);
+ } catch (error) {
+ if (error instanceof RangeError) {
+ decoder = binaryTextDecoder;
+ } else {
+ throw error;
+ }
+ }
+ }
+
+ decoders.set(key, decoder);
+ return decoder;
+}
diff --git a/javascript/src/index.js b/javascript/src/index.js
index 3c40c4c16b..47b6a594ad 100644
--- a/javascript/src/index.js
+++ b/javascript/src/index.js
@@ -5,15 +5,19 @@ import { fileURLToPath } from "node:url";
import { ParseResult } from "./deserialize.js";
import { parsePrism } from "./parsePrism.js";
+export * from "./location.js";
+export * from "./source.js";
export * from "./visitor.js";
export * from "./nodes.js";
/**
- * Load the prism wasm module and return a parse function.
+ * Load the prism wasm module and return a parse function. The function takes
+ * the source as bytes in the encoding it is written in, which is what the
+ * parser reads and what the locations in the result are offsets into.
*
* @typedef {import("./parsePrism.js").Options} Options
*
- * @returns {Promise<(source: string, options?: Options) => ParseResult>}
+ * @returns {Promise<(source: Uint8Array, options?: Options) => ParseResult>}
*/
export async function loadPrism() {
const wasm = await WebAssembly.compile(await readFile(fileURLToPath(new URL("prism.wasm", import.meta.url))));
diff --git a/javascript/src/location.js b/javascript/src/location.js
new file mode 100644
index 0000000000..26fa507f47
--- /dev/null
+++ b/javascript/src/location.js
@@ -0,0 +1,123 @@
+import { Source } from "./source.js";
+
+/**
+ * A location in the source code. Locations are stored as byte offsets into the
+ * source, because bytes are the unit that the parser itself works in. They hold
+ * a pointer to the source they came from so that they can resolve line numbers,
+ * columns, and the source code that they represent.
+ */
+export class Location {
+ /**
+ * The source that this location points into. It is private so that it stays
+ * off of the instance itself, which keeps locations cheap to serialize with
+ * JSON.stringify.
+ *
+ * @type {Source}
+ */
+ #source;
+
+ /**
+ * The byte offset from the beginning of the source where this location
+ * starts.
+ *
+ * @type {number}
+ */
+ startOffset;
+
+ /**
+ * The length of this location in bytes.
+ *
+ * @type {number}
+ */
+ length;
+
+ /**
+ * Construct a new Location.
+ *
+ * @param {Source} source
+ * @param {number} startOffset
+ * @param {number} length
+ */
+ constructor(source, startOffset, length) {
+ this.#source = source;
+ this.startOffset = startOffset;
+ this.length = length;
+ }
+
+ /**
+ * The byte offset from the beginning of the source where this location ends.
+ *
+ * @returns {number}
+ */
+ endOffset() {
+ return this.startOffset + this.length;
+ }
+
+ /**
+ * The line number where this location starts.
+ *
+ * @returns {number}
+ */
+ startLine() {
+ return this.#source.line(this.startOffset);
+ }
+
+ /**
+ * The line number where this location ends.
+ *
+ * @returns {number}
+ */
+ endLine() {
+ return this.#source.line(this.endOffset());
+ }
+
+ /**
+ * The column in bytes where this location starts from the start of its line.
+ *
+ * @returns {number}
+ */
+ startColumn() {
+ return this.#source.column(this.startOffset);
+ }
+
+ /**
+ * The column in bytes where this location ends from the start of its line.
+ *
+ * @returns {number}
+ */
+ endColumn() {
+ return this.#source.column(this.endOffset());
+ }
+
+ /**
+ * The column in code units where this location starts from the start of its
+ * line, counted in the given position encoding.
+ *
+ * @param {"utf-8" | "utf-16" | "utf-32"} encoding
+ * @returns {number}
+ */
+ startCodeUnitsColumn(encoding = "utf-16") {
+ return this.#source.codeUnitsColumn(this.startOffset, encoding);
+ }
+
+ /**
+ * The column in code units where this location ends from the start of its
+ * line, counted in the given position encoding.
+ *
+ * @param {"utf-8" | "utf-16" | "utf-32"} encoding
+ * @returns {number}
+ */
+ endCodeUnitsColumn(encoding = "utf-16") {
+ return this.#source.codeUnitsColumn(this.endOffset(), encoding);
+ }
+
+ /**
+ * The source code that this location represents.
+ *
+ * @param {TextDecoder | null} decoder
+ * @returns {string}
+ */
+ slice(decoder = null) {
+ return this.#source.slice(this.startOffset, this.length, decoder);
+ }
+}
diff --git a/javascript/src/parsePrism.js b/javascript/src/parsePrism.js
index 615cc83d13..1b350aa212 100644
--- a/javascript/src/parsePrism.js
+++ b/javascript/src/parsePrism.js
@@ -1,7 +1,7 @@
import { ParseResult, deserialize } from "./deserialize.js";
/**
- * Parse the given source code.
+ * Parse the given source code represented as a Uint8Array.
*
* @typedef {{
* locals?: string[],
@@ -18,35 +18,33 @@ import { ParseResult, deserialize } from "./deserialize.js";
* main_script?: boolean,
* partial_script?: boolean,
* scopes?: (string[] | Scope)[]
- * }} Options
+ * }} Options
*
* @param {WebAssembly.Exports} prism
- * @param {string} source
+ * @param {Uint8Array} source
* @param {Options} options
* @returns {ParseResult}
*/
export function parsePrism(prism, source, options = {}) {
- const sourceArray = new TextEncoder().encode(source);
- const sourcePointer = prism.calloc(1, sourceArray.length);
-
- const packedOptions = dumpOptions(options);
- const optionsPointer = prism.calloc(1, packedOptions.length);
+ const dumpedOptions = dumpOptions(options);
const bufferPointer = prism.pm_buffer_new();
+ const sourcePointer = prism.malloc(source.length);
+ const optionsPointer = prism.malloc(dumpedOptions.length);
- const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, sourceArray.length);
- sourceView.set(sourceArray);
+ const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, source.length);
+ sourceView.set(source);
- const optionsView = new Uint8Array(prism.memory.buffer, optionsPointer, packedOptions.length);
- optionsView.set(packedOptions);
+ const optionsView = new Uint8Array(prism.memory.buffer, optionsPointer, dumpedOptions.length);
+ optionsView.set(dumpedOptions);
- prism.pm_serialize_parse(bufferPointer, sourcePointer, sourceArray.length, optionsPointer);
+ prism.pm_serialize_parse(bufferPointer, sourcePointer, source.length, optionsPointer);
const serializedView = new Uint8Array(prism.memory.buffer, prism.pm_buffer_value(bufferPointer), prism.pm_buffer_length(bufferPointer));
- const result = deserialize(serializedView);
+ const deserialized = deserialize(source, serializedView);
prism.pm_buffer_free(bufferPointer);
prism.free(sourcePointer);
prism.free(optionsPointer);
- return result;
+ return deserialized;
}
/**
@@ -118,7 +116,7 @@ function dumpOptions(options) {
}
template.push("l");
- values.push(options.line || 1);
+ values.push(options.line === undefined ? 1 : options.line);
template.push("L");
if (options.encoding) {
diff --git a/javascript/src/source.js b/javascript/src/source.js
new file mode 100644
index 0000000000..e69610cf2c
--- /dev/null
+++ b/javascript/src/source.js
@@ -0,0 +1,228 @@
+import { getDecoder } from "./decoding.js";
+
+/**
+ * The encoder used to count code units for the utf-8 position encoding. The
+ * TextEncoder interface only ever emits utf-8, which is exactly what is needed
+ * here.
+ */
+const encoder = new TextEncoder();
+
+/**
+ * A source of Ruby code that has been parsed. Locations hold a pointer to the
+ * source they came from, which is what allows them to resolve line numbers,
+ * columns, and the source code that they represent.
+ */
+export class Source {
+ /**
+ * The bytes of the source code, in the encoding that it was parsed in.
+ *
+ * @type {Uint8Array}
+ */
+ bytes;
+
+ /**
+ * The name of the encoding that the source code is in, as determined by the
+ * parser options or by the encoding magic comment.
+ *
+ * @type {string}
+ */
+ encoding;
+
+ /**
+ * The line number that the source starts on.
+ *
+ * @type {number}
+ */
+ startLine;
+
+ /**
+ * The byte offset of the start of each line in the source code. The first
+ * element is always 0 to mark the first line.
+ *
+ * @type {number[]}
+ */
+ offsets;
+
+ /**
+ * The decoder used to convert this source's bytes into strings. It is created
+ * on first use because not every encoding that the parser accepts has a
+ * TextDecoder equivalent, and parsing should not fail on that basis.
+ *
+ * @type {Decoder | null}
+ */
+ #decoder;
+
+ /**
+ * Construct a new Source.
+ *
+ * @param {Uint8Array} bytes
+ * @param {string} encoding
+ * @param {number} startLine
+ * @param {number[]} offsets
+ */
+ constructor(bytes, encoding, startLine, offsets) {
+ this.bytes = bytes;
+ this.encoding = encoding;
+ this.startLine = startLine;
+ this.offsets = offsets;
+ this.#decoder = null;
+ }
+
+ /**
+ * Decode the given byte range of the source code into a string. Because a
+ * byte range can begin or end in the middle of a multi-byte character, bytes
+ * that do not form a whole character are decoded into replacement
+ * characters.
+ *
+ * @param {number} byteOffset
+ * @param {number} length
+ * @param {TextDecoder | null} decoder
+ * @returns {string}
+ */
+ slice(byteOffset, length, decoder = null) {
+ if (decoder === null) {
+ decoder = this.#defaultDecoder();
+ }
+
+ return decoder.decode(this.bytes.subarray(byteOffset, byteOffset + length));
+ }
+
+ /**
+ * The decoder for this source's own encoding. It is created on first use
+ * because not every encoding that the parser accepts has a TextDecoder
+ * equivalent, and parsing should not fail on that basis.
+ *
+ * @returns {Decoder}
+ */
+ #defaultDecoder() {
+ if (this.#decoder === null) {
+ this.#decoder = getDecoder(this.encoding);
+ }
+
+ return this.#decoder;
+ }
+
+ /**
+ * The line number that the given byte offset is on.
+ *
+ * @param {number} byteOffset
+ * @returns {number}
+ */
+ line(byteOffset) {
+ return this.startLine + this.findLine(byteOffset);
+ }
+
+ /**
+ * The byte offset of the start of the line that the given byte offset is on.
+ *
+ * @param {number} byteOffset
+ * @returns {number}
+ */
+ lineStart(byteOffset) {
+ return this.offsets[this.findLine(byteOffset)];
+ }
+
+ /**
+ * The byte offset of the end of the line that the given byte offset is on.
+ *
+ * @param {number} byteOffset
+ * @returns {number}
+ */
+ lineEnd(byteOffset) {
+ const offset = this.offsets[this.findLine(byteOffset) + 1];
+ return offset === undefined ? this.bytes.length : offset;
+ }
+
+ /**
+ * The column in bytes of the given byte offset from the start of its line.
+ *
+ * @param {number} byteOffset
+ * @returns {number}
+ */
+ column(byteOffset) {
+ return byteOffset - this.lineStart(byteOffset);
+ }
+
+ /**
+ * The column in code units of the given byte offset from the start of its
+ * line, counted in the given position encoding. A code unit is the smallest
+ * unit of an encoding form, so utf-8 counts bytes, utf-16 counts sixteen bit
+ * units where characters outside the basic multilingual plane take two, and
+ * utf-32 counts whole codepoints.
+ *
+ * These are the three position encodings of the language server protocol,
+ * spelled the way the protocol spells them, so a negotiated value can be
+ * passed straight through. It defaults to utf-16 because that is what the
+ * protocol defaults to.
+ *
+ * The prefix of the line is decoded through this source's own encoding
+ * first, so a given character resolves to the same column no matter which
+ * encoding the source was parsed in.
+ *
+ * @param {number} byteOffset
+ * @param {"utf-8" | "utf-16" | "utf-32"} encoding
+ * @returns {number}
+ */
+ codeUnitsColumn(byteOffset, encoding = "utf-16") {
+ const lineStart = this.lineStart(byteOffset);
+
+ /* Byte offsets are themselves utf-8 code units when the source is utf-8,
+ * and counting them directly keeps bytes that do not decode from inflating
+ * the column into the width of the replacement character. */
+ if (encoding === "utf-8" && this.encoding.toLowerCase() === "utf-8") {
+ return byteOffset - lineStart;
+ }
+
+ const prefix = this.#defaultDecoder().decode(this.bytes.subarray(lineStart, byteOffset));
+
+ switch (encoding) {
+ case "utf-8":
+ return encoder.encode(prefix).length;
+ case "utf-16":
+ return prefix.length;
+ case "utf-32": {
+ /* Every codepoint is one utf-16 code unit except those outside the
+ * basic multilingual plane, which are a surrogate pair. Decoders only
+ * ever produce well-formed utf-16, substituting the replacement
+ * character for anything they cannot pair up, so every low surrogate
+ * here closes a pair and dropping them leaves the codepoint count. */
+ let count = prefix.length;
+
+ for (let index = 0; index < prefix.length; index++) {
+ const unit = prefix.charCodeAt(index);
+ if (unit >= 0xdc00 && unit <= 0xdfff) count--;
+ }
+
+ return count;
+ }
+ default:
+ throw new Error(`Unsupported position encoding '${encoding}'`);
+ }
+ }
+
+ /**
+ * Binary search through the offsets to find the index of the line that the
+ * given byte offset is on.
+ *
+ * @param {number} byteOffset
+ * @returns {number}
+ */
+ findLine(byteOffset) {
+ let low = 0;
+ let high = this.offsets.length;
+
+ /* Find the first line that starts after the given byte offset; the line
+ * that contains the offset is the one before it. */
+ while (low < high) {
+ const middle = (low + high) >>> 1;
+
+ if (this.offsets[middle] > byteOffset) {
+ high = middle;
+ } else {
+ low = middle + 1;
+ }
+ }
+
+ return low - 1;
+ }
+}
diff --git a/javascript/test.js b/javascript/test.js
index 26dca10b74..3131ec30dd 100644
--- a/javascript/test.js
+++ b/javascript/test.js
@@ -1,10 +1,16 @@
-import test from "node:test";
+import test, { suite } from "node:test";
import assert from "node:assert";
import { loadPrism } from "./src/index.js";
import * as nodes from "./src/nodes.js";
+import { Location } from "./src/location.js";
+import { Source } from "./src/source.js";
import { Visitor } from "./src/visitor.js";
-const parse = await loadPrism();
+const parseArray = await loadPrism();
+
+function parse(source, options = {}) {
+ return parseArray(new TextEncoder().encode(source), options);
+}
function statement(result) {
return result.value.statements.body[0];
@@ -18,185 +24,197 @@ function eachNode(node, callback) {
}
}
-test("node", () => {
- const result = parse("foo");
- assert(result.value instanceof nodes.ProgramNode);
-});
-
-test("node? present", () => {
- const result = parse("foo.bar");
- assert(statement(result).receiver instanceof nodes.CallNode);
-});
-
-test("node? absent", () => {
- const result = parse("foo");
- assert(statement(result).receiver === null);
-});
-
-test("node[]", () => {
- const result = parse("foo.bar");
- assert(result.value.statements.body instanceof Array);
-});
-
-test("string", () => {
- const result = parse('"foo"');
- const node = statement(result);
-
- assert(!node.isForcedUtf8Encoding())
- assert(!node.isForcedBinaryEncoding())
-
- assert(node.unescaped.value === "foo");
- assert(node.unescaped.encoding === "utf-8");
- assert(node.unescaped.validEncoding);
-});
-
-test("forced utf-8 string using \\u syntax", () => {
- const result = parse('# encoding: utf-8\n"\\u{9E7F}"');
- const node = statement(result);
- const str = node.unescaped;
-
- assert(node.isForcedUtf8Encoding());
- assert(!node.isForcedBinaryEncoding());
-
- assert(str.value === "鹿");
- assert(str.encoding === "utf-8");
- assert(str.validEncoding);
-});
-
-test("forced utf-8 string with invalid byte sequence", () => {
- const result = parse('# encoding: utf-8\n"\\xFF\\xFF\\xFF"');
- const node = statement(result);
- const str = node.unescaped;
-
- assert(node.isForcedUtf8Encoding());
- assert(!node.isForcedBinaryEncoding());
-
- assert(str.value === "ÿÿÿ");
- assert(str.encoding === "utf-8");
- assert(!str.validEncoding);
-});
-
-test("ascii string with embedded utf-8 character", () => {
- // # encoding: ascii\n"鹿"'
- // # encoding: ascii\n"鹿"'
- const ascii_str = new Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 233, 185, 191, 34]);
- const result = parse(ascii_str);
- const node = statement(result);
- const str = node.unescaped;
-
- assert(!node.isForcedUtf8Encoding());
- assert(node.isForcedBinaryEncoding());
-
- assert(str.value === "鹿");
- assert(str.encoding === "ascii");
- assert(str.validEncoding);
-});
-
-test("forced binary string", () => {
- const result = parse('# encoding: ascii\n"\\xFF\\xFF\\xFF"');
- const node = statement(result);
- const str = node.unescaped;
-
- assert(!node.isForcedUtf8Encoding());
- assert(node.isForcedBinaryEncoding());
-
- assert(str.value === "ÿÿÿ");
- assert(str.encoding === "ascii");
- assert(str.validEncoding);
-});
+suite("fields", () => {
+ test("node", () => {
+ const result = parse("foo");
+ assert(result.value instanceof nodes.ProgramNode);
+ });
+
+ test("node? present", () => {
+ const result = parse("foo.bar");
+ assert(statement(result).receiver instanceof nodes.CallNode);
+ });
+
+ test("node? absent", () => {
+ const result = parse("foo");
+ assert(statement(result).receiver === null);
+ });
+
+ test("node[]", () => {
+ const result = parse("foo.bar");
+ assert(result.value.statements.body instanceof Array);
+ });
+
+ suite("string", () => {
+ test("basic", () => {
+ const result = parse('"foo"');
+ const node = statement(result);
+
+ assert(!node.isForcedUtf8Encoding())
+ assert(!node.isForcedBinaryEncoding())
+
+ assert(node.unescaped.value === "foo");
+ assert(node.unescaped.encoding === "utf-8");
+ assert(node.unescaped.validEncoding);
+ });
-test("forced binary string with Unicode character", () => {
- // # encoding: us-ascii\n"\\xFF鹿\\xFF"
- const ascii_str = Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 92, 120, 70, 70, 233, 185, 191, 92, 120, 70, 70, 34]);
- const result = parse(ascii_str);
- const node = statement(result);
- const str = node.unescaped;
+ test("forced utf-8 using \\u syntax", () => {
+ const result = parse('# encoding: utf-8\n"\\u{9E7F}"');
+ const node = statement(result);
+ const str = node.unescaped;
- assert(!node.isForcedUtf8Encoding());
- assert(node.isForcedBinaryEncoding());
+ assert(node.isForcedUtf8Encoding());
+ assert(!node.isForcedBinaryEncoding());
- assert(str.value === "ÿ鹿ÿ");
- assert(str.encoding === "ascii");
- assert(str.validEncoding);
-});
+ assert(str.value === "鹿");
+ assert(str.encoding === "utf-8");
+ assert(str.validEncoding);
+ });
-test("constant", () => {
- const result = parse("foo = 1");
- assert(result.value.locals[0] === "foo");
-});
+ test("forced utf-8 string with invalid byte sequence", () => {
+ const result = parse('# encoding: utf-8\n"\\xFF\\xFF\\xFF"');
+ const node = statement(result);
+ const str = node.unescaped;
-test("constant? present", () => {
- const result = parse("def foo(*bar); end");
- assert(statement(result).parameters.rest.name === "bar");
-});
+ assert(node.isForcedUtf8Encoding());
+ assert(!node.isForcedBinaryEncoding());
-test("constant? absent", () => {
- const result = parse("def foo(*); end");
- assert(statement(result).parameters.rest.name === null);
-});
+ assert(str.value === "ÿÿÿ");
+ assert(str.encoding === "utf-8");
+ assert(!str.validEncoding);
+ });
-test("constant[]", async() => {
- const result = parse("foo = 1");
- assert(result.value.locals instanceof Array);
-});
+ test("ascii with embedded utf-8 character", () => {
+ // # encoding: ascii\n"鹿"'
+ // # encoding: ascii\n"鹿"'
+ const ascii_str = new Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 233, 185, 191, 34]);
+ const result = parse(ascii_str);
+ const node = statement(result);
+ const str = node.unescaped;
-test("location", () => {
- const result = parse("foo = 1");
- assert(typeof result.value.location.startOffset === "number");
-});
+ assert(!node.isForcedUtf8Encoding());
+ assert(node.isForcedBinaryEncoding());
-test("location? present", () => {
- const result = parse("def foo = bar");
- assert(statement(result).equalLoc !== null);
-});
+ assert(str.value === "鹿");
+ assert(str.encoding === "ascii-8bit");
+ assert(str.validEncoding);
+ });
-test("location? absent", () => {
- const result = parse("def foo; bar; end");
- assert(statement(result).equalLoc === null);
-});
+ test("forced binary", () => {
+ const result = parse('# encoding: ascii\n"\\xFF\\xFF\\xFF"');
+ const node = statement(result);
+ const str = node.unescaped;
-test("uint8", () => {
- const result = parse("-> { _3 }");
- assert(statement(result).parameters.maximum === 3);
-});
+ assert(!node.isForcedUtf8Encoding());
+ assert(node.isForcedBinaryEncoding());
-test("uint32", () => {
- const result = parse("foo = 1");
- assert(statement(result).depth === 0);
-});
+ assert(str.value === "ÿÿÿ");
+ assert(str.encoding === "ascii-8bit");
+ assert(str.validEncoding);
+ });
-test("flags", () => {
- const result = parse("/foo/mi");
- const regexp = statement(result);
+ test("forced binary with Unicode character", () => {
+ // # encoding: us-ascii\n"\\xFF鹿\\xFF"
+ const ascii_str = Buffer.from([35, 32, 101, 110, 99, 111, 100, 105, 110, 103, 58, 32, 97, 115, 99, 105, 105, 10, 34, 92, 120, 70, 70, 233, 185, 191, 92, 120, 70, 70, 34]);
+ const result = parse(ascii_str);
+ const node = statement(result);
+ const str = node.unescaped;
- assert(regexp.isIgnoreCase());
- assert(regexp.isMultiLine());
- assert(!regexp.isExtended());
-});
+ assert(!node.isForcedUtf8Encoding());
+ assert(node.isForcedBinaryEncoding());
-test("integer (decimal)", () => {
- const result = parse("10");
- assert(statement(result).value === 10);
-});
+ assert(str.value === "ÿ鹿ÿ");
+ assert(str.encoding === "ascii-8bit");
+ assert(str.validEncoding);
+ });
+ });
+
+ test("constant", () => {
+ const result = parse("foo = 1");
+ assert(result.value.locals[0] === "foo");
+ });
+
+ test("constant? present", () => {
+ const result = parse("def foo(*bar); end");
+ assert(statement(result).parameters.rest.name === "bar");
+ });
+
+ test("constant? absent", () => {
+ const result = parse("def foo(*); end");
+ assert(statement(result).parameters.rest.name === null);
+ });
+
+ test("constant[]", async() => {
+ const result = parse("foo = 1");
+ assert(result.value.locals instanceof Array);
+ });
+
+ test("location", () => {
+ const result = parse("foo = 1");
+ const location = result.value.location;
+
+ assert(location instanceof Location);
+ assert(location.startOffset === 0);
+ assert(location.length === 7);
+ assert(location.endOffset() === 7);
+ });
+
+ test("location? present", () => {
+ const result = parse("def foo = bar");
+
+ assert(statement(result).equalLoc instanceof Location);
+ });
+
+ test("location? absent", () => {
+ const result = parse("def foo; bar; end");
+ assert(statement(result).equalLoc === null);
+ });
+
+ test("uint8", () => {
+ const result = parse("-> { _3 }");
+ assert(statement(result).parameters.maximum === 3);
+ });
+
+ test("uint32", () => {
+ const result = parse("foo = 1");
+ assert(statement(result).depth === 0);
+ });
+
+ test("flags", () => {
+ const result = parse("/foo/mi");
+ const regexp = statement(result);
+
+ assert(regexp.isIgnoreCase());
+ assert(regexp.isMultiLine());
+ assert(!regexp.isExtended());
+ });
+
+ suite("integer", () => {
+ test("decimal", () => {
+ const result = parse("10");
+ assert(statement(result).value === 10);
+ });
-test("integer (hex)", () => {
- const result = parse("0xA");
- assert(statement(result).value === 10);
-});
+ test("hex", () => {
+ const result = parse("0xA");
+ assert(statement(result).value === 10);
+ });
-test("integer (2 nodes)", () => {
- const result = parse("4294967296");
- assert(statement(result).value === 4294967296n);
-});
+ test("2 nodes", () => {
+ const result = parse("4294967296");
+ assert(statement(result).value === 4294967296n);
+ });
-test("integer (3 nodes)", () => {
- const result = parse("18446744073709552000");
- assert(statement(result).value === 18446744073709552000n);
-});
+ test("3 nodes", () => {
+ const result = parse("18446744073709552000");
+ assert(statement(result).value === 18446744073709552000n);
+ });
+ });
-test("double", () => {
- const result = parse("1.0");
- assert(statement(result).value === 1.0);
+ test("double", () => {
+ const result = parse("1.0");
+ assert(statement(result).value === 1.0);
+ });
});
test("scopes", () => {
@@ -259,3 +277,153 @@ test("visitor visits nodes inside node list fields", () => {
assert.deepStrictEqual(collect("begin\n a\nrescue B\n c\nend"), ["a", "c"]);
assert.deepStrictEqual(collect("def foo(a = b); end"), ["b"]);
});
+
+test("source", () => {
+ const source = new Source(new TextEncoder().encode("foo\nbar\nbaz"), "utf-8", 1, [0, 4, 8]);
+
+ assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.line(offset)), [1, 2, 3, 3]);
+ assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.lineStart(offset)), [0, 4, 8, 8]);
+ assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.lineEnd(offset)), [4, 8, 11, 11]);
+ assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.column(offset)), [0, 0, 0, 3]);
+ assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.codeUnitsColumn(offset)), [0, 0, 0, 3]);
+
+ for (const encoding of ["utf-8", "utf-16", "utf-32"]) {
+ assert.deepStrictEqual([0, 4, 8, 11].map((offset) => source.codeUnitsColumn(offset, encoding)), [0, 0, 0, 3]);
+ }
+
+ assert.throws(() => source.codeUnitsColumn(0, "utf-64"));
+
+ assert(source.slice(4, 3) === "bar");
+ assert(source.slice(0, 11) === "foo\nbar\nbaz");
+});
+
+suite("location", () => {
+ test("lines and columns", () => {
+ const result = parse("foo\nbar\nbaz");
+ const body = result.value.statements.body;
+
+ assert.deepStrictEqual(body.map((node) => node.location.startLine()), [1, 2, 3]);
+ assert.deepStrictEqual(body.map((node) => node.location.endLine()), [1, 2, 3]);
+ assert.deepStrictEqual(body.map((node) => node.location.startColumn()), [0, 0, 0]);
+ assert.deepStrictEqual(body.map((node) => node.location.endColumn()), [3, 3, 3]);
+ });
+
+ test("lines and columns spanning multiple lines", () => {
+ const result = parse("foo(\n bar\n)");
+ const location = statement(result).location;
+
+ assert(location.startLine() === 1);
+ assert(location.startColumn() === 0);
+ assert(location.endLine() === 3);
+ assert(location.endColumn() === 1);
+ });
+
+ test("lines respect the line option", () => {
+ for (const line of [-2147483648, -1073741824, -5, -1, 0, 1, 10, 1073741824, 2147483646]) {
+ const result = parse("foo\nbar", { line });
+ const body = result.value.statements.body;
+
+ assert.deepStrictEqual(body.map((node) => node.location.startLine()), [line, line + 1]);
+ }
+ });
+
+ test("slice", () => {
+ const result = parse("foo(bar)");
+ const node = statement(result);
+
+ assert(node.location.slice() === "foo(bar)");
+ assert(node.messageLoc.slice() === "foo");
+ assert(node.arguments_.arguments_[0].location.slice() === "bar");
+ });
+
+ test("slice with multibyte characters", () => {
+ const result = parse('"鹿" + "foo"');
+ const node = statement(result);
+
+ assert(node.receiver.location.slice() === '"鹿"');
+ assert(node.receiver.location.startColumn() === 0);
+
+ assert(node.receiver.location.endColumn() === 5);
+ assert(node.arguments_.arguments_[0].location.slice() === '"foo"');
+ });
+
+ test("code units columns default to utf-16 and count code units, not bytes", () => {
+ const result = parse('"鹿" + "foo"');
+ const node = statement(result);
+
+ assert(node.receiver.location.startCodeUnitsColumn() === 0);
+ assert(node.receiver.location.endCodeUnitsColumn() === 3);
+
+ assert(node.arguments_.arguments_[0].location.startCodeUnitsColumn() === 6);
+ assert(node.arguments_.arguments_[0].location.endCodeUnitsColumn() === 11);
+ });
+
+ test("code units columns count in the requested position encoding", () => {
+ const result = parse('"鹿" + "foo"');
+ const node = statement(result);
+ const argument = node.arguments_.arguments_[0];
+
+ // 鹿 is three bytes and one code unit in every form, so only utf-8 differs.
+ assert.deepStrictEqual(
+ ["utf-8", "utf-16", "utf-32"].map((encoding) => node.receiver.location.endCodeUnitsColumn(encoding)),
+ [5, 3, 3]
+ );
+
+ assert.deepStrictEqual(
+ ["utf-8", "utf-16", "utf-32"].map((encoding) => argument.location.startCodeUnitsColumn(encoding)),
+ [8, 6, 6]
+ );
+ });
+
+ test("code units columns count characters outside the BMP per encoding", () => {
+ const result = parse('"\u{1F600}" + "foo"');
+ const node = statement(result);
+ const argument = node.arguments_.arguments_[0];
+
+ // The emoji is four bytes, two utf-16 code units, and one codepoint.
+ assert(node.receiver.location.endColumn() === 6);
+ assert.deepStrictEqual(
+ ["utf-8", "utf-16", "utf-32"].map((encoding) => node.receiver.location.endCodeUnitsColumn(encoding)),
+ [6, 4, 3]
+ );
+
+ assert.deepStrictEqual(
+ ["utf-8", "utf-16", "utf-32"].map((encoding) => argument.location.endCodeUnitsColumn(encoding)),
+ [14, 12, 11]
+ );
+ });
+
+ test("code units columns count bytes that do not decode as themselves in utf-8", () => {
+ // 0xff is not valid utf-8, and the parser tolerates it in the source. The
+ // editor still holds that one byte, so the utf-8 column stays 7 rather than
+ // widening to the three bytes a replacement character would encode to.
+ const bytes = new Uint8Array([0x78, 0x20, 0x3d, 0x20, 0x22, 0xff, 0x22]);
+ const source = new Source(bytes, "UTF-8", 1, [0]);
+
+ assert(source.codeUnitsColumn(7, "utf-8") === 7);
+ assert(source.codeUnitsColumn(7, "utf-16") === 7);
+ });
+
+ test("code units columns in a source that is not utf-8", () => {
+ // x = "ソ" in Shift_JIS, where ソ is 0x83 0x5c and that trailing byte is an
+ // ASCII backslash.
+ const bytes = new Uint8Array([0x78, 0x20, 0x3d, 0x20, 0x22, 0x83, 0x5c, 0x22]);
+ const source = new Source(bytes, "Shift_JIS", 1, [0]);
+
+ assert(source.column(8) === 8);
+ assert.deepStrictEqual(
+ ["utf-8", "utf-16", "utf-32"].map((encoding) => source.codeUnitsColumn(8, encoding)),
+ [9, 7, 7]
+ );
+ });
+
+ test("code units columns are relative to the start of the line", () => {
+ const result = parse('x = 1\n"鹿" + "foo"');
+ const node = result.value.statements.body[1];
+
+ assert(node.location.startLine() === 2);
+ assert(node.receiver.location.startCodeUnitsColumn() === 0);
+ assert(node.receiver.location.endCodeUnitsColumn() === 3);
+ assert(node.receiver.location.endCodeUnitsColumn("utf-8") === 5);
+ });
+});
diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb
index 49499c43ea..5673edb181 100644
--- a/templates/javascript/src/deserialize.js.erb
+++ b/templates/javascript/src/deserialize.js.erb
@@ -1,13 +1,16 @@
import * as nodes from "./nodes.js";
+import { getDecoder } from "./decoding.js";
+import { Location } from "./location.js";
+import { Source } from "./source.js";
const MAJOR_VERSION = 1;
const MINOR_VERSION = 9;
const PATCH_VERSION = 0;
-// The DataView getFloat64 function takes an optional second argument that
-// specifies whether the number is little-endian or big-endian. It does not
-// appear to have a native endian mode, so we need to determine the endianness
-// of the system at runtime.
+/* The DataView getFloat64 function takes an optional second argument that
+ * specifies whether the number is little-endian or big-endian. It does not
+ * appear to have a native endian mode, so we need to determine the endianness
+ * of the system at runtime. */
const LITTLE_ENDIAN = (() => {
let uint32 = new Uint32Array([0x11223344]);
let uint8 = new Uint8Array(uint32.buffer);
@@ -22,18 +25,11 @@ const LITTLE_ENDIAN = (() => {
})();
class SerializationBuffer {
- FORCED_UTF8_ENCODING_FLAG = 1 << 2;
- FORCED_BINARY_ENCODING_FLAG = 1 << 3;
-
- DECODER_MAP = new Map([
- ["ascii-8bit", "ascii"]
- ]);
-
constructor(array) {
this.array = array;
this.index = 0;
- this.fileEncoding = "utf-8";
- this.decoders = new Map();
+ this.encoding = "utf-8";
+ this.source = null;
}
readByte() {
@@ -52,7 +48,7 @@ class SerializationBuffer {
return this.decodeString(this.readBytes(length), flags).value;
}
- // Read a 32-bit unsigned integer in little-endian format.
+ /* Read a 32-bit unsigned integer in little-endian format. */
readUint32() {
const result = this.scanUint32(this.index);
this.index += 4;
@@ -64,6 +60,17 @@ class SerializationBuffer {
return bytes[0] | (bytes[1] << 8) | (bytes[2] << 16) | (bytes[3] << 24);
}
+ /**
+ * Read a signed integer that was written in zigzag encoding, where the low
+ * bit carries the sign. Both operators here are 32-bit, which is what makes
+ * this correct for zigzag values above 2^31, where readVarInt overflows and
+ * returns the bit pattern as a negative number.
+ */
+ readVarSInt() {
+ const value = this.readVarInt();
+ return (value >>> 1) ^ -(value & 1);
+ }
+
readVarInt() {
let result = 0;
let shift = 0;
@@ -82,7 +89,9 @@ class SerializationBuffer {
}
readLocation() {
- return { startOffset: this.readVarInt(), length: this.readVarInt() };
+ const startOffset = this.readVarInt();
+ const length = this.readVarInt();
+ return new Location(this.source, startOffset, length);
}
readOptionalLocation() {
@@ -102,8 +111,7 @@ class SerializationBuffer {
const offset = constantPoolOffset + constantIndex * 8;
const startOffset = this.scanUint32(offset);
const length = this.scanUint32(offset + 4);
-
- return this.getDecoder(this.fileEncoding).decode(this.array.slice(startOffset, startOffset + length));
+ return getDecoder(this.encoding, { fatal: true }).decode(this.array.slice(startOffset, startOffset + length));
}
readDouble() {
@@ -116,67 +124,37 @@ class SerializationBuffer {
}
decodeString(bytes, flags) {
- const forcedBin = (flags & this.FORCED_BINARY_ENCODING_FLAG) !== 0;
- const forcedUtf8 = (flags & this.FORCED_UTF8_ENCODING_FLAG) !== 0;
-
- if (forcedBin) {
- // just use raw bytes
- return {
- encoding: "ascii",
- validEncoding: true,
- value: this.asciiDecoder.decode(bytes)
- };
- } else {
- const encoding = forcedUtf8 ? "utf-8" : this.fileEncoding.toLowerCase();
- const decoder = this.getDecoder(encoding);
-
- try {
- // decode with encoding
- return {
- encoding,
- validEncoding: true,
- value: decoder.decode(bytes)
- };
- } catch(e) {
- // just use raw bytes, capture what the encoding should be, set flag saying encoding is invalid
- if (e.code === "ERR_ENCODING_INVALID_ENCODED_DATA") {
- return {
- encoding,
- validEncoding: false,
- value: this.asciiDecoder.decode(bytes)
- };
- }
-
- throw e;
- }
- }
- }
-
- getDecoder(encoding) {
- encoding = this.DECODER_MAP.get(encoding) || encoding;
+ let value, encoding, validEncoding;
- if (!this.decoders.has(encoding)) {
- this.decoders.set(encoding, new TextDecoder(encoding, {fatal: true}));
+ if ((flags & nodes.StringFlags.FORCED_BINARY_ENCODING) !== 0) {
+ encoding = "ascii-8bit";
+ } else if ((flags & nodes.StringFlags.FORCED_UTF8_ENCODING) !== 0) {
+ encoding = "utf-8";
+ } else {
+ encoding = this.encoding.toLowerCase();
}
- return this.decoders.get(encoding);
- }
-
- get asciiDecoder() {
- if (!this._asciiDecoder) {
- this._asciiDecoder = new TextDecoder("ascii");
+ const decoder = getDecoder(encoding, { fatal: true });
+
+ try {
+ value = decoder.decode(bytes);
+ validEncoding = true;
+ } catch (error) {
+ /* A fatal decoder reports bytes that are not valid in its encoding by
+ * throwing a TypeError, and that is the whole of what it throws, so the
+ * string is read back as binary and flagged. */
+ if (error instanceof TypeError) {
+ value = getDecoder("ascii-8bit").decode(bytes);
+ validEncoding = false;
+ } else {
+ throw error;
+ }
}
- return this._asciiDecoder;
+ return { value, encoding, validEncoding };
}
}
-/**
- * A location in the source code.
- *
- * @typedef {{ startOffset: number, length: number }} Location
- */
-
/**
* A comment in the source code.
*
@@ -229,6 +207,7 @@ export class ParseResult {
/**
* @type {Location | null}
*/
+ dataLoc;
/**
* @type {ParseError[]}
@@ -240,20 +219,28 @@ export class ParseResult {
*/
warnings;
+ /**
+ * @type {boolean}
+ */
+ continuable;
+
/**
* @param {nodes.ProgramNode} value
* @param {Comment[]} comments
* @param {MagicComment[]} magicComments
+ * @param {Location | null} dataLoc
* @param {ParseError[]} errors
* @param {ParseWarning[]} warnings
+ * @param {boolean} continuable
*/
- constructor(value, comments, magicComments, dataLoc, errors, warnings) {
+ constructor(value, comments, magicComments, dataLoc, errors, warnings, continuable) {
this.value = value;
this.comments = comments;
this.magicComments = magicComments;
this.dataLoc = dataLoc;
this.errors = errors;
this.warnings = warnings;
+ this.continuable = continuable;
}
}
@@ -275,11 +262,12 @@ const warningTypes = [
* Accept two Uint8Arrays, one for the source and one for the serialized format.
* Return the AST corresponding to the serialized form.
*
+ * @param {Uint8Array} source
* @param {Uint8Array} array
* @returns {ParseResult}
* @throws {Error}
*/
-export function deserialize(array) {
+export function deserialize(source, array) {
const buffer = new SerializationBuffer(array);
if (buffer.readString(5) !== "PRISM") {
@@ -295,18 +283,14 @@ export function deserialize(array) {
}
// Read the file's encoding.
- buffer.fileEncoding = buffer.readString(buffer.readVarInt());
+ buffer.encoding = buffer.readString(buffer.readVarInt());
- // Skip past the start line, as we don't support that option yet in
- // JavaScript.
- buffer.readVarInt();
+ // Read the line that the source starts on along with the byte offset of the
+ // start of each line, which together let locations resolve lines and columns.
+ const startLine = buffer.readVarSInt();
+ const lineOffsets = Array.from({ length: buffer.readVarInt() }, () => buffer.readVarInt());
- // Skip past the line offsets, as there is no Source object yet in JavaScript.
- // const lineOffsets = Array.from({ length: buffer.readVarInt() }, () => buffer.readVarInt());
- const lineOffsetsCount = buffer.readVarInt();
- for (let i = 0; i < lineOffsetsCount; i ++) {
- buffer.readVarInt();
- }
+ buffer.source = new Source(source, buffer.encoding, startLine, lineOffsets);
const comments = Array.from({ length: buffer.readVarInt() }, () => ({
type: buffer.readVarInt(),
diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb
index f31c2ad216..fcab285461 100644
--- a/templates/javascript/src/nodes.js.erb
+++ b/templates/javascript/src/nodes.js.erb
@@ -24,6 +24,7 @@ def jstype(field)
end
end
-%>
+import { Location } from "./location.js"
import * as visitors from "./visitor.js"
<%- flags.each do |flag| -%>
@@ -31,19 +32,13 @@ import * as visitors from "./visitor.js"
/**
* <%= flag.comment %>
*/
-const <%= flag.name %> = {
+export const <%= flag.name %> = {
<%- flag.values.each_with_index do |value, index| -%>
<%= value.name %>: 1 << <%= index + Prism::Template::COMMON_FLAGS_COUNT %>,
<%- end -%>
};
<%- end -%>
-/**
- * A location in the source code.
- *
- * @typedef {{ startOffset: number, length: number }} Location
- */
-
/**
* An encoded Ruby string.
*
@@ -137,7 +132,7 @@ export class <%= node.name -%> {
when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "this.#{prop(field)}"
when Prism::Template::NodeListField then "...this.#{prop(field)}"
end
- }.compact.join(", ") %>]
+ }.compact.join(", ") %>];
}
/**