From e09aab2ed4089b2e62f9f44600e9498da74f83f4 Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Mon, 7 Sep 2026 19:44:48 -0700 Subject: [PATCH 1/3] Carry a script's top-level declarations across the wombat block The rewriter wraps a script in a block so wombat can shadow window, document and the rest. A block is a scope, so every top-level const, let and class in that script becomes block-scoped with it and stops being visible to any other script on the page. The script still runs, so nothing reports an error; the page is simply wrong. This ports wabac.js's approach (parseGlobals in src/rewrite/jsrewriter.ts), keeping its names and its output byte for byte: * let is declared before the block and its keyword removed inside, so the assignment writes the outer binding * const and class are handed out through self.___WB_const_ and re-declared after the block, and the carrier deleted * a name that shadows one of the wombat globals is left to the script and that global is dropped from the wrapper * a top-level document.write() still gets its document.close() Parsing is behind rewriting/js_ast.py so the parser is one import to change. tree-sitter rather than a pure-Python parser because the input is whatever the live web served: esprima is ES2017 and refuses optional chaining, class fields and for-await, all ordinary in shipped code, and a parse failure silently restores the bug. A script that still cannot be parsed is wrapped exactly as before rather than corrupted. Two existing fixtures encoded the old behaviour and are updated. One of them, a script declaring 'var self', was emitting 'let self' twice in the same block: a SyntaxError that killed the whole script. Fixes #329 --- pyproject.toml | 7 ++ src/zimscraperlib/rewriting/js.py | 91 +++++++++++++++- src/zimscraperlib/rewriting/js_ast.py | 146 ++++++++++++++++++++++++++ tests/rewriting/test_js_globals.py | 140 ++++++++++++++++++++++++ tests/rewriting/test_js_rewriting.py | 61 ++++++++--- 5 files changed, 428 insertions(+), 17 deletions(-) create mode 100644 src/zimscraperlib/rewriting/js_ast.py create mode 100644 tests/rewriting/test_js_globals.py diff --git a/pyproject.toml b/pyproject.toml index 06a731d..cc8f341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,13 @@ dependencies = [ "piexif==1.1.3", # this dep is a nightmare in terms of release management, better pinned just like in optimize-images anyway "idna>=2.5,<4.0", "xxhash>=2.0,<4.0", + # Parsing JavaScript well enough to know which names a script declares at its + # top level (see rewriting/js_ast.py). tree-sitter rather than a pure-Python + # parser because the input is whatever the live web served: esprima is ES2017 + # and refuses optional chaining, class fields and `for await`, all ordinary in + # shipped code, and a parse failure here silently restores the bug this fixes. + "tree-sitter>=0.23,<1.0", + "tree-sitter-javascript>=0.23,<1.0", "types-xxhash>=2.0,<4.0", ] dynamic = ["authors", "classifiers", "keywords", "license", "version", "urls"] diff --git a/src/zimscraperlib/rewriting/js.py b/src/zimscraperlib/rewriting/js.py index 2743fde..171fb4e 100644 --- a/src/zimscraperlib/rewriting/js.py +++ b/src/zimscraperlib/rewriting/js.py @@ -21,6 +21,7 @@ from collections.abc import Callable, Iterable from typing import Any, Literal +from zimscraperlib.rewriting.js_ast import parse_top_level from zimscraperlib.rewriting.rx_replacer import ( RxRewriter, TransformationAction, @@ -348,13 +349,97 @@ def rewrite(self, text: str | bytes, opts: dict[str, Any] | None = None) -> str: if opts.get("inline", False): new_text = new_text.replace("\n", " ") - # This is not totally correctly handling globals, - # see https://github.com/openzim/python-scraperlib/issues/329 if wrap_globals: - new_text = self.first_buff + new_text + self.last_buff + new_text = self._wrap(new_text, GLOBAL_OVERRIDES) + if opts.get("inline", False): + new_text = new_text.replace("\n", " ") return new_text + def _wrap(self, new_text: str, overrides: list[str]) -> str: + """Put the script inside the wombat block, and put its globals back. + + The block is a scope, so `const`, `let` and `class` declared at the top + level of the script stop being reachable from any other script on the + page — which is how a page that declares its data in one