From 5047a8d46fb4fc840202d5c635f66f50a236e023 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Wed, 29 Jul 2026 07:04:01 +0000 Subject: [PATCH 1/4] feat(parser): cache generated parse tables Co-authored-by: GPT-5.6-Sol --- README.md | 11 + plare/parser.py | 420 +++++++++++++++++++++++++++++++++++-- tests/test_parser_cache.py | 281 +++++++++++++++++++++++++ 3 files changed, 696 insertions(+), 16 deletions(-) create mode 100644 tests/test_parser_cache.py diff --git a/README.md b/README.md index 5bc083d..9a55b40 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ classes and dictionaries — no code generation, no external grammar files. modes mid-stream (e.g., to skip comments) - **LALR(1) parser** — efficient shift/reduce parser with automatic conflict detection +- **Persistent parse-table cache** — opt in to skip repeated LALR table construction - **Operator precedence** — resolve shift/reduce conflicts by setting `precedence` and `associative` class variables on token classes - **No build step** — install and import @@ -39,6 +40,16 @@ lexer = Lexer({"start": [(r"\d+", NUM), (r"\+", PLUS), (r" +", "start")]}) parser = Parser({"exp": [(["exp", PLUS, "exp"], Add, [0, 2]), ([NUM], Const, [0])]}) ``` +## Parse-table cache + +Pass a cache file path when constructing a parser to reuse its generated LALR table across process runs: + +```python +parser = Parser(grammar, cache_path=".cache/plare/expressions.json") +``` + +Plare creates missing parent directories and writes the cache atomically. Grammar structure, rule order, token precedence or associativity, and `%prec` changes automatically invalidate it; corrupt or unreadable cache files are ignored and rebuilt. Semantic action classes and argument lists are always taken from the current grammar rather than the cached file. + ## Examples - [`examples/calc/`](examples/calc/) — integer arithmetic with operator precedence diff --git a/plare/parser.py b/plare/parser.py index dc85131..86ad773 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -12,19 +12,30 @@ 4. Compute LALR(1) per-item lookahead sets (ASU §9.6). 5. Populate the action/goto table; resolve shift/reduce and reduce/reduce conflicts using token precedence and associativity. + +When ``cache_path`` points to a compatible table, construction stops after +grammar normalization and rebinds the cached actions to the current classes. """ from __future__ import annotations +import hashlib +import json +import os from collections import deque from collections.abc import Mapping, Sequence from itertools import chain -from typing import Iterable, Protocol, TypeGuard +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Iterable, Protocol, TypeGuard, cast from plare.exception import ParserError, ParsingError from plare.token import Token from plare.utils import logger +PARSE_TABLE_CACHE_VERSION = 1 +"""Version of the on-disk parse-table schema and parser-building algorithm.""" + class EOS(Token): """Sentinel token appended to every token stream to signal end-of-input.""" @@ -799,6 +810,330 @@ def compute_lalr1_lookaheads[T]( return lookahead_table +class InvalidParseTableCache(ValueError): + """Internal signal for a stale, corrupt, or incompatible cache file.""" + + +class StaleParseTableCache(InvalidParseTableCache): + """Internal signal for a valid cache built for another grammar or version.""" + + +def _cache_digest(value: object) -> str: + """Return a deterministic SHA-256 digest for a JSON-compatible value.""" + encoded = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _cache_list(value: object) -> list[object]: + if not isinstance(value, list): + raise InvalidParseTableCache("expected a list") + return cast(list[object], value) + + +def _cache_dict(value: object) -> dict[str, object]: + if not isinstance(value, dict): + raise InvalidParseTableCache("expected an object with string keys") + mapping = cast(dict[object, object], value) + if not all(isinstance(key, str) for key in mapping): + raise InvalidParseTableCache("expected an object with string keys") + return cast(dict[str, object], mapping) + + +def _cache_int(value: object) -> int: + if type(value) is not int: + raise InvalidParseTableCache("expected an integer") + return value + + +def _cache_str(value: object) -> str: + if not isinstance(value, str): + raise InvalidParseTableCache("expected a string") + return value + + +def _encode_cache_symbol( + symbol: Symbol, token_ids: dict[type[Token], int] +) -> list[str | int]: + if symbol is EOS: + return ["eos"] + if isinstance(symbol, type): + return ["token", token_ids[symbol]] + return ["nonterminal", symbol] + + +def _decode_cache_symbol(value: object, tokens: list[type[Token]]) -> Symbol: + encoded = _cache_list(value) + if not encoded: + raise InvalidParseTableCache("empty symbol") + kind = _cache_str(encoded[0]) + if kind == "eos" and len(encoded) == 1: + return EOS + if kind == "token" and len(encoded) == 2: + token_id = _cache_int(encoded[1]) + if 0 <= token_id < len(tokens): + return tokens[token_id] + if kind == "nonterminal" and len(encoded) == 2: + return _cache_str(encoded[1]) + raise InvalidParseTableCache("invalid symbol") + + +def _encode_cache_action[T](action: Action[T] | None) -> list[str | int]: + match action: + case Shift(next=next_state): + return ["shift", next_state] + case Reduce(definition_index=definition_index): + return ["reduce", definition_index] + case Accept(symbol=symbol): + return ["accept", symbol] + case Goto(next=next_state): + return ["goto", next_state] + case _: + raise ValueError(f"Unsupported parse-table action: {action}") + + +def _copy_reduce[T](reduction: Reduce[T]) -> Reduce[T]: + """Return a distinct reduce action bound to the same current-grammar maker.""" + return Reduce( + reduction.left, + reduction.n, + reduction.maker, + reduction.precedence, + reduction.definition_index, + ) + + +def _decode_cache_action[T]( + value: object, + state_count: int, + reductions: dict[int, Reduce[T]], + entry_names: list[str], +) -> Action[T]: + encoded = _cache_list(value) + if len(encoded) != 2: + raise InvalidParseTableCache("invalid action") + kind = _cache_str(encoded[0]) + target = encoded[1] + if kind in {"shift", "goto"}: + next_state = _cache_int(target) + if not 0 <= next_state < state_count: + raise InvalidParseTableCache("state target out of range") + return Shift(next_state) if kind == "shift" else Goto(next_state) + if kind == "reduce": + definition_index = _cache_int(target) + try: + return _copy_reduce(reductions[definition_index]) + except KeyError: + raise InvalidParseTableCache("unknown production") from None + if kind == "accept": + symbol = _cache_str(target) + if symbol not in entry_names: + raise InvalidParseTableCache("unknown accept symbol") + return Accept(symbol) + raise InvalidParseTableCache("unknown action") + + +def _encode_parse_table_cache[T]( + fingerprint: str, + table: Table[T], + entry_state: dict[str, int], + token_ids: dict[type[Token], int], +) -> dict[str, object]: + rows: list[object] = [] + for row in table.table: + rows.append( + [ + [ + _encode_cache_symbol(symbol, token_ids), + _encode_cache_action(action), + ] + for symbol, action in row.items() + ] + ) + data: dict[str, object] = { + "entry_state": entry_state, + "table": rows, + } + return { + "version": PARSE_TABLE_CACHE_VERSION, + "grammar": fingerprint, + "checksum": _cache_digest(data), + "data": data, + } + + +def _decode_parse_table_cache[T]( + raw: object, + fingerprint: str, + tokens: list[type[Token]], + reductions: dict[int, Reduce[T]], + entry_names: list[str], +) -> tuple[Table[T], dict[str, int]]: + payload = _cache_dict(raw) + if _cache_int(payload.get("version")) != PARSE_TABLE_CACHE_VERSION: + raise StaleParseTableCache("cache version mismatch") + if _cache_str(payload.get("grammar")) != fingerprint: + raise StaleParseTableCache("grammar mismatch") + + data = _cache_dict(payload.get("data")) + if _cache_str(payload.get("checksum")) != _cache_digest(data): + raise InvalidParseTableCache("checksum mismatch") + + encoded_rows = _cache_list(data.get("table")) + state_count = len(encoded_rows) + encoded_entry_state = _cache_dict(data.get("entry_state")) + if set(encoded_entry_state) != set(entry_names): + raise InvalidParseTableCache("entry symbols mismatch") + entry_state: dict[str, int] = {} + for symbol in entry_names: + state = _cache_int(encoded_entry_state[symbol]) + if not 0 <= state < state_count: + raise InvalidParseTableCache("entry state out of range") + entry_state[symbol] = state + + table = Table[T](state_count) + for state, encoded_row in enumerate(encoded_rows): + for encoded_cell in _cache_list(encoded_row): + cell = _cache_list(encoded_cell) + if len(cell) != 2: + raise InvalidParseTableCache("invalid table cell") + symbol = _decode_cache_symbol(cell[0], tokens) + action = _decode_cache_action(cell[1], state_count, reductions, entry_names) + if isinstance(symbol, type): + if isinstance(action, Goto): + raise InvalidParseTableCache("goto action for a token") + else: + if symbol not in entry_names: + raise InvalidParseTableCache("unknown nonterminal") + if not isinstance(action, Goto): + raise InvalidParseTableCache("non-goto action for a nonterminal") + if symbol in table.table[state]: + raise InvalidParseTableCache("duplicate table cell") + table.table[state][symbol] = action + return table, entry_state + + +def _load_parse_table_cache[T]( + path: Path, + fingerprint: str, + tokens: list[type[Token]], + reductions: dict[int, Reduce[T]], + entry_names: list[str], +) -> tuple[Table[T], dict[str, int]] | None: + try: + with path.open(encoding="utf-8") as cache_file: + raw = cast(object, json.load(cache_file)) + return _decode_parse_table_cache( + raw, fingerprint, tokens, reductions, entry_names + ) + except FileNotFoundError: + return None + except StaleParseTableCache as error: + logger.info("Rebuilding stale parse-table cache %s: %s", path, error) + return None + except (OSError, RecursionError, UnicodeDecodeError, ValueError) as error: + logger.warning("Ignoring parse-table cache %s: %s", path, error) + return None + + +def _parse_table_cache_identity[T]( + rules: list[Rule[T]], +) -> tuple[str, list[type[Token]], dict[type[Token], int]]: + """Build the grammar fingerprint and terminal registry used by the cache.""" + tokens: list[type[Token]] = [] + token_ids: dict[type[Token], int] = {} + encoded_rules: list[object] = [] + for rule in rules: + encoded_productions: list[object] = [] + for right, _, prec_override in rule.rights: + encoded_right: list[object] = [] + for symbol in right: + if isinstance(symbol, type): + if symbol not in token_ids: + token_ids[symbol] = len(tokens) + tokens.append(symbol) + encoded_right.append(["token", token_ids[symbol]]) + else: + encoded_right.append(["nonterminal", symbol]) + encoded_productions.append( + { + "right": encoded_right, + "precedence": prec_override, + } + ) + encoded_rules.append( + { + "left": rule.left, + "productions": encoded_productions, + } + ) + + token_descriptors = [ + { + "module": token.__module__, + "qualname": token.__qualname__, + "name": token.__name__, + "precedence": token.precedence, + "associative": token.associative, + } + for token in tokens + ] + fingerprint = _cache_digest( + { + "rules": encoded_rules, + "tokens": token_descriptors, + } + ) + return fingerprint, tokens, token_ids + + +def _write_parse_table_cache[T]( + path: Path, + fingerprint: str, + table: Table[T], + entry_state: dict[str, int], + token_ids: dict[type[Token], int], +) -> None: + temporary_path: Path | None = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + payload = _encode_parse_table_cache(fingerprint, table, entry_state, token_ids) + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as cache_file: + temporary_path = Path(cache_file.name) + json.dump( + payload, + cache_file, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + cache_file.flush() + os.fsync(cache_file.fileno()) + os.replace(temporary_path, path) + temporary_path = None + except (KeyError, OSError, TypeError, ValueError) as error: + logger.warning("Unable to write parse-table cache %s: %s", path, error) + finally: + if temporary_path is not None: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + + class Parser[T]: """LALR(1) parser that builds a parse table from a grammar and drives LR parsing. @@ -822,14 +1157,20 @@ class Parser[T]: through a single child unchanged. * ``arg_indices``: which RHS children to forward to ``action_type.__init__``. + Pass ``cache_path`` to persist the generated table. Compatible cache files + are rebound to the current grammar's token and semantic-action classes. + Attributes: table: The completed LR action/goto table. entry_state: Mapping from non-terminal name → initial state id for that entry point (one entry point per top-level key in the grammar). + cache_hit: Whether this instance loaded its parse table from + ``cache_path`` instead of building it. """ table: Table[T] entry_state: dict[str, int] + cache_hit: bool def __init__( self, @@ -842,6 +1183,8 @@ def __init__( ] ], ], + *, + cache_path: str | os.PathLike[str] | None = None, ) -> None: # ── Phase 1: Augment grammar ───────────────────────────────────────── # For each entry non-terminal X, add an augmented rule @@ -857,6 +1200,7 @@ def __init__( # definition_index (global counter) so equal-precedence R/R conflicts # can be resolved by definition order. rules: dict[str, Rule[T]] = {} + user_rules: list[Rule[T]] = [] entry_rules: list[tuple[StartVariable, Rule[T]]] = [] start_variables: set[StartVariable] = set() global_idx = 0 @@ -867,13 +1211,15 @@ def __init__( for entry in productions: if len(entry) == 4: right, action, args, prec_token = entry - norm_rights.append( - (list(right), action, args, prec_token.precedence) - ) + prec_override = prec_token.precedence else: right, action, args = entry - norm_rights.append((list(right), action, args, None)) - rules[left] = Rule[T](left, norm_rights, global_idx) + prec_override = None + normalized_right = list(right) + norm_rights.append((normalized_right, action, args, prec_override)) + rule = Rule[T](left, norm_rights, global_idx) + rules[left] = rule + user_rules.append(rule) start_var = StartVariable(left) augmented = Rule[T](start_var, [([left], None, [0], None)], 0) rules[start_var] = augmented @@ -881,16 +1227,54 @@ def __init__( start_variables.add(start_var) global_idx += len(norm_rights) + reductions: dict[int, Reduce[T]] = {} + for rule in user_rules: + for (right, maker, prec_override), definition_index in zip( + rule.rights, rule.definition_indices + ): + item = Item( + rule.left, + right, + maker, + definition_index, + prec_override=prec_override, + ) + reductions[definition_index] = Reduce( + rule.left, + len(right), + maker, + item.precedence, + definition_index, + ) + + self.cache_hit = False + cache_file = Path(cache_path) if cache_path is not None else None + cache_fingerprint: str | None = None + cache_token_ids: dict[type[Token], int] = {} + entry_names = [left.orig for left, _ in entry_rules] + if cache_file is not None: + cache_fingerprint, cache_tokens, cache_token_ids = ( + _parse_table_cache_identity(user_rules) + ) + cached = _load_parse_table_cache( + cache_file, + cache_fingerprint, + cache_tokens, + reductions, + entry_names, + ) + if cached is not None: + self.table, self.entry_state = cached + self.cache_hit = True + logger.info("Parser loaded from cache: %s", cache_file) + return + # ── Phase 2: Compute FIRST sets ────────────────────────────────────── # FIRST(A) is needed to propagate ε through nullable non-terminals # during LALR(1) lookahead propagation in Phase 4. first_sets = compute_first_sets(rules) all_items = {left: rule.items for left, rule in rules.items()} - all_tokens = set[type[Token]]() - for rule in rules.values(): - for right, _, _ in rule.rights: - all_tokens.update(t for t in right if isinstance(t, type)) # ── Phase 3: Build LR(0) canonical collection ──────────────────────── # BFS over the LR(0) automaton. ``state_index`` maps a frozenset of @@ -979,12 +1363,8 @@ def __init__( ) else: for symbol in state.lookaheads.get(item, set()): - reduce_action = Reduce( - item.left, - len(item.right), - item.maker, - item.precedence, - item.definition_index, + reduce_action = _copy_reduce( + reductions[item.definition_index] ) try: self.table[state.id, symbol] = reduce_action @@ -1018,6 +1398,14 @@ def __init__( self.table.resolve_conflict( state.id, symbol, reduce_action ) + if cache_file is not None and cache_fingerprint is not None: + _write_parse_table_cache( + cache_file, + cache_fingerprint, + self.table, + self.entry_state, + cache_token_ids, + ) logger.info("Parser created") def parse(self, var: str, lexbuf: Iterable[Token]) -> T | Token: diff --git a/tests/test_parser_cache.py b/tests/test_parser_cache.py new file mode 100644 index 0000000..1d80c10 --- /dev/null +++ b/tests/test_parser_cache.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Barrier +from typing import Any + +import pytest + +import plare.parser as parser_module +from plare.parser import Parser +from plare.token import Token + + +class NUM_C(Token): + def __init__(self, value: str, *, lineno: int, offset: int) -> None: + super().__init__(value, lineno=lineno, offset=offset) + self.value = int(value) + + +class PLUS_C(Token): + precedence = 1 + associative = "left" + + +class PREC_C(Token): + precedence = 1 + + +class NumC: + def __init__(self, token: NUM_C) -> None: + self.value = token.value + + +class AddC: + def __init__(self, left: Any, right: Any) -> None: + self.left = left + self.right = right + + +def expression_grammar() -> dict[ + str, + list[tuple[list[type[Token] | str], type[Any] | None, list[int]]], +]: + return { + "expr": [ + (["expr", PLUS_C, "expr"], AddC, [0, 2]), + ([NUM_C], NumC, [0]), + ] + } + + +def expression_tokens() -> list[Token]: + return [ + NUM_C("1", lineno=1, offset=0), + PLUS_C("+", lineno=1, offset=1), + NUM_C("2", lineno=1, offset=2), + PLUS_C("+", lineno=1, offset=3), + NUM_C("3", lineno=1, offset=4), + ] + + +def assert_expression_result(result: object, *, left_associative: bool) -> None: + assert isinstance(result, AddC) + branch = result.left if left_associative else result.right + assert isinstance(branch, AddC) + + +def fail_if_table_is_built(*args: object, **kwargs: object) -> None: + raise AssertionError("parse table should have been loaded from cache") + + +def test_cache_miss_writes_file_and_hit_skips_table_build( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_path = tmp_path / "nested" / "expression.json" + + first = Parser(expression_grammar(), cache_path=str(cache_path)) + + assert first.cache_hit is False + assert cache_path.is_file() + assert_expression_result( + first.parse("expr", expression_tokens()), left_associative=True + ) + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + second = Parser(expression_grammar(), cache_path=cache_path) + + assert second.cache_hit is True + assert second.entry_state == first.entry_state + assert_expression_result( + second.parse("expr", expression_tokens()), left_associative=True + ) + + +def test_cache_rebinds_current_local_semantic_action( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class First: + def __init__(self, left: NUM_C, right: NUM_C) -> None: + self.values = (left.value, right.value) + + class Second: + def __init__(self, right: NUM_C, left: NUM_C) -> None: + self.values = (right.value, left.value) + + cache_path = tmp_path / "local-action.json" + Parser( + {"value": [([NUM_C, PLUS_C, NUM_C], First, [0, 2])]}, + cache_path=cache_path, + ) + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + + cached = Parser( + {"value": [([NUM_C, PLUS_C, NUM_C], Second, [2, 0])]}, + cache_path=cache_path, + ) + result = cached.parse( + "value", + [ + NUM_C("4", lineno=1, offset=0), + PLUS_C("+", lineno=1, offset=1), + NUM_C("7", lineno=1, offset=2), + ], + ) + + assert cached.cache_hit is True + assert isinstance(result, Second) + assert result.values == (7, 4) + + +def test_cache_preserves_non_alphabetical_entry_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + grammar = { + "z_value": [([NUM_C], NumC, [0])], + "a_value": [([NUM_C], NumC, [0])], + } + cache_path = tmp_path / "entry-order.json" + first = Parser(grammar, cache_path=cache_path) + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + + cached = Parser(grammar, cache_path=cache_path) + + assert cached.cache_hit is True + assert ( + list(cached.entry_state) + == list(first.entry_state) + == [ + "z_value", + "a_value", + ] + ) + + +def test_token_conflict_metadata_change_invalidates_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_path = tmp_path / "associativity.json" + first = Parser(expression_grammar(), cache_path=cache_path) + assert_expression_result( + first.parse("expr", expression_tokens()), left_associative=True + ) + + monkeypatch.setattr(PLUS_C, "associative", "right") + second = Parser(expression_grammar(), cache_path=cache_path) + + assert second.cache_hit is False + assert_expression_result( + second.parse("expr", expression_tokens()), left_associative=False + ) + + +def test_grammar_structure_change_invalidates_cache(tmp_path: Path) -> None: + cache_path = tmp_path / "grammar.json" + Parser({"value": [([NUM_C], NumC, [0])]}, cache_path=cache_path) + + changed = Parser( + {"value": [([PLUS_C, NUM_C], NumC, [1])]}, + cache_path=cache_path, + ) + result = changed.parse( + "value", + [ + PLUS_C("+", lineno=1, offset=0), + NUM_C("9", lineno=1, offset=1), + ], + ) + + assert changed.cache_hit is False + assert isinstance(result, NumC) + assert result.value == 9 + + +def test_prec_token_change_invalidates_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_path = tmp_path / "prec.json" + grammar = {"value": [([NUM_C], NumC, [0], PREC_C)]} + Parser(grammar, cache_path=cache_path) + + monkeypatch.setattr(PREC_C, "precedence", 2) + changed = Parser(grammar, cache_path=cache_path) + + assert changed.cache_hit is False + + +@pytest.mark.parametrize("contents", ["", "{", "[]", '{"version": 1}']) +def test_corrupt_cache_is_rebuilt( + contents: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_path = tmp_path / "corrupt.json" + cache_path.write_text(contents, encoding="utf-8") + + rebuilt = Parser(expression_grammar(), cache_path=cache_path) + + assert rebuilt.cache_hit is False + assert_expression_result( + rebuilt.parse("expr", expression_tokens()), left_associative=True + ) + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + cached = Parser(expression_grammar(), cache_path=cache_path) + assert cached.cache_hit is True + + +def test_deeply_nested_cache_is_rebuilt(tmp_path: Path) -> None: + cache_path = tmp_path / "deeply-nested.json" + cache_path.write_text("[" * 10_000 + "]" * 10_000, encoding="utf-8") + + rebuilt = Parser(expression_grammar(), cache_path=cache_path) + + assert rebuilt.cache_hit is False + assert_expression_result( + rebuilt.parse("expr", expression_tokens()), left_associative=True + ) + + +def test_cache_io_failure_does_not_break_parser(tmp_path: Path) -> None: + cache_path = tmp_path / "cache-is-a-directory" + cache_path.mkdir() + + parser = Parser(expression_grammar(), cache_path=cache_path) + + assert parser.cache_hit is False + assert_expression_result( + parser.parse("expr", expression_tokens()), left_associative=True + ) + assert not list(tmp_path.glob(f".{cache_path.name}.*.tmp")) + + +def test_concurrent_cache_initialization_is_atomic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_path = tmp_path / "concurrent" / "expression.json" + barrier = Barrier(4) + + def construct_and_parse(_: int) -> object: + barrier.wait() + parser = Parser(expression_grammar(), cache_path=cache_path) + return parser.parse("expr", expression_tokens()) + + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(construct_and_parse, range(8))) + + for result in results: + assert_expression_result(result, left_associative=True) + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + cached = Parser(expression_grammar(), cache_path=cache_path) + assert cached.cache_hit is True From 7a4975a159190ea121d9e2ce1839a05974730fba Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Wed, 29 Jul 2026 07:22:32 +0000 Subject: [PATCH 2/4] fix(parser): align parse-table cache defaults Co-authored-by: GPT-5.6-Sol --- .gitignore | 1 + README.md | 10 +- plare/parser.py | 124 ++++++++++++------------ tests/test_determinism.py | 10 +- tests/test_error_reporting.py | 8 +- tests/test_grammar_logic.py | 17 ++-- tests/test_integration.py | 3 +- tests/test_parser.py | 8 +- tests/test_parser_cache.py | 164 +++++++++++++++++++++++++------- tests/test_parser_safety_net.py | 26 +++-- tests/test_prec_override.py | 8 +- 11 files changed, 249 insertions(+), 130 deletions(-) diff --git a/.gitignore b/.gitignore index b6e4761..8151f0d 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.plare/ # Translations *.mo diff --git a/README.md b/README.md index 9a55b40..1eec2be 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ classes and dictionaries — no code generation, no external grammar files. modes mid-stream (e.g., to skip comments) - **LALR(1) parser** — efficient shift/reduce parser with automatic conflict detection -- **Persistent parse-table cache** — opt in to skip repeated LALR table construction +- **Persistent parse-table cache** — automatically skip repeated LALR table construction - **Operator precedence** — resolve shift/reduce conflicts by setting `precedence` and `associative` class variables on token classes - **No build step** — install and import @@ -42,13 +42,15 @@ parser = Parser({"exp": [(["exp", PLUS, "exp"], Add, [0, 2]), ([NUM], Const, [0] ## Parse-table cache -Pass a cache file path when constructing a parser to reuse its generated LALR table across process runs: +Plare automatically caches generated LALR tables in `/.plare`, treating the current working directory as the project root. Each grammar uses its own fingerprinted JSON file, and compatible tables are reused across process runs: ```python -parser = Parser(grammar, cache_path=".cache/plare/expressions.json") +parser = Parser(grammar) ``` -Plare creates missing parent directories and writes the cache atomically. Grammar structure, rule order, token precedence or associativity, and `%prec` changes automatically invalidate it; corrupt or unreadable cache files are ignored and rebuilt. Semantic action classes and argument lists are always taken from the current grammar rather than the cached file. +Pass `cache_dir=None` to disable caching or provide another directory with `cache_dir="path/to/cache"`. + +Plare creates missing cache directories and writes each table atomically. Grammar structure, rule order, token precedence or associativity, `%prec`, and the Plare version automatically determine cache compatibility; corrupt or unreadable cache files are ignored and rebuilt. Semantic action classes and argument lists are always taken from the current grammar rather than the cached file. ## Examples diff --git a/plare/parser.py b/plare/parser.py index 86ad773..a2ba768 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -13,8 +13,8 @@ 5. Populate the action/goto table; resolve shift/reduce and reduce/reduce conflicts using token precedence and associativity. -When ``cache_path`` points to a compatible table, construction stops after -grammar normalization and rebinds the cached actions to the current classes. +When a compatible cached table exists, construction stops after grammar +normalization and rebinds the cached actions to the current classes. """ from __future__ import annotations @@ -29,12 +29,13 @@ from tempfile import NamedTemporaryFile from typing import Iterable, Protocol, TypeGuard, cast +from plare import __version__ as PLARE_VERSION from plare.exception import ParserError, ParsingError from plare.token import Token from plare.utils import logger -PARSE_TABLE_CACHE_VERSION = 1 -"""Version of the on-disk parse-table schema and parser-building algorithm.""" +PARSER_TABLE_CACHE_VERSION = PLARE_VERSION +"""Plare version used to validate on-disk parse tables.""" class EOS(Token): @@ -818,7 +819,7 @@ class StaleParseTableCache(InvalidParseTableCache): """Internal signal for a valid cache built for another grammar or version.""" -def _cache_digest(value: object) -> str: +def cache_digest(value: object) -> str: """Return a deterministic SHA-256 digest for a JSON-compatible value.""" encoded = json.dumps( value, @@ -830,13 +831,13 @@ def _cache_digest(value: object) -> str: return hashlib.sha256(encoded).hexdigest() -def _cache_list(value: object) -> list[object]: +def cache_list(value: object) -> list[object]: if not isinstance(value, list): raise InvalidParseTableCache("expected a list") return cast(list[object], value) -def _cache_dict(value: object) -> dict[str, object]: +def cache_dict(value: object) -> dict[str, object]: if not isinstance(value, dict): raise InvalidParseTableCache("expected an object with string keys") mapping = cast(dict[object, object], value) @@ -845,19 +846,19 @@ def _cache_dict(value: object) -> dict[str, object]: return cast(dict[str, object], mapping) -def _cache_int(value: object) -> int: +def cache_int(value: object) -> int: if type(value) is not int: raise InvalidParseTableCache("expected an integer") return value -def _cache_str(value: object) -> str: +def cache_str(value: object) -> str: if not isinstance(value, str): raise InvalidParseTableCache("expected a string") return value -def _encode_cache_symbol( +def encode_cache_symbol( symbol: Symbol, token_ids: dict[type[Token], int] ) -> list[str | int]: if symbol is EOS: @@ -867,23 +868,23 @@ def _encode_cache_symbol( return ["nonterminal", symbol] -def _decode_cache_symbol(value: object, tokens: list[type[Token]]) -> Symbol: - encoded = _cache_list(value) +def decode_cache_symbol(value: object, tokens: list[type[Token]]) -> Symbol: + encoded = cache_list(value) if not encoded: raise InvalidParseTableCache("empty symbol") - kind = _cache_str(encoded[0]) + kind = cache_str(encoded[0]) if kind == "eos" and len(encoded) == 1: return EOS if kind == "token" and len(encoded) == 2: - token_id = _cache_int(encoded[1]) + token_id = cache_int(encoded[1]) if 0 <= token_id < len(tokens): return tokens[token_id] if kind == "nonterminal" and len(encoded) == 2: - return _cache_str(encoded[1]) + return cache_str(encoded[1]) raise InvalidParseTableCache("invalid symbol") -def _encode_cache_action[T](action: Action[T] | None) -> list[str | int]: +def encode_cache_action[T](action: Action[T] | None) -> list[str | int]: match action: case Shift(next=next_state): return ["shift", next_state] @@ -897,7 +898,7 @@ def _encode_cache_action[T](action: Action[T] | None) -> list[str | int]: raise ValueError(f"Unsupported parse-table action: {action}") -def _copy_reduce[T](reduction: Reduce[T]) -> Reduce[T]: +def copy_reduce[T](reduction: Reduce[T]) -> Reduce[T]: """Return a distinct reduce action bound to the same current-grammar maker.""" return Reduce( reduction.left, @@ -908,37 +909,37 @@ def _copy_reduce[T](reduction: Reduce[T]) -> Reduce[T]: ) -def _decode_cache_action[T]( +def decode_cache_action[T]( value: object, state_count: int, reductions: dict[int, Reduce[T]], entry_names: list[str], ) -> Action[T]: - encoded = _cache_list(value) + encoded = cache_list(value) if len(encoded) != 2: raise InvalidParseTableCache("invalid action") - kind = _cache_str(encoded[0]) + kind = cache_str(encoded[0]) target = encoded[1] if kind in {"shift", "goto"}: - next_state = _cache_int(target) + next_state = cache_int(target) if not 0 <= next_state < state_count: raise InvalidParseTableCache("state target out of range") return Shift(next_state) if kind == "shift" else Goto(next_state) if kind == "reduce": - definition_index = _cache_int(target) + definition_index = cache_int(target) try: - return _copy_reduce(reductions[definition_index]) + return copy_reduce(reductions[definition_index]) except KeyError: raise InvalidParseTableCache("unknown production") from None if kind == "accept": - symbol = _cache_str(target) + symbol = cache_str(target) if symbol not in entry_names: raise InvalidParseTableCache("unknown accept symbol") return Accept(symbol) raise InvalidParseTableCache("unknown action") -def _encode_parse_table_cache[T]( +def encode_parse_table_cache[T]( fingerprint: str, table: Table[T], entry_state: dict[str, int], @@ -949,8 +950,8 @@ def _encode_parse_table_cache[T]( rows.append( [ [ - _encode_cache_symbol(symbol, token_ids), - _encode_cache_action(action), + encode_cache_symbol(symbol, token_ids), + encode_cache_action(action), ] for symbol, action in row.items() ] @@ -960,50 +961,50 @@ def _encode_parse_table_cache[T]( "table": rows, } return { - "version": PARSE_TABLE_CACHE_VERSION, + "version": PARSER_TABLE_CACHE_VERSION, "grammar": fingerprint, - "checksum": _cache_digest(data), + "checksum": cache_digest(data), "data": data, } -def _decode_parse_table_cache[T]( +def decode_parse_table_cache[T]( raw: object, fingerprint: str, tokens: list[type[Token]], reductions: dict[int, Reduce[T]], entry_names: list[str], ) -> tuple[Table[T], dict[str, int]]: - payload = _cache_dict(raw) - if _cache_int(payload.get("version")) != PARSE_TABLE_CACHE_VERSION: + payload = cache_dict(raw) + if cache_str(payload.get("version")) != PARSER_TABLE_CACHE_VERSION: raise StaleParseTableCache("cache version mismatch") - if _cache_str(payload.get("grammar")) != fingerprint: + if cache_str(payload.get("grammar")) != fingerprint: raise StaleParseTableCache("grammar mismatch") - data = _cache_dict(payload.get("data")) - if _cache_str(payload.get("checksum")) != _cache_digest(data): + data = cache_dict(payload.get("data")) + if cache_str(payload.get("checksum")) != cache_digest(data): raise InvalidParseTableCache("checksum mismatch") - encoded_rows = _cache_list(data.get("table")) + encoded_rows = cache_list(data.get("table")) state_count = len(encoded_rows) - encoded_entry_state = _cache_dict(data.get("entry_state")) + encoded_entry_state = cache_dict(data.get("entry_state")) if set(encoded_entry_state) != set(entry_names): raise InvalidParseTableCache("entry symbols mismatch") entry_state: dict[str, int] = {} for symbol in entry_names: - state = _cache_int(encoded_entry_state[symbol]) + state = cache_int(encoded_entry_state[symbol]) if not 0 <= state < state_count: raise InvalidParseTableCache("entry state out of range") entry_state[symbol] = state table = Table[T](state_count) for state, encoded_row in enumerate(encoded_rows): - for encoded_cell in _cache_list(encoded_row): - cell = _cache_list(encoded_cell) + for encoded_cell in cache_list(encoded_row): + cell = cache_list(encoded_cell) if len(cell) != 2: raise InvalidParseTableCache("invalid table cell") - symbol = _decode_cache_symbol(cell[0], tokens) - action = _decode_cache_action(cell[1], state_count, reductions, entry_names) + symbol = decode_cache_symbol(cell[0], tokens) + action = decode_cache_action(cell[1], state_count, reductions, entry_names) if isinstance(symbol, type): if isinstance(action, Goto): raise InvalidParseTableCache("goto action for a token") @@ -1018,7 +1019,7 @@ def _decode_parse_table_cache[T]( return table, entry_state -def _load_parse_table_cache[T]( +def load_parse_table_cache[T]( path: Path, fingerprint: str, tokens: list[type[Token]], @@ -1028,7 +1029,7 @@ def _load_parse_table_cache[T]( try: with path.open(encoding="utf-8") as cache_file: raw = cast(object, json.load(cache_file)) - return _decode_parse_table_cache( + return decode_parse_table_cache( raw, fingerprint, tokens, reductions, entry_names ) except FileNotFoundError: @@ -1041,7 +1042,7 @@ def _load_parse_table_cache[T]( return None -def _parse_table_cache_identity[T]( +def parse_table_cache_identity[T]( rules: list[Rule[T]], ) -> tuple[str, list[type[Token]], dict[type[Token], int]]: """Build the grammar fingerprint and terminal registry used by the cache.""" @@ -1050,7 +1051,10 @@ def _parse_table_cache_identity[T]( encoded_rules: list[object] = [] for rule in rules: encoded_productions: list[object] = [] - for right, _, prec_override in rule.rights: + for production in rule.rights: + # Semantic makers are rebound from the current grammar on cache load. + right = production[0] + prec_override = production[2] encoded_right: list[object] = [] for symbol in right: if isinstance(symbol, type): @@ -1083,7 +1087,7 @@ def _parse_table_cache_identity[T]( } for token in tokens ] - fingerprint = _cache_digest( + fingerprint = cache_digest( { "rules": encoded_rules, "tokens": token_descriptors, @@ -1092,7 +1096,7 @@ def _parse_table_cache_identity[T]( return fingerprint, tokens, token_ids -def _write_parse_table_cache[T]( +def write_parse_table_cache[T]( path: Path, fingerprint: str, table: Table[T], @@ -1102,7 +1106,7 @@ def _write_parse_table_cache[T]( temporary_path: Path | None = None try: path.parent.mkdir(parents=True, exist_ok=True) - payload = _encode_parse_table_cache(fingerprint, table, entry_state, token_ids) + payload = encode_parse_table_cache(fingerprint, table, entry_state, token_ids) with NamedTemporaryFile( mode="w", encoding="utf-8", @@ -1157,15 +1161,15 @@ class Parser[T]: through a single child unchanged. * ``arg_indices``: which RHS children to forward to ``action_type.__init__``. - Pass ``cache_path`` to persist the generated table. Compatible cache files - are rebound to the current grammar's token and semantic-action classes. + By default, generated tables are stored under ``.plare`` in the current + project directory. Pass ``cache_dir=None`` to disable caching. Attributes: table: The completed LR action/goto table. entry_state: Mapping from non-terminal name → initial state id for that entry point (one entry point per top-level key in the grammar). cache_hit: Whether this instance loaded its parse table from - ``cache_path`` instead of building it. + ``cache_dir`` instead of building it. """ table: Table[T] @@ -1184,7 +1188,7 @@ def __init__( ], ], *, - cache_path: str | os.PathLike[str] | None = None, + cache_dir: str | os.PathLike[str] | None = ".plare", ) -> None: # ── Phase 1: Augment grammar ───────────────────────────────────────── # For each entry non-terminal X, add an augmented rule @@ -1248,15 +1252,17 @@ def __init__( ) self.cache_hit = False - cache_file = Path(cache_path) if cache_path is not None else None + cache_directory = Path(cache_dir) if cache_dir is not None else None + cache_file: Path | None = None cache_fingerprint: str | None = None cache_token_ids: dict[type[Token], int] = {} entry_names = [left.orig for left, _ in entry_rules] - if cache_file is not None: + if cache_directory is not None: cache_fingerprint, cache_tokens, cache_token_ids = ( - _parse_table_cache_identity(user_rules) + parse_table_cache_identity(user_rules) ) - cached = _load_parse_table_cache( + cache_file = cache_directory / f"{cache_fingerprint}.json" + cached = load_parse_table_cache( cache_file, cache_fingerprint, cache_tokens, @@ -1363,7 +1369,7 @@ def __init__( ) else: for symbol in state.lookaheads.get(item, set()): - reduce_action = _copy_reduce( + reduce_action = copy_reduce( reductions[item.definition_index] ) try: @@ -1399,7 +1405,7 @@ def __init__( state.id, symbol, reduce_action ) if cache_file is not None and cache_fingerprint is not None: - _write_parse_table_cache( + write_parse_table_cache( cache_file, cache_fingerprint, self.table, diff --git a/tests/test_determinism.py b/tests/test_determinism.py index 9005010..1c4418a 100644 --- a/tests/test_determinism.py +++ b/tests/test_determinism.py @@ -102,8 +102,8 @@ def make_expr_grammar() -> Grammar: def test_parser_entry_state_is_deterministic() -> None: """Two Parser instances built from identical grammars have the same entry_state.""" - p1 = Parser(make_expr_grammar()) - p2 = Parser(make_expr_grammar()) + p1 = Parser(make_expr_grammar(), cache_dir=None) + p2 = Parser(make_expr_grammar(), cache_dir=None) assert p1.entry_state == p2.entry_state @@ -113,8 +113,8 @@ def test_parser_table_actions_are_deterministic() -> None: Compares every cell by its string representation so the test does not rely on Action subclass identity. """ - p1 = Parser(make_expr_grammar()) - p2 = Parser(make_expr_grammar()) + p1 = Parser(make_expr_grammar(), cache_dir=None) + p2 = Parser(make_expr_grammar(), cache_dir=None) assert len(p1.table.table) == len(p2.table.table), "table row count differs" for state_id, (row1, row2) in enumerate(zip(p1.table.table, p2.table.table)): @@ -174,7 +174,7 @@ def test_parser_build_time_large_grammar() -> None: """ grammar = make_chain_grammar(50) start = time.perf_counter() - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) elapsed = time.perf_counter() - start assert p.entry_state, "parser must have at least one entry state" print(f"\nLarge grammar (50 levels) build time: {elapsed * 1000:.1f} ms") diff --git a/tests/test_error_reporting.py b/tests/test_error_reporting.py index 04a48d8..a23a73b 100644 --- a/tests/test_error_reporting.py +++ b/tests/test_error_reporting.py @@ -53,7 +53,7 @@ def make_tok(cls: type[Token], *, lineno: int = 1, offset: int = 0) -> Token: def test_parsing_error_unexpected_token_fields() -> None: """ParsingError carries the offending token, its position, and expected classes.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) wrong_tok = make_tok(Plus, lineno=3, offset=7) with pytest.raises(ParsingError) as exc_info: @@ -68,7 +68,7 @@ def test_parsing_error_unexpected_token_fields() -> None: def test_parsing_error_unexpected_token_str() -> None: """str(ParsingError) starts with the 'Line X, col Y:' prefix.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) wrong_tok = make_tok(Plus, lineno=2, offset=5) with pytest.raises(ParsingError) as exc_info: @@ -86,7 +86,7 @@ def test_parsing_error_unexpected_token_str() -> None: def test_parsing_error_truncated_input_expected_nonempty() -> None: """When input is too short, ParsingError.expected is non-empty.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) num_tok = make_tok(Num, lineno=1, offset=0) with pytest.raises(ParsingError) as exc_info: @@ -99,7 +99,7 @@ def test_parsing_error_truncated_input_expected_nonempty() -> None: def test_parsing_error_truncated_input_str() -> None: """str(ParsingError) for truncated input contains the expected class name.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) num_tok = make_tok(Num, lineno=1, offset=0) with pytest.raises(ParsingError) as exc_info: diff --git a/tests/test_grammar_logic.py b/tests/test_grammar_logic.py index d683d87..c7effdb 100644 --- a/tests/test_grammar_logic.py +++ b/tests/test_grammar_logic.py @@ -139,7 +139,7 @@ def eval_c(node: ExprC) -> int: } ) -calc_parser = Parser(CALC_GRAMMAR) +calc_parser = Parser(CALC_GRAMMAR, cache_dir=None) def num(v: int, o: int = 0) -> NUM_C: @@ -282,7 +282,8 @@ def test_default_sr_conflict_is_left_associative() -> None: (["expr", PLUS_SR, "expr"], AddSR, [0, 2]), ([NUM_SR], NumSR, [0]), ] - } + }, + cache_dir=None, ) result = p.parse( "expr", @@ -371,7 +372,8 @@ def test_dangling_else_inner_if_gets_else() -> None: ([IF_D, BASE_D, THEN_D, "stmt"], IfThenD, [1, 3]), ([BASE_D], BaseStmtD, [0]), ] - } + }, + cache_dir=None, ) tokens = [ IF_D("if", lineno=1, offset=0), @@ -459,7 +461,8 @@ def __init__(self, l: ExprUP, r: ExprUP) -> None: (["expr", STAR_UP, "expr"], MulUP, [0, 2]), ([MINUS_UP, "expr"], NegUP, [1], UMINUS_UP), # prec override → 3 ] - } + }, + cache_dir=None, ) @@ -559,7 +562,7 @@ def __init__(self) -> None: ([NUM_L], SingleItemL, [0]), ], } -list_parser = Parser(LIST_GRAMMAR) +list_parser = Parser(LIST_GRAMMAR, cache_dir=None) def num_l(v: int, o: int = 0) -> NUM_L: @@ -673,7 +676,7 @@ def __init__(self, name: ID_F) -> None: ([NUM_F], SingleArgF, [0]), ], } -call_parser = Parser(CALL_GRAMMAR) +call_parser = Parser(CALL_GRAMMAR, cache_dir=None) def num_f(v: int, o: int = 0) -> NUM_F: @@ -859,7 +862,7 @@ def __init__(self, stmts: StmtsE) -> None: ([], EmptyStmtsE, []), ], } -program_parser = Parser(PROGRAM_GRAMMAR) +program_parser = Parser(PROGRAM_GRAMMAR, cache_dir=None) def test_empty_program() -> None: diff --git a/tests/test_integration.py b/tests/test_integration.py index 77dcc3a..0b5e887 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -365,7 +365,8 @@ def lex_word(matched: str, state: None, lineno: int, offset: int) -> Token: ([IF, "expr", THEN, "expr", ELSE, "expr"], IfExpr, [1, 3, 5]), ([LET, ID, EQ, "expr", IN, "expr"], LetExpr, [1, 3, 5]), ] - } + }, + cache_dir=None, ) # --------------------------------------------------------------------------- diff --git a/tests/test_parser.py b/tests/test_parser.py index e717c14..b8b7cd2 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -50,7 +50,8 @@ def make_positive_integer_parser() -> Parser[Tree]: "num": [ ([NUM], Num, [0]), ], - } + }, + cache_dir=None, ) @@ -62,7 +63,7 @@ def test_parse_positive_integer_without_add(): def test_minimal_empty_rule_parser(): - parser = Parser({"pgm": [([], list[int], [])]}) + parser = Parser({"pgm": [([], list[int], [])]}, cache_dir=None) parsed = parser.parse("pgm", []) assert isinstance(parsed, list) assert len(parsed) == 0 @@ -102,7 +103,8 @@ def make_list_parser() -> Parser[IntList]: ([NUM, COMMA, "items"], IntList, [0, 1]), ([], EmptyIntList, []), ], - } + }, + cache_dir=None, ) diff --git a/tests/test_parser_cache.py b/tests/test_parser_cache.py index 1d80c10..3790e80 100644 --- a/tests/test_parser_cache.py +++ b/tests/test_parser_cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from concurrent.futures import ThreadPoolExecutor from pathlib import Path from threading import Barrier @@ -8,6 +9,7 @@ import pytest import plare.parser as parser_module +from plare import __version__ as PLARE_VERSION from plare.parser import Parser from plare.token import Token @@ -70,15 +72,102 @@ def fail_if_table_is_built(*args: object, **kwargs: object) -> None: raise AssertionError("parse table should have been loaded from cache") +def only_cache_file(cache_dir: Path) -> Path: + cache_files = sorted(cache_dir.glob("*.json")) + assert len(cache_files) == 1 + return cache_files[0] + + +def test_cache_version_follows_plare_version( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "version" + Parser(expression_grammar(), cache_dir=cache_dir) + cache_file = only_cache_file(cache_dir) + payload = json.loads(cache_file.read_text(encoding="utf-8")) + + assert parser_module.PARSER_TABLE_CACHE_VERSION == PLARE_VERSION + assert payload["version"] == PLARE_VERSION + + monkeypatch.setattr(parser_module, "PARSER_TABLE_CACHE_VERSION", "999.0.0") + rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) + rewritten = json.loads(cache_file.read_text(encoding="utf-8")) + + assert rebuilt.cache_hit is False + assert only_cache_file(cache_dir) == cache_file + assert rewritten["version"] == "999.0.0" + + +def test_default_cache_directory_is_project_root_plare( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + + first = Parser(expression_grammar()) + + cache_dir = tmp_path / ".plare" + assert first.cache_hit is False + assert only_cache_file(cache_dir).is_file() + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + cached = Parser(expression_grammar()) + + assert cached.cache_hit is True + + +def test_default_cache_keeps_multiple_grammars( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + first_grammar = {"value": [([NUM_C], NumC, [0])]} + second_grammar = { + "value": [([PLUS_C, NUM_C], NumC, [1])], + } + + Parser(first_grammar) + Parser(second_grammar) + + cache_dir = tmp_path / ".plare" + assert len(list(cache_dir.glob("*.json"))) == 2 + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + assert Parser(first_grammar).cache_hit is True + assert Parser(second_grammar).cache_hit is True + + +def test_cache_directory_none_disables_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + + parser = Parser(expression_grammar(), cache_dir=None) + + assert parser.cache_hit is False + assert not (tmp_path / ".plare").exists() + + +def test_parser_module_has_no_private_top_level_names() -> None: + private_names = sorted( + name + for name in vars(parser_module) + if name.startswith("_") and not name.startswith("__") + ) + assert private_names == [] + + def test_cache_miss_writes_file_and_hit_skips_table_build( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - cache_path = tmp_path / "nested" / "expression.json" + cache_dir = tmp_path / "nested" / "cache" - first = Parser(expression_grammar(), cache_path=str(cache_path)) + first = Parser(expression_grammar(), cache_dir=str(cache_dir)) assert first.cache_hit is False - assert cache_path.is_file() + assert only_cache_file(cache_dir).is_file() assert_expression_result( first.parse("expr", expression_tokens()), left_associative=True ) @@ -86,7 +175,7 @@ def test_cache_miss_writes_file_and_hit_skips_table_build( monkeypatch.setattr( parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built ) - second = Parser(expression_grammar(), cache_path=cache_path) + second = Parser(expression_grammar(), cache_dir=cache_dir) assert second.cache_hit is True assert second.entry_state == first.entry_state @@ -106,10 +195,10 @@ class Second: def __init__(self, right: NUM_C, left: NUM_C) -> None: self.values = (right.value, left.value) - cache_path = tmp_path / "local-action.json" + cache_dir = tmp_path / "local-action" Parser( {"value": [([NUM_C, PLUS_C, NUM_C], First, [0, 2])]}, - cache_path=cache_path, + cache_dir=cache_dir, ) monkeypatch.setattr( parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built @@ -117,7 +206,7 @@ def __init__(self, right: NUM_C, left: NUM_C) -> None: cached = Parser( {"value": [([NUM_C, PLUS_C, NUM_C], Second, [2, 0])]}, - cache_path=cache_path, + cache_dir=cache_dir, ) result = cached.parse( "value", @@ -140,13 +229,13 @@ def test_cache_preserves_non_alphabetical_entry_order( "z_value": [([NUM_C], NumC, [0])], "a_value": [([NUM_C], NumC, [0])], } - cache_path = tmp_path / "entry-order.json" - first = Parser(grammar, cache_path=cache_path) + cache_dir = tmp_path / "entry-order" + first = Parser(grammar, cache_dir=cache_dir) monkeypatch.setattr( parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built ) - cached = Parser(grammar, cache_path=cache_path) + cached = Parser(grammar, cache_dir=cache_dir) assert cached.cache_hit is True assert ( @@ -162,28 +251,29 @@ def test_cache_preserves_non_alphabetical_entry_order( def test_token_conflict_metadata_change_invalidates_cache( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - cache_path = tmp_path / "associativity.json" - first = Parser(expression_grammar(), cache_path=cache_path) + cache_dir = tmp_path / "associativity" + first = Parser(expression_grammar(), cache_dir=cache_dir) assert_expression_result( first.parse("expr", expression_tokens()), left_associative=True ) monkeypatch.setattr(PLUS_C, "associative", "right") - second = Parser(expression_grammar(), cache_path=cache_path) + second = Parser(expression_grammar(), cache_dir=cache_dir) assert second.cache_hit is False + assert len(list(cache_dir.glob("*.json"))) == 2 assert_expression_result( second.parse("expr", expression_tokens()), left_associative=False ) def test_grammar_structure_change_invalidates_cache(tmp_path: Path) -> None: - cache_path = tmp_path / "grammar.json" - Parser({"value": [([NUM_C], NumC, [0])]}, cache_path=cache_path) + cache_dir = tmp_path / "grammar" + Parser({"value": [([NUM_C], NumC, [0])]}, cache_dir=cache_dir) changed = Parser( {"value": [([PLUS_C, NUM_C], NumC, [1])]}, - cache_path=cache_path, + cache_dir=cache_dir, ) result = changed.parse( "value", @@ -194,6 +284,7 @@ def test_grammar_structure_change_invalidates_cache(tmp_path: Path) -> None: ) assert changed.cache_hit is False + assert len(list(cache_dir.glob("*.json"))) == 2 assert isinstance(result, NumC) assert result.value == 9 @@ -201,24 +292,27 @@ def test_grammar_structure_change_invalidates_cache(tmp_path: Path) -> None: def test_prec_token_change_invalidates_cache( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - cache_path = tmp_path / "prec.json" + cache_dir = tmp_path / "prec" grammar = {"value": [([NUM_C], NumC, [0], PREC_C)]} - Parser(grammar, cache_path=cache_path) + Parser(grammar, cache_dir=cache_dir) monkeypatch.setattr(PREC_C, "precedence", 2) - changed = Parser(grammar, cache_path=cache_path) + changed = Parser(grammar, cache_dir=cache_dir) assert changed.cache_hit is False + assert len(list(cache_dir.glob("*.json"))) == 2 @pytest.mark.parametrize("contents", ["", "{", "[]", '{"version": 1}']) def test_corrupt_cache_is_rebuilt( contents: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - cache_path = tmp_path / "corrupt.json" - cache_path.write_text(contents, encoding="utf-8") + cache_dir = tmp_path / "corrupt" + Parser(expression_grammar(), cache_dir=cache_dir) + cache_file = only_cache_file(cache_dir) + cache_file.write_text(contents, encoding="utf-8") - rebuilt = Parser(expression_grammar(), cache_path=cache_path) + rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) assert rebuilt.cache_hit is False assert_expression_result( @@ -228,15 +322,17 @@ def test_corrupt_cache_is_rebuilt( monkeypatch.setattr( parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built ) - cached = Parser(expression_grammar(), cache_path=cache_path) + cached = Parser(expression_grammar(), cache_dir=cache_dir) assert cached.cache_hit is True def test_deeply_nested_cache_is_rebuilt(tmp_path: Path) -> None: - cache_path = tmp_path / "deeply-nested.json" - cache_path.write_text("[" * 10_000 + "]" * 10_000, encoding="utf-8") + cache_dir = tmp_path / "deeply-nested" + Parser(expression_grammar(), cache_dir=cache_dir) + cache_file = only_cache_file(cache_dir) + cache_file.write_text("[" * 10_000 + "]" * 10_000, encoding="utf-8") - rebuilt = Parser(expression_grammar(), cache_path=cache_path) + rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) assert rebuilt.cache_hit is False assert_expression_result( @@ -245,27 +341,27 @@ def test_deeply_nested_cache_is_rebuilt(tmp_path: Path) -> None: def test_cache_io_failure_does_not_break_parser(tmp_path: Path) -> None: - cache_path = tmp_path / "cache-is-a-directory" - cache_path.mkdir() + cache_dir = tmp_path / "cache-is-a-file" + cache_dir.write_text("not a directory", encoding="utf-8") - parser = Parser(expression_grammar(), cache_path=cache_path) + parser = Parser(expression_grammar(), cache_dir=cache_dir) assert parser.cache_hit is False assert_expression_result( parser.parse("expr", expression_tokens()), left_associative=True ) - assert not list(tmp_path.glob(f".{cache_path.name}.*.tmp")) + assert cache_dir.is_file() def test_concurrent_cache_initialization_is_atomic( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - cache_path = tmp_path / "concurrent" / "expression.json" + cache_dir = tmp_path / "concurrent" barrier = Barrier(4) - def construct_and_parse(_: int) -> object: + def construct_and_parse(task_index: int) -> object: barrier.wait() - parser = Parser(expression_grammar(), cache_path=cache_path) + parser = Parser(expression_grammar(), cache_dir=cache_dir) return parser.parse("expr", expression_tokens()) with ThreadPoolExecutor(max_workers=4) as executor: @@ -277,5 +373,5 @@ def construct_and_parse(_: int) -> object: monkeypatch.setattr( parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built ) - cached = Parser(expression_grammar(), cache_path=cache_path) + cached = Parser(expression_grammar(), cache_dir=cache_dir) assert cached.cache_hit is True diff --git a/tests/test_parser_safety_net.py b/tests/test_parser_safety_net.py index 5e45754..64fa93d 100644 --- a/tests/test_parser_safety_net.py +++ b/tests/test_parser_safety_net.py @@ -69,7 +69,7 @@ def test_epsilon_optional_suffix() -> None: ([], NoLabel, []), ], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) result_no = p.parse( "stmt", @@ -138,7 +138,7 @@ def test_left_recursive_addition_chain() -> None: ], "num": [([NUM2], Num2, [0])], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) result = p.parse( "expr", @@ -204,7 +204,7 @@ def test_right_recursive_cons_list() -> None: ], "num": [([NUM3], Num3, [0])], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) result = p.parse( "list", @@ -284,7 +284,8 @@ def test_operator_precedence_mul_over_add() -> None: (["expr", STAR4, "expr"], Mul4, [0, 2]), ([NUM4], Num4, [0]), ] - } + }, + cache_dir=None, ) # 1 + 2 * 3 → Add4(Num4(1), Mul4(Num4(2), Num4(3))) @@ -352,7 +353,8 @@ def test_left_associative_subtraction() -> None: (["expr", MINUS5, "expr"], Sub5, [0, 2]), ([NUM5], Num5, [0]), ] - } + }, + cache_dir=None, ) # 1 - 2 - 3 → Sub5(Sub5(Num5(1), Num5(2)), Num5(3)) @@ -420,7 +422,8 @@ def test_right_associative_exponentiation() -> None: (["expr", POW6, "expr"], Pow6, [0, 2]), ([NUM6], Num6, [0]), ] - } + }, + cache_dir=None, ) # 2 ^ 3 ^ 4 → Pow6(Num6(2), Pow6(Num6(3), Num6(4))) @@ -484,7 +487,7 @@ def test_multiple_entry_points() -> None: "str_expr": [([WORD7], StrVal, [0])], "int_expr": [([NUM7], IntVal, [0])], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) str_result = p.parse("str_expr", [WORD7("hello", lineno=1, offset=0)]) assert isinstance(str_result, StrVal) @@ -543,7 +546,8 @@ def test_shift_reduce_resolution_prefers_shift() -> None: (["expr", PLUS8, "expr"], Add8, [0, 2]), ([NUM8], Num8, [0]), ] - } + }, + cache_dir=None, ) result = p.parse( @@ -622,7 +626,8 @@ def test_lalr1_resolves_rr_conflict_variant_1() -> None: ], "A_nt": [([E8x], None, [0])], "B_nt": [([E8x], None, [0])], - } + }, + cache_dir=None, ) result = p.parse( "start", @@ -682,7 +687,8 @@ def test_lalr1_resolves_rr_conflict_variant_2() -> None: ], "X_nt": [([T8y], None, [0])], "Y_nt": [([T8y], None, [0])], - } + }, + cache_dir=None, ) result = p.parse( "start2", diff --git a/tests/test_prec_override.py b/tests/test_prec_override.py index cc115e8..b2361c3 100644 --- a/tests/test_prec_override.py +++ b/tests/test_prec_override.py @@ -89,7 +89,8 @@ def parser_with_prec_override() -> Parser[Expr]: (["expr", STAR, "expr"], Mul, [0, 2]), ([MINUS, "expr"], Neg, [1], UMINUS), ] - } + }, + cache_dir=None, ) @@ -102,7 +103,8 @@ def parser_without_prec_override() -> Parser[Expr]: (["expr", STAR, "expr"], Mul, [0, 2]), ([MINUS, "expr"], Neg, [1]), ] - } + }, + cache_dir=None, ) @@ -199,7 +201,7 @@ def test_rr_equal_precedence_first_defined_wins() -> None: "first_val": [([Lit], None, [0])], "second_val": [([Lit], None, [0])], } - parser = Parser(grammar) + parser = Parser(grammar, cache_dir=None) result = parser.parse("result", [Lit("x", lineno=1, offset=0)]) assert isinstance( result, FirstResult From fff0257fa95bd301f782ef88089a67957a4c5151 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Wed, 29 Jul 2026 08:08:42 +0000 Subject: [PATCH 3/4] refactor(parser): use package version for cache Co-authored-by: GPT-5.6-Sol --- plare/parser.py | 9 +++------ tests/test_parser_cache.py | 8 ++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/plare/parser.py b/plare/parser.py index a2ba768..c8a50f7 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -29,14 +29,11 @@ from tempfile import NamedTemporaryFile from typing import Iterable, Protocol, TypeGuard, cast -from plare import __version__ as PLARE_VERSION +import plare from plare.exception import ParserError, ParsingError from plare.token import Token from plare.utils import logger -PARSER_TABLE_CACHE_VERSION = PLARE_VERSION -"""Plare version used to validate on-disk parse tables.""" - class EOS(Token): """Sentinel token appended to every token stream to signal end-of-input.""" @@ -961,7 +958,7 @@ def encode_parse_table_cache[T]( "table": rows, } return { - "version": PARSER_TABLE_CACHE_VERSION, + "version": plare.__version__, "grammar": fingerprint, "checksum": cache_digest(data), "data": data, @@ -976,7 +973,7 @@ def decode_parse_table_cache[T]( entry_names: list[str], ) -> tuple[Table[T], dict[str, int]]: payload = cache_dict(raw) - if cache_str(payload.get("version")) != PARSER_TABLE_CACHE_VERSION: + if cache_str(payload.get("version")) != plare.__version__: raise StaleParseTableCache("cache version mismatch") if cache_str(payload.get("grammar")) != fingerprint: raise StaleParseTableCache("grammar mismatch") diff --git a/tests/test_parser_cache.py b/tests/test_parser_cache.py index 3790e80..abfe132 100644 --- a/tests/test_parser_cache.py +++ b/tests/test_parser_cache.py @@ -8,8 +8,8 @@ import pytest +import plare import plare.parser as parser_module -from plare import __version__ as PLARE_VERSION from plare.parser import Parser from plare.token import Token @@ -86,10 +86,10 @@ def test_cache_version_follows_plare_version( cache_file = only_cache_file(cache_dir) payload = json.loads(cache_file.read_text(encoding="utf-8")) - assert parser_module.PARSER_TABLE_CACHE_VERSION == PLARE_VERSION - assert payload["version"] == PLARE_VERSION + assert "PARSER_TABLE_CACHE_VERSION" not in vars(parser_module) + assert payload["version"] == plare.__version__ - monkeypatch.setattr(parser_module, "PARSER_TABLE_CACHE_VERSION", "999.0.0") + monkeypatch.setattr(plare, "__version__", "999.0.0") rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) rewritten = json.loads(cache_file.read_text(encoding="utf-8")) From 26849cf5677c633c78cfaa4452bd837a409b5b40 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Wed, 29 Jul 2026 08:21:41 +0000 Subject: [PATCH 4/4] docs: trim parse-table cache details Co-authored-by: GPT-5.6-Sol --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index 1eec2be..66bd77d 100644 --- a/README.md +++ b/README.md @@ -40,18 +40,6 @@ lexer = Lexer({"start": [(r"\d+", NUM), (r"\+", PLUS), (r" +", "start")]}) parser = Parser({"exp": [(["exp", PLUS, "exp"], Add, [0, 2]), ([NUM], Const, [0])]}) ``` -## Parse-table cache - -Plare automatically caches generated LALR tables in `/.plare`, treating the current working directory as the project root. Each grammar uses its own fingerprinted JSON file, and compatible tables are reused across process runs: - -```python -parser = Parser(grammar) -``` - -Pass `cache_dir=None` to disable caching or provide another directory with `cache_dir="path/to/cache"`. - -Plare creates missing cache directories and writes each table atomically. Grammar structure, rule order, token precedence or associativity, `%prec`, and the Plare version automatically determine cache compatibility; corrupt or unreadable cache files are ignored and rebuilt. Semantic action classes and argument lists are always taken from the current grammar rather than the cached file. - ## Examples - [`examples/calc/`](examples/calc/) — integer arithmetic with operator precedence