diff --git a/.gitignore b/.gitignore index b6e4761..8151f0d 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.plare/ # Translations *.mo diff --git a/README.md b/README.md index 5bc083d..66bd77d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ classes and dictionaries — no code generation, no external grammar files. modes mid-stream (e.g., to skip comments) - **LALR(1) parser** — efficient shift/reduce parser with automatic conflict detection +- **Persistent parse-table cache** — automatically skip repeated LALR table construction - **Operator precedence** — resolve shift/reduce conflicts by setting `precedence` and `associative` class variables on token classes - **No build step** — install and import diff --git a/plare/parser.py b/plare/parser.py index dc85131..c8a50f7 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -12,15 +12,24 @@ 4. Compute LALR(1) per-item lookahead sets (ASU §9.6). 5. Populate the action/goto table; resolve shift/reduce and reduce/reduce conflicts using token precedence and associativity. + +When a compatible cached table exists, construction stops after grammar +normalization and rebinds the cached actions to the current classes. """ from __future__ import annotations +import hashlib +import json +import os from collections import deque from collections.abc import Mapping, Sequence from itertools import chain -from typing import Iterable, Protocol, TypeGuard +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Iterable, Protocol, TypeGuard, cast +import plare from plare.exception import ParserError, ParsingError from plare.token import Token from plare.utils import logger @@ -799,6 +808,333 @@ def compute_lalr1_lookaheads[T]( return lookahead_table +class InvalidParseTableCache(ValueError): + """Internal signal for a stale, corrupt, or incompatible cache file.""" + + +class StaleParseTableCache(InvalidParseTableCache): + """Internal signal for a valid cache built for another grammar or version.""" + + +def cache_digest(value: object) -> str: + """Return a deterministic SHA-256 digest for a JSON-compatible value.""" + encoded = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def cache_list(value: object) -> list[object]: + if not isinstance(value, list): + raise InvalidParseTableCache("expected a list") + return cast(list[object], value) + + +def cache_dict(value: object) -> dict[str, object]: + if not isinstance(value, dict): + raise InvalidParseTableCache("expected an object with string keys") + mapping = cast(dict[object, object], value) + if not all(isinstance(key, str) for key in mapping): + raise InvalidParseTableCache("expected an object with string keys") + return cast(dict[str, object], mapping) + + +def cache_int(value: object) -> int: + if type(value) is not int: + raise InvalidParseTableCache("expected an integer") + return value + + +def cache_str(value: object) -> str: + if not isinstance(value, str): + raise InvalidParseTableCache("expected a string") + return value + + +def encode_cache_symbol( + symbol: Symbol, token_ids: dict[type[Token], int] +) -> list[str | int]: + if symbol is EOS: + return ["eos"] + if isinstance(symbol, type): + return ["token", token_ids[symbol]] + return ["nonterminal", symbol] + + +def decode_cache_symbol(value: object, tokens: list[type[Token]]) -> Symbol: + encoded = cache_list(value) + if not encoded: + raise InvalidParseTableCache("empty symbol") + kind = cache_str(encoded[0]) + if kind == "eos" and len(encoded) == 1: + return EOS + if kind == "token" and len(encoded) == 2: + token_id = cache_int(encoded[1]) + if 0 <= token_id < len(tokens): + return tokens[token_id] + if kind == "nonterminal" and len(encoded) == 2: + return cache_str(encoded[1]) + raise InvalidParseTableCache("invalid symbol") + + +def encode_cache_action[T](action: Action[T] | None) -> list[str | int]: + match action: + case Shift(next=next_state): + return ["shift", next_state] + case Reduce(definition_index=definition_index): + return ["reduce", definition_index] + case Accept(symbol=symbol): + return ["accept", symbol] + case Goto(next=next_state): + return ["goto", next_state] + case _: + raise ValueError(f"Unsupported parse-table action: {action}") + + +def copy_reduce[T](reduction: Reduce[T]) -> Reduce[T]: + """Return a distinct reduce action bound to the same current-grammar maker.""" + return Reduce( + reduction.left, + reduction.n, + reduction.maker, + reduction.precedence, + reduction.definition_index, + ) + + +def decode_cache_action[T]( + value: object, + state_count: int, + reductions: dict[int, Reduce[T]], + entry_names: list[str], +) -> Action[T]: + encoded = cache_list(value) + if len(encoded) != 2: + raise InvalidParseTableCache("invalid action") + kind = cache_str(encoded[0]) + target = encoded[1] + if kind in {"shift", "goto"}: + next_state = cache_int(target) + if not 0 <= next_state < state_count: + raise InvalidParseTableCache("state target out of range") + return Shift(next_state) if kind == "shift" else Goto(next_state) + if kind == "reduce": + definition_index = cache_int(target) + try: + return copy_reduce(reductions[definition_index]) + except KeyError: + raise InvalidParseTableCache("unknown production") from None + if kind == "accept": + symbol = cache_str(target) + if symbol not in entry_names: + raise InvalidParseTableCache("unknown accept symbol") + return Accept(symbol) + raise InvalidParseTableCache("unknown action") + + +def encode_parse_table_cache[T]( + fingerprint: str, + table: Table[T], + entry_state: dict[str, int], + token_ids: dict[type[Token], int], +) -> dict[str, object]: + rows: list[object] = [] + for row in table.table: + rows.append( + [ + [ + encode_cache_symbol(symbol, token_ids), + encode_cache_action(action), + ] + for symbol, action in row.items() + ] + ) + data: dict[str, object] = { + "entry_state": entry_state, + "table": rows, + } + return { + "version": plare.__version__, + "grammar": fingerprint, + "checksum": cache_digest(data), + "data": data, + } + + +def decode_parse_table_cache[T]( + raw: object, + fingerprint: str, + tokens: list[type[Token]], + reductions: dict[int, Reduce[T]], + entry_names: list[str], +) -> tuple[Table[T], dict[str, int]]: + payload = cache_dict(raw) + if cache_str(payload.get("version")) != plare.__version__: + raise StaleParseTableCache("cache version mismatch") + if cache_str(payload.get("grammar")) != fingerprint: + raise StaleParseTableCache("grammar mismatch") + + data = cache_dict(payload.get("data")) + if cache_str(payload.get("checksum")) != cache_digest(data): + raise InvalidParseTableCache("checksum mismatch") + + encoded_rows = cache_list(data.get("table")) + state_count = len(encoded_rows) + encoded_entry_state = cache_dict(data.get("entry_state")) + if set(encoded_entry_state) != set(entry_names): + raise InvalidParseTableCache("entry symbols mismatch") + entry_state: dict[str, int] = {} + for symbol in entry_names: + state = cache_int(encoded_entry_state[symbol]) + if not 0 <= state < state_count: + raise InvalidParseTableCache("entry state out of range") + entry_state[symbol] = state + + table = Table[T](state_count) + for state, encoded_row in enumerate(encoded_rows): + for encoded_cell in cache_list(encoded_row): + cell = cache_list(encoded_cell) + if len(cell) != 2: + raise InvalidParseTableCache("invalid table cell") + symbol = decode_cache_symbol(cell[0], tokens) + action = decode_cache_action(cell[1], state_count, reductions, entry_names) + if isinstance(symbol, type): + if isinstance(action, Goto): + raise InvalidParseTableCache("goto action for a token") + else: + if symbol not in entry_names: + raise InvalidParseTableCache("unknown nonterminal") + if not isinstance(action, Goto): + raise InvalidParseTableCache("non-goto action for a nonterminal") + if symbol in table.table[state]: + raise InvalidParseTableCache("duplicate table cell") + table.table[state][symbol] = action + return table, entry_state + + +def load_parse_table_cache[T]( + path: Path, + fingerprint: str, + tokens: list[type[Token]], + reductions: dict[int, Reduce[T]], + entry_names: list[str], +) -> tuple[Table[T], dict[str, int]] | None: + try: + with path.open(encoding="utf-8") as cache_file: + raw = cast(object, json.load(cache_file)) + return decode_parse_table_cache( + raw, fingerprint, tokens, reductions, entry_names + ) + except FileNotFoundError: + return None + except StaleParseTableCache as error: + logger.info("Rebuilding stale parse-table cache %s: %s", path, error) + return None + except (OSError, RecursionError, UnicodeDecodeError, ValueError) as error: + logger.warning("Ignoring parse-table cache %s: %s", path, error) + return None + + +def parse_table_cache_identity[T]( + rules: list[Rule[T]], +) -> tuple[str, list[type[Token]], dict[type[Token], int]]: + """Build the grammar fingerprint and terminal registry used by the cache.""" + tokens: list[type[Token]] = [] + token_ids: dict[type[Token], int] = {} + encoded_rules: list[object] = [] + for rule in rules: + encoded_productions: list[object] = [] + for production in rule.rights: + # Semantic makers are rebound from the current grammar on cache load. + right = production[0] + prec_override = production[2] + encoded_right: list[object] = [] + for symbol in right: + if isinstance(symbol, type): + if symbol not in token_ids: + token_ids[symbol] = len(tokens) + tokens.append(symbol) + encoded_right.append(["token", token_ids[symbol]]) + else: + encoded_right.append(["nonterminal", symbol]) + encoded_productions.append( + { + "right": encoded_right, + "precedence": prec_override, + } + ) + encoded_rules.append( + { + "left": rule.left, + "productions": encoded_productions, + } + ) + + token_descriptors = [ + { + "module": token.__module__, + "qualname": token.__qualname__, + "name": token.__name__, + "precedence": token.precedence, + "associative": token.associative, + } + for token in tokens + ] + fingerprint = cache_digest( + { + "rules": encoded_rules, + "tokens": token_descriptors, + } + ) + return fingerprint, tokens, token_ids + + +def write_parse_table_cache[T]( + path: Path, + fingerprint: str, + table: Table[T], + entry_state: dict[str, int], + token_ids: dict[type[Token], int], +) -> None: + temporary_path: Path | None = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + payload = encode_parse_table_cache(fingerprint, table, entry_state, token_ids) + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as cache_file: + temporary_path = Path(cache_file.name) + json.dump( + payload, + cache_file, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + cache_file.flush() + os.fsync(cache_file.fileno()) + os.replace(temporary_path, path) + temporary_path = None + except (KeyError, OSError, TypeError, ValueError) as error: + logger.warning("Unable to write parse-table cache %s: %s", path, error) + finally: + if temporary_path is not None: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + + class Parser[T]: """LALR(1) parser that builds a parse table from a grammar and drives LR parsing. @@ -822,14 +1158,20 @@ class Parser[T]: through a single child unchanged. * ``arg_indices``: which RHS children to forward to ``action_type.__init__``. + By default, generated tables are stored under ``.plare`` in the current + project directory. Pass ``cache_dir=None`` to disable caching. + Attributes: table: The completed LR action/goto table. entry_state: Mapping from non-terminal name → initial state id for that entry point (one entry point per top-level key in the grammar). + cache_hit: Whether this instance loaded its parse table from + ``cache_dir`` instead of building it. """ table: Table[T] entry_state: dict[str, int] + cache_hit: bool def __init__( self, @@ -842,6 +1184,8 @@ def __init__( ] ], ], + *, + cache_dir: str | os.PathLike[str] | None = ".plare", ) -> None: # ── Phase 1: Augment grammar ───────────────────────────────────────── # For each entry non-terminal X, add an augmented rule @@ -857,6 +1201,7 @@ def __init__( # definition_index (global counter) so equal-precedence R/R conflicts # can be resolved by definition order. rules: dict[str, Rule[T]] = {} + user_rules: list[Rule[T]] = [] entry_rules: list[tuple[StartVariable, Rule[T]]] = [] start_variables: set[StartVariable] = set() global_idx = 0 @@ -867,13 +1212,15 @@ def __init__( for entry in productions: if len(entry) == 4: right, action, args, prec_token = entry - norm_rights.append( - (list(right), action, args, prec_token.precedence) - ) + prec_override = prec_token.precedence else: right, action, args = entry - norm_rights.append((list(right), action, args, None)) - rules[left] = Rule[T](left, norm_rights, global_idx) + prec_override = None + normalized_right = list(right) + norm_rights.append((normalized_right, action, args, prec_override)) + rule = Rule[T](left, norm_rights, global_idx) + rules[left] = rule + user_rules.append(rule) start_var = StartVariable(left) augmented = Rule[T](start_var, [([left], None, [0], None)], 0) rules[start_var] = augmented @@ -881,16 +1228,56 @@ def __init__( start_variables.add(start_var) global_idx += len(norm_rights) + reductions: dict[int, Reduce[T]] = {} + for rule in user_rules: + for (right, maker, prec_override), definition_index in zip( + rule.rights, rule.definition_indices + ): + item = Item( + rule.left, + right, + maker, + definition_index, + prec_override=prec_override, + ) + reductions[definition_index] = Reduce( + rule.left, + len(right), + maker, + item.precedence, + definition_index, + ) + + self.cache_hit = False + cache_directory = Path(cache_dir) if cache_dir is not None else None + cache_file: Path | None = None + cache_fingerprint: str | None = None + cache_token_ids: dict[type[Token], int] = {} + entry_names = [left.orig for left, _ in entry_rules] + if cache_directory is not None: + cache_fingerprint, cache_tokens, cache_token_ids = ( + parse_table_cache_identity(user_rules) + ) + cache_file = cache_directory / f"{cache_fingerprint}.json" + cached = load_parse_table_cache( + cache_file, + cache_fingerprint, + cache_tokens, + reductions, + entry_names, + ) + if cached is not None: + self.table, self.entry_state = cached + self.cache_hit = True + logger.info("Parser loaded from cache: %s", cache_file) + return + # ── Phase 2: Compute FIRST sets ────────────────────────────────────── # FIRST(A) is needed to propagate ε through nullable non-terminals # during LALR(1) lookahead propagation in Phase 4. first_sets = compute_first_sets(rules) all_items = {left: rule.items for left, rule in rules.items()} - all_tokens = set[type[Token]]() - for rule in rules.values(): - for right, _, _ in rule.rights: - all_tokens.update(t for t in right if isinstance(t, type)) # ── Phase 3: Build LR(0) canonical collection ──────────────────────── # BFS over the LR(0) automaton. ``state_index`` maps a frozenset of @@ -979,12 +1366,8 @@ def __init__( ) else: for symbol in state.lookaheads.get(item, set()): - reduce_action = Reduce( - item.left, - len(item.right), - item.maker, - item.precedence, - item.definition_index, + reduce_action = copy_reduce( + reductions[item.definition_index] ) try: self.table[state.id, symbol] = reduce_action @@ -1018,6 +1401,14 @@ def __init__( self.table.resolve_conflict( state.id, symbol, reduce_action ) + if cache_file is not None and cache_fingerprint is not None: + write_parse_table_cache( + cache_file, + cache_fingerprint, + self.table, + self.entry_state, + cache_token_ids, + ) logger.info("Parser created") def parse(self, var: str, lexbuf: Iterable[Token]) -> T | Token: diff --git a/tests/test_determinism.py b/tests/test_determinism.py index 9005010..1c4418a 100644 --- a/tests/test_determinism.py +++ b/tests/test_determinism.py @@ -102,8 +102,8 @@ def make_expr_grammar() -> Grammar: def test_parser_entry_state_is_deterministic() -> None: """Two Parser instances built from identical grammars have the same entry_state.""" - p1 = Parser(make_expr_grammar()) - p2 = Parser(make_expr_grammar()) + p1 = Parser(make_expr_grammar(), cache_dir=None) + p2 = Parser(make_expr_grammar(), cache_dir=None) assert p1.entry_state == p2.entry_state @@ -113,8 +113,8 @@ def test_parser_table_actions_are_deterministic() -> None: Compares every cell by its string representation so the test does not rely on Action subclass identity. """ - p1 = Parser(make_expr_grammar()) - p2 = Parser(make_expr_grammar()) + p1 = Parser(make_expr_grammar(), cache_dir=None) + p2 = Parser(make_expr_grammar(), cache_dir=None) assert len(p1.table.table) == len(p2.table.table), "table row count differs" for state_id, (row1, row2) in enumerate(zip(p1.table.table, p2.table.table)): @@ -174,7 +174,7 @@ def test_parser_build_time_large_grammar() -> None: """ grammar = make_chain_grammar(50) start = time.perf_counter() - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) elapsed = time.perf_counter() - start assert p.entry_state, "parser must have at least one entry state" print(f"\nLarge grammar (50 levels) build time: {elapsed * 1000:.1f} ms") diff --git a/tests/test_error_reporting.py b/tests/test_error_reporting.py index 04a48d8..a23a73b 100644 --- a/tests/test_error_reporting.py +++ b/tests/test_error_reporting.py @@ -53,7 +53,7 @@ def make_tok(cls: type[Token], *, lineno: int = 1, offset: int = 0) -> Token: def test_parsing_error_unexpected_token_fields() -> None: """ParsingError carries the offending token, its position, and expected classes.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) wrong_tok = make_tok(Plus, lineno=3, offset=7) with pytest.raises(ParsingError) as exc_info: @@ -68,7 +68,7 @@ def test_parsing_error_unexpected_token_fields() -> None: def test_parsing_error_unexpected_token_str() -> None: """str(ParsingError) starts with the 'Line X, col Y:' prefix.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) wrong_tok = make_tok(Plus, lineno=2, offset=5) with pytest.raises(ParsingError) as exc_info: @@ -86,7 +86,7 @@ def test_parsing_error_unexpected_token_str() -> None: def test_parsing_error_truncated_input_expected_nonempty() -> None: """When input is too short, ParsingError.expected is non-empty.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) num_tok = make_tok(Num, lineno=1, offset=0) with pytest.raises(ParsingError) as exc_info: @@ -99,7 +99,7 @@ def test_parsing_error_truncated_input_expected_nonempty() -> None: def test_parsing_error_truncated_input_str() -> None: """str(ParsingError) for truncated input contains the expected class name.""" - p: Parser[Expr] = Parser(GRAMMAR) + p: Parser[Expr] = Parser(GRAMMAR, cache_dir=None) num_tok = make_tok(Num, lineno=1, offset=0) with pytest.raises(ParsingError) as exc_info: diff --git a/tests/test_grammar_logic.py b/tests/test_grammar_logic.py index d683d87..c7effdb 100644 --- a/tests/test_grammar_logic.py +++ b/tests/test_grammar_logic.py @@ -139,7 +139,7 @@ def eval_c(node: ExprC) -> int: } ) -calc_parser = Parser(CALC_GRAMMAR) +calc_parser = Parser(CALC_GRAMMAR, cache_dir=None) def num(v: int, o: int = 0) -> NUM_C: @@ -282,7 +282,8 @@ def test_default_sr_conflict_is_left_associative() -> None: (["expr", PLUS_SR, "expr"], AddSR, [0, 2]), ([NUM_SR], NumSR, [0]), ] - } + }, + cache_dir=None, ) result = p.parse( "expr", @@ -371,7 +372,8 @@ def test_dangling_else_inner_if_gets_else() -> None: ([IF_D, BASE_D, THEN_D, "stmt"], IfThenD, [1, 3]), ([BASE_D], BaseStmtD, [0]), ] - } + }, + cache_dir=None, ) tokens = [ IF_D("if", lineno=1, offset=0), @@ -459,7 +461,8 @@ def __init__(self, l: ExprUP, r: ExprUP) -> None: (["expr", STAR_UP, "expr"], MulUP, [0, 2]), ([MINUS_UP, "expr"], NegUP, [1], UMINUS_UP), # prec override → 3 ] - } + }, + cache_dir=None, ) @@ -559,7 +562,7 @@ def __init__(self) -> None: ([NUM_L], SingleItemL, [0]), ], } -list_parser = Parser(LIST_GRAMMAR) +list_parser = Parser(LIST_GRAMMAR, cache_dir=None) def num_l(v: int, o: int = 0) -> NUM_L: @@ -673,7 +676,7 @@ def __init__(self, name: ID_F) -> None: ([NUM_F], SingleArgF, [0]), ], } -call_parser = Parser(CALL_GRAMMAR) +call_parser = Parser(CALL_GRAMMAR, cache_dir=None) def num_f(v: int, o: int = 0) -> NUM_F: @@ -859,7 +862,7 @@ def __init__(self, stmts: StmtsE) -> None: ([], EmptyStmtsE, []), ], } -program_parser = Parser(PROGRAM_GRAMMAR) +program_parser = Parser(PROGRAM_GRAMMAR, cache_dir=None) def test_empty_program() -> None: diff --git a/tests/test_integration.py b/tests/test_integration.py index 77dcc3a..0b5e887 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -365,7 +365,8 @@ def lex_word(matched: str, state: None, lineno: int, offset: int) -> Token: ([IF, "expr", THEN, "expr", ELSE, "expr"], IfExpr, [1, 3, 5]), ([LET, ID, EQ, "expr", IN, "expr"], LetExpr, [1, 3, 5]), ] - } + }, + cache_dir=None, ) # --------------------------------------------------------------------------- diff --git a/tests/test_parser.py b/tests/test_parser.py index e717c14..b8b7cd2 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -50,7 +50,8 @@ def make_positive_integer_parser() -> Parser[Tree]: "num": [ ([NUM], Num, [0]), ], - } + }, + cache_dir=None, ) @@ -62,7 +63,7 @@ def test_parse_positive_integer_without_add(): def test_minimal_empty_rule_parser(): - parser = Parser({"pgm": [([], list[int], [])]}) + parser = Parser({"pgm": [([], list[int], [])]}, cache_dir=None) parsed = parser.parse("pgm", []) assert isinstance(parsed, list) assert len(parsed) == 0 @@ -102,7 +103,8 @@ def make_list_parser() -> Parser[IntList]: ([NUM, COMMA, "items"], IntList, [0, 1]), ([], EmptyIntList, []), ], - } + }, + cache_dir=None, ) diff --git a/tests/test_parser_cache.py b/tests/test_parser_cache.py new file mode 100644 index 0000000..abfe132 --- /dev/null +++ b/tests/test_parser_cache.py @@ -0,0 +1,377 @@ +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Barrier +from typing import Any + +import pytest + +import plare +import plare.parser as parser_module +from plare.parser import Parser +from plare.token import Token + + +class NUM_C(Token): + def __init__(self, value: str, *, lineno: int, offset: int) -> None: + super().__init__(value, lineno=lineno, offset=offset) + self.value = int(value) + + +class PLUS_C(Token): + precedence = 1 + associative = "left" + + +class PREC_C(Token): + precedence = 1 + + +class NumC: + def __init__(self, token: NUM_C) -> None: + self.value = token.value + + +class AddC: + def __init__(self, left: Any, right: Any) -> None: + self.left = left + self.right = right + + +def expression_grammar() -> dict[ + str, + list[tuple[list[type[Token] | str], type[Any] | None, list[int]]], +]: + return { + "expr": [ + (["expr", PLUS_C, "expr"], AddC, [0, 2]), + ([NUM_C], NumC, [0]), + ] + } + + +def expression_tokens() -> list[Token]: + return [ + NUM_C("1", lineno=1, offset=0), + PLUS_C("+", lineno=1, offset=1), + NUM_C("2", lineno=1, offset=2), + PLUS_C("+", lineno=1, offset=3), + NUM_C("3", lineno=1, offset=4), + ] + + +def assert_expression_result(result: object, *, left_associative: bool) -> None: + assert isinstance(result, AddC) + branch = result.left if left_associative else result.right + assert isinstance(branch, AddC) + + +def fail_if_table_is_built(*args: object, **kwargs: object) -> None: + raise AssertionError("parse table should have been loaded from cache") + + +def only_cache_file(cache_dir: Path) -> Path: + cache_files = sorted(cache_dir.glob("*.json")) + assert len(cache_files) == 1 + return cache_files[0] + + +def test_cache_version_follows_plare_version( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "version" + Parser(expression_grammar(), cache_dir=cache_dir) + cache_file = only_cache_file(cache_dir) + payload = json.loads(cache_file.read_text(encoding="utf-8")) + + assert "PARSER_TABLE_CACHE_VERSION" not in vars(parser_module) + assert payload["version"] == plare.__version__ + + monkeypatch.setattr(plare, "__version__", "999.0.0") + rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) + rewritten = json.loads(cache_file.read_text(encoding="utf-8")) + + assert rebuilt.cache_hit is False + assert only_cache_file(cache_dir) == cache_file + assert rewritten["version"] == "999.0.0" + + +def test_default_cache_directory_is_project_root_plare( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + + first = Parser(expression_grammar()) + + cache_dir = tmp_path / ".plare" + assert first.cache_hit is False + assert only_cache_file(cache_dir).is_file() + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + cached = Parser(expression_grammar()) + + assert cached.cache_hit is True + + +def test_default_cache_keeps_multiple_grammars( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + first_grammar = {"value": [([NUM_C], NumC, [0])]} + second_grammar = { + "value": [([PLUS_C, NUM_C], NumC, [1])], + } + + Parser(first_grammar) + Parser(second_grammar) + + cache_dir = tmp_path / ".plare" + assert len(list(cache_dir.glob("*.json"))) == 2 + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + assert Parser(first_grammar).cache_hit is True + assert Parser(second_grammar).cache_hit is True + + +def test_cache_directory_none_disables_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + + parser = Parser(expression_grammar(), cache_dir=None) + + assert parser.cache_hit is False + assert not (tmp_path / ".plare").exists() + + +def test_parser_module_has_no_private_top_level_names() -> None: + private_names = sorted( + name + for name in vars(parser_module) + if name.startswith("_") and not name.startswith("__") + ) + assert private_names == [] + + +def test_cache_miss_writes_file_and_hit_skips_table_build( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "nested" / "cache" + + first = Parser(expression_grammar(), cache_dir=str(cache_dir)) + + assert first.cache_hit is False + assert only_cache_file(cache_dir).is_file() + assert_expression_result( + first.parse("expr", expression_tokens()), left_associative=True + ) + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + second = Parser(expression_grammar(), cache_dir=cache_dir) + + assert second.cache_hit is True + assert second.entry_state == first.entry_state + assert_expression_result( + second.parse("expr", expression_tokens()), left_associative=True + ) + + +def test_cache_rebinds_current_local_semantic_action( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class First: + def __init__(self, left: NUM_C, right: NUM_C) -> None: + self.values = (left.value, right.value) + + class Second: + def __init__(self, right: NUM_C, left: NUM_C) -> None: + self.values = (right.value, left.value) + + cache_dir = tmp_path / "local-action" + Parser( + {"value": [([NUM_C, PLUS_C, NUM_C], First, [0, 2])]}, + cache_dir=cache_dir, + ) + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + + cached = Parser( + {"value": [([NUM_C, PLUS_C, NUM_C], Second, [2, 0])]}, + cache_dir=cache_dir, + ) + result = cached.parse( + "value", + [ + NUM_C("4", lineno=1, offset=0), + PLUS_C("+", lineno=1, offset=1), + NUM_C("7", lineno=1, offset=2), + ], + ) + + assert cached.cache_hit is True + assert isinstance(result, Second) + assert result.values == (7, 4) + + +def test_cache_preserves_non_alphabetical_entry_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + grammar = { + "z_value": [([NUM_C], NumC, [0])], + "a_value": [([NUM_C], NumC, [0])], + } + cache_dir = tmp_path / "entry-order" + first = Parser(grammar, cache_dir=cache_dir) + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + + cached = Parser(grammar, cache_dir=cache_dir) + + assert cached.cache_hit is True + assert ( + list(cached.entry_state) + == list(first.entry_state) + == [ + "z_value", + "a_value", + ] + ) + + +def test_token_conflict_metadata_change_invalidates_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "associativity" + first = Parser(expression_grammar(), cache_dir=cache_dir) + assert_expression_result( + first.parse("expr", expression_tokens()), left_associative=True + ) + + monkeypatch.setattr(PLUS_C, "associative", "right") + second = Parser(expression_grammar(), cache_dir=cache_dir) + + assert second.cache_hit is False + assert len(list(cache_dir.glob("*.json"))) == 2 + assert_expression_result( + second.parse("expr", expression_tokens()), left_associative=False + ) + + +def test_grammar_structure_change_invalidates_cache(tmp_path: Path) -> None: + cache_dir = tmp_path / "grammar" + Parser({"value": [([NUM_C], NumC, [0])]}, cache_dir=cache_dir) + + changed = Parser( + {"value": [([PLUS_C, NUM_C], NumC, [1])]}, + cache_dir=cache_dir, + ) + result = changed.parse( + "value", + [ + PLUS_C("+", lineno=1, offset=0), + NUM_C("9", lineno=1, offset=1), + ], + ) + + assert changed.cache_hit is False + assert len(list(cache_dir.glob("*.json"))) == 2 + assert isinstance(result, NumC) + assert result.value == 9 + + +def test_prec_token_change_invalidates_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "prec" + grammar = {"value": [([NUM_C], NumC, [0], PREC_C)]} + Parser(grammar, cache_dir=cache_dir) + + monkeypatch.setattr(PREC_C, "precedence", 2) + changed = Parser(grammar, cache_dir=cache_dir) + + assert changed.cache_hit is False + assert len(list(cache_dir.glob("*.json"))) == 2 + + +@pytest.mark.parametrize("contents", ["", "{", "[]", '{"version": 1}']) +def test_corrupt_cache_is_rebuilt( + contents: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "corrupt" + Parser(expression_grammar(), cache_dir=cache_dir) + cache_file = only_cache_file(cache_dir) + cache_file.write_text(contents, encoding="utf-8") + + rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) + + assert rebuilt.cache_hit is False + assert_expression_result( + rebuilt.parse("expr", expression_tokens()), left_associative=True + ) + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + cached = Parser(expression_grammar(), cache_dir=cache_dir) + assert cached.cache_hit is True + + +def test_deeply_nested_cache_is_rebuilt(tmp_path: Path) -> None: + cache_dir = tmp_path / "deeply-nested" + Parser(expression_grammar(), cache_dir=cache_dir) + cache_file = only_cache_file(cache_dir) + cache_file.write_text("[" * 10_000 + "]" * 10_000, encoding="utf-8") + + rebuilt = Parser(expression_grammar(), cache_dir=cache_dir) + + assert rebuilt.cache_hit is False + assert_expression_result( + rebuilt.parse("expr", expression_tokens()), left_associative=True + ) + + +def test_cache_io_failure_does_not_break_parser(tmp_path: Path) -> None: + cache_dir = tmp_path / "cache-is-a-file" + cache_dir.write_text("not a directory", encoding="utf-8") + + parser = Parser(expression_grammar(), cache_dir=cache_dir) + + assert parser.cache_hit is False + assert_expression_result( + parser.parse("expr", expression_tokens()), left_associative=True + ) + assert cache_dir.is_file() + + +def test_concurrent_cache_initialization_is_atomic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cache_dir = tmp_path / "concurrent" + barrier = Barrier(4) + + def construct_and_parse(task_index: int) -> object: + barrier.wait() + parser = Parser(expression_grammar(), cache_dir=cache_dir) + return parser.parse("expr", expression_tokens()) + + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(construct_and_parse, range(8))) + + for result in results: + assert_expression_result(result, left_associative=True) + + monkeypatch.setattr( + parser_module, "compute_lalr1_lookaheads", fail_if_table_is_built + ) + cached = Parser(expression_grammar(), cache_dir=cache_dir) + assert cached.cache_hit is True diff --git a/tests/test_parser_safety_net.py b/tests/test_parser_safety_net.py index 5e45754..64fa93d 100644 --- a/tests/test_parser_safety_net.py +++ b/tests/test_parser_safety_net.py @@ -69,7 +69,7 @@ def test_epsilon_optional_suffix() -> None: ([], NoLabel, []), ], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) result_no = p.parse( "stmt", @@ -138,7 +138,7 @@ def test_left_recursive_addition_chain() -> None: ], "num": [([NUM2], Num2, [0])], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) result = p.parse( "expr", @@ -204,7 +204,7 @@ def test_right_recursive_cons_list() -> None: ], "num": [([NUM3], Num3, [0])], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) result = p.parse( "list", @@ -284,7 +284,8 @@ def test_operator_precedence_mul_over_add() -> None: (["expr", STAR4, "expr"], Mul4, [0, 2]), ([NUM4], Num4, [0]), ] - } + }, + cache_dir=None, ) # 1 + 2 * 3 → Add4(Num4(1), Mul4(Num4(2), Num4(3))) @@ -352,7 +353,8 @@ def test_left_associative_subtraction() -> None: (["expr", MINUS5, "expr"], Sub5, [0, 2]), ([NUM5], Num5, [0]), ] - } + }, + cache_dir=None, ) # 1 - 2 - 3 → Sub5(Sub5(Num5(1), Num5(2)), Num5(3)) @@ -420,7 +422,8 @@ def test_right_associative_exponentiation() -> None: (["expr", POW6, "expr"], Pow6, [0, 2]), ([NUM6], Num6, [0]), ] - } + }, + cache_dir=None, ) # 2 ^ 3 ^ 4 → Pow6(Num6(2), Pow6(Num6(3), Num6(4))) @@ -484,7 +487,7 @@ def test_multiple_entry_points() -> None: "str_expr": [([WORD7], StrVal, [0])], "int_expr": [([NUM7], IntVal, [0])], } - p = Parser(grammar) + p = Parser(grammar, cache_dir=None) str_result = p.parse("str_expr", [WORD7("hello", lineno=1, offset=0)]) assert isinstance(str_result, StrVal) @@ -543,7 +546,8 @@ def test_shift_reduce_resolution_prefers_shift() -> None: (["expr", PLUS8, "expr"], Add8, [0, 2]), ([NUM8], Num8, [0]), ] - } + }, + cache_dir=None, ) result = p.parse( @@ -622,7 +626,8 @@ def test_lalr1_resolves_rr_conflict_variant_1() -> None: ], "A_nt": [([E8x], None, [0])], "B_nt": [([E8x], None, [0])], - } + }, + cache_dir=None, ) result = p.parse( "start", @@ -682,7 +687,8 @@ def test_lalr1_resolves_rr_conflict_variant_2() -> None: ], "X_nt": [([T8y], None, [0])], "Y_nt": [([T8y], None, [0])], - } + }, + cache_dir=None, ) result = p.parse( "start2", diff --git a/tests/test_prec_override.py b/tests/test_prec_override.py index cc115e8..b2361c3 100644 --- a/tests/test_prec_override.py +++ b/tests/test_prec_override.py @@ -89,7 +89,8 @@ def parser_with_prec_override() -> Parser[Expr]: (["expr", STAR, "expr"], Mul, [0, 2]), ([MINUS, "expr"], Neg, [1], UMINUS), ] - } + }, + cache_dir=None, ) @@ -102,7 +103,8 @@ def parser_without_prec_override() -> Parser[Expr]: (["expr", STAR, "expr"], Mul, [0, 2]), ([MINUS, "expr"], Neg, [1]), ] - } + }, + cache_dir=None, ) @@ -199,7 +201,7 @@ def test_rr_equal_precedence_first_defined_wins() -> None: "first_val": [([Lit], None, [0])], "second_val": [([Lit], None, [0])], } - parser = Parser(grammar) + parser = Parser(grammar, cache_dir=None) result = parser.parse("result", [Lit("x", lineno=1, offset=0)]) assert isinstance( result, FirstResult