diff --git a/scripts/build_observability_fixture.py b/scripts/build_observability_fixture.py index 9885dd62..bedd88f6 100644 --- a/scripts/build_observability_fixture.py +++ b/scripts/build_observability_fixture.py @@ -16,7 +16,7 @@ SEED, GENERATED_AT = 20260913, datetime(2026, 9, 13, 12, tzinfo=UTC) FIXED_MTIME = int(GENERATED_AT.timestamp()) -HELDOUT_SHA256, MANIFEST = "7c14c1457dd3702b0980e8a755b4aadec59a2a62d99091cd3950d94682296ea9", Path(__file__).resolve().parents[1] / "tests/fixtures/observability/cases.json" # fmt: skip +HELDOUT_SHA256, MANIFEST = "a945fed90d3b943a994e952f7b041b6745991d68bf6e2d16de66ad7aa5475228", Path(__file__).resolve().parents[1] / "tests/fixtures/observability/cases.json" # fmt: skip @dataclass(frozen=True) @@ -27,6 +27,8 @@ class CaseDefinition: @property def split(self) -> str: + if self.case_id == "legacy-no-op-dev": + return "dev" return "heldout" if hashlib.sha256(self.case_id.encode()).digest()[0] < 0x60 else "dev" @property @@ -34,6 +36,10 @@ def db_path(self) -> str: return f"db/{self.case_id}.sqlite" +def _mtime_iso(*, future: bool = False) -> str: + return datetime.fromtimestamp(FIXED_MTIME + (4 * 3600 if future else 0), tz=UTC).isoformat().replace("+00:00", "Z") + + def case_definitions() -> list[CaseDefinition]: document = json.loads(MANIFEST.read_text(encoding="utf-8")) result = [CaseDefinition(case["case_id"], case["failure"], case["profile"]) for case in document["cases"]] @@ -60,8 +66,9 @@ def _rows() -> list[tuple[object, ...]]: # fmt: off rows.append((f"synthetic-{index:02d}", f"Synthetic observability fixture row {index:02d}", "{}", source_file, "brainlayer-fixture", "assistant_text", source, sender, created, - provenance[index % len(provenance)], source_classes[index % len(source_classes)], - ("knowledge", "decision", "operational", "noise")[index % 4], + (None if index in {5, 17} else provenance[index % len(provenance)]), + source_classes[index % len(source_classes)], + ("knowledge", "decision", "operational", "test")[index % 4], "synthetic-00" if index == 21 else None, GENERATED_AT.isoformat().replace("+00:00", "Z") if index == 20 else None)) # fmt: on @@ -97,8 +104,11 @@ def _build_db(path: Path, case: CaseDefinition, pid_root: Path) -> None: def _jsonl_log(profile: str) -> str: - if profile == "no_op": - return json.dumps({"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": False, "verified": True}) + "\n" # fmt: skip + if profile in {"no_op", "legacy_no_op"}: + receipt = {"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": False, "verified": True} + if profile == "no_op": + receipt["attempted_at"] = "2026-09-13T10:00:00Z" + return json.dumps(receipt) + "\n" # fmt: off shapes = [ {"status": "uploaded", "archive": "claude-jsonl-2026-09-10.tar.gz", "uploaded": True, @@ -132,8 +142,8 @@ def _daily_log(profile: str) -> str: def build_fixture_bundle(root: Path, *, seed: int) -> None: if seed != SEED: raise ValueError(f"seed must be {SEED}") - manifest_text = MANIFEST.read_text(encoding="utf-8") - if json.loads(manifest_text)["heldout_goldens_sha256"] != HELDOUT_SHA256: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + if manifest["heldout_goldens_sha256"] != HELDOUT_SHA256: raise ValueError("cases.json held-out digest does not match the sealed digest") root.mkdir(parents=True, exist_ok=True) for generated_dir in ("db", "logs", "launchd", ".writer-pids"): @@ -161,7 +171,13 @@ def build_fixture_bundle(root: Path, *, seed: int) -> None: db_path.with_name(db_path.name + suffix).unlink(missing_ok=True) os.utime(db_path, (FIXED_MTIME, FIXED_MTIME)) shutil.rmtree(root / ".writer-pids", ignore_errors=True) - _write(root / "cases.json", manifest_text) + for case in manifest["cases"]: + future = case["failure"] == "clock_skew" + case["input_mtimes"] = { + path: _mtime_iso(future=future and path.endswith("jsonl-backup.log")) + for path in case["declared_inputs"] + } + _write(root / "cases.json", json.dumps(manifest, indent=2) + "\n") def _build_case(root: Path, case: CaseDefinition) -> None: @@ -174,12 +190,14 @@ def _build_case(root: Path, case: CaseDefinition) -> None: launchd = root / "launchd" / f"{case.case_id}.txt" if case.failure == "missing_launchd": _write(launchd, "") - elif case.profile == "no_op": + elif case.profile in {"no_op", "legacy_no_op"}: _write(launchd, 'Bad request.\nCould not find service "com.brainlayer.jsonl-backup" in domain for user gui: 501\n') # fmt: skip else: _write(launchd, "gui/501/com.brainlayer.jsonl-backup = {\n\tstate = running\n\truns = 6\n\tpid = 4242\n\tlast exit code = 0\n}\n") # fmt: skip if case.profile == "healthy": - _write(root / "launchd" / f"{case.case_id}.disabled/com.brainlayer.jsonl-backup.plist", "synthetic disabled fixture\n") # fmt: skip + disabled_dir = root / "launchd" / f"{case.case_id}.disabled" + _write(disabled_dir / "com.brainlayer.jsonl-backup.plist", "synthetic disabled fixture\n") # fmt: skip + os.utime(disabled_dir, (FIXED_MTIME, FIXED_MTIME)) def main() -> int: diff --git a/scripts/derive_observability_goldens.py b/scripts/derive_observability_goldens.py new file mode 100644 index 00000000..f690bffd --- /dev/null +++ b/scripts/derive_observability_goldens.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Derive observability DB sections directly from built fixture SQLite files. + +This is deliberately independent from the observability producer. It is a +plain-SQL oracle for reviewing the committed fixture artifacts. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sqlite3 +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +DERIVATION_SQL = Path(__file__).resolve().parents[1] / "tests/fixtures/observability/derivation.sql" + + +def _iso(value: datetime) -> str: + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _input(case: dict[str, Any], root: Path) -> dict[str, Any]: + path = case["inputs"]["db"] + db = root / path + with sqlite3.connect(db) as connection: + count = connection.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] + return { + "mtime": case["input_mtimes"][path], + "path": path, + "rows_or_bytes": count, + "sha256_first_64kb": None, + "skipped_lines": 0, + "status": "read", + } + + +def _columns(connection: sqlite3.Connection) -> set[str]: + return {row[1] for row in connection.execute("PRAGMA table_info(chunks)")} + + +def _preview(value: str) -> str: + return re.sub(r"\s+", " ", value).strip()[:80] + + +def _emitter(row: sqlite3.Row) -> tuple[str, str]: + if row["source"]: + return row["source"], "source" + if row["sender"]: + return row["sender"], "sender" + return row["source_file"].rsplit("/", 1)[-1], "source_file" + + +def _db_sections(case: dict[str, Any], root: Path, template: dict[str, Any]) -> dict[str, Any]: + path = root / case["inputs"]["db"] + connection = sqlite3.connect(path) + connection.row_factory = sqlite3.Row + try: + inputs = [_input(case, root)] + if "source_class" not in _columns(connection): + reason = "required column missing: chunks.source_class" + return { + name: {"inputs": inputs, "reason": reason, "state": "unmeasurable"} + for name in ("stores", "emitters", "author_unknown") + } + generated = datetime.fromisoformat(case["generated_at"].replace("Z", "+00:00")) + window_start = generated - timedelta(hours=case["window_hours"]) + rows = connection.execute( + "SELECT id, content, source_file, source, sender, created_at, provenance_class, " + "source_class, content_class, archived_at, superseded_by FROM chunks" + ).fetchall() + window = [row for row in rows if window_start <= datetime.fromisoformat(row["created_at"].replace("Z", "+00:00")) <= generated] + by_class = connection.execute( + "SELECT COALESCE(content_class, 'knowledge') AS content_class, COUNT(*) AS count " + "FROM chunks GROUP BY COALESCE(content_class, 'knowledge') ORDER BY content_class" + ).fetchall() + by_hour = connection.execute( + "SELECT substr(created_at, 1, 13) || ':00:00Z' AS hour, COUNT(*) AS count FROM chunks " + "WHERE created_at >= ? AND created_at <= ? GROUP BY hour ORDER BY hour", + (_iso(window_start), _iso(generated)), + ).fetchall() + by_source_class = connection.execute( + "SELECT source_class, COUNT(*) AS count, SUM(CASE WHEN created_at >= ? AND created_at <= ? THEN 1 ELSE 0 END) AS in_window " + "FROM chunks GROUP BY source_class ORDER BY source_class IS NOT NULL DESC, source_class", + (_iso(window_start), _iso(generated)), + ).fetchall() + by_emitter: dict[tuple[str, str], int] = {} + for row in window: + key = _emitter(row) + by_emitter[key] = by_emitter.get(key, 0) + 1 + latest = [] + for row in sorted(window, key=lambda item: item["created_at"], reverse=True)[:5]: + emitter, _ = _emitter(row) + latest.append({"chunk_id": row["id"], "emitter": emitter, "preview": _preview(row["content"]), "source_class": row["source_class"], "stored_at": row["created_at"]}) + live_count = connection.execute("SELECT COUNT(*) FROM chunks WHERE archived_at IS NULL").fetchone()[0] + unknown_count = connection.execute("SELECT COUNT(*) FROM chunks WHERE provenance_class = 'unknown' AND archived_at IS NULL").fetchone()[0] + never_count = connection.execute("SELECT COUNT(*) FROM chunks WHERE (provenance_class IS NULL OR source_class IS NULL) AND archived_at IS NULL").fetchone()[0] + top_files = connection.execute( + "SELECT source_file, COUNT(*) AS count FROM chunks WHERE archived_at IS NULL AND " + "(provenance_class IS NULL OR source_class IS NULL OR provenance_class = 'unknown') " + "GROUP BY source_file ORDER BY count DESC, source_file LIMIT 2" + ).fetchall() + trend = [] + for offset in range(6, -1, -1): + day = (generated - timedelta(days=offset)).date().isoformat() + trend.append({ + "classified_unknown": connection.execute("SELECT COUNT(*) FROM chunks WHERE provenance_class = 'unknown' AND archived_at IS NULL AND substr(created_at, 1, 10) = ?", (day,)).fetchone()[0], + "day": day, + "never_classified": connection.execute("SELECT COUNT(*) FROM chunks WHERE (provenance_class IS NULL OR source_class IS NULL) AND archived_at IS NULL AND substr(created_at, 1, 10) = ?", (day,)).fetchone()[0], + }) + stores = { + "by_content_class": [{"content_class": row["content_class"], "count": row["count"]} for row in by_class], + "in_window": {"by_hour": [{"count": row["count"], "hour": row["hour"]} for row in by_hour], "count": len(window)}, + "inputs": inputs, + "latest": latest, + "reason": "", + "state": "measured", + "total_chunks": len(rows), + } + emitters = { + "by_emitter": [{"count_in_window": count, "derived_from": source, "emitter": emitter} for (emitter, source), count in sorted(by_emitter.items())], + "by_source_class": [{"count": row["count"], "in_window": row["in_window"], "source_class": row["source_class"]} for row in by_source_class], + "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", + "hidden_from_default_search": connection.execute("SELECT COUNT(*) FROM chunks WHERE source_class IN ('desktop', 'brain-worker')").fetchone()[0], + "inputs": inputs, + "reason": "", + "state": "measured", + } + author_unknown = { + "census_2026_09_13": template["author_unknown"]["census_2026_09_13"], + "classified_unknown": {"count": unknown_count, "definition": "provenance_class = 'unknown', archived_at IS NULL", "share": round(unknown_count / live_count, 6) if live_count else 0.0}, + "inputs": inputs, + "never_classified": {"count": never_count, "definition": "provenance_class IS NULL OR source_class IS NULL, archived_at IS NULL", "share": round(never_count / live_count, 6) if live_count else 0.0}, + "reason": "", + "state": "measured", + "top_source_files": [{"count": row["count"], "source_file": row["source_file"]} for row in top_files], + "trend_7d": trend, + } + return {"stores": stores, "emitters": emitters, "author_unknown": author_unknown} + finally: + connection.close() + + +def derive(root: Path, golden_root: Path) -> None: + manifest = json.loads((root / "cases.json").read_text()) + for case in manifest["cases"]: + path = golden_root / case["golden"] + if not path.exists(): + continue + golden = json.loads(path.read_text()) + golden.update(_db_sections(case, root, golden)) + path.write_text(json.dumps(golden, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--fixture-root", type=Path, required=True) + parser.add_argument("--golden-root", type=Path, required=True) + args = parser.parse_args() + derive(args.fixture_root.resolve(), args.golden_root.resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/observability_eval.py b/scripts/observability_eval.py index 7d47fbad..e90346ef 100644 --- a/scripts/observability_eval.py +++ b/scripts/observability_eval.py @@ -6,11 +6,13 @@ import io import json import os +import shutil import subprocess import sys import tarfile import tempfile from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Any @@ -97,7 +99,38 @@ def grade_payload( *(f"undeclared opened input: {item}" for item in sorted(opened - declared)), ] return Grade(case["case_id"], _field_diff(expected, actual), mock_green, traceability) -def _run_case(case: dict[str, Any], root: Path, producer_root: Path, golden_root: Path | None) -> Grade: + + +def _stage_case_inputs(case: dict[str, Any], source_root: Path, staged_root: Path) -> None: + source_root, staged_root = source_root.resolve(), staged_root.resolve() + declared = case["declared_inputs"] + mtimes = case.get("input_mtimes", {}) + missing = sorted(set(declared) - set(mtimes)) + if missing: + raise ValueError(f"missing input_mtimes for declared inputs: {', '.join(missing)}") + unexpected = sorted(set(mtimes) - set(declared)) + if unexpected: + raise ValueError(f"input_mtimes contains undeclared inputs: {', '.join(unexpected)}") + for relative in declared: + source = (source_root / relative).resolve() + target = (staged_root / relative).resolve() + if source_root not in source.parents and source != source_root: + raise ValueError(f"declared input escapes fixture root: {relative}") + if staged_root not in target.parents and target != staged_root: + raise ValueError(f"declared input escapes staging root: {relative}") + if source.is_dir(): + shutil.copytree(source, target) + elif source.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + if target.exists(): + timestamp = datetime.fromisoformat(mtimes[relative].replace("Z", "+00:00")).timestamp() + os.utime(target, (timestamp, timestamp)) + + +def _run_case( + case: dict[str, Any], root: Path, producer_root: Path, golden_root: Path | None, *, stage_inputs: bool = True +) -> Grade: root, producer_root = root.resolve(), producer_root.resolve() golden_root = golden_root.resolve() if golden_root is not None else None try: @@ -106,14 +139,22 @@ def _run_case(case: dict[str, Any], root: Path, producer_root: Path, golden_root return Grade(case["case_id"], [f"$: golden unavailable: {exc}"], [], []) with tempfile.TemporaryDirectory(prefix="observability-eval-") as temp: output, trace = Path(temp) / "observability.json", Path(temp) / "inputs.json" + input_root = root + if stage_inputs: + input_root = Path(temp) / "inputs" + input_root.mkdir() + try: + _stage_case_inputs(case, root, input_root) + except (OSError, ValueError) as exc: + return Grade(case["case_id"], [f"$: input staging failed: {exc}"], [], []) env = {key: os.environ[key] for key in ("HOME", "PATH") if key in os.environ} env.update({ - "BRAINLAYER_DB": str(root / case["inputs"]["db"]), "BRAINLAYER_OBSERVABILITY_PATH": str(output), - "BRAINLAYER_OBSERVABILITY_TRACE_PATH": str(trace), "BRAINLAYER_OBSERVABILITY_INPUT_ROOT": str(root), - "BRAINLAYER_OBSERVABILITY_JSONL_BACKUP_LOG": str(root / case["inputs"]["jsonl_backup_log"]), - "BRAINLAYER_OBSERVABILITY_BACKUP_DAILY_LOG": str(root / case["inputs"]["backup_daily_log"]), - "BRAINLAYER_OBSERVABILITY_LAUNCHD_OUTPUT": str(root / case["inputs"]["launchd_output"]), - "BRAINLAYER_OBSERVABILITY_DISABLED_DIR": str(root / case["inputs"]["disabled_dir"]), + "BRAINLAYER_DB": str(input_root / case["inputs"]["db"]), "BRAINLAYER_OBSERVABILITY_PATH": str(output), + "BRAINLAYER_OBSERVABILITY_TRACE_PATH": str(trace), "BRAINLAYER_OBSERVABILITY_INPUT_ROOT": str(input_root), + "BRAINLAYER_OBSERVABILITY_JSONL_BACKUP_LOG": str(input_root / case["inputs"]["jsonl_backup_log"]), + "BRAINLAYER_OBSERVABILITY_BACKUP_DAILY_LOG": str(input_root / case["inputs"]["backup_daily_log"]), + "BRAINLAYER_OBSERVABILITY_LAUNCHD_OUTPUT": str(input_root / case["inputs"]["launchd_output"]), + "BRAINLAYER_OBSERVABILITY_DISABLED_DIR": str(input_root / case["inputs"]["disabled_dir"]), "BRAINLAYER_OBSERVABILITY_NOW": case["generated_at"], "PYTHONPATH": str(producer_root / "src"), }) try: diff --git a/tests/fixtures/observability/cases.json b/tests/fixtures/observability/cases.json index 35618e7e..ab7f9224 100644 --- a/tests/fixtures/observability/cases.json +++ b/tests/fixtures/observability/cases.json @@ -23,7 +23,14 @@ "profile": "healthy", "split": "dev", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/healthy-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/healthy-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/healthy-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/healthy-dev.txt": "2026-09-13T12:00:00Z", + "launchd/healthy-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "missing-source-class-dev", @@ -52,7 +59,14 @@ "emitters", "author_unknown" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/missing-source-class-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/missing-source-class-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/missing-source-class-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/missing-source-class-dev.txt": "2026-09-13T12:00:00Z", + "launchd/missing-source-class-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "missing-log-dev", @@ -79,7 +93,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/missing-log-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/missing-log-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/missing-log-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/missing-log-dev.txt": "2026-09-13T12:00:00Z", + "launchd/missing-log-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "malformed-log-dev-1", @@ -106,7 +127,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/malformed-log-dev-1.sqlite": "2026-09-13T12:00:00Z", + "logs/malformed-log-dev-1/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/malformed-log-dev-1/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/malformed-log-dev-1.txt": "2026-09-13T12:00:00Z", + "launchd/malformed-log-dev-1.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "missing-launchd-dev", @@ -133,7 +161,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/missing-launchd-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/missing-launchd-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/missing-launchd-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/missing-launchd-dev.txt": "2026-09-13T12:00:00Z", + "launchd/missing-launchd-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "empty-db-dev", @@ -158,7 +193,14 @@ "profile": "healthy", "split": "dev", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/empty-db-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/empty-db-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/empty-db-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/empty-db-dev.txt": "2026-09-13T12:00:00Z", + "launchd/empty-db-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "clock-skew-dev", @@ -185,7 +227,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/clock-skew-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/clock-skew-dev/jsonl-backup.log": "2026-09-13T16:00:00Z", + "logs/clock-skew-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/clock-skew-dev.txt": "2026-09-13T12:00:00Z", + "launchd/clock-skew-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "no-op-dev", @@ -210,7 +259,46 @@ "profile": "no_op", "split": "dev", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/no-op-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/no-op-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/no-op-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/no-op-dev.txt": "2026-09-13T12:00:00Z", + "launchd/no-op-dev.disabled": "2026-09-13T12:00:00Z" + } + }, + { + "case_id": "legacy-no-op-dev", + "db": "db/legacy-no-op-dev.sqlite", + "declared_inputs": [ + "db/legacy-no-op-dev.sqlite", + "logs/legacy-no-op-dev/jsonl-backup.log", + "logs/legacy-no-op-dev/backup-daily.log", + "launchd/legacy-no-op-dev.txt", + "launchd/legacy-no-op-dev.disabled" + ], + "failure": "none", + "generated_at": "2026-09-13T12:00:00Z", + "golden": "golden/legacy-no-op-dev.json", + "inputs": { + "backup_daily_log": "logs/legacy-no-op-dev/backup-daily.log", + "db": "db/legacy-no-op-dev.sqlite", + "disabled_dir": "launchd/legacy-no-op-dev.disabled", + "jsonl_backup_log": "logs/legacy-no-op-dev/jsonl-backup.log", + "launchd_output": "launchd/legacy-no-op-dev.txt" + }, + "profile": "legacy_no_op", + "split": "dev", + "unmeasurable_sections": [], + "window_hours": 24, + "input_mtimes": { + "db/legacy-no-op-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/legacy-no-op-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/legacy-no-op-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/legacy-no-op-dev.txt": "2026-09-13T12:00:00Z", + "launchd/legacy-no-op-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "backup-errors-dev", @@ -235,7 +323,14 @@ "profile": "errors", "split": "dev", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/backup-errors-dev.sqlite": "2026-09-13T12:00:00Z", + "logs/backup-errors-dev/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/backup-errors-dev/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/backup-errors-dev.txt": "2026-09-13T12:00:00Z", + "launchd/backup-errors-dev.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "healthy-heldout-3", @@ -260,7 +355,14 @@ "profile": "healthy", "split": "heldout", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/healthy-heldout-3.sqlite": "2026-09-13T12:00:00Z", + "logs/healthy-heldout-3/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/healthy-heldout-3/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/healthy-heldout-3.txt": "2026-09-13T12:00:00Z", + "launchd/healthy-heldout-3.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "missing-source-class-heldout", @@ -289,7 +391,14 @@ "emitters", "author_unknown" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/missing-source-class-heldout.sqlite": "2026-09-13T12:00:00Z", + "logs/missing-source-class-heldout/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/missing-source-class-heldout/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/missing-source-class-heldout.txt": "2026-09-13T12:00:00Z", + "launchd/missing-source-class-heldout.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "missing-log-heldout-2", @@ -316,7 +425,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/missing-log-heldout-2.sqlite": "2026-09-13T12:00:00Z", + "logs/missing-log-heldout-2/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/missing-log-heldout-2/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/missing-log-heldout-2.txt": "2026-09-13T12:00:00Z", + "launchd/missing-log-heldout-2.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "malformed-log-heldout", @@ -343,7 +459,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/malformed-log-heldout.sqlite": "2026-09-13T12:00:00Z", + "logs/malformed-log-heldout/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/malformed-log-heldout/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/malformed-log-heldout.txt": "2026-09-13T12:00:00Z", + "launchd/malformed-log-heldout.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "missing-launchd-heldout-2", @@ -370,7 +493,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/missing-launchd-heldout-2.sqlite": "2026-09-13T12:00:00Z", + "logs/missing-launchd-heldout-2/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/missing-launchd-heldout-2/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/missing-launchd-heldout-2.txt": "2026-09-13T12:00:00Z", + "launchd/missing-launchd-heldout-2.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "empty-db-heldout-3", @@ -395,7 +525,14 @@ "profile": "healthy", "split": "heldout", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/empty-db-heldout-3.sqlite": "2026-09-13T12:00:00Z", + "logs/empty-db-heldout-3/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/empty-db-heldout-3/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/empty-db-heldout-3.txt": "2026-09-13T12:00:00Z", + "launchd/empty-db-heldout-3.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "clock-skew-heldout", @@ -422,7 +559,14 @@ "unmeasurable_sections": [ "backups" ], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/clock-skew-heldout.sqlite": "2026-09-13T12:00:00Z", + "logs/clock-skew-heldout/jsonl-backup.log": "2026-09-13T16:00:00Z", + "logs/clock-skew-heldout/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/clock-skew-heldout.txt": "2026-09-13T12:00:00Z", + "launchd/clock-skew-heldout.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "no-op-heldout-2", @@ -447,7 +591,14 @@ "profile": "no_op", "split": "heldout", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/no-op-heldout-2.sqlite": "2026-09-13T12:00:00Z", + "logs/no-op-heldout-2/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/no-op-heldout-2/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/no-op-heldout-2.txt": "2026-09-13T12:00:00Z", + "launchd/no-op-heldout-2.disabled": "2026-09-13T12:00:00Z" + } }, { "case_id": "backup-errors-heldout", @@ -472,10 +623,17 @@ "profile": "errors", "split": "heldout", "unmeasurable_sections": [], - "window_hours": 24 + "window_hours": 24, + "input_mtimes": { + "db/backup-errors-heldout.sqlite": "2026-09-13T12:00:00Z", + "logs/backup-errors-heldout/jsonl-backup.log": "2026-09-13T12:00:00Z", + "logs/backup-errors-heldout/backup-daily.log": "2026-09-13T12:00:00Z", + "launchd/backup-errors-heldout.txt": "2026-09-13T12:00:00Z", + "launchd/backup-errors-heldout.disabled": "2026-09-13T12:00:00Z" + } } ], - "heldout_goldens_sha256": "7c14c1457dd3702b0980e8a755b4aadec59a2a62d99091cd3950d94682296ea9", + "heldout_goldens_sha256": "a945fed90d3b943a994e952f7b041b6745991d68bf6e2d16de66ad7aa5475228", "schema_version": 1, "seed": 20260913 } diff --git a/tests/fixtures/observability/db/backup-errors-dev.sqlite b/tests/fixtures/observability/db/backup-errors-dev.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/backup-errors-dev.sqlite and b/tests/fixtures/observability/db/backup-errors-dev.sqlite differ diff --git a/tests/fixtures/observability/db/backup-errors-heldout.sqlite b/tests/fixtures/observability/db/backup-errors-heldout.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/backup-errors-heldout.sqlite and b/tests/fixtures/observability/db/backup-errors-heldout.sqlite differ diff --git a/tests/fixtures/observability/db/clock-skew-dev.sqlite b/tests/fixtures/observability/db/clock-skew-dev.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/clock-skew-dev.sqlite and b/tests/fixtures/observability/db/clock-skew-dev.sqlite differ diff --git a/tests/fixtures/observability/db/clock-skew-heldout.sqlite b/tests/fixtures/observability/db/clock-skew-heldout.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/clock-skew-heldout.sqlite and b/tests/fixtures/observability/db/clock-skew-heldout.sqlite differ diff --git a/tests/fixtures/observability/db/healthy-dev.sqlite b/tests/fixtures/observability/db/healthy-dev.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/healthy-dev.sqlite and b/tests/fixtures/observability/db/healthy-dev.sqlite differ diff --git a/tests/fixtures/observability/db/healthy-heldout-3.sqlite b/tests/fixtures/observability/db/healthy-heldout-3.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/healthy-heldout-3.sqlite and b/tests/fixtures/observability/db/healthy-heldout-3.sqlite differ diff --git a/tests/fixtures/observability/db/legacy-no-op-dev.sqlite b/tests/fixtures/observability/db/legacy-no-op-dev.sqlite new file mode 100644 index 00000000..0550e610 Binary files /dev/null and b/tests/fixtures/observability/db/legacy-no-op-dev.sqlite differ diff --git a/tests/fixtures/observability/db/malformed-log-dev-1.sqlite b/tests/fixtures/observability/db/malformed-log-dev-1.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/malformed-log-dev-1.sqlite and b/tests/fixtures/observability/db/malformed-log-dev-1.sqlite differ diff --git a/tests/fixtures/observability/db/malformed-log-heldout.sqlite b/tests/fixtures/observability/db/malformed-log-heldout.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/malformed-log-heldout.sqlite and b/tests/fixtures/observability/db/malformed-log-heldout.sqlite differ diff --git a/tests/fixtures/observability/db/missing-launchd-dev.sqlite b/tests/fixtures/observability/db/missing-launchd-dev.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/missing-launchd-dev.sqlite and b/tests/fixtures/observability/db/missing-launchd-dev.sqlite differ diff --git a/tests/fixtures/observability/db/missing-launchd-heldout-2.sqlite b/tests/fixtures/observability/db/missing-launchd-heldout-2.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/missing-launchd-heldout-2.sqlite and b/tests/fixtures/observability/db/missing-launchd-heldout-2.sqlite differ diff --git a/tests/fixtures/observability/db/missing-log-dev.sqlite b/tests/fixtures/observability/db/missing-log-dev.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/missing-log-dev.sqlite and b/tests/fixtures/observability/db/missing-log-dev.sqlite differ diff --git a/tests/fixtures/observability/db/missing-log-heldout-2.sqlite b/tests/fixtures/observability/db/missing-log-heldout-2.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/missing-log-heldout-2.sqlite and b/tests/fixtures/observability/db/missing-log-heldout-2.sqlite differ diff --git a/tests/fixtures/observability/db/missing-source-class-dev.sqlite b/tests/fixtures/observability/db/missing-source-class-dev.sqlite index bd9dd0dd..498ad582 100644 Binary files a/tests/fixtures/observability/db/missing-source-class-dev.sqlite and b/tests/fixtures/observability/db/missing-source-class-dev.sqlite differ diff --git a/tests/fixtures/observability/db/missing-source-class-heldout.sqlite b/tests/fixtures/observability/db/missing-source-class-heldout.sqlite index bd9dd0dd..498ad582 100644 Binary files a/tests/fixtures/observability/db/missing-source-class-heldout.sqlite and b/tests/fixtures/observability/db/missing-source-class-heldout.sqlite differ diff --git a/tests/fixtures/observability/db/no-op-dev.sqlite b/tests/fixtures/observability/db/no-op-dev.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/no-op-dev.sqlite and b/tests/fixtures/observability/db/no-op-dev.sqlite differ diff --git a/tests/fixtures/observability/db/no-op-heldout-2.sqlite b/tests/fixtures/observability/db/no-op-heldout-2.sqlite index 19dd7a1e..0550e610 100644 Binary files a/tests/fixtures/observability/db/no-op-heldout-2.sqlite and b/tests/fixtures/observability/db/no-op-heldout-2.sqlite differ diff --git a/tests/fixtures/observability/derivation.sql b/tests/fixtures/observability/derivation.sql new file mode 100644 index 00000000..8843328f --- /dev/null +++ b/tests/fixtures/observability/derivation.sql @@ -0,0 +1,23 @@ +-- Review aid for scripts/derive_observability_goldens.py. +-- Run against a built case DB with: sqlite3 -readonly db/.sqlite < derivation.sql +SELECT COALESCE(content_class, 'knowledge') AS content_class, COUNT(*) AS count +FROM chunks GROUP BY COALESCE(content_class, 'knowledge') ORDER BY content_class; +SELECT source_class, COUNT(*) AS count +FROM chunks GROUP BY source_class ORDER BY source_class IS NOT NULL DESC, source_class; +SELECT COUNT(*) AS total_chunks FROM chunks; +SELECT COUNT(*) AS never_classified FROM chunks +WHERE (provenance_class IS NULL OR source_class IS NULL) AND archived_at IS NULL; +SELECT COUNT(*) AS classified_unknown FROM chunks +WHERE provenance_class = 'unknown' AND archived_at IS NULL; +SELECT source_file, COUNT(*) AS count FROM chunks +WHERE archived_at IS NULL + AND (provenance_class IS NULL OR source_class IS NULL OR provenance_class = 'unknown') +GROUP BY source_file ORDER BY count DESC, source_file LIMIT 2; + +-- author_unknown.trend_7d: live-only seven-day counts, grouped by UTC day. +SELECT substr(created_at, 1, 10) AS day, + SUM(CASE WHEN provenance_class = 'unknown' AND archived_at IS NULL THEN 1 ELSE 0 END) AS classified_unknown, + SUM(CASE WHEN (provenance_class IS NULL OR source_class IS NULL) AND archived_at IS NULL THEN 1 ELSE 0 END) AS never_classified +FROM chunks +GROUP BY day +ORDER BY day; diff --git a/tests/fixtures/observability/expect-red-da297f5eab46d82dd7ec6a1fb669b123d2c66cbf.log b/tests/fixtures/observability/expect-red-da297f5eab46d82dd7ec6a1fb669b123d2c66cbf.log index 98446889..db5aedf0 100644 --- a/tests/fixtures/observability/expect-red-da297f5eab46d82dd7ec6a1fb669b123d2c66cbf.log +++ b/tests/fixtures/observability/expect-red-da297f5eab46d82dd7ec6a1fb669b123d2c66cbf.log @@ -10,6 +10,7 @@ | empty-db-dev | 1 | 0 | 0 | FAIL | | clock-skew-dev | 1 | 0 | 0 | FAIL | | no-op-dev | 1 | 0 | 0 | FAIL | +| legacy-no-op-dev | 1 | 0 | 0 | FAIL | | backup-errors-dev | 1 | 0 | 0 | FAIL | | healthy-heldout-3 | 1 | 0 | 0 | FAIL | | missing-source-class-heldout | 1 | 0 | 0 | FAIL | @@ -21,9 +22,9 @@ | no-op-heldout-2 | 1 | 0 | 0 | FAIL | | backup-errors-heldout | 1 | 0 | 0 | FAIL | -- Cases: 18 +- Cases: 19 - Passed: 0 -- Field mismatches: 18 +- Field mismatches: 19 - MOCK_GREEN: 0 - Traceability failures: 0 @@ -35,6 +36,7 @@ - `empty-db-dev`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface - `clock-skew-dev`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface - `no-op-dev`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface +- `legacy-no-op-dev`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface - `backup-errors-dev`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface - `healthy-heldout-3`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface - `missing-source-class-heldout`: $: producer failed rc=1: /Library/Frameworks/Python.framework/Versions/3.13/bin/python3: No module named brainlayer.observability_surface diff --git a/tests/fixtures/observability/golden/backup-errors-dev.json b/tests/fixtures/observability/golden/backup-errors-dev.json index 01dc6ed3..6abb0e6e 100644 --- a/tests/fixtures/observability/golden/backup-errors-dev.json +++ b/tests/fixtures/observability/golden/backup-errors-dev.json @@ -115,7 +115,7 @@ "reason": "", "retention_invariant": "PASS", "state": "measured", - "surviving_archives_30d": 7, + "surviving_archives_30d": 1, "threshold_hours": 36 }, "db_path": "db/backup-errors-dev.sqlite", @@ -154,13 +154,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -174,8 +174,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -185,7 +185,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -212,11 +212,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -288,7 +288,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -302,7 +302,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -316,7 +316,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/golden/clock-skew-dev.json b/tests/fixtures/observability/golden/clock-skew-dev.json index 0c387bb0..446b2dd2 100644 --- a/tests/fixtures/observability/golden/clock-skew-dev.json +++ b/tests/fixtures/observability/golden/clock-skew-dev.json @@ -137,13 +137,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -157,8 +157,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -168,7 +168,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -195,11 +195,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -271,7 +271,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -285,7 +285,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -299,7 +299,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/golden/empty-db-dev.json b/tests/fixtures/observability/golden/empty-db-dev.json index 9c41aecb..a47d40d5 100644 --- a/tests/fixtures/observability/golden/empty-db-dev.json +++ b/tests/fixtures/observability/golden/empty-db-dev.json @@ -24,7 +24,43 @@ "reason": "", "state": "measured", "top_source_files": [], - "trend_7d": [] + "trend_7d": [ + { + "classified_unknown": 0, + "day": "2026-09-07", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-08", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-09", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-10", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-11", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-12", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-13", + "never_classified": 0 + } + ] }, "backups": { "db_snapshot": { @@ -74,7 +110,7 @@ "reason": "", "retention_invariant": "PASS", "state": "measured", - "surviving_archives_30d": 7, + "surviving_archives_30d": 1, "threshold_hours": 36 }, "db_path": "db/empty-db-dev.sqlite", diff --git a/tests/fixtures/observability/golden/healthy-dev.json b/tests/fixtures/observability/golden/healthy-dev.json index 76ffb591..2525b661 100644 --- a/tests/fixtures/observability/golden/healthy-dev.json +++ b/tests/fixtures/observability/golden/healthy-dev.json @@ -119,7 +119,7 @@ "reason": "", "retention_invariant": "PASS", "state": "measured", - "surviving_archives_30d": 7, + "surviving_archives_30d": 1, "threshold_hours": 36 }, "db_path": "db/healthy-dev.sqlite", @@ -158,13 +158,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -178,8 +178,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -189,7 +189,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -216,11 +216,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -292,7 +292,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -306,7 +306,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -320,7 +320,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/golden/legacy-no-op-dev.json b/tests/fixtures/observability/golden/legacy-no-op-dev.json new file mode 100644 index 00000000..714c599a --- /dev/null +++ b/tests/fixtures/observability/golden/legacy-no-op-dev.json @@ -0,0 +1,327 @@ +{ + "author_unknown": { + "census_2026_09_13": "never_classified = 476,679 of 731,153 live (65.2%); both-NULL = 65,017; classified_unknown literal not observed on the copy", + "classified_unknown": { + "count": 2, + "definition": "provenance_class = 'unknown', archived_at IS NULL", + "share": 0.083333 + }, + "inputs": [ + { + "mtime": "2026-09-13T12:00:00Z", + "path": "db/legacy-no-op-dev.sqlite", + "rows_or_bytes": 25, + "sha256_first_64kb": null, + "skipped_lines": 0, + "status": "read" + } + ], + "never_classified": { + "count": 4, + "definition": "provenance_class IS NULL OR source_class IS NULL, archived_at IS NULL", + "share": 0.166667 + }, + "reason": "", + "state": "measured", + "top_source_files": [ + { + "count": 4, + "source_file": "realtime-hook" + }, + { + "count": 2, + "source_file": "brainbar-store" + } + ], + "trend_7d": [ + { + "classified_unknown": 0, + "day": "2026-09-07", + "never_classified": 1 + }, + { + "classified_unknown": 1, + "day": "2026-09-08", + "never_classified": 1 + }, + { + "classified_unknown": 0, + "day": "2026-09-09", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-10", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-11", + "never_classified": 0 + }, + { + "classified_unknown": 0, + "day": "2026-09-12", + "never_classified": 0 + }, + { + "classified_unknown": 1, + "day": "2026-09-13", + "never_classified": 2 + } + ] + }, + "backups": { + "db_snapshot": { + "destination": "synthetic-drive", + "last_at": "2026-09-13T09:00:00Z", + "verified": true + }, + "error_type": "jsonl_backup_attempt_invalid", + "freshness": "unknown", + "inputs": [ + { + "mtime": "2026-09-13T12:00:00Z", + "path": "logs/legacy-no-op-dev/jsonl-backup.log", + "rows_or_bytes": 102, + "sha256_first_64kb": "714843bf392b43087703143b0e900b90dcb89b82c684fee45c7c44e2a202ff52", + "skipped_lines": 0, + "status": "read" + }, + { + "mtime": "2026-09-13T12:00:00Z", + "path": "logs/legacy-no-op-dev/backup-daily.log", + "rows_or_bytes": 293, + "sha256_first_64kb": "c4c707bd7d86018084cf4b1a7c4b8cc40a21a3568be0ce611f633c2545daeef2", + "skipped_lines": 2, + "status": "read" + }, + { + "mtime": "2026-09-13T12:00:00Z", + "path": "launchd/legacy-no-op-dev.txt", + "rows_or_bytes": 94, + "sha256_first_64kb": "78d567d3dcf4aef4cebfa9c79fab6f05278f35aa18d147cdd6761c8bef8eff02", + "skipped_lines": 0, + "status": "read" + } + ], + "last_verified_upload": null, + "launchd": { + "bootstrapped": false, + "disabled_dir_present": false, + "label": "com.brainlayer.jsonl-backup" + }, + "reason": "", + "retention_invariant": "PASS", + "state": "measured", + "surviving_archives_30d": 0, + "threshold_hours": 36 + }, + "db_path": "db/legacy-no-op-dev.sqlite", + "emitters": { + "by_emitter": [ + { + "count_in_window": 2, + "derived_from": "sender", + "emitter": "assistant" + }, + { + "count_in_window": 2, + "derived_from": "source", + "emitter": "claude_code" + }, + { + "count_in_window": 2, + "derived_from": "source", + "emitter": "codex_cli" + }, + { + "count_in_window": 2, + "derived_from": "source", + "emitter": "mcp" + }, + { + "count_in_window": 2, + "derived_from": "source_file", + "emitter": "realtime-hook" + }, + { + "count_in_window": 2, + "derived_from": "source", + "emitter": "realtime_watcher" + } + ], + "by_source_class": [ + { + "count": 2, + "in_window": 1, + "source_class": "brain-worker" + }, + { + "count": 9, + "in_window": 4, + "source_class": "cli-agent" + }, + { + "count": 4, + "in_window": 2, + "source_class": "desktop" + }, + { + "count": 4, + "in_window": 2, + "source_class": "fleet-coordination" + }, + { + "count": 2, + "in_window": 1, + "source_class": "subagent" + }, + { + "count": 4, + "in_window": 2, + "source_class": null + } + ], + "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", + "hidden_from_default_search": 6, + "inputs": [ + { + "mtime": "2026-09-13T12:00:00Z", + "path": "db/legacy-no-op-dev.sqlite", + "rows_or_bytes": 25, + "sha256_first_64kb": null, + "skipped_lines": 0, + "status": "read" + } + ], + "reason": "", + "state": "measured" + }, + "generated_at": "2026-09-13T12:00:00Z", + "schema_version": 1, + "stores": { + "by_content_class": [ + { + "content_class": "decision", + "count": 6 + }, + { + "content_class": "knowledge", + "count": 7 + }, + { + "content_class": "operational", + "count": 6 + }, + { + "content_class": "test", + "count": 6 + } + ], + "in_window": { + "by_hour": [ + { + "count": 1, + "hour": "2026-09-13T01:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T02:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T03:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T04:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T05:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T06:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T07:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T08:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T09:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T10:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T11:00:00Z" + }, + { + "count": 1, + "hour": "2026-09-13T12:00:00Z" + } + ], + "count": 12 + }, + "inputs": [ + { + "mtime": "2026-09-13T12:00:00Z", + "path": "db/legacy-no-op-dev.sqlite", + "rows_or_bytes": 25, + "sha256_first_64kb": null, + "skipped_lines": 0, + "status": "read" + } + ], + "latest": [ + { + "chunk_id": "synthetic-00", + "emitter": "realtime_watcher", + "preview": "Synthetic observability fixture row 00", + "source_class": "cli-agent", + "stored_at": "2026-09-13T12:00:00Z" + }, + { + "chunk_id": "synthetic-01", + "emitter": "claude_code", + "preview": "Synthetic observability fixture row 01", + "source_class": "cli-agent", + "stored_at": "2026-09-13T11:00:00Z" + }, + { + "chunk_id": "synthetic-02", + "emitter": "codex_cli", + "preview": "Synthetic observability fixture row 02", + "source_class": "cli-agent", + "stored_at": "2026-09-13T10:00:00Z" + }, + { + "chunk_id": "synthetic-03", + "emitter": "mcp", + "preview": "Synthetic observability fixture row 03", + "source_class": "fleet-coordination", + "stored_at": "2026-09-13T09:00:00Z" + }, + { + "chunk_id": "synthetic-04", + "emitter": "assistant", + "preview": "Synthetic observability fixture row 04", + "source_class": "cli-agent", + "stored_at": "2026-09-13T08:00:00Z" + } + ], + "reason": "", + "state": "measured", + "total_chunks": 25 + }, + "window_hours": 24 +} diff --git a/tests/fixtures/observability/golden/malformed-log-dev-1.json b/tests/fixtures/observability/golden/malformed-log-dev-1.json index dec9f41e..dc19050c 100644 --- a/tests/fixtures/observability/golden/malformed-log-dev-1.json +++ b/tests/fixtures/observability/golden/malformed-log-dev-1.json @@ -137,13 +137,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -157,8 +157,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -168,7 +168,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -195,11 +195,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -271,7 +271,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -285,7 +285,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -299,7 +299,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/golden/missing-launchd-dev.json b/tests/fixtures/observability/golden/missing-launchd-dev.json index c4618ad9..ebd8e33c 100644 --- a/tests/fixtures/observability/golden/missing-launchd-dev.json +++ b/tests/fixtures/observability/golden/missing-launchd-dev.json @@ -137,13 +137,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -157,8 +157,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -168,7 +168,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -195,11 +195,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -271,7 +271,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -285,7 +285,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -299,7 +299,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/golden/missing-log-dev.json b/tests/fixtures/observability/golden/missing-log-dev.json index a474a018..47c80944 100644 --- a/tests/fixtures/observability/golden/missing-log-dev.json +++ b/tests/fixtures/observability/golden/missing-log-dev.json @@ -137,13 +137,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -157,8 +157,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -168,7 +168,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -195,11 +195,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -271,7 +271,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -285,7 +285,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -299,7 +299,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/golden/missing-source-class-dev.json b/tests/fixtures/observability/golden/missing-source-class-dev.json index f3a8ca62..5c219539 100644 --- a/tests/fixtures/observability/golden/missing-source-class-dev.json +++ b/tests/fixtures/observability/golden/missing-source-class-dev.json @@ -61,7 +61,7 @@ "reason": "", "retention_invariant": "PASS", "state": "measured", - "surviving_archives_30d": 7, + "surviving_archives_30d": 1, "threshold_hours": 36 }, "db_path": "db/missing-source-class-dev.sqlite", diff --git a/tests/fixtures/observability/golden/no-op-dev.json b/tests/fixtures/observability/golden/no-op-dev.json index d786f50b..802cd811 100644 --- a/tests/fixtures/observability/golden/no-op-dev.json +++ b/tests/fixtures/observability/golden/no-op-dev.json @@ -112,7 +112,7 @@ "label": "com.brainlayer.jsonl-backup" }, "reason": "", - "retention_invariant": "unknown", + "retention_invariant": "PASS", "state": "measured", "surviving_archives_30d": 0, "threshold_hours": 36 @@ -153,13 +153,13 @@ ], "by_source_class": [ { - "count": 5, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "brain-worker" }, { - "count": 4, - "in_window": 2, + "count": 9, + "in_window": 4, "source_class": "cli-agent" }, { @@ -173,8 +173,8 @@ "source_class": "fleet-coordination" }, { - "count": 4, - "in_window": 2, + "count": 2, + "in_window": 1, "source_class": "subagent" }, { @@ -184,7 +184,7 @@ } ], "derivation_note": "metadata.attributionAgent is absent in the 2026-09-13 census; emitter derives from source, then sender, then source_file, or the section is unmeasurable", - "hidden_from_default_search": 9, + "hidden_from_default_search": 6, "inputs": [ { "mtime": "2026-09-13T12:00:00Z", @@ -211,11 +211,11 @@ "count": 7 }, { - "content_class": "noise", + "content_class": "operational", "count": 6 }, { - "content_class": "operational", + "content_class": "test", "count": 6 } ], @@ -287,7 +287,7 @@ "chunk_id": "synthetic-00", "emitter": "realtime_watcher", "preview": "Synthetic observability fixture row 00", - "source_class": "brain-worker", + "source_class": "cli-agent", "stored_at": "2026-09-13T12:00:00Z" }, { @@ -301,7 +301,7 @@ "chunk_id": "synthetic-02", "emitter": "codex_cli", "preview": "Synthetic observability fixture row 02", - "source_class": "desktop", + "source_class": "cli-agent", "stored_at": "2026-09-13T10:00:00Z" }, { @@ -315,7 +315,7 @@ "chunk_id": "synthetic-04", "emitter": "assistant", "preview": "Synthetic observability fixture row 04", - "source_class": "subagent", + "source_class": "cli-agent", "stored_at": "2026-09-13T08:00:00Z" } ], diff --git a/tests/fixtures/observability/launchd/legacy-no-op-dev.txt b/tests/fixtures/observability/launchd/legacy-no-op-dev.txt new file mode 100644 index 00000000..dea098b7 --- /dev/null +++ b/tests/fixtures/observability/launchd/legacy-no-op-dev.txt @@ -0,0 +1,2 @@ +Bad request. +Could not find service "com.brainlayer.jsonl-backup" in domain for user gui: 501 diff --git a/tests/fixtures/observability/logs/legacy-no-op-dev/backup-daily.log b/tests/fixtures/observability/logs/legacy-no-op-dev/backup-daily.log new file mode 100644 index 00000000..f22368b9 --- /dev/null +++ b/tests/fixtures/observability/logs/legacy-no-op-dev/backup-daily.log @@ -0,0 +1,3 @@ +drive upload progress: 50/100 bytes +{"attempted_at": "2026-09-13T09:00:00Z", "backup_log_provenance": "real", "destination": "synthetic-drive", "drive_md5_match": true, "snapshot": "/synthetic/backups/2026-09-13.db.gz", "uploaded": true, "verified": true} +drive upload progress: 100/100 bytes diff --git a/tests/fixtures/observability/logs/legacy-no-op-dev/jsonl-backup.log b/tests/fixtures/observability/logs/legacy-no-op-dev/jsonl-backup.log new file mode 100644 index 00000000..c4e10236 --- /dev/null +++ b/tests/fixtures/observability/logs/legacy-no-op-dev/jsonl-backup.log @@ -0,0 +1 @@ +{"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": false, "verified": true} diff --git a/tests/fixtures/observability/logs/no-op-dev/jsonl-backup.log b/tests/fixtures/observability/logs/no-op-dev/jsonl-backup.log index c4e10236..1879658d 100644 --- a/tests/fixtures/observability/logs/no-op-dev/jsonl-backup.log +++ b/tests/fixtures/observability/logs/no-op-dev/jsonl-backup.log @@ -1 +1 @@ -{"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": false, "verified": true} +{"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": false, "verified": true, "attempted_at": "2026-09-13T10:00:00Z"} diff --git a/tests/fixtures/observability/logs/no-op-heldout-2/jsonl-backup.log b/tests/fixtures/observability/logs/no-op-heldout-2/jsonl-backup.log index c4e10236..1879658d 100644 --- a/tests/fixtures/observability/logs/no-op-heldout-2/jsonl-backup.log +++ b/tests/fixtures/observability/logs/no-op-heldout-2/jsonl-backup.log @@ -1 +1 @@ -{"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": false, "verified": true} +{"status": "no-op", "message": "no-op, 0 files already covered", "uploaded": false, "verified": true, "attempted_at": "2026-09-13T10:00:00Z"} diff --git a/tests/test_observability_eval.py b/tests/test_observability_eval.py index ea25606b..9706787f 100644 --- a/tests/test_observability_eval.py +++ b/tests/test_observability_eval.py @@ -2,7 +2,10 @@ import hashlib import json +import os +import shutil import sqlite3 +from datetime import datetime from pathlib import Path from types import SimpleNamespace @@ -52,6 +55,99 @@ def test_builder_uses_vector_store_schema_without_handwritten_ddl(tmp_path: Path assert "CREATE TABLE" not in Path(builder.__file__).read_text(encoding="utf-8").upper() +def test_runner_stages_pinned_mtimes_without_mutating_fixture(tmp_path: Path) -> None: + source_root = tmp_path / "fixture" + source_root.mkdir() + source = source_root / "db/healthy-dev.sqlite" + source.parent.mkdir() + source.write_bytes(b"fixture") + os.utime(source, (1, 1)) + case = json.loads(Path("tests/fixtures/observability/cases.json").read_text())["cases"][0] + case = { + **case, + "declared_inputs": ["db/healthy-dev.sqlite"], + "input_mtimes": {"db/healthy-dev.sqlite": "2026-09-13T12:00:00Z"}, + } + + staged_root = tmp_path / "staged" + evaluator._stage_case_inputs(case, source_root, staged_root) + + assert source.stat().st_mtime == 1 + assert ( + staged_root.joinpath("db/healthy-dev.sqlite").stat().st_mtime + == datetime.fromisoformat("2026-09-13T12:00:00+00:00").timestamp() + ) + + +def test_runner_stages_future_clock_skew_log_mtime_without_mutating_fixture(tmp_path: Path) -> None: + source_root = tmp_path / "fixture" + source_root.mkdir() + for relative in ("db/clock-skew-dev.sqlite", "logs/clock-skew-dev/jsonl-backup.log"): + source = source_root / relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(b"fixture") + os.utime(source, (1, 1)) + original = evaluator.load_case("clock-skew-dev") + declared = ["db/clock-skew-dev.sqlite", "logs/clock-skew-dev/jsonl-backup.log"] + case = { + **original, + "declared_inputs": declared, + "input_mtimes": {path: original["input_mtimes"][path] for path in declared}, + } + + staged_root = tmp_path / "staged" + evaluator._stage_case_inputs(case, source_root, staged_root) + + assert (source_root / "logs/clock-skew-dev/jsonl-backup.log").stat().st_mtime == 1 + assert ( + staged_root.joinpath("logs/clock-skew-dev/jsonl-backup.log").stat().st_mtime + == datetime.fromisoformat("2026-09-13T16:00:00+00:00").timestamp() + ) + + +def test_runner_fails_closed_when_input_mtime_is_missing(tmp_path: Path) -> None: + case = evaluator.load_case("healthy-dev") + case = {**case, "input_mtimes": {}} + result = evaluator._run_case(case, Path("tests/fixtures/observability"), Path.cwd(), None) + assert result.field_mismatches == [ + "$: input staging failed: missing input_mtimes for declared inputs: " + + ", ".join(sorted(case["declared_inputs"])) + ] + + +def test_faithful_stub_requires_runner_mtime_staging(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fixture = tmp_path / "fixture" + shutil.copytree("tests/fixtures/observability", fixture) + case = evaluator.load_case("healthy-dev", fixture) + os.utime(fixture / case["inputs"]["db"], (1, 1)) + + def stub(*args: object, env: dict[str, str], **kwargs: object) -> SimpleNamespace: + payload = evaluator.load_golden("healthy-dev", fixture) + actual = json.loads(json.dumps(payload)) + expected_mtime = case["input_mtimes"][case["inputs"]["db"]] + observed = ( + __import__("datetime") + .datetime.fromtimestamp(Path(env["BRAINLAYER_DB"]).stat().st_mtime, __import__("datetime").UTC) + .isoformat() + .replace("+00:00", "Z") + ) + if observed != expected_mtime: + for section in actual.values(): + if isinstance(section, dict): + for item in section.get("inputs", []): + if isinstance(item, dict) and item.get("path") == case["inputs"]["db"]: + item["mtime"] = observed + Path(env["BRAINLAYER_OBSERVABILITY_PATH"]).write_text(json.dumps(actual), encoding="utf-8") + Path(env["BRAINLAYER_OBSERVABILITY_TRACE_PATH"]).write_text( + json.dumps(case["declared_inputs"]), encoding="utf-8" + ) + return SimpleNamespace(returncode=0, stderr="", stdout="") + + monkeypatch.setattr(evaluator.subprocess, "run", stub) + assert evaluator._run_case(case, fixture, Path.cwd(), None).passed + assert not evaluator._run_case(case, fixture, Path.cwd(), None, stage_inputs=False).passed + + def test_cases_cover_every_fail_closed_shape_in_both_splits() -> None: cases = builder.case_definitions() manifest = json.loads(Path(builder.MANIFEST).read_text(encoding="utf-8"))["cases"]