Release 0.10.0: resumable benchmarks by default

.github/workflows/ci.yml

@@ -30,7 +30,9 @@ jobs:
       # registry importable without the benchmark stack — bitmind-subnet
       # depends on this contract. This job fails if anyone adds an eager
       # heavy import to the registry path or a heavy dep to base deps.
-      - run: pip install .
+      - run: pip install . pytest
+      # Checkpoint recovery is CPU-only and must not require the ML extras.
+      - run: pytest tests/unit/test_checkpoint.py
       - run: |
           python - <<'EOF'
           from gasbench.dataset.config import load_benchmark_datasets_from_yaml
@@ -52,6 +54,32 @@ jobs:
           assert not leaked, f"heavy deps in base install: {leaked}"
           print("base install contract holds:", dists)
           EOF
+      # Recorder recovery shares the existing metrics and parquet paths, without
+      # loading torch or the inference backends (same lazy-import policy).
+      - run: pip install numpy pandas pyarrow
+      - run: pytest tests/unit/test_recorder_checkpoint.py
+
+  benchmark-resume:
+    name: benchmark interruption recovery (CPU)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      # Use the project's versions, with CPU wheels for the inference test doubles.
+      - run: |
+          python - <<'EOF'
+          import subprocess, sys, tomllib
+          with open("pyproject.toml", "rb") as f:
+              requirements = tomllib.load(f)["project"]["optional-dependencies"]["gpu"]
+          cpu = [r for r in requirements if r.split("==")[0] in {"torch", "torchvision", "torchaudio"}]
+          subprocess.check_call([sys.executable, "-m", "pip", "install", *cpu,
+                                 "--index-url", "https://download.pytorch.org/whl/cpu"])
+          EOF
+      - run: pip install '.[gpu]' pytest
+      - run: pytest tests/unit/test_benchmark_resume.py tests/unit/test_cached_sample_labels.py
 
   typecheck:
     name: mypy (advisory)

README.md

@@ -77,6 +77,24 @@ Model directory must contain: `model_config.yaml`, `model.py`, `*.safetensors`
 
 Results are automatically saved to a timestamped JSON file.
 
+Every benchmark run checkpoints through `BenchmarkRunRecorder` after each inference
+batch. Checkpoints default to `<cache-dir>/runs/<run-id>/checkpoint`; use
+`--checkpoint-dir` to choose another location. To resume, rerun the same command
+with the same `--run-id` and storage directory. Completed predictions are restored
+before decoding, and scores and parquet output are rebuilt through the normal
+recorder path. A new run ID starts a new evaluation.
+
+The checkpoint freezes sample selection (including the robustness pass), source
+content hashes, model/evaluator identity, seed (42 when omitted), and scoring
+settings. Changed inputs fail closed. Preparing a new run reads the selected files
+to fingerprint them; resumed runs validate only pending samples. Model setup and
+uncommitted work may repeat after interruption.
+
+Checkpoint storage must survive the process or container being replaced. The
+Python API accepts `checkpoint_persist(directory)` for filesystems requiring an
+explicit remote commit. One coordinator must own each run directory; automatic
+container replacement and distributed ownership are the caller's responsibility.
+
 ---
 
 ## Python API

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gasbench"
-version = "0.9.10"
+version = "0.10.0"
 description = "GASBench - ML model benchmark evaluation package"
 readme = "README.md"
 requires-python = ">=3.10"

src/gasbench/benchmark.py

@@ -41,9 +41,16 @@ async def run_benchmark(
     aug_weight: float = 0.2,
     aug_cache_dir: Optional[str] = None,
     aug_cache_readonly: bool = False,
+    checkpoint_dir: Optional[str] = None,
+    checkpoint_persist=None,
 ) -> Dict:
     """
     Args:
+        checkpoint_dir: Storage directory; defaults to <cache_dir>/runs/<run_id>/checkpoint.
+            Every run checkpoints. Reuse run_id and the same inputs to resume.
+        checkpoint_persist: Optional filesystem commit callback, called after each
+            durable write. Required when a mounted filesystem needs an explicit
+            remote commit; failures abort the run.
         model_path: Path to a custom PyTorch model directory
         modality: Type of modality to test ("image" or "video")
         mode: Benchmark mode - "debug", "small", or "full" (default: "full")
@@ -127,6 +134,8 @@ async def run_benchmark(
             content_category,
             score_composition,
             multiclass_scoring,
+            checkpoint_dir=checkpoint_dir,
+            checkpoint_persist=checkpoint_persist,
             n_aug_per_dataset=n_aug_per_dataset,
             aug_weight=aug_weight,
             aug_cache_dir=aug_cache_dir,
@@ -232,6 +241,8 @@ async def execute_benchmark(
     aug_weight: float = 0.2,
     aug_cache_dir: Optional[str] = None,
     aug_cache_readonly: bool = False,
+    checkpoint_dir: Optional[str] = None,
+    checkpoint_persist=None,
 ) -> float:
     """Execute the actual benchmark evaluation."""
 
@@ -260,6 +271,8 @@ async def execute_benchmark(
             content_category=content_category,
             score_composition=score_composition,
             multiclass_scoring=multiclass_scoring,
+            checkpoint_dir=checkpoint_dir,
+            checkpoint_persist=checkpoint_persist,
             n_aug_per_dataset=n_aug_per_dataset,
             aug_weight=aug_weight,
             aug_cache_dir=aug_cache_dir,
@@ -288,6 +301,8 @@ async def execute_benchmark(
             content_category=content_category,
             score_composition=score_composition,
             multiclass_scoring=multiclass_scoring,
+            checkpoint_dir=checkpoint_dir,
+            checkpoint_persist=checkpoint_persist,
             n_aug_per_dataset=n_aug_per_dataset,
             aug_weight=aug_weight,
             aug_cache_dir=aug_cache_dir,
@@ -316,6 +331,8 @@ async def execute_benchmark(
             content_category=content_category,
             score_composition=score_composition,
             multiclass_scoring=multiclass_scoring,
+            checkpoint_dir=checkpoint_dir,
+            checkpoint_persist=checkpoint_persist,
         )
         benchmark_score = benchmark_results.get("audio_results", {}).get("benchmark_score", 0.0)
     else:

src/gasbench/benchmarks/__init__.py

@@ -1,18 +1,23 @@
-"""Benchmark execution and metrics."""
+"""Lazy exports, matching gasbench: recorder recovery needs no inference backend."""
 
-from .image_bench import run_image_benchmark
-from .video_bench import run_video_benchmark
-from .utils import (
-    Metrics,
-    update_generator_stats,
-    calculate_per_source_accuracy,
-)
+from importlib import import_module
 
-__all__ = [
-    "run_image_benchmark",
-    "run_video_benchmark",
-    "Metrics",
-    "update_generator_stats",
-    "calculate_per_source_accuracy",
-]
+_LAZY_EXPORTS = {
+    "run_image_benchmark": (".image_bench", "run_image_benchmark"),
+    "run_video_benchmark": (".video_bench", "run_video_benchmark"),
+    "Metrics": (".utils.metrics", "Metrics"),
+    "update_generator_stats": (".utils.metrics", "update_generator_stats"),
+    "calculate_per_source_accuracy": (
+        ".utils.metrics",
+        "calculate_per_source_accuracy",
+    ),
+}
 
+__all__ = list(_LAZY_EXPORTS)
+
+
+def __getattr__(name):
+    if name not in _LAZY_EXPORTS:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module_name, attr = _LAZY_EXPORTS[name]
+    return getattr(import_module(module_name, __name__), attr)