fix(protspace): own every retained cache by what produced it

apps/protspace/CLAUDE.md

@@ -268,24 +268,25 @@ For a live count run `uv run pytest tests/ --collect-only -q`.
 
 | File | What it covers |
 |------|---------------|
-| `test_annotation_manager.py` | Annotation fetch, merge, cache, configuration, evidence parsing |
+| `test_annotation_manager.py` | Annotation fetch, merge, cache, configuration, evidence parsing; per-identifier reuse (a source is fetched only for the identifiers the cache lacks, taxonomy only for unseen organisms, rows outside the run are kept, a failed fill-in caches nothing) |
 | `test_transformer.py` | Annotation transformers (field normalization, EC names) |
 | `test_reducers.py` | All 6 DR methods: shapes, finite output, float16, config validation |
 | `test_interpro_annotation_retriever.py` | InterPro API mocking, parsing |
 | `test_settings_converter.py` | Settings table ↔ visualization state conversion |
 | `test_uniprot_annotation_retriever.py` | UniProt API mocking, inactive entry resolution |
-| `test_pipeline_utils.py` | ReductionPipeline, EmbeddingSet, method parsing, multi-input merging, inline param overrides |
+| `test_pipeline_utils.py` | ReductionPipeline, projection cache identity (a changed, reordered or grown matrix under one embedding name misses; an unchanged rerun hits; `--refetch projections` always recomputes), annotation cache fill-in wiring, EmbeddingSet, method parsing, multi-input merging, inline param overrides |
 | `test_stats.py` | Projection statistics: elbow, annotation-based validity (silhouette/DBI/CH per annotation), auto-cluster ARI/NMI agreement, auto-cluster self-validity (filed under the membership column, gated on it, and equal to driving `AnnotationValidityStatistic` directly so an out-of-band re-score cannot drift), faithfulness (dual continuity + global metrics), cluster-selection (elbow/silhouette/both), subsample determinism/order-invariance, silhouette consistency, `_align` no-id guard, silhouette→elbow fallback |
 | `test_stats_cli.py` | `protspace stats` CLI + `prepare` stats wiring, `--stats-annotation` (auto/list) wiring, `--settings-out` guard, `--cluster-selection` validation |
 | `test_stats_carriage.py` | Routing rows to bundle parts (metadata quality, annotation columns, cluster legend) |
 | `test_stats_bundle.py` | Optional 5th (statistics) bundle part round-trip |
 | `test_annotation_select.py` | Annotation selection: suitability filter (cardinality/numeric/id-like exclusion), `auto` vs explicit-list label building (explicit names bypass the heuristic), missing-value dropping |
 | `test_annotation_validity.py` | `AnnotationValidityStatistic`: silhouette/DBI/CH scored per annotation on `ctx.coords`, embedding vs. projection `space_kind`, missing-value exclusion, single-category no-op, id-canonical subsample determinism |
-| `test_biocentral_embedder.py` | Biocentral API client, embedding flow, completeness gate (reads the .h5, not a counter), `/`-in-header rejection |
-| `test_embed_completeness.py` | Shared embed contract (`data/embedding/store.py`): `expected = requested - skipped`, skip-vs-fail, skip reporting, resume-covered runs, FASTA coverage direction + identifier normalisation |
-| `test_backend_switch.py` | Embedding backend switch: `resolve_default_backend` (Colab+GPU→local), `embed_fasta` local/biocentral dispatch (short key vs resolved name), `protspace embed --backend` CLI wiring + enum validation + non-positive batch_size rejection |
-| `test_local_embedder.py` | Local embedding backend: checkpoint resolution (12 short keys, Synthyra ESM-C), the notebook-gating sets pinned to the registry each constrains (`COLAB_OVERSIZED`→`LOCAL_CHECKPOINTS`, `BIOCENTRAL_INVALID`→`ALL_SHORT_KEYS`), per-family preprocessing/residue pooling, `/`-in-header guard, LocalEmbedConfig validation, over-length + OOM skips reported not failed, non-skip shortfall fails, esm2_8m end-to-end + resume (slow) |
+| `test_biocentral_embedder.py` | Biocentral API client, embedding flow, completeness gate (reads the .h5, not a counter), `/`-in-header rejection, producer stamping, and a local-written cache refused before any API call |
+| `test_embed_completeness.py` | Shared embed contract (`data/embedding/store.py`): `expected = requested - skipped`, skip-vs-fail, skip reporting, resume-covered runs, FASTA coverage direction + identifier normalisation; producer ownership (another backend or model is refused with the remedies and the file left byte-identical, an unstamped file is adopted then owned) and residue identity (a changed sequence is outstanding again and replaces its vector + digest, an unchanged one resumes, a digest-less protein is trusted, digests read in one file open) |
+| `test_backend_switch.py` | Embedding backend switch: `embed_fasta` refuses another backend's cache and resumes its own, and returns only the requested FASTA's proteins; `resolve_default_backend` (Colab+GPU→local), `embed_fasta` local/biocentral dispatch (short key vs resolved name), `protspace embed --backend` CLI wiring + enum validation + non-positive batch_size rejection |
+| `test_local_embedder.py` | Local embedding backend: producer/digest stamping, refusing a Biocentral cache before a checkpoint loads, re-embedding a changed sequence; checkpoint resolution (12 short keys, Synthyra ESM-C), the notebook-gating sets pinned to the registry each constrains (`COLAB_OVERSIZED`→`LOCAL_CHECKPOINTS`, `BIOCENTRAL_INVALID`→`ALL_SHORT_KEYS`), per-family preprocessing/residue pooling, `/`-in-header guard, LocalEmbedConfig validation, over-length + OOM skips reported not failed, non-skip shortfall fails, esm2_8m end-to-end + resume (slow) |
 | `test_fasta.py` | FASTA parsing, edge cases, CSV annotation loading |
+| `test_query.py` | UniProt query FASTA download: a truncated download is never published, atomic cache publication, umask-derived permissions, and a retained FASTA owned by its query text (`prepare -q A` then `-q B` in one output directory) |
 | `test_biocentral_retriever.py` | Biocentral prediction retriever (TMbed parsing, per-sequence) |
 | `test_taxonomy_annotation_retriever.py` | Taxonomy via UniProt Taxonomy API (mocked + integration) |
 | `test_config_validation.py` | DimensionReductionConfig parameter validation |
@@ -303,14 +304,15 @@ For a live count run `uv run pytest tests/ --collect-only -q`.
 | `test_display_decode.py` | Display-side decoding of encoded values, multi-hit rendering, gated-off passthrough |
 | `test_toxprot_demo.py` | Signal-peptide bound parsing, mature-FASTA stripping, bundle post-processing (column filter/reorder) |
 | `test_bundle_overlay.py` | Round-trip replacement of the annotations part of a bundle |
+| `test_atomic_publication.py` | `data/io/atomic.py`: staged rename keeps the previous content on failure, and a published file (bundle, statistics parquet, retained FASTA) carries the process umask rather than `mkstemp`'s owner-only mode |
 | `test_classification.py` | Query/reference rules: id-prefix and case-insensitive `where` substring, query-over-reference precedence, empty-match and missing-column errors |
 | `test_bundle_version.py` | `format_version=2` stamped into the annotations parquet |
 | `test_uniprot_parser_encoding.py` | UniProtEntry free-text emit points percent-encode reserved chars |
 | `test_cath_names.py` | CATH names file parsing |
 | `test_cli_no_frontend.py` | CLI imports without the optional `frontend` extra (plotly, dash) |
 | `test_cli_no_similarity.py` | `-s/--similarity` without the optional `similarity` extra: up-front CLI guard (before any load/embed), loader `ImportError` backstop, `EMBEDDER_MODELS` pinned to the embedder registry |
 | `test_docs_extras_sync.py` | `README.md` (PyPI) and `docs/guide/python-cli.md` (protspace.app) hold the same extras section; the guide's embedder shortcut list matches `EMBEDDER_MODELS` |
-| `test_notebooks.py` | Colab notebooks: cell magics only on line 1, every code cell compiles after IPython transformation, cell ids present for `nbformat >= 4.5`, `except ImportError` fallback sets equal the package constants they stand in for — read structurally off the guarded import, so a fallback that is missing, emptied or written in an unrecognised shape fails instead of matching nothing |
+| `test_notebooks.py` | Colab notebooks: cell magics only on line 1, every code cell compiles after IPython transformation, cell ids present for `nbformat >= 4.5`, no notebook imports a private `protspace` name (cell 1 installs the *released* package, so a private name added this release breaks setup until the next one), each Generate action names its bundle distinctly, `except ImportError` fallback sets equal the package constants they stand in for — read structurally off the guarded import, so a fallback that is missing, emptied or written in an unrecognised shape fails instead of matching nothing |
 | `test_encoding_e2e.py` | Backend end-to-end round-trip proof for v2 annotation encoding |
 | `test_scores_ted.py` | `--no-scores` strips TED domains |
 

apps/protspace/notebooks/ProtSpace_Preparation.ipynb

@@ -49,6 +49,7 @@
     "# instead — and cover every import below, not just protspace's: h5py, pandas and\n",
     "# tqdm are the preloaded ones, so they are the likeliest to be the casualty.\n",
     "try:\n",
+    "    import hashlib\n",
     "    import urllib.request\n",
     "    from pathlib import Path\n",
     "\n",
@@ -82,7 +83,7 @@
     "        f\"Import failed after install ({_exc}).\\n\"\n",
     "        \"This usually means an upgraded package needs a fresh interpreter: \"\n",
     "        \"Runtime > Restart session, then run this cell again.\"\n",
-    "    ) from _exc"
+    "    ) from _exc\n"
    ]
   },
   {
@@ -518,7 +519,7 @@
     "    # reading, and that is also when a group *is* visible.\n",
     "    notes = []\n",
     "    if \"PCA\" in sel:\n",
-    "        notes.append(\"PCA has no parameters\")\n",
+    "        notes.append(\"PCA has no parameters, so these sliders do not change it\")\n",
     "    if \"MDS\" in sel:\n",
     "        notes.append(\n",
     "            \"MDS runs with this notebook's fixed defaults\"\n",
@@ -652,7 +653,10 @@
     "        out_dir.mkdir(exist_ok=True)\n",
     "        cache_dir = out_dir / \"tmp\"\n",
     "        cache_dir.mkdir(exist_ok=True)\n",
-    "        output_path = out_dir / \"data.parquetbundle\"\n",
+    "        # Distinct per run: a fixed name downloads as \"data (1).parquetbundle\"\n",
+    "        # beside the previous one, and opening the older file looks exactly like\n",
+    "        # a projection that never refreshed (issue #338).\n",
+    "        output_path = out_dir / f\"protspace_{_time.strftime('%Y%m%d-%H%M%S')}.parquetbundle\"\n",
     "\n",
     "        step_html = widgets.HTML(value=\"<b>Loading embeddings...</b>\")\n",
     "        display(step_html)\n",
@@ -661,59 +665,52 @@
     "            t0 = _time.time()\n",
     "            embedding_sets = []\n",
     "\n",
-    "            if inp[\"type\"] == \"query\":\n",
+    "            if inp[\"type\"] in (\"query\", \"fasta\"):\n",
     "                gated = _embedders_and_backend()\n",
     "                if gated is None:\n",
     "                    return\n",
     "                embs, backend, _emb_cfg = gated\n",
-    "                step_html.value = \"<b>Fetching sequences from UniProt...</b>\"\n",
-    "                fasta_cache = cache_dir / \"sequences.fasta\"\n",
-    "                if fasta_cache.exists() and fasta_cache.stat().st_size > 0:\n",
-    "                    from protspace.data.loaders.query import (\n",
-    "                        extract_identifiers_from_fasta,\n",
-    "                    )\n",
-    "                    headers = extract_identifiers_from_fasta(fasta_cache)\n",
-    "                    fasta_path = fasta_cache\n",
-    "                    print(f\"Reusing cached sequences from output/tmp ({len(headers):,} sequences) — delete output/tmp to re-run the query\")\n",
+    "                if inp[\"type\"] == \"query\":\n",
+    "                    step_html.value = \"<b>Fetching sequences from UniProt...</b>\"\n",
+    "                    # Addressed by the query text: one shared name would hand a\n",
+    "                    # later query the previous one's sequences.\n",
+    "                    _q = hashlib.sha256(inp[\"query\"].encode()).hexdigest()[:12]\n",
+    "                    fasta_cache = cache_dir / \"queries\" / f\"{_q}.fasta\"\n",
+    "                    if fasta_cache.exists() and fasta_cache.stat().st_size > 0:\n",
+    "                        from protspace.data.loaders.query import (\n",
+    "                            extract_identifiers_from_fasta,\n",
+    "                        )\n",
+    "                        headers = extract_identifiers_from_fasta(fasta_cache)\n",
+    "                        fasta_path = fasta_cache\n",
+    "                        print(f\"Reusing cached sequences from output/tmp ({len(headers):,} sequences) — delete output/tmp to re-run the query\")\n",
+    "                    else:\n",
+    "                        headers, fasta_path = query_uniprot(inp[\"query\"], save_to=fasta_cache)\n",
+    "                    if not headers:\n",
+    "                        print(f\"No sequences found for query: {inp['query']}\")\n",
+    "                        return\n",
     "                else:\n",
-    "                    headers, fasta_path = query_uniprot(inp[\"query\"], save_to=fasta_cache)\n",
-    "                if not headers:\n",
-    "                    print(f\"No sequences found for query: {inp['query']}\")\n",
-    "                    return\n",
+    "                    fasta_path = Path(inp[\"path\"])\n",
     "                for emb_name in embs:\n",
     "                    step_html.value = f\"<b>Computing {emb_name} embeddings ({backend})...</b>\"\n",
     "                    emb_set = embed_fasta(\n",
     "                        fasta_path, emb_name,\n",
     "                        backend=backend,\n",
     "                        embed_config=_emb_cfg,\n",
-    "                        embedding_cache=cache_dir / f\"{emb_name}.h5\",\n",
+    "                        # Backend in the name so switching the toggle keeps both\n",
+    "                        # caches; the store refuses to mix them either way.\n",
+    "                        embedding_cache=cache_dir / f\"{backend}-{emb_name}.h5\",\n",
     "                    )\n",
     "                    emb_set.fasta_path = fasta_path\n",
     "                    embedding_sets.append(emb_set)\n",
-    "            elif inp[\"type\"] == \"fasta\":\n",
-    "                gated = _embedders_and_backend()\n",
-    "                if gated is None:\n",
-    "                    return\n",
-    "                embs, backend, _emb_cfg = gated\n",
-    "                for emb_name in embs:\n",
-    "                    step_html.value = f\"<b>Computing {emb_name} embeddings ({backend})...</b>\"\n",
-    "                    emb_set = embed_fasta(\n",
-    "                        Path(inp[\"path\"]), emb_name,\n",
-    "                        backend=backend,\n",
-    "                        embed_config=_emb_cfg,\n",
-    "                        embedding_cache=cache_dir / f\"{emb_name}.h5\",\n",
-    "                    )\n",
-    "                    emb_set.fasta_path = Path(inp[\"path\"])\n",
-    "                    embedding_sets.append(emb_set)\n",
     "            else:\n",
     "                h5_path = Path(inp[\"path\"])\n",
     "                name_override = inp.get(\"name\")\n",
     "                emb_set = load_h5([h5_path], name_override=name_override)\n",
     "                embedding_sets.append(emb_set)\n",
     "\n",
-    "            n_proteins = len(embedding_sets[0].headers)\n",
-    "\n",
-    "            # Build pipeline with caching enabled\n",
+    "            # Cached intermediates are keyed by what they were computed from\n",
+    "            # (matrix, sequence, query), so reuse here is always reuse of this\n",
+    "            # input's own work.\n",
     "            reducer_params = ReducerParams(\n",
     "                n_neighbors=pw[\"n_neighbors\"].value,\n",
     "                min_dist=pw[\"min_dist\"].value,\n",
@@ -734,15 +731,20 @@
     "            )\n",
     "            pipeline = ReductionPipeline(config)\n",
     "\n",
+    "            # Embedders can skip different proteins (an OOM on the larger model,\n",
+    "            # say), and bundling pairs projection rows with one header list, so\n",
+    "            # align every set to the shared identifiers as ReductionPipeline.run() does.\n",
+    "            all_headers = pipeline._validate_headers(embedding_sets)\n",
+    "            n_proteins = len(all_headers)\n",
+    "\n",
     "            # Step 2: Annotations (cached after first run)\n",
     "            step_html.value = \"<b>Fetching annotations...</b>\"\n",
-    "            metadata = pipeline._fetch_annotations(embedding_sets[0].headers)\n",
+    "            metadata = pipeline._fetch_annotations(all_headers, embedding_sets)\n",
     "\n",
     "            # Step 3: Dimensionality reduction\n",
     "            step_html.value = \"<b>Reducing dimensions...</b>\"\n",
     "\n",
     "            # Build full metadata\n",
-    "            all_headers = embedding_sets[0].headers\n",
     "            full_metadata = _pd.DataFrame({\"identifier\": all_headers})\n",
     "            if len(metadata.columns) > 1:\n",
     "                metadata = metadata.astype(str)\n",
@@ -790,7 +792,7 @@
     "            print(f\"Processed {n_proteins} proteins with {len(method_specs)} method(s)\")\n",
     "            print(f\"\\nBundle written to {output_path.resolve()} (inside this Colab runtime).\")\n",
     "            print(\"Downloading it to your computer now, then open it at https://protspace.app/explore\")\n",
-    "            print(\"(Download blocked by the browser? Take it from the Colab Files pane: output/data.parquetbundle)\")\n",
+    "            print(f\"(Download blocked by the browser? Take it from the Colab Files pane: output/{output_path.name})\")\n",
     "            files.download(str(output_path))\n",
     "\n",
     "        except Exception as e:\n",
@@ -839,7 +841,7 @@
     "display(widgets.VBox([\n",
     "    _panel,\n",
     "    gen_btn,\n",
-    "    widgets.HTML(\"<p><i>The bundle is written to <code>output/data.parquetbundle</code> in this \"\n",
+    "    widgets.HTML(\"<p><i>Each run writes a timestamped bundle to <code>output/</code> in this \"\n",
     "         \"runtime, then downloaded to your computer.</i></p>\"),\n",
     "    gen_out,\n",
     "]))\n",

apps/protspace/src/protspace/cli/prepare.py

@@ -420,32 +420,16 @@ def prepare(
     # --- Build embedding sets ---
     from protspace.data.loaders import EmbeddingSet, load_h5
     from protspace.data.loaders.h5 import EMBEDDING_EXTENSIONS
-    from protspace.data.loaders.query import (
-        extract_identifiers_from_fasta,
-        query_uniprot,
-    )
 
     embed_config = build_embed_config(backend, batch_size, max_length)
     embedding_sets: list[EmbeddingSet] = []
     fasta_for_similarity: Path | None = fasta
 
     try:
         if query:
-            fasta_save = cache_dir / "sequences.fasta" if cache_dir else None
-            if (
-                fasta_save
-                and fasta_save.exists()
-                and fasta_save.stat().st_size > 0
-                and "query" not in refetch_stages
-            ):
-                headers = extract_identifiers_from_fasta(fasta_save)
-                logger.warning(
-                    "Using cached FASTA (%s sequences)",
-                    f"{len(headers):,}",
-                )
-                fasta_path = fasta_save
-            else:
-                headers, fasta_path = query_uniprot(query, save_to=fasta_save)
+            from protspace.data.loaders.query import resolve_query_fasta
+
+            headers, fasta_path = resolve_query_fasta(query, cache_dir, refetch_stages)
             if not headers:
                 raise typer.BadParameter(f"No sequences for query: '{query}'")