From 681f156ae4a7ef8154ac724c17124b46da80a7c1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 1 Sep 2026 10:05:11 -0400 Subject: [PATCH 01/20] feat(orchestration): the keep list, and the script that acts on it persist_exp.py copies one exposure's named PSF products off /scratch onto the persistent root and records what went, with sizes. The threat it answers is the 60-day purge, not clean_exposure: run_dir is scratch and products_dir is /project, so the only way a per-exposure product outlives its campaign is to leave the filesystem. Exempting files from reclamation would not have done it. The search is recursive beneath the PSF chain's four module output dirs, because setools writes into mask/, rand_split/, new_cat/, plot/ and stat/ rather than flat -- so the config's patterns stay plain file names and the layout stays ours. A pattern that matches nothing is a recorded warning (setools rejects sparse CCDs); nothing matching at all is a failure, since a green manifest over an empty copy is what would let reclamation delete an unsaved exposure. config.yaml's persist_exp: defaults to validation_psf-*.fits -- the psfex_interp VALIDATION catalogue, the rho/tau statistics input, the minimum. The opt-in candidates are documented there with what each buys; sizes are still to be measured. PSFEx residuals and XML are not candidates as the chain stands: the committed default.psfex sets CHECKIMAGE_TYPE NONE and WRITE_XML N. Co-Authored-By: Claude Opus 5 --- workflow/config.yaml | 59 ++++++++++++ workflow/scripts/persist_exp.py | 156 ++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 workflow/scripts/persist_exp.py diff --git a/workflow/config.yaml b/workflow/config.yaml index 63b29413c..4f85b24e3 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -68,6 +68,65 @@ outputs: # would otherwise have to rebuild from tile headers. index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite +# Per-exposure PSF products to COPY onto the persistent root before the scratch +# store goes (`exp_persist`, exposure.smk). A list of plain file-name globs, +# matched recursively under the PSF chain's four module output dirs +# (/exp///output/run_sp_exp_SxSePsfPi/*/output/ — +# sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner). +# Matches land flat in /exp///psf/. +# +# WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat. +# run_dir is /scratch and is PURGED on a 60-day window whether or not +# clean_exposure ever ran; products_dir is /project, backed up and not purged. +# The only way a per-exposure product outlives its campaign is to leave the +# filesystem. (Ordering is free: clean_exposure takes the exp_persist manifest +# as an input, so a store is never reclaimed before its keepers are written.) +# +# EDITING THIS LIST IS CHEAP. It rides on exp_persist's `params`, so a change +# reruns the copy (seconds) and NOT exp_psf (four hours per exposure). That +# separation is the whole reason exp_persist is a rule of its own. +# +# The default is the minimum: the psfex_interp VALIDATION catalogue, one per +# CCD, which is the input to the rho/tau statistics. Without it the PSF +# diagnostics cannot be recomputed after a purge without rebuilding the exposure +# chain from VOS. +# +# OPT-IN CANDIDATES, and what each buys (sizes to be measured): +# star_split_ratio_80-*.fits setools' 80% TRAINING star sample, the set PSFEx +# actually fitted. Refit or perturb the model. +# +# star_split_ratio_20-*.fits the 20% VALIDATION sample — the positions the +# validation_psf rows correspond to, with the +# measured star shapes beside them. +# star_selection-*.fits the PRE-SPLIT selection (setools writes it under +# mask/). The only file that can answer "which +# stars were rejected, and why" — the split +# samples have already lost the rejects. +# +# star_stat-*.txt setools' per-CCD STAT block (star counts, +# stars/deg^2, FWHM mode and cuts, under stat/): +# the selection's summary without its catalogue. +# +# *.psf the PSFEx model itself. Keeping it means the PSF +# can be re-interpolated at ANY position later +# without rebuilding the exposure chain — the +# single most capability-adding entry here. +# +# psfex_cat*.cat PSFEx's own output catalogue (FITS_LDAC). +# +# PSFEx residual/check images and its XML diagnostics are NOT candidates as the +# chain stands: the committed default.psfex sets CHECKIMAGE_TYPE NONE and +# WRITE_XML N, so nothing is emitted to match. They are a config change first, +# a pattern second. +# +# NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the copy +# then lands beside the store on the same filesystem and buys nothing, and the +# manifest sits in the exposure's own manifests/ dir, which clean_exposure +# deletes wholesale — so a one-root run re-persists after every reclamation. +# Harmless, and exactly the pre-D5 behaviour a one-root run asks for. +persist_exp: + - validation_psf-*.fits + # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py new file mode 100644 index 000000000..1a17c40cb --- /dev/null +++ b/workflow/scripts/persist_exp.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Copy ONE exposure's keepable PSF products off scratch, and record what went. + +Run as the shell of the in-DAG ``exp_persist`` rule, never by hand. + +WHY A COPY AND NOT AN EXEMPTION FROM CLEANUP. The obvious alternative — teach +``clean_exposure`` to spare these files — does not work, because reclamation is +not what threatens them. The exposure store lives on ``run_dir``, which is +/scratch: a 60-day purge takes everything there whether or not this workflow +ever cleaned it. ``products_dir`` is /project, backed up and not purged. So the +only way a per-exposure product outlives its campaign is to LEAVE THE +FILESYSTEM, and that is a copy. Reclamation ordering then falls out for free: +``clean_exposure`` takes this rule's manifest as an input, so the store is never +deleted before its keepers have been written elsewhere. + +WHY A SEPARATE RULE AND NOT A ``cp`` APPENDED TO ``exp_psf``. The list of what +to keep is a decision that will be revisited — rho statistics want one file +today, a residual study may want three tomorrow — and ``exp_psf`` is four hours +per exposure. The list rides on this rule's ``params``, so editing it makes +snakemake rerun THIS rule (seconds of cp) and leaves the PSF chain alone. Folded +into ``exp_psf``, the same edit would re-derive every PSF model in the campaign. + +WHAT IT SEARCHES. ``/output/run_sp_exp_SxSePsfPi/*/output/`` — the four +module output dirs of the PSF config (sextractor, setools, psfex, psfex_interp) +— RECURSIVELY. The recursion is not laziness: setools does not write flat, it +writes into ``mask/``, ``rand_split/``, ``new_cat/``, ``plot/`` and ``stat/`` +beneath its own output dir, so a caller who wrote ``star_split_ratio_80-*.fits`` +meaning "the training star sample" would match nothing under a non-recursive +glob. Patterns are therefore plain FILE names and the layout is ours to know, +not the config author's. + +ZERO MATCHES FOR ONE PATTERN IS A WARNING, NOT A FAILURE. setools rejects sparse +CCDs (~0.2% attrition, tolerated by exp_psf's own count floor), so per-CCD +counts are not fixed, and a pattern naming an optional diagnostic may legitimately +find nothing. ZERO FILES IN TOTAL IS A FAILURE: it means the store was not what +we think it is, and writing a green manifest over that would let +``clean_exposure`` delete an exposure whose products were never saved. + +The destination is FLAT — one ``psf/`` dir per exposure, no module subtree — +because the module a file came from is already in its name and the consumer +(rho/tau statistics) globs the directory. A name collision between two modules +is therefore a hard error rather than a silent overwrite; nothing in the current +config can produce one, and if a future one can we want to hear about it. + +The manifest is the rule's ONLY declared output, and it lives on the persistent +root beside the copies (``/exp///manifests/``), NOT in +the exposure's scratch ``manifests/`` dir which ``clean_exposure`` deletes +wholesale. It is deliberately NOT a ``directory()`` output: what was copied, and +how big each file was, is provenance we want written down, and a directory +output attests only that some directory exists. + +It carries no timestamp and is written tmp-then-``cmp``-then-``mv`` (the pattern +``exp_star_cat`` uses), so a rerun that copies the same files leaves the mtime +alone — mtime is a rerun trigger, and an unconditional rewrite would make every +downstream ``clean_exposure`` look out of date once per invocation. +""" + +import argparse +import filecmp +import json +import shutil +import sys +from pathlib import Path + +# The PSF chain's run dir (RUN_NAME in config_exp_psfex.ini). Hardcoded rather +# than passed: this rule persists the PSF stage's products and nothing else, and +# a knob here would be a knob for "persist some other stage", which is a +# different rule. +RUN_NAME = "run_sp_exp_SxSePsfPi" + + +def collect(exp_dir: Path, patterns: list) -> tuple: + """Matched files per pattern, in a stable order, plus the empty patterns.""" + root = exp_dir / "output" / RUN_NAME + found, empty = {}, [] + for pat in patterns: + # One glob per module output dir, recursive beneath it (see the module + # docstring on setools' subdirectories). sorted() over the union keeps + # the manifest byte-stable across filesystem readdir order. + hits = sorted({p for mod in sorted(root.glob("*/output")) + for p in mod.rglob(pat) if p.is_file()}) + if hits: + found[pat] = hits + else: + empty.append(pat) + return found, empty + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path, + help="the exposure's scratch store") + p.add_argument("--exp", required=True) + p.add_argument("--dest", required=True, type=Path, + help="/exp///psf") + p.add_argument("--manifest", required=True, type=Path) + p.add_argument("--pattern", action="append", default=[], + help="repeatable; a plain file-name glob") + args = p.parse_args() + + if not args.pattern: + sys.exit("persist_exp: no --pattern given (config persist_exp is empty)") + + found, empty = collect(args.exp_dir, args.pattern) + if not found: + sys.exit(f"persist_exp: {args.exp}: no file matched any of " + f"{args.pattern} under {args.exp_dir}/output/{RUN_NAME}") + + args.dest.mkdir(parents=True, exist_ok=True) + seen, files = {}, [] + for pat, hits in found.items(): + for src in hits: + dst = args.dest / src.name + if src.name in seen: + sys.exit(f"persist_exp: {args.exp}: two source files are both " + f"named {src.name} ({seen[src.name]} and {src}); the " + f"destination is flat, so this would silently overwrite") + seen[src.name] = src + # Skip a byte-identical copy: it is not just an I/O saving, it keeps + # the destination's mtimes still for anything downstream that reads + # them. + if not (dst.exists() and filecmp.cmp(src, dst, shallow=False)): + tmp = dst.with_name(dst.name + ".tmp") + shutil.copy2(src, tmp) + tmp.replace(dst) # atomic: no half-copied product + files.append({"name": src.name, "pattern": pat, + "src": str(src), "bytes": dst.stat().st_size}) + + body = { + "stage": "exp_persist", "level": "exp", "unit": args.exp, + "status": "complete", + "dest": str(args.dest), + "patterns": list(args.pattern), + # The warning the docstring argues for: named patterns that matched + # nothing. Present as a key even when empty, so a reader never has to + # wonder whether an old manifest predates the field. + "patterns_unmatched": empty, + "n_files": len(files), + "bytes": sum(f["bytes"] for f in files), + "files": sorted(files, key=lambda f: f["name"]), + } + args.manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = args.manifest.with_name(args.manifest.name + ".tmp") + tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + if args.manifest.exists() and filecmp.cmp(tmp, args.manifest, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(args.manifest) + + warn = f" ({len(empty)} pattern(s) matched nothing: {empty})" if empty else "" + print(f"[persist_exp] {args.exp}: {len(files)} file(s), " + f"{body['bytes'] / 1e6:.1f} MB -> {args.dest}{warn}") + + +if __name__ == "__main__": + main() From de243a8f47ebfbc7d84854cdbb43880f1c45f066 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 1 Sep 2026 10:05:25 -0400 Subject: [PATCH 02/20] feat(orchestration): exp_persist, between exp_psf and clean_exposure One rule per exposure, output = ONE manifest on the persistent root at /exp///manifests/exp_persist.json. Not a directory() output: what we want written down is which files were copied and how big each was, and a directory attests only that a directory exists. Byte-stable, so a no-op rerun does not move an mtime clean_exposure reads. Its own rule rather than a cp on the end of exp_psf, and that is the whole point: the keep list rides on params, so adding a pattern reruns seconds of copying instead of four hours of PSF fitting per exposure. A localrule, by the arithmetic that made exp_star_cat one -- a few MB of cp, ~20k of them at DR6 scale, each shorter than the scheduling latency that would submit it. The mid-chain grouping constraint does not bite: its neighbours are exp_psf (too heavy to fuse) and clean_exposure (local already). clean_exposure gains the manifest as an input, so a store is never reclaimed before its keepers have left scratch -- conditional only on there being a keep list, since "keep nothing" must not become a dependency on a rule that would fail for having nothing to copy. rule all requests the persist manifests DIRECTLY, not only through clean_exposure: the purge takes the store whether or not clean: is on, so hanging the copy off reclamation alone would lose everything in a clean:false campaign. Cleaned exposures are excluded -- their exp_psf manifest is gone, so asking would rebuild the chain from VOS, and a tombstone already means the copy happened. Co-Authored-By: Claude Opus 5 --- workflow/Snakefile | 58 ++++++++++++++++++++++++++++- workflow/rules/exposure.smk | 73 ++++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index bbdfeefb6..a5fa5c54a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -263,6 +263,7 @@ EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") # The persistent root mirrors the scratch one, shard for shard, so the two trees # read as the same campaign seen from two filesystems. PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") +PROD_EXP_DIR = str(PRODUCTS_DIR / "exp" / "{shard}" / "{exp}") def tile_dir(tile): return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" @@ -276,6 +277,18 @@ def tile_manifest(tile, stage): def exp_manifest(exp, stage): return f"{exp_dir(exp)}/manifests/{stage}.json" +def prod_exp_dir(exp): + """The exposure's dir on the PERSISTENT root — where exp_persist writes. + + Sharded identically to the scratch one, so the two trees read as the same + campaign seen from two filesystems, exposure side as well as tile side.""" + return f"{PRODUCTS_DIR}/exp/{exp[:2]}/{exp}" + +def prod_exp_manifest(exp, stage): + """A manifest that must SURVIVE reclamation, so it is not in the exposure's + scratch manifests/ dir (clean_exposure deletes that wholesale).""" + return f"{prod_exp_dir(exp)}/manifests/{stage}.json" + def forest_dir(tile): return f"{tile_dir(tile)}/exp_forest" @@ -314,6 +327,7 @@ SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") +PERSIST_HASH = script_hash("persist_exp.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -431,6 +445,40 @@ def clean_targets(): out.append(tombstone(exp)) return sorted(out) +# --- persisted exposure products (D5) -------------------------------------- +# The keep list is config, not a rule input, and it is READ HERE so that exactly +# one place converts it into the form the rule carries. An empty list is a +# deliberate "keep nothing" and produces no jobs at all. +PERSIST_EXP = list(config.get("persist_exp") or []) + + +def persist_targets(): + """Which exposures this invocation must copy PSF products off scratch for. + + `rule all` requests these DIRECTLY rather than reaching them only through + clean_exposure. Persistence and reclamation are different concerns — the + /scratch purge takes the store whether or not `clean:` is on — and hanging + the copy off the clean rule alone would mean a campaign run with clean:false + persists nothing and loses everything at the purge. + + Scope is the ready tiles' exposures, which `all` already builds through the + tile chain, so nothing new is pulled into the DAG by asking. + + EXCEPT A CLEANED EXPOSURE. Its exp_psf manifest was deleted by + clean_exposure, so requesting its persist manifest would make the DAG + rebuild the whole exposure chain from VOS — the avalanche tile.smk's + reclaimed-edge cut exists to prevent, arriving through a new target instead. + A tombstone means the copy already happened (clean_exposure cannot run + before exp_persist), so there is nothing to ask for. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. + """ + if not PERSIST_EXP or not workflow.is_main_process: + return [] + exps = {e for t in TILES_READY for e in tile_exposures(t)} + return sorted(prod_exp_manifest(e, "exp_persist") for e in exps + if not Path(tombstone(e)).exists()) + # --- tile reclamation (D5) -------------------------------------------------- # A separate flag from `clean:` (config.yaml carries the full # argument): exposure reclamation costs nothing but a rebuild if a tile is @@ -601,11 +649,19 @@ include: "rules/tile.smk" # localrule would: a local job cannot be fused into a submitted group. The old # star-catalogue rules were exactly that, and they are gone with the internal # mask generation.) -localrules: all, prepare_all_tiles, clean_exposure, clean_tile +# +# exp_persist joins them for the same arithmetic — one tar of a few MB per +# exposure, ~20k of them at DR6 scale, each far shorter than the scheduling +# latency that would submit it (exposure.smk argues the placement in full). It +# sits mid-chain between exp_psf and clean_exposure, but both of those are +# outside every group already (exp_psf is heavy, clean_exposure is local), so it +# adds no new grouping constraint. +localrules: all, prepare_all_tiles, clean_exposure, clean_tile, exp_persist rule all: input: [final_cat(t) for t in TILES_READY], + persist_targets(), clean_targets(), clean_tile_targets(), diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 4e1ef45a4..0bf80b9c6 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,6 +1,6 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -> exp_psf + exp_get_images -> exp_split -> exp_psf -> exp_persist Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a @@ -17,6 +17,13 @@ the per-band ``MASK_`` columns on the tile side. Neither needs a rule, a star catalogue, or a network fetch — hence no ``star_catalogue`` / ``exp_star_cat`` here, and no ``exp_mask``. +``exp_persist`` is the one rule here that writes to the PERSISTENT root: it +packs the PSF products named by `persist_exp:` into one tar per exposure off +/scratch before the purge (or clean_exposure) can take them. It is a separate +rule from exp_psf precisely so that editing that list costs a re-pack and not a +four-hour refit; the full +argument is in workflow/scripts/persist_exp.py. + NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, not over one invocation — reclamation here is clean_exposure's job (S5), driven @@ -106,6 +113,59 @@ rule exp_psf: sp_shell("exp_psf", f"config_exp_{PSF_MODEL}.ini") +# --- persistence (D5) ------------------------------------------------------- +# The counterpart of reclamation, and it must come first in the DAG: this copies +# the exposure's keepable PSF products onto the persistent root, and +# clean_exposure below takes its manifest as an input so the store is never +# reclaimed before the keepers have left /scratch. The purge would take them +# anyway — that, not clean_exposure, is what this rule exists for +# (persist_exp.py's docstring argues both halves, and config.yaml's +# `persist_exp:` block carries the keep list and its candidates). +# +# A LOCALRULE (declared in the Snakefile), by exactly the arithmetic that made +# exp_star_cat one: the body is `cp` of a few MB from one shared filesystem to +# another, seconds of work, and one sbatch per exposure would be ~20k +# submissions at DR6 scale for jobs shorter than the scheduling latency. The +# grouping constraint that binds mid-chain localrules (this file's docstring) +# does not bite here: exp_persist's only neighbours are exp_psf, which is too +# heavy to ever fuse, and clean_exposure, which is local itself. +# +# ONE DECLARED OUTPUT, AND IT IS A MANIFEST, NOT A directory(). The copies are +# not declared: a directory output would attest that a directory exists, where +# what we want written down is WHICH files were copied and how big each was — +# the provenance a rho-statistics run months from now needs in order to know +# what it is reading. The manifest is byte-stable, so a no-op rerun does not +# move its mtime and does not make clean_exposure look out of date. +# +# THE KEEP LIST RIDES ON params. That is the entire reason this is not three +# lines of cp appended to exp_psf's shell: `params` is a rerun trigger, so +# adding a pattern reruns the copy and leaves the PSF chain alone. +rule exp_persist: + input: + rules.exp_psf.output.manifest + output: + manifest = f"{PROD_EXP_DIR}/manifests/exp_persist.json" + # No `log:`: the script's only failure modes are "nothing matched" and a + # name collision, both of which it reports on stderr and neither of which + # has a per-CCD verdict worth a completeness record. + params: + patterns = " ".join(f"--pattern '{p}'" for p in PERSIST_EXP), + exp_dir = lambda wc: exp_dir(wc.exp), + dest = lambda wc: f"{prod_exp_dir(wc.exp)}/psf", + script_hash = PERSIST_HASH + threads: 1 + retries: 2 + resources: + mem_mb = 2000, + runtime = 10 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/persist_exp.py" + " --exp-dir '{params.exp_dir}' --exp {wildcards.exp}" + " --dest '{params.dest}' --manifest {output.manifest}" + " {params.patterns}" + + # --- reclamation (D5) ------------------------------------------------------- # The one exception to "no reclamation in this file": clean_exposure OWNS # exposure-level deletion, and it is a real job, not temp() bookkeeping, because @@ -137,7 +197,16 @@ rule clean_exposure: # spatial neighbours. In-scope consumers keep their edge: they may run in # this DAG, so the clean must be ordered after them. lambda wc: [tile_manifest(t, "tile_vignets") - for t in clean_consumers(wc.exp) if t in READY_SET] + for t in clean_consumers(wc.exp) if t in READY_SET], + # The keepers must be off /scratch before the store goes. Unlike the + # consumer edges above, this edge does not depend on scope: it is the + # same exposure's own rule, so it drags nothing into the DAG that this + # exposure's chain did not already put there. It is conditional only on + # there being a keep list at all — with `persist_exp:` empty, "keep + # nothing" is a coherent instruction and must not become a dependency on + # a rule that would fail for having nothing to copy. + lambda wc: ([prod_exp_manifest(wc.exp, "exp_persist")] + if PERSIST_EXP else []) output: tombstone = f"{EXP_DIR}/cleaned.json" params: From 4cf639f75a7295160fbd630aad7fc7251b57f366 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 1 Sep 2026 10:05:25 -0400 Subject: [PATCH 03/20] docs(orchestration): exp_persist in the rule list, and why the report omits it run_report disk-scans the scratch run_dir, and exp_persist's manifest is the one exposure manifest that lives on products_dir instead -- the placement that makes it survive clean_exposure. Listed in EXP_STAGES it would read as "not run" for every exposure in the campaign, so it is deliberately absent, with the reason on the line. Co-Authored-By: Claude Opus 5 --- workflow/README.md | 17 ++++++++++++++++- workflow/scripts/run_report.py | 7 +++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/workflow/README.md b/workflow/README.md index 55611cedc..f4e00c2ea 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -156,7 +156,7 @@ workflow/ bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures - exposure.smk per-exposure: get_images, split, psf (no temp()) + exposure.smk per-exposure: get_images, split, psf, persist (no temp()) tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count check) @@ -165,6 +165,7 @@ workflow/ completeness.py the ported count table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) container.py image layers + the resolution order behind `sp container` (stdlib-only) + persist_exp.py ONE exposure's keepable PSF products -> products_dir (the exp_persist rule) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` @@ -240,6 +241,20 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee trigger reads that cut as a reason to rerun the very tiles it protects. Know the consequence — `--forcerun` on a tile whose `final_cat` exists will not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **PSF products leave scratch before the purge does.** `exp_persist` copies + the files named by `persist_exp:` in `config.yaml` (default: the psfex_interp + `validation_psf-*.fits`, the rho/tau statistics input) from the exposure's + scratch store into `/exp///psf/`, and writes ONE + manifest beside them recording the patterns, the files and their sizes. The + threat it answers is the /scratch purge, not `clean_exposure` — the store goes + in 60 days whether or not the workflow reclaimed it — so it runs even with + `clean: false`, requested directly by `rule all`. `clean_exposure` takes its + manifest as an input, so reclamation can never overtake the copy. It is a + rule of its own rather than a `cp` on the end of `exp_psf` because the keep + list rides on `params`: adding a pattern reruns seconds of copying, not four + hours of PSF fitting per exposure. A pattern that matches nothing is a + recorded warning (setools rejects sparse CCDs); matching nothing at all is a + failure. A `localrule`, like `exp_star_cat` and for the same arithmetic. - **A dead tile can be told to stop pinning exposures.** An exposure is cleanable only once every consuming tile has its vignets, so one permanently-failed tile holds its ~80 exposures for the life of the diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 24c07670b..11cdcffe4 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -61,6 +61,13 @@ "tile_ngmix", "tile_merge_cats", "tile_make_cat"] EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] +# exp_persist is DELIBERATELY NOT in that list. This report disk-scans the +# scratch run_dir, and exp_persist's manifest is the one exposure manifest that +# lives on products_dir instead — that placement is what makes it survive +# clean_exposure. Listed here it would read as "not run" for every exposure in +# the campaign. Reporting on the persisted products means scanning the second +# root, which is a report this one does not yet do. + # The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names # the mechanism that owns each). Their presence is therefore NOT evidence that a # tile's chain was rebuilt, which absorb_tombstones needs to know From 1fa64551675e3c6a6a4b88a6776ba3c1c27e95e6 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 2 Sep 2026 20:16:01 -0400 Subject: [PATCH 04/20] feat(orchestration): exp_persist packs one tar per exposure, not loose copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inodes, not bytes, bind on /project (~1 M-file group quota): smk-m2 measured ~200 loose files per exposure with all candidates on — 25k for 64 tiles, ~2 M at DR6 scale, for 7 GB. persist_exp.py now writes /exp// /psf/.tar (uncompressed, flat members, deterministic: ownership zeroed, sorted, tmp-cmp-mv so a no-op rerun keeps the mtime) and the manifest lists every member. Manifest path, rule wiring and params are unchanged. config.yaml's candidate table carries the smk-m2 per-exposure sizes; psfex_cat and star_stat are marked unmeasured (no live store held them). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015qtLUV3bVLPV5p6un7aTFR --- workflow/README.md | 12 ++--- workflow/Snakefile | 2 +- workflow/config.yaml | 57 ++++++++++++++--------- workflow/rules/exposure.smk | 18 ++++---- workflow/scripts/persist_exp.py | 80 ++++++++++++++++++++++----------- 5 files changed, 107 insertions(+), 62 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index f4e00c2ea..f2ce81024 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -165,7 +165,7 @@ workflow/ completeness.py the ported count table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) container.py image layers + the resolution order behind `sp container` (stdlib-only) - persist_exp.py ONE exposure's keepable PSF products -> products_dir (the exp_persist rule) + persist_exp.py ONE exposure's keepable PSF products -> one tar on products_dir (the exp_persist rule) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` @@ -241,17 +241,19 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee trigger reads that cut as a reason to rerun the very tiles it protects. Know the consequence — `--forcerun` on a tile whose `final_cat` exists will not rebuild its reclaimed exposures. Delete the `final_cat` first. -- **PSF products leave scratch before the purge does.** `exp_persist` copies +- **PSF products leave scratch before the purge does.** `exp_persist` packs the files named by `persist_exp:` in `config.yaml` (default: the psfex_interp `validation_psf-*.fits`, the rho/tau statistics input) from the exposure's - scratch store into `/exp///psf/`, and writes ONE - manifest beside them recording the patterns, the files and their sizes. The + scratch store into ONE uncompressed tar, + `/exp///psf/.tar` (inodes, not bytes, bind + on /project), and writes ONE manifest beside it recording the patterns, the + members and their sizes. The threat it answers is the /scratch purge, not `clean_exposure` — the store goes in 60 days whether or not the workflow reclaimed it — so it runs even with `clean: false`, requested directly by `rule all`. `clean_exposure` takes its manifest as an input, so reclamation can never overtake the copy. It is a rule of its own rather than a `cp` on the end of `exp_psf` because the keep - list rides on `params`: adding a pattern reruns seconds of copying, not four + list rides on `params`: adding a pattern reruns seconds of packing, not four hours of PSF fitting per exposure. A pattern that matches nothing is a recorded warning (setools rejects sparse CCDs); matching nothing at all is a failure. A `localrule`, like `exp_star_cat` and for the same arithmetic. diff --git a/workflow/Snakefile b/workflow/Snakefile index a5fa5c54a..fbba2c851 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -453,7 +453,7 @@ PERSIST_EXP = list(config.get("persist_exp") or []) def persist_targets(): - """Which exposures this invocation must copy PSF products off scratch for. + """Which exposures this invocation must pack PSF products off scratch for. `rule all` requests these DIRECTLY rather than reaching them only through clean_exposure. Persistence and reclamation are different concerns — the diff --git a/workflow/config.yaml b/workflow/config.yaml index 4f85b24e3..ce434314c 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -68,12 +68,17 @@ outputs: # would otherwise have to rebuild from tile headers. index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite -# Per-exposure PSF products to COPY onto the persistent root before the scratch +# Per-exposure PSF products to carry onto the persistent root before the scratch # store goes (`exp_persist`, exposure.smk). A list of plain file-name globs, # matched recursively under the PSF chain's four module output dirs # (/exp///output/run_sp_exp_SxSePsfPi/*/output/ — # sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner). -# Matches land flat in /exp///psf/. +# Matches are packed, flat, into ONE uncompressed tar per exposure: +# /exp///psf/.tar, with a manifest listing the +# members beside it. One tar rather than loose copies because inodes, not bytes, +# bind on /project (~1 M-file group quota; loose copies would be ~200 files per +# exposure, ~2 M at DR6 scale). FITS members read straight from the tar: +# fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())). # # WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat. # run_dir is /scratch and is PURGED on a 60-day window whether or not @@ -83,7 +88,7 @@ outputs: # as an input, so a store is never reclaimed before its keepers are written.) # # EDITING THIS LIST IS CHEAP. It rides on exp_persist's `params`, so a change -# reruns the copy (seconds) and NOT exp_psf (four hours per exposure). That +# reruns the packing (seconds) and NOT exp_psf (four hours per exposure). That # separation is the whole reason exp_persist is a rule of its own. # # The default is the minimum: the psfex_interp VALIDATION catalogue, one per @@ -91,35 +96,43 @@ outputs: # diagnostics cannot be recomputed after a purge without rebuilding the exposure # chain from VOS. # -# OPT-IN CANDIDATES, and what each buys (sizes to be measured): -# star_split_ratio_80-*.fits setools' 80% TRAINING star sample, the set PSFEx -# actually fitted. Refit or perturb the model. -# -# star_split_ratio_20-*.fits the 20% VALIDATION sample — the positions the -# validation_psf rows correspond to, with the -# measured star shapes beside them. +# OPT-IN CANDIDATES, and what each buys. Sizes are per exposure (40 CCDs), +# measured on smk-m2 (127 exposures, 64 tiles); a 64-tile campaign with all of +# the measured ones on came to 7.2 GB: +# validation_psf-*.fits (the default) 2.0 MB +# *.psf the PSFEx model itself. Keeping it means the PSF +# can be re-interpolated at ANY position later +# without rebuilding the exposure chain — the +# single most capability-adding entry here. +# 2.8 MB +# psfex_cat-*.cat PSFEx's own output catalogue (FITS_LDAC): the +# per-star FLAGS_PSF / CHI2_PSF, i.e. WHICH stars +# outlier rejection clipped. Not recoverable from +# anything else (the .psf header keeps only the +# LOADED/ACCEPTED counts). unmeasured # star_selection-*.fits the PRE-SPLIT selection (setools writes it under # mask/). The only file that can answer "which -# stars were rejected, and why" — the split -# samples have already lost the rejects. -# +# stars were rejected by the selection cuts, and +# why" — the split samples have already lost the +# rejects. 24.5 MB +# star_split_ratio_80-*.fits setools' 80% TRAINING star sample, the set PSFEx +# actually fitted. Rows duplicate star_selection. +# 19.9 MB +# star_split_ratio_20-*.fits the 20% VALIDATION sample — the positions the +# validation_psf rows correspond to. Rows +# duplicate star_selection. 7.1 MB # star_stat-*.txt setools' per-CCD STAT block (star counts, # stars/deg^2, FWHM mode and cuts, under stat/): # the selection's summary without its catalogue. -# -# *.psf the PSFEx model itself. Keeping it means the PSF -# can be re-interpolated at ANY position later -# without rebuilding the exposure chain — the -# single most capability-adding entry here. -# -# psfex_cat*.cat PSFEx's own output catalogue (FITS_LDAC). -# +# unmeasured +# A production keep list is `validation_psf` + `*.psf` + `psfex_cat` (~5 MB per +# exposure); the star_split files are only worth it if star_selection is off. # PSFEx residual/check images and its XML diagnostics are NOT candidates as the # chain stands: the committed default.psfex sets CHECKIMAGE_TYPE NONE and # WRITE_XML N, so nothing is emitted to match. They are a config change first, # a pattern second. # -# NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the copy +# NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the tar # then lands beside the store on the same filesystem and buys nothing, and the # manifest sits in the exposure's own manifests/ dir, which clean_exposure # deletes wholesale — so a one-root run re-persists after every reclamation. diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 0bf80b9c6..47bb4ae26 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -114,8 +114,8 @@ rule exp_psf: # --- persistence (D5) ------------------------------------------------------- -# The counterpart of reclamation, and it must come first in the DAG: this copies -# the exposure's keepable PSF products onto the persistent root, and +# The counterpart of reclamation, and it must come first in the DAG: this packs +# the exposure's keepable PSF products into one tar on the persistent root, and # clean_exposure below takes its manifest as an input so the store is never # reclaimed before the keepers have left /scratch. The purge would take them # anyway — that, not clean_exposure, is what this rule exists for @@ -123,23 +123,23 @@ rule exp_psf: # `persist_exp:` block carries the keep list and its candidates). # # A LOCALRULE (declared in the Snakefile), by exactly the arithmetic that made -# exp_star_cat one: the body is `cp` of a few MB from one shared filesystem to -# another, seconds of work, and one sbatch per exposure would be ~20k +# exp_star_cat one: the body is a `tar` of a few MB from one shared filesystem +# to another, seconds of work, and one sbatch per exposure would be ~20k # submissions at DR6 scale for jobs shorter than the scheduling latency. The # grouping constraint that binds mid-chain localrules (this file's docstring) # does not bite here: exp_persist's only neighbours are exp_psf, which is too # heavy to ever fuse, and clean_exposure, which is local itself. # -# ONE DECLARED OUTPUT, AND IT IS A MANIFEST, NOT A directory(). The copies are -# not declared: a directory output would attest that a directory exists, where -# what we want written down is WHICH files were copied and how big each was — +# ONE DECLARED OUTPUT, AND IT IS A MANIFEST, NOT THE TAR OR A directory(). The +# tar is not declared: a directory output would attest that a directory exists, +# where what we want written down is WHICH files were packed and how big each was — # the provenance a rho-statistics run months from now needs in order to know # what it is reading. The manifest is byte-stable, so a no-op rerun does not # move its mtime and does not make clean_exposure look out of date. # # THE KEEP LIST RIDES ON params. That is the entire reason this is not three -# lines of cp appended to exp_psf's shell: `params` is a rerun trigger, so -# adding a pattern reruns the copy and leaves the PSF chain alone. +# lines of tar appended to exp_psf's shell: `params` is a rerun trigger, so +# adding a pattern reruns the packing and leaves the PSF chain alone. rule exp_persist: input: rules.exp_psf.output.manifest diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index 1a17c40cb..a23ab150d 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Copy ONE exposure's keepable PSF products off scratch, and record what went. +"""Pack ONE exposure's keepable PSF products into a tar off scratch, and record what went. Run as the shell of the in-DAG ``exp_persist`` rule, never by hand. @@ -36,21 +36,39 @@ we think it is, and writing a green manifest over that would let ``clean_exposure`` delete an exposure whose products were never saved. -The destination is FLAT — one ``psf/`` dir per exposure, no module subtree — -because the module a file came from is already in its name and the consumer -(rho/tau statistics) globs the directory. A name collision between two modules -is therefore a hard error rather than a silent overwrite; nothing in the current -config can produce one, and if a future one can we want to hear about it. +The manifest lists every member (name, pattern, source path, bytes), so a reader +knows what the tar holds without opening it. + +ONE UNCOMPRESSED TAR PER EXPOSURE, ``/.tar``, NOT LOOSE COPIES. +Inodes, not bytes, are what bind on /project: the group quota is ~1 M files, +and loose per-CCD copies are ~200 per exposure with all candidates on — ~25k for +a 64-tile campaign, ~2 M at DR6 scale, against ~7 GB of bytes. A tar collapses +that to one inode per exposure and costs nothing to read: FITS members go +``tarfile.open(t).extractfile(m).read()`` -> ``fits.open(io.BytesIO(...))``, +which is why a tar rather than a multi-HDU FITS bundle (the keep list mixes +FITS, ``.psf`` and ``.txt``; a FITS container could not hold the last two). +Uncompressed because FITS barely compresses and a plain tar is seekable. + +Members are FLAT — file name only, no module subtree — because the module a +file came from is already in its name and the consumer globs member names. A +name collision between two modules is therefore a hard error rather than a +silent overwrite; nothing in the current config can produce one, and if a +future one can we want to hear about it. + +The tar is written DETERMINISTICALLY (ownership zeroed, members in sorted +order, source mtimes kept), tmp-then-``cmp``-then-``mv``: a rerun over an +unchanged store produces a byte-identical tar and leaves the existing one's +mtime alone. The manifest is the rule's ONLY declared output, and it lives on the persistent -root beside the copies (``/exp///manifests/``), NOT in +root beside the tar (``/exp///manifests/``, beside the tar's ``psf/``), NOT in the exposure's scratch ``manifests/`` dir which ``clean_exposure`` deletes wholesale. It is deliberately NOT a ``directory()`` output: what was copied, and how big each file was, is provenance we want written down, and a directory output attests only that some directory exists. It carries no timestamp and is written tmp-then-``cmp``-then-``mv`` (the pattern -``exp_star_cat`` uses), so a rerun that copies the same files leaves the mtime +``exp_star_cat`` uses), so a rerun that packs the same files leaves the mtime alone — mtime is a rerun trigger, and an unconditional rewrite would make every downstream ``clean_exposure`` look out of date once per invocation. """ @@ -58,8 +76,8 @@ import argparse import filecmp import json -import shutil import sys +import tarfile from pathlib import Path # The PSF chain's run dir (RUN_NAME in config_exp_psfex.ini). Hardcoded rather @@ -92,7 +110,8 @@ def main() -> None: help="the exposure's scratch store") p.add_argument("--exp", required=True) p.add_argument("--dest", required=True, type=Path, - help="/exp///psf") + help="/exp///psf; the tar is " + "/.tar") p.add_argument("--manifest", required=True, type=Path) p.add_argument("--pattern", action="append", default=[], help="repeatable; a plain file-name glob") @@ -107,29 +126,40 @@ def main() -> None: f"{args.pattern} under {args.exp_dir}/output/{RUN_NAME}") args.dest.mkdir(parents=True, exist_ok=True) + tar_path = args.dest / f"{args.exp}.tar" seen, files = {}, [] for pat, hits in found.items(): for src in hits: - dst = args.dest / src.name if src.name in seen: sys.exit(f"persist_exp: {args.exp}: two source files are both " - f"named {src.name} ({seen[src.name]} and {src}); the " - f"destination is flat, so this would silently overwrite") - seen[src.name] = src - # Skip a byte-identical copy: it is not just an I/O saving, it keeps - # the destination's mtimes still for anything downstream that reads - # them. - if not (dst.exists() and filecmp.cmp(src, dst, shallow=False)): - tmp = dst.with_name(dst.name + ".tmp") - shutil.copy2(src, tmp) - tmp.replace(dst) # atomic: no half-copied product + f"named {src.name} ({seen[src.name][0]} and {src}); tar " + f"members are flat, so this would silently overwrite") + seen[src.name] = (src, pat) files.append({"name": src.name, "pattern": pat, - "src": str(src), "bytes": dst.stat().st_size}) + "src": str(src), "bytes": src.stat().st_size}) + files.sort(key=lambda f: f["name"]) + + def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: + # Ownership is the one thing that would differ between two writes of + # the same files from different accounts/nodes; drop it. mtime stays: + # it is the product's, and it is stable while the store is. + ti.uid = ti.gid = 0 + ti.uname = ti.gname = "" + return ti + + tmp = tar_path.with_name(tar_path.name + ".tmp") + with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: + for f in files: + tf.add(seen[f["name"]][0], arcname=f["name"], filter=anonymous) + if tar_path.exists() and filecmp.cmp(tmp, tar_path, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(tar_path) # atomic: no half-written archive body = { "stage": "exp_persist", "level": "exp", "unit": args.exp, "status": "complete", - "dest": str(args.dest), + "tar": str(tar_path), "patterns": list(args.pattern), # The warning the docstring argues for: named patterns that matched # nothing. Present as a key even when empty, so a reader never has to @@ -137,7 +167,7 @@ def main() -> None: "patterns_unmatched": empty, "n_files": len(files), "bytes": sum(f["bytes"] for f in files), - "files": sorted(files, key=lambda f: f["name"]), + "files": files, } args.manifest.parent.mkdir(parents=True, exist_ok=True) tmp = args.manifest.with_name(args.manifest.name + ".tmp") @@ -149,7 +179,7 @@ def main() -> None: warn = f" ({len(empty)} pattern(s) matched nothing: {empty})" if empty else "" print(f"[persist_exp] {args.exp}: {len(files)} file(s), " - f"{body['bytes'] / 1e6:.1f} MB -> {args.dest}{warn}") + f"{body['bytes'] / 1e6:.1f} MB -> {tar_path}{warn}") if __name__ == "__main__": From a14a2f36658177bec2cfc6c3ef1a41c87ef7d93f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 2 Sep 2026 20:19:22 -0400 Subject: [PATCH 05/20] fix(orchestration): persist_exp never leaves a .tmp behind on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An orphaned tar.tmp on /project is an inode nothing revisits — the leak the tar design exists to avoid. try/finally around both tmp writes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015qtLUV3bVLPV5p6un7aTFR --- workflow/scripts/persist_exp.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index a23ab150d..c13fca604 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -147,14 +147,20 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: ti.uname = ti.gname = "" return ti + # tmp-then-cmp-then-mv, and the tmp NEVER outlives a failure: an orphaned + # .tmp on /project is an inode nothing revisits — the leak this whole tar + # design exists to avoid, one per failed attempt at DR6 scale. tmp = tar_path.with_name(tar_path.name + ".tmp") - with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: - for f in files: - tf.add(seen[f["name"]][0], arcname=f["name"], filter=anonymous) - if tar_path.exists() and filecmp.cmp(tmp, tar_path, shallow=False): - tmp.unlink() # unchanged: leave the mtime alone - else: - tmp.replace(tar_path) # atomic: no half-written archive + try: + with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: + for f in files: + tf.add(seen[f["name"]][0], arcname=f["name"], filter=anonymous) + if tar_path.exists() and filecmp.cmp(tmp, tar_path, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(tar_path) # atomic: no half-written archive + finally: + tmp.unlink(missing_ok=True) body = { "stage": "exp_persist", "level": "exp", "unit": args.exp, @@ -171,11 +177,14 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: } args.manifest.parent.mkdir(parents=True, exist_ok=True) tmp = args.manifest.with_name(args.manifest.name + ".tmp") - tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") - if args.manifest.exists() and filecmp.cmp(tmp, args.manifest, shallow=False): - tmp.unlink() # unchanged: leave the mtime alone - else: - tmp.replace(args.manifest) + try: + tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + if args.manifest.exists() and filecmp.cmp(tmp, args.manifest, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(args.manifest) + finally: + tmp.unlink(missing_ok=True) warn = f" ({len(empty)} pattern(s) matched nothing: {empty})" if empty else "" print(f"[persist_exp] {args.exp}: {len(files)} file(s), " From d91da36ce11731fc65b256aed761b010c8764499 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 08:58:57 -0400 Subject: [PATCH 06/20] docs(orchestration): drop references to the removed star-catalogue rules develop no longer has exp_star_cat / star_catalogue (PR #847); the three comments that cited exp_star_cat as the localrule precedent now cite clean_exposure, which makes the same argument. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- workflow/README.md | 2 +- workflow/rules/exposure.smk | 2 +- workflow/scripts/persist_exp.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index f2ce81024..2b6745eb8 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -256,7 +256,7 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee list rides on `params`: adding a pattern reruns seconds of packing, not four hours of PSF fitting per exposure. A pattern that matches nothing is a recorded warning (setools rejects sparse CCDs); matching nothing at all is a - failure. A `localrule`, like `exp_star_cat` and for the same arithmetic. + failure. A `localrule`, by the same arithmetic as `clean_exposure`. - **A dead tile can be told to stop pinning exposures.** An exposure is cleanable only once every consuming tile has its vignets, so one permanently-failed tile holds its ~80 exposures for the life of the diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 47bb4ae26..40a4044d4 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -123,7 +123,7 @@ rule exp_psf: # `persist_exp:` block carries the keep list and its candidates). # # A LOCALRULE (declared in the Snakefile), by exactly the arithmetic that made -# exp_star_cat one: the body is a `tar` of a few MB from one shared filesystem +# clean_exposure one: the body is a `tar` of a few MB from one shared filesystem # to another, seconds of work, and one sbatch per exposure would be ~20k # submissions at DR6 scale for jobs shorter than the scheduling latency. The # grouping constraint that binds mid-chain localrules (this file's docstring) diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index c13fca604..e78d3e9a4 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -68,7 +68,7 @@ output attests only that some directory exists. It carries no timestamp and is written tmp-then-``cmp``-then-``mv`` (the pattern -``exp_star_cat`` uses), so a rerun that packs the same files leaves the mtime +``clean_exposure`` uses), so a rerun that packs the same files leaves the mtime alone — mtime is a rerun trigger, and an unconditional rewrite would make every downstream ``clean_exposure`` look out of date once per invocation. """ From 626aabacf6f2f1f9013f2bd874366838f4d2e793 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 10:08:36 -0400 Subject: [PATCH 07/20] feat(orchestration): the two campaign-level merges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every rule in this workflow was per unit, and the two products downstream analysis actually opens are per campaign. So a run ended one file short on each side and both merges were a manual pass afterwards. These are the last links of chains the workflow already had. star_cat_merge stacks every exposure's every CCD's validation_psf into one /full_starcat-0000000.fits — the rho/tau statistics input, at the path sp_validation hardcodes. It reads the members straight out of the per-exposure tars exp_persist wrote (tarfile + BytesIO); unpacking ~800k files to merge them would defeat the tar's whole purpose. The stacking is MergeStarCatPSFEX, the class the old merge_starcat_runner called, so the column list keeps exactly one definition. That class gains one thing: an entry may be [fileobj, name] rather than [path], fits.open taking the first element and the CCD_NB regex the last — the same string for the runner's one-element entries. final_cat_merge collects every ready tile's final_cat into /final_cat_.hdf5: one dataset per tile under a group named for the campaign, the final_cat.param columns, an n_tiles attribute. That schema is sp_validation's reader's, so it is fixed. The COLUMN EXTRACTION reuses scripts/python/create_final_cat.py (read_param_file, read_data, copy_data) so the column grammar keeps one definition; the file is written here, because that script's own discovery walks a directory layout this workflow does not have and groups by a unit ShapePipe v2 has dropped. Two places where the reference implementation is not reproducible are pinned down at the call site rather than copied: copy_data leaves every non-requested column as uninitialised memory, and read_param_file's column order varies with the process hash seed. bin/sp now snapshots the repo's scripts/ so the campaign pins that file like everything else it runs. Neither merge puts its input paths in its shell: ~20k of them is an order of magnitude over Linux's 128 KiB MAX_ARG_STRLEN for one argv entry. Each job is handed the two small files the Snakefile itself started from — the tile list and the run index — and DERIVES the same set from them, through readers that now live in build_index.py beside the schema. The rule's params carries that set's fingerprint, which is the rerun trigger, and the equality of the two sides is what makes the trigger mean anything: a glob over products_dir would merge tiles or exposures from an earlier, larger tile list sharing the root, rows no trigger could see. star_cat_merge depends on a live exposure through its exp_persist manifest and on a RECLAIMED one through its tar, which no rule declares and which therefore requires nothing to be built. Requesting a reclaimed exposure's manifest instead rebuilds its whole chain from VOS, and ancient() does not prevent that: measured on smk-g6 with one reclaimed exposure given a manifest by hand, the dry run grew exp_get_images, exp_split, exp_psf and exp_persist jobs. Reclaimed exposures belong in the star catalogue — carrying their PSF products off scratch is what exp_persist is for. Both rebuild rather than append, so the output is a function of its input set: byte-stable on a no-op rerun (tmp-then-cmp-then-mv), rebuilt when a unit is appended. Neither is a localrule — one job over ~20k units is real work. star_cat_merge produces no job, and a parse-time warning rather than a runtime failure, when persist_exp keeps no validation_psf-shaped file. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- .../merge_starcat_package/merge_starcat.py | 21 +- workflow/README.md | 42 +++- workflow/Snakefile | 180 +++++++++++++- workflow/bin/sp | 9 +- workflow/config.yaml | 23 ++ workflow/rules/exposure.smk | 67 +++++ workflow/rules/tile.smk | 60 +++++ workflow/scripts/build_index.py | 39 +++ workflow/scripts/merge_final_cat.py | 222 +++++++++++++++++ workflow/scripts/merge_star_cat.py | 234 ++++++++++++++++++ 10 files changed, 886 insertions(+), 11 deletions(-) create mode 100644 workflow/scripts/merge_final_cat.py create mode 100644 workflow/scripts/merge_star_cat.py diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 7bc0cb76b..1717825f9 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -524,7 +524,15 @@ class MergeStarCatPSFEX(object): Parameters ---------- input_file_list : list - Input files + Input entries. Each entry is a list, as the module runner builds them: + ``[path]`` from the file handler. An entry may also carry a name + alongside an already-open source, ``[fileobj, name]`` — ``fits.open`` + takes the first element and the CCD number is parsed from the LAST, + which is the same string in the one-element case. That is what lets a + caller merge catalogues it never wrote to disk (the Snakemake + workflow's ``star_cat_merge`` reads them out of the per-exposure tars + with ``tarfile`` + ``BytesIO``), without this class learning anything + about where they came from. output_dir : str Output directory w_log : logging.Logger @@ -569,10 +577,15 @@ def process(self): ) for name in self._input_file_list: + # The source to read and the NAME to parse the CCD number out of. + # Identical for a plain [path] entry; different only when the caller + # hands over an open file-like object plus the member name it came + # under (see the class docstring). + source, label = name[0], name[-1] try: - starcat_j = fits.open(name[0], memmap=False, ignore_missing_simple=True) + starcat_j = fits.open(source, memmap=False, ignore_missing_simple=True) except OSError as e: - print(f"Error while opening file '{name[0]}'") + print(f"Error while opening file '{label}'") #raise continue @@ -614,7 +627,7 @@ def process(self): psfex_acc += list(np.zeros_like(data_j["X"])) # CCD number - ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", name[0])[-2]] * len( + ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2]] * len( data_j["RA"] ) diff --git a/workflow/README.md b/workflow/README.md index 2b6745eb8..824eddc73 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -107,7 +107,9 @@ and the run fails if either phase failed. ## The launch code snapshot `sp run` copies the code it is about to launch — `workflow/` (config symlinks -dereferenced), `src/` and the profile — into `/code`, records HEAD +dereferenced), `src/`, the repo's `scripts/` (`final_cat_merge` loads +`scripts/python/create_final_cat.py` by path) and the profile — into +`/code`, records HEAD plus a dirty flag in `/code/snapshot.json`, and runs the campaign entirely out of that copy. It matters because a campaign is not one process: the SLURM executor re-invokes snakemake on every job's node, so jobs re-parse the @@ -156,8 +158,8 @@ workflow/ bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures - exposure.smk per-exposure: get_images, split, psf, persist (no temp()) - tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat + exposure.smk per-exposure: get_images, split, psf, persist (no temp()); campaign star_cat_merge + tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat; campaign final_cat_merge scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count check) build_index.py prepare-phase run_index.sqlite builder (plain script) @@ -166,6 +168,8 @@ workflow/ run_report.py standalone report (NOT a DAG node; run_report hooks call it) container.py image layers + the resolution order behind `sp container` (stdlib-only) persist_exp.py ONE exposure's keepable PSF products -> one tar on products_dir (the exp_persist rule) + merge_star_cat.py ALL exposures' validation_psf, read out of the tars -> full_starcat (the star_cat_merge rule) + merge_final_cat.py ALL tiles' final_cat -> final_cat_.hdf5 (the final_cat_merge rule) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` @@ -257,6 +261,38 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee hours of PSF fitting per exposure. A pattern that matches nothing is a recorded warning (setools rejects sparse CCDs); matching nothing at all is a failure. A `localrule`, by the same arithmetic as `clean_exposure`. +- **The campaign ends in two merged catalogues, and the workflow now makes + both.** Everything above is per unit; the two products downstream analysis + actually opens are per *campaign*, and until these rules existed each was a + manual pass after the run. + `star_cat_merge` stacks every exposure's every CCD's `validation_psf-*.fits` + into one `/full_starcat-0000000.fits` — the rho/tau statistics + input, at the path sp_validation hardcodes. It reads the members straight out + of the per-exposure tars (`tarfile` + `BytesIO`; unpacking ~800k files to + merge them would defeat the tar's whole purpose) and stacks them with + `MergeStarCatPSFEX`, the same class the old `merge_starcat_runner` called, so + the column list has exactly one definition. Its input is the same + `exp_persist` manifest set `rule all` already requests, so it pulls nothing + new into the DAG, and it exists only when `persist_exp:` keeps a + `validation_psf-*.fits`-shaped file — otherwise no job, and a warning at parse + time rather than a failure on a node. + `final_cat_merge` collects every ready tile's `final_cat-.fits` into + `/final_cat_.hdf5`: one dataset per tile under a group + named for the campaign, the `final_cat.param` columns, an `n_tiles` attribute. + That schema is what sp_validation's reader opens, so it is fixed; the column + extraction reuses `scripts/python/create_final_cat.py` while the file is + written here, because that script's own discovery walks a directory layout + this workflow does not have. `campaign:` in `config.yaml` names the group and + defaults to the persistent root's basename. + Both rebuild from the whole persistent root rather than appending, so the + output is a function of its input set: byte-stable on a no-op rerun + (tmp-then-`cmp`-then-`mv`), and rebuilt when a tile or exposure is appended + (the input list's fingerprint rides on `params`). Neither is a `localrule` — + one job over ~20k units is real work — and neither puts its input paths in its + shell, which is not fastidiousness: ~20k paths is an order of magnitude over + Linux's 128 KiB `MAX_ARG_STRLEN` for a single argv entry, so each script + rediscovers the set under `products_dir` while the fingerprint travels on + `params`. - **A dead tile can be told to stop pinning exposures.** An exposure is cleanable only once every consuming tile has its vignets, so one permanently-failed tile holds its ~80 exposures for the life of the diff --git a/workflow/Snakefile b/workflow/Snakefile index fbba2c851..2954222bc 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -31,6 +31,7 @@ the manifest says "this stage succeeded", the log says "here is what happened" (the contract is argued in completeness.py's docstring). """ +import fnmatch import functools import hashlib import json @@ -40,6 +41,10 @@ import sys from pathlib import Path from snakemake.exceptions import WorkflowError +# Explicit rather than relying on the name snakemake injects into this +# namespace: the one place we log at parse time is a branch that only a +# non-default keep list reaches, and a NameError there would be found by a user. +from snakemake.logging import logger # Resolved relative to THIS file, not the working directory: snakemake runs with # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project @@ -92,6 +97,14 @@ RUN_DIR = Path(OUTPUTS["run_dir"]) # second path: one root, exactly the pre-D5 layout. PRODUCTS_DIR = Path(OUTPUTS.get("products_dir") or RUN_DIR) INDEX_DB = Path(OUTPUTS["index_db"]) +# The campaign's NAME — what the two campaign-level merges label their output +# with (`final_cat_.hdf5`, and the group inside it that holds the +# campaign's per-tile datasets). It +# defaults to the persistent root's basename, which is already how every +# campaign here is named (smk-g4, smk-g5, smk-g6: run_dir, products_dir and +# index all end in it), so the common case needs no key at all. Set `campaign:` +# in config.yaml when the two must differ. +CAMPAIGN = config.get("campaign") or PRODUCTS_DIR.name SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There @@ -328,6 +341,8 @@ FOREST_HASH = script_hash("build_forest.py") CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") PERSIST_HASH = script_hash("persist_exp.py") +MERGE_STAR_HASH = script_hash("merge_star_cat.py") +MERGE_FINAL_HASH = script_hash("merge_final_cat.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -473,12 +488,173 @@ def persist_targets(): HEAD PROCESS ONLY, for the same reason as clean_targets() above. """ - if not PERSIST_EXP or not workflow.is_main_process: + if not workflow.is_main_process: + return [] + return persist_manifests() + + +@functools.lru_cache(maxsize=1) +def persist_manifests(): + """persist_targets() without the head-process guard, memoised. + + The guard on persist_targets() is a cost decision, not a correctness one: + `rule all` reads it at MODULE level, so a job parse would pay a whole + campaign's index walk for a target it can never schedule. star_cat_merge + reads the same list through an INPUT FUNCTION, which snakemake evaluates + only for parses that actually build that job — the head process, and the one + merge job's own re-parse under the slurm executor, which genuinely needs it. + So this half carries no guard and the memo keeps either parse to one walk. + """ + if not PERSIST_EXP: return [] exps = {e for t in TILES_READY for e in tile_exposures(t)} return sorted(prod_exp_manifest(e, "exp_persist") for e in exps if not Path(tombstone(e)).exists()) +# --- the campaign-level merges --------------------------------------------- +# Two rules, one job each per campaign, both writing to the persistent root, and +# both the LAST link of a chain whose per-unit half the workflow already had: +# the exposure side ends in one `full_starcat-0000000.fits` (every CCD's PSF +# validation catalogue, stacked — the rho/tau statistics input) and the tile side +# in one `final_cat_.hdf5` (every tile's final catalogue — the shear +# catalogue sp_validation reads). Until they existed the workflow's product set +# was two files short of what the old `combine_runs.bash` + `create_final_cat.py` +# chain delivered, and every campaign ended with a manual merge. +# +# NEITHER IS A LOCALRULE, and the arithmetic runs the opposite way from +# exp_persist's. Those rules are ~20k jobs of seconds each, so submitting them +# costs more in scheduling latency than the work; these are ONE job each over the +# whole campaign — ~800k catalogues stacked in memory, or ~20k catalogues read +# end to end at DR6 scale. That is a compute job, and it belongs on a node. +# +# NEITHER PUTS ITS INPUT PATHS IN ITS SHELL. `{input}` at DR6 scale is ~20k paths +# in a single argv entry, an order of magnitude over Linux's 128 KiB +# MAX_ARG_STRLEN, and the job would die on exec. So each rule's `input` is the +# DAG EDGE (what must exist first) and each script rediscovers the same set under +# products_dir; what travels is a FINGERPRINT of the input list, on `params`, +# which is what makes the merge rerun when the set changes and not otherwise. +# The scripts' docstrings argue the rediscovery — it is also what lets the merges +# cover exposures whose scratch stores reclamation has since taken. + + +def input_fingerprint(paths): + """A short digest of an input list, for `params`. + + `params` is a rerun trigger and a path list is not: two campaigns' worth of + manifests hashes to two different values, so appending a tile reruns the + merge, while a rerun over the same set leaves it alone. Sorted before + hashing because the list's ORDER is not part of what changed. + """ + joined = "\n".join(sorted(str(p) for p in paths)) + return f"{len(paths)}:{hashlib.md5(joined.encode()).hexdigest()[:12]}" + + +# The tar member the star merge consumes. The keep list is globs, so the test is +# "would this member be kept", not a string comparison — `validation_psf-*.fits`, +# `validation_psf*`, `*.fits` and a bare `*` all say yes, and all are things a +# user might reasonably write. +_STAR_CAT_MEMBER = "validation_psf-2605805-12.fits" +STAR_CAT_MERGE = any(fnmatch.fnmatch(_STAR_CAT_MEMBER, p) for p in PERSIST_EXP) + +# LOUD AT PARSE TIME, once, and only where it can be acted on. A keep list +# without the validation catalogues is a legitimate configuration (persist the +# PSF models alone, say) — it is not an error, so it must not become a job that +# fails on a node an hour later. It is worth SAYING, because the omission is +# silent otherwise and the missing product only surfaces when a rho-statistics +# run cannot find its input. +if PERSIST_EXP and not STAR_CAT_MERGE and workflow.is_main_process \ + and PHASE == "compute": + logger.warning( + f"star_cat_merge: no job — persist_exp {PERSIST_EXP} keeps no " + f"'{_STAR_CAT_MEMBER}'-shaped file, so there is nothing to stack into " + f"{PRODUCTS_DIR}/full_starcat-0000000.fits (the rho/tau statistics " + f"input). Add 'validation_psf-*.fits' to persist_exp: to get it.") + + +def full_starcat(): + """The campaign's merged star catalogue. The NAME is not ours to choose: + sp_validation hardcodes `full_starcat-0000000.fits` beside its data dir.""" + return f"{PRODUCTS_DIR}/full_starcat-0000000.fits" + + +def final_cat_hdf5(): + """The campaign's merged shear catalogue — sp_validation's galaxy_cat_path.""" + return f"{PRODUCTS_DIR}/final_cat_{CAMPAIGN}.hdf5" + + +def prod_exp_tar(exp): + """The tar exp_persist writes. Not a declared output of anything — see + star_cat_inputs().""" + return f"{prod_exp_dir(exp)}/psf/{exp}.tar" + + +@functools.lru_cache(maxsize=1) +def star_cat_inputs(): + """What star_cat_merge waits for: every exposure of TILES_READY whose PSF + products are on the persistent root, live and reclaimed alike. + + THE SAME SET merge_star_cat.py derives at job time, and that equality is + load-bearing — the fingerprint on `params` is taken over THIS list, so + anything the job stacked that was not in it would be rows no rerun trigger + could see. The job states the rule from its own side: same tile list, same + index, exp_persist manifest present on the persistent root. By the time it + runs, every exposure below has one. + + RECLAIMED EXPOSURES BELONG IN THE STAR CATALOGUE. Carrying their PSF + products off scratch is exactly what exp_persist is for, and a merge that + dropped them would shrink the campaign's star catalogue every time + reclamation ran. But their exp_psf manifest is gone, so REQUESTING their + exp_persist manifest rebuilds the whole exposure chain from VOS — the + avalanche persist_targets() drops them to avoid. + ancient() DOES NOT HELP: it suppresses the timestamp comparison, not the + missing input, and snakemake schedules the chain anyway. Measured on smk-g6 + with one reclaimed exposure given a manifest by hand: the dry run grew + exp_get_images, exp_split, exp_psf and exp_persist jobs. + + So a reclaimed exposure is depended on through its TAR instead. The tar is + not a declared output of any rule (exp_persist declares only its manifest, + deliberately — persist_exp.py says why), so a tar that exists is a DAG leaf: + snakemake requires it and builds nothing. A live exposure keeps its manifest + edge, which is what orders the merge after the packing; its tar does not + exist yet, so it could not serve as the edge. + + An exposure reclaimed by a workflow PREDATING exp_persist has neither tar nor + manifest and is in no set at all. Nothing short of rebuilding its chain from + VOS recovers it; the merge reports how many exposures it found. + """ + if not PERSIST_EXP: + return [] + live = set(persist_manifests()) + exps = {e for t in TILES_READY for e in tile_exposures(t)} + reclaimed = sorted(prod_exp_tar(e) for e in exps + if prod_exp_manifest(e, "exp_persist") not in live + and Path(prod_exp_manifest(e, "exp_persist")).exists() + and Path(prod_exp_tar(e)).exists()) + return sorted(live) + reclaimed + + +def star_cat_targets(): + """`full_starcat` when there is anything to stack into it, else nothing. + + Three ways to get nothing, and all three are states rather than errors: the + keep list holds no validation catalogue (warned about above), `persist_exp:` + is empty at all, or every exposure in scope is already tombstoned — a + campaign resumed after reclamation, whose exposures were cleaned by a + workflow that predates exp_persist and therefore left neither tar nor + manifest to read. A rule with an empty input list would still be a JOB, and + it would write an empty star catalogue over a good one. + """ + if not STAR_CAT_MERGE or not workflow.is_main_process: + return [] + return [full_starcat()] if star_cat_inputs() else [] + + +def final_cat_targets(): + """The merged hdf5, whenever this campaign has a tile to put in it.""" + if not workflow.is_main_process or not TILES_READY: + return [] + return [final_cat_hdf5()] + # --- tile reclamation (D5) -------------------------------------------------- # A separate flag from `clean:` (config.yaml carries the full # argument): exposure reclamation costs nothing but a rebuild if a tile is @@ -662,6 +838,8 @@ rule all: input: [final_cat(t) for t in TILES_READY], persist_targets(), + star_cat_targets(), + final_cat_targets(), clean_targets(), clean_tile_targets(), diff --git a/workflow/bin/sp b/workflow/bin/sp index fdc3d4f59..e3aec91a4 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -72,7 +72,10 @@ STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and # runs the campaign entirely out of that copy: the Snakefile, the rules, the # scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points -# into example/, and the copy must be self-contained), src/, and the profile. +# into example/, and the copy must be self-contained), src/, the repo's own +# scripts/ (final_cat_merge loads scripts/python/create_final_cat.py by path -- +# it is a script, not an installed module, and the hdf5 layout it defines must +# be pinned to the campaign like everything else here), and the profile. # Every workflow-internal path hangs off `workflow.basedir`, which IS the # snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the # one that cannot (YAML splices nothing) and is rewritten below. @@ -91,10 +94,10 @@ snapshot_code() { mkdir -p "$SNAPSHOT" if command -v rsync >/dev/null 2>&1; then rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ - "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + "$HERE" "$REPO/src" "$REPO/scripts" "$REPO/profiles" "$SNAPSHOT/" else rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" - cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + cp -rL "$HERE" "$REPO/src" "$REPO/scripts" "$REPO/profiles" "$SNAPSHOT/" find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + fi diff --git a/workflow/config.yaml b/workflow/config.yaml index ce434314c..70ace906e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -57,6 +57,19 @@ outputs: # (-state; bin/sp explains why). products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 +# The campaign's NAME. It labels the two campaign-level merges' output — +# /final_cat_.hdf5 and the group inside it holding that +# campaign's per-tile datasets — +# and nothing else; the per-unit stores are named by their own IDs. UNSET means +# the persistent root's basename, which is already how every campaign here is +# named (run_dir, products_dir and index_db all end in smk-g6), so this key only +# earns its place when the two must differ. +# +# It is not a rule input, so renaming a campaign mid-flight changes the merged +# catalogue's PATH and therefore builds a new one; the per-tile catalogues it +# reads are untouched. +# campaign: smk-g6 + # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / # $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules @@ -96,6 +109,16 @@ outputs: # diagnostics cannot be recomputed after a purge without rebuilding the exposure # chain from VOS. # +# THIS LIST GATES `star_cat_merge`. That campaign-level rule stacks every +# exposure's every CCD's validation_psf into ONE +# /full_starcat-0000000.fits, reading the members straight out of +# the tars. A keep list that matches no `validation_psf-*.fits` is a legitimate +# configuration (keep the PSF models alone, say) and produces NO merge job and a +# warning at parse time — not a failure on a node an hour later. Note the +# corollary: an exposure already reclaimed by a workflow that predates +# exp_persist left no tar, so it contributes nothing and cannot be recovered +# short of rebuilding its chain from VOS. +# # OPT-IN CANDIDATES, and what each buys. Sizes are per exposure (40 CCDs), # measured on smk-m2 (127 exposures, 64 tiles); a 64-tile campaign with all of # the measured ones on came to 7.2 GB: diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 40a4044d4..9c6831549 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -220,3 +220,70 @@ rule clean_exposure: f"python {SCRIPTS}/clean_exposure.py" " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" " --tombstone {output.tombstone} --consumers '{params.consumers}'" + + +# --- the campaign's star catalogue ------------------------------------------ +# ONE job per campaign: every exposure's every CCD's `validation_psf--.fits`, +# stacked into `/full_starcat-0000000.fits`. That file is the +# rho/tau statistics input and sp_validation reads it at exactly that path, +# doing no merging of its own; the old bash chain built it with +# `combine_runs.bash psf` + a `merge_starcat_runner` pass, and the workflow +# emitted neither. The stacking itself is `MergeStarCatPSFEX` — the same class +# the old runner called, reused rather than restated, so a column added to the +# module is a column added here (merge_star_cat.py argues the reuse and the +# tar-member reading). +# +# THE INPUT IS star_cat_inputs() (Snakefile): every exposure of TILES_READY whose +# PSF products are on the persistent root — the live ones through the exp_persist +# manifest edge `rule all` already requests, the RECLAIMED ones through their TAR, +# which no rule declares and which therefore requires nothing to be built. That +# asymmetry is not a flourish; requesting a reclaimed exposure's manifest +# rebuilds its whole chain from VOS, and ancient() does not prevent it (measured +# — the Snakefile carries the numbers). Nothing new enters the DAG either way. It +# is read through an INPUT FUNCTION rather than at module level so that only a +# parse which actually builds this job pays for the walk. +# +# THE PATHS DO NOT REACH THE SHELL, and that is not a style choice: ~20k manifest +# paths is an order of magnitude over Linux's 128 KiB MAX_ARG_STRLEN for a single +# argv entry, so `{input}` here would be a job that dies on exec at DR6 scale. +# The job is handed the two small files the Snakefile itself started from — the +# tile list and the index — and derives THE SAME SET from them; `params.inputs` +# carries that set's FINGERPRINT, which is the rerun trigger. The equality is +# the point: a job that stacked anything the fingerprint did not see would be +# rows no rerun trigger could notice, which is what a glob over products_dir +# would have given on a root shared with an earlier, larger tile list. +# Byte-stable output otherwise (tmp-then-cmp-then-mv), so a no-op rerun does not +# move its mtime. +# +# NOT A LOCALRULE. exp_persist is local because it is 20k jobs of seconds; this +# is one job that holds a campaign's stars in memory (~800k catalogues at DR6 +# scale). mem_mb is a guess scaled by attempt, not a measurement — the campaigns +# run so far are 127 exposures, three orders of magnitude short of the case this +# sizing is for, and the first DR6-scale run should replace this number with a +# benchmark. +# +# NO JOB AT ALL when `persist_exp:` keeps no validation catalogue, or when every +# exposure in scope is tombstoned: star_cat_targets() (Snakefile) simply does not +# request the output, and the parse says so rather than a node failing later. +rule star_cat_merge: + input: + lambda wc: star_cat_inputs() + output: + star_cat = full_starcat() + params: + products_dir = str(PRODUCTS_DIR), + tile_list = str(config["tile_list"]), + index_db = str(INDEX_DB), + inputs = lambda wc, input: input_fingerprint(input), + script_hash = MERGE_STAR_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/merge_star_cat.py" + " --products-dir '{params.products_dir}'" + " --tile-list '{params.tile_list}' --index-db '{params.index_db}'" + " --output {output.star_cat}" + f" --psf-model {PSF_MODEL}" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 68d6da9ba..cd10ce6cc 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -898,3 +898,63 @@ rule clean_tile: f"python {SCRIPTS}/clean_tile.py" " --tile-dir $(dirname {output.tombstone}) --tile {wildcards.tile}" " --tombstone {output.tombstone}" + + +# --- the campaign's shear catalogue ----------------------------------------- +# ONE job per campaign, the tile-side twin of exposure.smk's star_cat_merge, and +# the same three design calls hold: the input is the list `rule all` already +# requests (every ready tile's final_cat), the paths never reach the shell +# (MAX_ARG_STRLEN), and a fingerprint on `params` is what makes it rerun when a +# tile is appended. The job derives the same set the fingerprint was taken over +# from the tile list and the index rather than globbing products_dir — on a +# products root shared with an earlier, larger tile list a glob would merge tiles +# no rerun trigger ever saw. +# +# THE OUTPUT SCHEMA IS AN INTERFACE, NOT A CHOICE. sp_validation opens this file +# as its `galaxy_cat_path`: one dataset per tile under a named group, the +# columns of workflow/config/cfis/final_cat.param, an `n_tiles` attribute on the +# root. The group is named for the CAMPAIGN, which is the only unit this +# workflow has above the tile. So the rule reuses +# scripts/python/create_final_cat.py's column extraction rather than restating +# it, and writes the file itself — merge_final_cat.py argues that split, the one +# legacy literal in the schema, and the two places where the reference +# implementation had to be pinned down to be reproducible. +# +# THE INPUT IS final_cat, NOT the tile_make_cat manifest, for the same reason +# clean_tile's is: final_cat on the persistent root IS the campaign's +# tile-finished marker (see final_cat() in the Snakefile), and it is the file +# this rule actually reads. +# +# NOT A LOCALRULE, and here the reason is IO rather than memory: the job reads +# every tile's catalogue end to end on every run — ~32-46 MB per tile, so ~2 GB +# for a 64-tile campaign and ~800 GB at DR6's 23k tiles. It rebuilds rather than +# appends because a DAG output must be a function of its input set +# (merge_final_cat.py); incremental update by hand is what +# `create_final_cat.py -s add` remains for. Memory is one tile's catalogue at a +# time plus the hdf5 write buffer, which is why mem_mb is modest where +# star_cat_merge's is not. +rule final_cat_merge: + input: + lambda wc: [final_cat(t) for t in TILES_READY] + output: + merged = final_cat_hdf5() + params: + products_dir = str(PRODUCTS_DIR), + tile_list = str(config["tile_list"]), + index_db = str(INDEX_DB), + param_file = str(CONFIG_DIR / "final_cat.param"), + campaign = CAMPAIGN, + inputs = lambda wc, input: input_fingerprint(input), + script_hash = MERGE_FINAL_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/merge_final_cat.py" + " --products-dir '{params.products_dir}'" + " --tile-list '{params.tile_list}' --index-db '{params.index_db}'" + " --output {output.merged}" + " --campaign '{params.campaign}'" + " --param-file '{params.param_file}'" diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 2dd191306..83e9c606b 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -152,6 +152,45 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, "n_missing": len(missing)} +# --- reading it back, for the campaign-level merges ------------------------- +# The Snakefile loads this index into dicts at parse time and derives the +# campaign's unit sets from them (TILES_READY, and the exposures those tiles +# read). A merge JOB has to derive the same two sets, and cannot be handed them +# on its command line — ~20k paths is an order of magnitude over Linux's 128 KiB +# MAX_ARG_STRLEN for a single argv entry. So it is given the two things the +# Snakefile itself started from, the tile list and this database, and rebuilds +# the sets here. Both halves therefore read the schema through one module rather +# than two hand-written queries that could drift apart. + + +def campaign_tiles(tile_list: Path, db_path: Path) -> list[str]: + """The campaign's ready tiles: declared in the list AND indexed. + + Exactly the Snakefile's TILES_READY, computed the same way from the same two + files — a declared tile with no indexed exposure list cannot have been + computed, so it has no catalogue to merge. + """ + with open(tile_list) as f: + declared = [ln.strip() for ln in f if ln.strip()] + con = sqlite3.connect(db_path, timeout=60) + indexed = {r[0] for r in con.execute("SELECT DISTINCT tile_id FROM tile_exposures")} + con.close() + return [t for t in declared if t in indexed] + + +def campaign_exposures(tile_list: Path, db_path: Path) -> list[str]: + """Every exposure the campaign's ready tiles read, sorted. + + Exactly the set the Snakefile's persist_manifests() builds its manifest + paths from. + """ + tiles = set(campaign_tiles(tile_list, db_path)) + con = sqlite3.connect(db_path, timeout=60) + rows = con.execute("SELECT tile_id, exp_id FROM tile_exposures").fetchall() + con.close() + return sorted({e for t, e in rows if t in tiles}) + + def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--tile-list", required=True, type=Path, diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py new file mode 100644 index 000000000..2d2974fc0 --- /dev/null +++ b/workflow/scripts/merge_final_cat.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Collect the campaign's per-tile final catalogues into ONE hdf5 file. + +Run as the shell of the campaign-level ``final_cat_merge`` rule, never by hand. + +WHAT IT PRODUCES, AND FOR WHOM. ``/final_cat_.hdf5``: +one dataset per tile, carrying the columns named by +``workflow/config/cfis/final_cat.param``, plus an ``n_tiles`` attribute on the +file root. sp_validation opens that file as its ``galaxy_cat_path`` +(``sp_validation/catalog.py``), so its SCHEMA is an interface and not a choice — +see ``SPVAL_GROUP`` below for the one legacy literal in it. + +(sp_validation's own ``merge_catalogues`` is a different layer entirely: it +works over already-calibrated ``shape_catalog_comprehensive_*.fits``. It does +not do this merge, and this does not do that one.) + +WHAT IT REUSES, AND WHAT IT DOES NOT. The column extraction is +``create_final_cat.py``'s — ``read_param_file`` for the parameter list, +``read_data`` and ``copy_data`` for pulling those columns out of one catalogue +with their FITS dtypes — so the column grammar keeps exactly one definition. +Its ``process()`` is NOT used and neither is any of its discovery: that function +walks a directory tree the workflow does not have and never will, and it groups +by a unit ShapePipe v2 no longer has. This script walks the workflow's own +products tree instead (``tiles/<2-char prefix>//final_cat-.fits``) and +writes the hdf5 itself. + +WHERE ``create_final_cat.py`` IS FOUND. Beside this workflow, at +``/scripts/python/create_final_cat.py`` — resolved relative to THIS file, +so it follows the launch code snapshot (``bin/sp``) exactly as +``workflow/scripts/*`` does, and a campaign never reads a mid-run edit. It is +loaded by path rather than imported: it is a script, not an installed module, +and the container's ``shapepipe`` install does not carry it. + +IT REBUILDS THE WHOLE FILE, IT DOES NOT APPEND. ``create_final_cat.py``'s own +``process()`` skips tiles already in the file, which is right for a hand-driven +incremental update (``-s add`` / ``-s remove`` are that tool's job). A DAG rule +wants the opposite: the output must be a pure function of the input set, so that +a no-op rerun is byte-stable and a changed set is visibly a different file. +Appending would make the result depend on the order campaigns were run in, and +would silently keep a tile whose catalogue was later rebuilt. The cost is +reading every tile's catalogue on every run of the rule — real work at DR6 scale +(~20k tiles), which is why this is not a localrule. + +BYTE-STABLE ON A NO-OP RERUN: written to a tmp path, compared, moved only if it +differs (the pattern ``persist_exp.py`` and ``clean_exposure.py`` use). Tiles +are visited in sorted ID order so the file is a function of the input set alone. +An unconditional rewrite would move the output's mtime every invocation. + +TWO PLACES WHERE THE REFERENCE IMPLEMENTATION IS NOT DETERMINISTIC, and where +this script therefore pins the behaviour down rather than copying it. Both are +in the DTYPE, and both are invisible when a human runs the tool once by hand: + + * ``copy_data`` allocates ``np.empty`` with the SOURCE catalogue's full + dtype and then fills only the requested columns, so every column NOT in + ``final_cat.param`` reaches the hdf5 file as uninitialised memory — + different bytes on every run, and meaningless data in the file besides. We + hand ``copy_data`` a dtype restricted to the requested columns, so every + field it writes is a field it fills. The file then carries exactly the + ``final_cat.param`` columns, which is what sp_validation reads and what the + parameter file is for. + * ``read_param_file`` returns ``list(set(...))``, whose order varies with the + process's string hash seed. Column ORDER in a structured dtype is part of + the file, so that alone would defeat the byte comparison. We order the + fields by the source catalogue's own column order instead. + +WHICH TILES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set +is the CAMPAIGN's: every tile both declared in ``tile_list`` and present in the +index, which is exactly the Snakefile's TILES_READY, rebuilt here from the same +two files the Snakefile started from (``--tile-list`` and ``--index-db``, read +through ``build_index.campaign_tiles`` so there is one definition and not two +that can drift). It is derived rather than passed because at DR6 scale the set +is ~20k paths and a shell command reaches ``execve`` as a SINGLE argv entry +capped at 128 KiB by ``MAX_ARG_STRLEN``; the rule's ``input`` is the DAG edge +and its ``params`` carries a fingerprint of that same list, which is the rerun +trigger. + +THE TWO SETS ARE THE SAME SET, which is the point of deriving it this way rather +than globbing ``/tiles``: a products root shared with an earlier, +larger tile list would hand the job tiles the fingerprint never saw and no rerun +trigger would notice. A tile in the derived set whose catalogue is missing is a +hard error here, not a skip — under the DAG it cannot happen, since every one of +them is a declared input of this job. +""" + +import argparse +import filecmp +import importlib.util +import sys +from pathlib import Path + +import h5py +import numpy as np + +# Same directory; the rule invokes this file by path, so it is sys.path[0]. +import build_index + +# /scripts/python/create_final_cat.py, from /workflow/scripts/this. +CFC_PATH = (Path(__file__).resolve().parents[2] + / "scripts" / "python" / "create_final_cat.py") + + +def spval_group(campaign: str) -> str: + """The hdf5 group the campaign's per-tile datasets live under. + + ``patches/`` is a LEGACY KEY IN sp_validation's FILE SCHEMA, kept verbatim + only so its reader works unchanged (CosmoStat/sp_validation#340 tracks + removing it); it names nothing in this workflow, which has campaigns and + tiles and no other unit. This is the one place the literal appears — + everything else here says campaign. + """ + return f"patches/{campaign}" + + +def load_create_final_cat(): + """The hdf5 layout's definition, loaded by path (see the module docstring).""" + if not CFC_PATH.exists(): + sys.exit(f"merge_final_cat: {CFC_PATH} is not there — the launch code " + f"snapshot must carry scripts/python/ (see bin/sp).") + spec = importlib.util.spec_from_file_location("create_final_cat", CFC_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: + """``(tile ID, path)`` for the campaign's tiles, in ID order. + + Not a glob over the products root: see the module docstring on why the set + is the campaign's and not the filesystem's. + """ + out, missing = [], [] + for tile in sorted(build_index.campaign_tiles(tile_list, index_db)): + path = (products_dir / "tiles" / tile[:2] / tile + / f"final_cat-{tile}.fits") + if path.exists(): + out.append((tile, path)) + else: + missing.append(tile) + if missing: + sys.exit(f"merge_final_cat: {len(missing)} campaign tile(s) have no " + f"final catalogue: {' '.join(missing[:5])}" + f"{' ...' if len(missing) > 5 else ''}") + return out + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--products-dir", required=True, type=Path, + help="the persistent root; per-tile catalogues are found " + "beneath it") + p.add_argument("--tile-list", required=True, type=Path, + help="the campaign's tile list (config tile_list)") + p.add_argument("--index-db", required=True, type=Path, + help="the campaign's run index (config outputs.index_db)") + p.add_argument("--output", required=True, type=Path) + p.add_argument("--campaign", required=True, + help="names the campaign's group in the output file") + p.add_argument("--param-file", required=True, type=Path, + help="workflow/config/cfis/final_cat.param — the column list") + p.add_argument("--hdu", type=int, default=1) + args = p.parse_args() + + cfc = load_create_final_cat() + param_list = cfc.read_param_file(str(args.param_file), verbose=False) + if not param_list: + sys.exit(f"merge_final_cat: no columns read from {args.param_file}") + # read_data/copy_data read their knobs out of this dict, exactly as + # create_final_cat.py's own main() builds it. + params = {"hdu_num": args.hdu, "param_list": param_list, "verbose": False} + + tiles = catalogues(args.products_dir, args.tile_list, args.index_db) + if not tiles: + # An empty hdf5 would satisfy every downstream existence check and + # produce an empty shear catalogue. + sys.exit(f"merge_final_cat: no tile in {args.tile_list} is indexed in " + f"{args.index_db}, so there is nothing to merge") + + # tmp-then-cmp-then-mv; the tmp never outlives this process. + args.output.parent.mkdir(parents=True, exist_ok=True) + tmp = args.output.with_name(args.output.name + ".tmp") + try: + tmp.unlink(missing_ok=True) # h5py "a" would reopen a stale one + with h5py.File(tmp, "w") as hdf5_file: + group = hdf5_file.create_group(spval_group(args.campaign)) + columns = None + for tile, path in tiles: + extracted, dtype = cfc.read_data(str(path), params) + # Requested columns, in the SOURCE catalogue's order (see the + # module docstring on determinism). Computed from the first + # tile and reused, so a tile whose catalogue is missing a + # column fails loudly on the assignment rather than quietly + # producing a differently-shaped dataset. + if columns is None: + columns = [c for c in dtype.names + if c in set(params["param_list"])] + missing = sorted(set(params["param_list"]) - set(columns)) + if missing: + sys.exit(f"merge_final_cat: {path} has none of the " + f"requested column(s): {' '.join(missing)}") + subset = np.dtype([(c, dtype[c]) for c in columns]) + group.create_dataset( + tile, + data=cfc.copy_data(columns, extracted, subset), + dtype=subset, + ) + # The same attribute create_final_cat.py's print_list() writes, and + # what sp_validation reads to know how many tiles it is holding. + hdf5_file.attrs["n_tiles"] = len(tiles) + + if args.output.exists() and filecmp.cmp(tmp, args.output, shallow=False): + print(f"[merge_final_cat] unchanged: {args.output}") + else: + tmp.replace(args.output) # atomic: same filesystem + print(f"[merge_final_cat] {len(tiles)} tile(s), " + f"{len(param_list)} column(s) -> {args.output} " + f"(group {spval_group(args.campaign)})") + finally: + tmp.unlink(missing_ok=True) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py new file mode 100644 index 000000000..a4f2e3d9f --- /dev/null +++ b/workflow/scripts/merge_star_cat.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Concatenate the campaign's per-CCD PSF validation catalogues into ONE full_starcat. + +Run as the shell of the campaign-level ``star_cat_merge`` rule, never by hand. + +WHAT IT PRODUCES, AND FOR WHOM. ``/full_starcat-0000000.fits``: +every exposure's every CCD's ``validation_psf--.fits`` row, stacked, +with a ``CCD_NB`` column recording which CCD each row came from. It is the input +to the rho/tau statistics — sp_validation reads exactly this path +(``star_cat_path`` in its ``scripts/calibration/params.py``) and does no merging +of its own. Historically it was ``combine_runs.bash psf`` + a +``merge_starcat_runner`` pass; the workflow emitted neither, so the product set +was short one file. This script is that pass, driven by the DAG instead of by +bash. + +IT DOES NOT REIMPLEMENT THE COLUMN LIST. The stacking, the column names and the +CCD_NB parse all live in ``MergeStarCatPSFEX`` +(``shapepipe.modules.merge_starcat_package.merge_starcat``), which is what the +old runner called. This script only decides WHICH catalogues that class is +handed, and where the result lands. A column added to the module is a column +added here for free — which is the entire reason for the indirection. + +IT READS THE TARS, IT DOES NOT UNPACK THEM. ``exp_persist`` packs each +exposure's keepers into one uncompressed tar on the persistent root +(``/exp///psf/.tar``) precisely because inodes, +not bytes, bind on /project. Unpacking ~20k tars × ~40 members to merge them +would materialise ~800k files on the filesystem that design exists to protect, +and then delete them. So members are read into memory +(``tarfile.extractfile(m).read()`` -> ``io.BytesIO``) one at a time and handed +to the merge class as ``[fileobj, member_name]`` pairs. The member NAME is what +the CCD_NB regex parses, which is why the pair carries it; the class takes the +name from the last element of the entry, so a plain ``[path]`` entry behaves +exactly as it always did. + +WHICH EXPOSURES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. +The set is the CAMPAIGN's: every exposure read by a tile that is both declared +in ``tile_list`` and present in the index, which is the Snakefile's TILES_READY +walked one edge further. This script rebuilds it from the same two files the +Snakefile started from (``--tile-list`` and ``--index-db``, both small, both on +the persistent root, both read through ``build_index.campaign_exposures`` so +there is one query and not two that can drift), and then takes the exposures +whose ``exp_persist`` manifest is on the persistent root. + +It is derived rather than passed because at DR6 scale the set is ~20k paths, and +a shell command reaches ``execve`` as a SINGLE argv entry capped at 128 KiB by +``MAX_ARG_STRLEN``. Passing them would be a job that dies before it starts. So +the rule's ``input`` is the DAG EDGE — what must exist before this runs — and +the rule's ``params`` carries a FINGERPRINT of that same list, which is what +makes the merge rerun when the set changes. + +THE TWO SETS ARE THE SAME SET, and that equality is the point of deriving it +this way rather than globbing the tree. The rule's input is ``star_cat_inputs()`` +(Snakefile): for each exposure of TILES_READY whose PSF products are on the +persistent root, an edge — the ``exp_persist`` manifest for a live exposure, the +TAR for one whose scratch store reclamation already took (that function argues +the asymmetry, which is about not rebuilding a reclaimed chain from VOS). +Nothing at all for an exposure reclaimed before ``exp_persist`` existed, which +left neither and is unrecoverable short of that rebuild. What this script +selects is the same rule stated from the job's side: same tiles, same index, +manifest present — and by the time the job runs, every exposure with an edge has +one. A glob over ``/exp`` would NOT be the same set: it would +sweep in exposures of an earlier, larger tile list sharing the products root, +stacking rows the fingerprint never saw and no rerun trigger would notice. + +THE MANIFEST, NOT THE TAR, IS WHAT IT READS FIRST: the manifest records what was +actually packed, pattern by pattern, member by member, with sizes. Selecting +members from it means this script never guesses at tar contents, and an exposure +whose keep list did not include the validation catalogues contributes nothing +visibly rather than silently. + +BYTE-STABLE ON A NO-OP RERUN: written to a tmp path, compared, and moved only +if it differs (the pattern ``persist_exp.py`` and ``clean_exposure.py`` use). +An unconditional rewrite would move the output's mtime on every invocation. +Members are visited in sorted (exposure, member) order so the row order is a +function of the input set alone. + +PSFEX ONLY, DELIBERATELY. ``PSF_MODEL`` is ``psfex`` in every campaign the +workflow has run; ``MergeStarCatMCCD`` and ``MergeStarCatSetools`` exist beside +it and take the same constructor, so the hook is the one-line class choice in +``merge_class()`` below — an implementation, not a design, away. +""" + +import argparse +import filecmp +import io +import json +import logging +import shutil +import sys +import tarfile +import tempfile +from fnmatch import fnmatch +from pathlib import Path + +from shapepipe.modules.merge_starcat_package import merge_starcat + +# Same directory; the rule invokes this file by path, so it is sys.path[0]. +import build_index + +# The output name is not ours to choose: sp_validation hardcodes it +# (`star_cat_path = f"{data_dir}/full_starcat-0000000.fits"`), and +# MergeStarCatPSFEX writes exactly this basename into the output dir it is +# given. Kept here as the name this script promises to produce. +OUT_NAME = "full_starcat-0000000.fits" + +# The keep-list pattern whose members this merge consumes. The rule refuses to +# exist unless `persist_exp:` contains a pattern matching this shape (the +# Snakefile does that check at parse time), so by the time we get here the +# members are expected to be present. +MEMBER_PATTERN = "validation_psf-*.fits" + + +def merge_class(psf_model: str): + """The merge class for this PSF model — the one-line MCCD/setools hook.""" + try: + return {"psfex": merge_starcat.MergeStarCatPSFEX, + "mccd": merge_starcat.MergeStarCatMCCD, + "setools": merge_starcat.MergeStarCatSetools}[psf_model] + except KeyError: + sys.exit(f"merge_star_cat: unknown psf_model {psf_model!r}") + + +def manifests(products_dir: Path, tile_list: Path, index_db: Path) -> list: + """The campaign's exp_persist manifests that are on disk, in exposure order. + + Not a glob over the products root: see the module docstring on why the set + is the campaign's and not the filesystem's. + """ + out = [] + for exp in build_index.campaign_exposures(tile_list, index_db): + path = (products_dir / "exp" / exp[:2] / exp / "manifests" + / "exp_persist.json") + if path.exists(): + out.append(path) + return out + + +def entries(manifest_paths: list, pattern: str) -> tuple: + """``[fileobj, member_name]`` for every matching member, and the tar count. + + One tar is opened at a time and its members are read into memory; the tars + are never unpacked to disk (see the module docstring). The returned file + objects are BytesIO, so nothing stays open on the filesystem — at ~50 KB per + member and ~40 members per exposure this is ~2 MB per exposure held only for + as long as the merge takes to consume it, but note that the merge class + holds the whole stack in python lists regardless, which is the real memory + term the rule's mem_mb is sized against. + """ + out, n_tars, empty = [], 0, [] + for man_path in manifest_paths: + man = json.loads(man_path.read_text()) + wanted = sorted(f["name"] for f in man["files"] + if fnmatch(f["name"], pattern)) + if not wanted: + empty.append(man["unit"]) + continue + tar_path = Path(man["tar"]) + if not tar_path.exists(): + sys.exit(f"merge_star_cat: {man_path} names a tar that is not " + f"there: {tar_path}") + with tarfile.open(tar_path) as tf: + for name in wanted: + member = tf.extractfile(name) + if member is None: + sys.exit(f"merge_star_cat: {tar_path} has no member " + f"{name}, which its manifest lists") + out.append([io.BytesIO(member.read()), name]) + n_tars += 1 + return out, n_tars, empty + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--products-dir", required=True, type=Path, + help="the persistent root; exp_persist manifests and tars " + "are found beneath it") + p.add_argument("--tile-list", required=True, type=Path, + help="the campaign's tile list (config tile_list)") + p.add_argument("--index-db", required=True, type=Path, + help="the campaign's run index (config outputs.index_db)") + p.add_argument("--output", required=True, type=Path, + help=f"the merged catalogue; its basename is {OUT_NAME}") + p.add_argument("--psf-model", default="psfex") + p.add_argument("--pattern", default=MEMBER_PATTERN, + help="tar-member glob to merge; default %(default)s") + args = p.parse_args() + + if args.output.name != OUT_NAME: + # The merge class writes OUT_NAME into a directory it is handed; a + # differently-named declared output would silently never be produced. + sys.exit(f"merge_star_cat: --output must be named {OUT_NAME} " + f"(got {args.output.name})") + + log = logging.getLogger("merge_star_cat") + logging.basicConfig(format="[merge_star_cat] %(message)s", + level=logging.INFO, stream=sys.stdout) + + manifest_paths = manifests(args.products_dir, args.tile_list, args.index_db) + file_list, n_tars, empty = entries(manifest_paths, args.pattern) + if not file_list: + # Not a no-op: an empty star catalogue would pass every downstream + # existence check and produce meaningless rho statistics. + sys.exit(f"merge_star_cat: no member matched {args.pattern!r} in any " + f"of {len(manifest_paths)} exp_persist manifest(s) for this " + f"campaign — is '{args.pattern}' in the persist_exp keep list?") + if empty: + log.info(f"{len(empty)} exposure(s) persisted no {args.pattern}: " + f"{', '.join(sorted(empty)[:5])}" + f"{' ...' if len(empty) > 5 else ''}") + + # tmp-then-cmp-then-mv. The merge class chooses its own basename inside the + # directory it is given, so the tmp is a DIRECTORY, not a file path, and it + # never outlives this process — an orphan on /project is an inode nothing + # revisits. + args.output.parent.mkdir(parents=True, exist_ok=True) + tmp_dir = Path(tempfile.mkdtemp(dir=args.output.parent, + prefix=".star_cat_merge.")) + try: + merge_class(args.psf_model)(file_list, str(tmp_dir), log).process() + tmp = tmp_dir / OUT_NAME + if not tmp.exists(): + sys.exit(f"merge_star_cat: the merge wrote no {OUT_NAME}") + if args.output.exists() and filecmp.cmp(tmp, args.output, shallow=False): + log.info(f"unchanged: {args.output}") + else: + tmp.replace(args.output) # atomic: same filesystem + log.info(f"{len(file_list)} catalogue(s) from {n_tars} exposure(s) " + f"-> {args.output}") + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() From fc9aa3ea298374c57c2547cdbe84b8da4ab2eb4a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 10:33:28 -0400 Subject: [PATCH 08/20] fix(orchestration): seven defects in the campaign-level merges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PERSISTENCE KEYED ON THE WRONG FILE, and the consequence was the avalanche it exists to prevent. persist_targets() skipped an exposure only when its SCRATCH tombstone was there, but clean_exposure is one of two ways a store disappears and the other leaves nothing behind: /scratch is purged on a 60-day window whether or not the workflow reclaimed anything. After a purge — or on any campaign with clean: false — every exposure looked live, its persist manifest was requested, its exp_psf manifest was gone, and snakemake rebuilt the whole exposure chain from VOS. The test is now exp_store_reclaimed(), used by persist_manifests() and by star_cat_merge's live/reclaimed split so the two cannot disagree, and it takes BOTH pieces of evidence that an exposure once had a store: a tombstone, or a persisted manifest with no exp_psf manifest beside it. Both are needed. The manifest clause alone regressed the tombstone case — measured on smk-g6, whose 126 exposures were cleaned before exp_persist existed and so have no manifest: the dry run grew 126 exp_get_images, exp_split, exp_psf and exp_persist jobs, a whole campaign rebuilt from VOS. The tombstone clause alone is the purge bug above. Neither file can be dropped, because the parse cannot otherwise tell a store that is GONE from one not BUILT yet, and a fresh campaign must still be asked to persist. OVERLAPPING KEEP PATTERNS FAILED EVERY EXPOSURE. persist_exp treated a file matched by two patterns as a flat-member name collision, so 'validation_psf-*.fits' alongside '*.fits' — an ordinary way to write a keep list — aborted the pack. Two DIFFERENT paths on one member name is still fatal; the same path twice is now one file, recorded under the first pattern that matched it. merge_star_cat MATERIALISED THE WHOLE CAMPAIGN before merging a row: every member's bytes, ~2 MB per exposure, ~40 GB at DR6's ~20k exposures against a rule asking for 16 GB. TarMembers hands the merge class the same entries one tar at a time, so peak memory is one member plus the class's own accumulators, which are the unavoidable term. It keeps __len__ off the manifests so the count is still logged before a tar is opened. THE STAR MERGE RERAN ON BOOKKEEPING. Its fingerprint was over input PATHS, and an exposure's edge flips from its manifest to its tar the moment its store is reclaimed — so every reclamation pass reran the merge over identical content. It is over the exposure IDS now, which move only when the set does, and which are what the job derives on its own side. final_cat_merge's is over tile ids for the same reason. merge_final_cat's MISSING-COLUMN CHECK WAS UNREACHABLE. create_final_cat's read_data wraps its column selection in a bare `except:` that prints and falls through, so a missing column left its return values unbound and the caller got UnboundLocalError from the return statement, naming nothing. The columns are checked against the catalogue's own header before read_data is called, and the message now names every missing one. A TILE LISTED TWICE killed the merge on the second create_dataset. The tile list is appended to by hand, so duplicates happen; campaign_tiles() dedupes it order-preserving, and the Snakefile's TILES does the same so the fingerprint and the job's derived set still name the same set. merge_class OFFERED MCCD AND SETOOLS while only MergeStarCatPSFEX had learned the [fileobj, name] entry shape. Both now take the entry's name from its last element like PSFEX does — unchanged for the module runner, whose entries are [path]. Setools needs one thing more before it can read a tar (it hands file_io input_file_list[0][0] as a template path), and merge_class says so rather than implying otherwise. Also: README no longer lists scripts/sp_rule.py, which does not exist. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- .../merge_starcat_package/merge_starcat.py | 17 ++- workflow/README.md | 1 - workflow/Snakefile | 123 ++++++++++++++---- workflow/rules/exposure.smk | 2 +- workflow/rules/tile.smk | 2 +- workflow/scripts/build_index.py | 11 +- workflow/scripts/merge_final_cat.py | 23 +++- workflow/scripts/merge_star_cat.py | 95 +++++++++----- workflow/scripts/persist_exp.py | 10 ++ 9 files changed, 214 insertions(+), 70 deletions(-) diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 1717825f9..2b1bf8c34 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -239,10 +239,15 @@ def process(self): my_mask[inside_circle] = True for name in self._input_file_list: + # The source to read and the NAME to report it by; identical for a + # plain [path] entry (see MergeStarCatPSFEX's docstring on the + # [fileobj, name] form). This class takes its CCD numbers from the + # data's own CCD_ID_LIST, so the name is only ever used in messages. + source, label = name[0], name[-1] try: - starcat_j = fits.open(name[0], memmap=False, ignore_missing_simple=True) + starcat_j = fits.open(source, memmap=False, ignore_missing_simple=True) except ValueError: - print(f"Error for file {name[0]}, check FITS file integrity") + print(f"Error for file {label}, check FITS file integrity") #raise continue @@ -799,7 +804,11 @@ def process(self): ) for name in self._input_file_list: - starcat_j = fits.open(name[0], memmap=False) + # The source to read and the NAME to parse the CCD number out of; + # identical for a plain [path] entry (see MergeStarCatPSFEX's + # docstring on the [fileobj, name] form). + source, label = name[0], name[-1] + starcat_j = fits.open(source, memmap=False) data_j = starcat_j[self._hdu_table].data @@ -824,7 +833,7 @@ def process(self): snr += list(data_j["SNR_WIN"]) # CCD number - ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", name[0])[-2]] * len( + ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2]] * len( data_j["XWIN_IMAGE"] ) diff --git a/workflow/README.md b/workflow/README.md index 824eddc73..eb801da6a 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -161,7 +161,6 @@ workflow/ exposure.smk per-exposure: get_images, split, psf, persist (no temp()); campaign star_cat_merge tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat; campaign final_cat_merge scripts/ - sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count check) build_index.py prepare-phase run_index.sqlite builder (plain script) build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count table (shared by sp_rule + run_report) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2954222bc..587f271b2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -136,8 +136,14 @@ if PHASE not in ("prepare", "compute", "passthrough"): raise WorkflowError( f"SP_PHASE={PHASE!r} is not one of prepare, compute, passthrough.") +# DEDUPED, order preserved. The tile list is appended to by hand across a +# campaign, so a tile can appear twice — harmless for a per-tile target, which +# is the same path requested twice, but not for the campaign-level merges: their +# fingerprint counts what is in this list and the job derives a deduped set from +# the same file (build_index.campaign_tiles), so a duplicate line would make the +# two disagree about a set they must name identically. with open(config["tile_list"]) as f: - TILES = [ln.strip() for ln in f if ln.strip()] + TILES = list(dict.fromkeys(ln.strip() for ln in f if ln.strip())) # --- ngmix scatter (D4) ---------------------------------------------------- # Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one @@ -479,12 +485,11 @@ def persist_targets(): Scope is the ready tiles' exposures, which `all` already builds through the tile chain, so nothing new is pulled into the DAG by asking. - EXCEPT A CLEANED EXPOSURE. Its exp_psf manifest was deleted by - clean_exposure, so requesting its persist manifest would make the DAG - rebuild the whole exposure chain from VOS — the avalanche tile.smk's - reclaimed-edge cut exists to prevent, arriving through a new target instead. - A tombstone means the copy already happened (clean_exposure cannot run - before exp_persist), so there is nothing to ask for. + EXCEPT AN EXPOSURE WHOSE STORE IS GONE. Its exp_psf manifest is not there, + so requesting its persist manifest would make the DAG rebuild the whole + exposure chain from VOS — the avalanche tile.smk's reclaimed-edge cut exists + to prevent, arriving through a new target instead. exp_store_reclaimed() + below is that test, and what it does NOT test is the tombstone. HEAD PROCESS ONLY, for the same reason as clean_targets() above. """ @@ -509,7 +514,46 @@ def persist_manifests(): return [] exps = {e for t in TILES_READY for e in tile_exposures(t)} return sorted(prod_exp_manifest(e, "exp_persist") for e in exps - if not Path(tombstone(e)).exists()) + if not exp_store_reclaimed(e)) + + +def exp_store_reclaimed(exp): + """True when this exposure's PSF products exist ONLY on the persistent root. + + The one condition both the persist target list and star_cat_merge's edge + choice turn on, and it deliberately does NOT read the tombstone. + + THE TOMBSTONE ALONE IS NOT THE EVIDENCE. It says clean_exposure ran, and + clean_exposure is only one of the two ways a scratch store disappears. + + WHAT THE TEST HAS TO SEPARATE is a store that is GONE from one that has not + been BUILT yet, and no single file says that. This runs at parse time, before + any exp_psf job of a fresh campaign has run, so "the exp_psf manifest is + missing" alone would skip every exposure of a new campaign and persist + nothing at all. The question is therefore: is there evidence this exposure + once had a store? Two files carry it, and either will do: + + * the TOMBSTONE — clean_exposure ran, so the store was built and reclaimed, + and whatever was going to be packed was packed before it went; + * a PERSISTED MANIFEST with no exp_psf manifest beside it — exp_persist + ran, so the store existed, and it is not there now. This is the purge + case, and it is the one the tombstone cannot see: /scratch is purged on + a 60-day window whether or not this workflow reclaimed anything, and it + leaves nothing behind. Keying on the tombstone alone meant that after a + purge, or on any campaign run with `clean: false`, every exposure looked + live, its persist manifest was requested, its exp_psf manifest was not + there, and snakemake rebuilt the entire exposure chain from VOS. + + An exposure with a LIVE store and a manifest is not reclaimed and is still + asked for, which is what lets an edit to `persist_exp:` re-pack in seconds + rather than be silently ignored — the whole reason exp_persist is a rule of + its own. An exposure with neither file is asked for too: either it has not + run yet, or it was purged having saved nothing, and only the DAG can tell + those apart by trying. + """ + return (Path(tombstone(exp)).exists() + or (Path(prod_exp_manifest(exp, "exp_persist")).exists() + and not Path(exp_manifest(exp, "exp_psf")).exists())) # --- the campaign-level merges --------------------------------------------- # Two rules, one job each per campaign, both writing to the persistent root, and @@ -530,23 +574,31 @@ def persist_manifests(): # NEITHER PUTS ITS INPUT PATHS IN ITS SHELL. `{input}` at DR6 scale is ~20k paths # in a single argv entry, an order of magnitude over Linux's 128 KiB # MAX_ARG_STRLEN, and the job would die on exec. So each rule's `input` is the -# DAG EDGE (what must exist first) and each script rediscovers the same set under -# products_dir; what travels is a FINGERPRINT of the input list, on `params`, -# which is what makes the merge rerun when the set changes and not otherwise. +# DAG EDGE (what must exist first) and each script rediscovers the same set from +# the tile list and the index; what travels is a FINGERPRINT of that set's unit +# ids, on `params`, which is what makes the merge rerun when the set changes and +# not otherwise. # The scripts' docstrings argue the rediscovery — it is also what lets the merges # cover exposures whose scratch stores reclamation has since taken. -def input_fingerprint(paths): - """A short digest of an input list, for `params`. +def unit_fingerprint(units): + """A short digest of a set of UNIT IDS, for a merge rule's `params`. + + `params` is a rerun trigger and a set of ids is not: appending a tile grows + the set, moves the digest and reruns the merge, while a rerun over the same + set leaves it alone. Sorted before hashing because the ORDER is not part of + what changed. - `params` is a rerun trigger and a path list is not: two campaigns' worth of - manifests hashes to two different values, so appending a tile reruns the - merge, while a rerun over the same set leaves it alone. Sorted before - hashing because the list's ORDER is not part of what changed. + IDS RATHER THAN THE RULE'S `input` PATHS. A path can change while the set + does not — star_cat_merge's edge for one exposure flips from its manifest to + its tar when the store is reclaimed — and a merge that reruns over identical + content on every reclamation pass is a rerun trigger firing on bookkeeping. + The ids are also exactly what the job derives on its own side, so both + halves agree on the set and on how it is named. """ - joined = "\n".join(sorted(str(p) for p in paths)) - return f"{len(paths)}:{hashlib.md5(joined.encode()).hexdigest()[:12]}" + joined = "\n".join(sorted(str(u) for u in units)) + return f"{len(units)}:{hashlib.md5(joined.encode()).hexdigest()[:12]}" # The tar member the star merge consumes. The keep list is globs, so the test is @@ -624,13 +676,32 @@ def star_cat_inputs(): """ if not PERSIST_EXP: return [] - live = set(persist_manifests()) - exps = {e for t in TILES_READY for e in tile_exposures(t)} - reclaimed = sorted(prod_exp_tar(e) for e in exps - if prod_exp_manifest(e, "exp_persist") not in live - and Path(prod_exp_manifest(e, "exp_persist")).exists() - and Path(prod_exp_tar(e)).exists()) - return sorted(live) + reclaimed + live, reclaimed = [], [] + for exp in sorted({e for t in TILES_READY for e in tile_exposures(t)}): + if not exp_store_reclaimed(exp): + live.append(prod_exp_manifest(exp, "exp_persist")) + elif Path(prod_exp_tar(exp)).exists(): + reclaimed.append(prod_exp_tar(exp)) + return live + reclaimed + + +@functools.lru_cache(maxsize=1) +def star_cat_exposures(): + """The exposure IDs star_cat_merge stacks — what its fingerprint is taken + over. + + THE IDS, NOT THE PATHS, and the difference is a rerun. An exposure's edge + FLIPS from its manifest to its tar the moment its store is reclaimed, so a + fingerprint over paths moves on every reclamation pass and reruns the merge + over content that did not change. The ids move only when the set does, which + is what the trigger is for. It is also what merge_star_cat.py derives on the + job side, so the two agree on the set AND on how it is named. + """ + if not PERSIST_EXP: + return [] + return sorted(e for e in {e for t in TILES_READY for e in tile_exposures(t)} + if Path(prod_exp_manifest(e, "exp_persist")).exists() + or not exp_store_reclaimed(e)) def star_cat_targets(): diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 9c6831549..7b6d33882 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -274,7 +274,7 @@ rule star_cat_merge: products_dir = str(PRODUCTS_DIR), tile_list = str(config["tile_list"]), index_db = str(INDEX_DB), - inputs = lambda wc, input: input_fingerprint(input), + inputs = unit_fingerprint(star_cat_exposures()), script_hash = MERGE_STAR_HASH threads: 1 resources: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index cd10ce6cc..ed5daceb4 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -944,7 +944,7 @@ rule final_cat_merge: index_db = str(INDEX_DB), param_file = str(CONFIG_DIR / "final_cat.param"), campaign = CAMPAIGN, - inputs = lambda wc, input: input_fingerprint(input), + inputs = unit_fingerprint(TILES_READY), script_hash = MERGE_FINAL_HASH threads: 1 resources: diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 83e9c606b..24290b561 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -170,8 +170,17 @@ def campaign_tiles(tile_list: Path, db_path: Path) -> list[str]: files — a declared tile with no indexed exposure list cannot have been computed, so it has no catalogue to merge. """ + # DEDUPED, order preserved. The tile list is appended to by hand across a + # campaign, so a tile can appear twice; a merge would then try to write that + # tile's dataset twice and die on the second. Deduping here rather than at + # the call sites keeps the answer the same for every reader of the index. + seen, declared = set(), [] with open(tile_list) as f: - declared = [ln.strip() for ln in f if ln.strip()] + for line in f: + tile = line.strip() + if tile and tile not in seen: + seen.add(tile) + declared.append(tile) con = sqlite3.connect(db_path, timeout=60) indexed = {r[0] for r in con.execute("SELECT DISTINCT tile_id FROM tile_exposures")} con.close() diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py index 2d2974fc0..c12d5fcee 100644 --- a/workflow/scripts/merge_final_cat.py +++ b/workflow/scripts/merge_final_cat.py @@ -90,6 +90,7 @@ import h5py import numpy as np +from astropy.io import fits # Same directory; the rule invokes this file by path, so it is sys.path[0]. import build_index @@ -143,6 +144,16 @@ def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: return out +def check_columns(path: Path, hdu: int, wanted: list) -> None: + """Fail loudly, and by name, when a catalogue lacks a requested column.""" + with fits.open(path, memmap=False) as hdu_list: + present = set(hdu_list[hdu].columns.names) + missing = sorted(c for c in wanted if c not in present) + if missing: + sys.exit(f"merge_final_cat: {path} is missing {len(missing)} of the " + f"{len(wanted)} requested column(s): {' '.join(missing)}") + + def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--products-dir", required=True, type=Path, @@ -184,6 +195,14 @@ def main() -> None: group = hdf5_file.create_group(spval_group(args.campaign)) columns = None for tile, path in tiles: + # BEFORE read_data, and not inside it. read_data wraps its + # column selection in a bare `except:` that prints and falls + # through, so a missing column leaves its return values unbound + # and the caller sees UnboundLocalError from the return + # statement — the real name, and every other missing name, never + # reaches the caller at all. Reading the header costs nothing + # next to reading the table. + check_columns(path, args.hdu, params["param_list"]) extracted, dtype = cfc.read_data(str(path), params) # Requested columns, in the SOURCE catalogue's order (see the # module docstring on determinism). Computed from the first @@ -193,10 +212,6 @@ def main() -> None: if columns is None: columns = [c for c in dtype.names if c in set(params["param_list"])] - missing = sorted(set(params["param_list"]) - set(columns)) - if missing: - sys.exit(f"merge_final_cat: {path} has none of the " - f"requested column(s): {' '.join(missing)}") subset = np.dtype([(c, dtype[c]) for c in columns]) group.create_dataset( tile, diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py index a4f2e3d9f..25c6e4c22 100644 --- a/workflow/scripts/merge_star_cat.py +++ b/workflow/scripts/merge_star_cat.py @@ -20,17 +20,18 @@ handed, and where the result lands. A column added to the module is a column added here for free — which is the entire reason for the indirection. -IT READS THE TARS, IT DOES NOT UNPACK THEM. ``exp_persist`` packs each -exposure's keepers into one uncompressed tar on the persistent root +IT READS THE TARS, IT DOES NOT UNPACK THEM, AND IT STREAMS. ``exp_persist`` +packs each exposure's keepers into one uncompressed tar on the persistent root (``/exp///psf/.tar``) precisely because inodes, not bytes, bind on /project. Unpacking ~20k tars × ~40 members to merge them would materialise ~800k files on the filesystem that design exists to protect, -and then delete them. So members are read into memory -(``tarfile.extractfile(m).read()`` -> ``io.BytesIO``) one at a time and handed -to the merge class as ``[fileobj, member_name]`` pairs. The member NAME is what -the CCD_NB regex parses, which is why the pair carries it; the class takes the -name from the last element of the entry, so a plain ``[path]`` entry behaves -exactly as it always did. +and then delete them. So members are read out of the tars in memory +(``tarfile.extractfile(m).read()`` -> ``io.BytesIO``) and handed to the merge +class as ``[fileobj, member_name]`` pairs — ONE AT A TIME, lazily, through +``TarMembers`` below, because materialising them all first is ~40 GB at DR6 +scale. The member NAME is what the CCD_NB regex parses, which is why the pair +carries it; the class takes the name from the last element of the entry, so a +plain ``[path]`` entry behaves exactly as it always did. WHICH EXPOSURES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set is the CAMPAIGN's: every exposure read by a tile that is both declared @@ -111,7 +112,15 @@ def merge_class(psf_model: str): - """The merge class for this PSF model — the one-line MCCD/setools hook.""" + """The merge class for this PSF model — the one-line MCCD/setools hook. + + Only psfex is exercised: it is what every campaign has run. MCCD reaches the + tars unchanged (it takes its CCD numbers from the data, and it now reports + by the entry's name like the others). SETOOLS would need one more thing — + it passes ``input_file_list[0][0]`` to file_io as a template path, which a + streamed entry is not — so wiring setools to this path is a change to that + class, not a change here. + """ try: return {"psfex": merge_starcat.MergeStarCatPSFEX, "mccd": merge_starcat.MergeStarCatMCCD, @@ -135,18 +144,14 @@ def manifests(products_dir: Path, tile_list: Path, index_db: Path) -> list: return out -def entries(manifest_paths: list, pattern: str) -> tuple: - """``[fileobj, member_name]`` for every matching member, and the tar count. +def selection(manifest_paths: list, pattern: str) -> tuple: + """``[(tar path, [member names])]`` for the merge, and the empty exposures. - One tar is opened at a time and its members are read into memory; the tars - are never unpacked to disk (see the module docstring). The returned file - objects are BytesIO, so nothing stays open on the filesystem — at ~50 KB per - member and ~40 members per exposure this is ~2 MB per exposure held only for - as long as the merge takes to consume it, but note that the merge class - holds the whole stack in python lists regardless, which is the real memory - term the rule's mem_mb is sized against. + Reads the manifests only. Every tar is checked for existence HERE, so a + products root missing a file fails before a single row is stacked rather + than an hour in. """ - out, n_tars, empty = [], 0, [] + chosen, empty = [], [] for man_path in manifest_paths: man = json.loads(man_path.read_text()) wanted = sorted(f["name"] for f in man["files"] @@ -158,15 +163,40 @@ def entries(manifest_paths: list, pattern: str) -> tuple: if not tar_path.exists(): sys.exit(f"merge_star_cat: {man_path} names a tar that is not " f"there: {tar_path}") - with tarfile.open(tar_path) as tf: - for name in wanted: - member = tf.extractfile(name) - if member is None: - sys.exit(f"merge_star_cat: {tar_path} has no member " - f"{name}, which its manifest lists") - out.append([io.BytesIO(member.read()), name]) - n_tars += 1 - return out, n_tars, empty + chosen.append((tar_path, wanted)) + return chosen, empty + + +class TarMembers: + """The merge class's input list, materialised ONE TAR AT A TIME. + + ``MergeStarCatPSFEX`` wants something it can take the length of and iterate + once, handing it ``[fileobj, name]`` entries; it never indexes and never + rewinds. So it does not need a list, and a list is the one thing we cannot + afford: reading every member up front is the whole campaign in memory at + once — ~2 MB per exposure, so ~40 GB at DR6's ~20k exposures, against a + rule asking for 16 GB. Read lazily, peak memory is ONE member's bytes plus + the merge class's own accumulators, which are the real and unavoidable term. + + ``__len__`` comes from the manifests, so the class can log the count before + a single tar is opened. + """ + + def __init__(self, chosen): + self._chosen = chosen + + def __len__(self): + return sum(len(names) for _, names in self._chosen) + + def __iter__(self): + for tar_path, names in self._chosen: + with tarfile.open(tar_path) as tf: + for name in names: + member = tf.extractfile(name) + if member is None: + sys.exit(f"merge_star_cat: {tar_path} has no member " + f"{name}, which its manifest lists") + yield [io.BytesIO(member.read()), name] def main() -> None: @@ -196,8 +226,9 @@ def main() -> None: level=logging.INFO, stream=sys.stdout) manifest_paths = manifests(args.products_dir, args.tile_list, args.index_db) - file_list, n_tars, empty = entries(manifest_paths, args.pattern) - if not file_list: + chosen, empty = selection(manifest_paths, args.pattern) + file_list = TarMembers(chosen) + if not len(file_list): # Not a no-op: an empty star catalogue would pass every downstream # existence check and produce meaningless rho statistics. sys.exit(f"merge_star_cat: no member matched {args.pattern!r} in any " @@ -224,8 +255,8 @@ def main() -> None: log.info(f"unchanged: {args.output}") else: tmp.replace(args.output) # atomic: same filesystem - log.info(f"{len(file_list)} catalogue(s) from {n_tars} exposure(s) " - f"-> {args.output}") + log.info(f"{len(file_list)} catalogue(s) from {len(chosen)} " + f"exposure(s) -> {args.output}") finally: shutil.rmtree(tmp_dir, ignore_errors=True) diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index e78d3e9a4..40e8c9354 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -127,10 +127,20 @@ def main() -> None: args.dest.mkdir(parents=True, exist_ok=True) tar_path = args.dest / f"{args.exp}.tar" + # A file matched by TWO patterns is one file, not a collision. Keep lists + # overlap on purpose — `validation_psf-*.fits` alongside `*.fits` is a + # perfectly ordinary way to say "the validation catalogues, and everything + # else FITS while we are here" — and treating the second match as a name + # clash failed every exposure in the campaign. What must still be fatal is + # two DIFFERENT paths landing on one flat member name, which would silently + # overwrite; that is a same-name/different-source test, and the first + # pattern to match a file is the one recorded for it. seen, files = {}, [] for pat, hits in found.items(): for src in hits: if src.name in seen: + if seen[src.name][0] == src: + continue # same file, a second matching pattern sys.exit(f"persist_exp: {args.exp}: two source files are both " f"named {src.name} ({seen[src.name][0]} and {src}); tar " f"members are flat, so this would silently overwrite") From d774cc3663fcb31148e932a39bbae2392dae8eba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:18:02 -0400 Subject: [PATCH 09/20] fix(create_final_cat): make the merged-catalogue writer reproducible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects in scripts/python/create_final_cat.py, fixed where they live rather than worked around in the workflow rule that now calls it. A hand-run of the tool deserves them as much as the rule does, and files it has already written carry the first one. copy_data allocated np.empty with the SOURCE catalogue's full dtype and then filled only the requested columns, so every column NOT in the parameter file reached the hdf5 file as uninitialised memory: meaningless values, and different bytes on every run over the same inputs. It allocates the requested columns alone now, in the source catalogue's order. The parameter file says what the merged catalogue is for; those are the columns it gets. read_param_file returned list(set(...)), whose order varies with the process's string hash seed. Column order is part of a structured dtype and therefore part of the file, so two runs over the same inputs disagreed. Ordered dedup via dict.fromkeys. (The duplicate-count message also only printed for more than one duplicate, and said {n} literally.) read_data wrapped its column selection in a bare `except:` that printed and fell through, leaving its return values unbound — so a missing column surfaced to the caller as UnboundLocalError from the return statement, naming neither the file nor the column. It raises a KeyError naming the file and every missing column, in parameter-file order. process()'s own create_dataset follows the array copy_data returns rather than the source dtype, which are no longer the same thing. merge_final_cat.py drops the equivalents it had been carrying at the call site and relies on the fixed functions. The fixture hdf5 is byte-identical either way (md5 6de2d261…): the workaround and the fix produce the same file, which is the point. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- scripts/python/create_final_cat.py | 59 ++++++++++++++++++---------- workflow/scripts/merge_final_cat.py | 61 +++++------------------------ 2 files changed, 47 insertions(+), 73 deletions(-) diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 2b583b857..efef24620 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -144,13 +144,17 @@ def read_param_file(path, verbose=False): print("No parameters read", end="") print(" into merged catalogue") - param_list_unique = list(set(param_list)) - + # Ordered dedup. list(set(...)) reordered the columns by the process's + # string hash seed, so two runs of this tool over the same inputs produced + # files whose datasets differed in column ORDER — which is part of a + # structured dtype, and therefore part of the file. + param_list_unique = list(dict.fromkeys(param_list)) + if verbose: n = len(param_list) - len(param_list_unique) - if n > 1: - print("Removed {n} duplicate entries") - + if n > 0: + print(f"Removed {n} duplicate entries") + return param_list_unique @@ -312,16 +316,20 @@ def read_data(fits_file, params): if params["param_list"] is None: params["param_list"] = [col for col in data.keys()] - try: - extracted_data = {col: data[col] for col in params["param_list"]} - dtype = data.dtype - except: - print(f"Error for ID {id}, path {fits_file}") - for col in params["param_list"]: - if col not in data: - print(col, end=" ") - print() - continue + # RAISE, do not print and fall through. The bare `except:` this replaces + # left extracted_data and dtype unbound, so the caller's own error was an + # UnboundLocalError from the return statement below, naming neither the + # file nor the column that was actually missing. + present = set(data.dtype.names or ()) + missing = [col for col in params["param_list"] if col not in present] + if missing: + raise KeyError( + f"{fits_file}: missing {len(missing)} of the " + f"{len(params['param_list'])} requested column(s): " + f"{' '.join(missing)}" + ) + extracted_data = {col: data[col] for col in params["param_list"]} + dtype = data.dtype return extracted_data, dtype @@ -330,16 +338,23 @@ def copy_data(param_list, extracted_data, dtype): """Copy Data. """ + # THE REQUESTED COLUMNS ONLY, in the SOURCE catalogue's order. Allocating + # with the source's full dtype and filling only the requested columns left + # every other column as uninitialised memory: meaningless values in the + # output file, and different bytes on every run of this tool over the same + # inputs. The parameter file says which columns the merged catalogue is + # for; those are the columns it gets. + columns = [col for col in (dtype.names or ()) if col in set(param_list)] + subset = np.dtype([(col, dtype[col]) for col in columns]) + # Initialize new data structure structured_data = np.empty( len(extracted_data[param_list[0]]), - dtype=dtype, + dtype=subset, ) # Loop over parameters - for col in param_list: - if not col in extracted_data: - print(f"Column {col} not in file with ID {id}") + for col in columns: structured_data[col] = extracted_data[col] #if isinstance(extracted_data[col][0], (np.ndarray, tuple, list)): @@ -467,12 +482,14 @@ def process(params): structured_data = copy_data(params["param_list"], extracted_data, dtype) - # Create a new dataset + # Create a new dataset. dtype comes from the array copy_data + # built, not from the source catalogue: they differ now that + # copy_data allocates the requested columns alone. try: patch_group.create_dataset( str(id), data=structured_data, - dtype=dtype, + dtype=structured_data.dtype, ) except: print(f"Error for {id}: Could not create dataset in group {patch}") diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py index c12d5fcee..58357fcd5 100644 --- a/workflow/scripts/merge_final_cat.py +++ b/workflow/scripts/merge_final_cat.py @@ -18,6 +18,13 @@ ``create_final_cat.py``'s — ``read_param_file`` for the parameter list, ``read_data`` and ``copy_data`` for pulling those columns out of one catalogue with their FITS dtypes — so the column grammar keeps exactly one definition. +Those three are REPRODUCIBLE FUNCTIONS, and this PR is what made them so: the +parameter list comes back ordered rather than through a set, ``copy_data`` +allocates the requested columns alone rather than leaving every other column of +the source as uninitialised memory, and a missing column raises with its own +name instead of falling out of a bare ``except:`` as an UnboundLocalError. The +fixes are upstream, in that script, because a hand-run of it deserves them as +much as this rule does. Its ``process()`` is NOT used and neither is any of its discovery: that function walks a directory tree the workflow does not have and never will, and it groups by a unit ShapePipe v2 no longer has. This script walks the workflow's own @@ -46,23 +53,6 @@ are visited in sorted ID order so the file is a function of the input set alone. An unconditional rewrite would move the output's mtime every invocation. -TWO PLACES WHERE THE REFERENCE IMPLEMENTATION IS NOT DETERMINISTIC, and where -this script therefore pins the behaviour down rather than copying it. Both are -in the DTYPE, and both are invisible when a human runs the tool once by hand: - - * ``copy_data`` allocates ``np.empty`` with the SOURCE catalogue's full - dtype and then fills only the requested columns, so every column NOT in - ``final_cat.param`` reaches the hdf5 file as uninitialised memory — - different bytes on every run, and meaningless data in the file besides. We - hand ``copy_data`` a dtype restricted to the requested columns, so every - field it writes is a field it fills. The file then carries exactly the - ``final_cat.param`` columns, which is what sp_validation reads and what the - parameter file is for. - * ``read_param_file`` returns ``list(set(...))``, whose order varies with the - process's string hash seed. Column ORDER in a structured dtype is part of - the file, so that alone would defeat the byte comparison. We order the - fields by the source catalogue's own column order instead. - WHICH TILES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set is the CAMPAIGN's: every tile both declared in ``tile_list`` and present in the index, which is exactly the Snakefile's TILES_READY, rebuilt here from the same @@ -89,8 +79,6 @@ from pathlib import Path import h5py -import numpy as np -from astropy.io import fits # Same directory; the rule invokes this file by path, so it is sys.path[0]. import build_index @@ -144,16 +132,6 @@ def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: return out -def check_columns(path: Path, hdu: int, wanted: list) -> None: - """Fail loudly, and by name, when a catalogue lacks a requested column.""" - with fits.open(path, memmap=False) as hdu_list: - present = set(hdu_list[hdu].columns.names) - missing = sorted(c for c in wanted if c not in present) - if missing: - sys.exit(f"merge_final_cat: {path} is missing {len(missing)} of the " - f"{len(wanted)} requested column(s): {' '.join(missing)}") - - def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--products-dir", required=True, type=Path, @@ -193,31 +171,10 @@ def main() -> None: tmp.unlink(missing_ok=True) # h5py "a" would reopen a stale one with h5py.File(tmp, "w") as hdf5_file: group = hdf5_file.create_group(spval_group(args.campaign)) - columns = None for tile, path in tiles: - # BEFORE read_data, and not inside it. read_data wraps its - # column selection in a bare `except:` that prints and falls - # through, so a missing column leaves its return values unbound - # and the caller sees UnboundLocalError from the return - # statement — the real name, and every other missing name, never - # reaches the caller at all. Reading the header costs nothing - # next to reading the table. - check_columns(path, args.hdu, params["param_list"]) extracted, dtype = cfc.read_data(str(path), params) - # Requested columns, in the SOURCE catalogue's order (see the - # module docstring on determinism). Computed from the first - # tile and reused, so a tile whose catalogue is missing a - # column fails loudly on the assignment rather than quietly - # producing a differently-shaped dataset. - if columns is None: - columns = [c for c in dtype.names - if c in set(params["param_list"])] - subset = np.dtype([(c, dtype[c]) for c in columns]) - group.create_dataset( - tile, - data=cfc.copy_data(columns, extracted, subset), - dtype=subset, - ) + data = cfc.copy_data(params["param_list"], extracted, dtype) + group.create_dataset(tile, data=data, dtype=data.dtype) # The same attribute create_final_cat.py's print_list() writes, and # what sp_validation reads to know how many tiles it is holding. hdf5_file.attrs["n_tiles"] = len(tiles) From 98532ceccad08c0e5051e03a733be2f71fc1557a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:22:07 -0400 Subject: [PATCH 10/20] perf(orchestration): size the two merges from the data, not from a guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both merges had a constant mem_mb, which is wrong by however much a campaign differs from the one it was tuned on — and these are the only two rules whose single job grows with the whole campaign. Both are now measured slopes, evaluated against the campaign's own bytes at DAG build, still * attempt. STAR SIDE, measured on this login node in the campaign container over synthetic tars, 20 and 80 exposures of 40 CCDs x 400 stars: input members peak RSS (getrusage RUSAGE_CHILDREN) 32.3 MB 383 MB 129.0 MB 1313 MB a slope of 10.1x input bytes over a ~73 MB interpreter floor. Tenfold because MergeStarCatPSFEX accumulates every column into python LISTS of python floats before building the output arrays. THE CONSEQUENCE IS A CEILING and the Snakefile says so: at ~2 MB of members per exposure a 16 GB job merges roughly 800 exposures, and DR6's ~20k would want ~400 GB. A full-survey full_starcat needs that accumulation changed to preallocated arrays or a two-pass count — a change to MergeStarCatPSFEX, not to this rule, and not in this PR. The formula is honest about the slope so the job asks for what it will use and fails at submission rather than most of the way through. TILE SIDE, measured against smk-g6's real catalogues, 2 tiles (73.9 MB in, largest 39.6 MB) and 6 tiles (235.5 MB in, largest 47.7 MB): peak RSS 129 MB and 139 MB. FLAT in the tile count, because the merge holds one catalogue at a time — so it is sized on the LARGEST tile at ~3x, not on the total. Runtime is the total, since every tile is read end to end. On smk-g6's 64 tiles the rule resolves to mem_mb=1002, runtime=51, against the flat 8000/120 it had. Sizes come from stat() on the tar or the catalogue, falling back to the measured per-unit default when a fresh campaign has not produced it yet. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- workflow/Snakefile | 72 +++++++++++++++++++++++++++++++++++++ workflow/rules/exposure.smk | 12 +++++-- workflow/rules/tile.smk | 11 ++++-- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 587f271b2..8889b7f38 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -704,6 +704,78 @@ def star_cat_exposures(): or not exp_store_reclaimed(e)) +# --- sizing the two merges (D4) --------------------------------------------- +# MEASURED, not guessed, and measured as a SLOPE rather than a single number: +# these are the only two rules whose one job's footprint grows with the whole +# campaign, so a constant is wrong by however much the campaign is not the one +# it was tuned on. +# +# Both slopes were measured on this login node, inside the campaign container, +# against synthetic tars for the star side and against smk-g6's real +# catalogues for the tile side. Peak RSS is getrusage(RUSAGE_CHILDREN). +# +# STAR SIDE, and it is the alarming one. Two points, 20 and 80 exposures of 40 +# CCDs x 400 stars (1.6 MB of members per exposure, against the 2.0 MB measured +# on smk-m2): +# +# input members peak RSS +# 32.3 MB 383 MB +# 129.0 MB 1313 MB +# +# a slope of 10.1x the input bytes and an intercept of ~73 MB (interpreter, +# astropy, shapepipe). Tenfold, because MergeStarCatPSFEX accumulates every +# column into PYTHON LISTS of python floats before building the output arrays — +# 8 bytes of payload becomes a 32-byte object plus an 8-byte pointer. THE +# CONSEQUENCE IS A CEILING, and it should be said plainly: at ~2 MB per +# exposure, a 16 GB job merges roughly 800 exposures, and DR6's ~20k exposures +# would want ~400 GB. A full-survey full_starcat needs the accumulation changed +# to preallocated arrays or a two-pass count — a change to MergeStarCatPSFEX, +# not to this rule, and not in this PR. The formula below is honest about the +# slope so the job asks for what it will use and fails at submission rather +# than at 90% of the way through a campaign-length merge. +# +# TILE SIDE, and it is the reassuring one. Two points against real smk-g6 +# catalogues, 2 tiles (73.9 MB in, largest 39.6 MB) and 6 tiles (235.5 MB in, +# largest 47.7 MB): peak RSS 129 MB and 139 MB. FLAT IN THE NUMBER OF TILES — +# the merge holds one catalogue at a time — so it is sized on the LARGEST tile, +# not the total, at ~3x it plus the interpreter. +STAR_MEM_BASE_MB = 500 # interpreter + astropy + shapepipe, rounded up +STAR_MEM_FACTOR = 12 # x input bytes; 10.1 measured, rounded up +FINAL_MEM_BASE_MB = 800 +FINAL_MEM_FACTOR = 4 # x the LARGEST tile; ~3 measured +# What one unit costs when its product is not on disk yet to be stat()ed — a +# fresh campaign sizes its merge before anything has been packed or made. +# Both are the measured medians in config.yaml's persist_exp block and D5 notes. +EXP_BYTES_DEFAULT = 2_000_000 +TILE_BYTES_DEFAULT = 46_000_000 + + +def _size(path, default): + """Bytes on disk, or the documented per-unit default if it is not there.""" + try: + return Path(path).stat().st_size + except OSError: + return default + + +def star_cat_bytes(): + """Total member bytes the star merge will read. + + The TAR is what gets stat()ed, not the manifest: it is one stat per + exposure rather than a json parse, and it is present for exactly the + exposures whose products already exist — live-and-already-packed as well as + reclaimed. An exposure not yet packed contributes the measured default. + """ + return sum(_size(prod_exp_tar(e), EXP_BYTES_DEFAULT) + for e in star_cat_exposures()) + + +def final_cat_max_bytes(): + """The LARGEST tile catalogue the hdf5 merge will read — what sizes it.""" + return max([_size(final_cat(t), TILE_BYTES_DEFAULT) for t in TILES_READY] + or [TILE_BYTES_DEFAULT]) + + def star_cat_targets(): """`full_starcat` when there is anything to stack into it, else nothing. diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 7b6d33882..789e9da1d 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -278,8 +278,16 @@ rule star_cat_merge: script_hash = MERGE_STAR_HASH threads: 1 resources: - mem_mb = lambda wc, attempt: 16000 * attempt, - runtime = 120 + # Sized on the campaign's own member bytes, slope and intercept + # measured (the Snakefile's sizing block carries both points, and the + # ceiling this rule runs into at DR6 scale). Still * attempt, because a + # measured slope on synthetic tars is not a guarantee about real ones. + mem_mb = lambda wc, attempt: attempt * ( + STAR_MEM_BASE_MB + STAR_MEM_FACTOR * star_cat_bytes() // 1_000_000), + # ~2 min per GB of members on the measurement above, doubled, over a + # floor that covers the fixed cost of opening ~40 members per exposure. + runtime = lambda wc, attempt: attempt * ( + 30 + 4 * star_cat_bytes() // 1_000_000_000) shell: "set -euo pipefail\n" f"python {SCRIPTS}/merge_star_cat.py" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index ed5daceb4..4eb293b04 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -948,8 +948,15 @@ rule final_cat_merge: script_hash = MERGE_FINAL_HASH threads: 1 resources: - mem_mb = lambda wc, attempt: 8000 * attempt, - runtime = 120 + # Sized on the LARGEST tile, not the total: the merge holds one + # catalogue at a time, and the measurement is flat in the tile count + # (the Snakefile's sizing block carries both points). + mem_mb = lambda wc, attempt: attempt * ( + FINAL_MEM_BASE_MB + + FINAL_MEM_FACTOR * final_cat_max_bytes() // 1_000_000), + # Runtime, unlike memory, is the TOTAL: every tile is read end to end. + # ~1 min per 10 tiles on the measurement, triply generous, over a floor. + runtime = lambda wc, attempt: attempt * (30 + len(TILES_READY) // 3) shell: "set -euo pipefail\n" f"python {SCRIPTS}/merge_final_cat.py" From d0ecfdd5283455460f333e0b5d556b7c9f2d241a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:25:23 -0400 Subject: [PATCH 11/20] feat(orchestration): persist_exp keeps NAMED products, not globs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the readability half of CosmoStat/shapepipe#844, and makes the 2026-09-08 call's request — keep the PSF model — something you can write down as `psf_model` rather than `*.psf`. `persist_exp:` entries are now names from a catalogue in persist_exp.py, which is the single source of truth for what each one means, what it costs per exposure and what keeping it buys: psf_validation validation_psf-*.fits 2.0 MB psf_model *.psf 2.8 MB psfex_cat psfex_cat-*.cat unmeasured star_selection star_selection-*.fits 24.5 MB star_train star_split_ratio_80-*.fits 19.9 MB star_test star_split_ratio_20-*.fits 7.1 MB star_stats star_stat-*.txt unmeasured `persist_exp.py --list-products` renders it, and config.yaml's block IS that rendering rather than a second copy of it — the old block was a long comment listing globs and their sizes, maintained by hand beside the code that actually knew them. THE DEFAULT BECOMES psf_validation + psf_model, ~4.8 MB per exposure. The model is the single most capability-adding thing an exposure can keep: with it the PSF can be re-interpolated at any position later without rebuilding the chain from VOS, and without it that capability dies with the /scratch purge. A raw glob is still accepted as an escape hatch for a file the catalogue does not name yet. The test is syntactic and cheap — a glob metacharacter or a dot means glob, a bare identifier means name — so `*.psf` and `psf_model` cannot be confused. An unknown NAME is a parse-time WorkflowError listing the valid ones, not a silently empty keep or a per-exposure failure an hour in. The manifest records both: `products` as written, `patterns` resolved, and each member's own `product`. star_cat_merge's gate and its member glob resolve through the same catalogue, so adding a product cannot leave the two disagreeing, and the "add this to persist_exp" hint now names the product. Tile-side retention is explicitly out of scope and noted as such in config.yaml: final_cat is the only tile product that persists today, and it is written straight to products_dir by tile_make_cat. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- workflow/README.md | 25 ++++- workflow/Snakefile | 29 ++++-- workflow/config.yaml | 118 +++++++++++---------- workflow/scripts/merge_star_cat.py | 16 +-- workflow/scripts/persist_exp.py | 159 +++++++++++++++++++++++++++-- 5 files changed, 267 insertions(+), 80 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index eb801da6a..1ad740b48 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -245,8 +245,7 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee Know the consequence — `--forcerun` on a tile whose `final_cat` exists will not rebuild its reclaimed exposures. Delete the `final_cat` first. - **PSF products leave scratch before the purge does.** `exp_persist` packs - the files named by `persist_exp:` in `config.yaml` (default: the psfex_interp - `validation_psf-*.fits`, the rho/tau statistics input) from the exposure's + the products named by `persist_exp:` in `config.yaml` from the exposure's scratch store into ONE uncompressed tar, `/exp///psf/.tar` (inodes, not bytes, bind on /project), and writes ONE manifest beside it recording the patterns, the @@ -260,6 +259,28 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee hours of PSF fitting per exposure. A pattern that matches nothing is a recorded warning (setools rejects sparse CCDs); matching nothing at all is a failure. A `localrule`, by the same arithmetic as `clean_exposure`. +- **The keep list names products, not globs.** `persist_exp:` entries are names + from a catalogue in `workflow/scripts/persist_exp.py`, which is the single + source of truth for what each one means and what keeping it buys + ([#844](https://github.com/CosmoStat/shapepipe/issues/844)); `config.yaml`'s + block is that catalogue rendered, and + `persist_exp.py --list-products` prints it. Sizes are per exposure, 40 CCDs, + measured on smk-m2. + + | product | glob | per exposure | what it buys | + |---|---|---|---| + | `psf_validation` | `validation_psf-*.fits` | 2.0 MB | the rho/tau statistics input, and `star_cat_merge`'s | + | `psf_model` | `*.psf` | 2.8 MB | re-interpolate the PSF anywhere later, no rebuild | + | `psfex_cat` | `psfex_cat-*.cat` | unmeasured | which stars PSFEx's outlier rejection clipped | + | `star_selection` | `star_selection-*.fits` | 24.5 MB | which stars the selection cuts rejected, and why | + | `star_train` | `star_split_ratio_80-*.fits` | 19.9 MB | the 80% sample PSFEx fitted | + | `star_test` | `star_split_ratio_20-*.fits` | 7.1 MB | the 20% sample `psf_validation` corresponds to | + | `star_stats` | `star_stat-*.txt` | unmeasured | setools' per-CCD counts, density and FWHM cuts | + + The default is `psf_validation` + `psf_model`. A raw glob is still accepted as + an escape hatch — anything with a glob metacharacter or a dot is read as one — + and an unknown *name* is a parse-time error listing the valid ones. The list + is exposure-side only; tile-side retention is #844 follow-up. - **The campaign ends in two merged catalogues, and the workflow now makes both.** Everything above is per unit; the two products downstream analysis actually opens are per *campaign*, and until these rules existed each was a diff --git a/workflow/Snakefile b/workflow/Snakefile index 8889b7f38..8f871df3b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -472,6 +472,21 @@ def clean_targets(): # deliberate "keep nothing" and produces no jobs at all. PERSIST_EXP = list(config.get("persist_exp") or []) +# The keep list names PRODUCTS (`psf_model`), not globs (`*.psf`); the +# catalogue that maps one to the other lives in persist_exp.py, which is also +# what the rule runs, so there is one definition and not a copy here. +# UNKNOWN NAMES DIE AT PARSE TIME, listing the valid ones — a typo in a keep +# list would otherwise be a silently-empty keep or a per-exposure failure an +# hour into a campaign. +import persist_exp as _persist # noqa: E402 + +for _entry in PERSIST_EXP: + try: + _persist.resolve(_entry) + except KeyError as _exc: + raise WorkflowError(f"config persist_exp: {_exc.args[0]}") +PERSIST_GLOBS = [_persist.resolve(e) for e in PERSIST_EXP] + def persist_targets(): """Which exposures this invocation must pack PSF products off scratch for. @@ -601,12 +616,14 @@ def unit_fingerprint(units): return f"{len(units)}:{hashlib.md5(joined.encode()).hexdigest()[:12]}" -# The tar member the star merge consumes. The keep list is globs, so the test is -# "would this member be kept", not a string comparison — `validation_psf-*.fits`, -# `validation_psf*`, `*.fits` and a bare `*` all say yes, and all are things a -# user might reasonably write. +# The tar member the star merge consumes, and the test is "would this member be +# kept" rather than "is `psf_validation` in the list". Both spellings must count: +# the product name, and a raw glob that happens to cover it (`validation_psf*`, +# `*.fits`, a bare `*`) — all things a user might reasonably write. So the keep +# list is RESOLVED to globs first and the member matched against those. +_STAR_CAT_PRODUCT = "psf_validation" _STAR_CAT_MEMBER = "validation_psf-2605805-12.fits" -STAR_CAT_MERGE = any(fnmatch.fnmatch(_STAR_CAT_MEMBER, p) for p in PERSIST_EXP) +STAR_CAT_MERGE = any(fnmatch.fnmatch(_STAR_CAT_MEMBER, g) for g in PERSIST_GLOBS) # LOUD AT PARSE TIME, once, and only where it can be acted on. A keep list # without the validation catalogues is a legitimate configuration (persist the @@ -620,7 +637,7 @@ if PERSIST_EXP and not STAR_CAT_MERGE and workflow.is_main_process \ f"star_cat_merge: no job — persist_exp {PERSIST_EXP} keeps no " f"'{_STAR_CAT_MEMBER}'-shaped file, so there is nothing to stack into " f"{PRODUCTS_DIR}/full_starcat-0000000.fits (the rho/tau statistics " - f"input). Add 'validation_psf-*.fits' to persist_exp: to get it.") + f"input). Add '{_STAR_CAT_PRODUCT}' to persist_exp: to get it.") def full_starcat(): diff --git a/workflow/config.yaml b/workflow/config.yaml index 70ace906e..03c282113 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -82,15 +82,54 @@ outputs: index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite # Per-exposure PSF products to carry onto the persistent root before the scratch -# store goes (`exp_persist`, exposure.smk). A list of plain file-name globs, -# matched recursively under the PSF chain's four module output dirs -# (/exp///output/run_sp_exp_SxSePsfPi/*/output/ — -# sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner). +# store goes (`exp_persist`, exposure.smk). A list of PRODUCT NAMES — not globs. +# The catalogue below is the rendering of workflow/scripts/persist_exp.py's +# PRODUCTS table, which is the single source of truth for what each name means +# and what keeping it buys (CosmoStat/shapepipe#844); print it any time with +# +# workflow/bin/sp container exec python workflow/scripts/persist_exp.py --list-products +# +# product glob size/exposure +# -------------- -------------------------- ------------- +# star_selection star_selection-*.fits 24.5 MB +# setools' PRE-SPLIT selection. The only file that answers which +# stars the selection cuts rejected and why; the split samples have +# already lost the rejects. +# star_train star_split_ratio_80-*.fits 19.9 MB +# the 80% TRAINING sample, the stars PSFEx actually fitted. Rows +# duplicate star_selection. +# star_test star_split_ratio_20-*.fits 7.1 MB +# the 20% VALIDATION sample — the positions psf_validation's rows +# correspond to. Rows duplicate star_selection. +# star_stats star_stat-*.txt unmeasured +# setools' per-CCD STAT block: star counts, stars/deg^2, FWHM mode +# and cuts. The selection's summary without its catalogue. +# psf_model *.psf 2.8 MB +# the PSFEx model itself. Keeping it means the PSF can be +# re-interpolated at ANY position later without rebuilding the +# exposure chain — the single most capability-adding entry here. +# psfex_cat psfex_cat-*.cat unmeasured +# PSFEx's own output catalogue (FITS_LDAC): the per-star FLAGS_PSF +# and CHI2_PSF, i.e. WHICH stars outlier rejection clipped. Not +# recoverable from anything else — the .psf header keeps only the +# LOADED/ACCEPTED counts. +# psf_validation validation_psf-*.fits 2.0 MB +# the psfex_interp validation catalogue, one per CCD: the input to +# the rho/tau statistics, and to the star_cat_merge rule that stacks +# them into the campaign's full_starcat. +# +# A RAW GLOB IS STILL ACCEPTED, as an escape hatch for a file the catalogue does +# not name yet: anything carrying a glob metacharacter or a dot is taken as a +# glob rather than a name (`*.psf` is a glob, `psf_model` is the name for it). +# An unknown NAME is a parse-time error listing the valid ones, never a silently +# empty keep. +# # Matches are packed, flat, into ONE uncompressed tar per exposure: # /exp///psf/.tar, with a manifest listing the -# members beside it. One tar rather than loose copies because inodes, not bytes, -# bind on /project (~1 M-file group quota; loose copies would be ~200 files per -# exposure, ~2 M at DR6 scale). FITS members read straight from the tar: +# members (and the product each came from) beside it. One tar rather than loose +# copies because inodes, not bytes, bind on /project (~1 M-file group quota; +# loose copies would be ~200 files per exposure, ~2 M at DR6 scale). FITS +# members read straight from the tar: # fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())). # # WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat. @@ -104,56 +143,24 @@ outputs: # reruns the packing (seconds) and NOT exp_psf (four hours per exposure). That # separation is the whole reason exp_persist is a rule of its own. # -# The default is the minimum: the psfex_interp VALIDATION catalogue, one per -# CCD, which is the input to the rho/tau statistics. Without it the PSF -# diagnostics cannot be recomputed after a purge without rebuilding the exposure -# chain from VOS. +# THE DEFAULT is psf_validation + psf_model (~4.8 MB per exposure): the rho/tau +# statistics input, and the model that lets the PSF be re-interpolated at any +# position later without rebuilding the exposure chain from VOS. Add psfex_cat +# for a production run if you want to know which stars PSFEx clipped; the +# star_* products are for selection studies and cost an order of magnitude more. +# +# THIS LIST IS EXPOSURE-SIDE ONLY. Tile-side retention is not configurable: the +# only tile product that persists today is final_cat, written by tile_make_cat +# straight to products_dir. A tile keep list is #844 follow-up. # # THIS LIST GATES `star_cat_merge`. That campaign-level rule stacks every -# exposure's every CCD's validation_psf into ONE +# exposure's every CCD's psf_validation into ONE # /full_starcat-0000000.fits, reading the members straight out of -# the tars. A keep list that matches no `validation_psf-*.fits` is a legitimate -# configuration (keep the PSF models alone, say) and produces NO merge job and a -# warning at parse time — not a failure on a node an hour later. Note the -# corollary: an exposure already reclaimed by a workflow that predates -# exp_persist left no tar, so it contributes nothing and cannot be recovered -# short of rebuilding its chain from VOS. -# -# OPT-IN CANDIDATES, and what each buys. Sizes are per exposure (40 CCDs), -# measured on smk-m2 (127 exposures, 64 tiles); a 64-tile campaign with all of -# the measured ones on came to 7.2 GB: -# validation_psf-*.fits (the default) 2.0 MB -# *.psf the PSFEx model itself. Keeping it means the PSF -# can be re-interpolated at ANY position later -# without rebuilding the exposure chain — the -# single most capability-adding entry here. -# 2.8 MB -# psfex_cat-*.cat PSFEx's own output catalogue (FITS_LDAC): the -# per-star FLAGS_PSF / CHI2_PSF, i.e. WHICH stars -# outlier rejection clipped. Not recoverable from -# anything else (the .psf header keeps only the -# LOADED/ACCEPTED counts). unmeasured -# star_selection-*.fits the PRE-SPLIT selection (setools writes it under -# mask/). The only file that can answer "which -# stars were rejected by the selection cuts, and -# why" — the split samples have already lost the -# rejects. 24.5 MB -# star_split_ratio_80-*.fits setools' 80% TRAINING star sample, the set PSFEx -# actually fitted. Rows duplicate star_selection. -# 19.9 MB -# star_split_ratio_20-*.fits the 20% VALIDATION sample — the positions the -# validation_psf rows correspond to. Rows -# duplicate star_selection. 7.1 MB -# star_stat-*.txt setools' per-CCD STAT block (star counts, -# stars/deg^2, FWHM mode and cuts, under stat/): -# the selection's summary without its catalogue. -# unmeasured -# A production keep list is `validation_psf` + `*.psf` + `psfex_cat` (~5 MB per -# exposure); the star_split files are only worth it if star_selection is off. -# PSFEx residual/check images and its XML diagnostics are NOT candidates as the -# chain stands: the committed default.psfex sets CHECKIMAGE_TYPE NONE and -# WRITE_XML N, so nothing is emitted to match. They are a config change first, -# a pattern second. +# the tars. A keep list without psf_validation is a legitimate configuration and +# produces NO merge job and a warning at parse time — not a failure on a node an +# hour later. Note the corollary: an exposure already reclaimed by a workflow +# that predates exp_persist left no tar, so it contributes nothing and cannot be +# recovered short of rebuilding its chain from VOS. # # NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the tar # then lands beside the store on the same filesystem and buys nothing, and the @@ -161,7 +168,8 @@ outputs: # deletes wholesale — so a one-root run re-persists after every reclamation. # Harmless, and exactly the pre-D5 behaviour a one-root run asks for. persist_exp: - - validation_psf-*.fits + - psf_validation + - psf_model # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py index 25c6e4c22..002d156ff 100644 --- a/workflow/scripts/merge_star_cat.py +++ b/workflow/scripts/merge_star_cat.py @@ -97,6 +97,7 @@ class as ``[fileobj, member_name]`` pairs — ONE AT A TIME, lazily, through # Same directory; the rule invokes this file by path, so it is sys.path[0]. import build_index +import persist_exp # The output name is not ours to choose: sp_validation hardcodes it # (`star_cat_path = f"{data_dir}/full_starcat-0000000.fits"`), and @@ -104,11 +105,13 @@ class as ``[fileobj, member_name]`` pairs — ONE AT A TIME, lazily, through # given. Kept here as the name this script promises to produce. OUT_NAME = "full_starcat-0000000.fits" -# The keep-list pattern whose members this merge consumes. The rule refuses to -# exist unless `persist_exp:` contains a pattern matching this shape (the -# Snakefile does that check at parse time), so by the time we get here the -# members are expected to be present. -MEMBER_PATTERN = "validation_psf-*.fits" +# The members this merge consumes, named as the keep list names them and +# resolved through the same catalogue persist_exp packs by — so the glob has one +# definition and adding a product cannot leave the two disagreeing. The rule +# refuses to exist unless `persist_exp:` keeps something of this shape (the +# Snakefile checks at parse time), so the members are expected here. +MEMBER_PRODUCT = "psf_validation" +MEMBER_PATTERN = persist_exp.resolve(MEMBER_PRODUCT) def merge_class(psf_model: str): @@ -233,7 +236,8 @@ def main() -> None: # existence check and produce meaningless rho statistics. sys.exit(f"merge_star_cat: no member matched {args.pattern!r} in any " f"of {len(manifest_paths)} exp_persist manifest(s) for this " - f"campaign — is '{args.pattern}' in the persist_exp keep list?") + f"campaign — is '{MEMBER_PRODUCT}' in the persist_exp keep " + f"list?") if empty: log.info(f"{len(empty)} exposure(s) persisted no {args.pattern}: " f"{', '.join(sorted(empty)[:5])}" diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index 40e8c9354..fef439b7b 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -86,40 +86,175 @@ # different rule. RUN_NAME = "run_sp_exp_SxSePsfPi" +# --- the product catalogue (CosmoStat/shapepipe#844) ------------------------ +# THE SINGLE SOURCE OF TRUTH for what an exposure can keep. `persist_exp:` in +# config.yaml names PRODUCTS, not globs: `psf_model`, not `*.psf`. The glob is +# an implementation detail of the module that writes the file, and a keep list +# written in globs is a keep list nobody can read — the argument that produced +# #844 and the 2026-09-08 call's request to keep the PSF model, which had to be +# spelled `*.psf` to be said at all. +# +# Each entry is (glob, per-exposure size, what keeping it buys). Sizes are for +# 40 CCDs, measured on smk-m2 (127 exposures, 64 tiles); "?" means not yet +# measured. `persist_exp.py --list-products` renders this table, and +# config.yaml's block is that rendering rather than a second copy of it. +# +# ORDER IS THE ORDER OF THE CHAIN — sextractor, setools, psfex, psfex_interp — +# so the table reads as the pipeline runs. +PRODUCTS = { + "star_selection": ( + "star_selection-*.fits", 24_500_000, + "setools' PRE-SPLIT selection. The only file that answers which stars " + "the selection cuts rejected and why; the split samples have already " + "lost the rejects."), + "star_train": ( + "star_split_ratio_80-*.fits", 19_900_000, + "the 80% TRAINING sample, the stars PSFEx actually fitted. Rows " + "duplicate star_selection."), + "star_test": ( + "star_split_ratio_20-*.fits", 7_100_000, + "the 20% VALIDATION sample — the positions psf_validation's rows " + "correspond to. Rows duplicate star_selection."), + "star_stats": ( + "star_stat-*.txt", None, + "setools' per-CCD STAT block: star counts, stars/deg^2, FWHM mode and " + "cuts. The selection's summary without its catalogue."), + "psf_model": ( + "*.psf", 2_800_000, + "the PSFEx model itself. Keeping it means the PSF can be " + "re-interpolated at ANY position later without rebuilding the exposure " + "chain — the single most capability-adding entry here."), + "psfex_cat": ( + "psfex_cat-*.cat", None, + "PSFEx's own output catalogue (FITS_LDAC): the per-star FLAGS_PSF and " + "CHI2_PSF, i.e. WHICH stars outlier rejection clipped. Not recoverable " + "from anything else — the .psf header keeps only the LOADED/ACCEPTED " + "counts."), + "psf_validation": ( + "validation_psf-*.fits", 2_000_000, + "the psfex_interp validation catalogue, one per CCD: the input to the " + "rho/tau statistics, and to the star_cat_merge rule that stacks them " + "into the campaign's full_starcat."), +} + +# PSFEx residual/check images and its XML diagnostics are deliberately absent: +# the committed default.psfex sets CHECKIMAGE_TYPE NONE and WRITE_XML N, so +# nothing is emitted to match. They are a config change first, a catalogue +# entry second. + +# A raw glob is still accepted, as an escape hatch for a file the catalogue does +# not name yet. The test is syntactic and deliberately cheap: a product name is +# a bare identifier, so anything carrying a glob metacharacter or a dot is a +# glob. That makes `*.psf`, `star_stat-*.txt` and `default.psfex` globs, and +# `psf_model` a name, with no ambiguity a user could stumble into. +_GLOBBY = set("*?[]. ") + + +def is_glob(entry: str) -> bool: + """True when this keep-list entry is a raw glob rather than a product name.""" + return any(ch in _GLOBBY for ch in entry) + + +def resolve(entry: str) -> str: + """The file-name glob for one keep-list entry, name or raw glob.""" + if is_glob(entry): + return entry + try: + return PRODUCTS[entry][0] + except KeyError: + raise KeyError( + f"unknown persist_exp product {entry!r}; the products are " + f"{', '.join(PRODUCTS)} (or write a raw glob such as '*.psf')" + ) from None + + +def product_of(entry: str) -> str: + """The NAME to record for an entry — the entry itself for a raw glob.""" + return entry + + +def render_products() -> str: + """The catalogue as a table, for --list-products and for config.yaml.""" + width = max(len(n) for n in PRODUCTS) + lines = [f"{'product'.ljust(width)} {'glob'.ljust(26)} size/exposure", + f"{'-' * width} {'-' * 26} -------------"] + for name, (glob, size, why) in PRODUCTS.items(): + size_s = "unmeasured" if size is None else f"{size / 1e6:.1f} MB" + lines.append(f"{name.ljust(width)} {glob.ljust(26)} {size_s}") + for i, chunk in enumerate(_wrap(why, 66)): + lines.append(f"{' ' * width} {chunk}") + return "\n".join(lines) + + +def _wrap(text: str, width: int) -> list: + out, line = [], "" + for word in text.split(): + if line and len(line) + 1 + len(word) > width: + out.append(line) + line = word + else: + line = f"{line} {word}".strip() + if line: + out.append(line) + return out + def collect(exp_dir: Path, patterns: list) -> tuple: - """Matched files per pattern, in a stable order, plus the empty patterns.""" + """Matched files per ENTRY, in a stable order, plus the entries that matched + nothing. Entries are product names or raw globs; resolve() takes either.""" root = exp_dir / "output" / RUN_NAME found, empty = {}, [] - for pat in patterns: + for entry in patterns: + pat = resolve(entry) # One glob per module output dir, recursive beneath it (see the module # docstring on setools' subdirectories). sorted() over the union keeps # the manifest byte-stable across filesystem readdir order. hits = sorted({p for mod in sorted(root.glob("*/output")) for p in mod.rglob(pat) if p.is_file()}) if hits: - found[pat] = hits + found[entry] = hits else: - empty.append(pat) + empty.append(entry) return found, empty def main() -> None: p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--exp-dir", required=True, type=Path, + p.add_argument("--exp-dir", type=Path, help="the exposure's scratch store") - p.add_argument("--exp", required=True) - p.add_argument("--dest", required=True, type=Path, + p.add_argument("--exp") + p.add_argument("--dest", type=Path, help="/exp///psf; the tar is " "/.tar") - p.add_argument("--manifest", required=True, type=Path) + p.add_argument("--manifest", type=Path) p.add_argument("--pattern", action="append", default=[], - help="repeatable; a plain file-name glob") + help="repeatable; a product name (see --list-products) or a " + "raw file-name glob") + p.add_argument("--list-products", action="store_true", + help="print the product catalogue and exit") args = p.parse_args() + # --list-products is a QUERY, not a run: it answers "what can I keep?" and + # needs no exposure, so the run arguments are optional at the parser and + # required here instead. + if args.list_products: + print(render_products()) + return + missing = [f"--{n.replace('_', '-')}" for n in + ("exp_dir", "exp", "dest", "manifest") + if getattr(args, n) is None] + if missing: + p.error(f"the following arguments are required: {', '.join(missing)}") + if not args.pattern: sys.exit("persist_exp: no --pattern given (config persist_exp is empty)") + for entry in args.pattern: # loud, and before any work + try: + resolve(entry) + except KeyError as exc: + sys.exit(f"persist_exp: {exc.args[0]}") + found, empty = collect(args.exp_dir, args.pattern) if not found: sys.exit(f"persist_exp: {args.exp}: no file matched any of " @@ -145,7 +280,8 @@ def main() -> None: f"named {src.name} ({seen[src.name][0]} and {src}); tar " f"members are flat, so this would silently overwrite") seen[src.name] = (src, pat) - files.append({"name": src.name, "pattern": pat, + files.append({"name": src.name, "product": pat, + "pattern": resolve(pat), "src": str(src), "bytes": src.stat().st_size}) files.sort(key=lambda f: f["name"]) @@ -176,7 +312,8 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: "stage": "exp_persist", "level": "exp", "unit": args.exp, "status": "complete", "tar": str(tar_path), - "patterns": list(args.pattern), + "products": list(args.pattern), + "patterns": [resolve(e) for e in args.pattern], # The warning the docstring argues for: named patterns that matched # nothing. Present as a key even when empty, so a reader never has to # wonder whether an old manifest predates the field. From 3b8c7d59ac9de8a258007ea2ce1cb121cd78500f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:32:30 -0400 Subject: [PATCH 12/20] perf(merge_starcat): accumulate arrays, not python lists of floats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three merge classes built every output column by extending a python list with one value per star: `x += list(data["X"])`. Four bytes of float32 payload became a 32-byte python object plus an 8-byte pointer in an overallocating list, measured end to end at ~10x the input bytes — which put a full-survey full_starcat (~20k exposures x 40 CCDs) at ~400 GB of RAM and out of reach of any node. One array per input catalogue per column, concatenated once at the end. Same values, same order, same dtypes — np.array() over a list of numpy scalars and np.concatenate() over the arrays they came from agree on both. The stacking helper empties the list it is handed, which is half the saving: concatenate holds the chunks and the result at once, so releasing column by column peaks at one campaign plus one column rather than two campaigns. MEASURED on the same two fixture points, 20 and 80 exposures of 40 CCDs x 400 stars: input members peak RSS, before after 32.3 MB 383 MB 238 MB 129.0 MB 1313 MB 740 MB 10.1x -> 5.5x, over a ~62 MB interpreter floor. The rule's mem_mb factor follows. A 16 GB job now merges ~2200 exposures rather than ~800. THE REMAINING 5.5x IS THE OUTPUT SIDE: file_io writes every float column as FITS 1D, so float32 inputs become a float64 table astropy then buffers. That is a change to the output FORMAT, which is what sp_validation reads, and a different decision from this one. BYTE-IDENTICAL OUTPUT, both ways in. The workflow's tar path and the module runner's plain [path] path produce the same file as before the change, md5 f7caa1cf… on the fixture — the runner path checked by calling MergeStarCatPSFEX directly with [[path]] entries as merge_starcat_runner builds them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- .../merge_starcat_package/merge_starcat.py | 247 +++++++++--------- workflow/Snakefile | 42 +-- 2 files changed, 153 insertions(+), 136 deletions(-) diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 2b1bf8c34..3463ccf43 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -17,6 +17,36 @@ from shapepipe.pipeline import file_io +def _stack(chunks, dtype=None): + """Concatenate one column's per-catalogue arrays into a single array. + + THE COLUMN ACCUMULATORS ARE LISTS OF ARRAYS, ONE PER INPUT CATALOGUE, and + not lists of values, because these classes are the last step of a whole + campaign. ``x += list(data["X"])`` turns 4 bytes of float32 payload into a + 32-byte python object plus an 8-byte pointer in a list that overallocates — + measured at ~10x the input bytes end to end, which put a full-survey merge + (~20k exposures x 40 CCDs) at ~400 GB of RAM and made it unrunnable on any + node. One array per catalogue plus one concatenate at the end holds ~1x, and + produces the identical output: np.array() over a list of numpy scalars and + np.concatenate() over the arrays they came from agree on dtype and on order. + + IT EMPTIES THE LIST IT IS GIVEN, and that is not a side effect to tidy away + later — it is half the saving. np.concatenate holds the chunks and the + result at once, so a caller that stacks sixteen columns while all sixteen + chunk lists are still alive peaks at twice the campaign. Released column by + column, the peak is one campaign plus one column. Callers stack once, at the + end, and do not touch the accumulators afterwards. + + An empty input list is a merge over no catalogues, which the callers guard + against; it returns an empty array so the output column still exists. + """ + if not chunks: + return np.array([], dtype=dtype or np.float64) + out = np.concatenate(chunks) + del chunks[:] + return out + + class MergeStarCatMCCD(object): """Merge Star Catalogue MCCD. @@ -320,66 +350,37 @@ def process(self): model_var.append(model_var_val) model_var_size.append(model_var_val.size) + # ONE ARRAY PER CATALOGUE PER COLUMN (see _stack): the per-value + # python lists this replaces cost ~10x the input bytes. # positions - x += list( - starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 0] - ) - y += list( - starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 1] - ) + pos = starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"] + x.append(np.asarray(pos[:, 0])) + y.append(np.asarray(pos[:, 1])) # RA and DEC positions try: - ra += list(starcat_j[self._hdu_table].data["RA_LIST"][:]) - dec += list(starcat_j[self._hdu_table].data["DEC_LIST"][:]) + ra.append(np.asarray(starcat_j[self._hdu_table].data["RA_LIST"][:])) + dec.append(np.asarray(starcat_j[self._hdu_table].data["DEC_LIST"][:])) except Exception: - ra += list( - np.zeros( - starcat_j[self._hdu_table] - .data["GLOB_POSITION_IMG_LIST"][:, 0] - .shape, - dtype=int, - ) - ) - dec += list( - np.zeros( - starcat_j[self._hdu_table] - .data["GLOB_POSITION_IMG_LIST"][:, 0] - .shape, - dtype=int, - ) - ) + ra.append(np.zeros(pos[:, 0].shape, dtype=int)) + dec.append(np.zeros(pos[:, 0].shape, dtype=int)) # shapes (convert sigmas to T = 2 sigma^2) - g1_psf += list( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 0] - ) - g2_psf += list( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 1] - ) - size_psf += list( - cs_size.sigma_to_T( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 2] - ) - ) - g1 += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 0]) - g2 += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 1]) - size += list( - cs_size.sigma_to_T( - starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 2] - ) - ) + psf_mom = starcat_j[self._hdu_table].data["PSF_MOM_LIST"] + star_mom = starcat_j[self._hdu_table].data["STAR_MOM_LIST"] + g1_psf.append(np.asarray(psf_mom[:, 0])) + g2_psf.append(np.asarray(psf_mom[:, 1])) + size_psf.append(np.asarray(cs_size.sigma_to_T(psf_mom[:, 2]))) + g1.append(np.asarray(star_mom[:, 0])) + g2.append(np.asarray(star_mom[:, 1])) + size.append(np.asarray(cs_size.sigma_to_T(star_mom[:, 2]))) # flags - flag_psf += list( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 3] - ) - flag_star += list( - starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 3] - ) + flag_psf.append(np.asarray(psf_mom[:, 3])) + flag_star.append(np.asarray(star_mom[:, 3])) # ccd id list - ccd_nb += list(starcat_j[self._hdu_table].data["CCD_ID_LIST"]) + ccd_nb.append(np.asarray(starcat_j[self._hdu_table].data["CCD_ID_LIST"])) starcat_j.close() @@ -452,15 +453,21 @@ def process(self): ) # Mask and transform to numpy arrays - flagmask = np.abs(np.array(flag_star) - 1) * np.abs( - np.array(flag_psf) - 1 - ) - psf_e1 = np.array(g1_psf)[flagmask.astype(bool)] - psf_e2 = np.array(g2_psf)[flagmask.astype(bool)] - psf_r2 = np.array(size_psf)[flagmask.astype(bool)] - star_e1 = np.array(g1)[flagmask.astype(bool)] - star_e2 = np.array(g2)[flagmask.astype(bool)] - star_r2 = np.array(size)[flagmask.astype(bool)] + # Concatenate once, here: everything below already wanted arrays and + # was calling np.array() on python lists to get them (see _stack). + x, y, ra, dec = _stack(x), _stack(y), _stack(ra), _stack(dec) + g1_psf, g2_psf, size_psf = _stack(g1_psf), _stack(g2_psf), _stack(size_psf) + g1, g2, size = _stack(g1), _stack(g2), _stack(size) + flag_psf, flag_star = _stack(flag_psf), _stack(flag_star) + ccd_nb = _stack(ccd_nb) + + flagmask = np.abs(flag_star - 1) * np.abs(flag_psf - 1) + psf_e1 = g1_psf[flagmask.astype(bool)] + psf_e2 = g2_psf[flagmask.astype(bool)] + psf_r2 = size_psf[flagmask.astype(bool)] + star_e1 = g1[flagmask.astype(bool)] + star_e2 = g2[flagmask.astype(bool)] + star_r2 = size[flagmask.astype(bool)] rmse, mean, std_dev = MSC.stats_calculator(star_e1, psf_e1) self._w_log.info( @@ -596,45 +603,49 @@ def process(self): data_j = starcat_j[self._hdu_table].data + # ONE ARRAY PER CATALOGUE PER COLUMN, concatenated once at the end + # (see _stack): the per-value python lists this replaces cost ~10x + # the input bytes and put a full-survey merge out of reach. # positions - x += list(data_j["X"]) - y += list(data_j["Y"]) - ra += list(data_j["RA"]) - dec += list(data_j["DEC"]) + x.append(np.asarray(data_j["X"])) + y.append(np.asarray(data_j["Y"])) + ra.append(np.asarray(data_j["RA"])) + dec.append(np.asarray(data_j["DEC"])) # shapes (size column already holds T = 2 sigma^2) - g1_psf += list(data_j["HSM_G1_PSF"]) - g2_psf += list(data_j["HSM_G2_PSF"]) - size_psf += list(data_j["HSM_T_PSF"]) - g1 += list(data_j["HSM_G1_STAR"]) - g2 += list(data_j["HSM_G2_STAR"]) - size += list(data_j["HSM_T_STAR"]) + g1_psf.append(np.asarray(data_j["HSM_G1_PSF"])) + g2_psf.append(np.asarray(data_j["HSM_G2_PSF"])) + size_psf.append(np.asarray(data_j["HSM_T_PSF"])) + g1.append(np.asarray(data_j["HSM_G1_STAR"])) + g2.append(np.asarray(data_j["HSM_G2_STAR"])) + size.append(np.asarray(data_j["HSM_T_STAR"])) # flags - flag_psf += list(data_j["HSM_FLAG_PSF"]) - flag_star += list(data_j["HSM_FLAG_STAR"]) + flag_psf.append(np.asarray(data_j["HSM_FLAG_PSF"])) + flag_star.append(np.asarray(data_j["HSM_FLAG_STAR"])) # misc # MKDEBUG: The following columns do not exist (yet) # for psf converted (pix2wcs) files. try: - mag += list(data_j["MAG"]) + mag.append(np.asarray(data_j["MAG"])) except: - mag += list(np.zeros_like(data_j["X"])) + mag.append(np.zeros_like(data_j["X"])) try: - snr += list(data_j["SNR"]) + snr.append(np.asarray(data_j["SNR"])) except: - snr += list(np.zeros_like(data_j["X"])) + snr.append(np.zeros_like(data_j["X"])) try: - psfex_acc += list(data_j["ACCEPTED"]) + psfex_acc.append(np.asarray(data_j["ACCEPTED"])) except: - psfex_acc += list(np.zeros_like(data_j["X"])) + psfex_acc.append(np.zeros_like(data_j["X"])) - # CCD number - ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2]] * len( - data_j["RA"] - ) + # CCD number: this catalogue's one value over its own rows, as an + # array rather than a python list holding the same string N times. + ccd_nb.append(np.full( + len(data_j["RA"]), + re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2])) # Prepare output FITS catalogue # MKDEBUG: SEx_cat=True -> False @@ -647,22 +658,22 @@ def process(self): # Collect columns (size stored as T = 2 sigma^2) data = { - "X": x, - "Y": y, - "RA": ra, - "DEC": dec, - "HSM_G1_PSF": g1_psf, - "HSM_G2_PSF": g2_psf, - "HSM_T_PSF": size_psf, - "HSM_G1_STAR": g1, - "HSM_G2_STAR": g2, - "HSM_T_STAR": size, - "HSM_FLAG_PSF": flag_psf, - "HSM_FLAG_STAR": flag_star, - "MAG": mag, - "SNR": snr, - "ACCEPTED": psfex_acc, - "CCD_NB": ccd_nb, + "X": _stack(x), + "Y": _stack(y), + "RA": _stack(ra), + "DEC": _stack(dec), + "HSM_G1_PSF": _stack(g1_psf), + "HSM_G2_PSF": _stack(g2_psf), + "HSM_T_PSF": _stack(size_psf), + "HSM_G1_STAR": _stack(g1), + "HSM_G2_STAR": _stack(g2), + "HSM_T_STAR": _stack(size), + "HSM_FLAG_PSF": _stack(flag_psf), + "HSM_FLAG_STAR": _stack(flag_star), + "MAG": _stack(mag), + "SNR": _stack(snr), + "ACCEPTED": _stack(psfex_acc), + "CCD_NB": _stack(ccd_nb, dtype="U1"), } # Write file @@ -813,29 +824,29 @@ def process(self): data_j = starcat_j[self._hdu_table].data # positions - x += list(data_j["XWIN_IMAGE"]) - y += list(data_j["YWIN_IMAGE"]) - ra += list(data_j["XWIN_WORLD"]) - dec += list(data_j["YWIN_WORLD"]) + x.append(np.asarray(data_j["XWIN_IMAGE"])) + y.append(np.asarray(data_j["YWIN_IMAGE"])) + ra.append(np.asarray(data_j["XWIN_WORLD"])) + dec.append(np.asarray(data_j["YWIN_WORLD"])) m11, m20, m02 = self.get_moments(data_j) eps1, eps2 = self.get_ellipticity(m11, m20, m02, "epsilon") chi1, chi2 = self.get_ellipticity(m11, m20, m02, "chi") - size += list(data_j["FLUX_RADIUS"]) + size.append(np.asarray(data_j["FLUX_RADIUS"])) # flags - flags += list(data_j["FLAGS_WIN"]) - flags_ext += list(data_j["IMAFLAGS_ISO"]) + flags.append(np.asarray(data_j["FLAGS_WIN"])) + flags_ext.append(np.asarray(data_j["IMAFLAGS_ISO"])) # misc - mag += list(data_j["MAG_WIN"]) - snr += list(data_j["SNR_WIN"]) + mag.append(np.asarray(data_j["MAG_WIN"])) + snr.append(np.asarray(data_j["SNR_WIN"])) # CCD number - ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2]] * len( - data_j["XWIN_IMAGE"] - ) + ccd_nb.append(np.full( + len(data_j["XWIN_IMAGE"]), + re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2])) # Prepare output FITS catalogue output = file_io.FITSCatalogue( @@ -847,20 +858,20 @@ def process(self): # Collect columns # convert back to sigma for consistency data = { - "X": x, - "Y": y, - "RA": ra, - "DEC": dec, + "X": _stack(x), + "Y": _stack(y), + "RA": _stack(ra), + "DEC": _stack(dec), "EPS1": eps1, "EPS2": eps2, "CHI1": chi1, "CHI2": chi2, - "SIZE": size, - "FLAGS": flags, - "FLAGS_EXT": flags_ext, - "MAG": mag, - "SNR": snr, - "CCD_NB": ccd_nb, + "SIZE": _stack(size), + "FLAGS": _stack(flags), + "FLAGS_EXT": _stack(flags_ext), + "MAG": _stack(mag), + "SNR": _stack(snr), + "CCD_NB": _stack(ccd_nb, dtype="U1"), } # Write file diff --git a/workflow/Snakefile b/workflow/Snakefile index 8f871df3b..47a8f7282 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -731,25 +731,31 @@ def star_cat_exposures(): # against synthetic tars for the star side and against smk-g6's real # catalogues for the tile side. Peak RSS is getrusage(RUSAGE_CHILDREN). # -# STAR SIDE, and it is the alarming one. Two points, 20 and 80 exposures of 40 -# CCDs x 400 stars (1.6 MB of members per exposure, against the 2.0 MB measured -# on smk-m2): +# STAR SIDE. Two points, 20 and 80 exposures of 40 CCDs x 400 stars (1.6 MB of +# members per exposure, against the 2.0 MB measured on smk-m2): # -# input members peak RSS -# 32.3 MB 383 MB -# 129.0 MB 1313 MB +# input members peak RSS, python lists peak RSS, arrays +# 32.3 MB 383 MB 238 MB +# 129.0 MB 1313 MB 740 MB # -# a slope of 10.1x the input bytes and an intercept of ~73 MB (interpreter, -# astropy, shapepipe). Tenfold, because MergeStarCatPSFEX accumulates every -# column into PYTHON LISTS of python floats before building the output arrays — -# 8 bytes of payload becomes a 32-byte object plus an 8-byte pointer. THE -# CONSEQUENCE IS A CEILING, and it should be said plainly: at ~2 MB per -# exposure, a 16 GB job merges roughly 800 exposures, and DR6's ~20k exposures -# would want ~400 GB. A full-survey full_starcat needs the accumulation changed -# to preallocated arrays or a two-pass count — a change to MergeStarCatPSFEX, -# not to this rule, and not in this PR. The formula below is honest about the -# slope so the job asks for what it will use and fails at submission rather -# than at 90% of the way through a campaign-length merge. +# a slope of 10.1x the input bytes before, 5.5x now, over a ~62 MB interpreter +# floor. The tenfold was MergeStarCat*'s accumulation of every column into +# PYTHON LISTS of python floats — 4 bytes of float32 payload becoming a 32-byte +# object plus an 8-byte pointer — and this PR replaced it with one array per +# catalogue and one concatenate at the end, output byte-identical. +# +# THE REMAINING 5.5x IS THE OUTPUT SIDE, and it is not a leak: file_io writes +# every float column as FITS 1D, so a float32 input becomes a float64 table +# that astropy then buffers to write. Halving it means changing the OUTPUT +# format, which is what sp_validation reads — a different decision from this +# one, and not ours to take here. +# +# THE CEILING MOVED BUT DID NOT GO. At ~2 MB of members per exposure a 16 GB +# job now merges ~2200 exposures rather than ~800, and DR6's ~20k would want +# ~280 GB rather than ~400 GB. A full-survey full_starcat still needs the +# output side addressed; the formula below is honest about the slope so the job +# asks for what it will use and fails at submission rather than most of the way +# through. # # TILE SIDE, and it is the reassuring one. Two points against real smk-g6 # catalogues, 2 tiles (73.9 MB in, largest 39.6 MB) and 6 tiles (235.5 MB in, @@ -757,7 +763,7 @@ def star_cat_exposures(): # the merge holds one catalogue at a time — so it is sized on the LARGEST tile, # not the total, at ~3x it plus the interpreter. STAR_MEM_BASE_MB = 500 # interpreter + astropy + shapepipe, rounded up -STAR_MEM_FACTOR = 12 # x input bytes; 10.1 measured, rounded up +STAR_MEM_FACTOR = 7 # x input bytes; 5.5 measured, rounded up FINAL_MEM_BASE_MB = 800 FINAL_MEM_FACTOR = 4 # x the LARGEST tile; ~3 measured # What one unit costs when its product is not on disk yet to be stat()ed — a From 0ad640350a390b2def72ce6f7e390e0ae3b6610b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:34:47 -0400 Subject: [PATCH 13/20] feat(orchestration): the star catalogue's inputs are not a user choice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `persist_exp:` was doing two jobs. It decided what a campaign keeps for later — a retention question, and the user's — and it also decided whether the campaign's star catalogue could be built at all, because star_cat_merge existed only when the keep list happened to name something validation_psf-shaped. That made the survey's PSF diagnostics an opt-in, and a typo away from silently absent. exp_persist now packs psf_validation for every exposure whatever the config says. It is the merged catalogue's PROVENANCE — a full_starcat with no per-exposure inputs beside it cannot be audited, re-cut or recomputed after a purge — and it is what keeps APPENDING TILES CHEAP, since a tile added next month brings exposures whose catalogues must join the existing stack and the alternative is rebuilding their chains from VOS. ~2 MB per exposure: ~40 GB and ~40k inodes at DR6 scale against a ~1 M-inode group quota, which is the price of being able to say where the number came from. `persist_exp:` is therefore purely additive retention, defaulting to psf_model, and an EMPTY list is now a coherent instruction rather than a switch that turns persistence off: the tar holds the merge's inputs and nothing else. The keep-list gate on star_cat_merge and its parse-time warning are gone with it, as is clean_exposure's conditional edge on exp_persist — there is no configuration left under which that rule has nothing to wait for. No transient/cleanup knob: these files are kept, not staged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- workflow/README.md | 58 ++++++++++----------------------- workflow/Snakefile | 49 +++++++++------------------- workflow/config.yaml | 56 ++++++++++++++++--------------- workflow/rules/exposure.smk | 13 ++++---- workflow/scripts/persist_exp.py | 39 +++++++++++++++++----- 5 files changed, 99 insertions(+), 116 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 1ad740b48..832a894f1 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -259,17 +259,24 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee hours of PSF fitting per exposure. A pattern that matches nothing is a recorded warning (setools rejects sparse CCDs); matching nothing at all is a failure. A `localrule`, by the same arithmetic as `clean_exposure`. -- **The keep list names products, not globs.** `persist_exp:` entries are names - from a catalogue in `workflow/scripts/persist_exp.py`, which is the single - source of truth for what each one means and what keeping it buys +- **The star catalogue's inputs are always kept; `persist_exp:` is what you + keep on top.** `exp_persist` packs `psf_validation` — the psfex_interp + validation catalogue, one per CCD — for every exposure whatever the config + says, because `star_cat_merge` stacks exactly those into the campaign's + `full_starcat`. They are that catalogue's provenance, and they are what keeps + appending a tile next month cheap rather than a rebuild from VOS. About 2 MB + per exposure: ~40 GB and ~40k inodes at DR6 scale, against a ~1 M-inode group + quota. `persist_exp:` is purely additive, and an empty list is legal — the tar + then holds the merge's inputs and nothing else. +- **The keep list names products, not globs.** Entries are names from a + catalogue in `workflow/scripts/persist_exp.py`, which is the single source of + truth for what each one means and what keeping it buys ([#844](https://github.com/CosmoStat/shapepipe/issues/844)); `config.yaml`'s - block is that catalogue rendered, and - `persist_exp.py --list-products` prints it. Sizes are per exposure, 40 CCDs, - measured on smk-m2. + block is that catalogue rendered, and `persist_exp.py --list-products` prints + it. Sizes are per exposure, 40 CCDs, measured on smk-m2. | product | glob | per exposure | what it buys | |---|---|---|---| - | `psf_validation` | `validation_psf-*.fits` | 2.0 MB | the rho/tau statistics input, and `star_cat_merge`'s | | `psf_model` | `*.psf` | 2.8 MB | re-interpolate the PSF anywhere later, no rebuild | | `psfex_cat` | `psfex_cat-*.cat` | unmeasured | which stars PSFEx's outlier rejection clipped | | `star_selection` | `star_selection-*.fits` | 24.5 MB | which stars the selection cuts rejected, and why | @@ -277,42 +284,11 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee | `star_test` | `star_split_ratio_20-*.fits` | 7.1 MB | the 20% sample `psf_validation` corresponds to | | `star_stats` | `star_stat-*.txt` | unmeasured | setools' per-CCD counts, density and FWHM cuts | - The default is `psf_validation` + `psf_model`. A raw glob is still accepted as - an escape hatch — anything with a glob metacharacter or a dot is read as one — + The default is `psf_model`. `psf_validation` is in the catalogue too but needs + no naming; naming it anyway is harmless. A raw glob is still accepted as an + escape hatch — anything with a glob metacharacter or a dot is read as one — and an unknown *name* is a parse-time error listing the valid ones. The list is exposure-side only; tile-side retention is #844 follow-up. -- **The campaign ends in two merged catalogues, and the workflow now makes - both.** Everything above is per unit; the two products downstream analysis - actually opens are per *campaign*, and until these rules existed each was a - manual pass after the run. - `star_cat_merge` stacks every exposure's every CCD's `validation_psf-*.fits` - into one `/full_starcat-0000000.fits` — the rho/tau statistics - input, at the path sp_validation hardcodes. It reads the members straight out - of the per-exposure tars (`tarfile` + `BytesIO`; unpacking ~800k files to - merge them would defeat the tar's whole purpose) and stacks them with - `MergeStarCatPSFEX`, the same class the old `merge_starcat_runner` called, so - the column list has exactly one definition. Its input is the same - `exp_persist` manifest set `rule all` already requests, so it pulls nothing - new into the DAG, and it exists only when `persist_exp:` keeps a - `validation_psf-*.fits`-shaped file — otherwise no job, and a warning at parse - time rather than a failure on a node. - `final_cat_merge` collects every ready tile's `final_cat-.fits` into - `/final_cat_.hdf5`: one dataset per tile under a group - named for the campaign, the `final_cat.param` columns, an `n_tiles` attribute. - That schema is what sp_validation's reader opens, so it is fixed; the column - extraction reuses `scripts/python/create_final_cat.py` while the file is - written here, because that script's own discovery walks a directory layout - this workflow does not have. `campaign:` in `config.yaml` names the group and - defaults to the persistent root's basename. - Both rebuild from the whole persistent root rather than appending, so the - output is a function of its input set: byte-stable on a no-op rerun - (tmp-then-`cmp`-then-`mv`), and rebuilt when a tile or exposure is appended - (the input list's fingerprint rides on `params`). Neither is a `localrule` — - one job over ~20k units is real work — and neither puts its input paths in its - shell, which is not fastidiousness: ~20k paths is an order of magnitude over - Linux's 128 KiB `MAX_ARG_STRLEN` for a single argv entry, so each script - rediscovers the set under `products_dir` while the fingerprint travels on - `params`. - **A dead tile can be told to stop pinning exposures.** An exposure is cleanable only once every consuming tile has its vignets, so one permanently-failed tile holds its ~80 exposures for the life of the diff --git a/workflow/Snakefile b/workflow/Snakefile index 47a8f7282..0473c7ac6 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -31,7 +31,6 @@ the manifest says "this stage succeeded", the log says "here is what happened" (the contract is argued in completeness.py's docstring). """ -import fnmatch import functools import hashlib import json @@ -470,6 +469,12 @@ def clean_targets(): # The keep list is config, not a rule input, and it is READ HERE so that exactly # one place converts it into the form the rule carries. An empty list is a # deliberate "keep nothing" and produces no jobs at all. +# OPTIONAL RETENTION, and only that. What star_cat_merge needs — every CCD's +# psf_validation — is packed by exp_persist whatever this list says +# (persist_exp.py's ALWAYS argues why: provenance for the merged catalogue, and +# a cheap tile append later). So an EMPTY list is a coherent instruction and not +# a switch that turns persistence off: the tar then holds the star catalogue's +# inputs and nothing else, and exp_persist still runs for every exposure. PERSIST_EXP = list(config.get("persist_exp") or []) # The keep list names PRODUCTS (`psf_model`), not globs (`*.psf`); the @@ -485,7 +490,6 @@ for _entry in PERSIST_EXP: _persist.resolve(_entry) except KeyError as _exc: raise WorkflowError(f"config persist_exp: {_exc.args[0]}") -PERSIST_GLOBS = [_persist.resolve(e) for e in PERSIST_EXP] def persist_targets(): @@ -525,8 +529,6 @@ def persist_manifests(): merge job's own re-parse under the slurm executor, which genuinely needs it. So this half carries no guard and the memo keeps either parse to one walk. """ - if not PERSIST_EXP: - return [] exps = {e for t in TILES_READY for e in tile_exposures(t)} return sorted(prod_exp_manifest(e, "exp_persist") for e in exps if not exp_store_reclaimed(e)) @@ -616,28 +618,12 @@ def unit_fingerprint(units): return f"{len(units)}:{hashlib.md5(joined.encode()).hexdigest()[:12]}" -# The tar member the star merge consumes, and the test is "would this member be -# kept" rather than "is `psf_validation` in the list". Both spellings must count: -# the product name, and a raw glob that happens to cover it (`validation_psf*`, -# `*.fits`, a bare `*`) — all things a user might reasonably write. So the keep -# list is RESOLVED to globs first and the member matched against those. -_STAR_CAT_PRODUCT = "psf_validation" -_STAR_CAT_MEMBER = "validation_psf-2605805-12.fits" -STAR_CAT_MERGE = any(fnmatch.fnmatch(_STAR_CAT_MEMBER, g) for g in PERSIST_GLOBS) - -# LOUD AT PARSE TIME, once, and only where it can be acted on. A keep list -# without the validation catalogues is a legitimate configuration (persist the -# PSF models alone, say) — it is not an error, so it must not become a job that -# fails on a node an hour later. It is worth SAYING, because the omission is -# silent otherwise and the missing product only surfaces when a rho-statistics -# run cannot find its input. -if PERSIST_EXP and not STAR_CAT_MERGE and workflow.is_main_process \ - and PHASE == "compute": - logger.warning( - f"star_cat_merge: no job — persist_exp {PERSIST_EXP} keeps no " - f"'{_STAR_CAT_MEMBER}'-shaped file, so there is nothing to stack into " - f"{PRODUCTS_DIR}/full_starcat-0000000.fits (the rho/tau statistics " - f"input). Add '{_STAR_CAT_PRODUCT}' to persist_exp: to get it.") +# NO GATE ON THE KEEP LIST. star_cat_merge used to exist only when +# `persist_exp:` named something validation_psf-shaped, which made the +# campaign's star catalogue an opt-in and a typo away from silently absent. +# exp_persist now packs psf_validation unconditionally, so the merge is +# requested whenever the campaign has a persisted exposure at all, and +# star_cat_targets() below is the only condition left. def full_starcat(): @@ -691,8 +677,6 @@ def star_cat_inputs(): manifest and is in no set at all. Nothing short of rebuilding its chain from VOS recovers it; the merge reports how many exposures it found. """ - if not PERSIST_EXP: - return [] live, reclaimed = [], [] for exp in sorted({e for t in TILES_READY for e in tile_exposures(t)}): if not exp_store_reclaimed(exp): @@ -714,8 +698,6 @@ def star_cat_exposures(): is what the trigger is for. It is also what merge_star_cat.py derives on the job side, so the two agree on the set AND on how it is named. """ - if not PERSIST_EXP: - return [] return sorted(e for e in {e for t in TILES_READY for e in tile_exposures(t)} if Path(prod_exp_manifest(e, "exp_persist")).exists() or not exp_store_reclaimed(e)) @@ -802,15 +784,14 @@ def final_cat_max_bytes(): def star_cat_targets(): """`full_starcat` when there is anything to stack into it, else nothing. - Three ways to get nothing, and all three are states rather than errors: the - keep list holds no validation catalogue (warned about above), `persist_exp:` - is empty at all, or every exposure in scope is already tombstoned — a + One way to get nothing, and it is a state rather than an error: every + exposure in scope is already tombstoned — a campaign resumed after reclamation, whose exposures were cleaned by a workflow that predates exp_persist and therefore left neither tar nor manifest to read. A rule with an empty input list would still be a JOB, and it would write an empty star catalogue over a good one. """ - if not STAR_CAT_MERGE or not workflow.is_main_process: + if not workflow.is_main_process: return [] return [full_starcat()] if star_cat_inputs() else [] diff --git a/workflow/config.yaml b/workflow/config.yaml index 03c282113..1b19784b5 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -81,11 +81,27 @@ outputs: # would otherwise have to rebuild from tile headers. index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite -# Per-exposure PSF products to carry onto the persistent root before the scratch -# store goes (`exp_persist`, exposure.smk). A list of PRODUCT NAMES — not globs. -# The catalogue below is the rendering of workflow/scripts/persist_exp.py's -# PRODUCTS table, which is the single source of truth for what each name means -# and what keeping it buys (CosmoStat/shapepipe#844); print it any time with +# OPTIONAL per-exposure retention: what to carry onto the persistent root ON TOP +# OF the star catalogue's own inputs, before the scratch store goes +# (`exp_persist`, exposure.smk). +# +# WHAT IS ALWAYS KEPT, AND IS NOT A CHOICE HERE: psf_validation, the psfex_interp +# validation catalogue, one per CCD. `star_cat_merge` stacks every one of them +# into the campaign's /full_starcat-0000000.fits, so they are that +# catalogue's PROVENANCE — a merged star catalogue with no per-exposure inputs +# beside it cannot be audited, re-cut, or recomputed after a purge — and they are +# what keeps APPENDING TILES CHEAP, since a tile added next month brings +# exposures whose catalogues must join the existing stack. ~2 MB per exposure: +# ~40 GB and ~40k inodes at DR6 scale, against a ~1 M-inode group quota. That is +# the price of being able to say where the number came from, and it is paid. +# +# SO THIS LIST IS PURELY ADDITIVE, and an empty one is a coherent instruction: +# the tar then holds the star catalogue's inputs and nothing else. +# +# Entries are PRODUCT NAMES — not globs. The catalogue below is the rendering of +# workflow/scripts/persist_exp.py's PRODUCTS table, which is the single source of +# truth for what each name means and what keeping it buys +# (CosmoStat/shapepipe#844); print it any time with # # workflow/bin/sp container exec python workflow/scripts/persist_exp.py --list-products # @@ -121,16 +137,13 @@ outputs: # A RAW GLOB IS STILL ACCEPTED, as an escape hatch for a file the catalogue does # not name yet: anything carrying a glob metacharacter or a dot is taken as a # glob rather than a name (`*.psf` is a glob, `psf_model` is the name for it). -# An unknown NAME is a parse-time error listing the valid ones, never a silently -# empty keep. +# An unknown NAME is a parse-time error listing the valid ones. # # Matches are packed, flat, into ONE uncompressed tar per exposure: # /exp///psf/.tar, with a manifest listing the # members (and the product each came from) beside it. One tar rather than loose -# copies because inodes, not bytes, bind on /project (~1 M-file group quota; -# loose copies would be ~200 files per exposure, ~2 M at DR6 scale). FITS -# members read straight from the tar: -# fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())). +# copies because inodes, not bytes, bind on /project. FITS members read straight +# from the tar: fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())). # # WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat. # run_dir is /scratch and is PURGED on a 60-day window whether or not @@ -143,32 +156,23 @@ outputs: # reruns the packing (seconds) and NOT exp_psf (four hours per exposure). That # separation is the whole reason exp_persist is a rule of its own. # -# THE DEFAULT is psf_validation + psf_model (~4.8 MB per exposure): the rho/tau -# statistics input, and the model that lets the PSF be re-interpolated at any -# position later without rebuilding the exposure chain from VOS. Add psfex_cat -# for a production run if you want to know which stars PSFEx clipped; the -# star_* products are for selection studies and cost an order of magnitude more. +# THE DEFAULT is psf_model (2.8 MB per exposure on top of psf_validation's 2.0): +# the model that lets the PSF be re-interpolated at any position later without +# rebuilding the exposure chain from VOS, which is the single most +# capability-adding thing an exposure can keep. Add psfex_cat for a production +# run if you want to know which stars PSFEx clipped; the star_* products are for +# selection studies and cost an order of magnitude more. # # THIS LIST IS EXPOSURE-SIDE ONLY. Tile-side retention is not configurable: the # only tile product that persists today is final_cat, written by tile_make_cat # straight to products_dir. A tile keep list is #844 follow-up. # -# THIS LIST GATES `star_cat_merge`. That campaign-level rule stacks every -# exposure's every CCD's psf_validation into ONE -# /full_starcat-0000000.fits, reading the members straight out of -# the tars. A keep list without psf_validation is a legitimate configuration and -# produces NO merge job and a warning at parse time — not a failure on a node an -# hour later. Note the corollary: an exposure already reclaimed by a workflow -# that predates exp_persist left no tar, so it contributes nothing and cannot be -# recovered short of rebuilding its chain from VOS. -# # NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the tar # then lands beside the store on the same filesystem and buys nothing, and the # manifest sits in the exposure's own manifests/ dir, which clean_exposure # deletes wholesale — so a one-root run re-persists after every reclamation. # Harmless, and exactly the pre-D5 behaviour a one-root run asks for. persist_exp: - - psf_validation - psf_model # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 789e9da1d..0cbafb448 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -149,6 +149,9 @@ rule exp_persist: # name collision, both of which it reports on stderr and neither of which # has a per-CCD verdict worth a completeness record. params: + # Only the OPTIONAL retention list travels: psf_validation is packed + # by persist_exp.py whatever this says. It still rides on params, so + # adding a product re-packs (seconds) rather than re-fitting the PSF. patterns = " ".join(f"--pattern '{p}'" for p in PERSIST_EXP), exp_dir = lambda wc: exp_dir(wc.exp), dest = lambda wc: f"{prod_exp_dir(wc.exp)}/psf", @@ -201,12 +204,10 @@ rule clean_exposure: # The keepers must be off /scratch before the store goes. Unlike the # consumer edges above, this edge does not depend on scope: it is the # same exposure's own rule, so it drags nothing into the DAG that this - # exposure's chain did not already put there. It is conditional only on - # there being a keep list at all — with `persist_exp:` empty, "keep - # nothing" is a coherent instruction and must not become a dependency on - # a rule that would fail for having nothing to copy. - lambda wc: ([prod_exp_manifest(wc.exp, "exp_persist")] - if PERSIST_EXP else []) + # exposure's chain did not already put there. It is UNCONDITIONAL now: + # exp_persist always packs the star catalogue's inputs, so there is no + # keep list under which this rule has nothing to wait for. + lambda wc: [prod_exp_manifest(wc.exp, "exp_persist")] output: tombstone = f"{EXP_DIR}/cleaned.json" params: diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index fef439b7b..4e2dff349 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -29,6 +29,11 @@ glob. Patterns are therefore plain FILE names and the layout is ours to know, not the config author's. +THE KEEP LIST IS WHAT THE CAMPAIGN KEEPS ON TOP OF THE MERGE'S INPUTS. +``psf_validation`` is packed unconditionally (see ALWAYS below); ``persist_exp:`` +is purely optional retention, and an EMPTY one is a coherent instruction — the +tar then holds the star catalogue's inputs and nothing else. + ZERO MATCHES FOR ONE PATTERN IS A WARNING, NOT A FAILURE. setools rejects sparse CCDs (~0.2% attrition, tolerated by exp_psf's own count floor), so per-CCD counts are not fixed, and a pattern naming an optional diagnostic may legitimately @@ -101,6 +106,19 @@ # # ORDER IS THE ORDER OF THE CHAIN — sextractor, setools, psfex, psfex_interp — # so the table reads as the pipeline runs. +# THE STAR CATALOGUE'S INPUTS ARE NOT A USER CHOICE. star_cat_merge stacks +# every CCD's psf_validation into the campaign's full_starcat, so exp_persist +# ALWAYS packs it, whatever `persist_exp:` says. Two reasons, and neither is +# about taste. It is the merged catalogue's PROVENANCE: a full_starcat with no +# per-exposure inputs beside it cannot be audited, re-cut or recomputed after a +# purge. And it is what keeps APPENDING TILES CHEAP: a tile added next month +# brings exposures whose validation catalogues must join the existing stack, and +# if the earlier ones are gone the merge either shrinks or rebuilds their chains +# from VOS. ~2 MB per exposure, so ~40 GB and ~40k inodes at DR6 scale, against +# a group quota of ~1 M inodes — the cost of being able to say where the number +# came from. +ALWAYS = "psf_validation" + PRODUCTS = { "star_selection": ( "star_selection-*.fits", 24_500_000, @@ -228,8 +246,9 @@ def main() -> None: "/.tar") p.add_argument("--manifest", type=Path) p.add_argument("--pattern", action="append", default=[], - help="repeatable; a product name (see --list-products) or a " - "raw file-name glob") + help=f"repeatable; a product name (see --list-products) or " + f"a raw file-name glob. {ALWAYS} is packed whether or " + f"not it is named — star_cat_merge needs it") p.add_argument("--list-products", action="store_true", help="print the product catalogue and exit") args = p.parse_args() @@ -246,19 +265,21 @@ def main() -> None: if missing: p.error(f"the following arguments are required: {', '.join(missing)}") - if not args.pattern: - sys.exit("persist_exp: no --pattern given (config persist_exp is empty)") + # The merge's input first and always, then whatever the campaign chose to + # keep on top of it (see ALWAYS). Deduped, so naming it explicitly in + # persist_exp: is harmless rather than a repeated pattern. + entries = [ALWAYS] + [e for e in args.pattern if e != ALWAYS] - for entry in args.pattern: # loud, and before any work + for entry in entries: # loud, and before any work try: resolve(entry) except KeyError as exc: sys.exit(f"persist_exp: {exc.args[0]}") - found, empty = collect(args.exp_dir, args.pattern) + found, empty = collect(args.exp_dir, entries) if not found: sys.exit(f"persist_exp: {args.exp}: no file matched any of " - f"{args.pattern} under {args.exp_dir}/output/{RUN_NAME}") + f"{entries} under {args.exp_dir}/output/{RUN_NAME}") args.dest.mkdir(parents=True, exist_ok=True) tar_path = args.dest / f"{args.exp}.tar" @@ -312,8 +333,8 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: "stage": "exp_persist", "level": "exp", "unit": args.exp, "status": "complete", "tar": str(tar_path), - "products": list(args.pattern), - "patterns": [resolve(e) for e in args.pattern], + "products": entries, + "patterns": [resolve(e) for e in entries], # The warning the docstring argues for: named patterns that matched # nothing. Present as a key even when empty, so a reader never has to # wonder whether an old manifest predates the field. From 90dfb00afb2fa42be559b61c2517faa933f8f135 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:43:10 -0400 Subject: [PATCH 14/20] fix(cfis): two stale columns in final_cat.param, and no mask column at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit final_cat_merge on smk-g6's real catalogues failed on three of the 67 columns the parameter file asks for. Two of them the file should not have been asking for. IMAFLAGS_ISO is DROPPED. The tile-side SExtractor runs with FLAG_IMAGE = False and DOT_PARAM_FILE = default_noimaflags.param (config_tile_Sx.ini), so the column is never written into a tile catalogue and asking for it could only fail. Instrument flags reach the pipeline on the EXPOSURE side, where exp_split delivers the flag image and SExtractor reads it. NGMIX_MOM_FAIL is RENAMED to NGMIX_MCAL_TYPES_FAIL, which is what f0fca23e called it in June and what the catalogues carry. NGMIX_NEIGHBOUR_FLAG STAYS. It was added to make_cat in fa6e0016 on 2026-07-12 and it is the blend flag the systematics tests need. smk-g6's catalogues do not have it — checked on the files — so final_cat_merge still fails there, and that failure is correct: the campaign's catalogues are missing a column the analysis wants, which is a fact about the data and not about this file. Its launch snapshot is gone (only .snakemake survives under smk-g6-state), so the run's HEAD cannot be read back; what remains is that its catalogues carry NGMIX_MCAL_TYPES_FAIL (June) but not NGMIX_NEIGHBOUR_FLAG (July), consistent with a snapshot taken between the two. NO MASK COLUMN REPLACES IMAFLAGS_ISO, AND THE FILE NOW SAYS WHY. The intended replacement is make_cat's per-band MASK_, queried from the healsparse maps named by MASK_EXT_PATHS — and the workflow sets none: config_tile_Mc.ini has no such entry, save_mask_ext_data is never called, no MASK_ column exists in any catalogue this workflow has produced, and smk-g6's carry none. Naming one here would fail every merge on every campaign. The merged catalogue therefore carries no mask information today; that is a CONFIG gap, and closing it is setting MASK_EXT_PATHS first and adding the column names second. No healsparse map is staged under /project/def-mjhudson yet. With NGMIX_NEIGHBOUR_FLAG set aside, the merge runs clean over all 64 of smk-g6's real catalogues: 2.50 GB read in 20 s at 151 MB peak RSS, producing a 0.94 GB hdf5 of 65 columns. That also confirms the tile-side sizing — the rule asks for 1002 MB and 51 minutes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- workflow/config/cfis/final_cat.param | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 00bcb3f73..f3fa39677 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -8,7 +8,26 @@ TILE_ID # flags FLAGS -IMAFLAGS_ISO +# NO IMAFLAGS_ISO, AND NO MASK COLUMN AT ALL — READ THIS BEFORE ADDING ONE. +# The tile-side SExtractor runs with FLAG_IMAGE = False and DOT_PARAM_FILE = +# default_noimaflags.param (config_tile_Sx.ini), so IMAFLAGS_ISO is never +# written into a tile catalogue and asking for it here only made the merge +# fail. Instrument flags reach the pipeline on the EXPOSURE side, where +# exp_split delivers the flag image and SExtractor reads it. +# +# Its intended replacement is make_cat's per-band MASK_ columns, queried +# from the sky-fixed healsparse maps named by MASK_EXT_PATHS. THE WORKFLOW SETS +# NO SUCH PATHS: config_tile_Mc.ini has no MASK_EXT_PATHS entry, so +# save_mask_ext_data is never called, no MASK_ column exists in any tile +# catalogue this workflow has produced, and smk-g6's carry none (checked). +# Naming one here would fail every merge on every campaign. +# +# So the merged catalogue carries NO mask information today, and that is a +# CONFIG gap and not a gap in this file: turning it on is setting +# MASK_EXT_PATHS in config_tile_Mc.ini (`band:path` pairs, the same grammar as +# the commented MASK_PATHS in config_exp_psfex.ini) and adding the matching +# MASK_ names here, in that order. No healsparse map is staged under +# /project/def-mjhudson yet. NGMIX_MCAL_FLAGS # PSF ellipticity (original image PSF) @@ -113,5 +132,6 @@ NGMIX_T_PSF_ORIG_NOSHEAR # PSF size measured on reconvolved image # NGMIX_T_PSF_RECONV_NOSHEAR -# ngmix moment failure flag -NGMIX_MOM_FAIL +# ngmix metacalibration type failure flag (renamed from NGMIX_MOM_FAIL in +# f0fca23e; catalogues written before that commit carry the old name) +NGMIX_MCAL_TYPES_FAIL From a434c9aa5bed174761084cae8062cabed64fda8d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:43:10 -0400 Subject: [PATCH 15/20] perf(merge_starcat): two passes, so nothing is held twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The array accumulation landed a commit ago took the star merge from ~10x the input bytes to ~5.5x. What was left was the accumulation itself: one array per input catalogue, then a concatenate that has to hold its inputs and its result at the same time. MergeStarCatPSFEX now makes two passes. The first reads only the FITS HEADER of every input — NAXIS2, the row count — and touches no data block; the second allocates each output column once, at its exact final length, and fills it slice by slice. There are no chunks and no concatenate, so peak memory is one output plus one input catalogue. The workflow's tar reader hands over the archive's own file object rather than a BytesIO of the whole member, so the counting pass costs a header rather than a member. Both it and a plain list of paths are iterable twice, which the two passes require; a one-shot iterable would fill nothing on the second pass, so the merge checks that the passes agree on the row count rather than writing a catalogue padded with uninitialised memory. MEASURED on the same two fixture points, 20 and 80 exposures of 40 CCDs x 400 stars: input members python lists arrays+concat two passes 32.3 MB 383 MB 238 MB 221 MB 129.0 MB 1313 MB 740 MB 661 MB slope 10.1x 5.5x 4.8x The rule's mem_mb factor follows. A 16 GB job now merges ~1300 exposures. THE REMAINING 4.8x IS THE OUTPUT SIDE: file_io writes every float column as FITS 1D, so float32 inputs become a float64 table astropy then buffers — 141 MB of table for 78 MB of payload at the 80-exposure point. What stands between here and a full-survey full_starcat is that format, not the merge. MergeStarCatMCCD and MergeStarCatSetools keep the array accumulation. Their process() computes campaign-wide statistics over the same columns, so a two-pass rewrite there is a larger change with no consumer today — psfex is what every campaign runs. BYTE-IDENTICAL OUTPUT, both ways in: the workflow's tar path and the module runner's plain [path] path both give md5 f7caa1cf… on the fixture, unchanged through both rewrites. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- .../merge_starcat_package/merge_starcat.py | 169 ++++++++++-------- workflow/Snakefile | 41 ++--- workflow/scripts/merge_star_cat.py | 12 +- 3 files changed, 127 insertions(+), 95 deletions(-) diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 3463ccf43..97e25e0c2 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -571,81 +571,119 @@ def __init__( self._hdu_table = hdu_table self._input_cat_type = input_cat_type + # The columns this class writes, and where each comes from. Kept as data + # rather than as sixteen repeated lines, because a two-pass merge would + # otherwise state every column three times: to size it, to allocate it and + # to fill it. + _COLUMNS = ( + ("X", "X"), ("Y", "Y"), ("RA", "RA"), ("DEC", "DEC"), + ("HSM_G1_PSF", "HSM_G1_PSF"), ("HSM_G2_PSF", "HSM_G2_PSF"), + ("HSM_T_PSF", "HSM_T_PSF"), ("HSM_G1_STAR", "HSM_G1_STAR"), + ("HSM_G2_STAR", "HSM_G2_STAR"), ("HSM_T_STAR", "HSM_T_STAR"), + ("HSM_FLAG_PSF", "HSM_FLAG_PSF"), ("HSM_FLAG_STAR", "HSM_FLAG_STAR"), + ) + # Present in psfex_interp output, absent from pix2wcs-converted files + # (MKDEBUG); zero-filled when missing rather than failing the merge. + _OPTIONAL = (("MAG", "MAG"), ("SNR", "SNR"), ("ACCEPTED", "ACCEPTED")) + + def _ccd_nb(self, label): + """The CCD number this catalogue's rows carry, parsed from its name.""" + return re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2] + def process(self): """Process. Process merging. + TWO PASSES, AND NEITHER HOLDS THE CAMPAIGN TWICE. The first reads only + the FITS HEADER of every input — NAXIS2, the row count — and never + touches a data block; the second allocates the output columns once, at + their exact final length, and fills them slice by slice. Peak memory is + therefore ONE output plus ONE input catalogue. + + What this replaces, in two steps, is instructive about the cost of the + obvious code. Accumulating each column into a python LIST OF VALUES — + ``x += list(data["X"])`` — turned 4 bytes of float32 payload into a + 32-byte object plus an 8-byte pointer, measured at ~10x the input bytes + end to end and putting a full-survey merge (~20k exposures x 40 CCDs) at + ~400 GB. Accumulating one ARRAY PER CATALOGUE and concatenating once + brought that to ~5.5x. This pass structure removes what was left of the + accumulation: there are no chunks, and no concatenate that must hold its + inputs and its result at the same time. + + ``self._input_file_list`` MUST BE ITERABLE TWICE. A list is; so is the + workflow's tar reader, whose ``__iter__`` opens the archives afresh. + A one-shot generator is not, and would silently merge nothing on the + second pass — hence the explicit length check below. """ - x, y, ra, dec = [], [], [], [] - g1_psf, g2_psf, size_psf = [], [], [] - g1, g2, size = [], [], [] - flag_psf, flag_star = [], [] - mag, snr, psfex_acc = [], [], [] - ccd_nb = [] - self._w_log.info( f"Merging {len(self._input_file_list)} star catalogues" ) + # --- pass 1: row counts and dtypes, from headers alone -------------- + counts, labels, dtypes, n_total = [], [], None, 0 for name in self._input_file_list: - # The source to read and the NAME to parse the CCD number out of. - # Identical for a plain [path] entry; different only when the caller - # hands over an open file-like object plus the member name it came - # under (see the class docstring). source, label = name[0], name[-1] try: - starcat_j = fits.open(source, memmap=False, ignore_missing_simple=True) - except OSError as e: + with fits.open(source, memmap=False, + ignore_missing_simple=True) as starcat_j: + hdu = starcat_j[self._hdu_table] + n_rows = hdu.header["NAXIS2"] + if dtypes is None: + # ColDefs.dtype describes the table without reading it. + dtypes = hdu.columns.dtype + except OSError: print(f"Error while opening file '{label}'") #raise continue - + counts.append(n_rows) + labels.append(label) + n_total += n_rows + + if dtypes is None: + raise ValueError("merge_starcat: no readable input catalogue") + + # --- allocate once, at the exact final length ----------------------- + present = set(dtypes.names) + data = {out: np.empty(n_total, dtype=dtypes[col]) + for out, col in self._COLUMNS} + for out, col in self._OPTIONAL: + data[out] = np.empty( + n_total, dtype=dtypes[col] if col in present else dtypes["X"]) + # CCD_NB is one string per catalogue, repeated over its rows; its width + # is the widest CCD number in the campaign, which pass 1 already knows. + width = max((len(self._ccd_nb(lb)) for lb in labels), default=1) + data["CCD_NB"] = np.empty(n_total, dtype=f"U{width}") + + # --- pass 2: fill --------------------------------------------------- + at = 0 + for name in self._input_file_list: + source, label = name[0], name[-1] + try: + starcat_j = fits.open(source, memmap=False, + ignore_missing_simple=True) + except OSError: + continue data_j = starcat_j[self._hdu_table].data + n_rows = len(data_j) + sl = slice(at, at + n_rows) - # ONE ARRAY PER CATALOGUE PER COLUMN, concatenated once at the end - # (see _stack): the per-value python lists this replaces cost ~10x - # the input bytes and put a full-survey merge out of reach. - # positions - x.append(np.asarray(data_j["X"])) - y.append(np.asarray(data_j["Y"])) - ra.append(np.asarray(data_j["RA"])) - dec.append(np.asarray(data_j["DEC"])) - - # shapes (size column already holds T = 2 sigma^2) - g1_psf.append(np.asarray(data_j["HSM_G1_PSF"])) - g2_psf.append(np.asarray(data_j["HSM_G2_PSF"])) - size_psf.append(np.asarray(data_j["HSM_T_PSF"])) - g1.append(np.asarray(data_j["HSM_G1_STAR"])) - g2.append(np.asarray(data_j["HSM_G2_STAR"])) - size.append(np.asarray(data_j["HSM_T_STAR"])) - - # flags - flag_psf.append(np.asarray(data_j["HSM_FLAG_PSF"])) - flag_star.append(np.asarray(data_j["HSM_FLAG_STAR"])) - - # misc + for out, col in self._COLUMNS: + data[out][sl] = data_j[col] + for out, col in self._OPTIONAL: + data[out][sl] = data_j[col] if col in present else 0 + data["CCD_NB"][sl] = self._ccd_nb(label) - # MKDEBUG: The following columns do not exist (yet) - # for psf converted (pix2wcs) files. - try: - mag.append(np.asarray(data_j["MAG"])) - except: - mag.append(np.zeros_like(data_j["X"])) - try: - snr.append(np.asarray(data_j["SNR"])) - except: - snr.append(np.zeros_like(data_j["X"])) - try: - psfex_acc.append(np.asarray(data_j["ACCEPTED"])) - except: - psfex_acc.append(np.zeros_like(data_j["X"])) + at += n_rows + starcat_j.close() - # CCD number: this catalogue's one value over its own rows, as an - # array rather than a python list holding the same string N times. - ccd_nb.append(np.full( - len(data_j["RA"]), - re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2])) + if at != n_total: + # The two passes disagreed: an input changed under us, or the list + # was a one-shot iterable. Either way the output would be padded + # with uninitialised memory, so say so rather than write it. + raise ValueError( + f"merge_starcat: pass 1 counted {n_total} rows, pass 2 filled " + f"{at} — is the input list iterable more than once?") # Prepare output FITS catalogue # MKDEBUG: SEx_cat=True -> False @@ -656,25 +694,8 @@ def process(self): SEx_catalogue=False, ) - # Collect columns (size stored as T = 2 sigma^2) - data = { - "X": _stack(x), - "Y": _stack(y), - "RA": _stack(ra), - "DEC": _stack(dec), - "HSM_G1_PSF": _stack(g1_psf), - "HSM_G2_PSF": _stack(g2_psf), - "HSM_T_PSF": _stack(size_psf), - "HSM_G1_STAR": _stack(g1), - "HSM_G2_STAR": _stack(g2), - "HSM_T_STAR": _stack(size), - "HSM_FLAG_PSF": _stack(flag_psf), - "HSM_FLAG_STAR": _stack(flag_star), - "MAG": _stack(mag), - "SNR": _stack(snr), - "ACCEPTED": _stack(psfex_acc), - "CCD_NB": _stack(ccd_nb, dtype="U1"), - } + # `data` was built by the two passes above (size stored as T = 2 + # sigma^2); every column is already an array of its final length. # Write file # MKDEBUG for psf conv (pix2WCS) files do not write as SExtractorCat; diff --git a/workflow/Snakefile b/workflow/Snakefile index 0473c7ac6..734d11add 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -714,30 +714,31 @@ def star_cat_exposures(): # catalogues for the tile side. Peak RSS is getrusage(RUSAGE_CHILDREN). # # STAR SIDE. Two points, 20 and 80 exposures of 40 CCDs x 400 stars (1.6 MB of -# members per exposure, against the 2.0 MB measured on smk-m2): +# members per exposure, against the 2.0 MB measured on smk-m2), across the two +# rewrites this PR made to the accumulation in MergeStarCatPSFEX: # -# input members peak RSS, python lists peak RSS, arrays -# 32.3 MB 383 MB 238 MB -# 129.0 MB 1313 MB 740 MB +# input members python lists arrays+concat two passes +# 32.3 MB 383 MB 238 MB 221 MB +# 129.0 MB 1313 MB 740 MB 661 MB # -# a slope of 10.1x the input bytes before, 5.5x now, over a ~62 MB interpreter -# floor. The tenfold was MergeStarCat*'s accumulation of every column into -# PYTHON LISTS of python floats — 4 bytes of float32 payload becoming a 32-byte -# object plus an 8-byte pointer — and this PR replaced it with one array per -# catalogue and one concatenate at the end, output byte-identical. +# slope 10.1x 5.5x 4.8x # -# THE REMAINING 5.5x IS THE OUTPUT SIDE, and it is not a leak: file_io writes +# The tenfold was one python float object (32 bytes) plus a list pointer (8) +# per 4 bytes of float32 payload. Arrays per catalogue removed that; the +# two-pass structure — count rows from the FITS headers, allocate once at the +# exact length, then fill — removed what remained, so nothing is held twice. +# +# THE REMAINING 4.8x IS THE OUTPUT SIDE, and it is not a leak: file_io writes # every float column as FITS 1D, so a float32 input becomes a float64 table -# that astropy then buffers to write. Halving it means changing the OUTPUT -# format, which is what sp_validation reads — a different decision from this -# one, and not ours to take here. +# that astropy then buffers to write — 141 MB of table for 78 MB of payload at +# the 80-exposure point, plus its write copy. Halving it means changing the +# OUTPUT format, which is what sp_validation reads: a different decision from +# this one, and not ours to take here. # -# THE CEILING MOVED BUT DID NOT GO. At ~2 MB of members per exposure a 16 GB -# job now merges ~2200 exposures rather than ~800, and DR6's ~20k would want -# ~280 GB rather than ~400 GB. A full-survey full_starcat still needs the -# output side addressed; the formula below is honest about the slope so the job -# asks for what it will use and fails at submission rather than most of the way -# through. +# THE CEILING MOVED AND IS NOW ELSEWHERE. At ~2 MB of members per exposure a +# 16 GB job merges ~1300 exposures rather than ~800, and DR6's ~20k would want +# ~240 GB rather than ~400 GB. What stands between here and a full-survey +# full_starcat is the float64 output, not the merge. # # TILE SIDE, and it is the reassuring one. Two points against real smk-g6 # catalogues, 2 tiles (73.9 MB in, largest 39.6 MB) and 6 tiles (235.5 MB in, @@ -745,7 +746,7 @@ def star_cat_exposures(): # the merge holds one catalogue at a time — so it is sized on the LARGEST tile, # not the total, at ~3x it plus the interpreter. STAR_MEM_BASE_MB = 500 # interpreter + astropy + shapepipe, rounded up -STAR_MEM_FACTOR = 7 # x input bytes; 5.5 measured, rounded up +STAR_MEM_FACTOR = 6 # x input bytes; 4.8 measured, rounded up FINAL_MEM_BASE_MB = 800 FINAL_MEM_FACTOR = 4 # x the LARGEST tile; ~3 measured # What one unit costs when its product is not on disk yet to be stat()ed — a diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py index 002d156ff..f39364932 100644 --- a/workflow/scripts/merge_star_cat.py +++ b/workflow/scripts/merge_star_cat.py @@ -183,6 +183,10 @@ class TarMembers: ``__len__`` comes from the manifests, so the class can log the count before a single tar is opened. + + IT IS ITERABLE MORE THAN ONCE, and must be: the merge makes two passes, one + for row counts from the headers and one to fill. Each ``__iter__`` opens the + archives afresh, so the second pass sees the same members in the same order. """ def __init__(self, chosen): @@ -199,7 +203,13 @@ def __iter__(self): if member is None: sys.exit(f"merge_star_cat: {tar_path} has no member " f"{name}, which its manifest lists") - yield [io.BytesIO(member.read()), name] + # The tar's own file object, not a BytesIO of the whole + # member: it is seekable (the archive is uncompressed by + # design) and astropy reads through it, so the merge's + # first pass costs a header rather than a member. The + # object is valid only until the next member is reached, + # which is exactly how the merge consumes it. + yield [member, name] def main() -> None: From 3c2982588738718fa028f5e581638a2d74a51f2e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 19:47:05 -0400 Subject: [PATCH 16/20] feat(orchestration): final_cat_merge reconciles instead of rebuilding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule read every tile's catalogue on every run, because a DAG output must be a function of its input set and rebuilding is the simple way to guarantee that. At DR6 scale it is also ~800 GB of IO to add one 35 MB tile. It now brings the file INTO AGREEMENT with the campaign: a tile with no dataset is added, a dataset whose tile has left the campaign is deleted, a dataset whose source catalogue CHANGED is re-read, and one that agrees with its source is left alone, unread. Each dataset records its source's size and mtime as attributes, and a mismatch is what changed means — which is also what keeps the file from drifting from its inputs the way an append-only tool does. create_final_cat.py's own process() implements only the append-only half of this, skipping any tile already present whatever the file on disk now says. WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET, since this is the guarantee being traded. The file's CONTENT is: the same tiles with the same catalogues give the same datasets, the same columns and the same n_tiles, whether they arrived at once or one campaign at a time. Its BYTE LAYOUT is not, because hdf5 lays a group out in the order things were added. That is the price of not re-reading the campaign. UNTOUCHED ON A NO-OP, which is stronger than the byte comparison it replaces and cheaper to establish: reconciling is PLANNED against a read-only open, and an empty plan never opens the file for writing, so its mtime cannot move. A non-empty plan is carried out on a copy which is then moved into place, so a crash mid-merge leaves the old catalogue intact. VERIFIED on a three-tile fixture: build (3 added), no-op (unchanged, mtime identical to the nanosecond), append one tile WHILE AN EXISTING TILE'S CATALOGUE IS UNREADABLE — chmod 000, which succeeds and reports 1 added, so the existing tiles were demonstrably not read — rewrite of one catalogue (1 refreshed), and dropping two tiles from the list (2 removed, datasets gone, n_tiles 1). A from-scratch build of the same set is byte-stable across reruns. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- workflow/README.md | 33 ++++++ workflow/rules/tile.smk | 20 ++-- workflow/scripts/merge_final_cat.py | 175 ++++++++++++++++++++++------ 3 files changed, 182 insertions(+), 46 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 832a894f1..ab0f72d89 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -289,6 +289,39 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee escape hatch — anything with a glob metacharacter or a dot is read as one — and an unknown *name* is a parse-time error listing the valid ones. The list is exposure-side only; tile-side retention is #844 follow-up. +- **The campaign ends in two merged catalogues, and the workflow makes both.** + Everything above is per unit; the two products downstream analysis actually + opens are per *campaign*, and until these rules existed each was a manual pass + after the run. + `star_cat_merge` stacks every exposure's every CCD's `psf_validation` into one + `/full_starcat-0000000.fits` — the rho/tau statistics input, at + the path sp_validation hardcodes. It reads the members straight out of the + per-exposure tars (`tarfile`; unpacking ~800k files to merge them would defeat + the tar's whole purpose) and stacks them with `MergeStarCatPSFEX`, the same + class the old `merge_starcat_runner` called, so the column list has exactly + one definition. It exists whenever the campaign has a persisted exposure. + `final_cat_merge` collects every ready tile's `final_cat-.fits` into + `/final_cat_.hdf5`: one dataset per tile under a group + named for the campaign, the `final_cat.param` columns, an `n_tiles` attribute. + That schema is what sp_validation's reader opens, so it is fixed; the column + extraction reuses `scripts/python/create_final_cat.py` while the file is + written here, because that script's own discovery walks a directory layout + this workflow does not have. `campaign:` in `config.yaml` names the group and + defaults to the persistent root's basename. + `star_cat_merge` restacks the whole campaign, so its output is a function of + its input set and byte-stable on a no-op rerun (tmp-then-`cmp`-then-`mv`). + `final_cat_merge` RECONCILES instead — adds the tiles that have no dataset, + drops datasets whose tile left the campaign, re-reads one whose catalogue + changed (each dataset records its source's size and mtime), and leaves the + rest unread — because re-reading a campaign to add one tile is ~800 GB of IO + at DR6 scale. Its *content* is still a function of the input set; its byte + layout is not, and a no-op leaves the file untouched rather than rewritten. + Both rerun when the set changes: the unit ids' fingerprint rides on `params`. + Neither is a `localrule` — one job over ~20k units is real work — and neither + puts its input paths in its shell, which is not fastidiousness: ~20k paths is + an order of magnitude over Linux's 128 KiB `MAX_ARG_STRLEN` for a single argv + entry, so each job is handed the tile list and the run index and derives the + same set from them. - **A dead tile can be told to stop pinning exposures.** An exposure is cleanable only once every consuming tile has its vignets, so one permanently-failed tile holds its ~80 exposures for the life of the diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 4eb293b04..deb90138c 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -925,14 +925,18 @@ rule clean_tile: # tile-finished marker (see final_cat() in the Snakefile), and it is the file # this rule actually reads. # -# NOT A LOCALRULE, and here the reason is IO rather than memory: the job reads -# every tile's catalogue end to end on every run — ~32-46 MB per tile, so ~2 GB -# for a 64-tile campaign and ~800 GB at DR6's 23k tiles. It rebuilds rather than -# appends because a DAG output must be a function of its input set -# (merge_final_cat.py); incremental update by hand is what -# `create_final_cat.py -s add` remains for. Memory is one tile's catalogue at a -# time plus the hdf5 write buffer, which is why mem_mb is modest where -# star_cat_merge's is not. +# NOT A LOCALRULE, and here the reason is IO rather than memory: a first build +# reads every tile's catalogue end to end — ~32-46 MB per tile, so ~2 GB for a +# 64-tile campaign and ~800 GB at DR6's 23k tiles. It RECONCILES rather than +# rebuilds or appends: a tile with no dataset is added, a dataset whose tile +# left the campaign is deleted, a dataset whose source catalogue changed is +# re-read, and one that agrees with its source is left alone. So an append +# reads the appended tiles and nothing else, while the file still cannot drift +# from its inputs the way an append-only tool does (merge_final_cat.py argues +# what is and is not a function of the input set here). Memory is one tile's +# catalogue at a time plus the hdf5 write buffer, which is why mem_mb is modest +# where star_cat_merge's is not — and why runtime, which is sized on the whole +# campaign, is the pessimistic first-build case. rule final_cat_merge: input: lambda wc: [final_cat(t) for t in TILES_READY] diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py index 58357fcd5..601c2d0b7 100644 --- a/workflow/scripts/merge_final_cat.py +++ b/workflow/scripts/merge_final_cat.py @@ -38,20 +38,41 @@ loaded by path rather than imported: it is a script, not an installed module, and the container's ``shapepipe`` install does not carry it. -IT REBUILDS THE WHOLE FILE, IT DOES NOT APPEND. ``create_final_cat.py``'s own -``process()`` skips tiles already in the file, which is right for a hand-driven -incremental update (``-s add`` / ``-s remove`` are that tool's job). A DAG rule -wants the opposite: the output must be a pure function of the input set, so that -a no-op rerun is byte-stable and a changed set is visibly a different file. -Appending would make the result depend on the order campaigns were run in, and -would silently keep a tile whose catalogue was later rebuilt. The cost is -reading every tile's catalogue on every run of the rule — real work at DR6 scale -(~20k tiles), which is why this is not a localrule. - -BYTE-STABLE ON A NO-OP RERUN: written to a tmp path, compared, moved only if it -differs (the pattern ``persist_exp.py`` and ``clean_exposure.py`` use). Tiles -are visited in sorted ID order so the file is a function of the input set alone. -An unconditional rewrite would move the output's mtime every invocation. +IT RECONCILES, IT NEITHER REBUILDS NOR BLINDLY APPENDS. The output must be a +function of the input set — that is what makes the rule's fingerprint mean +something — but reading every tile's catalogue to add one tile is ~800 GB of IO +at DR6 scale for ~35 MB of new data. So the file is brought INTO AGREEMENT with +the campaign instead: + + * a campaign tile with no dataset is read and added; + * a dataset whose tile is no longer in the campaign is deleted; + * a dataset whose source catalogue has CHANGED is re-read. Each one records + its source's size and mtime as attributes, and a mismatch is what "changed" + means. This is the only reason a finished tile is ever read twice, and it is + the reason the file cannot drift from its inputs the way an append-only + tool does; + * a dataset that agrees with its source is left alone, unread. + +An append therefore reads exactly the appended tiles. ``create_final_cat.py``'s +own ``process()`` implements the append-only half of this — it skips a tile +already in the file, whatever the file on disk now says — which is right for a +hand-driven update and wrong for a DAG output; ``-s add`` / ``-s remove`` +remain that tool's way to do this by hand. + +WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET. The file's CONTENT is: the same +tiles with the same catalogues give the same datasets, the same columns and the +same n_tiles, whether they arrived at once or one campaign at a time. Its BYTE +LAYOUT is not, because hdf5 lays out a group in the order things were added. +That is the trade for not re-reading the campaign, and it is why the no-op case +below compares actions rather than bytes. +UNTOUCHED ON A NO-OP RERUN, which is stronger than byte-stable and cheaper to +establish. Reconciling is planned before anything is written: if the plan is +empty the file is not opened for writing at all, so its mtime cannot move — and +mtime is a rerun trigger, so an unconditional rewrite would make every +invocation look like a change. When the plan is NOT empty the existing file is +copied to a tmp path, changed there and moved into place, so a crash mid-merge +leaves the old catalogue intact rather than a half-written one. The copy is a +fraction of the reading it replaces. WHICH TILES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set is the CAMPAIGN's: every tile both declared in ``tile_list`` and present in the @@ -73,8 +94,8 @@ """ import argparse -import filecmp import importlib.util +import shutil import sys from pathlib import Path @@ -132,6 +153,97 @@ def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: return out +class Plan: + """What reconciling this campaign into this file requires: three tile lists. + + ``add`` and ``refresh`` are both "read the catalogue and write the dataset"; + they are separate only so the log can say which happened, because a refresh + means a finished tile's catalogue moved under us and that is worth seeing. + """ + + def __init__(self, add, refresh, remove): + self.add, self.refresh, self.remove = add, refresh, remove + + def empty(self): + return not (self.add or self.refresh or self.remove) + + def describe(self): + return (f"{len(self.add)} added, {len(self.refresh)} refreshed, " + f"{len(self.remove)} removed") + + +def stamp(path: Path) -> tuple: + """The source catalogue's identity, as recorded on its dataset. + + Size and mtime, not a checksum: the file is ~35 MB and the question is + "did this change since we read it", which mtime answers for a pipeline + that writes a catalogue once. A campaign that rewrites a final_cat in + place with identical size and mtime would defeat it, and nothing does. + """ + st = path.stat() + return st.st_size, st.st_mtime_ns + + +def reconcile_plan(output: Path, group_path: str, tiles: list) -> Plan: + """Compare the file on disk with the campaign, WITHOUT writing anything. + + Opened read-only, so a no-op invocation cannot move the output's mtime. + """ + if not output.exists(): + return Plan([t for t, _ in tiles], [], []) + + want = {tile: path for tile, path in tiles} + add, refresh = [], [] + with h5py.File(output, "r") as f: + have = dict(f[group_path].items()) if group_path in f else {} + present = set(have) + for tile, path in tiles: + if tile not in present: + add.append(tile) + continue + attrs = have[tile].attrs + if (int(attrs.get("src_bytes", -1)), + int(attrs.get("src_mtime_ns", -1))) != stamp(path): + refresh.append(tile) + return Plan(add, refresh, sorted(present - set(want))) + + +def apply_plan(output: Path, group_path: str, plan: Plan, tiles: list, + cfc, params: dict) -> None: + """Carry the plan out on a COPY, then move it into place. + + The copy is what makes a crash mid-merge leave the old catalogue intact, + and it costs a fraction of the reading it replaces — an append that copies + a 1 GB file to add one 35 MB tile still beats re-reading the campaign. + """ + paths = dict(tiles) + tmp = output.with_name(output.name + ".tmp") + try: + tmp.unlink(missing_ok=True) + if output.exists(): + shutil.copy2(output, tmp) + with h5py.File(tmp, "a") as f: + group = f[group_path] if group_path in f else f.create_group(group_path) + for tile in plan.remove: + del group[tile] + for tile in plan.refresh: + del group[tile] + for tile in plan.add + plan.refresh: + path = paths[tile] + extracted, dtype = cfc.read_data(str(path), params) + data = cfc.copy_data(params["param_list"], extracted, dtype) + dset = group.create_dataset(tile, data=data, dtype=data.dtype) + # The dataset's own record of what it was read from; this is + # what makes a later invocation able to leave it alone. + dset.attrs["src_bytes"], dset.attrs["src_mtime_ns"] = stamp(path) + # The same attribute create_final_cat.py's print_list() writes, and + # what sp_validation reads to know how many tiles it is holding. + f.attrs["n_tiles"] = len(group) + tmp.replace(output) # atomic: same filesystem + finally: + tmp.unlink(missing_ok=True) + + def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--products-dir", required=True, type=Path, @@ -164,30 +276,17 @@ def main() -> None: sys.exit(f"merge_final_cat: no tile in {args.tile_list} is indexed in " f"{args.index_db}, so there is nothing to merge") - # tmp-then-cmp-then-mv; the tmp never outlives this process. args.output.parent.mkdir(parents=True, exist_ok=True) - tmp = args.output.with_name(args.output.name + ".tmp") - try: - tmp.unlink(missing_ok=True) # h5py "a" would reopen a stale one - with h5py.File(tmp, "w") as hdf5_file: - group = hdf5_file.create_group(spval_group(args.campaign)) - for tile, path in tiles: - extracted, dtype = cfc.read_data(str(path), params) - data = cfc.copy_data(params["param_list"], extracted, dtype) - group.create_dataset(tile, data=data, dtype=data.dtype) - # The same attribute create_final_cat.py's print_list() writes, and - # what sp_validation reads to know how many tiles it is holding. - hdf5_file.attrs["n_tiles"] = len(tiles) - - if args.output.exists() and filecmp.cmp(tmp, args.output, shallow=False): - print(f"[merge_final_cat] unchanged: {args.output}") - else: - tmp.replace(args.output) # atomic: same filesystem - print(f"[merge_final_cat] {len(tiles)} tile(s), " - f"{len(param_list)} column(s) -> {args.output} " - f"(group {spval_group(args.campaign)})") - finally: - tmp.unlink(missing_ok=True) + group_path = spval_group(args.campaign) + plan = reconcile_plan(args.output, group_path, tiles) + if not plan.empty(): + apply_plan(args.output, group_path, plan, tiles, cfc, params) + print(f"[merge_final_cat] {plan.describe()} -> {args.output} " + f"({len(tiles)} tile(s), {len(param_list)} column(s), " + f"group {group_path})") + else: + print(f"[merge_final_cat] unchanged: {args.output} " + f"({len(tiles)} tile(s))") if __name__ == "__main__": From 78d2ce9c8620e6e07d377833cdd8ba0490ad581a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 20:03:30 -0400 Subject: [PATCH 17/20] fix(orchestration): eight defects found reviewing the merge work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OPTIONAL COLUMNS WERE DECIDED ONCE FOR THE WHOLE MERGE. MergeStarCatPSFEX read MAG/SNR/ACCEPTED out of the first catalogue's dtype and applied that verdict to every file behind it, so a merge over a mix of ordinary and pix2wcs-converted catalogues was wrong in both directions: ordinary first raised KeyError on the first converted file, and converted first SILENTLY ZEROED the real values of every ordinary file. The dtype now comes from any file that carries the column and pass 2 asks each file for its own schema, so only the files that actually lack a column are zero-filled. Both orderings verified on fixtures. A FAILED psfex_interp COULD GET A GREEN MANIFEST, and clean_exposure takes that manifest as its go-ahead to delete the store. With the default retention list, an exposure whose interpolation failed but whose PSFEx model landed had a non-empty match set, so the pack succeeded and the stars went with the store — unrecoverable short of rebuilding the chain from VOS. psf_validation is not optional: nothing matching it now fails the job, while the store is still on disk. Retention products that match nothing stay warnings. SHRINKING THE KEEP LIST DELETED PRODUCTS FROM /project. The list rides on `params`, so editing it reruns the pack — which rewrote the tar without what had been dropped, on the backed-up filesystem, with the scratch store it came from usually already reclaimed. RETENTION IS NOW ADDITIVE: an existing tar is a FLOOR, its members carried into the new one whatever the current list says, so a config change can only ever add. Removing a product is a deliberate act on products_dir, not a config edit. Verified: pack with [psf_model], rerun with an empty list, the .psf is still there under its own product name and the tar is byte-identical. THE COLUMN SET REACHED NO RERUN TRIGGER. final_cat_merge's reconcile keyed staleness on each source catalogue's size and mtime, and the column set is not a source catalogue: final_cat.param arrives through `params`, and the hash covered workflow/scripts/ only, not scripts/python/create_final_cat.py. So this PR's own edit to final_cat.param would have left every dataset in an existing hdf5 written to the old schema with nothing to notice. The file now carries a digest of the resolved column list on its root and refreshes every tile when it moves, and MERGE_FINAL_HASH covers all three files the rule's behaviour comes from. Verified: build, edit the parameter file, rerun -> 3 refreshed. STAR_CAT_MERGE'S MEMORY WAS SIZED ON THE TAR, which holds whatever the campaign retains, while the merge reads the psf_validation members alone. Measured on a fixture with a 3 MB PSF model kept: the tar is 92x the members it will read, and the default retention is 2.4x. It also jumped discontinuously as exposures were packed. The manifests record the product each member came from — exactly so this is answerable without opening a tar — so the sizing sums those members. NO REQUEST WAS CAPPED. A mem_mb above the partition maximum is a job SLURM never schedules and snakemake never diagnoses: it sits PENDING while the campaign looks alive. Both merge formulas grow with the campaign, so at some size they cross it. `max_mem_mb:` (default 750000, for Nibi's 766 GB standard node) caps both, with a parse-time warning naming the rule that was capped. copy_data ORDERED ITS OUTPUT BY THE SOURCE CATALOGUE, which made the merged dtype a property of the catalogue rather than of the parameter file: the ordered dedup added to read_param_file had no effect, and two tiles written by different ShapePipe versions landed in one group with two different structured dtypes, which np.concatenate refuses. It orders by param_list now. Verified on two catalogues with reversed column orders and an extra column: one dtype, concatenate works. The fixture hdf5 md5 moves with the column order, d2882294… -> 43ff946d…. RECONCILE LEAKED SPACE. It copied the file and deleted datasets in place, and HDF5 never reclaims that, so every refresh of a tile grew the file by that tile. A plan that removes or refreshes anything now builds the tmp fresh, moving the datasets it keeps across with h5py's own group copy — a dataset-level copy that never reads a row into numpy — so the result is compact; pure-append plans still copy and append. Verified: five successive full refreshes leave the file the same size, and dropping a tile shrinks it. Also: comments referring to the deleted parse-time keep-list gate are gone; `-s add` is documented as what it is (accepted by create_final_cat.py's validator, then falling through to the ordinary walk, so not a way to add one tile by hand); and three latent issues are noted where they live rather than fixed — hdu.columns.dtype ignoring TSCAL/TZERO (no validation_psf column is scaled), MergeStarCatSetools rebinding its ellipticity accumulators so only the last file's reach the output (pre-existing, setools is not wired to any workflow path), and the .tmp a SIGKILL can orphan next to the catalogue. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- scripts/python/create_final_cat.py | 20 ++-- .../merge_starcat_package/merge_starcat.py | 41 +++++-- workflow/README.md | 6 +- workflow/Snakefile | 105 +++++++++++++++--- workflow/config.yaml | 9 ++ workflow/rules/exposure.smk | 3 +- workflow/rules/tile.smk | 3 +- workflow/scripts/merge_final_cat.py | 85 ++++++++++---- workflow/scripts/merge_star_cat.py | 12 +- workflow/scripts/persist_exp.py | 77 ++++++++++++- 10 files changed, 299 insertions(+), 62 deletions(-) diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index efef24620..ea8a4d723 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -338,13 +338,19 @@ def copy_data(param_list, extracted_data, dtype): """Copy Data. """ - # THE REQUESTED COLUMNS ONLY, in the SOURCE catalogue's order. Allocating - # with the source's full dtype and filling only the requested columns left - # every other column as uninitialised memory: meaningless values in the - # output file, and different bytes on every run of this tool over the same - # inputs. The parameter file says which columns the merged catalogue is - # for; those are the columns it gets. - columns = [col for col in (dtype.names or ()) if col in set(param_list)] + # THE REQUESTED COLUMNS ONLY, IN THE PARAMETER FILE'S ORDER. Two things + # are being fixed here and they are easy to conflate. Allocating with the + # source's full dtype and filling only the requested columns left every + # other column as uninitialised memory — meaningless values, and different + # bytes on every run over the same inputs. And ordering the result by the + # SOURCE catalogue's columns made the output dtype a property of the + # catalogue rather than of the parameter file: two tiles written by + # different ShapePipe versions, whose catalogues order or extend their + # columns differently, then landed in one merged file with two different + # structured dtypes, which np.concatenate refuses. The parameter file is + # the schema; it says which columns AND in what order. + wanted = set(dtype.names or ()) + columns = [col for col in param_list if col in wanted] subset = np.dtype([(col, dtype[col]) for col in columns]) # Initialize new data structure diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 97e25e0c2..7f751006c 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -621,7 +621,15 @@ def process(self): ) # --- pass 1: row counts and dtypes, from headers alone -------------- - counts, labels, dtypes, n_total = [], [], None, 0 + # THE OPTIONAL COLUMNS ARE A PER-FILE QUESTION, NOT A PER-MERGE ONE. + # A pix2wcs-converted catalogue has no MAG/SNR/ACCEPTED while an + # ordinary one does, and a merge can be handed both. Deciding from the + # first file alone got it wrong in both directions: converted-first + # zero-filled the real values of every ordinary file behind it, and + # ordinary-first raised KeyError on the first converted one. So the + # dtype comes from ANY file that carries the column, and pass 2 asks + # each file for itself. + labels, dtypes, opt_dtypes, n_total = [], None, {}, 0 for name in self._input_file_list: source, label = name[0], name[-1] try: @@ -629,14 +637,21 @@ def process(self): ignore_missing_simple=True) as starcat_j: hdu = starcat_j[self._hdu_table] n_rows = hdu.header["NAXIS2"] + # ColDefs.dtype describes the table without reading it. + # NOTE: it is the RAW storage dtype and ignores TSCAL/TZERO, + # so a scaled column would be allocated narrower than the + # values .data returns. Latent, not live: no validation_psf + # column is scaled. Read the dtype off .data if one ever is. + cols = hdu.columns.dtype if dtypes is None: - # ColDefs.dtype describes the table without reading it. - dtypes = hdu.columns.dtype + dtypes = cols + for _, col in self._OPTIONAL: + if col not in opt_dtypes and col in (cols.names or ()): + opt_dtypes[col] = cols[col] except OSError: print(f"Error while opening file '{label}'") #raise continue - counts.append(n_rows) labels.append(label) n_total += n_rows @@ -644,12 +659,12 @@ def process(self): raise ValueError("merge_starcat: no readable input catalogue") # --- allocate once, at the exact final length ----------------------- - present = set(dtypes.names) data = {out: np.empty(n_total, dtype=dtypes[col]) for out, col in self._COLUMNS} for out, col in self._OPTIONAL: - data[out] = np.empty( - n_total, dtype=dtypes[col] if col in present else dtypes["X"]) + # A column no file carries still gets a column, zero-filled, in the + # positional dtype the old code used for it. + data[out] = np.empty(n_total, dtype=opt_dtypes.get(col, dtypes["X"])) # CCD_NB is one string per catalogue, repeated over its rows; its width # is the widest CCD number in the campaign, which pass 1 already knows. width = max((len(self._ccd_nb(lb)) for lb in labels), default=1) @@ -668,10 +683,13 @@ def process(self): n_rows = len(data_j) sl = slice(at, at + n_rows) + have = set(data_j.dtype.names or ()) for out, col in self._COLUMNS: data[out][sl] = data_j[col] for out, col in self._OPTIONAL: - data[out][sl] = data_j[col] if col in present else 0 + # THIS file's schema, not the merge's: zero-fill only the files + # that actually lack the column. + data[out][sl] = data_j[col] if col in have else 0 data["CCD_NB"][sl] = self._ccd_nb(label) at += n_rows @@ -850,6 +868,13 @@ def process(self): ra.append(np.asarray(data_j["XWIN_WORLD"])) dec.append(np.asarray(data_j["YWIN_WORLD"])) + # PRE-EXISTING BUG, LEFT ALONE DELIBERATELY: these four REBIND the + # accumulators initialised above rather than appending to them, so + # only the LAST input file's ellipticities reach the output while + # every other column carries the whole merge. Setools is not wired + # to any workflow path today; fixing it is its own change with its + # own verification, and doing it silently inside a memory rewrite + # would bury it. m11, m20, m02 = self.get_moments(data_j) eps1, eps2 = self.get_ellipticity(m11, m20, m02, "epsilon") chi1, chi2 = self.get_ellipticity(m11, m20, m02, "chi") diff --git a/workflow/README.md b/workflow/README.md index ab0f72d89..85cf4f465 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -285,7 +285,11 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee | `star_stats` | `star_stat-*.txt` | unmeasured | setools' per-CCD counts, density and FWHM cuts | The default is `psf_model`. `psf_validation` is in the catalogue too but needs - no naming; naming it anyway is harmless. A raw glob is still accepted as an + no naming; naming it anyway is harmless. **Retention is additive**: an + existing tar is a floor, so shrinking the list adds nothing and removes + nothing. Dropping a product is a deliberate act on `products_dir`, not a + config edit — otherwise editing a config would delete products from the + backed-up filesystem whose scratch originals are long gone. A raw glob is still accepted as an escape hatch — anything with a glob metacharacter or a dot is read as one — and an unknown *name* is a parse-time error listing the valid ones. The list is exposure-side only; tile-side retention is #844 follow-up. diff --git a/workflow/Snakefile b/workflow/Snakefile index 734d11add..c509b770e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -339,7 +339,21 @@ def unit_num(unit): # gets its own hash, on the forest rule only. def script_hash(name): """The 12-hex fingerprint of one script under workflow/scripts/.""" - return hashlib.md5((SCRIPTS / name).read_bytes()).hexdigest()[:12] + return path_hash(SCRIPTS / name) + + +def path_hash(path): + """The 12-hex fingerprint of any file the rules depend on but do not own. + + A rule whose behaviour comes from more than its own script needs all of it + in one trigger. final_cat_merge is the case: what it writes is decided by + scripts/python/create_final_cat.py (the column extraction) and by + config/cfis/final_cat.param (which columns), and NEITHER is under + workflow/scripts/ nor a declared input. Without them in the hash, this PR's + own edits to both would have left every finished campaign's hdf5 untouched + and nothing would have said so. + """ + return hashlib.md5(Path(path).read_bytes()).hexdigest()[:12] SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") @@ -347,7 +361,13 @@ CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") PERSIST_HASH = script_hash("persist_exp.py") MERGE_STAR_HASH = script_hash("merge_star_cat.py") -MERGE_FINAL_HASH = script_hash("merge_final_cat.py") +# Three files, one trigger: the rule's script, the column extraction it calls, +# and the parameter file that says which columns (path_hash argues why). +MERGE_FINAL_HASH = ":".join(( + script_hash("merge_final_cat.py"), + path_hash(Path(workflow.basedir).parent / "scripts" / "python" + / "create_final_cat.py"), + path_hash(CONFIG_DIR / "final_cat.param"))) # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -467,8 +487,8 @@ def clean_targets(): # --- persisted exposure products (D5) -------------------------------------- # The keep list is config, not a rule input, and it is READ HERE so that exactly -# one place converts it into the form the rule carries. An empty list is a -# deliberate "keep nothing" and produces no jobs at all. +# one place converts it into the form the rule carries. +# # OPTIONAL RETENTION, and only that. What star_cat_merge needs — every CCD's # psf_validation — is packed by exp_persist whatever this list says # (persist_exp.py's ALWAYS argues why: provenance for the merged catalogue, and @@ -745,14 +765,49 @@ def star_cat_exposures(): # largest 47.7 MB): peak RSS 129 MB and 139 MB. FLAT IN THE NUMBER OF TILES — # the merge holds one catalogue at a time — so it is sized on the LARGEST tile, # not the total, at ~3x it plus the interpreter. +# THE CEILING ON ANY REQUEST, and it is not a formatting nicety: a mem_mb above +# the partition maximum is a job SLURM will never schedule and snakemake will +# never diagnose — it sits PENDING with a reason nobody reads while the campaign +# looks alive. The two merge formulas grow with the campaign, so at some size +# they WILL cross it; capping turns "silently never runs" into "runs on the +# biggest node there is, and possibly dies with a diagnosable OOM". +# +# Nibi's standard compute node is 766 GB (192 cores, 4 GB/core); 750000 leaves +# room for the OS and the slurm accounting overhead. Override with `max_mem_mb:` +# for a cluster with smaller nodes, or to reserve headroom. +MAX_MEM_MB = int(config.get("max_mem_mb", 750_000)) +_capped_warned = set() + + +def capped_mem(mb, rule): + """min(mb, MAX_MEM_MB), and say so ONCE at parse time when it bites.""" + mb = int(mb) + if mb > MAX_MEM_MB: + if rule not in _capped_warned and workflow.is_main_process: + _capped_warned.add(rule) + logger.warning( + f"{rule}: sized at {mb} MB, capped to max_mem_mb={MAX_MEM_MB} " + f"(Nibi's standard node is 766 GB). The job will run with less " + f"memory than the measurement says it wants — expect an OOM, " + f"and split the campaign or fix the merge rather than raising " + f"this number past what a node has.") + return MAX_MEM_MB + return mb + + STAR_MEM_BASE_MB = 500 # interpreter + astropy + shapepipe, rounded up STAR_MEM_FACTOR = 6 # x input bytes; 4.8 measured, rounded up FINAL_MEM_BASE_MB = 800 FINAL_MEM_FACTOR = 4 # x the LARGEST tile; ~3 measured -# What one unit costs when its product is not on disk yet to be stat()ed — a -# fresh campaign sizes its merge before anything has been packed or made. -# Both are the measured medians in config.yaml's persist_exp block and D5 notes. +# What one unit costs when its product is not on disk yet to be measured — a +# fresh campaign sizes its merge before anything has been packed or made. The +# exposure figure is psf_validation's alone (the only members the star merge +# reads), not a whole tar's; both are the measured medians in config.yaml's +# persist_exp block and the D5 notes. EXP_BYTES_DEFAULT = 2_000_000 +# The product whose members star_cat_merge stacks — named once, here and in +# merge_star_cat.py, and resolved through persist_exp.py's catalogue. +STAR_CAT_PRODUCT = _persist.ALWAYS TILE_BYTES_DEFAULT = 46_000_000 @@ -765,15 +820,35 @@ def _size(path, default): def star_cat_bytes(): - """Total member bytes the star merge will read. - - The TAR is what gets stat()ed, not the manifest: it is one stat per - exposure rather than a json parse, and it is present for exactly the - exposures whose products already exist — live-and-already-packed as well as - reclaimed. An exposure not yet packed contributes the measured default. + """Total bytes of the members the star merge will actually read. + + THE TAR'S SIZE IS THE WRONG NUMBER, and increasingly wrong as the keep list + grows: the merge reads the psf_validation members and nothing else, while + the tar also holds whatever `persist_exp:` retains. With the default + retention that is 2.4x too much, and with the star_* products on it is ~36x + — a memory request that misses by more than an order of magnitude, and one + that would jump the moment an exposure got packed, since an unpacked one + contributed the per-exposure default instead. So the MANIFEST is read and + only the psf_validation members are counted; persist_exp records the product + each member came from, exactly so this is answerable without opening a tar. + + One json parse per exposure at DAG build, and only for the parse that builds + this job. An exposure not yet packed has no manifest and contributes the + measured default, which is the psf_validation figure and not the tar's. """ - return sum(_size(prod_exp_tar(e), EXP_BYTES_DEFAULT) - for e in star_cat_exposures()) + total = 0 + for exp in star_cat_exposures(): + manifest = Path(prod_exp_manifest(exp, "exp_persist")) + if not manifest.exists(): + total += EXP_BYTES_DEFAULT + continue + try: + body = json.loads(manifest.read_text()) + total += sum(f["bytes"] for f in body["files"] + if f.get("product") == STAR_CAT_PRODUCT) + except (OSError, ValueError, KeyError): + total += EXP_BYTES_DEFAULT + return total def final_cat_max_bytes(): diff --git a/workflow/config.yaml b/workflow/config.yaml index 1b19784b5..99ae1b560 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -239,6 +239,15 @@ clean_tiles: true # it is dead, not while you are still debugging it. clean_ignore_tiles: [] +# The ceiling on any rule's mem_mb. A request above the partition maximum is a +# job SLURM never schedules and snakemake never diagnoses: it sits PENDING while +# the campaign looks alive. Nibi's standard compute node is 766 GB (192 cores at +# 4 GB/core), so 750000 leaves room for the OS and slurm's own overhead. The two +# campaign-level merges size themselves from the campaign's bytes and will cross +# this at survey scale — the cap turns "never runs" into "runs on the biggest +# node there is", with a parse-time warning saying which rule was capped. +max_mem_mb: 750000 + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 0cbafb448..0b6a7a715 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -283,8 +283,9 @@ rule star_cat_merge: # measured (the Snakefile's sizing block carries both points, and the # ceiling this rule runs into at DR6 scale). Still * attempt, because a # measured slope on synthetic tars is not a guarantee about real ones. - mem_mb = lambda wc, attempt: attempt * ( + mem_mb = lambda wc, attempt: capped_mem(attempt * ( STAR_MEM_BASE_MB + STAR_MEM_FACTOR * star_cat_bytes() // 1_000_000), + "star_cat_merge"), # ~2 min per GB of members on the measurement above, doubled, over a # floor that covers the fixed cost of opening ~40 members per exposure. runtime = lambda wc, attempt: attempt * ( diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index deb90138c..46d84f02f 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -955,9 +955,10 @@ rule final_cat_merge: # Sized on the LARGEST tile, not the total: the merge holds one # catalogue at a time, and the measurement is flat in the tile count # (the Snakefile's sizing block carries both points). - mem_mb = lambda wc, attempt: attempt * ( + mem_mb = lambda wc, attempt: capped_mem(attempt * ( FINAL_MEM_BASE_MB + FINAL_MEM_FACTOR * final_cat_max_bytes() // 1_000_000), + "final_cat_merge"), # Runtime, unlike memory, is the TOTAL: every tile is read end to end. # ~1 min per 10 tiles on the measurement, triply generous, over a floor. runtime = lambda wc, attempt: attempt * (30 + len(TILES_READY) // 3) diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py index 601c2d0b7..4bae00f98 100644 --- a/workflow/scripts/merge_final_cat.py +++ b/workflow/scripts/merge_final_cat.py @@ -56,8 +56,10 @@ An append therefore reads exactly the appended tiles. ``create_final_cat.py``'s own ``process()`` implements the append-only half of this — it skips a tile already in the file, whatever the file on disk now says — which is right for a -hand-driven update and wrong for a DAG output; ``-s add`` / ``-s remove`` -remain that tool's way to do this by hand. +hand-driven update and wrong for a DAG output. (Its ``-s`` single-ID mode +implements ``check`` and ``remove``; ``add`` is accepted by the argument +validator and then falls through to the ordinary walk, so it is not a way to +add one tile by hand.) WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET. The file's CONTENT is: the same tiles with the same catalogues give the same datasets, the same columns and the @@ -94,6 +96,7 @@ """ import argparse +import hashlib import importlib.util import shutil import sys @@ -153,6 +156,20 @@ def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: return out +def schema_digest(param_list: list) -> str: + """A fingerprint of the COLUMN SET the datasets were written with. + + Recorded on the file's root and compared on every reconcile, because the + column set is the one input to this merge that nothing else can see. It is + not a source catalogue, so no dataset's size/mtime stamp moves when it + changes; it reaches the job through --param-file, which is a `params` value + and not a rule input. Without this, editing final_cat.param — which this PR + itself does — would leave every dataset in an existing hdf5 written to the + OLD schema, and nothing would ever notice. + """ + return hashlib.md5("\n".join(param_list).encode()).hexdigest()[:16] + + class Plan: """What reconciling this campaign into this file requires: three tile lists. @@ -184,7 +201,8 @@ def stamp(path: Path) -> tuple: return st.st_size, st.st_mtime_ns -def reconcile_plan(output: Path, group_path: str, tiles: list) -> Plan: +def reconcile_plan(output: Path, group_path: str, tiles: list, + digest: str) -> Plan: """Compare the file on disk with the campaign, WITHOUT writing anything. Opened read-only, so a no-op invocation cannot move the output's mtime. @@ -195,39 +213,62 @@ def reconcile_plan(output: Path, group_path: str, tiles: list) -> Plan: want = {tile: path for tile, path in tiles} add, refresh = [], [] with h5py.File(output, "r") as f: + # A changed column set invalidates every dataset at once — they were + # written to the old schema and nothing about their sources moved. + stale_schema = f.attrs.get("param_digest") != digest have = dict(f[group_path].items()) if group_path in f else {} present = set(have) for tile, path in tiles: if tile not in present: add.append(tile) - continue - attrs = have[tile].attrs - if (int(attrs.get("src_bytes", -1)), - int(attrs.get("src_mtime_ns", -1))) != stamp(path): + elif stale_schema: refresh.append(tile) + else: + attrs = have[tile].attrs + if (int(attrs.get("src_bytes", -1)), + int(attrs.get("src_mtime_ns", -1))) != stamp(path): + refresh.append(tile) return Plan(add, refresh, sorted(present - set(want))) def apply_plan(output: Path, group_path: str, plan: Plan, tiles: list, - cfc, params: dict) -> None: - """Carry the plan out on a COPY, then move it into place. - - The copy is what makes a crash mid-merge leave the old catalogue intact, - and it costs a fraction of the reading it replaces — an append that copies - a 1 GB file to add one 35 MB tile still beats re-reading the campaign. + cfc, params: dict, digest: str) -> None: + """Carry the plan out on a tmp file, then move it into place. + + TWO WAYS TO BUILD THE TMP, and which one is used is about SPACE, not speed. + HDF5 never reclaims the space a deleted dataset occupied, so a file that is + copied and then edited in place grows for the life of the campaign — every + refresh of a 15 MB tile leaks 15 MB. So: + + * a plan that only ADDS copies the existing file and appends to it. There + is nothing to reclaim, and copying beats rewriting. + * a plan that removes or refreshes anything builds the tmp FRESH, moving + the datasets it keeps across with h5py's own group copy — which is a + dataset-level copy inside the library and never reads a row into numpy — + and writing only the tiles that actually changed. The result is compact. + + Either way the tmp is moved into place at the end, so a crash mid-merge + leaves the old catalogue intact rather than a half-written one. A SIGKILL + between writing the tmp and renaming it leaves the tmp behind — one file, + next to the catalogue, overwritten by the next run; the rename itself is + atomic, which is the property that matters. """ paths = dict(tiles) + rewrite = bool(plan.remove or plan.refresh) + keep = [t for t, _ in tiles if t not in set(plan.add) | set(plan.refresh)] tmp = output.with_name(output.name + ".tmp") try: tmp.unlink(missing_ok=True) - if output.exists(): + if output.exists() and not rewrite: shutil.copy2(output, tmp) with h5py.File(tmp, "a") as f: - group = f[group_path] if group_path in f else f.create_group(group_path) - for tile in plan.remove: - del group[tile] - for tile in plan.refresh: - del group[tile] + group = (f[group_path] if group_path in f + else f.create_group(group_path)) + if rewrite and output.exists(): + with h5py.File(output, "r") as src: + for tile in keep: + src[f"{group_path}/{tile}"].copy( + src[f"{group_path}/{tile}"], group, name=tile) for tile in plan.add + plan.refresh: path = paths[tile] extracted, dtype = cfc.read_data(str(path), params) @@ -239,6 +280,7 @@ def apply_plan(output: Path, group_path: str, plan: Plan, tiles: list, # The same attribute create_final_cat.py's print_list() writes, and # what sp_validation reads to know how many tiles it is holding. f.attrs["n_tiles"] = len(group) + f.attrs["param_digest"] = digest tmp.replace(output) # atomic: same filesystem finally: tmp.unlink(missing_ok=True) @@ -278,9 +320,10 @@ def main() -> None: args.output.parent.mkdir(parents=True, exist_ok=True) group_path = spval_group(args.campaign) - plan = reconcile_plan(args.output, group_path, tiles) + digest = schema_digest(param_list) + plan = reconcile_plan(args.output, group_path, tiles, digest) if not plan.empty(): - apply_plan(args.output, group_path, plan, tiles, cfc, params) + apply_plan(args.output, group_path, plan, tiles, cfc, params, digest) print(f"[merge_final_cat] {plan.describe()} -> {args.output} " f"({len(tiles)} tile(s), {len(param_list)} column(s), " f"group {group_path})") diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py index f39364932..131758f57 100644 --- a/workflow/scripts/merge_star_cat.py +++ b/workflow/scripts/merge_star_cat.py @@ -107,9 +107,10 @@ class as ``[fileobj, member_name]`` pairs — ONE AT A TIME, lazily, through # The members this merge consumes, named as the keep list names them and # resolved through the same catalogue persist_exp packs by — so the glob has one -# definition and adding a product cannot leave the two disagreeing. The rule -# refuses to exist unless `persist_exp:` keeps something of this shape (the -# Snakefile checks at parse time), so the members are expected here. +# definition and adding a product cannot leave the two disagreeing. They are +# always there to find: persist_exp packs this product for every exposure +# whatever `persist_exp:` says, and fails the pack rather than writing a +# manifest without it. MEMBER_PRODUCT = "psf_validation" MEMBER_PATTERN = persist_exp.resolve(MEMBER_PRODUCT) @@ -246,8 +247,9 @@ def main() -> None: # existence check and produce meaningless rho statistics. sys.exit(f"merge_star_cat: no member matched {args.pattern!r} in any " f"of {len(manifest_paths)} exp_persist manifest(s) for this " - f"campaign — is '{MEMBER_PRODUCT}' in the persist_exp keep " - f"list?") + f"campaign. persist_exp packs {MEMBER_PRODUCT} for every " + f"exposure, so this means the manifests are not what we think " + f"they are.") if empty: log.info(f"{len(empty)} exposure(s) persisted no {args.pattern}: " f"{', '.join(sorted(empty)[:5])}" diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index 4e2dff349..24e0c1d66 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -29,6 +29,15 @@ glob. Patterns are therefore plain FILE names and the layout is ours to know, not the config author's. +RETENTION IS ADDITIVE, AND THAT IS A SAFETY PROPERTY. The keep list rides on +the rule's ``params``, so SHRINKING it reruns this script — and a naive rerun +would rewrite the tar without the products that were dropped, deleting them +from the backed-up filesystem because someone edited a config, with the scratch +store they came from usually long gone. An existing tar is therefore a FLOOR: +its members are carried into the new one whatever the current list says, and a +config change can only ever add. Removing a product is a deliberate act on +products_dir, not a config edit. + THE KEEP LIST IS WHAT THE CAMPAIGN KEEPS ON TOP OF THE MERGE'S INPUTS. ``psf_validation`` is packed unconditionally (see ALWAYS below); ``persist_exp:`` is purely optional retention, and an EMPTY one is a coherent instruction — the @@ -277,6 +286,21 @@ def main() -> None: sys.exit(f"persist_exp: {exc.args[0]}") found, empty = collect(args.exp_dir, entries) + # THE MERGE'S INPUTS ARE NOT ALLOWED TO BE MISSING, and this is a harder + # rule than "something matched". An exposure whose psfex_interp failed but + # whose PSFEx model landed has a non-empty match set under the default + # retention list, so it used to get a green manifest — and clean_exposure + # takes that manifest as its go-ahead and deletes the store, taking the + # stars with it. There is no recovering them afterwards short of rebuilding + # the chain from VOS, so a missing psf_validation fails the job here, while + # the store is still on disk. Retention products that match nothing stay + # warnings: they are optional by construction. + if ALWAYS not in found: + sys.exit(f"persist_exp: {args.exp}: nothing matched {ALWAYS} " + f"({resolve(ALWAYS)}) under {args.exp_dir}/output/{RUN_NAME}. " + f"That is the star catalogue's input and it is not optional — " + f"refusing to write a manifest that would let clean_exposure " + f"reclaim this store.") if not found: sys.exit(f"persist_exp: {args.exp}: no file matched any of " f"{entries} under {args.exp_dir}/output/{RUN_NAME}") @@ -304,6 +328,35 @@ def main() -> None: files.append({"name": src.name, "product": pat, "pattern": resolve(pat), "src": str(src), "bytes": src.stat().st_size}) + + # --- RETENTION IS ADDITIVE: an existing tar is a FLOOR, never a draft ---- + # Shrinking `persist_exp:` used to rerun this rule (the list rides on + # params, which is the whole point of the rule) and overwrite the tar with + # a smaller one — deleting products from the BACKED-UP filesystem because + # someone edited a config. The scratch store they came from is usually gone + # by then, so nothing could put them back. Whatever is already in the tar + # therefore stays in it: a config change can only ever ADD. + # + # Removing a product is consequently not a config edit. It is a deliberate + # act on products_dir, and it should look like one. + carried, prior_products = [], {} + if tar_path.exists(): + prior = args.manifest + if prior.exists(): + try: + prior_products = {f["name"]: f.get("product", "?") + for f in json.loads(prior.read_text())["files"]} + except (OSError, ValueError, KeyError): + pass # a damaged manifest loses only labels + with tarfile.open(tar_path) as tf: + for ti in tf.getmembers(): + if ti.name in seen or not ti.isfile(): + continue # a live source supersedes it + carried.append(ti.name) + files.append({"name": ti.name, + "product": prior_products.get(ti.name, "?"), + "pattern": None, "src": None, "bytes": ti.size}) + files.sort(key=lambda f: f["name"]) def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: @@ -319,9 +372,24 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: # design exists to avoid, one per failed attempt at DR6 scale. tmp = tar_path.with_name(tar_path.name + ".tmp") try: + # One pass in sorted member order, taking each member from whichever + # side has it: a live source on disk, or the existing tar. Members are + # copied across with their own TarInfo, so a carried member is + # byte-for-byte what it was and a rerun that changes nothing still + # produces an identical archive. with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: - for f in files: - tf.add(seen[f["name"]][0], arcname=f["name"], filter=anonymous) + old_tar = (tarfile.open(tar_path) if carried else None) + try: + for f in files: + if f["name"] in seen: + tf.add(seen[f["name"]][0], arcname=f["name"], + filter=anonymous) + else: + ti = anonymous(old_tar.getmember(f["name"])) + tf.addfile(ti, old_tar.extractfile(f["name"])) + finally: + if old_tar is not None: + old_tar.close() if tar_path.exists() and filecmp.cmp(tmp, tar_path, shallow=False): tmp.unlink() # unchanged: leave the mtime alone else: @@ -354,7 +422,10 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: finally: tmp.unlink(missing_ok=True) - warn = f" ({len(empty)} pattern(s) matched nothing: {empty})" if empty else "" + warn = (f" ({len(empty)} retention product(s) matched nothing: {empty})" + if empty else "") + if carried: + warn += f" ({len(carried)} member(s) carried from the existing tar)" print(f"[persist_exp] {args.exp}: {len(files)} file(s), " f"{body['bytes'] / 1e6:.1f} MB -> {tar_path}{warn}") From 0fb514b85d88c85c8da5d91949bef8738500fae1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 20:16:43 -0400 Subject: [PATCH 18/20] feat(orchestration): the star catalogue becomes hdf5, reconciled like the tile one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The campaign's two products behaved differently for no reason anyone chose. The shear catalogue reconciled — an append read the appended tiles — while the star catalogue was one flat FITS table that had to be restacked from every exposure the campaign had ever seen to add one: ~40 GB of members at DR6 scale to add ~2 MB, held in memory while it happened. Now they are the same thing. /full_starcat_.hdf5, one dataset per exposure at exposures/, each holding that exposure's every CCD's rows with a CCD_NB column, an n_exposures root attribute and the same column digest the tile side carries. Named for the campaign exactly as the shear catalogue beside it is. CCD_NB IS AN INT: it is parsed out of the member name, where it is always digits, so a string buys nothing and costs 8 bytes a row against 4. DTYPES ARE NATIVE: float32 stays float32, where the FITS writer widened every float column to 1D, doubling both the file and the peak memory of the job that wrote it for no information. MEMORY IS NOW FLAT IN THE CAMPAIGN — one exposure at a time — so the rule is sized on the largest exposure's members rather than the campaign's, and the ~240 GB a DR6-scale flat table would have wanted is simply not a number any more. The Snakefile's sizing block keeps the measurements that got us here, because they are the argument for the format. THE RECONCILE MACHINERY IS NOW ONE MODULE, workflow/scripts/hdf5_reconcile.py, used by both merges rather than duplicated: plan against a read-only open, add/refresh/remove, refresh everything when the column digest moves, compact rewrite when anything is removed or refreshed, untouched on a no-op. Writing it twice would have been two chances to disagree about what an output owes its inputs. THE WORKFLOW NO LONGER CALLS MergeStarCat* AT ALL, so merge_star_cat.py drops the shapepipe import and the psf-model switch, and the [fileobj, name] entry shape those classes learned for it is REVERTED — with no caller it was upstream surface with nothing behind it. What stays upstream is what fixes the module runner's own problems: the two-pass allocation, and asking each file for its own optional columns instead of deciding once for the merge. The runner path is byte-identical to before all of it, md5 f7caa1cf… on the fixture. VERIFIED on the fixtures: build (2 added); no-op (unchanged, mtime identical to the nanosecond); append one exposure with the others' tars at chmod 000, which succeeds and reports 1 added, so they were demonstrably not read; remove one (1 removed, dataset gone, n_exposures 2). Every one of the 16 columns equals the FITS version's values. The tile side's compaction sequence was re-run against a fix this work exposed — the keep-what-changed path called Dataset.copy, which does not exist, and only bites when a plan both rewrites and keeps something. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- .../merge_starcat_package/merge_starcat.py | 54 +-- workflow/README.md | 17 +- workflow/Snakefile | 88 ++-- workflow/rules/exposure.smk | 45 ++- workflow/scripts/hdf5_reconcile.py | 167 ++++++++ workflow/scripts/merge_final_cat.py | 207 ++-------- workflow/scripts/merge_star_cat.py | 380 +++++++++--------- 7 files changed, 500 insertions(+), 458 deletions(-) create mode 100644 workflow/scripts/hdf5_reconcile.py diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 7f751006c..1e0cc7256 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -269,15 +269,10 @@ def process(self): my_mask[inside_circle] = True for name in self._input_file_list: - # The source to read and the NAME to report it by; identical for a - # plain [path] entry (see MergeStarCatPSFEX's docstring on the - # [fileobj, name] form). This class takes its CCD numbers from the - # data's own CCD_ID_LIST, so the name is only ever used in messages. - source, label = name[0], name[-1] try: - starcat_j = fits.open(source, memmap=False, ignore_missing_simple=True) + starcat_j = fits.open(name[0], memmap=False, ignore_missing_simple=True) except ValueError: - print(f"Error for file {label}, check FITS file integrity") + print(f"Error for file {name[0]}, check FITS file integrity") #raise continue @@ -536,15 +531,7 @@ class MergeStarCatPSFEX(object): Parameters ---------- input_file_list : list - Input entries. Each entry is a list, as the module runner builds them: - ``[path]`` from the file handler. An entry may also carry a name - alongside an already-open source, ``[fileobj, name]`` — ``fits.open`` - takes the first element and the CCD number is parsed from the LAST, - which is the same string in the one-element case. That is what lets a - caller merge catalogues it never wrote to disk (the Snakemake - workflow's ``star_cat_merge`` reads them out of the per-exposure tars - with ``tarfile`` + ``BytesIO``), without this class learning anything - about where they came from. + Input files output_dir : str Output directory w_log : logging.Logger @@ -586,9 +573,9 @@ def __init__( # (MKDEBUG); zero-filled when missing rather than failing the merge. _OPTIONAL = (("MAG", "MAG"), ("SNR", "SNR"), ("ACCEPTED", "ACCEPTED")) - def _ccd_nb(self, label): + def _ccd_nb(self, path): """The CCD number this catalogue's rows carry, parsed from its name.""" - return re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2] + return re.split(r"\-([0-9]*)\-([0-9]+)\.", path)[-2] def process(self): """Process. @@ -611,10 +598,9 @@ def process(self): accumulation: there are no chunks, and no concatenate that must hold its inputs and its result at the same time. - ``self._input_file_list`` MUST BE ITERABLE TWICE. A list is; so is the - workflow's tar reader, whose ``__iter__`` opens the archives afresh. - A one-shot generator is not, and would silently merge nothing on the - second pass — hence the explicit length check below. + ``self._input_file_list`` MUST BE ITERABLE TWICE, which the module + runner's list is. A one-shot generator is not, and would silently merge + nothing on the second pass — hence the explicit row-count check below. """ self._w_log.info( f"Merging {len(self._input_file_list)} star catalogues" @@ -629,11 +615,10 @@ def process(self): # ordinary-first raised KeyError on the first converted one. So the # dtype comes from ANY file that carries the column, and pass 2 asks # each file for itself. - labels, dtypes, opt_dtypes, n_total = [], None, {}, 0 + names, dtypes, opt_dtypes, n_total = [], None, {}, 0 for name in self._input_file_list: - source, label = name[0], name[-1] try: - with fits.open(source, memmap=False, + with fits.open(name[0], memmap=False, ignore_missing_simple=True) as starcat_j: hdu = starcat_j[self._hdu_table] n_rows = hdu.header["NAXIS2"] @@ -649,10 +634,10 @@ def process(self): if col not in opt_dtypes and col in (cols.names or ()): opt_dtypes[col] = cols[col] except OSError: - print(f"Error while opening file '{label}'") + print(f"Error while opening file '{name[0]}'") #raise continue - labels.append(label) + names.append(name[0]) n_total += n_rows if dtypes is None: @@ -667,15 +652,14 @@ def process(self): data[out] = np.empty(n_total, dtype=opt_dtypes.get(col, dtypes["X"])) # CCD_NB is one string per catalogue, repeated over its rows; its width # is the widest CCD number in the campaign, which pass 1 already knows. - width = max((len(self._ccd_nb(lb)) for lb in labels), default=1) + width = max((len(self._ccd_nb(n)) for n in names), default=1) data["CCD_NB"] = np.empty(n_total, dtype=f"U{width}") # --- pass 2: fill --------------------------------------------------- at = 0 for name in self._input_file_list: - source, label = name[0], name[-1] try: - starcat_j = fits.open(source, memmap=False, + starcat_j = fits.open(name[0], memmap=False, ignore_missing_simple=True) except OSError: continue @@ -690,7 +674,7 @@ def process(self): # THIS file's schema, not the merge's: zero-fill only the files # that actually lack the column. data[out][sl] = data_j[col] if col in have else 0 - data["CCD_NB"][sl] = self._ccd_nb(label) + data["CCD_NB"][sl] = self._ccd_nb(name[0]) at += n_rows starcat_j.close() @@ -854,11 +838,7 @@ def process(self): ) for name in self._input_file_list: - # The source to read and the NAME to parse the CCD number out of; - # identical for a plain [path] entry (see MergeStarCatPSFEX's - # docstring on the [fileobj, name] form). - source, label = name[0], name[-1] - starcat_j = fits.open(source, memmap=False) + starcat_j = fits.open(name[0], memmap=False) data_j = starcat_j[self._hdu_table].data @@ -892,7 +872,7 @@ def process(self): # CCD number ccd_nb.append(np.full( len(data_j["XWIN_IMAGE"]), - re.split(r"\-([0-9]*)\-([0-9]+)\.", label)[-2])) + re.split(r"\-([0-9]*)\-([0-9]+)\.", name[0])[-2])) # Prepare output FITS catalogue output = file_io.FITSCatalogue( diff --git a/workflow/README.md b/workflow/README.md index 85cf4f465..40371f2b4 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -312,14 +312,15 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee written here, because that script's own discovery walks a directory layout this workflow does not have. `campaign:` in `config.yaml` names the group and defaults to the persistent root's basename. - `star_cat_merge` restacks the whole campaign, so its output is a function of - its input set and byte-stable on a no-op rerun (tmp-then-`cmp`-then-`mv`). - `final_cat_merge` RECONCILES instead — adds the tiles that have no dataset, - drops datasets whose tile left the campaign, re-reads one whose catalogue - changed (each dataset records its source's size and mtime), and leaves the - rest unread — because re-reading a campaign to add one tile is ~800 GB of IO - at DR6 scale. Its *content* is still a function of the input set; its byte - layout is not, and a no-op leaves the file untouched rather than rewritten. + BOTH RECONCILE, through one shared module (`hdf5_reconcile.py`) so the + campaign's two products cannot disagree about what an output owes its inputs. + Each adds the units that have no dataset, drops datasets whose unit left the + campaign, re-reads one whose source changed (every dataset records its + source's size and mtime) or whose column set moved (a digest on the file's + root), and leaves the rest unread — because re-reading a campaign to add one + unit is ~800 GB of IO at DR6 scale. The *content* is still a function of the + input set; the byte layout is not, and a no-op leaves the file untouched + rather than rewritten. Both rerun when the set changes: the unit ids' fingerprint rides on `params`. Neither is a `localrule` — one job over ~20k units is real work — and neither puts its input paths in its shell, which is not fastidiousness: ~20k paths is diff --git a/workflow/Snakefile b/workflow/Snakefile index c509b770e..ea4129f25 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -31,6 +31,7 @@ the manifest says "this stage succeeded", the log says "here is what happened" (the contract is argued in completeness.py's docstring). """ +import fnmatch import functools import hashlib import json @@ -647,9 +648,14 @@ def unit_fingerprint(units): def full_starcat(): - """The campaign's merged star catalogue. The NAME is not ours to choose: - sp_validation hardcodes `full_starcat-0000000.fits` beside its data dir.""" - return f"{PRODUCTS_DIR}/full_starcat-0000000.fits" + """The campaign's merged star catalogue — the rho/tau statistics input. + + hdf5, one dataset per exposure, named for the campaign exactly as the shear + catalogue beside it is. The old flat FITS table it replaces was called + `full_starcat-0000000.fits` and sp_validation still opens that name; + CosmoStat/sp_validation#340 moves its readers to this file, the same + migration that retires the `patches/` key on the galaxy side.""" + return f"{PRODUCTS_DIR}/full_starcat_{CAMPAIGN}.hdf5" def final_cat_hdf5(): @@ -733,32 +739,31 @@ def star_cat_exposures(): # against synthetic tars for the star side and against smk-g6's real # catalogues for the tile side. Peak RSS is getrusage(RUSAGE_CHILDREN). # -# STAR SIDE. Two points, 20 and 80 exposures of 40 CCDs x 400 stars (1.6 MB of -# members per exposure, against the 2.0 MB measured on smk-m2), across the two -# rewrites this PR made to the accumulation in MergeStarCatPSFEX: +# STAR SIDE, AND IT IS FLAT IN THE CAMPAIGN. The merge writes one hdf5 dataset +# per exposure and reads one exposure at a time, so it is sized on the LARGEST +# exposure's members — ~2 MB — not on the campaign's. What follows is the +# history of how that came to be true, because the numbers are the argument. +# +# The FITS full_starcat this replaced was one flat table, so the job held the +# whole campaign. Two fixture points, 20 and 80 exposures of 40 CCDs x 400 stars +# (1.6 MB of members per exposure, against the 2.0 MB measured on smk-m2), +# across the rewrites this PR made to MergeStarCatPSFEX: # # input members python lists arrays+concat two passes # 32.3 MB 383 MB 238 MB 221 MB # 129.0 MB 1313 MB 740 MB 661 MB -# # slope 10.1x 5.5x 4.8x # -# The tenfold was one python float object (32 bytes) plus a list pointer (8) -# per 4 bytes of float32 payload. Arrays per catalogue removed that; the -# two-pass structure — count rows from the FITS headers, allocate once at the -# exact length, then fill — removed what remained, so nothing is held twice. +# The tenfold was one python float object (32 bytes) plus a list pointer (8) per +# 4 bytes of float32 payload. Arrays per catalogue removed that; counting rows +# from the headers and filling a preallocated array removed the rest. What +# remained at 4.8x was the OUTPUT: file_io writes every float column as FITS 1D, +# so float32 became a float64 table astropy then buffered. # -# THE REMAINING 4.8x IS THE OUTPUT SIDE, and it is not a leak: file_io writes -# every float column as FITS 1D, so a float32 input becomes a float64 table -# that astropy then buffers to write — 141 MB of table for 78 MB of payload at -# the 80-exposure point, plus its write copy. Halving it means changing the -# OUTPUT format, which is what sp_validation reads: a different decision from -# this one, and not ours to take here. -# -# THE CEILING MOVED AND IS NOW ELSEWHERE. At ~2 MB of members per exposure a -# 16 GB job merges ~1300 exposures rather than ~800, and DR6's ~20k would want -# ~240 GB rather than ~400 GB. What stands between here and a full-survey -# full_starcat is the float64 output, not the merge. +# Per-exposure hdf5 removes the term entirely rather than shrinking it — and +# with it the ~240 GB a DR6-scale flat table would have wanted. Those +# improvements stay upstream regardless: the module runner still merges to one +# FITS table, and they are its fix. # # TILE SIDE, and it is the reassuring one. Two points against real smk-g6 # catalogues, 2 tiles (73.9 MB in, largest 39.6 MB) and 6 tiles (235.5 MB in, @@ -795,8 +800,8 @@ def capped_mem(mb, rule): return mb -STAR_MEM_BASE_MB = 500 # interpreter + astropy + shapepipe, rounded up -STAR_MEM_FACTOR = 6 # x input bytes; 4.8 measured, rounded up +STAR_MEM_BASE_MB = 500 # interpreter + astropy + h5py, rounded up +STAR_MEM_FACTOR = 6 # x the LARGEST exposure's members FINAL_MEM_BASE_MB = 800 FINAL_MEM_FACTOR = 4 # x the LARGEST tile; ~3 measured # What one unit costs when its product is not on disk yet to be measured — a @@ -808,6 +813,7 @@ EXP_BYTES_DEFAULT = 2_000_000 # The product whose members star_cat_merge stacks — named once, here and in # merge_star_cat.py, and resolved through persist_exp.py's catalogue. STAR_CAT_PRODUCT = _persist.ALWAYS +STAR_CAT_PATTERN = _persist.resolve(_persist.ALWAYS) TILE_BYTES_DEFAULT = 46_000_000 @@ -819,6 +825,17 @@ def _size(path, default): return default +def star_cat_max_bytes(): + """The LARGEST exposure's psf_validation members — what sizes the merge. + + The star merge holds ONE exposure at a time now that its output is hdf5 + with a dataset per exposure, so its memory is flat in the campaign exactly + as the tile side's is. Sizing on the total would ask a node for a campaign's + worth of memory to hold ~2 MB. + """ + return max(_star_cat_exposure_bytes() or [EXP_BYTES_DEFAULT]) + + def star_cat_bytes(): """Total bytes of the members the star merge will actually read. @@ -836,19 +853,30 @@ def star_cat_bytes(): this job. An exposure not yet packed has no manifest and contributes the measured default, which is the psf_validation figure and not the tar's. """ - total = 0 + return sum(_star_cat_exposure_bytes()) + + +@functools.lru_cache(maxsize=1) +def _star_cat_exposure_bytes(): + """Per exposure, the bytes of the members the star merge will read.""" + out = [] for exp in star_cat_exposures(): manifest = Path(prod_exp_manifest(exp, "exp_persist")) if not manifest.exists(): - total += EXP_BYTES_DEFAULT + out.append(EXP_BYTES_DEFAULT) continue try: body = json.loads(manifest.read_text()) - total += sum(f["bytes"] for f in body["files"] - if f.get("product") == STAR_CAT_PRODUCT) + # By product name or, for a manifest written before that field + # existed or by a raw-glob keep list, by file name — the same test + # merge_star_cat.is_member() applies, so the sizing counts exactly + # the members the job will read. + out.append(sum(f["bytes"] for f in body["files"] + if f.get("product") == STAR_CAT_PRODUCT + or fnmatch.fnmatch(f["name"], STAR_CAT_PATTERN))) except (OSError, ValueError, KeyError): - total += EXP_BYTES_DEFAULT - return total + out.append(EXP_BYTES_DEFAULT) + return out def final_cat_max_bytes(): diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 0b6a7a715..5845adb8d 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -225,14 +225,20 @@ rule clean_exposure: # --- the campaign's star catalogue ------------------------------------------ # ONE job per campaign: every exposure's every CCD's `validation_psf--.fits`, -# stacked into `/full_starcat-0000000.fits`. That file is the -# rho/tau statistics input and sp_validation reads it at exactly that path, -# doing no merging of its own; the old bash chain built it with -# `combine_runs.bash psf` + a `merge_starcat_runner` pass, and the workflow -# emitted neither. The stacking itself is `MergeStarCatPSFEX` — the same class -# the old runner called, reused rather than restated, so a column added to the -# module is a column added here (merge_star_cat.py argues the reuse and the -# tar-member reading). +# collected into `/full_starcat_.hdf5`, one dataset per +# exposure. That file is the rho/tau statistics input; the old bash chain built +# a flat FITS table with `combine_runs.bash psf` + a `merge_starcat_runner` +# pass, and the workflow emitted neither. sp_validation still opens the FITS +# name today — CosmoStat/sp_validation#340 moves its readers to this file, the +# same migration that retires the `patches/` key on the tile side. +# +# ONE DATASET PER EXPOSURE, NOT ONE TABLE, and it is the same decision as the +# tile side's: it makes the file RECONCILABLE. A flat table had to be restacked +# from every exposure the campaign had ever seen to add one — ~40 GB of members +# at DR6 scale to add ~2 MB — and held the whole campaign in memory while it did +# so. Reconciled, an append reads the appended exposures and nothing else, and +# the job holds one exposure at a time. hdf5_reconcile.py is the shared +# machinery; merge_star_cat.py argues the format and the tar reading. # # THE INPUT IS star_cat_inputs() (Snakefile): every exposure of TILES_READY whose # PSF products are on the persistent root — the live ones through the exp_persist @@ -253,19 +259,14 @@ rule clean_exposure: # the point: a job that stacked anything the fingerprint did not see would be # rows no rerun trigger could notice, which is what a glob over products_dir # would have given on a root shared with an earlier, larger tile list. -# Byte-stable output otherwise (tmp-then-cmp-then-mv), so a no-op rerun does not -# move its mtime. # # NOT A LOCALRULE. exp_persist is local because it is 20k jobs of seconds; this -# is one job that holds a campaign's stars in memory (~800k catalogues at DR6 -# scale). mem_mb is a guess scaled by attempt, not a measurement — the campaigns -# run so far are 127 exposures, three orders of magnitude short of the case this -# sizing is for, and the first DR6-scale run should replace this number with a -# benchmark. +# is one job that reads the campaign's tars end to end. Its MEMORY is flat in +# the campaign (one exposure at a time) and sized on the largest exposure; its +# RUNTIME is the total. # -# NO JOB AT ALL when `persist_exp:` keeps no validation catalogue, or when every -# exposure in scope is tombstoned: star_cat_targets() (Snakefile) simply does not -# request the output, and the parse says so rather than a node failing later. +# NO JOB AT ALL when every exposure in scope is tombstoned with no tar left +# behind: star_cat_targets() (Snakefile) simply does not request the output. rule star_cat_merge: input: lambda wc: star_cat_inputs() @@ -275,6 +276,7 @@ rule star_cat_merge: products_dir = str(PRODUCTS_DIR), tile_list = str(config["tile_list"]), index_db = str(INDEX_DB), + campaign = CAMPAIGN, inputs = unit_fingerprint(star_cat_exposures()), script_hash = MERGE_STAR_HASH threads: 1 @@ -283,8 +285,11 @@ rule star_cat_merge: # measured (the Snakefile's sizing block carries both points, and the # ceiling this rule runs into at DR6 scale). Still * attempt, because a # measured slope on synthetic tars is not a guarantee about real ones. + # Sized on the LARGEST exposure, not the total: the merge holds one + # exposure at a time (the Snakefile's sizing block carries the history). mem_mb = lambda wc, attempt: capped_mem(attempt * ( - STAR_MEM_BASE_MB + STAR_MEM_FACTOR * star_cat_bytes() // 1_000_000), + STAR_MEM_BASE_MB + + STAR_MEM_FACTOR * star_cat_max_bytes() // 1_000_000), "star_cat_merge"), # ~2 min per GB of members on the measurement above, doubled, over a # floor that covers the fixed cost of opening ~40 members per exposure. @@ -296,4 +301,4 @@ rule star_cat_merge: " --products-dir '{params.products_dir}'" " --tile-list '{params.tile_list}' --index-db '{params.index_db}'" " --output {output.star_cat}" - f" --psf-model {PSF_MODEL}" + " --campaign '{params.campaign}'" diff --git a/workflow/scripts/hdf5_reconcile.py b/workflow/scripts/hdf5_reconcile.py new file mode 100644 index 000000000..0635b5976 --- /dev/null +++ b/workflow/scripts/hdf5_reconcile.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Bring an hdf5 catalogue into agreement with a campaign, one dataset per unit. + +Shared by the two campaign-level merges — ``merge_final_cat.py`` (one dataset +per tile) and ``merge_star_cat.py`` (one per exposure) — because they want +exactly the same thing of their output and disagreeing about it would be a bug +waiting to happen rather than a difference worth having. + +WHY RECONCILE RATHER THAN REBUILD. The output must be a function of the input +set — that is what makes the rules' fingerprints mean anything — but reading +every unit to add one is ~800 GB of IO at DR6 scale for a few tens of MB of new +data. So the file is brought INTO AGREEMENT with the campaign instead: + + * a unit with no dataset is read and added; + * a dataset whose unit has left the campaign is deleted; + * a dataset whose SOURCE has changed is re-read. Each records its source's + size and mtime as attributes, and a mismatch is what changed means. This is + the only reason a finished unit is read twice, and it is why the file + cannot drift from its inputs the way an append-only tool does; + * a dataset whose column set was written under a DIFFERENT SCHEMA is re-read. + The column set is the one input nothing else can see: it is not a source + file, so no stamp moves when it changes. It travels as a digest on the + file's root. + * a dataset that agrees with its source and its schema is left alone, unread. + +An append therefore reads exactly the appended units. + +WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET. The file's CONTENT is: the same +units with the same sources give the same datasets, the same columns and the +same count attribute, whether they arrived at once or one batch at a time. Its +BYTE LAYOUT is not, because hdf5 lays a group out in the order things were +added. That is the trade for not re-reading the campaign, and it is why the +no-op case compares ACTIONS rather than bytes. + +UNTOUCHED ON A NO-OP, which is stronger than byte-stable and cheaper to +establish. Reconciling is PLANNED against a read-only open; an empty plan never +opens the file for writing, so its mtime cannot move — and mtime is a rerun +trigger, so an unconditional rewrite would make every invocation look like a +change. +""" + +import hashlib +import shutil +from pathlib import Path + +import h5py + + +def schema_digest(columns) -> str: + """A fingerprint of the COLUMN SET the datasets were written with.""" + return hashlib.md5("\n".join(columns).encode()).hexdigest()[:16] + + +def stamp(path: Path) -> tuple: + """A source's identity, as recorded on the dataset built from it. + + Size and mtime, not a checksum: the question is "did this change since we + read it", which mtime answers for a pipeline that writes a file once. A + campaign that rewrote a source in place with identical size and mtime would + defeat it, and nothing does. + """ + st = Path(path).stat() + return st.st_size, st.st_mtime_ns + + +class Plan: + """What reconciling requires: three unit lists. + + ``add`` and ``refresh`` are both "read the source and write the dataset"; + they are separate only so the log can say which happened, because a refresh + means a finished unit's source moved under us and that is worth seeing. + """ + + def __init__(self, add, refresh, remove): + self.add, self.refresh, self.remove = add, refresh, remove + + def empty(self): + return not (self.add or self.refresh or self.remove) + + def describe(self): + return (f"{len(self.add)} added, {len(self.refresh)} refreshed, " + f"{len(self.remove)} removed") + + +def plan(output: Path, group_path: str, units: list, digest: str) -> Plan: + """Compare the file on disk with the campaign, WITHOUT writing anything.""" + if not output.exists(): + return Plan([u for u, _ in units], [], []) + + want = {unit for unit, _ in units} + add, refresh = [], [] + with h5py.File(output, "r") as f: + stale_schema = f.attrs.get("param_digest") != digest + have = dict(f[group_path].items()) if group_path in f else {} + present = set(have) + for unit, source in units: + if unit not in present: + add.append(unit) + elif stale_schema: + refresh.append(unit) + else: + attrs = have[unit].attrs + if (int(attrs.get("src_bytes", -1)), + int(attrs.get("src_mtime_ns", -1))) != stamp(source): + refresh.append(unit) + return Plan(add, refresh, sorted(present - want)) + + +def apply(output: Path, group_path: str, todo: Plan, units: list, read, + digest: str, count_attr: str) -> None: + """Carry the plan out on a tmp file, then move it into place. + + ``read(unit, source)`` returns the structured array for one unit; it is + called only for the units the plan names, which is what makes an append + cheap. + + TWO WAYS TO BUILD THE TMP, and which one is used is about SPACE, not speed. + HDF5 never reclaims the space a deleted dataset occupied, so a file that is + copied and then edited in place grows for the life of the campaign — every + refresh of a unit leaks that unit. So: + + * a plan that only ADDS copies the existing file and appends to it. There + is nothing to reclaim, and copying beats rewriting. + * a plan that removes or refreshes anything builds the tmp FRESH, moving + the datasets it keeps across with h5py's own group copy — a + dataset-level copy inside the library that never reads a row into numpy + — and writing only the units that actually changed. The result is + compact. + + Either way the tmp is moved into place at the end, so a crash mid-merge + leaves the old catalogue intact rather than a half-written one. A SIGKILL + between writing the tmp and renaming it leaves the tmp behind — one file, + beside the catalogue, overwritten by the next run; the rename itself is + atomic, which is the property that matters. + """ + sources = dict(units) + rewrite = bool(todo.remove or todo.refresh) + written = set(todo.add) | set(todo.refresh) + keep = [u for u, _ in units if u not in written] + tmp = output.with_name(output.name + ".tmp") + try: + tmp.unlink(missing_ok=True) + if output.exists() and not rewrite: + shutil.copy2(output, tmp) + with h5py.File(tmp, "a") as f: + group = (f[group_path] if group_path in f + else f.create_group(group_path)) + if rewrite and output.exists(): + with h5py.File(output, "r") as src: + for unit in keep: + # File.copy, not Dataset.copy — the latter does not + # exist, and the difference only shows when a plan both + # rewrites and keeps something. + src.copy(f"{group_path}/{unit}", group, name=unit) + for unit in todo.add + todo.refresh: + source = sources[unit] + data = read(unit, source) + dset = group.create_dataset(unit, data=data, dtype=data.dtype) + # The dataset's own record of what it was read from; this is + # what lets a later invocation leave it alone. + dset.attrs["src_bytes"], dset.attrs["src_mtime_ns"] = \ + stamp(source) + f.attrs[count_attr] = len(group) + f.attrs["param_digest"] = digest + tmp.replace(output) # atomic: same filesystem + finally: + tmp.unlink(missing_ok=True) diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py index 4bae00f98..2db61e7e0 100644 --- a/workflow/scripts/merge_final_cat.py +++ b/workflow/scripts/merge_final_cat.py @@ -38,43 +38,18 @@ loaded by path rather than imported: it is a script, not an installed module, and the container's ``shapepipe`` install does not carry it. -IT RECONCILES, IT NEITHER REBUILDS NOR BLINDLY APPENDS. The output must be a -function of the input set — that is what makes the rule's fingerprint mean -something — but reading every tile's catalogue to add one tile is ~800 GB of IO -at DR6 scale for ~35 MB of new data. So the file is brought INTO AGREEMENT with -the campaign instead: - - * a campaign tile with no dataset is read and added; - * a dataset whose tile is no longer in the campaign is deleted; - * a dataset whose source catalogue has CHANGED is re-read. Each one records - its source's size and mtime as attributes, and a mismatch is what "changed" - means. This is the only reason a finished tile is ever read twice, and it is - the reason the file cannot drift from its inputs the way an append-only - tool does; - * a dataset that agrees with its source is left alone, unread. - -An append therefore reads exactly the appended tiles. ``create_final_cat.py``'s -own ``process()`` implements the append-only half of this — it skips a tile -already in the file, whatever the file on disk now says — which is right for a -hand-driven update and wrong for a DAG output. (Its ``-s`` single-ID mode -implements ``check`` and ``remove``; ``add`` is accepted by the argument -validator and then falls through to the ordinary walk, so it is not a way to -add one tile by hand.) - -WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET. The file's CONTENT is: the same -tiles with the same catalogues give the same datasets, the same columns and the -same n_tiles, whether they arrived at once or one campaign at a time. Its BYTE -LAYOUT is not, because hdf5 lays out a group in the order things were added. -That is the trade for not re-reading the campaign, and it is why the no-op case -below compares actions rather than bytes. -UNTOUCHED ON A NO-OP RERUN, which is stronger than byte-stable and cheaper to -establish. Reconciling is planned before anything is written: if the plan is -empty the file is not opened for writing at all, so its mtime cannot move — and -mtime is a rerun trigger, so an unconditional rewrite would make every -invocation look like a change. When the plan is NOT empty the existing file is -copied to a tmp path, changed there and moved into place, so a crash mid-merge -leaves the old catalogue intact rather than a half-written one. The copy is a -fraction of the reading it replaces. +IT RECONCILES, IT NEITHER REBUILDS NOR BLINDLY APPENDS, and the machinery for +that is ``hdf5_reconcile.py``, shared with the star side so the campaign's two +products cannot disagree about what an output owes its inputs. That module +carries the argument in full: an append reads the appended tiles, a source that +changed is re-read, a tile that left the campaign is deleted, a column-set +change refreshes everything, and a no-op leaves the file untouched. +``create_final_cat.py``'s own ``process()`` implements only the append-only half +— it skips a tile already in the file, whatever the file on disk now says — +which is right for a hand-driven update and wrong for a DAG output. (Its ``-s`` +single-ID mode implements ``check`` and ``remove``; ``add`` is accepted by the +argument validator and then falls through to the ordinary walk, so it is not a +way to add one tile by hand.) WHICH TILES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set is the CAMPAIGN's: every tile both declared in ``tile_list`` and present in the @@ -96,16 +71,13 @@ """ import argparse -import hashlib import importlib.util -import shutil import sys from pathlib import Path -import h5py - # Same directory; the rule invokes this file by path, so it is sys.path[0]. import build_index +import hdf5_reconcile # /scripts/python/create_final_cat.py, from /workflow/scripts/this. CFC_PATH = (Path(__file__).resolve().parents[2] @@ -156,136 +128,6 @@ def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: return out -def schema_digest(param_list: list) -> str: - """A fingerprint of the COLUMN SET the datasets were written with. - - Recorded on the file's root and compared on every reconcile, because the - column set is the one input to this merge that nothing else can see. It is - not a source catalogue, so no dataset's size/mtime stamp moves when it - changes; it reaches the job through --param-file, which is a `params` value - and not a rule input. Without this, editing final_cat.param — which this PR - itself does — would leave every dataset in an existing hdf5 written to the - OLD schema, and nothing would ever notice. - """ - return hashlib.md5("\n".join(param_list).encode()).hexdigest()[:16] - - -class Plan: - """What reconciling this campaign into this file requires: three tile lists. - - ``add`` and ``refresh`` are both "read the catalogue and write the dataset"; - they are separate only so the log can say which happened, because a refresh - means a finished tile's catalogue moved under us and that is worth seeing. - """ - - def __init__(self, add, refresh, remove): - self.add, self.refresh, self.remove = add, refresh, remove - - def empty(self): - return not (self.add or self.refresh or self.remove) - - def describe(self): - return (f"{len(self.add)} added, {len(self.refresh)} refreshed, " - f"{len(self.remove)} removed") - - -def stamp(path: Path) -> tuple: - """The source catalogue's identity, as recorded on its dataset. - - Size and mtime, not a checksum: the file is ~35 MB and the question is - "did this change since we read it", which mtime answers for a pipeline - that writes a catalogue once. A campaign that rewrites a final_cat in - place with identical size and mtime would defeat it, and nothing does. - """ - st = path.stat() - return st.st_size, st.st_mtime_ns - - -def reconcile_plan(output: Path, group_path: str, tiles: list, - digest: str) -> Plan: - """Compare the file on disk with the campaign, WITHOUT writing anything. - - Opened read-only, so a no-op invocation cannot move the output's mtime. - """ - if not output.exists(): - return Plan([t for t, _ in tiles], [], []) - - want = {tile: path for tile, path in tiles} - add, refresh = [], [] - with h5py.File(output, "r") as f: - # A changed column set invalidates every dataset at once — they were - # written to the old schema and nothing about their sources moved. - stale_schema = f.attrs.get("param_digest") != digest - have = dict(f[group_path].items()) if group_path in f else {} - present = set(have) - for tile, path in tiles: - if tile not in present: - add.append(tile) - elif stale_schema: - refresh.append(tile) - else: - attrs = have[tile].attrs - if (int(attrs.get("src_bytes", -1)), - int(attrs.get("src_mtime_ns", -1))) != stamp(path): - refresh.append(tile) - return Plan(add, refresh, sorted(present - set(want))) - - -def apply_plan(output: Path, group_path: str, plan: Plan, tiles: list, - cfc, params: dict, digest: str) -> None: - """Carry the plan out on a tmp file, then move it into place. - - TWO WAYS TO BUILD THE TMP, and which one is used is about SPACE, not speed. - HDF5 never reclaims the space a deleted dataset occupied, so a file that is - copied and then edited in place grows for the life of the campaign — every - refresh of a 15 MB tile leaks 15 MB. So: - - * a plan that only ADDS copies the existing file and appends to it. There - is nothing to reclaim, and copying beats rewriting. - * a plan that removes or refreshes anything builds the tmp FRESH, moving - the datasets it keeps across with h5py's own group copy — which is a - dataset-level copy inside the library and never reads a row into numpy — - and writing only the tiles that actually changed. The result is compact. - - Either way the tmp is moved into place at the end, so a crash mid-merge - leaves the old catalogue intact rather than a half-written one. A SIGKILL - between writing the tmp and renaming it leaves the tmp behind — one file, - next to the catalogue, overwritten by the next run; the rename itself is - atomic, which is the property that matters. - """ - paths = dict(tiles) - rewrite = bool(plan.remove or plan.refresh) - keep = [t for t, _ in tiles if t not in set(plan.add) | set(plan.refresh)] - tmp = output.with_name(output.name + ".tmp") - try: - tmp.unlink(missing_ok=True) - if output.exists() and not rewrite: - shutil.copy2(output, tmp) - with h5py.File(tmp, "a") as f: - group = (f[group_path] if group_path in f - else f.create_group(group_path)) - if rewrite and output.exists(): - with h5py.File(output, "r") as src: - for tile in keep: - src[f"{group_path}/{tile}"].copy( - src[f"{group_path}/{tile}"], group, name=tile) - for tile in plan.add + plan.refresh: - path = paths[tile] - extracted, dtype = cfc.read_data(str(path), params) - data = cfc.copy_data(params["param_list"], extracted, dtype) - dset = group.create_dataset(tile, data=data, dtype=data.dtype) - # The dataset's own record of what it was read from; this is - # what makes a later invocation able to leave it alone. - dset.attrs["src_bytes"], dset.attrs["src_mtime_ns"] = stamp(path) - # The same attribute create_final_cat.py's print_list() writes, and - # what sp_validation reads to know how many tiles it is holding. - f.attrs["n_tiles"] = len(group) - f.attrs["param_digest"] = digest - tmp.replace(output) # atomic: same filesystem - finally: - tmp.unlink(missing_ok=True) - - def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--products-dir", required=True, type=Path, @@ -320,16 +162,23 @@ def main() -> None: args.output.parent.mkdir(parents=True, exist_ok=True) group_path = spval_group(args.campaign) - digest = schema_digest(param_list) - plan = reconcile_plan(args.output, group_path, tiles, digest) - if not plan.empty(): - apply_plan(args.output, group_path, plan, tiles, cfc, params, digest) - print(f"[merge_final_cat] {plan.describe()} -> {args.output} " - f"({len(tiles)} tile(s), {len(param_list)} column(s), " - f"group {group_path})") - else: + digest = hdf5_reconcile.schema_digest(param_list) + + def read_tile(tile, path): + """One tile's requested columns, via create_final_cat.py's own reader.""" + extracted, dtype = cfc.read_data(str(path), params) + return cfc.copy_data(params["param_list"], extracted, dtype) + + todo = hdf5_reconcile.plan(args.output, group_path, tiles, digest) + if todo.empty(): print(f"[merge_final_cat] unchanged: {args.output} " f"({len(tiles)} tile(s))") + return + hdf5_reconcile.apply(args.output, group_path, todo, tiles, read_tile, + digest, "n_tiles") + print(f"[merge_final_cat] {todo.describe()} -> {args.output} " + f"({len(tiles)} tile(s), {len(param_list)} column(s), " + f"group {group_path})") if __name__ == "__main__": diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py index 131758f57..25c9db0e1 100644 --- a/workflow/scripts/merge_star_cat.py +++ b/workflow/scripts/merge_star_cat.py @@ -1,37 +1,52 @@ #!/usr/bin/env python3 -"""Concatenate the campaign's per-CCD PSF validation catalogues into ONE full_starcat. +"""Collect the campaign's per-CCD PSF validation catalogues into ONE hdf5 file. Run as the shell of the campaign-level ``star_cat_merge`` rule, never by hand. -WHAT IT PRODUCES, AND FOR WHOM. ``/full_starcat-0000000.fits``: -every exposure's every CCD's ``validation_psf--.fits`` row, stacked, -with a ``CCD_NB`` column recording which CCD each row came from. It is the input -to the rho/tau statistics — sp_validation reads exactly this path -(``star_cat_path`` in its ``scripts/calibration/params.py``) and does no merging -of its own. Historically it was ``combine_runs.bash psf`` + a -``merge_starcat_runner`` pass; the workflow emitted neither, so the product set -was short one file. This script is that pass, driven by the DAG instead of by -bash. - -IT DOES NOT REIMPLEMENT THE COLUMN LIST. The stacking, the column names and the -CCD_NB parse all live in ``MergeStarCatPSFEX`` -(``shapepipe.modules.merge_starcat_package.merge_starcat``), which is what the -old runner called. This script only decides WHICH catalogues that class is -handed, and where the result lands. A column added to the module is a column -added here for free — which is the entire reason for the indirection. - -IT READS THE TARS, IT DOES NOT UNPACK THEM, AND IT STREAMS. ``exp_persist`` -packs each exposure's keepers into one uncompressed tar on the persistent root +WHAT IT PRODUCES, AND FOR WHOM. ``/full_starcat_.hdf5``: +one dataset per exposure at ``exposures/``, holding that exposure's every +CCD's ``validation_psf--.fits`` rows stacked, with a ``CCD_NB`` column +recording which CCD each row came from. It is the input to the rho/tau +statistics. Historically this was ``combine_runs.bash psf`` plus a +``merge_starcat_runner`` pass producing one flat FITS table, +``full_starcat-0000000.fits``, and sp_validation still opens that name today; +its readers move to this hdf5 under CosmoStat/sp_validation#340, the same +migration that retires the ``patches/`` key on the galaxy side. + +WHY HDF5, AND WHY ONE DATASET PER EXPOSURE. The campaign's two products should +behave the same way, and one flat table cannot: appending a tile meant +restacking every exposure the campaign had ever seen — ~40 GB of members at DR6 +scale to add ~2 MB. Per-exposure datasets make the file RECONCILABLE +(hdf5_reconcile.py carries that argument, and merge_final_cat.py is the same +machinery on the tile side), so an append reads the appended exposures and +nothing else while the file still cannot drift from its inputs. Memory follows: +one exposure at a time, not one campaign. + +NATIVE DTYPES. Columns are written as the validation_psf files store them — +float32 stays float32. The FITS writer this replaces widened every float column +to ``1D``, doubling both the file and the peak memory of the job that wrote it, +for no information. + +CCD_NB IS AN INTEGER. It is parsed out of the member name +(``validation_psf--.fits``), where it is always digits, so a string +buys nothing — and an int column costs 4 bytes a row against the 8 a +two-character fixed-width string does. + +IT READS THE TARS, IT DOES NOT UNPACK THEM. ``exp_persist`` packs each +exposure's keepers into one uncompressed tar on the persistent root (``/exp///psf/.tar``) precisely because inodes, -not bytes, bind on /project. Unpacking ~20k tars × ~40 members to merge them -would materialise ~800k files on the filesystem that design exists to protect, -and then delete them. So members are read out of the tars in memory -(``tarfile.extractfile(m).read()`` -> ``io.BytesIO``) and handed to the merge -class as ``[fileobj, member_name]`` pairs — ONE AT A TIME, lazily, through -``TarMembers`` below, because materialising them all first is ~40 GB at DR6 -scale. The member NAME is what the CCD_NB regex parses, which is why the pair -carries it; the class takes the name from the last element of the entry, so a -plain ``[path]`` entry behaves exactly as it always did. +not bytes, bind on /project. Unpacking ~20k tars x ~40 members to merge them +would materialise ~800k files on the filesystem that design exists to protect. +Members are read through the archive's own file object — seekable, the tar +being uncompressed by design — so the counting pass costs a header rather than +a member. + +THE OPTIONAL COLUMNS ARE A PER-FILE QUESTION. A pix2wcs-converted catalogue has +no MAG/SNR/ACCEPTED where an ordinary one does, and a campaign can hold both. +Deciding once for the merge is wrong in both directions: it either fails on the +first converted file or silently zeroes the real values of every ordinary one. +Each file is asked for its own schema, and only the files that lack a column are +zero-filled. WHICH EXPOSURES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set is the CAMPAIGN's: every exposure read by a tile that is both declared @@ -42,99 +57,96 @@ class as ``[fileobj, member_name]`` pairs — ONE AT A TIME, lazily, through there is one query and not two that can drift), and then takes the exposures whose ``exp_persist`` manifest is on the persistent root. -It is derived rather than passed because at DR6 scale the set is ~20k paths, and +It is derived rather than passed because at DR6 scale the set is ~20k paths and a shell command reaches ``execve`` as a SINGLE argv entry capped at 128 KiB by -``MAX_ARG_STRLEN``. Passing them would be a job that dies before it starts. So -the rule's ``input`` is the DAG EDGE — what must exist before this runs — and -the rule's ``params`` carries a FINGERPRINT of that same list, which is what -makes the merge rerun when the set changes. - -THE TWO SETS ARE THE SAME SET, and that equality is the point of deriving it -this way rather than globbing the tree. The rule's input is ``star_cat_inputs()`` -(Snakefile): for each exposure of TILES_READY whose PSF products are on the -persistent root, an edge — the ``exp_persist`` manifest for a live exposure, the -TAR for one whose scratch store reclamation already took (that function argues -the asymmetry, which is about not rebuilding a reclaimed chain from VOS). -Nothing at all for an exposure reclaimed before ``exp_persist`` existed, which -left neither and is unrecoverable short of that rebuild. What this script -selects is the same rule stated from the job's side: same tiles, same index, -manifest present — and by the time the job runs, every exposure with an edge has -one. A glob over ``/exp`` would NOT be the same set: it would -sweep in exposures of an earlier, larger tile list sharing the products root, -stacking rows the fingerprint never saw and no rerun trigger would notice. +``MAX_ARG_STRLEN``. So the rule's ``input`` is the DAG EDGE — what must exist +before this runs — and its ``params`` carries a FINGERPRINT of the same set, +which is what makes the merge rerun when the set changes. A glob over +``/exp`` would NOT be the same set: it would sweep in exposures of +an earlier, larger tile list sharing the products root, stacking rows the +fingerprint never saw and no rerun trigger would notice. THE MANIFEST, NOT THE TAR, IS WHAT IT READS FIRST: the manifest records what was -actually packed, pattern by pattern, member by member, with sizes. Selecting -members from it means this script never guesses at tar contents, and an exposure -whose keep list did not include the validation catalogues contributes nothing -visibly rather than silently. - -BYTE-STABLE ON A NO-OP RERUN: written to a tmp path, compared, and moved only -if it differs (the pattern ``persist_exp.py`` and ``clean_exposure.py`` use). -An unconditional rewrite would move the output's mtime on every invocation. -Members are visited in sorted (exposure, member) order so the row order is a -function of the input set alone. - -PSFEX ONLY, DELIBERATELY. ``PSF_MODEL`` is ``psfex`` in every campaign the -workflow has run; ``MergeStarCatMCCD`` and ``MergeStarCatSetools`` exist beside -it and take the same constructor, so the hook is the one-line class choice in -``merge_class()`` below — an implementation, not a design, away. +actually packed, member by member, with sizes and the product each came from, so +this script never guesses at tar contents. """ import argparse -import filecmp -import io import json -import logging -import shutil import sys import tarfile -import tempfile from fnmatch import fnmatch from pathlib import Path -from shapepipe.modules.merge_starcat_package import merge_starcat +import numpy as np +from astropy.io import fits # Same directory; the rule invokes this file by path, so it is sys.path[0]. import build_index +import hdf5_reconcile import persist_exp -# The output name is not ours to choose: sp_validation hardcodes it -# (`star_cat_path = f"{data_dir}/full_starcat-0000000.fits"`), and -# MergeStarCatPSFEX writes exactly this basename into the output dir it is -# given. Kept here as the name this script promises to produce. -OUT_NAME = "full_starcat-0000000.fits" - # The members this merge consumes, named as the keep list names them and # resolved through the same catalogue persist_exp packs by — so the glob has one # definition and adding a product cannot leave the two disagreeing. They are # always there to find: persist_exp packs this product for every exposure # whatever `persist_exp:` says, and fails the pack rather than writing a # manifest without it. -MEMBER_PRODUCT = "psf_validation" +MEMBER_PRODUCT = persist_exp.ALWAYS MEMBER_PATTERN = persist_exp.resolve(MEMBER_PRODUCT) +# The group holding the per-exposure datasets. Unlike the galaxy side's +# `patches/`, this name is ours and says what it holds. +GROUP = "exposures" + +# The validation_psf table's HDU: what MergeStarCatPSFEX defaulted to and what +# psfex_interp writes — a SExtractor-style file, empty primary, header-carrying +# image extension, then the table. +HDU = 2 + +# The columns, in the order the FITS full_starcat carried them, which is the +# order every consumer has seen. The optional three are zero-filled per file. +COLUMNS = ("X", "Y", "RA", "DEC", + "HSM_G1_PSF", "HSM_G2_PSF", "HSM_T_PSF", + "HSM_G1_STAR", "HSM_G2_STAR", "HSM_T_STAR", + "HSM_FLAG_PSF", "HSM_FLAG_STAR") +OPTIONAL = ("MAG", "SNR", "ACCEPTED") +CCD_COLUMN = "CCD_NB" +ALL_COLUMNS = COLUMNS + OPTIONAL + (CCD_COLUMN,) -def merge_class(psf_model: str): - """The merge class for this PSF model — the one-line MCCD/setools hook. - Only psfex is exercised: it is what every campaign has run. MCCD reaches the - tars unchanged (it takes its CCD numbers from the data, and it now reports - by the entry's name like the others). SETOOLS would need one more thing — - it passes ``input_file_list[0][0]`` to file_io as a template path, which a - streamed entry is not — so wiring setools to this path is a change to that - class, not a change here. +def ccd_number(member_name: str) -> int: + """The CCD this member's rows belong to: ``validation_psf--.fits``. + + Always digits, which is why the column is an int; a member name that does + not carry one is a tar we do not understand, and saying so beats writing a + sentinel into the catalogue. + """ + ccd = member_name.rsplit(".", 1)[0].rsplit("-", 1)[-1] + if not ccd.isdigit(): + sys.exit(f"merge_star_cat: cannot read a CCD number out of member " + f"name {member_name!r}") + return int(ccd) + + +def is_member(entry: dict) -> bool: + """Is this manifest entry one of the members this merge reads? + + BY PRODUCT NAME, OR FAILING THAT BY FILE NAME. persist_exp records the + product every member came from and always packs psf_validation, so the name + is the answer for anything it writes today. The glob is the fallback, and it + earns its place twice over: a tar packed before the product field existed + has no label at all, and a keep list written as a raw glob + (`validation_psf-*.fits` rather than `psf_validation`) labels its members + with the glob. Neither should make the campaign's star catalogue silently + empty. """ - try: - return {"psfex": merge_starcat.MergeStarCatPSFEX, - "mccd": merge_starcat.MergeStarCatMCCD, - "setools": merge_starcat.MergeStarCatSetools}[psf_model] - except KeyError: - sys.exit(f"merge_star_cat: unknown psf_model {psf_model!r}") + return (entry.get("product") == MEMBER_PRODUCT + or fnmatch(entry["name"], MEMBER_PATTERN)) def manifests(products_dir: Path, tile_list: Path, index_db: Path) -> list: - """The campaign's exp_persist manifests that are on disk, in exposure order. + """``(exposure, manifest path)`` for the campaign's packed exposures. Not a glob over the products root: see the module docstring on why the set is the campaign's and not the filesystem's. @@ -144,73 +156,94 @@ def manifests(products_dir: Path, tile_list: Path, index_db: Path) -> list: path = (products_dir / "exp" / exp[:2] / exp / "manifests" / "exp_persist.json") if path.exists(): - out.append(path) + out.append((exp, path)) return out -def selection(manifest_paths: list, pattern: str) -> tuple: - """``[(tar path, [member names])]`` for the merge, and the empty exposures. +def tars(manifest_paths: list) -> tuple: + """``[(exposure, tar path)]`` for the merge, and the exposures with nothing. - Reads the manifests only. Every tar is checked for existence HERE, so a - products root missing a file fails before a single row is stacked rather - than an hour in. + Every tar is checked for existence HERE, so a products root missing a file + fails before a single row is read rather than an hour in. The tar is also + the unit's SOURCE for reconciling: its size and mtime are what a later + invocation compares against to decide whether this exposure changed. """ chosen, empty = [], [] - for man_path in manifest_paths: + for exp, man_path in manifest_paths: man = json.loads(man_path.read_text()) - wanted = sorted(f["name"] for f in man["files"] - if fnmatch(f["name"], pattern)) - if not wanted: - empty.append(man["unit"]) + if not any(is_member(f) for f in man["files"]): + empty.append(exp) continue tar_path = Path(man["tar"]) if not tar_path.exists(): sys.exit(f"merge_star_cat: {man_path} names a tar that is not " f"there: {tar_path}") - chosen.append((tar_path, wanted)) + chosen.append((exp, tar_path)) return chosen, empty -class TarMembers: - """The merge class's input list, materialised ONE TAR AT A TIME. - - ``MergeStarCatPSFEX`` wants something it can take the length of and iterate - once, handing it ``[fileobj, name]`` entries; it never indexes and never - rewinds. So it does not need a list, and a list is the one thing we cannot - afford: reading every member up front is the whole campaign in memory at - once — ~2 MB per exposure, so ~40 GB at DR6's ~20k exposures, against a - rule asking for 16 GB. Read lazily, peak memory is ONE member's bytes plus - the merge class's own accumulators, which are the real and unavoidable term. +def read_exposure(exp: str, tar_path: Path) -> np.ndarray: + """One exposure's every CCD, stacked, as a structured array. - ``__len__`` comes from the manifests, so the class can log the count before - a single tar is opened. - - IT IS ITERABLE MORE THAN ONCE, and must be: the merge makes two passes, one - for row counts from the headers and one to fill. Each ``__iter__`` opens the - archives afresh, so the second pass sees the same members in the same order. + TWO PASSES over the tar's members, and neither holds the exposure twice: + the first reads only each member's FITS HEADER — NAXIS2, the row count — + and the second allocates the columns once at their exact final length and + fills them slice by slice. Members are visited in sorted name order, so the + row order is a function of the tar's contents alone. """ - - def __init__(self, chosen): - self._chosen = chosen - - def __len__(self): - return sum(len(names) for _, names in self._chosen) - - def __iter__(self): - for tar_path, names in self._chosen: - with tarfile.open(tar_path) as tf: - for name in names: - member = tf.extractfile(name) - if member is None: - sys.exit(f"merge_star_cat: {tar_path} has no member " - f"{name}, which its manifest lists") - # The tar's own file object, not a BytesIO of the whole - # member: it is seekable (the archive is uncompressed by - # design) and astropy reads through it, so the merge's - # first pass costs a header rather than a member. The - # object is valid only until the next member is reached, - # which is exactly how the merge consumes it. - yield [member, name] + with tarfile.open(tar_path) as tf: + names = sorted(n for n in tf.getnames() + if Path(n).match(MEMBER_PATTERN)) + if not names: + sys.exit(f"merge_star_cat: {tar_path} holds no {MEMBER_PATTERN}") + + # --- pass 1: row counts and dtypes, from headers alone -------------- + counts, dtypes, opt_dtypes, n_total = [], None, {}, 0 + for name in names: + with fits.open(tf.extractfile(name), memmap=False, + ignore_missing_simple=True) as hdul: + hdu = hdul[HDU] + counts.append(hdu.header["NAXIS2"]) + # ColDefs.dtype describes the table without reading it. NOTE: + # it is the RAW storage dtype and ignores TSCAL/TZERO, so a + # scaled column would be allocated narrower than the values + # .data returns. Latent, not live: no validation_psf column is + # scaled. Read the dtype off .data if one ever is. + cols = hdu.columns.dtype + if dtypes is None: + dtypes = cols + for col in OPTIONAL: + if col not in opt_dtypes and col in (cols.names or ()): + opt_dtypes[col] = cols[col] + n_total += counts[-1] + + fields = [(c, dtypes[c]) for c in COLUMNS] + # A column no file of this exposure carries still gets a column, + # zero-filled, in the dtype the positional column X uses. + fields += [(c, opt_dtypes.get(c, dtypes["X"])) for c in OPTIONAL] + fields += [(CCD_COLUMN, np.int32)] + data = np.empty(n_total, dtype=np.dtype(fields)) + + # --- pass 2: fill --------------------------------------------------- + at = 0 + for name, n_rows in zip(names, counts): + with fits.open(tf.extractfile(name), memmap=False, + ignore_missing_simple=True) as hdul: + rows = hdul[HDU].data + have = set(rows.dtype.names or ()) + sl = slice(at, at + n_rows) + for col in COLUMNS: + data[col][sl] = rows[col] + for col in OPTIONAL: + # THIS file's schema, not the exposure's. + data[col][sl] = rows[col] if col in have else 0 + data[CCD_COLUMN][sl] = ccd_number(name) + at += n_rows + + if at != n_total: + raise ValueError(f"merge_star_cat: {tar_path}: pass 1 counted " + f"{n_total} rows, pass 2 filled {at}") + return data def main() -> None: @@ -222,59 +255,38 @@ def main() -> None: help="the campaign's tile list (config tile_list)") p.add_argument("--index-db", required=True, type=Path, help="the campaign's run index (config outputs.index_db)") - p.add_argument("--output", required=True, type=Path, - help=f"the merged catalogue; its basename is {OUT_NAME}") - p.add_argument("--psf-model", default="psfex") - p.add_argument("--pattern", default=MEMBER_PATTERN, - help="tar-member glob to merge; default %(default)s") + p.add_argument("--output", required=True, type=Path) + p.add_argument("--campaign", required=True, + help="named in the log; the group name is fixed") args = p.parse_args() - if args.output.name != OUT_NAME: - # The merge class writes OUT_NAME into a directory it is handed; a - # differently-named declared output would silently never be produced. - sys.exit(f"merge_star_cat: --output must be named {OUT_NAME} " - f"(got {args.output.name})") - - log = logging.getLogger("merge_star_cat") - logging.basicConfig(format="[merge_star_cat] %(message)s", - level=logging.INFO, stream=sys.stdout) - manifest_paths = manifests(args.products_dir, args.tile_list, args.index_db) - chosen, empty = selection(manifest_paths, args.pattern) - file_list = TarMembers(chosen) - if not len(file_list): + chosen, empty = tars(manifest_paths) + if not chosen: # Not a no-op: an empty star catalogue would pass every downstream # existence check and produce meaningless rho statistics. - sys.exit(f"merge_star_cat: no member matched {args.pattern!r} in any " - f"of {len(manifest_paths)} exp_persist manifest(s) for this " + sys.exit(f"merge_star_cat: no {MEMBER_PRODUCT} member in any of " + f"{len(manifest_paths)} exp_persist manifest(s) for this " f"campaign. persist_exp packs {MEMBER_PRODUCT} for every " f"exposure, so this means the manifests are not what we think " f"they are.") if empty: - log.info(f"{len(empty)} exposure(s) persisted no {args.pattern}: " - f"{', '.join(sorted(empty)[:5])}" - f"{' ...' if len(empty) > 5 else ''}") - - # tmp-then-cmp-then-mv. The merge class chooses its own basename inside the - # directory it is given, so the tmp is a DIRECTORY, not a file path, and it - # never outlives this process — an orphan on /project is an inode nothing - # revisits. + print(f"[merge_star_cat] {len(empty)} exposure(s) persisted no " + f"{MEMBER_PRODUCT}: {', '.join(sorted(empty)[:5])}" + f"{' ...' if len(empty) > 5 else ''}") + args.output.parent.mkdir(parents=True, exist_ok=True) - tmp_dir = Path(tempfile.mkdtemp(dir=args.output.parent, - prefix=".star_cat_merge.")) - try: - merge_class(args.psf_model)(file_list, str(tmp_dir), log).process() - tmp = tmp_dir / OUT_NAME - if not tmp.exists(): - sys.exit(f"merge_star_cat: the merge wrote no {OUT_NAME}") - if args.output.exists() and filecmp.cmp(tmp, args.output, shallow=False): - log.info(f"unchanged: {args.output}") - else: - tmp.replace(args.output) # atomic: same filesystem - log.info(f"{len(file_list)} catalogue(s) from {len(chosen)} " - f"exposure(s) -> {args.output}") - finally: - shutil.rmtree(tmp_dir, ignore_errors=True) + digest = hdf5_reconcile.schema_digest(ALL_COLUMNS) + todo = hdf5_reconcile.plan(args.output, GROUP, chosen, digest) + if todo.empty(): + print(f"[merge_star_cat] unchanged: {args.output} " + f"({len(chosen)} exposure(s))") + return + hdf5_reconcile.apply(args.output, GROUP, todo, chosen, read_exposure, + digest, "n_exposures") + print(f"[merge_star_cat] {todo.describe()} -> {args.output} " + f"({len(chosen)} exposure(s), {len(ALL_COLUMNS)} column(s), " + f"campaign {args.campaign})") if __name__ == "__main__": From 4ed9fb0d8ccb0f5912513519678e3458691ea354 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 20:27:06 -0400 Subject: [PATCH 19/20] fix(orchestration): nine findings from the third review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit path_hash KILLED EVERY INVOCATION over a file one rule needs. It runs at module level, so a snapshot without scripts/python/ raised FileNotFoundError during the parse — taking `sp --unlock`, `sp report` and every dry run with it, and taking them with a bare traceback rather than the diagnosis merge_final_cat.py already carries for exactly this case, which the job could never reach. The hash degrades to a sentinel and warns once; the parse survives, the rule still exists, and the job prints the message written for it. Verified both halves with the file moved aside. THE STAR MERGE'S OPTIONAL COLUMNS HAD NO CANONICAL DTYPE. MAG/SNR/ACCEPTED took the dtype of whichever file carried them, falling back to X's float32 when none did — so an exposure whose files are all pix2wcs-converted got a float32 ACCEPTED while its neighbours got int32, and datasets under exposures/ differed in dtype. np.concatenate refuses that, and no digest can repair it because nothing about the schema CHANGED. The three are pinned (int32, float32, float32) and cast. Verified on two exposures, one carrying them and one not: identical dtypes, concatenate works. A MISLABELED MANIFEST ABORTED THE CAMPAIGN. is_member() accepts a member by product label OR by file name — right for "did this exposure keep the product" — but read_exposure() selects by name alone, so an entry labelled psf_validation whose name did not match put the exposure in the merge and then killed the whole job when the tar held nothing selectable. Membership is now the name on both sides, with one line saying an entry was labelled and skipped. RENAMING `campaign:` WOULD HAVE HALF-UPDATED THE FILE. The tile hdf5 carries the campaign in its GROUP, so a rename pointed the rule at a new group inside the same file: a second group beside the first, the first frozen and stale, and n_tiles describing one of them. One file is one campaign — apply refuses and names what is already there. APPEND IS CHEAP IN READS, NOT IN WRITES, and the docstrings said otherwise. The existing file is copied so the result can be moved into place atomically: one pass over it and, briefly, twice its size on disk. Corrected, and apply now refuses when the filesystem cannot hold it rather than filling /project and leaving a truncated tmp beside a catalogue people trust. A CORRUPT TAR RAISED A RAW ReadError, on both sides. persist_exp now refuses to write a new tar and says the old one is untouched and may hold products nothing else has; merge_star_cat names the tar and says not to delete it. THE 16-COLUMN SCHEMA IS DEFINED TWICE and nothing held the two together. MergeStarCatPSFEX writes the flat FITS table the module runner emits; merge_star_cat.py writes the hdf5. Separate implementations are right — only one of them reads tars, keeps native dtypes and reconciles — but a column added to one writer would simply be missing from the other's product, found by whoever next computed rho statistics from the wrong one. tests/unit/test_star_cat_columns.py asserts the names and their order agree; verified passing, and verified failing when one list is changed. Also noted where it lives: adding a retention product re-packs the tar and moves its mtime, so the star merge refreshes those exposures although their validation members are byte-for-byte unchanged — seconds per exposure against per-member bookkeeping on every exposure, which is not a trade worth making. Stale docs updated to the hdf5 product: the Snakefile's merges header, the README's star_cat_merge paragraph and scripts list (the hdf5 paragraph written last round never landed — its edit script aborted before writing), and config.yaml's psf_validation block. The README now says there are two writers and names the test that keeps their schema together. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- tests/unit/test_star_cat_columns.py | 95 +++++++++++++++++++++++++++++ workflow/README.md | 27 +++++--- workflow/Snakefile | 25 ++++++-- workflow/config.yaml | 2 +- workflow/scripts/hdf5_reconcile.py | 63 ++++++++++++++++++- workflow/scripts/merge_star_cat.py | 60 +++++++++++++----- workflow/scripts/persist_exp.py | 11 +++- 7 files changed, 253 insertions(+), 30 deletions(-) create mode 100644 tests/unit/test_star_cat_columns.py diff --git a/tests/unit/test_star_cat_columns.py b/tests/unit/test_star_cat_columns.py new file mode 100644 index 000000000..20975815d --- /dev/null +++ b/tests/unit/test_star_cat_columns.py @@ -0,0 +1,95 @@ +"""The star catalogue's 16 columns are defined twice, and must not drift. + +Two writers emit a full_starcat, for two consumers that have to agree about it: + + * ``MergeStarCatPSFEX`` (``src/shapepipe/modules/merge_starcat_package``), + which the ``merge_starcat`` MODULE RUNNER calls, writing the flat FITS table + sp_validation opens today; + * ``workflow/scripts/merge_star_cat.py``, the Snakemake workflow's + ``star_cat_merge`` rule, writing the per-exposure hdf5 that replaces it + (CosmoStat/sp_validation#340 moves the readers). + +They were one definition until the workflow stopped calling the module class: +the rule reads validation_psf members out of the per-exposure tars, keeps their +native dtypes and reconciles its output, none of which the class does or should +do. Two implementations is the right answer for the behaviour; two COLUMN LISTS +is not, and nothing else would notice them diverging — a column added to one +writer would simply be absent from the other's product, discovered by whoever +next tried to compute rho statistics from the wrong one. + +Hence this module, which asserts the one thing they must share. It does NOT +assert the dtypes: the whole point of the hdf5 writer is that they differ (the +FITS one widens every float to 1D). Only the names, and their order. + +Deliberately import-light on the workflow side: merge_star_cat.py pulls in h5py +and astropy, which the class does too, so a container-free run is not on offer +here and is not worth contorting for. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" +SCRIPT = SCRIPTS / "merge_star_cat.py" + + +def _load_workflow_merge(): + """Import the rule's script by path — ``workflow/scripts`` is not a package. + + Its own imports (build_index, hdf5_reconcile, persist_exp) are siblings it + reaches through ``sys.path[0]``, which is how the rule invokes it, so the + directory goes on the path here too. + """ + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location("_merge_star_cat", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +@pytest.fixture(scope="module") +def writers(): + """The two column lists, each in the order its writer emits them.""" + h5py = pytest.importorskip("h5py") # noqa: F841 - workflow dep + pytest.importorskip("astropy") + workflow = _load_workflow_merge() + from shapepipe.modules.merge_starcat_package.merge_starcat import ( + MergeStarCatPSFEX, + ) + # The class carries (output name, source column) pairs plus its optional + # set and appends CCD_NB last; the script carries output names throughout. + module_columns = ( + tuple(out for out, _ in MergeStarCatPSFEX._COLUMNS) + + tuple(out for out, _ in MergeStarCatPSFEX._OPTIONAL) + + ("CCD_NB",) + ) + return module_columns, tuple(workflow.ALL_COLUMNS) + + +def test_column_names_and_order_agree(writers): + """Same names, same order — the schema both products promise.""" + module_columns, workflow_columns = writers + assert workflow_columns == module_columns + + +def test_sixteen_columns(writers): + """The count is itself the documented contract (README, config.yaml).""" + module_columns, workflow_columns = writers + assert len(module_columns) == 16 + assert len(workflow_columns) == 16 + + +def test_ccd_nb_is_last(writers): + """CCD_NB is appended per input file rather than read from one, in both.""" + module_columns, workflow_columns = writers + assert module_columns[-1] == "CCD_NB" + assert workflow_columns[-1] == "CCD_NB" diff --git a/workflow/README.md b/workflow/README.md index 40371f2b4..c00d66c18 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -167,7 +167,8 @@ workflow/ run_report.py standalone report (NOT a DAG node; run_report hooks call it) container.py image layers + the resolution order behind `sp container` (stdlib-only) persist_exp.py ONE exposure's keepable PSF products -> one tar on products_dir (the exp_persist rule) - merge_star_cat.py ALL exposures' validation_psf, read out of the tars -> full_starcat (the star_cat_merge rule) + hdf5_reconcile.py bring an hdf5 catalogue into agreement with a campaign (shared by both merges) + merge_star_cat.py ALL exposures' validation_psf, out of the tars -> full_starcat_.hdf5 merge_final_cat.py ALL tiles' final_cat -> final_cat_.hdf5 (the final_cat_merge rule) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going @@ -297,13 +298,23 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee Everything above is per unit; the two products downstream analysis actually opens are per *campaign*, and until these rules existed each was a manual pass after the run. - `star_cat_merge` stacks every exposure's every CCD's `psf_validation` into one - `/full_starcat-0000000.fits` — the rho/tau statistics input, at - the path sp_validation hardcodes. It reads the members straight out of the - per-exposure tars (`tarfile`; unpacking ~800k files to merge them would defeat - the tar's whole purpose) and stacks them with `MergeStarCatPSFEX`, the same - class the old `merge_starcat_runner` called, so the column list has exactly - one definition. It exists whenever the campaign has a persisted exposure. + `star_cat_merge` collects every exposure's every CCD's `psf_validation` into + `/full_starcat_.hdf5`, one dataset per exposure at + `exposures/` — the rho/tau statistics input. It reads the members + straight out of the per-exposure tars (`tarfile`; unpacking ~800k files to + merge them would defeat the tar's whole purpose), keeps their native dtypes, + and stores `CCD_NB` as an int. sp_validation still opens the old flat FITS + name, `full_starcat-0000000.fits`; its readers move to this file under + [sp_validation#340](https://github.com/CosmoStat/sp_validation/issues/340), + the same migration that retires the `patches/` key on the tile side. The rule + exists whenever the campaign has a persisted exposure. + **Two writers, one schema.** The module runner still emits the flat FITS + table through `MergeStarCatPSFEX`, and this rule emits the hdf5; they are + separate implementations on purpose, because only one of them reads tars, + keeps native dtypes and reconciles. Their 16 COLUMN NAMES must not drift + apart, and nothing else would notice if they did — a column added to one + writer would just be missing from the other's product. `tests/unit/` + `test_star_cat_columns.py` is what holds them together. `final_cat_merge` collects every ready tile's `final_cat-.fits` into `/final_cat_.hdf5`: one dataset per tile under a group named for the campaign, the `final_cat.param` columns, an `n_tiles` attribute. diff --git a/workflow/Snakefile b/workflow/Snakefile index ea4129f25..4317e871f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -353,8 +353,25 @@ def path_hash(path): workflow/scripts/ nor a declared input. Without them in the hash, this PR's own edits to both would have left every finished campaign's hdf5 untouched and nothing would have said so. + + A MISSING FILE IS NOT A PARSE ERROR. This runs at module level, so raising + here kills EVERY invocation of the workflow — `sp --unlock`, `sp report`, + a dry run — over a file only one rule needs. Worse, it kills them with a + bare FileNotFoundError, which is exactly the diagnosis merge_final_cat.py + already carries and would print if the job were allowed to reach it. So the + hash degrades to a sentinel and says so once; the parse survives, the rule + still exists, and the job fails with the message written for it. """ - return hashlib.md5(Path(path).read_bytes()).hexdigest()[:12] + path = Path(path) + try: + return hashlib.md5(path.read_bytes()).hexdigest()[:12] + except OSError: + if workflow.is_main_process: + logger.warning( + f"missing: {path} — it is part of a rule's rerun trigger, so " + f"that rule cannot tell whether it is out of date. The job " + f"that needs the file will say so when it runs.") + return "missing" SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") @@ -596,9 +613,9 @@ def exp_store_reclaimed(exp): # --- the campaign-level merges --------------------------------------------- # Two rules, one job each per campaign, both writing to the persistent root, and # both the LAST link of a chain whose per-unit half the workflow already had: -# the exposure side ends in one `full_starcat-0000000.fits` (every CCD's PSF -# validation catalogue, stacked — the rho/tau statistics input) and the tile side -# in one `final_cat_.hdf5` (every tile's final catalogue — the shear +# the exposure side ends in one `full_starcat_.hdf5` (every CCD's PSF +# validation catalogue — the rho/tau statistics input) and the tile side in one +# `final_cat_.hdf5` (every tile's final catalogue — the shear # catalogue sp_validation reads). Until they existed the workflow's product set # was two files short of what the old `combine_runs.bash` + `create_final_cat.py` # chain delivered, and every campaign ended with a manual merge. diff --git a/workflow/config.yaml b/workflow/config.yaml index 99ae1b560..38979b2d0 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -87,7 +87,7 @@ outputs: # # WHAT IS ALWAYS KEPT, AND IS NOT A CHOICE HERE: psf_validation, the psfex_interp # validation catalogue, one per CCD. `star_cat_merge` stacks every one of them -# into the campaign's /full_starcat-0000000.fits, so they are that +# into /full_starcat_.hdf5, so they are that # catalogue's PROVENANCE — a merged star catalogue with no per-exposure inputs # beside it cannot be audited, re-cut, or recomputed after a purge — and they are # what keeps APPENDING TILES CHEAP, since a tile added next month brings diff --git a/workflow/scripts/hdf5_reconcile.py b/workflow/scripts/hdf5_reconcile.py index 0635b5976..f2e3ff325 100644 --- a/workflow/scripts/hdf5_reconcile.py +++ b/workflow/scripts/hdf5_reconcile.py @@ -23,7 +23,12 @@ file's root. * a dataset that agrees with its source and its schema is left alone, unread. -An append therefore reads exactly the appended units. +An append therefore READS exactly the appended units. It still WRITES the whole +file: the existing one is copied so the result can be moved into place +atomically, which costs one pass over it and, briefly, twice its size on disk. +That is the cheap half by orders of magnitude — copying a 1 GB hdf5 against +re-reading 800 GB of catalogues — but it is not free, and `apply` refuses rather +than filling the filesystem when the free space is not there. WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET. The file's CONTENT is: the same units with the same sources give the same datasets, the same columns and the @@ -41,6 +46,7 @@ import hashlib import shutil +import sys from pathlib import Path import h5py @@ -106,6 +112,56 @@ def plan(output: Path, group_path: str, units: list, digest: str) -> Plan: return Plan(add, refresh, sorted(present - want)) +# Twice the file, plus a tenth of it again: the copy and the original coexist, +# and hdf5 is not a format to run to the last byte of a filesystem on. +FREE_SPACE_MARGIN = 2.1 + + +def check_free_space(output: Path) -> None: + """Refuse to start a rewrite the filesystem cannot hold. + + A merge that fills /project does not just fail: it fails everything else + writing there at the same time, and it can leave a truncated tmp beside a + catalogue people trust. Cheaper to say so first. + """ + if not output.exists(): + return + size = output.stat().st_size + free = shutil.disk_usage(output.parent).free + if free < size * FREE_SPACE_MARGIN: + sys.exit( + f"hdf5_reconcile: {output.parent} has {free / 1e9:.1f} GB free and " + f"this merge needs about {size * FREE_SPACE_MARGIN / 1e9:.1f} GB — " + f"it rewrites {output.name} ({size / 1e9:.1f} GB) through a tmp " + f"copy beside it. Free space or move products_dir; the existing " + f"catalogue is untouched.") + + +def check_sole_group(output: Path, group_path: str) -> None: + """One file, one campaign — refuse to half-update a file holding two. + + Renaming `campaign:` mid-flight points the rule at a NEW group inside the + SAME file (the path carries the campaign only on the tile side, where the + group does). Reconciling would then add a second group beside the first, + leave the first frozen and stale, and set a count attribute describing only + one of them. Nothing downstream reads such a file correctly, and no rule + here means to produce one. Say what is there and stop. + """ + if not output.exists() or "/" not in group_path: + return + parent, leaf = group_path.rsplit("/", 1) + with h5py.File(output, "r") as f: + if parent not in f: + return + others = sorted(k for k in f[parent] if k != leaf) + if others: + sys.exit( + f"hdf5_reconcile: {output} already holds {parent}/" + f"{', '.join(others)} beside {group_path}. One file is one " + f"campaign: reconciling would freeze the other group and count " + f"only this one. Point `campaign:` back, or write to a new path.") + + def apply(output: Path, group_path: str, todo: Plan, units: list, read, digest: str, count_attr: str) -> None: """Carry the plan out on a tmp file, then move it into place. @@ -120,7 +176,8 @@ def apply(output: Path, group_path: str, todo: Plan, units: list, read, refresh of a unit leaks that unit. So: * a plan that only ADDS copies the existing file and appends to it. There - is nothing to reclaim, and copying beats rewriting. + is nothing to reclaim, and copying beats rewriting. It is still a pass + over the whole file — an append is cheap in READS, not in writes. * a plan that removes or refreshes anything builds the tmp FRESH, moving the datasets it keeps across with h5py's own group copy — a dataset-level copy inside the library that never reads a row into numpy @@ -135,6 +192,8 @@ def apply(output: Path, group_path: str, todo: Plan, units: list, read, """ sources = dict(units) rewrite = bool(todo.remove or todo.refresh) + check_free_space(output) + check_sole_group(output, group_path) written = set(todo.add) | set(todo.refresh) keep = [u for u, _ in units if u not in written] tmp = output.with_name(output.name + ".tmp") diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py index 25c9db0e1..84ba633d1 100644 --- a/workflow/scripts/merge_star_cat.py +++ b/workflow/scripts/merge_star_cat.py @@ -110,9 +110,16 @@ "HSM_G1_PSF", "HSM_G2_PSF", "HSM_T_PSF", "HSM_G1_STAR", "HSM_G2_STAR", "HSM_T_STAR", "HSM_FLAG_PSF", "HSM_FLAG_STAR") -OPTIONAL = ("MAG", "SNR", "ACCEPTED") +# CANONICAL DTYPES, not whatever the first file that carries the column happens +# to use. These three are absent from pix2wcs-converted catalogues, so an +# exposure whose files all lack them would otherwise be allocated a fallback +# dtype while its neighbours got the real one — and datasets under exposures/* +# would then differ in dtype, which np.concatenate refuses and no digest can +# repair, since nothing about the schema CHANGED. Pinning the dtype here is what +# makes every exposure's dataset the same shape whatever its files carry. +OPTIONAL = {"MAG": np.float32, "SNR": np.float32, "ACCEPTED": np.int32} CCD_COLUMN = "CCD_NB" -ALL_COLUMNS = COLUMNS + OPTIONAL + (CCD_COLUMN,) +ALL_COLUMNS = COLUMNS + tuple(OPTIONAL) + (CCD_COLUMN,) def ccd_number(member_name: str) -> int: @@ -171,7 +178,21 @@ def tars(manifest_paths: list) -> tuple: chosen, empty = [], [] for exp, man_path in manifest_paths: man = json.loads(man_path.read_text()) - if not any(is_member(f) for f in man["files"]): + # MEMBERSHIP IS THE MEMBER NAME, and only the member name. is_member() + # will also accept a manifest's own product LABEL, which is the right + # test for "did this exposure keep the product" — but a label is not + # what read_exposure() selects on, and a mislabeled entry whose name + # does not match would put this exposure in the merge and then abort + # the whole campaign when the tar turned out to hold nothing selectable. + # So the two agree by construction: both ask the name. + if not any(fnmatch(f["name"], MEMBER_PATTERN) for f in man["files"]): + if any(is_member(f) for f in man["files"]): + # Labelled as the product, named as something else. Worth one + # line — it means a manifest we did not write, or a keep list + # whose glob does not match the member it matched. + print(f"[merge_star_cat] {exp}: manifest labels a " + f"{MEMBER_PRODUCT} member whose name does not match " + f"{MEMBER_PATTERN}; not merging it") empty.append(exp) continue tar_path = Path(man["tar"]) @@ -190,15 +211,29 @@ def read_exposure(exp: str, tar_path: Path) -> np.ndarray: and the second allocates the columns once at their exact final length and fills them slice by slice. Members are visited in sorted name order, so the row order is a function of the tar's contents alone. + + NOTE ON WHEN THIS IS CALLED AGAIN. The unit's source is the TAR, so adding a + retention product re-packs it, moves its mtime, and refreshes this exposure + even though its validation members are byte-for-byte what they were. Reading + one exposure is seconds and the alternative — stamping the members rather + than the archive — buys a rarely-taken shortcut for a per-member bookkeeping + cost on every exposure. Not worth it. """ - with tarfile.open(tar_path) as tf: + try: + tf = tarfile.open(tar_path) + except tarfile.TarError as exc: + sys.exit(f"merge_star_cat: cannot read {tar_path}: {exc}. That tar is " + f"this exposure's only copy of its PSF products — do not " + f"delete it; re-pack the exposure if its scratch store is " + f"still there, and treat the exposure as lost if it is not.") + with tf: names = sorted(n for n in tf.getnames() - if Path(n).match(MEMBER_PATTERN)) + if fnmatch(n, MEMBER_PATTERN)) if not names: sys.exit(f"merge_star_cat: {tar_path} holds no {MEMBER_PATTERN}") # --- pass 1: row counts and dtypes, from headers alone -------------- - counts, dtypes, opt_dtypes, n_total = [], None, {}, 0 + counts, dtypes, n_total = [], None, 0 for name in names: with fits.open(tf.extractfile(name), memmap=False, ignore_missing_simple=True) as hdul: @@ -209,18 +244,15 @@ def read_exposure(exp: str, tar_path: Path) -> np.ndarray: # scaled column would be allocated narrower than the values # .data returns. Latent, not live: no validation_psf column is # scaled. Read the dtype off .data if one ever is. - cols = hdu.columns.dtype if dtypes is None: - dtypes = cols - for col in OPTIONAL: - if col not in opt_dtypes and col in (cols.names or ()): - opt_dtypes[col] = cols[col] + dtypes = hdu.columns.dtype n_total += counts[-1] fields = [(c, dtypes[c]) for c in COLUMNS] - # A column no file of this exposure carries still gets a column, - # zero-filled, in the dtype the positional column X uses. - fields += [(c, opt_dtypes.get(c, dtypes["X"])) for c in OPTIONAL] + # The optional three take their CANONICAL dtype, not one file's (see + # OPTIONAL): every exposure's dataset must have the same dtype whether + # or not its files carry the column. + fields += list(OPTIONAL.items()) fields += [(CCD_COLUMN, np.int32)] data = np.empty(n_total, dtype=np.dtype(fields)) diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py index 24e0c1d66..bab1a5f0a 100644 --- a/workflow/scripts/persist_exp.py +++ b/workflow/scripts/persist_exp.py @@ -348,7 +348,15 @@ def main() -> None: for f in json.loads(prior.read_text())["files"]} except (OSError, ValueError, KeyError): pass # a damaged manifest loses only labels - with tarfile.open(tar_path) as tf: + try: + old_read = tarfile.open(tar_path) + except tarfile.TarError as exc: + sys.exit(f"persist_exp: {args.exp}: cannot read the existing " + f"{tar_path}: {exc}. Refusing to write a new one — the " + f"old tar is left exactly as it is, and it may still hold " + f"products nothing else has. Move it aside deliberately " + f"if you have decided it is lost.") + with old_read as tf: for ti in tf.getmembers(): if ti.name in seen or not ti.isfile(): continue # a live source supersedes it @@ -378,6 +386,7 @@ def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: # byte-for-byte what it was and a rerun that changes nothing still # produces an identical archive. with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: + # Already proven readable above, where the members were listed. old_tar = (tarfile.open(tar_path) if carried else None) try: for f in files: From e86d8c8d1149abab965d26976be284e63dae9044 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 20:41:35 -0400 Subject: [PATCH 20/20] test(unit): property-based state machines for reconcile and persist_exp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two rules carry state across invocations and are correct only over SEQUENCES: hdf5_reconcile brings a catalogue into agreement with a campaign that changes under it, and persist_exp packs a tar whose existing members are a floor. Neither is a claim the example tests can finish making, so each gets a hypothesis state machine that walks a random sequence of campaign edits and asserts the model after every one. hdf5_reconcile: units added, refreshed and removed, and the column set flipped, against a real file. After every step the datasets are the campaign's units with their sources' content, the count and digest attributes agree, every dataset shares one dtype, and a no-op leaves the mtime alone. Source mtimes are set explicitly, so a same-size rewrite inside one filesystem tick cannot masquerade as a refresh. Compaction is asserted where the module actually claims it — the rebuild path — and stated as "does not grow with history": a rebuild costs ~1.4 kB more than a from-scratch build (h5py's group copy writes more metadata than create_dataset does) and that overhead is constant, which test_repeated_refresh_does_not_grow_the_file pins directly. Plus a crash injected at the rename, which must leave the previous file byte-identical. persist_exp: random keep lists of product names, raw globs and overlapping mixtures over a store that gains and loses products. Members are additive across packs, never duplicated, and the manifest agrees with the tar down to the product labels; a missing psf_validation fails without writing a manifest, an unknown product name is refused before any work, a corrupt tar is left exactly as it is, and two different sources with one member name are still fatal. Both files were checked against five mutants (never rebuild, never refresh, non-additive retention, optional psf_validation, tolerated collision); each is caught. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QbnPCyzuDNTgkg715pHhar --- tests/unit/test_hdf5_reconcile_props.py | 336 ++++++++++++++++++++++ tests/unit/test_persist_exp_props.py | 360 ++++++++++++++++++++++++ 2 files changed, 696 insertions(+) create mode 100644 tests/unit/test_hdf5_reconcile_props.py create mode 100644 tests/unit/test_persist_exp_props.py diff --git a/tests/unit/test_hdf5_reconcile_props.py b/tests/unit/test_hdf5_reconcile_props.py new file mode 100644 index 000000000..a990c3f89 --- /dev/null +++ b/tests/unit/test_hdf5_reconcile_props.py @@ -0,0 +1,336 @@ +"""Property-based state machine over ``workflow/scripts/hdf5_reconcile.py``. + +The module's contract is that an hdf5 catalogue reconciled against a campaign +is a FUNCTION OF ITS INPUT SET — the same units with the same sources give the +same datasets, the same dtypes and the same count attribute, however they got +there. That is a claim about every reachable sequence of appends, refreshes and +removals, not about the three the unit tests happen to walk, so it is tested +here against a model: a random sequence of campaign edits, each followed by a +real plan/apply against a real file on disk, with the model asserted after +every step. + +The operations are the four things a campaign can do between invocations — +add a unit, change a unit's source, drop a unit, change the column set — plus +a no-op, which is the one that must leave the file's mtime alone. + +Source mtimes are set EXPLICITLY with ``os.utime`` rather than left to the +clock. ``stamp()`` is (size, mtime_ns), so a test that rewrote a file with the +same length inside one filesystem tick would silently exercise "nothing +changed" while believing it exercised a refresh. +""" + +import importlib.util +import os +import sys +from pathlib import Path + +import numpy as np +import pytest +from hypothesis import HealthCheck, settings +from hypothesis import strategies as st +from hypothesis.stateful import ( + RuleBasedStateMachine, + initialize, + invariant, + precondition, + rule, +) + +h5py = pytest.importorskip("h5py") + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" + + +def _load(name): + path = SCRIPTS / f"{name}.py" + assert path.exists(), f"{path} not found; the rules call it by path" + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(f"_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +reconcile = _load("hdf5_reconcile") + +GROUP = "cat/campaign_a" +COUNT_ATTR = "n_units" +UNITS = ["u0", "u1", "u2", "u3"] +# Two column sets, so a schema change is a real change of dtype and width. +COLUMN_SETS = [("RA", "DEC", "E1"), ("RA", "DEC", "E1", "FWHM")] +# What a rebuild may cost over a from-scratch build of the same campaign: the +# metadata h5py's group copy writes for a moved dataset. Measured at ~1.4 kB +# and constant in the number of rebuilds; the allowance is generous because the +# property being defended is "does not grow with history", not an exact size. +COPY_SLACK = 8192 + + +def _array(columns, rows, seed): + rng = np.random.default_rng(seed) + dtype = [(c, " None: + np.save(path, array, allow_pickle=False) + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def _read(unit, source): + return np.load(source, allow_pickle=False) + + +def _build(output: Path, sources: dict, columns): + """Plan and apply once, exactly as the two merge rules do; return the plan.""" + units = sorted(sources.items()) + digest = reconcile.schema_digest(columns) + todo = reconcile.plan(output, GROUP, units, digest) + if todo.empty(): + return todo + reconcile.apply(output, GROUP, todo, units, _read, digest, COUNT_ATTR) + return todo + + +class ReconcileMachine(RuleBasedStateMachine): + """A campaign that changes under a catalogue that must keep up with it.""" + + @initialize() + def setup(self): + self.dir = Path( + __import__("tempfile").mkdtemp(prefix="reconcile-props-") + ) + self.output = self.dir / "cat.h5" + self.columns = COLUMN_SETS[0] + self.sources = {} # unit -> source path + self.expected = {} # unit -> array as last written + self.clock = 1_000_000_000_000_000_000 + # Compaction is only claimed of the rebuild path (a plan that removes + # or refreshes). An add-only plan copies the file and appends, so its + # layout carries whatever the previous writes left behind. + self.rebuilt = False + + def teardown(self): + __import__("shutil").rmtree(self.dir, ignore_errors=True) + + # --- the campaign's moves ------------------------------------------- + def _tick(self): + self.clock += 1_000_000_000 + return self.clock + + def _step(self, changed): + before = (self.output.stat().st_mtime_ns + if self.output.exists() else None) + todo = _build(self.output, self.sources, self.columns) + self.rebuilt = bool(todo.remove or todo.refresh) + if not changed and before is not None: + assert self.output.stat().st_mtime_ns == before, ( + "a no-op reconcile rewrote the file; mtime is a rerun trigger" + ) + + @rule(pick=st.integers(0, 2**16), rows=st.integers(1, 5), + seed=st.integers(0, 2**16)) + @precondition(lambda self: len(self.sources) < len(UNITS)) + def add_unit(self, pick, rows, seed): + free = sorted(set(UNITS) - set(self.sources)) + unit = free[pick % len(free)] + path = self.dir / f"{unit}.npy" + array = _array(self.columns, rows, seed) + _write_source(path, array, self._tick()) + self.sources[unit] = path + self.expected[unit] = array + self._step(changed=True) + + @rule(pick=st.integers(0, 2**16), rows=st.integers(1, 5), + seed=st.integers(0, 2**16), resize=st.booleans()) + @precondition(lambda self: bool(self.sources)) + def modify_source(self, pick, rows, seed, resize): + unit = sorted(self.sources)[pick % len(self.sources)] + old = self.expected[unit] + rows = rows if resize else len(old) + array = _array(self.columns, rows, seed) + _write_source(self.sources[unit], array, self._tick()) + self.expected[unit] = array + self._step(changed=True) + + @rule(pick=st.integers(0, 2**16)) + @precondition(lambda self: bool(self.sources)) + def remove_unit(self, pick): + unit = sorted(self.sources)[pick % len(self.sources)] + self.sources.pop(unit).unlink() + self.expected.pop(unit) + self._step(changed=True) + + @rule() + def change_columns(self): + """Flip to the other column set — a digest change, so every unit + refreshes.""" + columns = next(c for c in COLUMN_SETS if c != self.columns) + self.columns = columns + # A schema change is a change to how the SOURCES are read, so the + # sources are rewritten under the new column set as the campaign would. + for i, (unit, path) in enumerate(sorted(self.sources.items())): + array = _array(columns, len(self.expected[unit]), 4242 + i) + _write_source(path, array, self._tick()) + self.expected[unit] = array + self._step(changed=True) + + @rule() + def no_op(self): + self._step(changed=False) + + # --- what must be true after every step ------------------------------ + @invariant() + def file_matches_campaign(self): + if not self.expected: + return + assert self.output.exists() + with h5py.File(self.output, "r") as f: + assert set(f[GROUP]) == set(self.expected), ( + "datasets and campaign units disagree") + assert f.attrs[COUNT_ATTR] == len(self.expected) + assert (f.attrs["param_digest"] + == reconcile.schema_digest(self.columns)) + dtypes = set() + for unit, want in self.expected.items(): + got = f[GROUP][unit][...] + assert got.dtype.names == want.dtype.names + np.testing.assert_array_equal(got, want) + dtypes.add(got.dtype) + stamp = reconcile.stamp(self.sources[unit]) + assert (int(f[GROUP][unit].attrs["src_bytes"]), + int(f[GROUP][unit].attrs["src_mtime_ns"])) == stamp + assert len(dtypes) == 1, ( + "sources share a column list; datasets must share a dtype") + + @invariant() + def compact(self): + """A rebuild does not carry the old file's dead space forward. + + HDF5 never reclaims a deleted dataset's space, which is why ``apply`` + builds the tmp FRESH whenever a plan removes or refreshes anything + instead of copying and editing in place. If that path stopped firing, + a long-lived campaign would grow by one unit per refresh forever. + + The bound is a from-scratch build of the same campaign plus a fixed + allowance: moving a dataset across with h5py's group copy costs a + little more metadata than creating it from an array does, measured at + ~1.4 kB here and — see the cycle test below — independent of how many + times the file has been rebuilt. What must never hold is growth that + tracks the history. + """ + if not self.rebuilt or not self.output.exists(): + return + fresh = self.dir / "fresh.h5" + fresh.unlink(missing_ok=True) + try: + _build(fresh, self.sources, self.columns) + if not fresh.exists(): + return + assert (self.output.stat().st_size + <= fresh.stat().st_size + COPY_SLACK), ( + "a rebuilt file is carrying dead space: " + f"{self.output.stat().st_size} bytes against " + f"{fresh.stat().st_size} from scratch") + finally: + fresh.unlink(missing_ok=True) + + +ReconcileMachine.TestCase.settings = settings( + max_examples=150, + stateful_step_count=14, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) +TestReconcileMachine = ReconcileMachine.TestCase + + +def test_crash_between_tmp_and_replace_leaves_the_file_untouched(): + """A failed rename must leave the previous catalogue byte-identical. + + Not a hypothesis case: the interesting axis is the crash point, and there + is one. ``os.replace`` is made to raise where the tmp is moved into place. + """ + import shutil + import tempfile + + work = Path(tempfile.mkdtemp(prefix="reconcile-crash-")) + try: + output = work / "cat.h5" + columns = COLUMN_SETS[0] + sources = {} + for i, unit in enumerate(UNITS[:2]): + path = work / f"{unit}.npy" + _write_source(path, _array(columns, 3, i), + 1_000_000_000_000_000_000 + i) + sources[unit] = path + _build(output, sources, columns) + before = output.read_bytes() + before_mtime = output.stat().st_mtime_ns + + # A third unit arrives, and the rename fails. + path = work / "u2.npy" + _write_source(path, _array(columns, 3, 99), 1_000_000_000_000_000_099) + sources["u2"] = path + units = sorted(sources.items()) + digest = reconcile.schema_digest(columns) + todo = reconcile.plan(output, GROUP, units, digest) + assert todo.add == ["u2"] + + real_replace = Path.replace + + def boom(self, target): + raise OSError("simulated crash between write and rename") + + Path.replace = boom + try: + with pytest.raises(OSError): + reconcile.apply(output, GROUP, todo, units, _read, digest, + COUNT_ATTR) + finally: + Path.replace = real_replace + + assert output.read_bytes() == before, "the old catalogue was modified" + assert output.stat().st_mtime_ns == before_mtime + assert not (work / "cat.h5.tmp").exists(), "tmp outlived the failure" + finally: + shutil.rmtree(work, ignore_errors=True) + + +def test_repeated_refresh_does_not_grow_the_file(): + """The leak the rebuild path exists to prevent, asserted directly. + + Twelve refreshes of one unit in a two-unit campaign. If ``apply`` ever + copied the file and edited it in place, each would strand the previous + dataset's bytes and the size would climb monotonically. + """ + import shutil + import tempfile + + work = Path(tempfile.mkdtemp(prefix="reconcile-growth-")) + try: + output = work / "cat.h5" + columns = COLUMN_SETS[0] + sources = {} + for i, unit in enumerate(("u0", "u1")): + path = work / f"{unit}.npy" + _write_source(path, _array(columns, 4, i), 10**18 + i) + sources[unit] = path + _build(output, sources, columns) + + sizes = [] + for k in range(12): + _write_source(sources["u0"], _array(columns, 4, 100 + k), + 10**18 + 100 + k) + todo = _build(output, sources, columns) + assert todo.refresh == ["u0"], todo.describe() + sizes.append(output.stat().st_size) + assert len(set(sizes)) == 1, f"file size drifted across refreshes: {sizes}" + finally: + shutil.rmtree(work, ignore_errors=True) diff --git a/tests/unit/test_persist_exp_props.py b/tests/unit/test_persist_exp_props.py new file mode 100644 index 000000000..4fc0bc039 --- /dev/null +++ b/tests/unit/test_persist_exp_props.py @@ -0,0 +1,360 @@ +"""Property-based state machine over ``workflow/scripts/persist_exp.py``. + +``exp_persist`` packs one exposure's keepable PSF products into a tar on +/project and writes a manifest describing it, and its central promise is that +RETENTION IS ADDITIVE: an existing tar is a floor, so shrinking the campaign's +keep list can never delete a product from the backed-up filesystem. That is a +claim about every sequence of keep lists and store states the campaign can +walk through, so it is tested here against a model — random keep lists over a +random set of present products, packed repeatedly, with the tar and the +manifest asserted after every pack. + +The keep lists mix product NAMES (``psf_model``), RAW GLOBS (``*.fits``) and +overlapping combinations of the two, because overlap is the case that once +failed every exposure in a campaign: two patterns matching one file is one +file, not a name collision. A genuine collision — two DIFFERENT source paths +landing on one flat member name — must still be fatal, and has its own test. + +The script is driven through ``main()`` with a patched ``sys.argv`` rather than +a subprocess: the rule invokes it as a script, but a subprocess per hypothesis +step would put this file out of reach of a login node's time budget. +""" + +import fnmatch +import hashlib +import importlib.util +import json +import shutil +import sys +import tarfile +import tempfile +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from hypothesis.stateful import ( + RuleBasedStateMachine, + initialize, + precondition, + rule, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" + + +def _load(name): + path = SCRIPTS / f"{name}.py" + assert path.exists(), f"{path} not found; the rule calls it by path" + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(f"_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +persist = _load("persist_exp") +ALWAYS = persist.ALWAYS + +# One concrete file name per catalogued product, in the module output dir the +# real chain writes it to. The names are shaped like the campaign's (module +# tag, exposure, CCD) so the catalogue's globs match them for the same reason +# they match the real thing. +LAYOUT = { + "star_selection": ("setools", "mask", "star_selection-2079614-5.fits"), + "star_train": ("setools", "rand_split", + "star_split_ratio_80-2079614-5.fits"), + "star_test": ("setools", "rand_split", + "star_split_ratio_20-2079614-5.fits"), + "star_stats": ("setools", "stat", "star_stat-2079614-5.txt"), + "psf_model": ("psfex", "", "star_split_ratio_80-2079614-5.psf"), + "psfex_cat": ("psfex", "", "psfex_cat-2079614-5.cat"), + "psf_validation": ("psfex_interp", "", "validation_psf-2079614-5.fits"), +} +OPTIONAL = sorted(set(LAYOUT) - {ALWAYS}) +# What a campaign can write in `persist_exp:` — names, raw globs, and one name +# the catalogue does not know, which must be refused before any work happens. +ENTRIES = OPTIONAL + ["*.fits", "*.psf", "star_*", "validation_psf-*.fits"] +UNKNOWN = "psf_residuals" + +EXP = "2079614" + + +def _md5(path: Path) -> str: + return hashlib.md5(path.read_bytes()).hexdigest() + + +def _members(tar: Path) -> list: + with tarfile.open(tar) as tf: + return [ti.name for ti in tf.getmembers() if ti.isfile()] + + +class Store: + """One exposure's scratch store, its destination, and how to pack it.""" + + def __init__(self): + self.root = Path(tempfile.mkdtemp(prefix="persist-exp-props-")) + self.exp_dir = self.root / "exp" / EXP + self.dest = self.root / "products" / "psf" + self.manifest = self.root / "products" / "manifests" / f"{EXP}.json" + self.tar = self.dest / f"{EXP}.tar" + + def close(self): + shutil.rmtree(self.root, ignore_errors=True) + + def path_of(self, product: str) -> Path: + module, sub, name = LAYOUT[product] + base = (self.exp_dir / "output" / persist.RUN_NAME + / f"run_sp_{module}" / "output") + return (base / sub / name) if sub else (base / name) + + def write(self, product: str, payload: bytes) -> None: + path = self.path_of(product) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + + def drop(self, product: str) -> None: + self.path_of(product).unlink(missing_ok=True) + + def pack(self, keep: list) -> int: + """Run the script's ``main`` as the rule does. 0 on success.""" + argv = ["persist_exp.py", "--exp-dir", str(self.exp_dir), + "--exp", EXP, "--dest", str(self.dest), + "--manifest", str(self.manifest)] + for entry in keep: + argv += ["--pattern", entry] + old = sys.argv + sys.argv = argv + try: + persist.main() + return 0 + except SystemExit as exc: + return 1 if exc.code not in (0, None) else 0 + finally: + sys.argv = old + + +class PersistExpMachine(RuleBasedStateMachine): + """A store that gains and loses products under a keep list that changes.""" + + @initialize() + def setup(self): + self.store = Store() + self.present = set() + self.prior_members = [] # members of the last tar written + self.prior_labels = {} # member -> product recorded for it + + def teardown(self): + self.store.close() + + # --- the store and the config move ---------------------------------- + @rule(product=st.sampled_from(sorted(LAYOUT)), size=st.integers(1, 64)) + def add_product(self, product, size): + self.store.write(product, bytes([len(product) % 251]) * size) + self.present.add(product) + + @rule(product=st.sampled_from(sorted(LAYOUT))) + def drop_product(self, product): + self.store.drop(product) + self.present.discard(product) + + @rule(keep=st.lists(st.sampled_from(ENTRIES), max_size=4, unique=True)) + def pack(self, keep): + self._pack_and_check(keep) + + @rule(keep=st.lists(st.sampled_from(ENTRIES), max_size=3, unique=True)) + def pack_with_unknown_product(self, keep): + """An unknown product name is refused before anything is written.""" + before = (_md5(self.store.tar) if self.store.tar.exists() else None) + code = self.store.pack(keep + [UNKNOWN]) + assert code != 0, "an unknown product name was accepted" + after = (_md5(self.store.tar) if self.store.tar.exists() else None) + assert after == before, "a refused keep list still touched the tar" + + @rule() + @precondition(lambda self: ALWAYS in self.present) + def pack_twice_unchanged(self): + """A rerun over an unchanged store must not move a single byte.""" + keep = sorted(OPTIONAL)[:2] + self._pack_and_check(keep) + tar_md5, man_md5 = _md5(self.store.tar), _md5(self.store.manifest) + tar_mtime = self.store.tar.stat().st_mtime_ns + man_mtime = self.store.manifest.stat().st_mtime_ns + assert self.store.pack(keep) == 0 + assert _md5(self.store.tar) == tar_md5, "the tar is not byte-stable" + assert _md5(self.store.manifest) == man_md5, "the manifest is not byte-stable" + assert self.store.tar.stat().st_mtime_ns == tar_mtime, ( + "an unchanged rerun rewrote the tar; mtime is a rerun trigger") + assert self.store.manifest.stat().st_mtime_ns == man_mtime, ( + "an unchanged rerun rewrote the manifest") + + # --- what a pack must leave behind ----------------------------------- + def _pack_and_check(self, keep): + had_tar = self.store.tar.exists() + tar_before = _md5(self.store.tar) if had_tar else None + man_before = (_md5(self.store.manifest) + if self.store.manifest.exists() else None) + code = self.store.pack(keep) + + if ALWAYS not in self.present: + # The star catalogue's input is not optional: the job fails and + # nothing downstream may be told the store is safe to reclaim. + assert code != 0, ( + f"{ALWAYS} is missing and the pack still succeeded") + assert (_md5(self.store.tar) if self.store.tar.exists() + else None) == tar_before, "a failed pack touched the tar" + assert (_md5(self.store.manifest) + if self.store.manifest.exists() + else None) == man_before, ( + "a failed pack wrote a manifest; clean_exposure would take " + "that as permission to delete the store") + return + + assert code == 0, f"pack failed with {ALWAYS} present and keep={keep}" + assert self.store.tar.exists() and self.store.manifest.exists() + members = _members(self.store.tar) + assert len(members) == len(set(members)), ( + f"duplicate member names in the tar: {members}") + + # ADDITIVE: an existing tar is a floor. + assert set(members) >= set(self.prior_members), ( + "members vanished from the tar: " + f"{sorted(set(self.prior_members) - set(members))}") + + body = json.loads(self.store.manifest.read_text()) + listed = {f["name"] for f in body["files"]} + assert listed == set(members), ( + "manifest and tar disagree about what was packed: " + f"{sorted(listed ^ set(members))}") + assert body["n_files"] == len(members) + assert body["unit"] == EXP and body["status"] == "complete" + + entries = [ALWAYS] + [e for e in keep if e != ALWAYS] + assert body["products"] == entries + for f in body["files"]: + if f["src"] is None: # carried from the previous tar + assert f["name"] in self.prior_members + assert f["product"] == self.prior_labels.get(f["name"], "?") + continue + assert f["product"] in entries, ( + f"{f['name']} labelled {f['product']!r}, not in the keep list") + assert fnmatch.fnmatch(f["name"], persist.resolve(f["product"])), ( + f"{f['name']} does not match {f['product']!r}'s glob") + assert Path(f["src"]).exists() + assert f["bytes"] == Path(f["src"]).stat().st_size + + # Every present product the keep list asks for is in there. + for entry in entries: + glob = persist.resolve(entry) + for product in self.present: + if fnmatch.fnmatch(LAYOUT[product][2], glob): + assert LAYOUT[product][2] in listed, ( + f"{product} matched {entry!r} but was not packed") + + self.prior_members = members + self.prior_labels = {f["name"]: f["product"] for f in body["files"]} + + +PersistExpMachine.TestCase.settings = settings( + max_examples=120, + stateful_step_count=12, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) +TestPersistExpMachine = PersistExpMachine.TestCase + + +@pytest.fixture() +def store(): + s = Store() + yield s + s.close() + + +def _seed(store, products=(ALWAYS,)): + for i, product in enumerate(products): + store.write(product, bytes([i + 1]) * (16 + i)) + + +def test_corrupt_existing_tar_is_refused_and_left_alone(store): + """A tar that cannot be read may still hold the only copy of something.""" + _seed(store, (ALWAYS, "psf_model")) + assert store.pack(["psf_model"]) == 0 + store.tar.write_bytes(b"not a tar at all, not even close" * 8) + corrupt = store.tar.read_bytes() + man_before = _md5(store.manifest) + + assert store.pack(["psf_model"]) != 0, "a corrupt tar was overwritten" + assert store.tar.read_bytes() == corrupt, "the corrupt tar was modified" + assert _md5(store.manifest) == man_before, ( + "a manifest was written over a tar that could not be read") + assert not store.tar.with_name(store.tar.name + ".tmp").exists() + + +def test_two_sources_with_one_member_name_is_fatal(store): + """Members are flat, so a real name clash would silently overwrite.""" + _seed(store, (ALWAYS,)) + # The same file name under a second module output dir. + clash = (store.exp_dir / "output" / persist.RUN_NAME / "run_sp_setools" + / "output" / "new_cat" / LAYOUT[ALWAYS][2]) + clash.parent.mkdir(parents=True, exist_ok=True) + clash.write_bytes(b"a different file with the same name") + + assert store.pack([]) != 0, "two different sources shared a member name" + assert not store.manifest.exists() + assert not store.tar.exists() + + +def test_shrinking_the_keep_list_cannot_delete_a_product(store): + """The property the additive rule exists for, stated end to end.""" + _seed(store, (ALWAYS, "psf_model", "star_train")) + assert store.pack(["psf_model", "star_train"]) == 0 + wide = set(_members(store.tar)) + assert LAYOUT["psf_model"][2] in wide + + # The campaign changes its mind, and the scratch store is gone. + for product in ("psf_model", "star_train"): + store.drop(product) + assert store.pack([]) == 0 + assert set(_members(store.tar)) == wide, ( + "shrinking persist_exp: deleted products from the backed-up tar") + body = json.loads(store.manifest.read_text()) + carried = {f["name"] for f in body["files"] if f["src"] is None} + assert LAYOUT["psf_model"][2] in carried + assert {f["name"]: f["product"] for f in body["files"]}[ + LAYOUT["psf_model"][2]] == "psf_model", ( + "a carried member lost the product label the old manifest had") + + +@settings(max_examples=80, deadline=None, + suppress_health_check=[HealthCheck.too_slow]) +@given(st.lists(st.sampled_from( + [ALWAYS, "*.fits", "validation_psf-*.fits", "psf_validation", + "star_*", "*.psf", "psf_model", "star_train"]), + min_size=1, max_size=5)) +def test_overlapping_patterns_never_fail(keep): + """Two patterns matching one file is one file, not a name collision. + + Overlap is ordinary — ``validation_psf-*.fits`` beside ``*.fits`` is a + perfectly reasonable way to say "the validation catalogues, and everything + else FITS while we are here" — and treating the second match as a clash + once failed every exposure in a campaign. + + A fresh store per example, because the additive rule makes packing + stateful and this property is about ONE pack. + """ + s = Store() + try: + _seed(s, tuple(LAYOUT)) + assert s.pack(keep) == 0, f"overlapping keep list failed: {keep}" + members = _members(s.tar) + assert len(members) == len(set(members)), members + # Every product present matched something, so all seven are packed. + assert set(members) == {name for _, _, name in LAYOUT.values()} & set( + members) + finally: + s.close()