From f92cb44b6747a010c4cf369d7e515e9278824244 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 15:46:49 -0400 Subject: [PATCH 001/100] feat(orchestration): Snakemake workflow for the real-data pipeline (#848) Implements the round-2 design in #848: static two-phase execution driven by a parse-time SQLite run index (never a rule input, so appending tiles never invalidates completed work); rules wrap the existing per-unit shapepipe_run config chains at tile/exposure granularity with deterministic RUN_DATETIME=False directory() outputs; in-job completeness count-floor (the ported bash complete_check table, workflow/scripts/completeness.py); one-allocation execution profile for nibi with apptainer software-deployment (the workflow never calls apptainer itself); out-of-DAG run_report verb + onsuccess/onerror hooks; workflow/bin/sp as the committed launcher. First pass touches no module code. Validated by dry-run against a 4-tile P3 subset (219-job compute DAG); prepare phase runs green on nibi. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + profiles/nibi/config.yaml | 101 +++++++ workflow/README.md | 105 ++++++++ workflow/Snakefile | 124 +++++++++ workflow/bin/sp | 63 +++++ workflow/config.yaml | 35 +++ workflow/rules/exposure.smk | 70 +++++ workflow/rules/prepare.smk | 67 +++++ workflow/rules/tile.smk | 147 ++++++++++ workflow/scripts/build_forest.py | 42 +++ workflow/scripts/build_index.py | 139 ++++++++++ workflow/scripts/clean_exposures.py | 64 +++++ workflow/scripts/completeness.py | 86 ++++++ workflow/scripts/run_report.py | 107 ++++++++ workflow/scripts/sp_rule.py | 405 ++++++++++++++++++++++++++++ 15 files changed, 1556 insertions(+) create mode 100644 profiles/nibi/config.yaml create mode 100644 workflow/README.md create mode 100644 workflow/Snakefile create mode 100755 workflow/bin/sp create mode 100644 workflow/config.yaml create mode 100644 workflow/rules/exposure.smk create mode 100644 workflow/rules/prepare.smk create mode 100644 workflow/rules/tile.smk create mode 100644 workflow/scripts/build_forest.py create mode 100644 workflow/scripts/build_index.py create mode 100644 workflow/scripts/clean_exposures.py create mode 100644 workflow/scripts/completeness.py create mode 100644 workflow/scripts/run_report.py create mode 100644 workflow/scripts/sp_rule.py diff --git a/.gitignore b/.gitignore index db20f3086..4c4b6d0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ code # .felt here is a machine-local symlink into it. Never track it in this repo. /.felt/ /.felt +.snakemake/ diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml new file mode 100644 index 000000000..e977803c1 --- /dev/null +++ b/profiles/nibi/config.yaml @@ -0,0 +1,101 @@ +# Snakemake profile for the Nibi cluster (Digital Research Alliance). +# +# ONE-ALLOCATION mode: sbatch one node up front, then run snakemake with the +# LOCAL scheduler inside it (no per-job sbatch; the local scheduler bin-packs the +# DAG into the allocation against `cores` + `resources: mem_mb`). This is the +# proven nibi path (job_p3_batch1) and the single honest first-pass mode; +# multi-node scaling = multiple tile-disjoint invocations, deferred. +# +# Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). +# software-deployment-method wraps every job's shell in `apptainer exec` — the +# user never types apptainer; the container is set in the Snakefile (local .sif). + +# Local scheduler (no `executor:` -> built-in local). Size to the allocation. +cores: 192 # node cores available to the DAG +local-cores: 4 # cap localrule/aggregation concurrency +resources: + mem_mb: 720000 # node RAM cap the local scheduler bin-packs under + +software-deployment-method: [apptainer] +# Explicit container environment (finding 2/3): the apptainer SDM otherwise drops +# the proven-recipe env and hardcodes --home , hiding ~/.ssl/cadcproxy.pem. +# --cleanenv strict host-env isolation (APPTAINERENV_*/SINGULARITYENV_* survive) +# OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) +# MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) +# --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# PYTHONPATH SETTLED CALL 3 — P0 pins the prod worktree src so science +# content matches the bash validation runs (which use this +# override). Production later rebuilds the sif at the exact +# validation commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-prod/src --home /home/cdaley --bind /project --bind /scratch" + +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true + +# rerun-triggers left at the full Snakemake v9 default +# {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. +# No override — the wrapper writes outputs only on change so mtimes move only +# when reality moves; protected() guards finished science products. + +default-resources: + mem_mb: 4000 + runtime: 120 # minutes + cpus_per_task: 1 + +# Per-stage resources — measured footprints from the nibi P3 runs (#808). +# threads == cpus_per_task == ShapePipe SMP fork width (sp_rule sets +# SMP_BATCH_SIZE={threads}); mem_mb == threads x measured per-worker footprint. +# +# set-threads is REQUIRED for {threads} to resolve above 1: the local scheduler +# derives cpus_per_task from threads, NOT the reverse (verified snakemake 9.23 — +# set-resources cpus_per_task alone leaves {threads}==1, i.e. SMP_BATCH_SIZE=1, +# serial). Keep the two blocks in lockstep. +set-threads: + exp_split: 4 + exp_mask: 4 + exp_psf: 8 + tile_mask: 16 + tile_detect: 16 + tile_merge_headers: 8 + tile_vignets: 8 + tile_ngmix: 8 + tile_merge_cats: 4 + tile_make_cat: 4 + +set-resources: + exp_split: + cpus_per_task: 4 + mem_mb: 8000 + exp_mask: + cpus_per_task: 4 + mem_mb: 8000 + exp_psf: + cpus_per_task: 8 + mem_mb: 16000 + runtime: 240 + tile_mask: + cpus_per_task: 16 + mem_mb: 16000 + tile_detect: + cpus_per_task: 16 + mem_mb: 16000 + tile_merge_headers: + cpus_per_task: 8 + mem_mb: 8000 + tile_vignets: + cpus_per_task: 8 + mem_mb: 24000 + runtime: 240 + tile_ngmix: + cpus_per_task: 8 + mem_mb: 24000 + runtime: 360 + tile_merge_cats: + cpus_per_task: 4 + mem_mb: 8000 + tile_make_cat: + cpus_per_task: 4 + mem_mb: 8000 diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 000000000..4d8abd1f1 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,105 @@ +# ShapePipe Snakemake orchestration + +Snakemake workflow that orchestrates real-data ShapePipe runs, replacing the +`curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` +bash layers (and per-site sbatch reimplementations). **Module code is untouched**: +rules call `shapepipe_run -c ` on the existing config chains. Design and +rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +(the living PRD). + +## Quick start (nibi) + +**One-allocation mode.** `sbatch` one node, then run Snakemake with the *local* +scheduler inside it: it bin-packs the DAG into the allocation (no per-job +sbatch). Each job's shell runs `shapepipe_run` **inside the container** (the +profile's apptainer software-deployment — you never type `apptainer`). + +```bash +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which is +# node-local; the executor re-invokes this python inside jobs). +uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 +source /project/def-mjhudson/cdaley/snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' + +# Edit workflow/config.yaml: tile_list, run_dir, config_src, container, star_cats. + +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a fresh +# shell always has the right state. Inside your allocation: +workflow/bin/sp prepare # prepare_tiles -> build_index.py -> prepare_exposures +workflow/bin/sp run # the main compute DAG +``` + +`sp` subcommands: `prepare` (chains all three prepare steps), `prepare_tiles`, +`index`, `prepare_exposures`, `run`, `rerun ` (forced recompute of +protected products), `cancel ` (scancel sweep before `--unlock`), +`clean-exposures [--apply]`, `report`. + +## Execution is three static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, so +it cannot be scheduled in the same static DAG that produces it. Hence: + +1. `snakemake prepare_tiles` — per-tile static DAG (Git_vos → Uz → Fe), + `keep-going` so tile failures are independent. +2. `build_index.py` — a **plain script**, not a DAG node. It iterates the + declared tile list, checks each tile's Fe output at its deterministic path (no + glob), writes `run_index.sqlite` + `missing.json`, and exits nonzero only if + the missing *fraction* exceeds a threshold. (There is no `--allow-missing` + Snakemake flag; that mechanism does not exist.) +3. `snakemake prepare_exposures` — per-exposure static DAG (Gie_vos + per-unit + star cats), parseable now the index exists. +4. `snakemake` (main DAG) — the tile/exposure compute chain. + +`sp prepare` chains 1–3 so the UX stays one command. + +## Layout + +``` +workflow/ + Snakefile parse-time index load; global container:; onsuccess/onerror report hooks + config.yaml the run: tile list, paths, container, chunk count + bin/sp committed launcher (module load + /project venv + subcommands) + rules/ + prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats + exposure.smk per-exposure: get_images, split, mask, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + scripts/ + sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) + build_index.py prepare-phase run_index.sqlite builder (plain script) + build_forest.py per-tile exposure symlink forest (group-compatible shell) + completeness.py the ported count-floor table (shared by sp_rule + run_report) + run_report.py standalone report (NOT a DAG node; run_report hooks call it) + clean_exposures.py index-driven exposure-store reclaim (replaces temp()) +profiles/nibi/config.yaml local scheduler; apptainer SDM; resources; keep-going +``` + +## How it works + +- **Completeness = the count floor.** Each stage declares its deterministic + `output/run_sp_/` directory as `directory()` output (the per-unit + config copy forces `RUN_DATETIME = False`, so the path is known at DAG time). + After the run, `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it; per-CCD attrition between + floor and `expect` is tolerated (the directory simply lacks that product). + This *is* the failure policy — no 3-class taxonomy, no error-signature + whitelist. `--keep-going` isolates a failure to its own cone. +- **Per-unit isolation is by work-dir content, not `-e/--exclusive`.** Every + tile/exposure runs its own `shapepipe_run` in `tiles//` or `exp//`, + with a `cfis` config symlink, `star_cat_{exp,tiles}` symlinks, a + `tile_numbers.txt` (tiles) or fabricated `exp_numbers-000-000.txt` pseudo-Fe + (exposures), and a config copy setting `RUN_DATETIME=False` and + `SMP_BATCH_SIZE={threads}`. `NUMBER_LIST=-` is injected **only** for + stages whose numbering scheme is the unit ID (tile-scheme stages + exp split), + never for get_images / exp_mask / exp_psf. +- **No `temp()` on shared exposure dirs.** Exposures overlap tiles, so temp() + would cascade destructive reruns when a tile is appended. The store is + persistent; `sp clean-exposures` reclaims space only for exposures all of + whose consuming tiles have a final_cat. +- **The index is parse-time data, never a rule input.** Appending tiles changes + which jobs exist without invalidating completed work. Star cats are re-keyed + per unit for the same reason. `final_cat` is `protected()`. +- **Failure is a report, not a gate.** `run_report.py` disk-scans the trees + against the count table and enumerates shortfalls (whole-unit absence vs + per-CCD attrition). It is a standalone script — a DAG report node would be + poisoned by the very failures it must enumerate — emitted automatically by the + `onsuccess`/`onerror` hooks and runnable any time via `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 000000000..04722a73c --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,124 @@ +"""ShapePipe real-data orchestration — Snakemake workflow. + +Design / rationale: CosmoStat/shapepipe#848 (the living PRD). + +Execution is honestly THREE static invocations, chained by ``workflow/bin/sp`` +so the UX stays one command (``sp prepare`` / ``sp run``): + + snakemake --profile profiles/nibi prepare_tiles # Git_vos + Uz + Fe (keep-going) + python workflow/scripts/build_index.py ... # PLAIN SCRIPT, not a DAG node + snakemake --profile profiles/nibi prepare_exposures # Gie_vos + star cats + snakemake --profile profiles/nibi # the main compute DAG + +The run index is loaded ONCE here at parse time into plain dicts; it is never a +rule input, so appending tiles and rebuilding it changes which jobs exist without +invalidating completed work. build_index.py is a plain script *between* +prepare_tiles and prepare_exposures (the exposure job set is data-derived from +the tiles' find_exposures output, so it cannot be scheduled in the same static +DAG that produces it). +""" + +import sqlite3 +from pathlib import Path + +configfile: "workflow/config.yaml" + +# Every job's shell runs inside this container (apptainer software-deployment in +# the profile); the user never types apptainer. +container: config["container"] + +# --- paths ----------------------------------------------------------------- +RUN_DIR = Path(config["run_dir"]) +CONFIG_SRC = Path(config["config_src"]) +STAR_CATS = Path(config["star_cats"]) +INDEX_DB = Path(config["index_db"]) +SCRIPTS = Path(workflow.basedir) / "scripts" +NGMIX_CHUNKS = config.get("ngmix_chunks", 8) + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- parse-time index load (compute phase only) ---------------------------- +# EXP: global set of exposure base-ids. TILE_EXP: tile -> [exp_ids]. +EXP, TILE_EXP = set(), {} +if INDEX_DB.exists(): + _con = sqlite3.connect(INDEX_DB) + EXP = {r[0] for r in _con.execute("SELECT exp_id FROM exposures")} + for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(tile, []).append(exp) + _con.close() + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + chunk = r"\d+", + +# --- shared wrapper invocation --------------------------------------------- +def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, + extra=""): + """Build the shell command that runs one unit through sp_rule.py. + + NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass + ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download + numbering must not be constrained. ``{threads}`` is filled by Snakemake and + becomes SMP_BATCH_SIZE (fork width == cpus_per_task). + """ + cmd = ( + f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " + f"--unit {unit} --level {level} --run-dir {RUN_DIR} " + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} --threads {{threads}}" + ) + if not isolate: + cmd += " --no-isolate" + if exp_forest: + cmd += f" --exp-forest {exp_forest}" + if extra: + cmd += " " + extra + return cmd + +def tile_dir(tile): + return RUN_DIR / "tiles" / tile + +def exp_dir(exp): + return RUN_DIR / "exp" / exp + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" + +# --- top-level targets ------------------------------------------------------ +# Only aggregation targets are localrules (no-ops under the local executor, but +# they keep the compute chain group-compatible: no mid-chain localrules). +localrules: all, prepare_tiles, prepare_exposures + +rule all: + input: + expand(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits"), tile=TILES) + +# Phase A.1 — per-tile static DAG (download + find_exposures). keep-going so +# tile failures are independent; build_index.py runs next as a plain script. +rule prepare_tiles: + input: + expand(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe"), tile=TILES) + +# Phase A.2 — per-exposure static DAG (download + star cats), parseable now the +# index exists. Star cats are re-keyed per unit (see prepare.smk). +rule prepare_exposures: + input: + expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)), + expand(str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), exp=sorted(EXP)), + expand(str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), tile=TILES) + +# --- report hooks ----------------------------------------------------------- +# run_report is NOT a DAG node (a descendant of every job would be poisoned by +# any hard failure — the exact case it exists for). It is a standalone script, +# emitted automatically at the end of every invocation, runnable any time. +def _report(status): + shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " + f"--index {INDEX_DB} --status {status} || true") + +onsuccess: + _report("success") + +onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp new file mode 100755 index 000000000..f226aa441 --- /dev/null +++ b/workflow/bin/sp @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# sp — the committed launcher for the ShapePipe Snakemake workflow. +# +# Removes the shell-state dependencies the review flagged (finding 20): it loads +# the apptainer module (Snakemake resolves `apptainer` via PATH at job runtime) +# and runs the Snakemake that lives on the shared /project venv (must be on a +# shared FS — the executor re-invokes it inside jobs). One entry point, so a +# fresh tmux / a restart after a crash always launches with the right state. +# +# Usage: +# sp prepare # prepare_tiles -> build_index.py -> prepare_exposures +# sp prepare_tiles # phase A.1 only +# sp index # build_index.py only (plain script, threshold-gated) +# sp prepare_exposures # phase A.2 only +# sp run [ARGS...] # the main compute DAG +# sp rerun # forced recompute of protected/finished products +# sp cancel # scancel this workflow's jobs (name sweep) before --unlock +# sp clean-exposures [--apply] # index-driven exposure-store reclaim +# sp report # emit run_report.json now (mid-run ok) +# Extra args after the subcommand pass through to snakemake. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ +REPO="$(dirname "$HERE")" +VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" +PROFILE="$REPO/profiles/nibi" +SCRIPTS="$HERE/scripts" +CONFIG="$HERE/config.yaml" + +module load apptainer/1.4.5 2>/dev/null || true +# shellcheck disable=SC1091 +source "$VENV/bin/activate" + +# Minimal scalar reader for workflow/config.yaml (key: value, no nesting). +cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } +RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" + +sm() { snakemake --profile "$PROFILE" "$@"; } +build_index() { + python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ + --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" +} + +cmd="${1:-}"; shift || true +case "$cmd" in + prepare) sm prepare_tiles "$@"; build_index; sm prepare_exposures "$@" ;; + prepare_tiles) sm prepare_tiles "$@" ;; + index) build_index ;; + prepare_exposures) sm prepare_exposures "$@" ;; + run) sm "$@" ;; + rerun) sm --forcerun "$@" ;; + cancel) + run="${1:?usage: sp cancel }" + squeue --me --noheader --format='%i %j' \ + | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; + clean-exposures) python "$SCRIPTS/clean_exposures.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" "$@" ;; + report) python "$SCRIPTS/run_report.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" --status manual "$@" ;; + *) echo "sp: unknown subcommand '${cmd:-}' (see the header of $0)"; exit 2 ;; +esac diff --git a/workflow/config.yaml b/workflow/config.yaml new file mode 100644 index 000000000..53542d1c2 --- /dev/null +++ b/workflow/config.yaml @@ -0,0 +1,35 @@ +# Run configuration for the ShapePipe Snakemake workflow. +# +# A "run" is declared by a tile list plus the paths below. Everything here is +# read at parse time; none of it is a rule input, so editing it (e.g. appending +# tiles) never invalidates completed work — it only changes which jobs exist. + +# The tile list that scopes this run (one "IDra.IDdec" per line). +tile_list: /scratch/cdaley/shapepipe-output/snakemake-p0/tiles4.txt + +# The container every job runs inside (apptainer software-deployment in the profile). +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif + +# Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: +# /tiles/// and /exp/// +run_dir: /scratch/cdaley/shapepipe-output/snakemake-p0 + +# The canonical config chain (example/cfis in the repo, or a run-specific copy). +config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis + +# Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). +# Star catalogues for masking, pre-generated (network step done in prepare): +star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats + +# The run index. Durable, low-volume products (index, missing.json, run_report, +# final_cat) should land on /project, not /scratch (60-day purge, finding 15). +index_db: /project/def-mjhudson/cdaley/shapepipe-output/snakemake-p0/index/run_index.sqlite + +# ngmix within-tile chunking: static N chunks (closed ID ranges computed +# per-tile, in-job, from the tile's own sexcat). +ngmix_chunks: 8 + +# The container's installed shapepipe is overridden by the prod worktree via +# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# build_index.py's missing-tile fraction floor is passed by workflow/bin/sp +# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk new file mode 100644 index 000000000..50403d386 --- /dev/null +++ b/workflow/rules/exposure.smk @@ -0,0 +1,70 @@ +"""Exposure chain — per exposure, keyed by exp_id (dedup is structural). + +Gie -> Sp -> Ma -> psf, each in the exposure's own work dir, chained by +ShapePipe's last:/run_sp_exp_* INPUT_DIR resolver. Completeness is the count +floor (completeness.py). + +NO temp() on these shared exposure directories: exposures overlap tiles by +construction (~7-10 tiles per exposure), so temp() would delete an exposure the +moment its forests are built and cascade destructive reruns across spatial +neighbours whenever a tile is appended (findings 4/5/6/19). The store is +persistent; space is reclaimed by the explicit index-driven `sp clean-exposures` +verb (scripts/clean_exposures.py), which deletes an exposure's intermediates +only once every consuming tile in the index has its final_cat present. + +NUMBER_LIST is injected only for exp_split (numbering scheme == exposure id); +NEVER for get_images (download) / exp_mask / exp_psf (per-CCD schemes where the +#746 startup validation would turn tolerated per-CCD attrition into a +whole-exposure hard failure). +""" + +EXP_OUT = str(RUN_DIR / "exp/{exp}/output") + +# Symlink/download exposure image/weight/flag; reads the fabricated pseudo-Fe +# exp_numbers-000-000.txt the wrapper drops (via last:find_exposures_runner). +rule exp_get_images: + output: + directory(f"{EXP_OUT}/run_sp_exp_Gie") + params: + cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", + "exp", wc.exp, isolate=False) + shell: + "{params.cmd}" + +# Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; +# persistent (see module docstring on temp()). +rule exp_split: + input: + rules.exp_get_images.output + output: + directory(f"{EXP_OUT}/run_sp_exp_Sp") + params: + cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) + shell: + "{params.cmd}" + +# Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). +rule exp_mask: + input: + rules.exp_split.output, + star=str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), + output: + directory(f"{EXP_OUT}/run_sp_exp_Ma") + params: + cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", + "exp", wc.exp, isolate=False) + shell: + "{params.cmd}" + +# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. +rule exp_psf: + input: + rules.exp_mask.output + output: + directory(f"{EXP_OUT}/run_sp_exp_SxSePsfPi") + params: + cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, + isolate=False) + shell: + "{params.cmd}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk new file mode 100644 index 000000000..2911c3b19 --- /dev/null +++ b/workflow/rules/prepare.smk @@ -0,0 +1,67 @@ +"""Phase A — prepare: tile staging, find_exposures, per-unit star cats. + +Split into two static invocations with build_index.py (a plain script) between: + * prepare_tiles — tile_get_images -> tile_uncompress -> tile_find_exposures + * [build_index.py] — plain script, threshold-gated (not a DAG node) + * prepare_exposures — exp_get_images (exposure.smk) + per-unit star cats + +Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). +Rules stay group-compatible: no mid-chain localrules, no run:/pipe outputs. +""" + +# get_images reads a one-line tile_numbers.txt the wrapper drops in $SP_RUN. +# NUMBER_LIST is NEVER injected here (download stage; nothing on disk to +# validate against — #746 would hard-fail). +rule tile_get_images: + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Git")) + params: + cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", + "tile", wc.tile, isolate=False) + shell: + "{params.cmd}" + +rule tile_uncompress: + input: + rules.tile_get_images.output + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Uz")) + params: + cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the +# data-derived tile->exposure edge build_index.py aggregates (as a plain script). +rule tile_find_exposures: + input: + rules.tile_uncompress.output + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe")) + params: + cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# Star catalogues for masking are pre-generated offline (create_star_cat.py on a +# networked login node — the one genuinely networked prepare step). They are +# re-keyed PER UNIT here: each tile/exposure's cat is its own DAG node, so +# appending tile 51 adds nodes rather than mutating a run-level output that would +# dirty every finished mask (the growth-invariant fix, finding 11). +rule exp_star_cat: + output: + str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits") + params: + src=lambda wc: str(STAR_CATS / "exp" / f"star_cat-{wc.exp}.fits") + shell: + "ln -sf {params.src} {output}" + +rule tile_star_cat: + output: + str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") + params: + src=lambda wc: str(STAR_CATS / "tiles" / f"star_cat-{wc.tile}.fits") + shell: + "ln -sf {params.src} {output}" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk new file mode 100644 index 000000000..845167710 --- /dev/null +++ b/workflow/rules/tile.smk @@ -0,0 +1,147 @@ +"""Tile post-chain — per tile: gather exposures, then detect/PSF/shape/catalogue. + +A per-tile exposure "forest" (symlink view of exactly this tile's exposures' +products) gives the tile modules a deterministic $SP_EXP to glob. But the forest +is only a runtime convenience, NEVER the DAG edge: every tile rule that reads +through $SP_EXP declares the underlying exposure directory() outputs as input via +an index dict lookup, so the last real reader is the last DAG consumer (findings +5/19). All rules are group-compatible (shell only, no mid-chain localrules). +""" + +TILE_OUT = str(RUN_DIR / "tiles/{tile}/output") +FOREST = str(RUN_DIR / "tiles/{tile}/exp_forest") + + +def _exp_out(wc, prefix): + return [str(exp_dir(e) / "output" / prefix) for e in TILE_EXP.get(wc.tile, [])] + +def tile_exp_split(wc): return _exp_out(wc, "run_sp_exp_Sp") +def tile_exp_mask(wc): return _exp_out(wc, "run_sp_exp_Ma") +def tile_exp_psf(wc): return _exp_out(wc, "run_sp_exp_SxSePsfPi") +def tile_exp_products(wc): + """All of this tile's exposures' product dirs — the forest's DAG inputs.""" + return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) + + +# Build the per-tile symlink forest (exp_forest//output/run_sp_exp_* -> the +# real exposure products). Declaring the exposure dirs as input makes this the +# tile's wait-on-its-exposures edge; the forest itself is just the SP_EXP view. +rule tile_exp_forest: + input: + tile_exp_products + output: + directory(FOREST) + params: + cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") + shell: + "{params.cmd}" + +# Merge single-exposure WCS headers into the tile-level sqlite log +# (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng +# consume via NUMBERING_SCHEME -000-000). Reads headers-*.npy through the forest, +# so the exposure split dirs are explicit inputs. +rule tile_merge_headers: + input: + forest=rules.tile_exp_forest.output, + split=tile_exp_split, + output: + directory(f"{TILE_OUT}/run_sp_tile_Mh_exp") + params: + cmd=lambda wc: sp_rule("tile_merge_headers", "config_tile_Mh_exp.ini", + "tile", wc.tile, isolate=True, + exp_forest=FOREST.format(tile=wc.tile)) + shell: + "{params.cmd}" + +# Mask tiles; this tile's star cat is a declared input (per-unit re-key). +rule tile_mask: + input: + git=rules.tile_get_images.output, + star=str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), + output: + directory(f"{TILE_OUT}/run_sp_tile_Ma") + params: + cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# SExtractor object detection on the tile. +rule tile_detect: + input: + mask=rules.tile_mask.output, + mh=rules.tile_merge_headers.output, + output: + directory(f"{TILE_OUT}/run_sp_tile_Sx") + params: + cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF +# models + split through the forest — those dirs are explicit inputs). +rule tile_vignets: + input: + sx=rules.tile_detect.output, + forest=rules.tile_exp_forest.output, + split=tile_exp_split, + psf=tile_exp_psf, + output: + directory(f"{TILE_OUT}/run_sp_tile_PiViVi") + params: + cmd=lambda wc: sp_rule("tile_vignets", "config_tile_PiViVi_canfar_sx.ini", + "tile", wc.tile, isolate=True, + exp_forest=FOREST.format(tile=wc.tile)) + shell: + "{params.cmd}" + +# ngmix shape measurement — static N chunks. Each chunk computes its own closed +# object-ID range in-job from the tile's own sexcat (a declared input); chunks +# never rewrite the shared log (--no-log-sync) and read literal INPUT_DIRs. +rule tile_ngmix: + input: + vignets=rules.tile_vignets.output, + sx=rules.tile_detect.output, + output: + directory(f"{TILE_OUT}/run_sp_tile_ngmix_Ng{{chunk}}u") + params: + cmd=lambda wc: sp_rule( + "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, + extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), + shell: + "{params.cmd}" + +def ngmix_chunks(wc): + return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" + for k in range(1, NGMIX_CHUNKS + 1)] + +# Merge the N chunk catalogues (DAG-serialized: the single per-tile log-sync +# point before make_cat). N_SPLIT_MAX == chunk count. +rule tile_merge_cats: + input: + ngmix_chunks + output: + directory(f"{TILE_OUT}/run_sp_Ms") + params: + cmd=lambda wc: sp_rule("tile_merge_cats", "config_merge_sep_cats_template.ini", + "tile", wc.tile, isolate=True, + extra=f"--n-split-max {NGMIX_CHUNKS}"), + shell: + "{params.cmd}" + +# Build the final catalogue: the run's science product. protected() so a +# code/params-drift rerun cannot silently discard a finished catalogue — the +# recompute must go through `sp rerun` (--forcerun). +rule tile_make_cat: + input: + rules.tile_merge_cats.output + output: + protected(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits")) + params: + cmd=lambda wc: sp_rule("tile_make_cat", "config_make_cat_psfex_nosm.ini", + "tile", wc.tile, isolate=True, + extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), + shell: + "{params.cmd}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py new file mode 100644 index 000000000..82df8b0c6 --- /dev/null +++ b/workflow/scripts/build_forest.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the SP_EXP view). + +A plain script (not a run: block) so the tile chain stays group-compatible. +Reads the tile's exposures from run_index.sqlite and symlinks each exposure's +``exp//output`` into ``//output`` by exact name (no glob), so +the tile modules' $SP_EXP globs resolve to exactly this tile's exposures' +products. The forest is a convenience view; the DAG edge to the exposures is +declared in the rule's input (tile.smk), not here. +""" + +import argparse +import sqlite3 +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + con = sqlite3.connect(args.index) + exps = [r[0] for r in con.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] + con.close() + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e / "output" + dst = args.forest / e / "output" + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.is_symlink() or dst.exists(): + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py new file mode 100644 index 000000000..13783314a --- /dev/null +++ b/workflow/scripts/build_index.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Build the run index (``run_index.sqlite``) that drives the compute DAG. + +The index is *parse-time data*, never a rule input: the Snakefile loads it once +at parse time into plain dicts, so extending a run's tile list changes which +jobs exist without touching the mtime chain of completed work. + +It records, per run: + + tiles(tile_id, ra_dir, n_exp, status) + exposures(exp_id) -- deduplicated union over tiles + tile_exposures(tile_id, exp_id) -- the tile->exposure edges + +The tile->exposure edges are *data-derived*: they are read from each tile's +``find_exposures`` output (``exp_numbers--.txt``), which +``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. +So this is a **plain script**, not a DAG node: it runs between the two static +invocations — after ``snakemake prepare_tiles`` (Git+Uz+Fe, keep-going) and +before ``snakemake prepare_exposures`` parses the now-existing index. It +iterates the *declared* tile list and checks each tile's Fe output at its +deterministic path (no globbing — O(tile-list) existence checks, respecting the +no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in +``missing.json`` and the index is built over the rest. The script exits nonzero +only if the missing *fraction* exceeds ``--missing-threshold`` (default 0.0), +so a keep-going download storm that lost a few tiles does not cost the run. +There is NO ``--allow-missing`` flag (that Snakemake flag does not exist). + +Exposure IDs are stored with their trailing single-char suffix stripped +(``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the +exposure-rule wildcard, matching the ``exp///`` forest layout. +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + + +def read_exposure_list(exp_numbers_file: Path) -> list[str]: + """Return the exp_base IDs listed in one tile's find_exposures output. + + Each line is an exposure name like ``2243881p``; we strip a trailing + alphabetic suffix so the base ID is the dedup key. + """ + ids = [] + for line in exp_numbers_file.read_text().splitlines(): + name = line.strip() + if not name: + continue + ids.append(name[:-1] if name[-1].isalpha() else name) + return ids + + +def build(tile_ids: list[str], run_dir: Path, db_path: Path, + missing_threshold: float = 0.0) -> dict: + """Build the index over ``tile_ids``; return a summary dict. + + For each tile, check its ``exp_numbers--.txt`` at the tile's + deterministic ``find_exposures`` run dir (RUN_DATETIME=False, no glob). A + tile whose exposure list is missing is recorded in ``missing.json`` and the + index is built over the rest (a bad tile costs that tile, not the run). The + build is fatal only if the missing fraction exceeds ``missing_threshold``. + """ + db_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db_path) + con.executescript( + """ + DROP TABLE IF EXISTS tiles; + DROP TABLE IF EXISTS exposures; + DROP TABLE IF EXISTS tile_exposures; + CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE exposures(exp_id TEXT PRIMARY KEY); + CREATE TABLE tile_exposures( + tile_id TEXT, exp_id TEXT, + PRIMARY KEY (tile_id, exp_id)); + """ + ) + + missing = [] + all_exposures: set[str] = set() + for tile_id in tile_ids: + ra_dir = tile_id.split(".")[0] + idra, iddec = tile_id.split(".") + # Flat forest, deterministic run dir (RUN_DATETIME=False). + exp_file = (run_dir / "tiles" / tile_id / "output" / "run_sp_tile_Fe" / + "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + if not exp_file.exists(): + missing.append(tile_id) + continue + exp_ids = read_exposure_list(exp_file) + con.execute("INSERT INTO tiles VALUES (?,?,?)", + (tile_id, ra_dir, len(exp_ids))) + for exp_id in exp_ids: + all_exposures.add(exp_id) + con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", + (tile_id, exp_id)) + + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?)", + [(e,) for e in sorted(all_exposures)]) + con.commit() + con.close() + + (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + return {"n_tiles": len(tile_ids) - len(missing), + "n_exposures": len(all_exposures), + "n_missing": len(missing)} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-list", required=True, type=Path, + help="file of tile IDs, one per line") + p.add_argument("--run-dir", required=True, type=Path, + help="$SP_RUN: root of the tiles/ work-dir forest") + p.add_argument("--db", required=True, type=Path, + help="output run_index.sqlite path") + p.add_argument("--missing-threshold", type=float, default=0.0, + help="fatal if the missing-tile fraction exceeds this " + "(default 0.0: any missing tile is fatal)") + args = p.parse_args() + + tile_ids = [ln.strip() for ln in args.tile_list.read_text().splitlines() + if ln.strip()] + summary = build(tile_ids, args.run_dir, args.db, args.missing_threshold) + print(f"run_index: {summary['n_tiles']} tiles, " + f"{summary['n_exposures']} exposures, " + f"{summary['n_missing']} missing -> {args.db}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposures.py b/workflow/scripts/clean_exposures.py new file mode 100644 index 000000000..4f46bf19b --- /dev/null +++ b/workflow/scripts/clean_exposures.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Index-driven exposure-store cleanup (`sp clean-exposures`). + +temp() is deliberately NOT used on the shared exposure directories (findings +4/5/6/19): they overlap tiles by construction, so temp() deletion would cascade +destructive reruns across spatial neighbours whenever a tile is appended. Instead +the store is persistent and space is reclaimed explicitly here. + +For each exposure in run_index.sqlite, delete its bulky intermediates +(run_sp_exp_Sp / run_sp_exp_Ma / run_sp_exp_SxSePsfPi) ONLY when every tile that +consumes it (tile_exposures) has its final_cat-.fits present across the whole +output root. An exposure still feeding an unfinished tile is left untouched. + +Caveat: if a later batch appends a tile that shares a deleted exposure, +Snakemake will regenerate that exposure — cleanup trades peak storage for +recompute. Run it between campaign phases, not mid-DAG. --dry-run by default. +""" + +import argparse +import shutil +import sqlite3 +import sys +from pathlib import Path + +PREFIXES = ("run_sp_exp_Sp", "run_sp_exp_Ma", "run_sp_exp_SxSePsfPi") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--apply", action="store_true", + help="actually delete (default: dry-run, list only)") + args = p.parse_args() + + con = sqlite3.connect(args.index) + edges = con.execute("SELECT exp_id, tile_id FROM tile_exposures").fetchall() + con.close() + + consumers: dict[str, list[str]] = {} + for exp_id, tile_id in edges: + consumers.setdefault(exp_id, []).append(tile_id) + + def tile_done(tile): + return (args.run_dir / "tiles" / tile / f"final_cat-{tile}.fits").exists() + + freed, held = 0, 0 + for exp_id, tiles in consumers.items(): + if not all(tile_done(t) for t in tiles): + held += 1 + continue + for prefix in PREFIXES: + d = args.run_dir / "exp" / exp_id / "output" / prefix + if d.is_dir(): + freed += 1 + print(f"{'DELETE' if args.apply else 'would delete'} {d}") + if args.apply: + shutil.rmtree(d) + print(f"[clean_exposures] {freed} dirs {'freed' if args.apply else 'reclaimable'}, " + f"{held} exposures held (consuming tile unfinished)", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py new file mode 100644 index 000000000..f35e81aee --- /dev/null +++ b/workflow/scripts/completeness.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""The count-floor completeness table — the single failure policy. + +This is the ported ``complete_check`` count table from the v2.0 bash layer +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). It is the *only* +failure policy in the design: there is no 3-class taxonomy and no error-signature +whitelist. A stage is a real failure iff a mandatory runner produced fewer than +its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) +sits between ``floor`` and ``expect`` and is tolerated. + +Two consumers share this table: + * ``sp_rule.py`` — after ``shapepipe_run``, counts products per runner and + exits nonzero if any mandatory runner is below its floor (``keep-going`` + then isolates that unit's cone; ``retries`` handle transient failures). + * ``run_report.py`` — disk-scans finished trees against ``expect`` to + enumerate shortfalls for the human, mid-run or after. + +Per-runner fields: + expect nominal file count for a fully complete unit (report yardstick) + floor the fail-loud minimum (below this the job exits nonzero) + warn if True the runner never fails the unit at all (bash ``:warn`` — + e.g. psfex_interp on tiles missing some epochs) + subpath count files in ``/output//`` instead of + ``/output/`` (bash ``:rand_split`` — setools split cats) + +Counts are file counts in the runner's output dir, matching the bash +``ls / | wc -l`` semantics (broken symlinks excluded by the caller). +""" + +# stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +COMPLETENESS = { + # --- tile prepare (phase A) --- + "tile_get_images": {"get_images_runner": dict(expect=4, floor=4)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, + + # --- exposure chain --- + "exp_get_images": {"get_images_runner": dict(expect=6, floor=6)}, + "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, + "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + "exp_psf": { + "sextractor_runner": dict(expect=80, floor=2), + "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), + "psfex_runner": dict(expect=80, floor=2), + "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + }, + + # --- tile post --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, + "tile_mask": {"mask_runner": dict(expect=1, floor=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, + "tile_vignets": { + "psfex_interp_runner": dict(expect=1, floor=1), + "vignetmaker_runner_run_1": dict(expect=1, floor=1), + "vignetmaker_runner_run_2": dict(expect=4, floor=4), + }, + "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1, floor=1)}, +} + + +def count_products(run_dir, runner, spec): + """Count files in ``run_dir//output[/]/`` (live links only).""" + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + return sum(1 for p in out.iterdir() if p.exists()) # p.exists() drops dead links + + +def check_floor(stage, run_dir): + """Return (ok, details). ok is False iff a mandatory runner is below floor. + + ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. + """ + table = COMPLETENESS[stage] + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + details.append((runner, n, spec["floor"], spec["expect"], + spec.get("warn", False))) + if not spec.get("warn", False) and n < spec["floor"]: + ok = False + return ok, details diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py new file mode 100644 index 000000000..bb3300405 --- /dev/null +++ b/workflow/scripts/run_report.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Standalone run report — NOT a DAG node. + +A report rule that declares all tiles' outputs as inputs is a descendant of every +job, so any single hard failure (under --keep-going) poisons its cone and the +report never runs — the exact scenario it exists for (finding 8). So this is a +plain script: it disk-scans the tiles/ and exp/ trees against the count-floor +table (completeness.py) and the index, and emits run_report.json. It runs +automatically via the Snakefile's onsuccess/onerror hooks, and by hand any time +(``sp report``), mid-run included. + +It distinguishes whole-exposure absence (all runners missing — a real gap or a +deletion/forest bug) from tolerated per-CCD attrition (counts between floor and +expect), so a future deletion bug cannot be silently absorbed (finding 5). +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from completeness import COMPLETENESS, check_floor # noqa: E402 + +# stage -> (level, run_sp_ dir name) +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +def classify(stage, run_dir): + """'absent' | 'under' | 'attrition' | 'complete' for one unit's stage.""" + if not run_dir.is_dir(): + return "absent", [] + ok, details = check_floor(stage, run_dir) + if not ok: + # all mandatory runners empty -> whole-unit absence; else below floor + mand = [d for d in details if not d[4]] + if mand and all(n == 0 for _, n, *_ in mand): + return "absent", details + return "under", details + if any(n < expect for _, n, floor, expect, warn in details if not warn): + return "attrition", details + return "complete", details + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--status", default="manual") + p.add_argument("--out", type=Path, default=None) + args = p.parse_args() + + tiles, exps = [], [] + if args.index.exists(): + con = sqlite3.connect(args.index) + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures")] + con.close() + + missing_json = args.index.parent / "missing.json" + missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] + + report = {"status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, "stages": {}} + for stage, (level, prefix) in STAGE_DIR.items(): + if stage not in COMPLETENESS: + continue + units = tiles if level == "tile" else exps + sub = "tiles" if level == "tile" else "exp" + tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} + for u in units: + run_dir = args.run_dir / sub / u / "output" / prefix + verdict, _ = classify(stage, run_dir) + if verdict in ("complete", "attrition"): + tally[verdict] += 1 + else: + tally[verdict].append(u) + report["stages"][stage] = tally + + finals = [t for t in tiles + if (args.run_dir / "tiles" / t / f"final_cat-{t}.fits").exists()] + report["final_cats"] = {"present": len(finals), "of": len(tiles), + "missing": [t for t in tiles if t not in finals]} + + out = args.out or (args.index.parent / "run_report.json") + out.write_text(json.dumps(report, indent=2)) + print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " + f"cats -> {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py new file mode 100644 index 000000000..cce92e2c8 --- /dev/null +++ b/workflow/scripts/sp_rule.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Thin per-unit ShapePipe rule wrapper — the single entrypoint every rule calls. + +Under the one-allocation local executor + apptainer software-deployment, +Snakemake runs each rule's shell *inside* the container on the allocation's node. +So this script already runs in-container: it calls ``shapepipe_run`` directly +(never apptainer itself), which is why the workflow never hand-invokes apptainer. + +What it owns — only what is intrinsically ShapePipe's, nothing Snakemake already +does: + +1. **Per-unit work dir + v2.0 isolation furniture.** Each tile/exposure gets its + own ``$SP_RUN`` (``tiles//`` or ``exp//``). Isolation is by work-dir + *content*, the proven v2.0 mechanism — NOT ``-e/--exclusive``: + - a ``cfis`` symlink to the canonical config dir (so ``$SP_CONFIG`` / ``.mask`` + refs resolve); + - ``star_cat_exp`` / ``star_cat_tiles`` symlinks to the shared cat pool; + - for a tile: ``tile_numbers.txt`` = this tile ID (dot format, real data — + what get_images reads); + - for an exposure: a fabricated + ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-000-000.txt`` + = this exposure ID (v2.0 ``init_exp_work_dir`` pseudo-Fe), so the Gie + config's ``last:find_exposures_runner`` resolver finds exactly one unit. + +2. **Per-unit config copy.** Forces ``RUN_DATETIME = False`` (deterministic + ``output/run_sp_/`` the rule declares as its ``directory()`` output), + sets ``SMP_BATCH_SIZE = {threads}`` (fork width == cpus_per_task, so SLURM + packing density and true per-job process count are one number), and — only + when ``isolate`` (tile-scheme stages + exp split; NEVER get_images / exp_mask / + exp_psf, whose per-CCD/download numbering would make #746 turn tolerated + per-CCD attrition into a whole-exposure hard failure) — ``NUMBER_LIST = -``. + +3. **Log sync at START and end.** ``update_runs_log_file.py`` regenerates + ``log_run_sp.txt`` before ``shapepipe_run`` (after the pre-delete, so + resolution never sees a just-deleted dir) and again after. Skipped for ngmix + chunks: the log is already complete from the prior tile stages, and the N + concurrent chunk jobs share the tile work dir — a full log *rewrite* would + race, whereas ShapePipe's own single-line ``O_APPEND`` of each chunk's run is + atomic. ngmix chunk configs additionally carry *literal* INPUT_DIRs + (deterministic under RUN_DATETIME=False), so they need no log read at all. + +4. **Count-floor check** (``completeness.py`` — the ported bash ``complete_check`` + table, the single failure policy: no 3-class taxonomy, no whitelist). After + the run, count products per mandatory runner and exit nonzero below the floor. + ``keep-going`` isolates that unit's cone; per-CCD attrition between floor and + ``expect`` is tolerated. Stages absent from the table fall back to a + zero-output floor. + +Stage-specific verbs (grounded in ``runs/p3-batch1/job_p3_batch1.sh``): + +- ``--ngmix-chunk K --ngmix-nchunks N`` — read this tile's SExtractor object + count from its ``run_sp_tile_Sx`` output, split ``[1, N_obj]`` into N disjoint + closed ID ranges, and generate chunk K's config from + ``config_tile_Ng_template_batch.ini``. +- ``--n-split-max N`` — generate the merge config from + ``config_merge_sep_cats_template.ini`` with ``N_SPLIT_MAX = N``. +- ``--final-cat PATH`` — after ``make_cat``, copy the produced ``final_cat`` FITS + to PATH (the rule's declared, protected science-product output). + +Run by hand to reproduce exactly what a rule does (the escape hatch — it +pre-deletes the run dir, just like Snakemake's delete-before-rerun):: + + python sp_rule.py --stage exp_split --config config_exp_Sp.ini \\ + --unit 2243881 --level exp --run-dir $SP_RUN --config-src $SP_CONFIG \\ + --star-cats $STAR_CATS --threads 8 +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from completeness import COMPLETENESS, check_floor # noqa: E402 + + +def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: + """Per-unit work dir in the forest: tiles/ or exp/. + + Flat (no IDra/prefix sharding): the shard existed only to keep the bash + layer's ``ls exp/`` bearable, but Snakemake never globs the forest — the + run index drives every path — so flattening keeps single-wildcard rules. + ``run`` level (aggregation stages) work directly in ``run_dir``. + """ + if level == "tile": + return run_dir / "tiles" / unit + if level == "exp": + base = unit[:-1] if unit[-1].isalpha() else unit + return run_dir / "exp" / base + return run_dir + + +def materialise_unit(work: Path, level: str, unit: str, config_src: Path, + star_cats: Path) -> None: + """Create the v2.0 unit-isolation furniture in the work dir.""" + (work / "output").mkdir(parents=True, exist_ok=True) + + cfis = work / "cfis" # $SP_CONFIG -> canonical config dir (.mask refs resolve) + if not cfis.exists(): + cfis.symlink_to(config_src) + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + link = work / name + if not link.exists() and (star_cats / sub).exists(): + link.symlink_to(star_cats / sub) + + if level == "tile": + (work / "tile_numbers.txt").write_text(unit + "\n") # dot format, real data + elif level == "exp": + fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" + fe.mkdir(parents=True, exist_ok=True) + exp_numbers = fe / "exp_numbers-000-000.txt" + if not exp_numbers.exists(): + exp_numbers.write_text(unit + "\n") + + +def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: + """Force RUN_DATETIME=False, SMP_BATCH_SIZE=threads, and (if isolating) + NUMBER_LIST=-. + + ``NUMBER_LIST`` uses ShapePipe's image-number convention (dots -> dashes), + the ``set_config_number_list`` mechanism that replaced the retired + ``-e/--exclusive`` flag (#746). + """ + lines = text.splitlines() + out, in_file, saw_datetime, saw_numberlist, saw_smp = [], False, False, False, False + number = "-" + unit.replace(".", "-") + for line in lines: + stripped = line.strip() + if stripped.startswith("[") and stripped.endswith("]"): + in_file = stripped.upper() == "[FILE]" + if stripped.upper().startswith("RUN_DATETIME"): + out.append("RUN_DATETIME = False") + saw_datetime = True + continue + if stripped.upper().startswith("SMP_BATCH_SIZE"): + out.append(f"SMP_BATCH_SIZE = {threads}") + saw_smp = True + continue + if isolate and stripped.upper().startswith("NUMBER_LIST"): + out.append(f"NUMBER_LIST = {number}") + saw_numberlist = True + continue + out.append(line) + if in_file and stripped.upper() == "[FILE]" and isolate and not saw_numberlist: + out.append(f"NUMBER_LIST = {number}") + saw_numberlist = True + if not saw_datetime: + # ShapePipe reads RUN_DATETIME from [DEFAULT]; inject after its header. + patched, done = [], False + for line in out: + patched.append(line) + if not done and line.strip().upper() == "[DEFAULT]": + patched.append("RUN_DATETIME = False") + done = True + out = patched if done else ["[DEFAULT]", "RUN_DATETIME = False"] + out + if not saw_smp: + # SMP_BATCH_SIZE lives in [JOB]; inject after its header. + patched, done = [], False + for line in out: + patched.append(line) + if not done and line.strip().upper() == "[JOB]": + patched.append(f"SMP_BATCH_SIZE = {threads}") + done = True + out = patched if done else out + ["[JOB]", f"SMP_BATCH_SIZE = {threads}"] + return "\n".join(out) + "\n" + + +def run_prefix_from_text(text: str, config_label: str) -> str: + """The RUN_NAME (== run_sp_) the module will write under.""" + for line in text.splitlines(): + s = line.strip() + if s.upper().startswith("RUN_NAME"): + return s.split("=", 1)[1].strip() + raise SystemExit(f"no RUN_NAME in {config_label}") + + +# --- ngmix chunking -------------------------------------------------------- + +# Literal, deterministic INPUT_DIRs for the ngmix chunk (RUN_DATETIME=False), so +# concurrent chunks need no log read at all (finding 18). +NGMIX_LITERAL_INPUT = ( + "./output/run_sp_tile_Sx/sextractor_runner/output, " + "./output/run_sp_tile_PiViVi/psfex_interp_runner/output, " + "./output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, " + "./output/run_sp_tile_Mh_exp/merge_headers_runner/output") + + +def ngmix_id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + """Split ``[1, n_obj]`` into ``n_chunks`` disjoint *closed* ID ranges. + + SExtractor's ``NUMBER`` column (what ngmix reads as ``obj_id``) runs 1..N + contiguous, so covering ``[1, N]`` processes every object exactly once. The + first n_chunks-1 chunks get ``n_obj // n_chunks`` each; the last gets the + remainder — a *closed* upper bound at ``n_obj`` (never ``ID_OBJ_MAX = -1``: + ngmix treats ``id_obj_max <= 0`` as unbounded, which would double-count). + + Deviation from the nibi monolith (deliberate): the monolith sizes every + tile's chunks from the *average* object count across all tiles and leaves the + last chunk open (``-1``); here each tile is an independent job, so we size + from *this* tile's own count and close the last chunk. The merged catalogue + is identical either way (merge_sep_cats recombines all chunks, ngmix fits + each object independently) — this just removes the open-ended overflow. + """ + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + size = base + (rem if k == n_chunks else 0) + hi = lo + size - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def tile_object_count(work: Path) -> int: + """Object count for this tile from its run_sp_tile_Sx SExtractor catalogue. + + Matches ``get_number_objects.py``: the ``NAXIS2`` of the catalogue's last + HDU. Per-unit isolation means exactly one ``sexcat*.fits`` lives here. + """ + from astropy.io import fits + + sexcats = sorted((work / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not sexcats: + raise SystemExit( + f"[sp_rule] FATAL ngmix: no sexcat under {work}/output/run_sp_tile_Sx") + with fits.open(sexcats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def apply_ngmix_subs(text: str, chunk: int, lo: int, hi: int) -> str: + """Fill the ngmix batch template for one chunk (the monolith's sed recipe) + plus literal INPUT_DIRs so the chunk needs no shared-log read.""" + text = text.replace("NgXu", f"Ng{chunk}u").replace("X_interp", "psfex_interp") + out = [] + for line in text.splitlines(): + s = line.strip() + if s.startswith("ID_OBJ_MIN"): + out.append(f"ID_OBJ_MIN = {lo}") + elif s.startswith("ID_OBJ_MAX"): + out.append(f"ID_OBJ_MAX = {hi}") + elif s.startswith("INPUT_DIR") and "sextractor_runner" in s: + out.append(f"INPUT_DIR = {NGMIX_LITERAL_INPUT}") + else: + out.append(line) + return "\n".join(out) + + +def apply_merge_subs(text: str, n_split_max: int) -> str: + """Fill N_SPLIT_MAX in the merge template (the monolith's sed recipe).""" + out = [] + for line in text.splitlines(): + if line.strip().startswith("N_SPLIT_MAX"): + out.append(f"N_SPLIT_MAX = {n_split_max}") + else: + out.append(line) + return "\n".join(out) + + +# --- in-job count floor ---------------------------------------------------- + +def count_floor(run_out: Path, stage: str) -> None: + """Fail loudly if any mandatory runner is below its ``floor`` (completeness.py). + + Stages absent from the table skip this floor (only the zero-output floor + applies to them). + """ + if stage not in COMPLETENESS: + return + ok, details = check_floor(stage, run_out) + for runner, n, floor, expect, warn in details: + tag = "warn" if warn else ("OK" if n >= floor else " <-- BELOW floor") + print(f"[sp_rule] {runner}: {n}/{expect} (floor {floor}) {tag}", + file=sys.stderr) + if not ok: + print(f"[sp_rule] FATAL {stage}: count floor breached in {run_out}", + file=sys.stderr) + sys.exit(1) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--stage", required=True) + p.add_argument("--config", required=True, help="config filename in --config-src") + p.add_argument("--unit", required=True) + p.add_argument("--level", required=True, choices=["tile", "exp", "run"]) + p.add_argument("--run-dir", required=True, type=Path, help="$SP_RUN root") + p.add_argument("--config-src", required=True, type=Path, + help="dir holding the canonical configs") + p.add_argument("--star-cats", type=Path, default=None, + help="shared star-cat pool (tiles/ + exp/ subdirs)") + p.add_argument("--threads", type=int, default=1, + help="SMP_BATCH_SIZE == cpus_per_task (fork width)") + p.add_argument("--exp-forest", type=Path, default=None, + help="$SP_EXP: per-tile exposure forest (tile post-stages)") + p.add_argument("--no-isolate", action="store_true", + help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") + p.add_argument("--ngmix-chunk", type=int, default=None, + help="1-indexed ngmix chunk K (with --ngmix-nchunks)") + p.add_argument("--ngmix-nchunks", type=int, default=None, + help="total ngmix chunks N (with --ngmix-chunk)") + p.add_argument("--n-split-max", type=int, default=None, + help="N_SPLIT_MAX for the merge_sep_cats template") + p.add_argument("--final-cat", type=Path, default=None, + help="copy the produced final_cat FITS here (make_cat)") + args = p.parse_args() + + ngmix_mode = args.ngmix_chunk is not None + if ngmix_mode and args.ngmix_nchunks is None: + raise SystemExit("--ngmix-chunk requires --ngmix-nchunks") + + work = unit_work_dir(args.run_dir, args.level, args.unit) + materialise_unit(work, args.level, args.unit, args.config_src, + args.star_cats or args.run_dir) + + # Build the config copy. ngmix chunks and merge fill template placeholders + # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, + # [+ NUMBER_LIST]). + src_text = (args.config_src / args.config).read_text() + isolate = not args.no_isolate + if ngmix_mode: + n_obj = tile_object_count(work) + lo, hi = ngmix_id_ranges(n_obj, args.ngmix_nchunks)[args.ngmix_chunk - 1] + print(f"[sp_rule] ngmix chunk {args.ngmix_chunk}/{args.ngmix_nchunks}: " + f"n_obj={n_obj} -> ID_OBJ [{lo}, {hi}]", file=sys.stderr) + src_text = apply_ngmix_subs(src_text, args.ngmix_chunk, lo, hi) + cfg_name = f"config_tile_Ng{args.ngmix_chunk}u.ini" # distinct per chunk + elif args.n_split_max is not None: + src_text = apply_merge_subs(src_text, args.n_split_max) + cfg_name = args.config + else: + cfg_name = args.config + + cfg_mod_dir = work / "cfis_mod" + cfg_mod = cfg_mod_dir / cfg_name + cfg_text = normalize_config_text(src_text, args.unit, isolate, args.threads) + cfg_mod_dir.mkdir(parents=True, exist_ok=True) + cfg_mod.write_text(cfg_text) + + prefix = run_prefix_from_text(cfg_text, str(cfg_mod)) + run_out = work / "output" / prefix + + # Pre-delete the deterministic run dir (delete-before-rerun for the hand-run + # escape hatch; ShapePipe's FileHandler.mkdir raises on an existing run dir). + if run_out.exists(): + shutil.rmtree(run_out) + + env = dict(os.environ) + env["SP_RUN"] = str(work) + env["SP_CONFIG"] = str(work / "cfis") # canonical dir (.mask refs), not cfis_mod + if args.exp_forest: + env["SP_EXP"] = str(args.exp_forest) + # Thread caps also come via apptainer-args; keep here for the hand-run path. + env.setdefault("OMP_NUM_THREADS", "1") + env.setdefault("OPENBLAS_NUM_THREADS", "1") + env.setdefault("MKL_NUM_THREADS", "1") + env.setdefault("NUMEXPR_NUM_THREADS", "1") + env.setdefault("MALLOC_ARENA_MAX", "2") + env.setdefault("MALLOC_TRIM_THRESHOLD_", "0") + + # Log sync before the run (after pre-delete). Skipped for ngmix chunks: N + # concurrent chunks would race a full rewrite (literal INPUT_DIRs make it moot). + if not ngmix_mode: + subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) + + print(f"[sp_rule] stage={args.stage} unit={args.unit} work={work}", + file=sys.stderr) + rc = subprocess.call(["shapepipe_run", "-c", str(cfg_mod)], + cwd=str(work), env=env) + if rc != 0: + print(f"[sp_rule] WARN shapepipe_run exit {rc} — checking count floor", + file=sys.stderr) + + if not ngmix_mode: + subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) + + # Zero-output floor for stages without a count table; count floor otherwise. + if args.stage in COMPLETENESS: + count_floor(run_out, args.stage) + else: + produced = list(run_out.glob("**/output/*")) if run_out.exists() else [] + if not produced: + print(f"[sp_rule] FATAL {args.stage} {args.unit}: zero output in {run_out}", + file=sys.stderr) + sys.exit(1) + + # make_cat: publish the science product to its declared, protected path. + if args.final_cat is not None: + finals = sorted(run_out.glob("**/output/final_cat*.fits")) + if not finals: + print(f"[sp_rule] FATAL {args.stage} {args.unit}: no final_cat under {run_out}", + file=sys.stderr) + sys.exit(1) + args.final_cat.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(finals[0], args.final_cat) + print(f"[sp_rule] final_cat -> {args.final_cat}", file=sys.stderr) + + print(f"[sp_rule] OK {args.stage} {args.unit}", file=sys.stderr) + + +if __name__ == "__main__": + main() From 187950c4ec95035b0e1b40ffaf69841122c31f89 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 15:57:40 -0400 Subject: [PATCH 002/100] fix(orchestration): resolve {threads} at rule level, not inside params.cmd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Snakemake formats params once, so a {threads} placeholder inside params.cmd survives literally and SMP_BATCH_SIZE never resolves (fork width silently 1, serial modules — the p3-first10 failure mode). Move '--threads {threads}' into each rule's shell: string; tile_exp_forest takes none (build_forest.py is single-threaded symlinking and has no such flag). Completeness table doc touch-ups ride along. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- workflow/Snakefile | 6 ++++-- workflow/rules/exposure.smk | 8 ++++---- workflow/rules/prepare.smk | 6 +++--- workflow/rules/tile.smk | 15 ++++++++------- workflow/scripts/completeness.py | 8 ++++++-- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 04722a73c..09d218318 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -60,13 +60,15 @@ def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download - numbering must not be constrained. ``{threads}`` is filled by Snakemake and + numbering must not be constrained. ``--threads {threads}`` is appended in + each rule's ``shell:`` string (NOT here: Snakemake formats the shell string + once, so a ``{threads}`` inside params.cmd would survive literally) and becomes SMP_BATCH_SIZE (fork width == cpus_per_task). """ cmd = ( f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} --threads {{threads}}" + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS}" ) if not isolate: cmd += " --no-isolate" diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 50403d386..3e934b185 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -29,7 +29,7 @@ rule exp_get_images: cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; # persistent (see module docstring on temp()). @@ -41,7 +41,7 @@ rule exp_split: params: cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). rule exp_mask: @@ -54,7 +54,7 @@ rule exp_mask: cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. @@ -67,4 +67,4 @@ rule exp_psf: cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 2911c3b19..4ca369ec9 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -19,7 +19,7 @@ rule tile_get_images: cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", "tile", wc.tile, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" rule tile_uncompress: input: @@ -30,7 +30,7 @@ rule tile_uncompress: cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the # data-derived tile->exposure edge build_index.py aggregates (as a plain script). @@ -43,7 +43,7 @@ rule tile_find_exposures: cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Star catalogues for masking are pre-generated offline (create_star_cat.py on a # networked login node — the one genuinely networked prepare step). They are diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 845167710..8abfdd18b 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -35,6 +35,7 @@ rule tile_exp_forest: cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") shell: + # no --threads: build_forest.py is single-threaded symlinking, takes no such flag "{params.cmd}" # Merge single-exposure WCS headers into the tile-level sqlite log @@ -52,7 +53,7 @@ rule tile_merge_headers: "tile", wc.tile, isolate=True, exp_forest=FOREST.format(tile=wc.tile)) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Mask tiles; this tile's star cat is a declared input (per-unit re-key). rule tile_mask: @@ -65,7 +66,7 @@ rule tile_mask: cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # SExtractor object detection on the tile. rule tile_detect: @@ -78,7 +79,7 @@ rule tile_detect: cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF # models + split through the forest — those dirs are explicit inputs). @@ -95,7 +96,7 @@ rule tile_vignets: "tile", wc.tile, isolate=True, exp_forest=FOREST.format(tile=wc.tile)) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # ngmix shape measurement — static N chunks. Each chunk computes its own closed # object-ID range in-job from the tile's own sexcat (a declared input); chunks @@ -111,7 +112,7 @@ rule tile_ngmix: "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" def ngmix_chunks(wc): return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" @@ -129,7 +130,7 @@ rule tile_merge_cats: "tile", wc.tile, isolate=True, extra=f"--n-split-max {NGMIX_CHUNKS}"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Build the final catalogue: the run's science product. protected() so a # code/params-drift rerun cannot silently discard a finished catalogue — the @@ -144,4 +145,4 @@ rule tile_make_cat: "tile", wc.tile, isolate=True, extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index f35e81aee..d52ad8d0a 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -30,12 +30,16 @@ # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} COMPLETENESS = { # --- tile prepare (phase A) --- - "tile_get_images": {"get_images_runner": dict(expect=4, floor=4)}, + # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 + # for the canfar vos flavor; the nibi symlink configs produce one file per + # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), + # verified against the p3-batch1 baseline tree (100 files / 50 tiles). + "tile_get_images": {"get_images_runner": dict(expect=2, floor=2)}, "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, # --- exposure chain --- - "exp_get_images": {"get_images_runner": dict(expect=6, floor=6)}, + "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, "exp_psf": { From ac6bddd7f5eddf1c9e514a4e759a1c4db7660dff Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:07:59 -0400 Subject: [PATCH 003/100] fix(orchestration): carry original exposure names through the index; star-cat store naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three P0-blocking fixes found by running against real data: - exposures(name): the fabricated per-unit exp_numbers list must carry the original exposure name verbatim (2605805p — get_images matches .fits.fz; the bare dedup id matches nothing). The index now stores both; exp_get_images passes --exp-name from the parse-time dict. - tile_star_cat: the pre-generated store names tiles in ShapePipe's image-number convention (dots->dashes); translate when linking. - profile: PYTHONPATH pinned to this branch's src/ (identical to develop@97e16d50 — orchestration commits never touch src/); NOT shapepipe-prod (drifted to a PR branch mid-run, and the live p3-batch1 job reads it) and not the sif default (frozen pre-#843). Run scoped to the 210/211 overlap quad (19 unique exposures, 15 shared — exercises structural dedup; the zero-overlap 196 quad is kept for the append-invariant test). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- profiles/nibi/config.yaml | 12 ++++++----- workflow/Snakefile | 8 +++++--- workflow/config.yaml | 9 ++++++--- workflow/rules/exposure.smk | 3 ++- workflow/rules/prepare.smk | 4 +++- workflow/scripts/build_index.py | 35 ++++++++++++++++++--------------- workflow/scripts/sp_rule.py | 17 ++++++++++++---- 7 files changed, 55 insertions(+), 33 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index e977803c1..07f68833a 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -23,11 +23,13 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp -# PYTHONPATH SETTLED CALL 3 — P0 pins the prod worktree src so science -# content matches the bash validation runs (which use this -# override). Production later rebuilds the sif at the exact -# validation commit and DROPS this --env line. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-prod/src --home /home/cdaley --bind /project --bind /scratch" +# PYTHONPATH SETTLED CALL 3 — P0 pins THIS branch's src/ (identical to +# develop@97e16d50: orchestration commits never touch src/). +# NOT shapepipe-prod (drifted to a PR branch mid-run, and the +# live p3-batch1 job reads it) and not the sif default +# (frozen pre-#843). Production later rebuilds the sif at the +# validated commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on diff --git a/workflow/Snakefile b/workflow/Snakefile index 09d218318..6e391bda2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -39,11 +39,13 @@ with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] # --- parse-time index load (compute phase only) ---------------------------- -# EXP: global set of exposure base-ids. TILE_EXP: tile -> [exp_ids]. -EXP, TILE_EXP = set(), {} +# EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes +# verbatim into the fabricated per-unit exp_numbers list so get_images matches +# .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. +EXP, TILE_EXP = {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) - EXP = {r[0] for r in _con.execute("SELECT exp_id FROM exposures")} + EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(tile, []).append(exp) _con.close() diff --git a/workflow/config.yaml b/workflow/config.yaml index 53542d1c2..ec529c4dc 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,14 +5,17 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -tile_list: /scratch/cdaley/shapepipe-output/snakemake-p0/tiles4.txt +# 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the +# structural exposure dedup (the 196 quad had zero overlap; kept for the +# append-invariant test). +tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: # /tiles/// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/snakemake-p0 +run_dir: /scratch/cdaley/shapepipe-output/smk-p0 # The canonical config chain (example/cfis in the repo, or a run-specific copy). config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis @@ -23,7 +26,7 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # The run index. Durable, low-volume products (index, missing.json, run_report, # final_cat) should land on /project, not /scratch (60-day purge, finding 15). -index_db: /project/def-mjhudson/cdaley/shapepipe-output/snakemake-p0/index/run_index.sqlite +index_db: /project/def-mjhudson/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 3e934b185..f39f6be23 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -27,7 +27,8 @@ rule exp_get_images: directory(f"{EXP_OUT}/run_sp_exp_Gie") params: cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", - "exp", wc.exp, isolate=False) + "exp", wc.exp, isolate=False, + extra=f"--exp-name {EXP[wc.exp]}") shell: "{params.cmd} --threads {threads}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 4ca369ec9..530ac9b31 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -62,6 +62,8 @@ rule tile_star_cat: output: str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") params: - src=lambda wc: str(STAR_CATS / "tiles" / f"star_cat-{wc.tile}.fits") + # the store names tiles in ShapePipe's image-number convention (dots->dashes) + src=lambda wc: str(STAR_CATS / "tiles" / + f"star_cat-{wc.tile.replace('.', '-')}.fits") shell: "ln -sf {params.src} {output}" diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 13783314a..7ba865d94 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -37,19 +37,22 @@ from pathlib import Path -def read_exposure_list(exp_numbers_file: Path) -> list[str]: - """Return the exp_base IDs listed in one tile's find_exposures output. - - Each line is an exposure name like ``2243881p``; we strip a trailing - alphabetic suffix so the base ID is the dedup key. +def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: + """Return ``(exp_id, name)`` pairs from one tile's find_exposures output. + + Each line is an exposure *name* like ``2243881p``; the bare base ID (suffix + stripped) is the dedup key everywhere in the DAG, but the original name is + kept in the index — the fabricated per-unit ``exp_numbers`` list must carry + it verbatim (``get_images`` matches ``.fits.fz`` in the store; the + bare ID matches nothing). """ - ids = [] + pairs = [] for line in exp_numbers_file.read_text().splitlines(): name = line.strip() if not name: continue - ids.append(name[:-1] if name[-1].isalpha() else name) - return ids + pairs.append((name[:-1] if name[-1].isalpha() else name, name)) + return pairs def build(tile_ids: list[str], run_dir: Path, db_path: Path, @@ -70,7 +73,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, DROP TABLE IF EXISTS exposures; DROP TABLE IF EXISTS tile_exposures; CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); - CREATE TABLE exposures(exp_id TEXT PRIMARY KEY); + CREATE TABLE exposures(exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); CREATE TABLE tile_exposures( tile_id TEXT, exp_id TEXT, PRIMARY KEY (tile_id, exp_id)); @@ -78,7 +81,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, ) missing = [] - all_exposures: set[str] = set() + all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: ra_dir = tile_id.split(".")[0] idra, iddec = tile_id.split(".") @@ -89,16 +92,16 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, if not exp_file.exists(): missing.append(tile_id) continue - exp_ids = read_exposure_list(exp_file) + exp_pairs = read_exposure_list(exp_file) con.execute("INSERT INTO tiles VALUES (?,?,?)", - (tile_id, ra_dir, len(exp_ids))) - for exp_id in exp_ids: - all_exposures.add(exp_id) + (tile_id, ra_dir, len(exp_pairs))) + for exp_id, _name in exp_pairs: con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", (tile_id, exp_id)) + all_exposures.update(exp_pairs) - con.executemany("INSERT OR IGNORE INTO exposures VALUES (?)", - [(e,) for e in sorted(all_exposures)]) + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + sorted(all_exposures)) con.commit() con.close() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py index cce92e2c8..b114fad08 100644 --- a/workflow/scripts/sp_rule.py +++ b/workflow/scripts/sp_rule.py @@ -93,7 +93,7 @@ def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: def materialise_unit(work: Path, level: str, unit: str, config_src: Path, - star_cats: Path) -> None: + star_cats: Path, exp_name: str | None = None) -> None: """Create the v2.0 unit-isolation furniture in the work dir.""" (work / "output").mkdir(parents=True, exist_ok=True) @@ -112,8 +112,13 @@ def materialise_unit(work: Path, level: str, unit: str, config_src: Path, fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" fe.mkdir(parents=True, exist_ok=True) exp_numbers = fe / "exp_numbers-000-000.txt" - if not exp_numbers.exists(): - exp_numbers.write_text(unit + "\n") + # Written UNCONDITIONALLY (cheap, deterministic): an exists-guard once + # pinned a stale pre-fix file with the bare base id. Content is the + # ORIGINAL exposure name (with its 'p'-style suffix, from the index) — + # the on-disk store is .fits.fz; the bare base id matches nothing. + # v2.0 copies the name verbatim from the tile's Fe output; the index + # carries it for us. + exp_numbers.write_text((exp_name or unit) + "\n") def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: @@ -296,6 +301,10 @@ def main() -> None: help="SMP_BATCH_SIZE == cpus_per_task (fork width)") p.add_argument("--exp-forest", type=Path, default=None, help="$SP_EXP: per-tile exposure forest (tile post-stages)") + p.add_argument("--exp-name", default=None, + help="original exposure name incl. suffix (e.g. 2605805p); " + "written into the fabricated exp_numbers list so " + "get_images finds .fits.fz in the store") p.add_argument("--no-isolate", action="store_true", help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") p.add_argument("--ngmix-chunk", type=int, default=None, @@ -314,7 +323,7 @@ def main() -> None: work = unit_work_dir(args.run_dir, args.level, args.unit) materialise_unit(work, args.level, args.unit, args.config_src, - args.star_cats or args.run_dir) + args.star_cats or args.run_dir, exp_name=args.exp_name) # Build the config copy. ngmix chunks and merge fill template placeholders # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, From 9b0456e72438ab7f6f10e304d231100c0e3423ae Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:09:45 -0400 Subject: [PATCH 004/100] fix(orchestration): exposure-name pass-through, star-cat dirs, P0 run config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - get_images looks up .fits.fz: the store keeps the CFHT 'p' suffix the index's base-id key strips. exposures gains a name column; rules pass --exp-name; the wrapper writes the ORIGINAL name into the fabricated exp_numbers list — and writes it unconditionally (an exists-guard pinned a stale pre-fix file). - Star cats are consumed as DIRECTORIES via the wrapper's $SP_RUN symlinks (the v2.0 mechanism; exp cats are per-CCD, 40/exposure) — dropped the per-unit star-cat file rules that linked nonexistent names nothing read. - get_images completeness counts follow the nibi symlink configs (tile 2, exp 3; the v2.0 4/6 were the canfar vos flavor), verified against p3-batch1. - P0 run config: 210/211 quad (19 unique exps, 13 shared across tiles — exercises structural dedup; the 196 quad had zero overlap), PYTHONPATH pinned to this branch's src (= develop@97e16d50; shapepipe-prod drifted mid-run). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 4 +--- workflow/rules/prepare.smk | 31 ++++++++++--------------------- workflow/rules/tile.smk | 3 +-- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 6e391bda2..774ae3ad2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -109,9 +109,7 @@ rule prepare_tiles: # index exists. Star cats are re-keyed per unit (see prepare.smk). rule prepare_exposures: input: - expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)), - expand(str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), exp=sorted(EXP)), - expand(str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), tile=TILES) + expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)) # --- report hooks ----------------------------------------------------------- # run_report is NOT a DAG node (a descendant of every job would be poisoned by diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 530ac9b31..d8698c826 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -46,24 +46,13 @@ rule tile_find_exposures: "{params.cmd} --threads {threads}" # Star catalogues for masking are pre-generated offline (create_star_cat.py on a -# networked login node — the one genuinely networked prepare step). They are -# re-keyed PER UNIT here: each tile/exposure's cat is its own DAG node, so -# appending tile 51 adds nodes rather than mutating a run-level output that would -# dirty every finished mask (the growth-invariant fix, finding 11). -rule exp_star_cat: - output: - str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits") - params: - src=lambda wc: str(STAR_CATS / "exp" / f"star_cat-{wc.exp}.fits") - shell: - "ln -sf {params.src} {output}" - -rule tile_star_cat: - output: - str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") - params: - # the store names tiles in ShapePipe's image-number convention (dots->dashes) - src=lambda wc: str(STAR_CATS / "tiles" / - f"star_cat-{wc.tile.replace('.', '-')}.fits") - shell: - "ln -sf {params.src} {output}" +# networked login node) and consumed as DIRECTORIES: the mask configs read +# INPUT_DIR $SP_RUN/star_cat_{exp,tiles} with per-CCD numbering (exp cats are +# star_cat--.fits, 40/exposure; tile cats star_cat--.fits) +# — the v2.0 mechanism. The wrapper materialises the two dir symlinks in every +# unit work dir (sp_rule.materialise_unit); there is NO per-unit star-cat DAG +# node: the store is pre-run input like the image store, and a missing cat fails +# the mask stage's count-floor loudly. (Earlier per-unit file rules linked names +# that don't exist — one file per exposure vs the store's 40 — and nothing read +# them: the configs only see the dir symlinks.) When star-cat GENERATION moves +# in-workflow (P2), it becomes real per-unit rules producing into the store. diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8abfdd18b..1bf7d4244 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -55,11 +55,10 @@ rule tile_merge_headers: shell: "{params.cmd} --threads {threads}" -# Mask tiles; this tile's star cat is a declared input (per-unit re-key). +# Mask tiles (star cats: pre-run store, read via the wrapper's dir symlink). rule tile_mask: input: git=rules.tile_get_images.output, - star=str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), output: directory(f"{TILE_OUT}/run_sp_tile_Ma") params: From 6fe141fc01221eb06b3016a883999195369282fe Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:35:07 -0400 Subject: [PATCH 005/100] fix(orchestration): {output} placeholder inside params.cmd survives literally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same single-format trap as {threads} (187950c4): Snakemake formats the shell string once, so the escaped {{output}} in tile_exp_forest's params reached build_forest as a literal './{output}' — all four forest jobs racing one garbage dir (FileExistsError at 50% of the P0 run). Move --forest {output} into the shell string. Swept the rules for further escaped placeholders: none. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ac/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- workflow/rules/tile.smk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 1bf7d4244..d36e91d95 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -32,11 +32,14 @@ rule tile_exp_forest: output: directory(FOREST) params: + # --forest {output} lives in the shell string: Snakemake formats shell + # ONCE — an {output} placeholder inside params.cmd survives literally + # (same trap as {threads}; all four forest jobs then race one './{output}'). cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") + f"--run-dir {RUN_DIR} --index {INDEX_DB}") shell: - # no --threads: build_forest.py is single-threaded symlinking, takes no such flag - "{params.cmd}" + # no --threads: build_forest.py is single-threaded symlinking + "{params.cmd} --forest {output}" # Merge single-exposure WCS headers into the tile-level sqlite log # (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng From a110d89faeca7f0ac6bb2bf6a0ac79862ea044fe Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:42:12 -0400 Subject: [PATCH 006/100] =?UTF-8?q?fix(orchestration):=20shard=20the=20exp?= =?UTF-8?q?osure=20forest=20=E2=80=94=20the=20module=20glob's=20shape=20is?= =?UTF-8?q?=20not=20cosmetic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exp_utils.get_exp_output_files hardwires the v2.0 sharded layout into its $SP_EXP glob (///output/run_sp_*/...); a flat forest fails every tile gather stage with 'No split_exp_runner output found'. The exposure STORE stays flat (nothing globs it — the index drives every path); only the module-facing forest view carries the 2-digit shard. Verified by hand-running tile_merge_headers on 210.296 (1/1 OK). Co-Authored-By: Claude Fable 5 --- workflow/scripts/build_forest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 82df8b0c6..86dc4354e 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -3,10 +3,14 @@ A plain script (not a run: block) so the tile chain stays group-compatible. Reads the tile's exposures from run_index.sqlite and symlinks each exposure's -``exp//output`` into ``//output`` by exact name (no glob), so -the tile modules' $SP_EXP globs resolve to exactly this tile's exposures' -products. The forest is a convenience view; the DAG edge to the exposures is -declared in the rule's input (tile.smk), not here. +``exp//output`` into ``///output`` by exact name (no +glob). The 2-digit ```` shard level is NOT cosmetic: ShapePipe's +``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its +$SP_EXP glob (``///output/run_sp_*/...``), so a flat +forest makes every tile gather stage fail "No split_exp_runner output found". +(The exposure STORE stays flat — only the module-facing view needs the shard.) +The forest is a convenience view; the DAG edge to the exposures is declared in +the rule's input (tile.smk), not here. """ import argparse @@ -30,7 +34,7 @@ def main() -> None: args.forest.mkdir(parents=True, exist_ok=True) for e in exps: src = args.run_dir / "exp" / e / "output" - dst = args.forest / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) if dst.is_symlink() or dst.exists(): dst.unlink() From 28c10d8cdca5947184fb5a2d2f831e84468b2c95 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:47:15 -0400 Subject: [PATCH 007/100] fix(orchestration): carry a wrapper-scripts content hash into every rule's params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default rerun-triggers' code trigger hashes only the rule's own shell string — NOT external scripts it calls. Bitten live: the forest-shard fix changed build_forest.py but no rule text, so stale flat forests survived a rerun and tile_merge_headers kept failing. SCRIPT_HASH (md5 of sp_rule.py + completeness.py + build_forest.py, computed at parse time) rides into params via an accept-and-ignore --script-hash flag, so script edits propagate exactly like code edits under the full default triggers. One-time cost: every rule's params change once, re-running completed work — the same tradeoff as any code change under full triggers, and the P0-scale rerun doubles as a live test of rerun semantics. Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 14 +++++++++++++- workflow/rules/tile.smk | 3 ++- workflow/scripts/build_forest.py | 5 +++++ workflow/scripts/sp_rule.py | 5 +++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 774ae3ad2..3c869482c 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -70,7 +70,8 @@ def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, cmd = ( f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS}" + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} " + f"--script-hash {SCRIPT_HASH}" ) if not isolate: cmd += " --no-isolate" @@ -86,6 +87,17 @@ def tile_dir(tile): def exp_dir(exp): return RUN_DIR / "exp" / exp +# Content hash of the wrapper scripts, computed once at parse time. Exposed as a +# param on every rule (via sp_rule() and tile_exp_forest) so the default +# rerun-triggers' *params* trigger covers script edits: the `code` trigger only +# hashes the rule's own shell string, NOT external scripts it calls — without +# this, a wrapper/forest-script fix silently leaves stale outputs in place +# (bitten live: the forest-shard fix rebuilt nothing on rerun). +import hashlib as _hashlib +SCRIPT_HASH = _hashlib.md5(b"".join( + (SCRIPTS / n).read_bytes() + for n in ("sp_rule.py", "completeness.py", "build_forest.py"))).hexdigest()[:12] + include: "rules/prepare.smk" include: "rules/exposure.smk" include: "rules/tile.smk" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d36e91d95..04cc4c6c9 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -36,7 +36,8 @@ rule tile_exp_forest: # ONCE — an {output} placeholder inside params.cmd survives literally # (same trap as {threads}; all four forest jobs then race one './{output}'). cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB}") + f"--run-dir {RUN_DIR} --index {INDEX_DB} " + f"--script-hash {SCRIPT_HASH}") shell: # no --threads: build_forest.py is single-threaded symlinking "{params.cmd} --forest {output}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 86dc4354e..e945820e0 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -24,6 +24,11 @@ def main() -> None: p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--index", required=True, type=Path) p.add_argument("--forest", required=True, type=Path) + p.add_argument("--script-hash", default=None, + help="ignored at runtime: carries the wrapper-scripts content " + "hash into the rule's params so the params rerun-trigger " + "covers script edits (the code trigger only hashes the " + "rule's own shell string)") args = p.parse_args() con = sqlite3.connect(args.index) diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py index b114fad08..8ee71c693 100644 --- a/workflow/scripts/sp_rule.py +++ b/workflow/scripts/sp_rule.py @@ -315,6 +315,11 @@ def main() -> None: help="N_SPLIT_MAX for the merge_sep_cats template") p.add_argument("--final-cat", type=Path, default=None, help="copy the produced final_cat FITS here (make_cat)") + p.add_argument("--script-hash", default=None, + help="ignored at runtime: carries the wrapper-scripts content " + "hash into the rule's params so the params rerun-trigger " + "covers script edits (the code trigger only hashes the " + "rule's own shell string)") args = p.parse_args() ngmix_mode = args.ngmix_chunk is not None From a273e76369119113d05251c3c8b33beb3e1ec654 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 19:59:57 -0400 Subject: [PATCH 008/100] fix(orchestration): keep snakemake state and index off /project (hard-full quota) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit def-mjhudson /project sits at its 27/27 TiB ceiling: a .snakemake metadata write died mid-run (killing an otherwise-green invocation after all 8 ngmix chunks of the first tile passed), and by end of night even git and file edits on /project failed. bin/sp now runs snakemake with --directory -state on /scratch (state only — data paths are absolute), the Snakefile resolves its configfile relative to itself so --directory can't break it, and the run index lives with the run on /scratch until project space is reclaimed (finding 15's /project placement stays the design intent; noted in config). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 5 ++++- workflow/bin/sp | 7 ++++++- workflow/config.yaml | 8 +++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3c869482c..5d93a1810 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -21,7 +21,10 @@ DAG that produces it). import sqlite3 from pathlib import Path -configfile: "workflow/config.yaml" +# Resolved relative to THIS file, not the working directory: snakemake runs with +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project +# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +configfile: str(Path(workflow.snakefile).parent / "config.yaml") # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. diff --git a/workflow/bin/sp b/workflow/bin/sp index f226aa441..d1f105548 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -34,9 +34,14 @@ source "$VENV/bin/activate" # Minimal scalar reader for workflow/config.yaml (key: value, no nesting). cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +# Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO +# THE RUN on /scratch, never on /project (hard 27/27 TiB group quota killed a +# metadata write mid-run live). --directory only moves state: all data paths are +# absolute, and the Snakefile resolves its own configfile. +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" -sm() { snakemake --profile "$PROFILE" "$@"; } +sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } build_index() { python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" diff --git a/workflow/config.yaml b/workflow/config.yaml index ec529c4dc..f92cbb69f 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -24,9 +24,11 @@ config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis # Star catalogues for masking, pre-generated (network step done in prepare): star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats -# The run index. Durable, low-volume products (index, missing.json, run_report, -# final_cat) should land on /project, not /scratch (60-day purge, finding 15). -index_db: /project/def-mjhudson/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +# The run index. Design intent (finding 15) is durable products on /project — +# BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and +# report live with the run on /scratch until space is reclaimed. Mind the +# 60-day purge for anything that must survive. +index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). From 0010def6c3928185b3c84334983d6625538fd749 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 17 Jul 2026 01:45:14 -0400 Subject: [PATCH 009/100] fix(orchestration): drop exp_mask's dead per-unit star-cat input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-unit star_cat file rules were removed (the store is per-CCD and the mask config reads it as a dir symlink), but exp_mask kept declaring exp/{exp}/star_cat-{exp}.fits as input. No producer exists, so the main DAG became unbuildable the moment the tile list grew past the prepared set — MissingInputException on every new exposure. Dormant for P0 (all 19 exposures pre-staged); caught live by the append-invariant test (196 quad appended: DAG build died before scheduling anything). With the fix the append dry-run schedules exactly the appended cone (185 jobs: 30 new exposures x4 stages once each, 4 tile chains) and nothing for the finished tiles; restored-state dry-run returns to 'Nothing to be done'. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0188niYp9MmhLmZou1KXWbye --- workflow/rules/exposure.smk | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index f39f6be23..82cdb4cd4 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -44,11 +44,14 @@ rule exp_split: shell: "{params.cmd} --threads {threads}" -# Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). +# Mask per-CCD. Star cats are NOT a per-unit input: the store is per-CCD +# (star_cat--.fits) and the mask config reads it as a DIR symlink +# (see prepare.smk) — a per-unit file input here has no producer and makes the +# DAG unbuildable the moment a new exposure is appended (caught live by the +# append-invariant test). A missing cat fails the mask count-floor loudly. rule exp_mask: input: rules.exp_split.output, - star=str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), output: directory(f"{EXP_OUT}/run_sp_exp_Ma") params: From 833aa9369232d37f0ccfeb35fecad1ef223fc00c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 21:50:51 -0400 Subject: [PATCH 010/100] fix(orchestration): nibi-flavor completeness counts; file-level attrition reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sextractor writes 3 files/CCD on nibi (120/exp, not v2.0's 80); vignetmaker run_2 writes 5 sqlites/tile (not canfar's 4) — both verified against the P0 tree and the bash baseline. run_report now aggregates found-vs-expected at file granularity per runner, so warn-runner attrition (psfex_interp) is reported instead of hidden behind unit-level completeness. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- workflow/scripts/completeness.py | 10 +++++++-- workflow/scripts/run_report.py | 36 ++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index d52ad8d0a..b2246abef 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -42,8 +42,11 @@ "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree + # AND the bash baseline (both 120/exposure). "exp_psf": { - "sextractor_runner": dict(expect=80, floor=2), + "sextractor_runner": dict(expect=120, floor=2), "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), "psfex_runner": dict(expect=80, floor=2), "psfex_interp_runner": dict(expect=40, floor=0, warn=True), @@ -56,7 +59,10 @@ "tile_vignets": { "psfex_interp_runner": dict(expect=1, floor=1), "vignetmaker_runner_run_1": dict(expect=1, floor=1), - "vignetmaker_runner_run_2": dict(expect=4, floor=4), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern + # (expect=floor: all-or-nothing, every vignette feeds ngmix). + "vignetmaker_runner_run_2": dict(expect=5, floor=5), }, "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index bb3300405..683433ecb 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -12,6 +12,14 @@ It distinguishes whole-exposure absence (all runners missing — a real gap or a deletion/forest bug) from tolerated per-CCD attrition (counts between floor and expect), so a future deletion bug cannot be silently absorbed (finding 5). + +Attrition is counted at file granularity, not unit granularity: each stage's +``products`` block aggregates found-vs-expected file counts per runner across +all passing units, with a per-unit shortfall map where files are missing. +``warn`` runners (psfex_interp) are exempt from *failing* a unit, not from +being *reported* — the P0 validation found 46/760 psfex_interp CCDs missing +(6%) behind a unit-level "zero attrition" claim, which is the failure mode +this granularity exists to prevent. """ import argparse @@ -52,7 +60,9 @@ def classify(stage, run_dir): if mand and all(n == 0 for _, n, *_ in mand): return "absent", details return "under", details - if any(n < expect for _, n, floor, expect, warn in details if not warn): + # warn runners are exempt from FAILING a unit (check_floor), not from + # attrition reporting — psfex_interp is precisely the runner that attrits. + if any(n < expect for _, n, _, expect, _ in details): return "attrition", details return "complete", details @@ -83,13 +93,28 @@ def main() -> None: units = tiles if level == "tile" else exps sub = "tiles" if level == "tile" else "exp" tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} + # File-level aggregate per runner over PASSING units (complete/attrition); + # under/absent units are enumerated by name above, not folded in here — + # mixing them would double-count whole-unit absence as attrition. + products = {r: {"found": 0, "expect": 0, "by_unit": {}} + for r in COMPLETENESS[stage]} for u in units: run_dir = args.run_dir / sub / u / "output" / prefix - verdict, _ = classify(stage, run_dir) + verdict, details = classify(stage, run_dir) if verdict in ("complete", "attrition"): tally[verdict] += 1 + for runner, n, _, expect, _ in details: + agg = products[runner] + agg["found"] += n + agg["expect"] += expect + if n < expect: + agg["by_unit"][u] = expect - n else: tally[verdict].append(u) + for agg in products.values(): + if not agg["by_unit"]: + del agg["by_unit"] + tally["products"] = products report["stages"][stage] = tally finals = [t for t in tiles @@ -101,6 +126,13 @@ def main() -> None: out.write_text(json.dumps(report, indent=2)) print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " f"cats -> {out}", file=sys.stderr) + for stage, tally in report["stages"].items(): + for runner, agg in tally["products"].items(): + if agg["found"] < agg["expect"]: + pct = 100 * (agg["expect"] - agg["found"]) / agg["expect"] + print(f"[run_report] attrition {stage}/{runner}: " + f"{agg['found']}/{agg['expect']} files (-{pct:.1f}%) " + f"across {len(agg['by_unit'])} units", file=sys.stderr) if __name__ == "__main__": From 20cc50630b0a9c3cfca922a1e136b19ffd89fb34 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:02:25 -0400 Subject: [PATCH 011/100] feat(ngmix): env-expanded chunk fields; position-seeded RNG becomes the only mode The three module touches named in #848: - ID_OBJ_MIN/MAX read via getexpanded, so the committed ngmix template can interpolate $NGMIX_ID_MIN/$NGMIX_ID_MAX per chunk. Unset vars fail loudly. - SEED_FROM_POSITION is retired as an option: each object's RNG derives from its own (ra, dec, ccd), so results are bit-identical under any chunking. Tile-seed path, Ngmix.get_prior, and self._rng removed; a config that still sets SEED_FROM_POSITION=False raises with a clear message. - RUN_NAME read via getexpanded, so per-chunk run dirs can come from $SP_NGMIX_CHUNK in the committed template. Tests: tests/module/test_ngmix.py 24 passed / 1 pre-existing failure (azgauss needs ngmix>=2.4.1, absent from the runtime image; fails on clean tree too); test_ngmix_weight_validation.py 14 passed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- .../modules/ngmix_package/__init__.py | 16 +++- src/shapepipe/modules/ngmix_package/ngmix.py | 73 ++++++------------- src/shapepipe/modules/ngmix_runner.py | 32 ++++---- src/shapepipe/run.py | 2 +- 4 files changed, 58 insertions(+), 65 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 230606bba..fca6ee0bd 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -40,15 +40,27 @@ (no batch saving) ID_OBJ_MIN : int ID of first galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, so an orchestrator can + set the object range per chunk, for example + ``ID_OBJ_MIN = $SP_NGMIX_ID_OBJ_MIN``. ID_OBJ_MAX : int ID of last galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, as for ``ID_OBJ_MIN``. BKG_RMS_VIGNET_PATH : str, optional Path to a ``background_rms_vignet*.sqlite`` file produced by ``vignetmaker_runner``. The string may contain ``{file_number_string}``, which is replaced by the current tile ID. +Random number generation +======================== + +Each object gets its own random number stream, seeded from its sky position +and CCD (``position_seed``, ngmix#796). The output is therefore identical +whether a tile is processed in one go or split into object chunks with +``ID_OBJ_MIN``/``ID_OBJ_MAX``. The older tile-seeded mode is retired; the +config option ``SEED_FROM_POSITION`` is obsolete. Setting it to ``False`` +raises an error, so a stale config cannot silently change the RNG. + """ __all__ = ["ngmix"] diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index d4f64a74e..fc9779c6a 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -335,13 +335,12 @@ class Ngmix(object): (robust for galaxies); ``"wcs"`` uses the catalog sky position projected through the WCS (better for stars, whose HSM moments are noisy). See :func:`make_ngmix_observation`. - seed_from_position : bool, optional - If ``True``, replace the tile-level RNG with a per-object RNG seeded - from the object's sky position (:func:`position_seed`) inside the - object loop, so metacal's ``fixnoise`` counter-noise and the fit - guesses cancel across Pujol image-simulation branches (ngmix#796). The - default ``False`` leaves the production path byte-identical. See - :func:`position_seed` for the physics and the seed construction. + Notes + ----- + The RNG is always per object and seeded from that object's sky position + (:func:`position_seed`). Results therefore do not depend on how the tile is + split into object chunks, and metacal's ``fixnoise`` counter-noise and the + fit guesses cancel across Pujol image-simulation branches (ngmix#796). Raises ------ @@ -364,7 +363,6 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="hsm", - seed_from_position=False, metacal_psf="fitgauss", ): @@ -418,20 +416,14 @@ def __init__( self._id_obj_max = id_obj_max self._bkg_sub = bkg_sub self._centroid_source = centroid_source - self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf self._w_log = w_log - # Initiatlise random generator - seed = int(''.join(re.findall(r'\d+', self._file_number_string))) - self._rng = np.random.RandomState(seed) - self._w_log.info(f'Random generator initialisation seed = {seed}') - if self._seed_from_position: - self._w_log.info( - 'SEED_FROM_POSITION on: per-object RNG seeded from sky position' - ' for Pujol noise cancellation (image sims, ngmix#796)' - ) + self._w_log.info( + 'Per-object RNG seeded from sky position (ngmix#796): results are' + ' invariant to how the tile is split into object chunks' + ) @classmethod def MegaCamFlip(self, vign, ccd_nb): @@ -461,18 +453,6 @@ def MegaCamFlip(self, vign, ccd_nb): # swap y axis so origin is on bottom-left return vign - def get_prior(self, T_range=None, F_range=None): - """Get Prior. - - Returns - ------- - ngmix.joint_prior.PriorSimpleSep - """ - return get_prior( - self._pixel_scale, self._rng, - T_range=T_range, F_range=F_range, - ) - def compile_results(self, results): """Compile Results. @@ -809,7 +789,6 @@ def process(self): vignet_cat = self._vignet_cat final_res = [] - prior = self.get_prior() count = 0 n_empty_cat = 0 @@ -843,24 +822,20 @@ def process(self): n_no_epoch += 1 continue - # Position-seeded per-object RNG for Pujol noise cancellation in - # image sims (ngmix#796): the same object gets the same fixnoise - # counter-noise and fit guesses in every shear branch, so both - # cancel in the branch difference. The prior is rebuilt from the - # same per-object RNG because the guesser draws its initial guess - # via prior.sample() (ngmix guessers.py), which consumes the RNG the - # prior was CONSTRUCTED with — so a per-object rng alone would leave - # the guess drawing from the shared tile stream and break - # cancellation. Off in production, where the single tile-level - # self._rng and the tile-level prior carry the whole loop. - if self._seed_from_position: - obj_rng = np.random.RandomState( - position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) - ) - obj_prior = get_prior(self._pixel_scale, obj_rng) - else: - obj_rng = self._rng - obj_prior = prior + # Position-seeded per-object RNG (ngmix#796). Each object draws from + # a stream fixed by its own (ra, dec, ccd), so the result is + # independent of which chunk the object lands in and of detection + # order, and the same object gets the same fixnoise counter-noise + # and fit guesses in every Pujol shear branch, so both cancel in the + # branch difference. The prior is rebuilt from the same per-object + # RNG because the guesser draws its initial guess via prior.sample() + # (ngmix guessers.py), which consumes the RNG the prior was + # CONSTRUCTED with — a per-object rng alone would leave the guess + # drawing from a shared stream and break both properties. + obj_rng = np.random.RandomState( + position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) + ) + obj_prior = get_prior(self._pixel_scale, obj_rng) try: flux_guess = ( diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 3d98356c4..fcb50be9f 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -86,9 +86,12 @@ def ngmix_runner( # No batch saving save_batch = -1 - # First and last galaxy ID to process - id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") - id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") + # First and last galaxy ID to process. Read via ``getexpanded`` so an + # orchestrator can drive the chunk bounds from environment variables + # (``$SP_NGMIX_ID_OBJ_MIN`` and friends); ``getexpanded`` is the only + # accessor in ShapePipe's config that expands ``$VAR``. + id_obj_min = int(config.getexpanded(module_config_sec, "ID_OBJ_MIN")) + id_obj_max = int(config.getexpanded(module_config_sec, "ID_OBJ_MAX")) # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) @@ -99,16 +102,20 @@ def ngmix_runner( else: centroid_source = "wcs" - # Seed the per-object RNG from sky position instead of per tile, so - # metacal's fixnoise counter-noise (and the fit guesses) cancel across - # Pujol image-simulation shear branches (ngmix#796). Default False leaves - # the production path byte-identical. + # Position-seeded RNG is the only mode: every object's RNG comes from its + # own (ra, dec, ccd), so results do not depend on how the tile is split + # into chunks, and metacal's fixnoise counter-noise cancels across Pujol + # image-simulation shear branches (ngmix#796). The retired tile-seed mode + # had neither property. Old configs that disable it must fail loudly. if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - seed_from_position = config.getboolean( - module_config_sec, "SEED_FROM_POSITION" - ) - else: - seed_from_position = False + if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): + raise ValueError( + "SEED_FROM_POSITION = False is no longer supported: the" + " tile-seeded RNG mode has been retired because it makes" + " results depend on the object chunking. Remove the" + " SEED_FROM_POSITION entry from the ngmix config section" + " (position-seeded RNG is now the only mode)." + ) # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. @@ -161,7 +168,6 @@ def ngmix_runner( id_obj_max=id_obj_max, bkg_sub=bkg_sub, centroid_source=centroid_source, - seed_from_position=seed_from_position, metacal_psf=metacal_psf, ) diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 962953ede..fee0dad3e 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -84,7 +84,7 @@ def _set_run_name(self): Set the name of the current pipeline run. """ - self._run_name = self.config.get("DEFAULT", "RUN_NAME") + self._run_name = self.config.getexpanded("DEFAULT", "RUN_NAME") if self.config.getboolean("DEFAULT", "RUN_DATETIME"): self._run_name += datetime.now().strftime("_%Y-%m-%d_%H-%M-%S") From d1c0afb01edbb3d4eee59bb76c38c73e8c39315a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:06:58 -0400 Subject: [PATCH 012/100] =?UTF-8?q?feat(orchestration):=20committed=20work?= =?UTF-8?q?flow=20configs=20=E2=80=94=20no=20wrapper,=20no=20runtime=20rew?= =?UTF-8?q?riting=20(#848=20D2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New workflow/config/cfis/ (example/cfis + p3-batch1 deltas, rewritten; example/cfis untouched for the bash flow): - RUN_DATETIME=False committed; every 'last:' INPUT_DIR replaced by the fixed $SP_RUN/output/run_sp_//output path of its producer. - NUMBER_LIST=$SP_UNIT_NUM on unit-ID-numbered stages (getlist env-expands; SP_UNIT_NUM carries the dashed form, e.g. -210-282 — rules do the transform). - ngmix template: $NGMIX_ID_MIN/$NGMIX_ID_MAX, SEED_FROM_POSITION=True, per-chunk RUN_NAME=run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u, SAVE_BATCH=250. - merge_sep_cats: N_SPLIT_MAX=$NGMIX_N_CHUNKS; module reads it env-expanded (same getexpanded treatment as the ngmix chunk fields). - make_cat from the nosm variant (what the P3 runs used) — no spread_model dependency. - p3 deltas folded: real INPUT_PATH/RETRIEVE=symlink, exp_Ma external star cat ($SP_RUN/star_cat_exp), PiViVi RUN_DATETIME fix. Rejected: SMP_BATCH_SIZE deltas (superseded by -b {threads}). - tile_Sx from the nomask variant (plain Sx needs an unlisted tile_Ma stage); tile_Uc kept but its run_sp_tile_Gic input path is unverified (Uc is not the production detect variant). All 14 configs pass a strict $VAR-expansion round-trip with the workflow's env stubbed. workflow/config.yaml config_src now points in-repo. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- .../modules/merge_sep_cats_runner.py | 2 +- workflow/config.yaml | 8 +- workflow/config/cfis/config_exp_Gie.ini | 99 ++++++++++ workflow/config/cfis/config_exp_Ma.ini | 86 +++++++++ workflow/config/cfis/config_exp_Sp.ini | 78 ++++++++ workflow/config/cfis/config_exp_psfex.ini | 181 ++++++++++++++++++ .../config/cfis/config_merge_sep_cats.ini | 87 +++++++++ workflow/config/cfis/config_onthefly.mask | 86 +++++++++ workflow/config/cfis/config_tile_Fe.ini | 76 ++++++++ workflow/config/cfis/config_tile_Git.ini | 93 +++++++++ workflow/config/cfis/config_tile_Mc.ini | 80 ++++++++ workflow/config/cfis/config_tile_Mh_exp.ini | 76 ++++++++ .../config/cfis/config_tile_Ng_template.ini | 104 ++++++++++ workflow/config/cfis/config_tile_PiViVi.ini | 172 +++++++++++++++++ workflow/config/cfis/config_tile_Sx.ini | 118 ++++++++++++ workflow/config/cfis/config_tile_Uc.ini | 92 +++++++++ workflow/config/cfis/config_tile_Uz.ini | 73 +++++++ .../config/cfis/config_tile_onthefly.mask | 90 +++++++++ workflow/config/cfis/default.conv | 5 + workflow/config/cfis/default.param | 68 +++++++ workflow/config/cfis/default.psfex | 85 ++++++++ workflow/config/cfis/default_exp.sex | 133 +++++++++++++ workflow/config/cfis/default_noimaflags.param | 65 +++++++ workflow/config/cfis/default_tile.sex | 133 +++++++++++++ workflow/config/cfis/final_cat.param | 114 +++++++++++ .../mask_default/MEGAPRIME_star_i_13.8.reg | 24 +++ .../cfis/mask_default/Messier_catalog.npy | Bin 0 -> 4209 bytes .../mask_default/Messier_catalog_updated.fits | Bin 0 -> 8640 bytes workflow/config/cfis/mask_default/default.ww | 40 ++++ .../config/cfis/mask_default/halo_mask.reg | 50 +++++ .../config/cfis/mask_default/ngc_cat.fits | Bin 0 -> 201600 bytes workflow/config/cfis/star_selection.setools | 103 ++++++++++ 32 files changed, 2418 insertions(+), 3 deletions(-) create mode 100644 workflow/config/cfis/config_exp_Gie.ini create mode 100644 workflow/config/cfis/config_exp_Ma.ini create mode 100644 workflow/config/cfis/config_exp_Sp.ini create mode 100644 workflow/config/cfis/config_exp_psfex.ini create mode 100644 workflow/config/cfis/config_merge_sep_cats.ini create mode 100644 workflow/config/cfis/config_onthefly.mask create mode 100644 workflow/config/cfis/config_tile_Fe.ini create mode 100644 workflow/config/cfis/config_tile_Git.ini create mode 100644 workflow/config/cfis/config_tile_Mc.ini create mode 100644 workflow/config/cfis/config_tile_Mh_exp.ini create mode 100644 workflow/config/cfis/config_tile_Ng_template.ini create mode 100644 workflow/config/cfis/config_tile_PiViVi.ini create mode 100644 workflow/config/cfis/config_tile_Sx.ini create mode 100644 workflow/config/cfis/config_tile_Uc.ini create mode 100644 workflow/config/cfis/config_tile_Uz.ini create mode 100644 workflow/config/cfis/config_tile_onthefly.mask create mode 100644 workflow/config/cfis/default.conv create mode 100644 workflow/config/cfis/default.param create mode 100644 workflow/config/cfis/default.psfex create mode 100644 workflow/config/cfis/default_exp.sex create mode 100644 workflow/config/cfis/default_noimaflags.param create mode 100644 workflow/config/cfis/default_tile.sex create mode 100644 workflow/config/cfis/final_cat.param create mode 100644 workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg create mode 100644 workflow/config/cfis/mask_default/Messier_catalog.npy create mode 100644 workflow/config/cfis/mask_default/Messier_catalog_updated.fits create mode 100644 workflow/config/cfis/mask_default/default.ww create mode 100644 workflow/config/cfis/mask_default/halo_mask.reg create mode 100644 workflow/config/cfis/mask_default/ngc_cat.fits create mode 100644 workflow/config/cfis/star_selection.setools diff --git a/src/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py index 927c79b76..6e99cf3ec 100644 --- a/src/shapepipe/modules/merge_sep_cats_runner.py +++ b/src/shapepipe/modules/merge_sep_cats_runner.py @@ -28,7 +28,7 @@ def merge_sep_cats_runner( ): """Define The Merge SEP Catalogues Runner.""" # Get config entries - n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") + n_split_max = int(config.getexpanded(module_config_sec, "N_SPLIT_MAX")) file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") file_ext = config.getlist(module_config_sec, "FILE_EXT") diff --git a/workflow/config.yaml b/workflow/config.yaml index f92cbb69f..592ecf989 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -17,8 +17,12 @@ container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # /tiles/// and /exp/// run_dir: /scratch/cdaley/shapepipe-output/smk-p0 -# The canonical config chain (example/cfis in the repo, or a run-specific copy). -config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis +# The canonical config chain: the committed, wrapper-free configs (#848 D2 config +# sweep) -- RUN_DATETIME=False, fixed INPUT_DIRs, NUMBER_LIST=$SP_UNIT_NUM baked +# in. example/cfis stays untouched for the bash flow; runs/p3-batch1/cfis's real +# divergences (real data paths, external star cats, SAVE_BATCH memory tuning) +# were folded in -- see the config-sweep report for the full list. +config_src: /home/cdaley/shapepipe-smk/workflow/config/cfis # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). # Star catalogues for masking, pre-generated (network step done in prepare): diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini new file mode 100644 index 000000000..7c6297cf0 --- /dev/null +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -0,0 +1,99 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini new file mode 100644 index 000000000..0b85fe86c --- /dev/null +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -0,0 +1,86 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Input file patterns: image, weight, external flag, external star catalogue +# (folded from runs/p3-batch1/cfis: offline cluster, no online star-cat fetch) +FILE_PATTERN = image, weight, flag, star_cat + +FILE_EXT = .fits, .fits, .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +# (folded from p3-batch1: True, using the pre-staged star_cat_exp pool) +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +# Path to check for existing output mask files +CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_exp_Sp.ini b/workflow/config/cfis/config_exp_Sp.ini new file mode 100644 index 000000000..dd27d6ccd --- /dev/null +++ b/workflow/config/cfis/config_exp_Sp.ini @@ -0,0 +1,78 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fitsfz, .fitsfz, .fitsfz + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini new file mode 100644 index 000000000..0af871d8e --- /dev/null +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -0,0 +1,181 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsfPi +#RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# Input from two modules +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini new file mode 100644 index 000000000..a033f28d5 --- /dev/null +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -0,0 +1,87 @@ +# ShapePipe post-run configuration file: merge separated catalogues + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Ms + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_sep_cats_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +# NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner +# derives the other chunks' paths itself from N_SPLIT_MAX and this pattern +# (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the +# same fixed $SP_RUN/output. +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_SEP_CATS_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Numbering convention, string that exemplifies a numbering pattern. +NUMBERING_SCHEME = -000-000 + +# WARNING (optional, default is 'error'). Use 'always'/'ignore' to +# display/ignore warnings, and not raise error +WARNING = always + +# Maximum number of separated catalogues per input. +# BLOCKER: merge_sep_cats_runner.py reads this with plain config.getint (not +# getexpanded) -- $NGMIX_N_CHUNKS would NOT expand here, unlike ID_OBJ_MIN/MAX +# in the ngmix module. Committed as a constant matching workflow/config.yaml's +# ngmix_chunks: 8; if that value changes, this line must be edited by hand +# (or the module gets the same getexpanded treatment ngmix_runner got). +# Chunk count comes from the workflow (single source of truth: the +# scattergather count); env-expanded by the module. +N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask new file mode 100644 index 000000000..7c185c602 --- /dev/null +++ b/workflow/config/cfis/config_onthefly.mask @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/workflow/config/cfis/config_tile_Fe.ini b/workflow/config/cfis/config_tile_Fe.ini new file mode 100644 index 000000000..9546f062e --- /dev/null +++ b/workflow/config/cfis/config_tile_Fe.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +COLNUM = 3 + +# Prefix to remove from exposure name +EXP_PREFIX = p + diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini new file mode 100644 index 000000000..72a0f0be7 --- /dev/null +++ b/workflow/config/cfis/config_tile_Git.ini @@ -0,0 +1,93 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/tiles, /project/def-mjhudson/unions-wl/tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini new file mode 100644 index 000000000..502731431 --- /dev/null +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -0,0 +1,80 @@ +# ShapePipe post-run configuration file: create final catalogs, with +# no spread model on input + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282) +NUMBER_LIST = $SP_UNIT_NUM + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/workflow/config/cfis/config_tile_Mh_exp.ini b/workflow/config/cfis/config_tile_Mh_exp.ini new file mode 100644 index 000000000..96512df1a --- /dev/null +++ b/workflow/config/cfis/config_tile_Mh_exp.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini new file mode 100644 index 000000000..6e3e33292 --- /dev/null +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -0,0 +1,104 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +# Per-chunk run dir: the workflow exports SP_NGMIX_CHUNK= for each chunk. +# RUN_NAME is env-expanded (run.py getexpanded); braces keep the trailing "u" +# out of the variable name. +RUN_NAME = run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as +# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for +# every tile (missing file -> error, no per-tile fallback); omit the option +# entirely to fall back to the scalar sigma_mad noise estimate. +BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite + +# Number of objects to batch save during processing, optional. Omit or set +# to -1 for no batch saving. +# 250 (folded from runs/p3-batch1/cfis): worker RSS grows ~4.8 MB/object +# until the flush recycles it -> peak ~1.1 GB + 250 x 4.8 MB ~= 2.3 GB/worker +# (A/B test, job 17607877). Not superseded by -b {threads} (that sets fork +# width, this bounds per-worker memory). +SAVE_BATCH = 250 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# SEED_FROM_POSITION: per-object RNG seeded from sky position (ra, dec, ccd) +# instead of one ordered per-tile stream, so results are bit-identical under +# any chunking (D4/#796). Required for the chunked ngmix scatter/gather. +SEED_FROM_POSITION = True + +# ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, +# computed at execution time from the tile's own object count and expanded +# via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not +# plain getint). +ID_OBJ_MIN = $NGMIX_ID_MIN +ID_OBJ_MAX = $NGMIX_ID_MAX diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini new file mode 100644 index 000000000..246ee18f9 --- /dev/null +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -0,0 +1,172 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# PSFEx PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[PSFEX_INTERP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +# Number of stars threshold +STAR_THRESH = 20 + +# chi^2 threshold +CHI2_THRESH = 2 + +# Multi-epoch mode parameters + +# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR +# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered +# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +ME_DOT_PSF_EXP_DIR = $SP_EXP + +# Input psf file pattern +ME_DOT_PSF_PATTERN = star_split_ratio_80 + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini new file mode 100644 index 000000000..0ce9ea226 --- /dev/null +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -0,0 +1,118 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini new file mode 100644 index 000000000..f2ff51e4a --- /dev/null +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -0,0 +1,92 @@ +# ShapePipe configuration file for tile object selection using +# an (external) catalogue + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = read_ext_sexcat_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[READ_EXT_SEXCAT_RUNNER] + +# NOTE(blocker): run_sp_tile_Gic (external-catalog get_images) is not one of +# the 13 configs in this sweep -- no committed config produces it, so this +# path is written by naming-convention analogy, unverified. Uc is otherwise +# fully de-wrapper-ified; do not rely on this input until Gic exists. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_cat, CFIS_image, log_exp_headers + +FILE_EXT = .cat, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +# Side length of the square postage stamp (vignet) extracted from the tile +# image, in pixels (must be odd). Default: 51 +VIGNET_SIZE = 51 + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = ALPHA_J2000,DELTA_J2000 + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uz.ini b/workflow/config/cfis/config_tile_Uz.ini new file mode 100644 index 000000000..fc8550af4 --- /dev/null +++ b/workflow/config/cfis/config_tile_Uz.ini @@ -0,0 +1,73 @@ +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +FILE_EXT = .fitsfz + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +HDU_DATA = 1 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask new file mode 100644 index 000000000..18cf5db2d --- /dev/null +++ b/workflow/config/cfis/config_tile_onthefly.mask @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 0 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 100644 index 000000000..09ad8405e --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1,68 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +# The following flag requires a flag image +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 100644 index 000000000..a9d1a906c --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1,85 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2017-11-30 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +BASIS_NAME basis.fits # Basis filename (FITS data-cube) +BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter +NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT + # or PCA_COMMON +NEWBASIS_NUMBER 8 # Number of new basis vectors +PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) +PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 51,51 # Image size of the PSF model +PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? +MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 2 # Polynom degree for each group +PSFVAR_NSNAP 9 # Number of PSF snapshots per axis +HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON +STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? + +BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? +BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed + +#----------------------- PSF homogeneisation kernel -------------------------- + +HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE +HOMOBASIS_NUMBER 10 # Kernel basis number or parameter +HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter +HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF +HOMOKERNEL_DIR # Where to write kernels (empty=same as input) +HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels + +#----------------------------- Output catalogs ------------------------------- + +OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS +CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +# Note: Check-image types can be set the ShapePipe config file, psfex_runner section +####### +CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames +#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? + +#----------------------------- Miscellaneous --------------------------------- + +PSF_SUFFIX .psf # Filename extension for output PSF filename +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 100644 index 000000000..b87275ecb --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE AUTO # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 100644 index 000000000..b251f5b74 --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1,65 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 100644 index 000000000..ff3b25213 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 100644 index 000000000..8ddfebd37 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1,114 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# flags +FLAGS +IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# spread class +#SPREAD_CLASS + +# spread model flag and error +#SPREAD_MODEL +#SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +#NGMIX_G1_ERR_1M +#NGMIX_G2_ERR_1M +#NGMIX_G1_ERR_1P +#NGMIX_G2_ERR_1P +#NGMIX_G1_ERR_2M +#NGMIX_G2_ERR_2M +#NGMIX_G1_ERR_2P +#NGMIX_G2_ERR_2P +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_T_PSF_RECONV_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL diff --git a/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg new file mode 100644 index 000000000..4e4164aaf --- /dev/null +++ b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg @@ -0,0 +1,24 @@ +-11.5 68 +-6 186.5 +7 188 +10 64.5 +31 55 +50 38.5 +56.5 11.5 +188 8 +192 -4 +59.5 -11.5 +45 -33 +13.5 -64 +5 -154 +-6 -155 +-11 -64.5 +-40 -44.5 +-51.5 -30.5 +-62.5 -22.5 +-68 -9.5 +-177 -2 +-176 3 +-78 12.5 +-67.5 14.5 +-38.5 50 diff --git a/workflow/config/cfis/mask_default/Messier_catalog.npy b/workflow/config/cfis/mask_default/Messier_catalog.npy new file mode 100644 index 0000000000000000000000000000000000000000..ef07eb032b08de4fcf508418524692e27f92ad75 GIT binary patch literal 4209 zcmb7{dvKIj6~H$S0z_V+Av_zF=Wf!F0D%A@VJ{fCEf+Q}Ge}K(fi%99nrA zDunWq77DdgDz+6YRTR<6s3U8&j-3Igj&%@bimd}oA2U?zlbKdM_uP9w(m(ve`$y*c zzCFM5y62pGb0i#Dx@=XB?^WN{Ks?dak_puMR|f+dThsACg+H)u`P4vJg+CbRYTc6H z4~^zMo@mjpb;;T0Jrn)RYq*17Tj|%eB$n*RU?!Sg*O7@Qw1q{{wyp$y?@C2G6Z9UO zJ*|SwZ1?}atFSw_CB3P=b8}@&M!PQkRQogQ9WJxR1Qh>397tE&bDwclCSdu1(AH6 z4(J=nqclLjNNy$y{Uf>I4k$$U+;$icOckc`Be|`!t%iZDp)MQ_Fa8fL{-b;tQzLp( z1zK|XR4X~i+6ssD2bS^mr21)@q|-L|e`)L2lj@%utcLWY`f8xj0Yf~cz08Nmysu}I z@{^g-7~8uRih{w2Z!HWBN-@I6ONt@pAt6OgeuG|8G~c*H>$1v0xQBsaDf~%ZQ5pK+ zj9#xu(RJZAt&dbL!Y~#qk)qmYi%L-!8?ToXsYiGI;Jwj}Di2{ei;a+?!V|0AWsRl> zym+Os9#lTUNERCeCHYt zJUw6aL%5fLQZM+Z%ud_-6Iwr&qT_?AH)5Y0Q~eRfvY5YD%-O+3yrf8%k}`2kQ5lS5 zF$!?}vw9tm%8M`a(@7E=S+VZ$>vz-ugb6G*QHqIP%nRl)Kw>5JofNeih%kx8CQH%J ziDe7S6Z!TxZ4=cXgaC_C$oH}-GAe=*UV1+mP=gW5Sgf4nY({85j5bP5FwOf$skxcg z`ew}uH3Z>41}X@U&gz10ej=di1-q4`b^hq(-Jdoa+e?F#{-Xl{Y#R$_Fm@dV%C0vB0w`Q0Pu=9N?r10aa z1fiP6W{_UL)7z_%e`NK-A8(x7p+->k!%PNhq!_BzUb;+rXK_@tft|MwtqK197itW` zEEbzBMTsX?^IN<3T|d4!Y5&KoR4Kw77ON%ms|DMaqS5$4ebivn6}t8M9V&xgsj&!i z8K@&*oT+IkM%@MMhIoJxJ1e4yYTsCB3-YHfhMU}t3w%~Us)i{I@1A>5M z-PR@B7L@6soDj-%^Uc>N)_z{GGL!si)iAfUj; z5chgC? z3g;oLX0bJ-x6I~7j4@k7dKc!~37<-tWQGvdGVllib}C9S%nNnqSzCL_=S?yN!a4@l z6EN0g|6g7*%bn5qnc^)}C2BrGlz|ul+q+-jSP-A>&68b=j-t$X$xCiGfE6@H@}#1%&`}s!}C4|3xRHS%ncEW8jNY^fzTg z8JXpd^5&~XtPx=|i*1pj&>|L+d5Nq!9g?sS0L0-4w@4|Wsp3qZx=0U6~aLVzCyqRm{F~3&hHc>Jfya3>+iC-sHUVhk$^KK6XW4(DKwegl8D|DgmX=&0@1}8gR!+><@H9 z+WAAZ9^qLAjuT*e#Csq>?XndVKW$FmD8h3Le2oCVEeJOk2$!g6{6`KDwc_*QyqTuQPCx0DJ5Z>D8!r%S;}ikXzqWafEL$@B#sK&c5wX z9YclwMt!znzPT4B5ME^9B?1bpOTA8Mbw^hpRnUe7=8}*^_$C9V2=Fdu-YK1(=oR%* zt|gU1IL*MfJ`;oX=kB5NT zfba?f-uzXxoJl@$H4aq@J{Kw+tpw#gl=-AJ5(Csd;(*Y{{>v*ks<&9 literal 0 HcmV?d00001 diff --git a/workflow/config/cfis/mask_default/Messier_catalog_updated.fits b/workflow/config/cfis/mask_default/Messier_catalog_updated.fits new file mode 100644 index 0000000000000000000000000000000000000000..6a9f00096170565aeb7fab3429c2256ce56f3533 GIT binary patch literal 8640 zcmeHL4NO&K82bxk?Xcg61^5lKgn(d%puWZoviN(qOUV;yoAdgbQ9S23R5C zGJc%mUqCidTB!L;nwUE8!IV-Emzk_l)0xbfKNvc7R`!19<}PlRM&quncFy*U!+YNM z`JV52-?M$^Ch6l6Vxt8?D|on!!gOJNj?I!|&(0GZIf71aOcL@OCY#x0H+#s(5YrnI z^eKJtg(9EHZZ{PQW|PAtI0|zm&n)h!0aEly_^(HOFCm{TKRZLR3ptjD%+_qlmS@ee z<+(|E-9@JuqYY7g@ELn0_VZRD>u0W-raGWHpgN#BpgN#BpgN#B@ZWcUA19Oa@dm9B z$n*vyu-NFpUb~tuWLRw`d!gVkWn@W$RN!zOPJ5+1di$@QpJ)8@^Rsdu5B7Y14xWX4 zS&}W&v4Ee)%@0rF0_OL1DSR_L^MwZY!H4;MT?t}*vH{QW{rFzUwCCjK3b`h`DO++# zcF*4nulyY!*XREJQ20EvxYZhU`smoGB+vRn9^rG1jKwNtnfqonipd7O(vLv1l&P%K zd;Z)D_w!ESQ~KdWJ`L|jpvgWz+iDYpf$N*;MLuI{LbS_2yUFFz!1U?j6XRTb_*Jc3 z1LNcR9`<9t7x=h77oYd^x%!)DEt2Npr~knA%k{bVyr<7KFN(fYZ|HOJc~4)6GS3UG zHswc^f&1s;18>Zq`uoOTE~2Q~q|Mla|%g za-h=JE%N?j_uw*?bnuLd%4718+1H`mV&Nd-x*iGk9UB7p&}C4$PlbH^_CmVeV&Nd- z=89JNK<*p#3k)C#Ou1FXUhvhhA8^?Mtb-}voMsd9Z@}Xk4L!{R2OH5KblZ)dP$=zM!+^*|xtz<#td_j+GQ1-&6ZGat%$AEx3wsX?S2kT*O z(P(-PbvTh(obuFT5Z{>yK3MWT-;4Ig4*6`)85)KG1QE-xCN>W;=W>HMh*)Vk)ja6M z@syJyu$7@FVSU62dK5DdMBK3RxExb`isC~K0*QVOCn9^$ zbp(OdhJ(xyr{#o-Fcz?5&!N=z7J3m$omj$lnD%Km2LvbsqXagOsg}PA-bQ25LIe>V zY_+_lgL<|@I z07LL-iY?$qV<2@4R4q?dbnI3#Dr+G=U=KWr-9NxdBlb3dj*Wz&SaOD!L_8P^mIHNs zdT16GHv7tAi0Nvf^(agkZEeVSNc<=n#-hoUN=9c0WOjG+Cb?dqlKPu$YDEYqwBwk(z^jfXBY2yG-{#>NlWyeBp( z@5@r2p?zSM!|o1OUcz)9@V|8#^oBHrL)mTU;l)flYcOm;Vdr^9f&XuELZ^<7!rdZ> z7d$Tc4;qc|FXe?ZLlMI-okBI`F@A;7=F| BE!zM9 literal 0 HcmV?d00001 diff --git a/workflow/config/cfis/mask_default/default.ww b/workflow/config/cfis/mask_default/default.ww new file mode 100644 index 000000000..c2797f904 --- /dev/null +++ b/workflow/config/cfis/mask_default/default.ww @@ -0,0 +1,40 @@ +#--------------------------------- Weights ------------------------------------ + +WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) + +WEIGHT_MIN 0. # Pixel below those thresholds will be flagged +WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged +WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels + +#---------------------------------- Flags ------------------------------------- + +FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) + +FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels +FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs +FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits + +#---------------------------------- Polygons ---------------------------------- + +POLY_NAMES "" # Filename(s) of input DS9 regions +POLY_OUTFLAGS # FLAG values for polygon masks +POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks +POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? + +#---------------------------------- Output ------------------------------------ + +OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename +OUTFLAG_NAME flag.fits # Output FLAG-map filename + +#----------------------------- Miscellaneous --------------------------------- + +GETAREA N # Compute area for flags and weights (Y/N)? +GETAREA_WEIGHT 0.0 # Weight threshold for area computation +GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area +MEMORY_BUFSIZE 256 # Buffer size in lines +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME ww.xml # Filename for XML output +XSL_URL file:///usr/local/share/weightwatcher/ww.xsl + # Filename for XSL style-sheet +NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/workflow/config/cfis/mask_default/halo_mask.reg b/workflow/config/cfis/mask_default/halo_mask.reg new file mode 100644 index 000000000..c44f25167 --- /dev/null +++ b/workflow/config/cfis/mask_default/halo_mask.reg @@ -0,0 +1,50 @@ + 274.66813 -1.25966 + 272.54579 32.47406 + 266.21222 65.67579 + 255.76731 97.82190 + 241.37579 128.40544 + 223.26462 156.94408 + 201.71942 182.98775 + 177.07997 206.12573 + 149.73486 225.99312 + 120.11532 242.27660 + 88.68848 254.71937 + 55.94996 263.12519 + 22.41606 267.36151 + -11.38436 267.36151 + -44.91826 263.12519 + -77.65678 254.71937 +-109.08362 242.27660 +-138.70315 225.99312 +-166.04827 206.12573 +-190.68772 182.98775 +-212.23292 156.94408 +-230.34409 128.40544 +-244.73561 97.82190 +-255.18052 65.67579 +-261.51409 32.47406 +-263.63643 -1.25966 +-261.51409 -34.99339 +-255.18052 -68.19511 +-244.73561 -100.34123 +-230.34409 -130.92476 +-212.23292 -159.46341 +-190.68772 -185.50708 +-166.04827 -208.64506 +-138.70315 -228.51245 +-109.08362 -244.79593 + -77.65678 -257.23870 + -44.91826 -265.64452 + -11.38436 -269.88084 + 22.41606 -269.88084 + 55.94996 -265.64452 + 88.68848 -257.23870 + 120.11532 -244.79593 + 149.73486 -228.51245 + 177.07997 -208.64506 + 201.71942 -185.50708 + 223.26462 -159.46341 + 241.37579 -130.92476 + 255.76731 -100.34123 + 266.21222 -68.19511 + 272.54579 -34.99339 diff --git a/workflow/config/cfis/mask_default/ngc_cat.fits b/workflow/config/cfis/mask_default/ngc_cat.fits new file mode 100644 index 0000000000000000000000000000000000000000..f51546da7fc88ae1ca919a3c13882fe1a7df6768 GIT binary patch literal 201600 zcmeGF2~?NW_CJoJC^(`d4mctzj+u&vn#6MsJc>gOJW`TaIFXu~17=1VYN#lVC{9`A zfF^3U*7~iL>ptto=A6A> zdpOfR16KEE`t~vd0zw1+=3oDSCjy4PGB*B|31cP(B)k&PwR?XnU}D1HvBL&W81^^i zL6Y73_w7F5Kk(}aeuF1W7(69l*x-c00SQya4gY&CZfb8apu2@1{}x{x@Ebd6%#h&| z0$z!CaUMBl_}GafUl}{m&EzltWe(_X_Kx@u{QCc;;Q#PHnc4ra>_5f+vA{nT_{ReO zSl}NE{9}QCEbxy7{=aVlnoe5X`}7VCXezt+?k{97v*};7dLm%R$gzVbObJLBJY@9n zfZ?wt=*j6{iu|Ad`Ty$tx!wQ&^JkuqxBdJ1bK4N`8$EpNO9>-rK0eGf6AwW7|MZXf zwfh@>ZQHi*(EdO0L;3&ouWxvt=-!ye{}0v+FHLx5(zt+egC`6gGdyAVguh=ebo>9s zukHU~Kj!!MT-<2;ckOQWim?73Ux)vLpAMP$k;6w1`_Ekahe!AB&i-gRZ1_to=dbbS z7W_Z_FXqSo_%HGcCVw;?JYndVkz)e_{`dIW{}=i7kLhb_`%M_EZS=px*R@a7GnyY( z)%@>&$B*#2{4w;uz>n~0e*Zmu+P@P=zCQd#EdBp?`Vv0P@4tsn#|z_&`ETIU{Qi6R zI&eI{JaQ~2RsQ$()BMDLBmVx$ZXxzixrB3$GW6Q(0ez6|7tpwsPev zE9_$Zz2elE+^{4Mk1&5CT)25il=*6ESZeMuO}OIgE6V;pk$!lZtK9R-la(jFLG{Ce zgM&?@R`xVmw#zfjG&lE{X_QTFpo;r{VpIYT4-XH^@R^e$4$l1-Ges54DBmR2@g6(K z08_?cRP44*R{!E(VYY_{OED^5k1^7gTU-U;EThuiC|UO7iAJ^HUxETctHr1ZQx=%Kcc z>e6aXiYz-~7y%<~Eu(H>f;j#D?66GD)H3QnxkmhJSO7Bx5rmtE)MtC1F!n(?CNwx6 z;tccqBy1<3L#$~8U8p0z+%$}G(2~t@Ljv7JmJ`7|UZzeZ5xO&1b%ycChkTEkj*#08u z1Gi0Q8in>3eKv%}$H!}6b9P-&&wXxi37FF|7NCiRz)1zJ){t5u)GmYsGGQu}SpIc} z+EXEvLmO30W5q_H_DtR9YJ`@t_>XjPZTfC%JWq3p#cMppwecNDq$VsG(^*_wn#fha z)HIeL=&r$0umx2Efq1;Ph{dwS(gFj)v=LVUeA7rfGL);JKIvl_OP3rL*Sg+(pRn$*MJnDbjw~neYf%DumDycd@N&)SDq-I(u$P>({Ll>$u(-1_>-aM z64G_!)`wXOI=OASZgW;xbaXUV2`1ledNEDxO4&kfMYost6MV#u?>}Z;n8*9hLb21= zmuwLVf@NeOaCbtijrhQ48d>r2VwZn8cYAr1P!uV4Rj$FVgtp*uJVfk5UkD6Zx%baz zJ96glRDy`zjm=k!9l5ZYyS-$Ej(6nyVQf>NwTx|Gy0iVWVK;Q|U>RGUIO51R_b`Et zydApiG)Hi&NH!l@A( zO!(x~Y4u{Sr`&$|!YCBI8<}p;efo8Vy&gSGyf%(1pUvGVWI1sCEHIs*7+BJ^!xPi^yWs?kwXT2H@Q-yKogI z{OOlyufM#Bbs^}Y%xBxA^d2rOj170uDT(r4dFs#ZHw}VLRfIR9@aB=544n)l%2&Lu zPLEq}OhMa+2`o_#eNdfl+l+~@M=n))giyX`E!AUMPeuywo1((zEY(W<$=y~a!a-E5 zc}{sfJH?2@phe1w3bD$hiy(&F$z`8CnqVixkC|^XXUAn9*lk1qQqlM z4y#=`dq{mEb3}!vUvc2F)&1vh6qk++A;*}glV3rKn0^Af7(b%z3;%6?Mi1~H=*+af=O{HQ zX$d`9*~>RgXeRVxl=9&FRPrFN^Tq2o!cwnyFb z@nM5Hdoxnjw%$|wjcCl2NGz(DqF%=+L+Xxzh^DANI8s%s(}yEjS9#({*hsV}*g_+% z7ZwyY3=N2+;Il-%S$Re<)CWWQEK$Gx1tapCpP9fV>h}#XI--LuK#3$$uD+EK7K`5U zJeb-{fVM-r&U43DeN1^`(a{5g_I`a#zAX)#nusMDPH82B4(to`9x^iE7cd8~e-|5o!87jd<{ax_ zY>LL4Zi$)d&xc)(XmaYbIQr))HZtqdNG^2ZzU$*|$W6YSp*F>SY{bVu}_=hRQ~-uHgymaniOq8qk1M2iPwFW{DQR z`N$^K284(Duad8l`Yo5Alcl5Da|HRXT-8^!7`Hy>5Y{h1Nxm^f%e8@GzgHeR3ihx> ztG*#(f3*SJIYO)@n&%!<2fS~SE;^E1qGQ{SUJ4?0IJ6#HEPmTjm3c7J)=h;Nj`%2% z0vt0(sr{JqXQB#+L(B9wYJcp<-1b?@<7iQ#eLrU&NfZu`f8YF|N8}0agIKx8cU^EI zKW<@o06JG`m*ioOTbIv%0nrle1I8GKR{!Y2VTq0vuZ!ZXGu`13;=f8w!Hf+kbrg0S zbUNpI48Q~o3R+fY(`cr6!Ek7sTK0J@-{5L3IjC!8p z`#;jTMKScJ+h0&$^g1RkAfyVYO6WJy>i4+D+!rz-tf^2T(|Q_RF+*uZVvRa|M46l* zcb8*eJ$EX+q09Ry@#Gr2Efx};blbOtv@*o0l-lG1?S}A1`QpK*Ta1)-3GeK!`eEFM z#dQM`!?K-P*B)b6t|VaHo;V^))opITsKnS3kr`v`Ow9Un!IXLQXm(3g7}NRT#98z> zS}W(b`8BvdWnFr9_O`#rkmLmu`V)^{_4nFE$3xsbOs`(Y>@zi#I~@9=qu)KggFD8a zX_BhSVS4 z*Qj|!VNG}wa|if@BES^=e@IhNYdo0;J1QEJyeLfa0GkYLis(PmRn($4*2IGJmEHMIesT(7)XvvquC6tF#(z=(k#z zSF8;C`kQ0PFFYF;uZ9KL*U=4sjY!TQH%vgke)IVR9->*f3H>hH-=RZ+Z*|a3#MJt+ zdwp3fBw6B7LaR9S*q^6dNW;ZM@c8z_imnD^ijm9DseNc-2sn*&WcD<*FYgm}WiMhn zY4T+Id(2=-IVOybj&>G3e?Y_`)-iE4#jDR%u=A%JWaV-hM}m*j|6UOrnIn1f;7Gdw z10P&Td`vO<)4O&7G9>t|Zu^r96LM~Ux11ftQeIo#$DUo#k2`uMOj(zob8l)h4hNPp zwMHAULoYS7J>Hmf$o^uW7rC;80O&uw)09mJv6h&c;A4MLn853(0wN?W3dsrIv6jWA zQUgjHTkOmXv8glB?e?wYyXq}0iu`piWr^vr&2yfa@hwLoGfjGDuTu%YAXa;S=6jJ! zR^G^s(1To=^ih%Xcc~>v>M{ZFO)$G=g*$9{%70VuAF6igpKhW#TH{6a}MW8N&tYNZiFeF zi)8(@0X*RBB&2gr@0Dc`QYB0+F()#`SoB9ai$yg{q?AjMWyTG3$5;xcaHTP8gC5$q zDR-h|={YA{Vl6QjQ(DhbyWMSW?zvRy^Ftc1_~J<0SxDQ?vzKrW1NAL2KV-5jlhmE6 z=%AZdr;k&m<5@OxDp_p)%|o)x+s!$WVfCS6am89T^02HNw6(+n_@zw3vxHRhNNrv0 zRK8Ww-5M6OE>?@%40J^ym|{8m>nA4-5A7&REb4bzdQ24SyHottWzz+9fgb6oIbb|u?_g)9qafvV zZ)e2)W<1`q=N6aR?TpC#gbbvarv2H|C^NqXG1LZHda$Q)VEsMj0W6kCYn<)WTn zz`!?_SiVt+I}m%r<)~#?Cfr0K(!Qd<3Sdg`&9 z<48l{x5U^KD116Kf~pqeF? zVryW~s+Qb5Y_iAFuQQxM>B_Ck@=M9a24tQH80^|tq+w#TWLdNrhi)97Dc(lzA~lH< zOu8RiwY9V3J*XF3j6L`EAWviV(@VLD0lq2LTw5<@Kx~pr{Z$i#otM`waN~IA**1pN z{U#F%D1zQy<0(CT*STZ;UH?^L%nh9`1yhv3)cpA;j__PtS3wGT^b*}$vKumPt&yd_ ziFU;ps^le#w_=;~1Rly+BBN6)lA!5vtp>F6ISVktb@OOJ|?bgn+ z;cw4!n`3P=dR|vMv73M&wD;F0-!&$O*xdDTiT4_3tNPX7VIJ&;wU|3pi~Ep)p@byFkE)v%m}`E+|(U-O)ig%lEsYu~Qj z66cgdNI@id4;E{Y<#@xDH$Y%r)=vyJwm$c^yUP0BGr}fxooB?LPfjIAZTL1vmf3#A zfMUAJhEu0y*)xG0Fw7%!$bIQmS0~)rV(X7y&w24mt~>pc`C^(Z3tfl@K1*Z{36y0| zDh`-TGU0|kvBnlmN5PPwn<6tONS57S)>Q>1uv)8%IUPW0T{5r7$g)`QLL|XQmEt3^ z%<<3N_So>V5H*qQp%P$fiH*_GvaDAgwJ7>bv2kD@S++qbW(vmOacrZgakZG+t*%n` zu4unexJzuru;!idfJ$hqZ^G!|-L9%z%BF^qqW6|xjnQzFLE66GBUG0CX@R>B+JyOf zEp&b47q-XylMczUCFx@(Y+7DHRL6V_QZ)2Uzg@`r3K4|g>f!bMW_64Sm`akP3P`#4 zv4TGJPE%U!>b&Lqv$u%`=n!BWEoJle^|GA*8SYR(L=~*FYoQ8$pbrFWmDItK*oLE+ zu<>89P9+dlLJI5wi=9h#Ds{~xN0IsU$68_y7+}Y{k|OMKw zu=L6V(o|x)JvusPBla3W&~DgKD@T?>XF$O40eZCz)fh(;;06u-0|c(m>Al=Zqnasp zbV(QOkb|WXFrg)~Zv^K|C|R1Lge7)j_F3Aaki{|~`^hyad9=cCXKtO}vM* zwk&o>?F7dM6L_H14QThg>mn<8JNI;4BKzWTr}watEVdUR-Tiz*&I6;64aD}?b!&~< z9UX0~MDs8R*q#M5q}Qaj959UIkt%zxt(Rrb-(lt0Q9ECYQcHARN~{0jwqpA`yE|=C zgC_!J&xkrXW51Z?9<=sQk_kgnfu4<9Vqe|nvNDFHO3;Oo?(3hZ3?xKSL5L>abRm_ku{(-^8h$Y9K`71mC@MkhL11DIlgz!ven~B?0%%O zF2(etoF#39S|aDbc~U*-WU;L&Ei{mV}aIOD9VRF775(bS=sjW_5UyvHmK2UH0c>b*$mQ|TD@mR!O{1U5hOGPf5d91NKZ9)}DMFty2*IXG4jQ0Xm3 zs!QOIO%vOL&!M2*0od?Vr|zGA+Pesrun*@s)gx2fCj8``y|VJ8*WGE#PgdlpJzz>m@mb<< z$YlE(PRsnTwN69MLN{Ec*R4s#g%R$J^u>tb2s zh|X(6A4~je_BLa7Nd}&!oWOQV5LTW5rJ3dh2g~v}ngCt2qfVT;ul68wMJ1R=o?R$? zOU_)eawlT;y5)=3zYAE--@S8hPBC1p))`fZGBcJ@;tNjfdp5rRtVz;{E zb8mB%0y5BPEM|6m-<;N_TK&Sv6ww!3&mfSIEm3fxjOB7C?Ec#-T+e3$g zDGI;N5Fgw4>h zLEw^KQmMpu*uZ=Wn{5zDpC!J3ElQkf*@ax7Ln?nvXUD4a6lI;rCP0ca69v9S$dt3F& zG%uuy9;A$@Am z;yZg8DT{r?4c{RfSkW55lL?I@!ABXG80G4zf+c>$QLm~9+=on02aHj5j6!Z=tnKM_ z`S$eB>bi9)Lb{{s_JPb4I80F_cFU^M|8yfQiinU^-yG{|gyEtHi;=1r#a8OsmL-1J zzTP;3yyXozicNSLBTzZaq%+}LxWW{tvB6!g?gZ)?=mU>=*Jah!KT?a*%Y>p2j>xKK z^_cBJHZ96MCaXTVhIyb0l)D@!Qjz?Fw%TH6a7wLOx8d%VeSUA0^3?~IbRTpEJ%80R zySS;NnkCMRJg%O>{tFDG-B9%Ve$f+E;58kmKOs+7^->oStGlf;7@lW|KWPR`?zF^D zJ!^?~KJs>jC75Ezd*F%ZjdTQ^Kh00i%^W4`%`w=ONu)e4I<*h0tpPwTUtDx>2lzWxK@+$m30wRI+og=kY; zxEv>|b$FejGq5wiT~M=eP6?jZJ9-qlbZ5jzLCO42}N;0f4 zqa`k$xGc*e5dlYQ92a52GF7gU6zzr!J7SDo^_H=-J28%98;xCFd9H9U#aZk$4>_z4 z95U0hgCm_G2lf##Duohaw==}%0gL<+E-qoaB^dJ{C`J*`U0i}2YT_6Ue$cvH_{iH{ zaN`Zm)4@nM+BC>h6=SM!1KI+dB`(!Kz7T8VT$m6JH^rse39`(;V%f+n_7XBiRo*|& zJwK{h;%xRbXDH4p_-j|frgow7MHE}QmqLl#lT|V{Fo9Czc*G6=e#A;#LdwMV#UAVz z00WMVLYAFAnh72hK^L$3*;_F1;kN|NdQ|L{RSDv=My7kAc?J7oWfycZP@g_iT)~cn z|L|}M7#;jqkm>OTflLUrmMCsJP25IK`XThI6NANaWF=9Bh^Dxj8822;i8at(j1~+iaEnlm{5G;vN+IGxWdvB*RW6c`?$xbBhW~% zZAcaOa5R#{0y==boNOP&egme~TK!*JCW|8Mo5E`pc;JCn*#nygwrh;K^lTgFrRK+} zMbgk_iR&Gs!~>sB>};R~0Sn&qyPl<;A|hJiMs&3J6Ip!lAkn6{4y3PReH)At;A4qj z-`T6G;-qWPF%M>X3jyQNM+Yt} z2BstG95{w!!1#d-4W`4g50*vOmjpsQ;kf-~kn}IVh35`nYKg1ueVo|`mbn}9ZTKZA zY!3%NOS$vdVQ~wg4NY~+zJu=bJkmcvNx;CgLa3^bBWFOWW9+tHqN@KxDIPs~k#ctn zR*9d_-{BqxOigk7OrZMc&9SVUKOwz?ZJ&zHLMAZMdpMTVA3MCzgKT1odoU0P)DeW{ zl7a4ecqrfOX|7gjiTnFn$#y+Lc>rdyH=x{{AN4|$n99-_W7HcsQv+Stx3cu6I%;aE zMlPh`(xXYL_@#0UCO}HKcz`4CDQlLwIwebbV0BWb&X1g7g~lMH1&fP6en<>ied#eO zR19D7>ci$CJz`dgCop|N3DUbBeK;_oce`GZNgI6 z@DIcg{jfx-Jrz_S5{TqEmVzqA@4mhqQb}YFPhI^|B}T?x(va&hZhLrcD^`9bOHn9@Dl-;3Rj_P25BUDX)T_!7 z@dH-#K_GySDa&F*Hv}sjvMHLxQ%4_oQgU zu5y(pMsZZ7oH0h6MXwED+Q^SoDGzJCi(~UJpGN9^cU8`ludQb2Nu+K0pgb{RP%nZ` zU$`v)cxN#J-35tdVBXj~SRh8bk(Nt2t~y}Cnu`)_k8gU|l3(&(zpVVWr`OpR8Dd1opm9R?MdtobK z(ArkKea>m}wmz_&N>VDX-fwS5!W|4L@GR;3>Kgf4wTmJE*$sTAtc>kiuQP$XjASEM zdA5yg3TK~pHRQ@?7OG4%4-m)=-DQ_DJAuWrwpF}^+OYkK zyG2z&BJ+Km$UcNAek;<}H?B}N!(ugFwW5^rrR_K@Q(1}=C<;oBHnIA?*ljAntt0Tz0rOZu zp?J{iDfS#B0T|Borrn;*+vrdbOuOcZ0_cJW(pCHpjglegKOjN7^05WQYHCS04l{y> zPq8|TbF8SZy=NY>P8HcdW zL)#?A@otciZ0E;E7ZSxZapG3IF%4*MaRsype6xXp{?bpj1&X7Eg*%sWWEmpl*ih3MVrxF5~V)8u#DQJMwe(}Rp zvKewpFfy+zWsPp5*8oQ(IE>Es;6KWxCT=`xR2MoYjvgY;><%fL_m{5mh!}3GzQ#iAE1JJod zt<9n0$Hl&^3tPR$TiN!kbKg+F(AQnotQF-{MPkMaz&t3%YQ0=bEo-1JqA;+UpWjmN zB1uik0hlRk;cin3Qa7-L_FV1S-m?7Vn(Qc>Tv;3ElRe?v0_|H%)^2-Qdj9+s$0;+d z-7!b}k~opqbKrv#(T8MdjJKn~lyTG^vr&4UxXe5N-ID%sg~A`_9a2$(dH6p_6B|!F z&p2RHl<=9MS_OSzpgs}7^z#(yIXIH7p2WZ&Asp*0Jzv<%eIYZgxuJp6t`=uD39TjT z^e&c-cDH35?3Y>(g47cj=fF@mu{v12HhR4;hm=1dt$$^stnkJU9NJu^_MXY&<380G z80%8|-85AR-39SLi74s6c$!nUbuph*VH~w~20Qgy58}4Zj;i~0uq@lB8f%+M%GF6o zRMl}WfL3d)t^X$x>aChO13WpHkfosHYr()K?-mzO;l}=yC57awjQ?8$|L0pJ!qBFSJuNWb6MQ= zq7opQvVPz7Mm6MucLIm5@(7O&sj3a;Qz)*(p~2I8#jj6Ru^q8LSHcw9 zww;!Y+g06nTg|OdB zvb1m(m>%#neDqOKjkJDloKYFGVB!JBC^0w7{ssX9hPp2dh|iOCk+-~z{CWVzN5Go# zvTo=;_ARP)mkrv*h{qSb$%BPWJnHtp;0(msFNzOl+MwTM;{ zErUF|Gj#B=WFss_f~v&2JA(iuM7F`V1zI^6X!u>_g_|V^I+~J$^X0hf#IXX;sfK| z&Tgn8pFV@>`3=_;F7V44;_Ql7dA`n$YE%(NG;dFKx9mo5rpfAqmQj4s2P5PBSg9Sq z@F)hmnz9j&oRvB~j-{~rfh|Jq9XKbAfYC7)ICPY%j&TkWwD$vRC#dmQi@|GT6Y$8p zAWI?E!H^6MLOw3D;ADln$~M3K{bJ7W5i6T;1U^VpWpLq=q_8U+4LWVSgYA}TAl#s^ zY}7DPRT^@ib@5>6$*E7T!Cf!Z*T5QOT@Tydc%<<*8W82_WFu^1%)sRh6xXex5$1){ zam|E~vT}_O4x@2{l1c!aC4)*YlmSQ)5>k``p;DX-K*|Iqw6hxr2CAB9>Oe^)pxlAe z>H@|}+Q02eQP`wqEjt|@>_Z++YF1EFP(0Tqwh6YcOBY2l4_FLUc1NqyS8^$6=?j-l zwr*6Du+fexI_R1_z!7gu6j7Xv1Yj>G8}XPsoCo7f65A9z&5d!H#fC0PWFV9{_+ZIl z>MqEPC{cC2n66LUY4w9zWU28;bU)N3XtD3Xsd&!-RrDmU$%{cMH6nsbu*I58J*`ro ztKhEEEFxQ$-{!}jW3f$NU2|{=QsDrlaWu^+G?rqKNCYSczkC(ryn*t)qubn_I!66gpVwL6)~b_+}IHfbEq>z!7!0gR-8F>s8I6a6uP2Pj>o#flse zjlkUr0^nP+B~rgp7;gzW+q7k8Z()^c6bexEh0B&96@-QUl*FQnDVyI*P!iMgO0WlO z+bl9gE&Q2p`Rd?rj{CPS&3d?JVagVnc{!&b1=i9fT4ATT8BXjH(o8bpBZQV{mB+Bo zPC}frE-kaJtK|0jun+j4N}nS+8!&OkZ?$r*PI-#{xDQmKizVA;`x-Wm69c+dt}X7G z+Iyqigw_d(GN|7=GBVAkP1$-xiVVUD_Y!TxW$VShGN_9_^9_5L5>j&31w5dRk)VTs zL%x+ThXE)ZFpqx_EBa#ho=PxMT7dlkOvlQMiNSbFd4J zXc92!1H%8cMC%1J)Uq0Fc-t6NV8X{@?3*w$A!V^`ho>HV2k%Vy>##Is+p?)@9QI8~ z3W%tJL2KNTYsk?$VA?K>weRScD0GQ-Et8Gskm`i$I)d6^Gqwo^VCbQxw8IgaCb;?d zkPy5xLmWBx4X5@BXmZuAZgVvr#~LA0M{?UX>s7qIO{%4||MaeGh1(0zLkC@lj!`lg zcZqFC;lS1d5Y+QxdZBJq#;WaWv6+Yc_9%* z-Vgb`k9-{a`!`^9%>%7r%#i!;0l3r5eAyD~>|WpjGg`7^%5m8oxg4qzOnp#CoQG*X z@eOuEE+Kt#eS})xI+NG?6RAXpf>ml9kivvyBvW?4?b>lTg9fI$MTJahrD}h)f_$q@ z7@;ENd+8Nf>;n<5^Bnw z^4ureOc-0NUdPq|iDl(NaJ}?(lz50VBrQX|j%PNYsgALb6Vr0$;~MmNS8E8Fm#mD8 zF@LFo8>s@j<~v~^e3tCoF-JB<+Jy)JX37wx)2HBwW;GBBsHY3LkfOrA|CoCR5KY;! z;cj&tcOwBzcNd*1gxdeY1)&mPmh6P<$MNn%i>B<Bq#ZYhDiiad5(3o! zcO!RhK2%lK52R?nbiu&`|JlPC7`w7_+smpJ&eYrhQec5DmEzQt&+2mjkxIIBR*~vh z$x#hm;)k@Hp1o&q&H&Xc8H%uMv}CH=4WZqS%i1e)*n~UTKv#=-{gUZ&!&A5`>W_dq z4Mx{b5B1gZg8tsJK=4x_`-IzGC*5^K!F4NUx2hjy3A z=L|3G&?LFqeiyuFFbCZQ8Hly*a{aI!iYXivBW+FDrT+!{Ev!9If~pObVVzdVCU{7r zga=9-Id05hj35VfFQE19@?FoFe&Y?`>*5mer7;Tct6-GnUqQY#;^}ni+4q zc4!#KBPGHnR}ii6oCERTDo_77MzzF&c0#(6;`Hfr4eUl3)$xNa5%zRX<%1^>Vn9GC zs2>%hd@xu%EYbO`2q$@dXNn+gJMn-`ENLQeo8WRW>T4sL z3CK*ms}I&Q^l&^0F%5GV6xSufH>IfYxbHz#{3$-n`=@0PF0w%}id9Q?ZImyYVrR94 zV9JQn_w72pXRy^lLH$KKIn=u#t~V& zNCv&79|XyzDv>yX*JQHI-e*Te%0SsiaVGllR$C-8W~+CfFDPLlO<;aLx5a9QL6SU`%3tR9~D# zpb{X$>gd;3z$euZFbpgbhe^E=$!;FaZrPs?E^s%oZdjbH?lH?ajKKo=K1)U}ub`HF zvS0|Bm)1AIK z`gj8~?TQzd-nzd`P?hs!kI0%8oaC2=leRj>dVCOTC&Pp`8W7N0vS*$BYBOGt$ptV? zcyG5;?XPD9dTvT_s;zyQ+^Bi|S%C1No)%*1II0>;|*u8^ftKh{Sy zA=;9?angP>W^BX*%9*k^)?%B{<1W%}Fl9fyQ}H{_^TRLHBBErJw~$TyC$gj19#K(; z(OuNyNQM;fz_mRe94R7QP|cKm=~1F5*BBpaf+hQ2sG}r?lbz5dlft2Io&8R!#v_?& zDz&J-SeoCNH<+&xbfwO*&z&5{Z)G`d9w=eSn7l-}xbb|R(IW!D@0Yk!^kGE?=0Tl-l^Um3VRPxB*#4C}t4vG-W8i(5z>@vnNLLf* zq;T)R@ew^N!hRR2PC(ZkY}D3`PWLY9JjabEi$zD?QgnR~#dRb{zdzJo`)zBsIx~$v zdR?u)(wSEl;A6>WdnVhTqanjuWJXg)<9rZ3J%Asyfu8+3Sk=REQ%KMTdiMAIsvdH% z&_l<1^xNr9Rh+q`7Im0$AZ2oQ-gOqsz@Ed47@tj^YyblVFT~_W%4 z>~{7GBYiG(yjttdDVg#S9s~Z|tEOS5N%VqpH^}M(GUL_8CoA&Uz$FG$4RI@%df9JGzqW5rYH%IA6$Zj&*syXt!EY@&pt7Ysu%aeo4clss%171M8-% zB{-x9kva;W$J-ryLu`8efqDptMZ2x+bR)5PG#&W)EqfV`6%Y@I1=CZf)mNAXo`){r z143dSr*iIF#`kd4apEy>#(lMHdLxQc@(l<$ny?#FIH1%4GjRHDwKU`jZiFCOav-MG zJ4-AE(O~+0y5sX=DPAKR#55MS!Cphc%ytFtz#)O^UfM+pN$t7Vb}?${`jJK&;uLvY zQx0ksmy?ML4@s!Po*UHKFDDc2mTU^K&^9N{X%nvx*zKhFh{Xi{lnI z(!PWTO56xmTd{`&9J)`A#kTRTnWMPZE+Ab7RlKft+lAZ+S2B*@eC+ha>$o!j!ICeS z4eaH30F;;lhb3dNPgnydW#bVr_)Pi2)bZ-MW7jx7H1y)oT!{*XT(9A54?sd_@d;C@AP!4jLM?!*WeN_mb>Nn*3<{NY^pN& z0tl~kWaupRkt&1HeeT6>6Y7&qEE$)Vs6N00T(AYTIa3bCl6Xl&ebFG7#+Si+CfhS` zIIh4IL64M(D?Xy?Vb==`sUd?0w$ZC%XiS0>5FjNv)oF*TNH<_H@PTrbj@fVNJgzP= zc=di&tz=3IEa0&!K~>u~iH0Nk!sU?qJ?$BAC4MN8g6#RSs&8^q#b%4)c+|Ag-M9fk z5_tT3iAQieqRWum6;v)>)gWzCN&O+6y`9JK#@$Xphd!1ZHZeoZ#72nWGGYACeB%df zq?V|UDl*V{rDSVyC29N8h&t+3%uonAl(XbWY%G_$vzn!_fnNHtP*uiuEkx>OI-*0g zeH4p#)Te5u9EamkDNVQWJctP+njcqHfg#g1+j>;Wk>jd7a^u0eAzO0jD@U9;$X3lakkSAi7HZ`p^rklEsy0ae$Z|h)gVZf!*&Y@ zx~Y#!?4#C|v|ETa<){sJ)q2n0*)QN@$q`8YgtV`>nn)NVyxsIhdE=aeUV zp9vH2%Go{~6}17KtBh}vMOUgM+N>PMQgF++U*8ZmGT@tX^a4NUEi9Tz7p^h^<4biU zBJhK@>4a59syed84@->0bCywPQ7d6}=8=F{uk=`Z?oX2_EGOZp*HM!nMAasoc&V0Z zhhyRoc}$#}(9?{3VFxJPHd^zb^)3U+3%=|%mz9{9#xNO5L&>c<3# zw&cX~@y_GJ!-Ws}pq=EG1oS}z@WU2FP$n^*_)WAkVB)9luJ5$~4^htVJWZG->XvQ{)@wbefwIUBWe?7pRFJ^ zmwfwbz!)bAPuh`zdQrRmzl)vJKGB(py)_iq7JDr|&k0L!!(%1fY01er0g$LC-#WBs z?i(u8)0L5mK|Fv;Bw{}1`I2z^<&CN#j_0(}q7(Yl7_!)uQ{$aD#Az_b+iGO7*FJ6F zBw~xAz=br?qtF?A)sK~9u~V@EzTd*1*FZrOFw>Axw6N*tuiKm@r@!SRPU32p2Rw%_ zTux6uBu=i~#S{3IbcXfKR>#y&$n*lKHu9S|Q1+Ai^0_6GlE#EZ<2?g-jd+0Yj-Rm- z^V5ffH};%!;;;Y010Ud9a``iz<(3J5Ng++cv^XwwABWTpDv`7*R4w>kzaRs&mV6V< z^abpR)3z~I^QN4BBGRbc{#jnlqsre9X5b~?ws@!JJb*zLFs-r1d9i*|ntG8LLHHro ziQH7j-F?nzIL&F_;Ct>rl3157dpe1MyWGbb=M0a=ji7F#F6?1biJ3D3)l_V@k%15a zrkrsuRlR{HL&(SseddJwDiLWHh$Oq3a_02QDiIqYAZUxt#5yVwIoNn01rJNkDCO%6 z%-JJ6;q2oiWyX?)jwo}IZ3?@Z@~xKFbGjfa35)9{_Ev|xIbpa?S%Slovq$8}={wr< z=`fg}yPWyheW!IvK1kzugfOs{6Zq0x?h8@Ek|}d;$tANs;K72$F6s!vwSnB@f&gIm zJ)NQ07=pE^4>4tO%l)e1vdUx}-H?-;p0i)VqM3j}-UtE z>yHx#I#?(eC1)X#xdA*NK^y3;7mqjtOYUeg(^;*Wt5gukruZy5tNJ08S~9-C4G?=c z-f4y_B2Y}1nB8yM!MF7jD!N0NjkEmC@!JA?I|=FRL7l@|lniUkV|H9&*dWhQ9H-D2 zNS7Tle#5y70;a{PY3dt1?MqVFm9tLfsqezw53|ZSl|93JaqB1-_5f4x7<1a*H+wk8 zDcf{bYdOvFxcyWj)|9il2?r^4?i^X{?5_^nv^t>@ zn($2ndo8Z~2f!~-5Xyaf%YF~XaDYK~O37Fh#g;r}n@Xel>|YY>bqFccr#M5EKaSa1 z2wbSHwVmT#!F~@-9lzC)JjZK|y#RwJO6ahBMB6!W9`-yOJAs4}9;tD5O41U8c(6U@ ztiEn1m%P&pK9-!b`>s6)&z<52t;-yoLn)8*4w+Dn?J>8Ek9c>reo_}`O*toKsM8w# z$9Y$Y*?AGp^CO1wDI_m4;and-yZel9Io44Hgg&u&C}0c^7A$ryrcUo7Z;4PKv8J5c z+t2P*c_X`lot^SYwi*Ck@LR3^-1*~eYr-$w1w$V&eT2vHLi=q@SaVUu<`Ty|?eq(E zsEKKd&CMuO%duL6gc7Fj^i<2k_R#D#6xA#_FL;#N5d0}OF;;)xAIDV9l6f+lFtyq! z;}U|OEXE~L{Vo{4eA}8N=sqa5YGF<#%;`wEOtMF6^dVIqz0^tsuCB7bx&0(swuL^Z z6^TtfwlU{t?3|s4O;O5{^XnI@b+7m3oIy2eb18_^nJGg>RdfbWu8*8QsXMxA7D{Ta3jnXcwr;Asi~fKmcaS1?>*2atmYKL(+nfbE@2; zH(8fnC@TJY;` zdp%ylq7q5umxWc*>@xbkSV@V}F;0`h16;y`OBf#ZN?b&tDq5GC+ngqNPa5?#j)jZH z*q-Qd$;hnSqQ(*SRvd026LOVB=3V=%zaE@k)VtXJ0tsfSqH$D8ciLUmZ!>_0Dd#77 zI5RT}36u_)MTJFn8*~P+mTpmtKCkoibS#=tT(_tN-}P}u;{lA5uG!YYe}}&KXtLcD zlRSXZ+Ag{kuO96Gfql!`F6ws92}7GB&oN=~FGcn~Jgi7!*|$qtc-vjk_DL*0Q!Z%| zZHsx=Ne_*5iHxzkVqdHhq!dsev?TJD-393%;(;obT#}HjCSflLztx@6;;o*Jx6-fj zXdLaZC_+g31IG;tx<@RQ*1T2Ni4r$Jv?**wxp{L>U z`BH2x;sTPtQvCRRLSAu_Xza_bOB0j zk7e5nRXgN^NEa=(V5ofuyIut#KoX>U+()&=-;p7v!9+-J7->I?GXm}shceVuyvbC} zB_;i*V6|8-WLyMk*hA}*j=MQ%a|9&QHkX@g2e$mu|v16z50 zp_6c^A&JyRUU|EMlYlGk{;s6`${&s>4@``5(M*}g%G1U6gOa%=N&(oDZPbIcy9_WS zKHcSty(vytQ|r=;EVi;R%c+5}60=vG#$H~rk25duK2M#v#H#1B)GtUM!^paKST%0F z`UOvfu5`7i)n@{YOn4vKY8)V_30Tfj3GnDHSK^QTj75SOKWN{sE4PUF+NT@JA3jmTJUeS-JOm_E8}?3h1wuGvzx=Bb@1%HuLO)OROE1 zsNTdfAs#qjMoX?Q<72;%#9c1LLP1Ne#g|BPS7$%$F*tI3&7fAwHPtyGO4(t0%p~JXmbzq*OJe1KqZXI55SB63$_J)52Jh0m3*Y;6TVNbZmqpm5^=> z&sVow=@+Qr7fWvZ5Pw*rWI=;crp$bAnj=5E?Y8MAY{s6%on{XJ!v`KyH#$9WM*^PL zZGYpOI!@11y;uq}eXmE6^X{&8B$7&)a`TC4cGdJYA^^bffl#n2=L(()y#@k4F#R;z zncebB!+;cgrrcCD#F>p%<3mDLMu@uh?)Uh)WOlJ>`8H=>YMfXE9$Y006V`H`zteJu zz7}$8Q}I*|j+-XwDp7K4M8wlxc)SL`CB`7cd4#URUjQP4O@#NK&vNDlXE3k~!m$<8 zrt~$Q^!G5xQBiUWvgaTCsnhywv8}@!s9E6+U9&AqzMp%{nb-Hthi`_;&6m@iq*Z;n z5pq~=35;=Gz>DOF9&N&{SH`PUtO{=EIEC1mqnz>{g}hdW2`#x5f7+;TEWgP_e7ei6 zb8b1kpX_gVK)HAdx(|jNR$EePa~=l3EV&)edSAr}L~<0wn(~9qp-zvI85=w5gRSG8 z9>~B$f^I-N_6-$bcvJ+%$!ezDQJkjmJSQ_{DIZ3r$oLrxT{AXQ?i`k_zUefHoeeDA z<*voP>cHRv_v~WxYjvFUU34}OAz{e_{dUW@KWxJWCte5{ zJ$*DYg^?_|8*9Qk7;jMm|AQN{&)j$BH3(v)3~U$jg*|SqaaY-Mag|tx=dm7E*@Nbh zi}@Jp(-$T8Cao76kgAB&1XJ$)Dn_is)6D^{u-rX7+IbVtNS;1@3O5k}-~SS7)Y6nK{W5Ucqm7{Gx+cjF)7 zMp^63I}9EEwd6+*au>MSM$l2plKafN&g)He?*OKz{P;nvI@zEHdoGhgYJY1S3BV5C z6ulz>N8y}OAIy_;!GnPvFvh5*80QdhdQx+sS97%#(;nM};~+e@9dleN!Br08l2HL( zi%v%e%eNN}Vm~{+79u!D{Aop`)5veGq!wR^>>4|l)Lt_&hz_V3ARJD#`jf`L%V zk_X?r>?B>PX_RI%h|OCv1ii6-$0Twx?4H<6Hs4A(QM;&?H)J!h;X3;1}j`WL%;8_)C3O5|%gRk>}2- z@3zil-{z9G$9yl#i9-S@<5a@br&x}|GjQOcL+WUFupA#dftj+Ak8Wrny)Z+;59CTf z&uk;ThVwI$IxK&{yTw^E7P2lks1N#LM3g#JcLF^xO!`>zC?+GDaCr+q&=)1YxE7)o zZ_8vH5N*k0WB1Ck1w5hAemOQFP?kk%4Tksh-lypSvs_{6}&4 z=^HF0k{~6H#eUaEokYM8p%+V8QAc{!`H_lKd_Wi6;r#}04|*Y_{3)bzahPLv2OV@E z0BQROXBtk+K{0LQGcEhbte0zXBlI9w9&ge>cEBY88=k{w%G^F_PSU*1%v2LbA9Iqj zlG$?*YsnLRJ!E?ve>v}p*aqJSi0Pbo4}i1^dnqrC3{ojQj{8uVAR`5%rhEOy2#CvL`NEBUp4Z{l;Ux z>`{UL@~Cb=Un9G5ZdqjxOV;INO(A>WXf@db@GW_=dn?&vRvveKY{I-Mv8n@@Cb=T{ z>t{TjW~R<-GxU>-3Z;BiKLiGSOnGvDoQ!;1|J5DD83>=(k&$>L5+aEYq&(hPR>EEZ zF%6~|%f}x+i^NH{c$g4`aQVaRZ8Bq^7hk%XLaw|T5F+39`hcxoK!p5yp0e%N7uhe+ zxVt>rvX4l|g@;zU3q^_aJQ=w|-%ZdsR<083acU2+wO>wVt`QsXzya#(M)+;0r|g28 zI)oHeELm7CMNS|66#EvwvE;WakBbPr2253e6eaS7_^99v1;6f-zcV~kf1K6_5AvQV ze+)Y!=MLHBT1a%0r=Ca?+c#%X&=DU?{)or-=8nx2rF0YPE(`mPai-ulSR8P`rk4CJ z_L!51ds=Xk2KIeetn(W3V}z7>oSqph4n~}0S53A5o|YOMRp_&Rjtc z?S>zErHj2dVMZnJS@QI2w^VIp1PVZ4ZGYHvPI`xpWxv2`mOS-ag!;bo3bMMvOuxIA z;JgtrpY6do9x3r7?l*Wp&u^z|i~Wesz#I3Yk^r5>{x~;OdXJw*47Dyr6%r(#qvKwR z#eV<(Drc_8heiNOtR$r0FDi8AKIO#?D4zWC{jEGFCHvuyW_MW>xKJGWc`gOMZc#

4h9YH;BjCK1aktyi9r? zzQt$C^H#pBd$rgdmOtYKs(xsJeBM`jmHUSF9~8T`SZDG*>9{Lg_Q0t9wl6z zA^Qy*Nt%Y{5|0aS$I1RBmwVU^KaDx<%*C-2s7@mgO4Lmk_e(aYSjr<+F8vlEmw10g z7zjFaX?NI}SLFydpbU1yzY{Kw_my>#=LZHdk|i%rik5Ye%qY--8zq0SGUW4kK?cUr zv3@ZtS>)p~Z6*u^aH#T0w)hrPIMk;x%959QB#0Tm`A{fm-(IPaCkOrBf-MH3DX$Ek zCSSl_BarGUSMbc&ko<=)E}8P`xb<>;$vqtQ%hd=UIpi;YFi2iqNcQ2=1j@o@p~ zurBy2hIQFVt}HH-EeF4H#ErDrD^Cu=+0i5zD40rI!Of)wd!r2C(7F^4xFv_+nky0T zp~M$ia>(_n?l>*RP=J)YJJzq>A1^H&(|U;GYTOvv0PjoygKiC1)7OZDnDdjxSnSm` zodxD~T!N((KXqN!#NrGlq$=p7Upi+LW3Mq6eqmj%-boQhO8zPWN~y68J$I`P#3(vInkocS5YUBK3>W^AE=#=OrEh1~*(e?ey#1m&d3~ z*0%QyQSw_|qPRHL*a;l4Cu!SPUiaxKpKbS-I}+Fa6(kZ%GHBrlwinEpV7zKtzf5*`fw+PX=I&f@ts<0}Jp-Br^LKt+FJEwcfC2POsVGV{hcA28zYX4)Au~#C{)4CwvgLTaF@eUZsl*2A(%P( z>y~ZFo4EKn0+)L6TW!-@7;wJAB@!|+9unAg@E*jW*k^#NV)h_-}eLE%2N2CsS z5439&Uf|KPEqOYMP3L&wDIvCUVi!=Zl&8Z)FQlrjm*bAM*UiWfNI?T3*;i!L<6?QWccHeFt* zIxKcrBek=&)h|p6cFI_$Oahp$^5?cjqJ$j@B&dd%F8|z-D7LTyUnlmponk$UD!|rl zUwll?zgM@0X{HK?y*=Q7_tn$_-PGZeWpA4m?0uCHiU(jY9V!3J-Qcw3>Tih8QGez5 z2`_mjz+mk0*MiaFAhRS0M*Y7gtQD`(Nh^zi1z@|1oN=U&4#FPVgnx@_v8MFVy7>5; z@hNYfXsfSR_W0|Gqn;x7))DHb%fFrr_H1<9i$W^@dgG*L1D&*zg2l4r-w&tlYoFCh zOVP?T7@r`YY0*wQN>`~}H9~w!_9&1lXQj(phE6No0X*7f$=c!t;?sbsb-iV6`DC$` z^wCH#5mdWppIE^#P#H)w{kO?Dv6W376w?ITkB2p1yML`VGLo`vKavn<9@FDO6WV`s z2z(ivk7d{+ALzEq)OVb7tZ|)f`+XZIJ~YRG zU0`!(IhhA0>meLc(gBv~yuky8hp-1W&9WQ5S}Irlx21(Wm4Ya@d9L@Rf9MXy_WNO>XVIr27sJ8!yAYf@DE?<(N1tUkxTRE_Wcf!qTk~jeQ@r?~;07%P zNtC#C0^3Hbl^)v2c7sQY#0RlD|Cmcyzgv95OdQy{$F&=*ND&(ls|be2x?-_`c_ju? zM3k!x_C~Y#SHuU0U%K5eEJExZd!t6GaWo=V)+!EhdeD{#NVgj|J}uV_D80C-VmFRR zkOgOkXwQ|=8tleBi)F#_)k+rwtTE}N3a&TfFNAM}Av&W6u-U$oW=-|iD>2{;qWV!028XcAzX~TXKJd0V}@dGK<;BiZ|wUK+# z{%!-Z8$TuF>gwNgl@eWITZPy_2AK>TMK#-Q_|hCPW#t6DWuuukYFaDPNZ*^SN@%j+NNdht7FHyU$PbYML>53#ulX`=_{iIMa{U{LTOaO8a!ckjWVoP|9` z9Q6kD4YhJ`C0HPky|Qp-vgHR4kBXqi%(T|^oT4L$YX6JCv}s-TSY1LRZG3~DY;)wK zR$t?2e7z9s$PWWo)kIJ(d$(BUUfqEyglj^bDXdC@!8F1qO~%RV?ro-dXrzr#UXa7N zm_u>(1II&?#X9Z*U#!x2QKc9(y+%YKh2Lnq@pob3+)qoi)wTLfZrLiYwpQy;W0q=_ zP2yV0CM3@bbw{vv6HYOv6-O!anby$cWU`$8>v(-90H#!VcaKb4Rjs_HklJ<=&vFQx z)gP)efbjk-nYQtm`Xi-Gd#*HI6WkK;_`ge(#mck?X8D*>=&S zlg78Q{GyRxKIkc8PKf!`j}rTYCzZ=^)HhvDld8j>QFM+T0>1Zs9c*iAZ_B zLS9b>{~!q39{vL=Wv55B`vN!1_HPmC>Bc}nF%4I~^=~@Ub3Iv#;;2IKsuEi~quCeB zfM8106mcD))Iy}eI9~`s{j%(U*2S_V zXV74%oHY`{IoYzZTy(n&%nrX#v|c(9Sr|O!qxLM8mX|utsri zJOiZsAc$(&Do7_pG*tpf(nMAaUnf1rZaQqOyp!{Vh&6txXVZY&i$s6=Acey)y=jqk z2Yb={VX6c?aa1hj)J-|S431^{L}R`U7fS@*xK%#O+7_ytF2}zfbf-`(W#L~b=cg5_5s8V-}@`TD|u&kDtA9;7S74M@qdn|?Q4&ib)NkApl) zO0x%M$$S6)R$C0Tl;9TDhfSB)8K`OOZc*_123-R9l!zT63Q9UFhAz|CYC#U7OQbwF zPqrXC5MoUfHqBZh3RZktS0Zp`g_&Cqc_Z6eW zEi1)pmO2q@da0%}S*UtLw@SsB5+CjpAMwprB`_g84g`yjJV$l^0dxqX&N}Nj74DXx z%0KZk=F4YwX8;jkOJsigVVH9l6HWVE zn9ty7c3*%T`EpKOx7GBuG-ux5Yt=x&0vJ--tQ1=xouvDZJVj_)b;io(Xp;)5e2rtk zO7TmzdHKs!$zRJ2?7HLocC%9_72kUs@@igg{uD z;BWTiEUS_M8hZumR8^K8*lM0k_-wQ4gVZO9Dlf0Ks_3;Ln7|F}5G{jP9o+>XPeBjt z94~|1vyy;=M?WE(bNp6Gp&C5C8RyKTb3{`UQh~&J6k8NC+|jrl$O6Eb-#*X*1ALg) zBTNosPkpuWtqI(~(4CGmX@pKD7No4s%W|^o%k6#YkF*1)R){Ov^HexAu;yPB$raNq z9k^AhmuhA&@jmj6PE>2G9>9az%jUFHlQWlal4AwG=%+XgA#mSz@$->qw3HIXBhXtZ zHmuQk6x|Zu0Fh5Vlnm2-X4%dA@0QI7?e%yLi$Q2`NaVlKPZJLSI zEB+vl5uP_DY~DUbJVtgP^ibX(V>iEHrtE1(C&rb{Ck4wMydhKLi}mKu&yty~LbRhW zLAu?%UyRtu`47d^;E0PAPqFBx2r#BZ-#y}~O|NN!sWK>8JjWWna<)bq)ONqj8@N%g z85ro+K2gNZF4RvEuuq-@^y~QZ(kxd^7s)_+!A-q6dPH_#wQAXmK~Il#zLodQ~^WmiU9G@lh11PVGPoGRR+{LJhP;$ zP>pby8|i&?Qa-1-ogky=Ps;(8q2puA1aDZg-!HCi*g=7FoQGNXk3w6IVj47T zr{`JL=oJp#DFsDkI*ZSlCk;`b5}(eKQ7=r<-v%^>fajtPn)EoD(4tY8Oyhl117r76u+5om~%NZu;GN_bx@XP8!oEL_w(1e)*=66T_V%BpBZY)#Gttgq)gpR9DeBTJX&tT>XOn&S@SkX73e= ztf3&mL}81&YsFx)N1?3|+v1u%p0C{drq~4Ncc)k_-OmZo$3R$VH6cHLhA-AzPB|^- zpR2O?uj+Er?H2cCJE4 zZHtWubUHdD7!$VGo8y$SJFBYTsC2s(+14HzW?stEgqEur2WLK@b1beKf#+r z#)Av^7UCGY7HWec6JlWkOC30f!Si~KcmD*1_Hp>Vx1Zy2p1@( z=;FUY1h`A`2FIyDXGw*5LxxgWb}*YM8*er515pL0vkFBtN1MQ)zPS*X@U~}-9t=h~ zMd-q?7=|X+TRgtVyPmU7${t!ut9QnUZJctALb-A!r4=*2C^kV$e7$9>U(Sfy;@fm5 zjU2_2260n@8W#Ut){sc;3o z_08MGn}ihgO$XbW<*+x4{;hLm>uC|sRs^NG=`LE|zSr47|50|ainCiRGdELQZT;h&E8MIayHsmv&5BcnJ0Q{V(VBIs zrSJf@Yt!0fv2@f+x?|J21as)<_FF&KQ}Zeng{^;_^X@E`!_Y%j0qIXm-o12~ngg|gKnf9R%etp0g`vyO=-|1~(;9LkidF2wn? zO}E>85hho$uTF6zHaIRv3} zl87x-wf|KjsnB|!9CtD95ed7^?~!s9-KTB`eoT69OZf9@e=)s&Q{8`X?@OlDQT|WO~BCjc1EoA>?Kbd*==5}6+1{th51aZ zx4E&>S?g!z(oo z!(&R0jK3|_*Ti7Ti;n$y7J%ZeO^Xl8{-Y+37&zNu#PFOC`OVFMwKw(z_ae72s%KyJeHmjSCfdtLQ0vkN60%I zCrq5D-C)GtcF6NQd*b}pEpcyzx|FA@7$eW$o!aTwHma?pP|CJLHbjaH`qMh3H=S{| zae~N$6iOlGtvP}>F|{kfGSUwHu~t4YZMnYl?lL{(tao!EQGzN-lN~>%k^EOF2;tEL z$0pkve<=0S?a=S`$`#SKiw`(-G(?pMZCWgT;O0^}>X)dH3cXM$AD#H7z!v)WtsU~` zO3zUytEz+%TYkv-nvW;NO(cgL%Jep3V>t>bxe`xwR@$@B!pX^Y( zNIbp#OS_)RD%Eo#7ioZOi2{6VUd(*^XGU~uk`GykjREWha z5K-K;hS1jzicmK z+Q`pta1IvztFP-DT|Te1mT_jHuF9rL=LGkos(eOnKRsF&t!}M*ZNoHok#i->4E*7$ zKXsOK<9tt`!MwYXJBOblI009$*E)YvgUeP?X*tS7Cx}66 zuWOI$SkK4w{E8{REW6!(rSb{#o|oenU8UWMt?~&LL-Ps#JY}G^YtP6AT-||(X>)B~ zDix;~OsEpMDhlfpUMLn-=bLHhNU`m<_b(7JW~8KC0nx0^6FLVmOny-f}hz4omc6k$1Q5Y)GRAJKGf;{jcgtotOL)*PAMJ`h>RU$)6nZ{vb zP9UR)f!^-REpqw2nHEM?aYhxU8b_FW#1W_PNw?cQkS}j~^IbK+HO_APSCJUwUJW%& zzn$m2&U3dxs9&Xd?3wADb#G25XkGq$+B1Y9e!j2EXM%fQ>jRc3pCu@P81lS}Ar7-k?mFQl`~s%zuK` zpv2}WSGJ2v^rUdlvmE8L)!VfSbT;smAR?5ZkuqSV$8xWihHI-Q1$%~bof5^AJ1H@~ zNZe1>Y9uJMF!c|0{$@)r*Efb~*LAeBfrrl!Y246m)gkddsRotB4AYV`vf<$Y>JNn$ zrbXvu!~Z;07clL}nNiEbRvA<&*HWfdcv5*|8IdM%+jlu6@1mnetck+KI3kr%A`3%*|dGb z0MDn~6Ibd34#FcRWl9T^vm64t{By=B5&tOlO_$T2yprEN{;|TLLV_xz13l+>lM4*0 zRi)eQcle3(2PW%Y3dRsPYnkk(2Zj#>HtZVnI%SmaOW2+v%M`MSV?On>?e;fhdYiIa zt(3z^NO?nusSJ!TpK*45=)6w3{{0JjFacw#Y@Y6%Azd;kl}*y^u(lV(Kc~0pO)ij9 zqA=Teh_M3_T;(2G=;R><*rwYGYY{0TNn@P{FkK}qW`kJ8DJ1?dCE6d%cTzcCsX~fs zS$242ptvI75B-VGP=v6z0%YPrV^bq`_H^%3XM~!Us-}*Szf17)!jy&TDjY`IVPy7M z+Hj@rP~ZXdD^EJ7Nt;+ol_1S@>~1G@%!eAf78|x?mNN_-9*~x(a0r{Y#aY6ciCj!* zaEyy|mL$#9eldilhn)a!qbuboMU{tRosn*)c=SP+6De}dT|X-+u02+)bgtm#VxXF~ zANJ84r-YvFA`f1vlB>&p)7NftbTj>Bl(&ctJ3p$_Xr|-O$_~$&3uH#hO=o0>1--Pz zv@YQc;UaE6rU*uC_@D}L73aqNP4fVlh!e}0dI8&5y~Ffux#rc@8V8an(ff!sg3VY& zy2|~W?9`HPD+(N@#M}xQ!6T6Tq2esv4(Gx34$tn<+QJyZeU;#7pzLXyY50SIKz~95 z!bTc26hMxaPDW_ zau-#!lnz{G={EO1-EOs!JA9WQr@5z0JjyQ-vyaF%H@>5nvZ$72cUVm#V@`)v(7+X&LD{;lMD!^N<9J(S-)c5O0y7~ zZr2OCuL0%BMHiy?0`F_;aNk%z(EkpT;b0 zkL*J4t*;zV8@l=;tlgpHu(R?%PwB%g+Ak4ZcFVZ7r*xcZrX9XJ;(Ym3R$a#y!TGTY zUPGd|3I2#d<798Sc-2Zd zjVxy{bgLYL2d95vsECL?K2to)6Ml;!%vUiM!D`@0-YE((M$WQ3^03u$4wGUF*B%dy zb6(}Rhq7s|YGNHzg*Z;y00j&JV(Srh70J0kCnQzyQsRzLvh!L;Yg?juY}+rVoHAD- z4I#ZkEFWR%E8SEZX?Kh|=WM_BJ^dxjQ|fo*&_Dz3Nbq0NUOKMbL@0w6gOYwj%Vh1yS&@5ZnV|$=SXi9pT=^e zPz9#7TfC8EOM>9Wn03?}b?t(>{-e{_LUD-HM-(?7i_T9SkOhx))XoNNmffjSq!`8O z`FfaYq@Bhki=Rlv@C4pBx^T;;-^K=dQsl4ganB-e-`ZK)4Tgt^^Y&*tTZ0KTuufOR zd;8^@i(4>83aLw4^5M&aX|8g1ry=LOEtyv)vHlUILaI|}j<>~Uo3*11kHS)KOQDag zxyqH3B9sjXP@0hH^cVSHIfPXP0j61Y=MHnc&HrhrqS#dF-0h%T#M!TMl+z_TU9rg9 z@)mtX)x{wedIvl=LLY2PP@YpIR7x2o#SpHE_r^c)rv5f)dvtD_EOW-4(*1`~?gJsJ zIMa&+RVCf-OkV4U9z3Q)%9NN*vfHbcY8;pVDS4UR7FXY-1{1FO_aF2IowRiX88;mF z6Q%#%s2g&Iil9!2~X{t2~_JZNq|lp3gv?4(#?ueqfGRg)2vO9vUXT4{ur*{GCo#cq2bA)y~!??7VH3 zNZ?bDWLhOv`tEVAA`uuEl=o3Tez_*_hjIgi{*#@ny4|L~K%m9he{A^{hh4bq4TP_K0 zGC(Bxt=3XJ%IcBr_Q>gswv?k_3b5RG=Ra(7#9=~RqI1Go=NO3=fNf%|^OhLr8~QQ+ ztBC)L$E$P1Hqx;8nb4*LM^>kEvtCItetF~o-v6|&A$R_KtFyxWSWpcUe!tW6K8F|b zFGlk5bDsBE{-M5!0ou{`-$NK-j)jP*$I-^+bP&Va<`WXwHt4Ldvfmn?r z^6Ex%#H9v`3|}ZljrmYRFW2hxCA@_y{Gk*?+o4OuD{Q1@!8A|nLXwpRcbw5-X)Km{ zQmo<>lB!aot4y0I+kSC_CWLFIK?@x9!nRVMU%DO19?n%fK)=g`6jIt`JI97<&$;$! zf57<65RoF@~6B2CB$8&atIZLtn3P zj8AYr;^R4=P6?YJ_Ue4+8!YC*x5hv`FaM2PsUw*x@VFM}Y=SE2s1M=R?andJ(Ym%D zIc>RftZb-)ZTj`dF@D|{26^BxA=SBZgg54#e(l#pa^&PC-tJ`PP-0;W4)p@+zBlw^ z*x4Fs)ft(x*4zm|EG2eVILVXB^#=z@|BLX$Zt)Q(W+-l2ROGd5z0*lq20`R#=wI%C zH-ljh^<6hqR$H5ynukN+2h-Xu!oy)frL8tlm)JtN{@FF!>KbWOvqI-x9@$YmbbLf{ zccl%7Yn3bcVW8g|d0Th;Q}r-L+o(x#vaNN=z$o(j9A_W>sgh!N1Xnv1BoFrkdYQ6E z)ZJUHjm3W|P$nFrmPLs1T+4bLXJ;#I zT#YnCBh5?nhP?ZY?t=^=X|gw%YXr(3`Km-0uI;VjhCj-wA7X#MV0}%VJpM327j-_- z`kLT}ptK!hcL|#*erH-8?(<6&ei_OPA`yOiva_kX`r zuYVX_f=4(Be4;6CtllLu&`G#p&iwRuriNb`~1YkRkiWHuAy?|d}4B?gS z&H?UuDyzdI-R|;UnpnXRycl?Rls%$@4~eTk4@gybE#Cn&XBgM_E)Zirb?FYw^ z?P8L7h}J|43>s%_ExIagBJENLd~AQSS0CCz=Bg6X|k5X*z@GiL=B*o{`GxCO)EfEEnHF zfT8&!M?ZbUxq3;YlA?a;cGvmUvVdca7V4L1d#tW?1lb$-uW5wQcf<>om;}XK9(M}y z5Gxa|6qNgAgUsNnTP{_Mwr2`tMhAUb&a~UEZEEG*ht0it_#ieYP2T31G^561Pv$$z z#y_QdDHx;vg*chv|EQLNUm#o#J$lbkab4j%dWR-RSNSSbHfsG~UGs=$zw%pph8%(} zMl1Jovg}Lx0;ODuCe$pE{W!D;Z1atZb`FUE9{5xQZT0B#h2l7`x#A?#ZAE`xD{f}X zm;Wlu1MJI4k;4myWxn}K^okT`IXCQ(;8uBujSwDRtb*XtRX%4)jqeL3Tq&PUu->X1 zp+CS`D)^&+8zCF5KcNR`#vb*;vU-6iC?BhR>y|hx8oz%*6O6^aohL%wkGL`N8vr~AwOps*X1q(>)It)wweE~I)Z1uKwTTBiD+)T! zVCEc@Q^4U*6~m)Tuyx9Wr0HS0lISUb`Aax{F(s@Bk?*dM?9#weCy1_OkIM6HFLWgx z%w+dfH&Y_JP%Mr6P0#qWJ-X&(ilv+l^w2y&I@0d?;$)dUQFm1~`2K^0A1SAG)jd=`0?wnu_i?xi`-3y=JvhhLOJ zY$a*V-HE2QN4M59MMpx4Ka_7M(JxSRWI+U57+~EdjgVarbkOZZGvzp@_FVR-7 z$KzP6=QoC5$}dK2@d9u1b@6pQe>YEr^BwCjiic*}z2QnRb8dh2MK=ro-8l|)inmJC zcuY&_-ha2q-=!D1jib6}$H`pI($?_^DH1m^K+7cxgS2uNszqC#MZ~x2mu`3OzF$a= zHc>+TAf!Z!o*c7K7Srl?AGTKXy!4z>_laA?H#Ad97<+UdbJUtfLNc;y7!$^PG1__h z{b|}jI4jHU{&A7aVV6fq(du`4@d|$imcDYTvXYwlbdcc~b#a><| zrbB>LNNOw?x>bJJAWl(bv9AGjFCXFTVHXg^RH-aGreUQxo}rIen0~$cuScCXxx|VD zGal>y?@4Fv@HKV7VtO19m8^|}(%2*BRYt*Gr`Lr;%&imTE#W70d&y9qi@CKx-a;Z{ zKU$?G)Y}G&IWWR`g7*na4D@t6W=_j@xYI1KxEj~NV{4cwVn<>2_IAm4ZTON zS~=yXPZS0dLG_t_=;saHIjC-3yT{N3ak#R)?vvDG*gF^%Tdlq z8N5M!$KkkqbTNdEX-+rytdrKZN5_LsH~Jue85})EP8KI8=`}WEakaHb5L(9+wjVZC9Nc48E06+j(J*0cr%5jg~q*u^TkPHLWR0@L~+;I^#}(m#7XuE6;o}op2J7URbF#18pcRDS}Bfjur2>0k9vgi z5#k8Si%m;5c-ygxuYyi9jX7KE6!lr`8-Kv`i?h~SoDflB0Y{ZDFIaEU<0>BE z+NQgTy!6TX<7t4AfUCtPAJ=tTJsT|%$2jDR3DplC^+M{4(cX|eeP!2_=*cYU8^6`s zFJ($>z5U7_Cn~*j$;D{-{L=Hz15&Wxpkf^pAm!fz@0|1Biz45CRL`-&)^sLCpft|z z`RHCL()DX!97;nm(|^R~L|Cid`^r)!v>RgiWG1d#HBz=6ZJ*=gQ& z5&At00~r}qVZss(Y|~2dHWSTU2%x3PS7+p{O_%$CL1;40yN>t8Kx$$vmZWBlhOZH6 zKrdIG>(!!>T)N=6Rt~FCsQ97N=mq{(q)9amGOr%vLQ4& zB36_5TS-AVn6{1(qe++>Os9nRAh6t#eOIv<3*ic-ZLcne#F1U^>s84z1-)0B04wFq zxtgigrI)Of*KRA*YZo(giF#5Z&RSDB_?VCP4G(VnwC1d}G9lp7kuuq7%^`3!b&+=1 z>U3kx0P`6W_Ue`5bYp?23Ie6T5xCN6Ko*G*pBs8{!LWejYG`VLuGf_@)@aV+=MmZ< zMasy@RtCo!m4Wy{SiM!w$T+BPE$ZW9y>2AIEJ-xh!-FbFnM^*&|2(Cm5VZeG{X0U% zb2rY>bJHMQ<(9415RO>6IFRz(WEuI;KUx>Vbmal@97uNool-D;e!qBrbGm8?rvK>m zOiSVA@O8ck2Y5VvR(Nl^OLtp_$BTuseX9w2C~1HdkwJzOT`*Lo((T^OxnIe0y3)l+ zc`I8y%q?FJ;ebsL`$L?(-hHu8SLyXvn9OodK^vz3P%!U2ZpZL-xltB4DVCiwJD>%G;R*~fn%4d(z`>cbqyB{@~NVM z^=7bnEf<7Ti5gvkbJ7tlZmPQ(+dJZ{;2A)DR>4$h6W~mASFYd#(|UwND3GVLb%sY&SKS?|0*Nfc5n6yI=<0Q z#h5Y@CDOaU(;4%Zq025+B`&&DP8a`J ze!g&s3k(pCkc*N3nsA5Iav{-^{$s8@cvv2a;RRkM_lF{*^;R_ z7+h`p9(D#kqPIAW3F8Ls7bRriR_Yrb^$5drMG2tzujx?YQV)pd$%g_`We=d&lX6?I zY{`?T3Ws5O(;ng3_l7o*vB%@1oEA(pOD+N%S?C0q`M%5a;nU6#H}k?S6%KJ`(tI&F zqaC+#o@mEDAq-Otjyo2KkdeQu`sVW*cV~<@_=yj-#i~?K7nh&r454MCt#M`C2NT5W zF*-AtiJ&+(x%yvvek^YMUZ;QG64fE8pKZroh!ZC&?y76Zan(CTCDTa$>++~qg5F^= z0|E4K!{Mj1*3BeoQZ@w(CgcHwvpnuwiaoGemffd&BYADwSNZ~~A+*d*9ZSnDM+#P> zMEk(ht1+Q6a)xq4pJka&dsgxk!B6v8x6kRoWCQ~lV11&NJCUq>s}c~??LM67?!YVV zKsQ$JbNh5Tk0p7f3ngHBUA!E|sc;Wp5B+KEb9<3<6In(SQw_b(n0;dZngtdjO*8E? zXMw!-p&5Ejx< zv@?lK(2|P=;@InB+?-ZCjQZ6Jw2j^RU71Z48(s;6!u{$e7t#Y%{5TXba`Xg zJ$h&j(vfzb(>&a``##0Pe1ZA~otAgPNaC-f6=P0|L9X2 zDYkK7NZErQMrQkI57%z6mI=$gcg_<#dJpr(X}aB)z}*q@gN9BO)Mv=D#`#CVc5%$f z5jzGq*Mu@{kMqmD?Q3W0vtQa`edh&=SIUMektVeJ&R-~AS@?}6fD|0jbL5kvTt^U$ zv+Ta}CyQ6gPiTS`+n0-dubilR_}%W?b-D~Lj?z8ME?s3nt<3I|tj`zvtN7?^hsoPs z3fIuF8U#+czFhjKPkjhUljZoUbjoE_3PMVSoKVtP$CxJ6+a&MH;@=x=9$z)3FCqD-e1**p~wqsuc3oL-FjR24%g2=iP=J`}3B0TUpc zk*rLp62^0D4|}iTqO@WPIK=YeMJ$WMHNKBU-y;*`B+><0v|EGYy;$*il{TNN%a;{0 zxrKSm1;%LW?Dn=}7f|siSBToU#{EwV;}>4d7%^T?jsuluNFT>=SY1QSapummNcWge7f}Gvz9~J z_aM~wD8KXzh>_E0>+2A%#lG4jZwt|l(0up%E{~J9H?29E?3+EWtjGB887BCxbnkM+JyartI2m^F6EUei}edRD;{T|`y%vy zZRUu_Z#$&br$oA4A3FCdH}NG@%2ACH{SS%9nKcHXT$Kja_l($eSDU)9?ALF**kux= z7z6bib57pLIt*?wU0=W9i$o$L1C=oaxBwelD2J0dUoq9L>^CA)lyba*KNMrcX6==2 z`W?{9!G|gn^F=A=r7(^w<+f^3${MVyVk~xdttjF;Lydt>m3uQqH@+_lhbi$uq~|?O z{QE&zuhuU5EsYo5IVGQA0vE!PND;$jCyZ<&sNeEJv59+p${r0@aZ6 zM1(VuaPYvC67X0P>rCVt0TN8}=*Pl;R_8t1w=ky4`pKdjpI3@2a1gi$`~};Spj4va zQL;g--QPtMjFfld#bT~8Dx|=t$~TcR^Xq%Ga^WhH>+|^Hh;vQQ!+IKNN>nZsyIHEJ z0eYE+UU67N^W>${#Q-}^k~h9UilBkTlc4Rc1-;@DcphW$Z)sMp@^utm?(F!f`PMabY%n#@n6j-NYbJ<*DtmF=Gd14}O#=-YFZ;T{`!@)7`c1XuCk$$x zsQpr-d5p=FIoFvNrDHUXhYyM`++3o zzm(y*63v4gRqcY!abzQw%#%MR_EW}D3W78}*@@sH8yJj{`%kZwf~y%-SWHXlpA)g~ zQLZ5=BO`?>^$6L)`&O8<3MPX3-y0$1U$cEJiYgBZ@e&u66fpQeSUO64FyZaG!0lg8 z+ZWEsiPLXUD@N*vkvElEpPOrprg`+wKIGX<=jic`a{9l%K~5-WbqRsPlcPCqT7)a} zRXFtjWRx7!z`X2~s{*F~%W0yB8{GV1BB=lM1>ytJ(dAKvA5}gOa>CdfbrhCp9Pk*y zw-V!+HrL6 zR1Tvjp-Z`HQ3J>lkh&s8TMVngbOujoxL-m)&2)OH=Rx-Of$dssc99&*qAWL`)c(9; zxS9-H^d+xRr1 zi-Hf~g$U1icg@jAd1stl@xo`i1f?j!*^Qgm-J{z}khTY@TW{mV&p6-G-he&l#0icB zP~VlZuGs0rM;r;J-6r&lk$!DY)va6&_^8(NCy6FOsj6U)iVY(5!8>$EU!##8*zG*U zF^gU3Vt{>f!Lhi3iTPaUReQvlKjvLx^dvxf@agOGuOesw>7?%Ne3!ytkPfJAWF6t8>};Pa6B?FUU-B-G(j`MxNeDQ=dAdX$L4bqM ze6+lqEv#~wdX!BQ8pL_}c6w5Wl!jivO29m4W1q$P&|;LX(upZ?o^DiByp1IF6tRM;kr@c0Pi&h!J|w=&psr+os~~k6CeI3MWJq4lEB3a&~>xYpV`^Ls(EGCm#Ax$5@Ciaeu7%@X@o{ zl>np4{RhPXCK7oU{qjhROu0VG=gKK|0^f!=nP-F2w5WuIXT+QAi+Ox)KY@fo-v*fX z)iJ>syFTHbJ)+`lwl9*g*ov*ZKcF4_*QI@I&ho<=D}Ta%b}ZLtFMv`Z02(z6tGw63$y*+B_1rjT0Y|2n>t6 zQAmF656O6dwcJMd>=~gh2`KfA33u!h|Jit~ZVZ#Kb+q`1%Yo&-a2Pl$K}_OyEf`!L zd)JB@j?mQjdYA++_SkBcvakutIljEmpg?~GgVJ#RyXR|V<4lghAdJm z90`i)z_4@s@_9XMDc!hMe_)r%;$McN{MQ8Tz^)vBA)Pj$RJD=z!0wHl$JwDlq#~r- z18**nE4bHK?o)1HWQ_A5mwWPkUCzJ>2j#pcpH(w)Rf;Ovlf`vpUVw*LnHYF)rnC0P zTlA6|B>*;mvh!5iXLYY_2ut>h$Dhj5210#!W*d|W96Yu)O09J#t47MU{L~94Zr5$UOtq-`C3atslA}EQ z*NplG?w%!{U{F&h<*jJ3mL)aUFYtJCp|j|pQF^FABtqb7#(=2TE%Z=db_i^vZ{T~o z{vy(Yk@8mDKyede13%dyZ++rJ6{E^Gwmqmpp?Hp7N>xD;y0D7lNwiuzH)e~);M zORQ+Bng=ER8*5GGNPwymuEY)sYAH5zM3euT5`!89i%qXxS0}cfltE-$`#1KHx;?Ey z{-aW>r)^VG4AMc(_lg4MrUgWDxt4M{jzR3IQPJQUCTs^nFL_^0&@aT9t^(;?-@`oCrAtiFV*tEQZ);3%N z>zQda<~p{*VF=Ns;yE+DH=V(tp376ecei{EumP1Kma97oDMm)hpkzn+X>)_dWQrGgGlk;nhbyl? zDMxZ75zo88ZWiJ>cc@^1P2J)g=HuxPuySqFDW~OCIBFN20YrdJ^Ap8f-av}_VULLw zVw0Ow#UPztsY}!W|={Dsam&RwtxVmjDC^ACw)_quJK z3c5jg=VZz$pSbdOFKWk>;oYq51lW#*BSdbEXskn-XPQOVKB7%Wz%J;!FsFWg54l(HDmcPw{) z;n^zwYnu9?dftCU$fa3DN~Gze2hHUm#zSUZ3iwDV%aKd(eOd=hzLxU(BH5dy0Lp~n z8pr-3vBph@YwW=dh&b*@6i@>A@HlYD8$}8xg#+Ix0tpQb4>w>AHgawupEL$`wLLY@ ziX$P}Y~S>L5INoA9-5&`7#`K-l=G&&N$6#4OL$76M43Ir3%sii^q;)v5vD}%3??%9A4WgJAHqfR^Q-wZ>Jpj z>v-!c{9*_^IrSVXSLmUf1ygicXT7egK!osJn9QxwI|;5E&ie@u`7RXFT?$8HvqIVW zx#gDKQk%B9*Tag?B|!u0xKsSV(MUuZBPVtV7C&&R z6CS1!CiW0c8}`;HZd{qz`>d10QQ~rdX#*wpA0hgDQKXqdL`q^oY6+VuOMRi8XeU^e zoDB8&qA+n_tjuC59I>iX0{Yl$+4rMQ^&+GJcKvqoJv(PH3|TM|OqU)Ox4Y@+s0QI? zo`_}US%Q>2<;wbmoA$_20UdooXWNMrcFL=cWNE9HX^TB_(izE}0Hqu_V0w49xXwN1 zX>_@Bf^4(QydpvsJU4fXRYjVw^{8)fupx8Dlqh}l%|IW2R?Hwph60wak=BEL?>Uj_ zZVZ_cE3)N{5&nAOZKM>8mUnPEj6d86+E^gIoBX&YAR44E##wKIG#eb@n!wvWInB@N zrfvwkj>tQUx+!g4&%F^N@7xsavj-M?MF`0aZVx3gG}4!MI!W9+$pyM53|St$Zy6t8 zM7ja<{y5qAqZGZCLkW2B4B+!bAyw202f*H{l@D_1s{A6b&kl((goDy0UrRY2A;R6G zURp}x4~N7}cTChnC*$lxN9E%4-&+bg)nHl`FDl8|r*N1m^$4dptxqZ!io1bZk159r z_L1H~0pkbsA12FSwx5xrE=}QEwp+@0hqns#jVmt{ip+@_IzbymaQuEj4&$Is2_-x_ zEPo4@7Ms%wn65IUQ?j?uRVjM?BVR!u@?e3tFG<~1m2hRCAwg?pDrcSMQ$_P2nWB97 zp80kAghP0P>RJv^T|pJVQDyuR>npyMQP^XbYPUl;NB;{qmlPpSm*{!aNoQUO2DeJb zmU0;DLHySxbS3ROpH~lrT&>IH>JNz>A^qJO5ZcH?Xr?J&w$REMT?VGf6vw2}H`vO+VTVGRoPY*$D@`)&|-)5jh8EazDc~oP@zhf=$k87xaZg~ z<4Ai51J=1Lh5J**XghSTe4yCmW%1M6jy@?KCKojSRT`(;L$a%#NS0hnr~(fN51zCR zyXnOZj~qWa_`U#(5^6F5uzFQ)J}KJ|?Ij*2PHVK0d2p=ySgZyGWWb{ zKyEk`owIgg50IKZdB`)d;umIGs)X^&YimXB_E5cYRiKf+eZ*SNtwpN1&fc~~F8=sz z-F@UCrDv=+*f><)Hzs^%i%dx#Bi8WEuF`HeQeeHzrCuCu8qkm<+pYcPz_RPguZyj> zCY7sCyh4kSj~9zOnorX0rA&1OLw;`L*la$cx~uKU3)b70u1eM~QGMFl$+09bxME35 zlfm_4i&mMSeS0>=deJ@djv2A~e-~ID-aAp+8hiYAr?s6g5Q@1mcA}DJX3d5Sr2uxe zRGcMSV>!SK;jg{cOaFYMUFp%`a4ud{lYNd7Dw>gUK0+L0XBB&xCYIEcgzQYb(bR-x zQllf*OC(lQB{YttfEe-P@P>L6JX=9e>afyDX2z#Vpi7pW)IQL9nZ6LHrejMAKV-ei zBGY=G#gfQ^_ak`&0i}NMXt703ySbole45m2j+1N-#~D{9#hn&Ea(h8lF*p*6tk+09 z0aG)3P3pN^CNWzJ0!NwZl#+N}KB>txUvG&Nxzf6WgOBrlgYTpPTjjKy_FPh>%^bOw zv85kU%2g9f8oSUcV0sTj(;AY-9M}V@-1gdO&7(%!^mg*mvV{dG zSGgH2t(VD@=K-lnG}4>H6i71ELkX=1P5jl(>Jb z`1L8B#oP2_No0;YnbpeoiMH*e$0DuoNoiMwfecbQgCAZB2Zr18Cq1`N+{^r>fKJJX z5uw47R-tjiQ zRMID!3vK-4xdQalRh5qF@@c>N>C#jI2#8;ut2=|2ttPbJxXfcS4evJ1`Q%@=hmFup% zR=Gh%^GG{s=OMXxN230Ec(ii#V-C3(GXREIke2VZ&d{+@#TDxfkhihdSOX5KAcZqm zPcsMdyC|ie)lVi{Ue>i$m2wU2WWHGPp{3)4YDhU>Y<4+D%5Nr zIwUCHQsQ!^NfnFalc%QY2r|kAwUoEbTB!VLJ}1eYL#-csp3^&cMas9y1EQ^;cxnMq zhH28}VVN9rm^c0YP!VJy8tBPetzVe(pqOz~G8w3bvSgA+c;h`vWLA1313%TH;20IS z$iezu*=yP`9OU4025E5X`8?~uf2El#^(^mZ}d zy)r`;c(5lP<)+A}(aoctE3+4hRZ9`SPh&A-c;k1U6DL&Q)n$@->dT1H@H?l+xCtnNA%i2fs5*o6uC5 zTOoR|4F+t}p(I}$D<7&bDX9#Ptg|w4@)6}E70K9xt)pY4u+LzWV`-)fMvGOiZ&1@F zMFiNKmeRl90g*+L$9P@h!5DEL>$nhHU6y5vv7d(M@j6nd^4J#7?>viG1`a=6WmTN# zckZjN$1kXlO&=)`53#zTN|-A7X<~G@+jO%1H+8IYhj%1Xz!0SD(h}0>k92kkCw7~ zzX;Dhp-pI%tJi<*jTLvHe%3{?^|WPAYaYX%IK+Y$=w$~zZ>=)d2MGt1JFwf@%Z^ou zZ$&Y=ELOB*gpz}D<+=fV5-&zMGt@BBbtT)^4e4_DuiKE1M>q{RDVsrOzz-?MT8jRA zGWB{P)gb)mto0#VM@VsTe3LBFIQ*n|*eD6^H1yMVVKed z$CIa~lDShL`FqXpOkP@yuH?)7Xb%;A> zU@(3eI_Sc_^Ne7X8~ja`Gvf7Cn>A82#vXrccYbwuc?^%?hpj1`yNCe?)nJ;EFS3qz z)2$NG>Gn{|Pu$FMSPtqVniBELWdi9OmZDtA|F+n5xz3v`{9i!{l%vXZ8$?5{GUXb3 zz?8>Z{n!*%(AR5Sa*B0k4vPB-nbyDWeWe<)X{JMS&WRibSC~&Rg~!ATB8X&BvmqE9 z(`H(KPR!69ERqmA>ySW#=t{TMSXY^SR$O8xXNgY*hSIeMm|Y1h@dQOvDI@#y7)(z@Mxw5OGGwzy@2ilduD<4$Tt;r zW2vF!gUus{NP_P$$IunEB97*v#O7%pY;d=Bd1Po44oP&9SyDrSiS?muw8xO^MnMN0 z5%f;JNaJ}|iYvw--4rdZH0MuMz(9Cyr??{Z4P^_pmIL9vwT>TYc@e4FKNfp^k7zULTANZ041K6lvPj0P9;lz*G{E+r7K!fdCqvk`T{Hl?(uFGU`0!|IC)OR1 zqQJr9dRNahBZm{M*b4&^DRh^%#*hsi`Aaf zC5HZUM&xo^5M7kRslsOc5|$bGuW5wC*aRKGhY#4sV#C_3^qloStb3Rst^Vb}hBZ$R z*KxGE3?A@->8U_(vgchI$aq40Vd2NU}AKMj0M(T>C=>N&G&X#J;JC2>rYNo!qBwrVL6E+kZehSGF|Yn z8?!}o9va1KrXL$NH`((O$#0Z$5Ye`ta?`U#8}2p2)O=rt%}x+^)6pwuS7~7P1?U4* zC6qAg-?K=Z9g?a0LWn50sMb2iGQ)9n!DxfdgIO@#9J{g86L@p5vEpr!-|E;armI zX4p@A#bJ(kD{V~`Mg~XHFVu}bhp}>XgtgBM6JtnWPWbJ%#iBPx&R6XvWe9h8t8{!B zBc~)y7T?tN)VknXs$3pHDLnR5$k-BFn4W*w_Xou%Y(OYiCMYAPw3sE1a8|*!2Lv{k z5=n}P9;zC;w5<@IR-G63(>(mO`j_(8n^f?=4SStlH!qhKO-e7Xf zn1-AZT`2~VSl|k*9;8cjTObk$%L24DghAs(=Ekc;7&gs!)wfRZ-pLFT$uJ#8RzTV< zQrsx~b-7%#@RBUeDR*q}_T|`X6uMw}s@#$5ZP~`8f-+3WI2|3M+lm=Ur%c{!J<0() z>}i^5%E4sI!=R=Nu^AdiPO%8%YPo_QVwi3aDdfz+TIz#=l-WzHtsit$)-s`;vSe23 zJA81!p!yJO`tVNKgu{aT;o4($tr*K>WEUm!RN$sO9x6_;8kpc4C#O6cZ><>6T8}P_ zl;^ity|^2xc=+imJ7!v6bCRjV*RoS?JiMGH;><`tR zt&Cyj^LuA{@D!)a z(xYHXQ6+h+x9h1DTDd3<{gm)V|9+n~a=0!LS>Ww<<#r|3bo9gfRCv3#Oj1luoDN?$ z(@R=A-8>AiekVOIGS$d4y~9X5Sr zdpcvFX@24m*P+UAwvn=`&>OufSeM9E6RY7@gvyZxMH+`;IypghV`hQZl#u|&u4s@o zQV&Oo5eOXf9L=Dn$`_}@7iEj&@&0 zzu=ALB3u4NV2_TrelPx0k1aLQ)GHQQHu)WLOy>v>QXq5>>#KFyr7iY!n5U9#jr`%N z|6qY^IO-)WHc$0I!wYLY-!eW`IE-?eD)+t2nJXnWLj!v*&Uzr*T*zIozH`HuMaYJ| zwu;%fLE{*{Y0f*vT$!;jG9|#Vr9ez2vmgvjBOJbUiI~D7LK0H2DJ3}lvytwz24H}L zu)C4fn-8BlGf|*g)bKYCiA3|-jSGxyYl%#Xf&fMpO1yDET-)hM#Z*ax2aB?4WGUjm zF7$c?r%x@4@uI*33p(d-eLIXrTe;bZLj8{#lD=CTFWHh3Y;CKrEHre zC#*5YfsELCggu#Z0wIN_<~x@fxY8T*uX*v)K!5*&xBFdf^i13sxiZe%eYQ^2;6~va z1#-Ome3mJ3Vyj5XYNfRe(H0}<)_mPp#~ARj$B%Qw5FYyhx*FJ|+o_>Dz0p4}*2Awj zrETg!6Jw}u43s+hq_-;<*r{*6*~7VKu{fr)9`)%e!@q1KZ$EaoZnp-V7OMSwKeeYH?WKqlMI0{`Z54zMz)%%*UDa?Oe6ZR)50Mi78)F_u3Aj%u;)+N?! zZSUUdIl*2udMGSFf1=nM^LeR`WK(5oBk?smc*5#mU9Xv%a zRaTUW-gl^w>R8X$eqmke={GzUB2^Viyu@)2PE;vExVBjNWZ7cc6x9+GffBzJS}Qp8 zoIwO>T~dGDCnoUC=CqvZ{y_Ry!oDxun;4Wzx2=|=Jma`{5l59MUE)vX`|N5cVEkZu zpin-3V{biTLo^0zbyDZlQl;SW{|GzxxT>o3 z|5KJ0sYzmLmM9vAk=a;z$;)1Aa})(bIijMdc%!`F6-6@+;-=Ag@7Jw_A(b`8O~%S0xOOm!}37@}x2tR28H=6D5vu79W&m*kk=O+aboT z@!Ww9j-QpQYO!y{Ib&I*Ry;5=JT}COG8U@T+~cBl_O1wLOoD!8*L*CjcWXIgIg?j* z)j%7Mi_hmw*M8}sL#hU0+iCGRx8iY<8Gc#Y(#7Ybg8{aO^rr;3H7Rx!Q+_^MgcZ82 zSI^c=sRl47B3Q;2#m4|!q>WsDIrt&V`dfjHKD}*gagOx!2Nw5qc|S(9W5iY$@AHEP zi&X1LCj~>#m2MsJ&BX1hxQAo!7TNmAz5;s)p}oI1OWs5-dCx$2?oh6%;#ua}SKG@s z#ZGOqYAC-LXV)-LX}awG${}3>1Ht2B2d9n6Ijp)av?XEP-3;Ir4{YMMMP#|#y8HD? zib6}3&nv{|q{2?ZK>Yp}p(00ozHFP;#gw2$2`^DT3R2~KmiRkE1wT@@Diex&Q=Ng_ zCsa%|Fj4P-Gmv2q7(7Rn*OEoPDD_G647BK+Txs$rs~!R#4_$WKK3}1mX+RtKM7T&V zh*Atq716l36vOCQl96)TLGciMGR4jJR&?Cy%*1O*QNXarLtUKK+?~ZZ28U>ORLm{@ zR!f0LZ(GEdI}N?t*Hi;X&q${shZYqN7(?i_(`obFC<|Rw1Au9tG^ZhVR?$`ggYd{^ z(c#sh;uh>-c%+|kmT)GI{~9BUUNKH&^S!FL3Vy027hB6&yFh|hBB9iY;oN6lD3Zmj zEVnrenK333sUk+ZQG?9W-|zw0E2rJ4apr_srgp=qG^Yv2PJ*GZ6ay1`JV&fg2bd`_ z{G1ie`LQ%(BzR;dI^pymTTOf*g=c5NIf+naX5tWfh0BI)#x^xWrxaUExfJ~PhqCIv zu%-!fGo8BhK^GB=w*SkfGb2T_!KFGZt2Fe2fO9iz7yO~%Q)NN26VB_DILY&VNt|;J zo>S0w>wwwXUgu>|RVuX`c#(}m1G+y&AE0l`abihQ4Gz!y>1SNY_%sXy5~e_{9^uGC>@pWuu2jST#p{_S8SLat3L^aZ$-xNreL*gw&hqPAj%J z`LAd7oks<0(%Q(GTH6B^PA`_^tCia^UVJmZmG1cs)5A@i-lZ`*3Mqv>+$-Wb?g2N% zruYWJduPOTw>_vGRigGh#oJfJ=WGk2m|~3jH3&zttj?p%t8Tax9@|r7mlkGc1Rjde zOb$qYNB2@j?9XQ$i|qx9dt$%3AZk^Fhk9Jwmi4+gXT6)`h=#`rA--TZs&HU6aGX3Q zzF=Y%Mra!X7laB~f<}VZ#K;;M%aElKDG)K?8yl@yF7;whRjs!zJ05b|wBJ%~LfNUA z+eWlfP0aXK#$*PM;Rde?I4Xo3M`8g#9X;^vys9LqAG$mk z>)ga(MBPmYDqSQGOH`b;s~a+DUYwyOiI@3`$tWQzxE)7*6(2CQE1Wnw6#g)7kVBK5 zp)6IPm{+B5EoaD|S6Eb0>QklNUh(qGD$#;r&r8a;vfV~enN?Q(DUkPV6DPT&sVv5i z5<{XL(rbmh507C7oQ=EyQkM>8xAMMxEZjMk`i-_JK?h-4ksDR_5gn(1gU1~QW&d{P zl(S7}%OxG%&g65<0~ka~^gZr&<~bmMc}5;H$4R{QsRc<&DsjJgvp!5cttcOgk^+?Q z@T$zIaQe|2$_y|Cy^i?&_fuNRMU8aS74ZcN|DJLn9TiP-=ofSq_&^vFFTSDy@rNn} zfzRsaKfSNjhp{3Y6`!AMt@JQ4Ci5-@A0xSIZ4*I=&AZg&5T~-#cxGuf&=dae{WM zbTL)d2yu!GcJT1LzlH*Pev0tyz#myEcxfl!v&kc-FJ|#7l($+A2FJ7GVMtu-q>YSCg zOnXjmf|hBqCyzO!NOu|ln5ptZs*}Nfb`=3*O8h-ioWN0@w(#H}*U6VMbpL^92w#+o zQA{-`?zN~Rh2jK@e;zPOFgs~P&d?wx+^s{qGD4gnTO%Spdwg_7Msisp^J<)Oko?3} zoyW*CGN-~%us){z!VjijH4!IBTcmgx-+sMXobU!D#$x2kI#K;v7A8DLXTTBx#g%eC z+rBf_$zCLNi^@~pw;N3j9^lZTGGRa&$Y$587c+~VbQvvld|WvoPVvG6#nlfUH=GeC zzHXsI3dR8Y*9CEsMf)(*UBKg}W6Fej{sNIIt?LIr2Zt&F4oduyhaG#?8BdN~Fc{_R zn2kY~eGf!>RcbJe%?%cN=Phv-Olwt$3fe8Ycy0f`r)6a6o2tGSr~fUMkq4J*0u}>U z1Q*!JmjkeF23W%m;uKdiszMFOCW+T6?jvb%oxavfNK`}H!5y7nY3eXk8bj!wx$g^}(1C{uOFLtWbBU&|bO~syx6KwO%t-{L zF~AIF4LU^Qu?D|!OZXQPL%*f!18Q$42ue9cNE6?2?*!whYKR?q zRvaghASgZMmK|^=^Fqu;ni!?1viMN&60hxfQXr7>@C%+bQ#cHdC0oRo#26TqJ1LQO zNqpI5Z74n<%$+JudfTLwKy0wkxsE$DibtlF@=A_4$$%s&gd_gOsqWn`*m}%wO7P(6 zN&G^6#TcgB!(?w~kG7aW`tLbT$6pRuVb}x4l&C@YJ=*C+?sU%{K0E%p1gGOM^E?ua zsq*1yagzLWC}x_6{pm)L3G@<72+?oH-xlrs{P4z5Xrs%!U1Z;$M>TZqm-t(Sb2o`e zI7??1cjB#%Zr2OXX%iYA^^OO3vF11%N5v_z@saVtpXn`?az>Z)VdA8>`G{zM{gNt9 zF=-B>T$si&w8HgO{!?tCQ!-4Szu@+KFJARfiVsXj#JKH-9oJ0Zfwrjuw_ch#GNPHr z-yZ4KC2c?@${Fa76a@dnf*BG#LeKWj^(?6cA!uOn4U@&0NA^|wHa_#H+b+!H8#g=> z!h-)`ac4RaWN?t%_RR06LTwHlPhD|uXU_r*#{2P0wz#*C_GzZNioI&JT>Xzu+NMpF z`tb!(&W|_W5eiFqhzMENJF*HmO7u+?Uofp!O!HJ|$7jbnVH};E4Pm!xFYy!h$s5Q? zon&GR0$01fJW;F-6wnr{L8t*eCtb$;^Yv?Z49S)wy~LXa>7gUeDK0DUhZUxQameP3 zmk-tuGLOoEj6e9p^o8*UGo5$6S3`BI$Gb&hh<9EaKIn3wKoRs{qD0$+H>ZPM+V?yy zh4b29bAgOMOq|v<;iztMAV&dllKR1v{2v3o1tP=rmu!3 z2-6Na?_hOWf}DkIPK7|QBKRp_(k%L#WDlFo8B>VpFw;a6lI zX7l`EOz0auzQ?C*3^lxJplLhAH(ZB;p^0Q48+NI`*HIWukL%lBDsL+stxKQ}V(&?j z?O3+0R*s#bAG~8pRHMXer^F8o;gax+F%Sp*&vN8l0fR`1cPeCL*BkX&$nUo%Nk3*)>-tG=z@f zZXDa3E$A(4D|_?|Pjl~kWO{YL^h`SA#xa5gl+m_l(rWiU(yx_Z4}(rA-j#D=6Svy~L zA`iR*X4KCpbvAICvPEc2*Rw7OxY)6JE2LacPA~t*oZL~N36JH8?!9EN^M{tgdk##r z9B;1xZ7DJ3nA6Y8*p{h*wK*-jdJ#kkpwHgpEa2jWQm#ZZWgRAuP8xz|sRO?MV@K$kx}rf#$|&t%jwBiNpA<^&5^L8FRC7r_L1@+t8& ziS8|vv^}2NA%5Y4q0%;}OEBdr_43rz)g>x(odpdCtMx+l1IPI=C!awt^_3(D7dJZ# zz3XyXO0S!aIbU(n0EVg*B{%@Fz*`_{s%?5U!|Fu(#y}`=do@aR{z(<3zA14!Rs8UH zGu;|6qu=h;y}eVwOIOchLd2rW&++n}mX)G6F;=BZa6tE0PVDlBX=1(Z3W(nr7;S}6 zqQv&tCeRM1mRA^y)v_a@cPGn1W6fmgPQu&7^SC>h~&%bL)Nb zjqZaqj$W(gxb@!uRXNExs@MAQZoP4fbqND380PkPe!2ccBMGJ)fBB7(qXK56Y)W)m zd0!x7!ndQH?S0Q%-=bWZ#!;h0`BW!?6e@Y9z4+{2+ls9#q^3ZEX+XVdl=!s0lf|{0 zG}JfH*B6L?d2@Ab^wJ1}5)b2?!0F$Qwwg3`+HbFreJ&$CS95z(bO^($NE3@j;g! zIpS7QVnXo5PK|IrpO&UisuZghmB2Iiw{d0vW`ZtGIVzz=n2h7}@os>j93=*4b$^8s z8=yQM*}MTk5Bc}E+ejckB@e8IhLcpJ?W#z^VZOlC5)u0%OgVqdJ2 zoYh53fe5e~1Tx0nasCGth3Z@PKX~-MZjXCiM(a@T0FPjn7{UEpr5tvp$~)slMdd%W z6hq*y#y_uJ|48_MS2`Y>A9(B-dw7_hp zglWR9r=7MeA<;}tC!KJZEHsSyRTB%S!0r7|z_~E9N;eOr`0Uc8BJb3FkYCf72Q$uBuUG_&GPO!8}o3vQBK)V#9Zc5ng&vs!`&uT23pNg2OZN zzstq1Rr-{y0oJ?uQ8AJ=0|ksyN^DjzpK(wg&NjgO`<%bfC-a8_M~O-2oF8{|)k_ss zy2`^x#ZZ#Hs1i+;=X!Sv6C?R(@rNg6O^jP-_6+?sg$E@bJ}M)4=v;|K5==#cjG$F| zVikcAN>F3)qVTNRt~z7Y#+8atrmOsAv=zyaRzc?nkHS6jR(fsz>)E5~sB<Ye6Ja3b7xam0?lE z*SO(A9kJXyWn!vC#97@hK3N?OeIDKD-oQr-re=|;&(O`HJxfS&K!*s^xAsW?4f+GD zfT4cbE;-DbiDxR$^@-gh9tIS8nE2?^AWVjFQ@CL$=xP#LCCf18`@lB7?bG&@NE}(I zW6UrO-yyraI#k<38>mlqeWz&e*P<4Z@|3oH28D}97>))IO9`LdXF$AIVqS4Kp6heR zDcOZ{^b}W&De*_;`V1)&UAdV84)vo1nJ2rN@0|I}_ZcDdH3H0M{L&}mn0SO6?byPE zRG;)+V$u3!?H2=V;w~|u`ZW`fzLepdBCqvDpl{U{Ta;g_G8`&`Q7#Z$den2aaTI3* z2Qs0l3WY--5^%}#^Gc*=vFYQ*`=Ush zM>@-hv!O$J5RRTH{X3Vx5NQIY&#@h)pLy$+M!7(&?6UKmg~e58fYo0P$XmQ6S0urs ze}Wv~CGpa_B!(51Rx%ac8p4rSKhden>5eLNp#(fK^W`{hP;EA&Pk1aymQU0z*T-Kd z<+l?j?RBm7YxDsbzY6Wd`bWiZZ~Hn=g;e5Q2dwY7bOk7769`O3zGEg1Y-L8Do%rA` z>-3Z@)r~OGx2<#`hiqztqfFT&@sSGa+m69%MRIHb_Aauf+S|?u}>75edWeZ)xt0JSe4bU^RHW6fbjr`&toRvkw#u^&g4hr{o;(H1<|ykEEAYyB#NVt&X6?f6sAm)~5qzer~ zR;R>}Z1FirJ6yHcic8i?HZhb5jgkLV;9QB0R*lfOA?Zw^)A39*-3WmX4}TLmX39{F z)Bt}O9)@t~UD?i&$dJ;$Ow42=mBzm1TqS;nS{(gz;@N)})JN$1BkCHk@{d@k`t zJwgF`Z@VuU${J47>G@3W(APQZS}%9eiwF%BkG^ekTrqq~^(I%}wmW1qrn6u$uI#&X zv$X`z2SUpA&FUh4BIzCGGf~)=w2AdMKBEU{EtJLju01Ax^pftP4^m3v#EGei_tv7bQOC3bI(QVR!J3jj4zVI$;7!8DS5iYq>XYo7p73%)ApSX;z!=p!9ePV zlw;ZAytj~P2poA|Jb1l^u6aCqFw>oQ=8`zi(Y4KCVafKR&Bj}G_~2aS@hC&MAI(<0 z3Cl-&C&-vuyLGHLRn6nk0U0vp*uDCiiD6oK*16c-WPs3OAN_2LbJ5&uG)N!)Zin+L zqrNJ1DO1Wly0@9g=Rm3=1eJ1+`p3&*u|1Uf%If&`FK6YZ?@jVws`%_ji!0>)tQjb) z8yssd$@@7{s2qin{C0B8UgzSZbJ}ydL~?^m)|1TUr}N=6RZ5P?PLm?lL_%pym4~97 zpPS6oUk{_p`HoUOCx#RcdQC!>YaVRHJ{Ewl-7Cp;c0kWm*JsK7O2x@2C5s?GNWlmg)K~*OMWIG?`AT> ze-&d&(Dl_`V(D-&z>-hPDeh;XC)!XhGgFpq*Dc!s>sPyd(BU9gLyTNoOL=RZY})kd z3(7F9LD);u|4X-M9L9uuj?3nK&6J^7)jTnc6sqKf1+RF=f$$9m>TyV}B=a@JO_$Rzx|?hqzf3C^rb~Rj zSx)8~1$0lBz?|R(a=KC9*dsYmE{o}LtFshX2D_sna7q-c!Wsh8AQ;w|OEm~cS zQM7+9gH13fSHP4JY}B_$EQ+gNirw$oaG92JPzRk>uKzt9gRhunN`)5Xn2Jt&qe{nV zK&$`V4%w0$A&ZE@G{v;vtC!^X7cW*jyMM3K)^i`+b_``U=|0A2l zx9s%z(bgDQ0mHmqjSs$>+mr3(5JbYbbn^6;Vz{DwC`Y+pN9pQFf0Nwbuf9<1k(yuxCx?JFY+N1xmqF`TUjU}OgNr{ejzCnV*p|$P*`~}(Q z$UNP$jkepfWFubQ0F(*-8U{*9?BLd6XJof&**-gEP>g$n^>p=uNPpgZyoJpr#2Tb2 zZFUA9K9!)ALp1jIF;h0UsGmpjbXmDhPDVK}c!W!dvI#NPGlVpyPqcN(40}xQr!<-> zoBgGSh7K0Lo$^qsypz=)Y@v({uun7HyBC;GG-m8=r|@1<<4UvCsV$b$C0e#JI6S8F z!o{uEwbIHZsi5o6OC-fzZ>(;&{bxtW`5)=0^OPBpvS6QR?!CxHm40^ruB+v;{387Y z(#}o^#EQnuZuO!vZ;@FnhBibw!{hy( zq7kcK3WrgSL<`r4FVcZqrmHZqcYQF$D-qlxZy}#ST}tF>MT9|GNDW^uux_s4x=wggq1IL~NGXB*qy0_F! z`<0xP8>akV@rSDBxBcx)MJByAe<*z*bl)qwaZp&{$kRy6yU4V$pNJ}EYZc03eyf>C z;T|9MFeUs;lSQMg=7CQbQ=$f8S*mDM{e+*-_P>|moF^#&UQ-sM$_uH%os5*QhZ$|9 zMx7SnESJL2n9%=R9V__bIiht=bGBO&&blnd3 zzNB(}49B=(cd|^6x=x1_jA2?9E7Px=sACaT;km@P}CLI;m;Y$861{zSS ziF+eQz0yLiObsh_JMmRlRq~Xh2F#5T0kSuML74zxH3$pqiva5ml<=4?P8R_dRh0L& z`UBqWVjbqioZX=wX281v>+rWdlnd05*0u+{5D*(U(p(m5Km$&s2meg0S0WZHemk{k zg7eEgW2y(NsZArC&&ZswkirL~XHrU6kvj~pdD{MYoorTYo_xSSlz4BKC}$}P#Y`g{ z@Y*SP>$ayC30BxyoqH_*cLkA&0)rDCpk zY%QQl)FhCcU@pD|l(AT9gG(aUEFgM>mV3ki`cs8eLr+aU6zto5m2T=_foW<~u(x+f zK>Iee`8n~|M&|ClDbcn_JnuabYY2}OJ4KuzQf>f#Z#&g@K!o4*j1p;P7O4rx#LWhn z5^MjD68Bb!OWb=6qlv)-1N9y)o@YhzB1{7s*!ViuFLf7crZ9%bxCHALIs+xv;J9at zyo+ONTd_J)(B=1Tt~KL{>S;pi!+T|Yq)#fs;_8Rk)rr=5uDT&Xm7+wG zj%7# zj>*SvcT~`s?jkk&jC?HePdXe7^o_M-#I=`<1~!TkH*hR^w?T^7uj6HdGh@_T%xF7s z!x{I!z$5xdGg$ogz;=G ztp_9*l}!hZzY_d`b*W0!->RjIj1#r`pVe&+&)N3CrU|mScgH(V8ED{RIo7FQZ*6sL zv4QOGog+^#5=?vye7vK${_{23x9~u@w~ojuw=@Yg51&1-4_A-!3QS;$# zs$*>6tWxVILK_LjrUMtoTR$`7Qv{=nERwfzDK8N6?Lc3Yye(;=0_IgI++(S2V-!=R z=#-{3lN%qbuS6h*Qvr^$178tdkt^CKODJ4cwjVl-yNIBh@$twT^eb{c%`m zq=eY@$ui>djOzZhrj+tjagam4M3B*C&<(q+d1M->j*r19assCqv6vA%h-YV7b74y1 z;ODmo)sM7JG2Gw}&mMn7I;c^qHHYgiA2Wn&r~~GsgR(9+>;f1FNg)07sI2QHN5ZZ! zjZU+2=?g2dm{xA!>20zB51%PuMr>TH^(E&|m4UDtRUX-FC3~5kVeGR9_25N&@5G&! zGRVGQ^=Bn-IymBViAN)?pkqU~<2Ewk6UJk?SK4t|L3pCI1h zK9J%8FvPZtl+6z=4NU?dOz0r|q_b2djB+KT_aA(vlRh2=4%ELu+GcLpy~S|DAYED@ zhVP8j(@3fzWk`{%*QlPhSm1vXY5;q_Kn!J!yA`BH>A_I_ zkWz!NFJ88M@uG!d>gTry-?uH;m-ib0WqR$w31LnO-DePOwJrm{m?MksyIqGAqG3u{ zwz=b8JwPi_AFV+b3*;>$t_w9&m~ub+J?|=H2c_-c8>8e692mN4MOt_RH~1*=bF<4oevO?X;Hb#OJ=tdcbN3dShSH>uDz0h6{hQ*`H4IGUOkmBvB840f9!81L641TR^oT*nThd1!m2*R8*Q6}h20 z(Oyu8=Ehyrv<%;#hVt35$@1dPrNh>!bHbPwSu6KlKdz{=Ndwrs>s(8BO?6h|B-AAOc>5#T*GUA~F z`ul<;aFEWy_wl!Sb)!nzW5}|p;xOkO6m%_i2#-NMw(B9aeqww(2d{oGacpgd59BEN;N+KO9$iDX+otYBMQ! z4$~cqanzfqrQE7}InOV@q{`dkwpVx3LmQ-sa~u`LTI%PshhD!!-o|bd^*sZ?SB*S0xmY%(8&^DF z43C$)h>LI6(Izy?i9J%b8>T=Rn+`o(VeKG^1{ZSGxQ;4s?+{z)J{41(1%au?#izPe zH;~hJAS=OuUuiE|0Oo#n0%D{A1#0md`gUE$2)bCcD4alL&}E< zVsl!5?H90M_0Ph@v&_VSZv2uqF+*;A`&|`ZreCkop$rY2kzJMzS54h$o9;XAEajbM zP@2whDA%#OFlnZQ0hWH>Hfxyog0dD{qq`XT{CJrtKhrIHx7K!Vq3rV7bD_XRIUeci z@?z6#l;cT|E*xkFscQQ^JH30Pbqn(bR5wOWf2Y7HX27}?&!HeCnpMa;WGZP0QcBTn zm8^4~x?!2_wtx>|ak|rh^QSRXfiYr#EtYNB2FLqK+unBi&1dDkcj^6XPyN_jdB?Kh zy4%uK(qBEce>dG0B29^5z4tg|xy=jcrN=D?_PlsyL9vZF}rfL z8-m(EKWE5P_SN~rtHRZcJHFeYP%23<9Z+93>%3ogD2B&?Gx84RQT(BtMTsHNvKgyi zO4~fG?N=9^EVAL?BoEk6qn)*+>8VW#6r_sF(G@P$U@g_uvDiOOi$O_`Xl)VO+aC5q zEwPFl{=txehLZr}d2hiTNq&1cPmuQ_rHm?J2;DcTkQy9uB=Je zbKH8K&HxOi#E3r8&JNt5fKd%8V^+&61oJwSQGWz4fIP#=t`cxi3Si-NWR|BVCy*+p_O* zkrIaR#}XO)V%>YZ)D&O=`jITzmS+H!E{KM}GjILwD^uq4aKuE4pBNeNUxOnfGf~{Y zbfua=x}bH@uj@W}t5PmwZL`v)%R8&xdlO?B$I={LywGldF;Xta$)4W`Ef&#!J0mh) z`qt_+fyVn8mFt{^SMJeFt2EQ5xw0o)ScH@b?Tp8>#JT+@?HO2l+oQ%FcUOG*j0T30 z{PxIZo1Nh^9?%vGQ&L7oop#i*69wH6TJLg(lZXRQiU4#{sPuekyLOcJZAQ`@*>k~e zWeWw2+qPjHdw3a^6&7^4xLMBQr9A4ZpKXuGKP8{Gmuo3VLM*v%MtOrg1MKBgXSugm zkfeBwWOd{p_%;t7rogxq%W&xo~F@ zO$>&{b0cqza`H;wRG>VYeqZ3sX9TO%*HSX}x0k)xenK%tM3+B8&v>y&4qDnodoG}m zW|SV1gD~OlP*{#)H*y%g)KwlMXGe&CykYWO8RgnvlD)|6;~5#G?Nem0A5DMyf0THs zT)fRCfl7b{wQ@5vWskgL)dm_>NA%c0-s0Nma#7hMV@jzkNeQ2iqq0sv6QA77xwaKTn2p-_@*`tPa z>G2_lIkrMTV@mJ@4_8zbQ)Bf}9gD^Gq91j@lqnpezTD1|X^(ddcU;B2V`e==MmPrH;!G#&mL11IKj91*L;z_9?cF$#J92-NK087$RJ_zf< zlj7=dof>Z=cJDFK_wBVxITH?}_SF}Cy(3me+gYWud1Jrs#^FPi3kR*%oG*+EeZ@wR z4rbnsuZEPfQ7%esAaJpTvDoMbGn@|IZBozz{d53sgAd z98vGQe!Hco)}?=(9Fd%^7#gdOsk2j#Xl<0csJe?$?C}l%p-c5wy#~|K*}J;)fPo%M zm1!gYG5ANnlk9Y)5eAWBO!#ZM9D4m{)m27w4khKzcQnE1(rTaV&)s_u7|M;zl|yPT z)gvWcV$76MXAF5ZGhm7dC^7Y{9254m4$G}dxiL8$*CnlBn|D9H%9(cfeH5KOnTvlC3%9O9|y)pFL*h(H_MtRe-@XpfSBp%e%R@ zhe#7+WBQfKyLrY-RnbVt&dhfL9BNf1@|2XZ^&_2*?42kaM(p%tCxb@}nGKVzk@PT0RN<%W(E%R6hYUoCM>+)WW;|9F^6l828)ciB zx2WQt8)^_-#c^^P+~J5PjOOs#7!Z^Sr8uG==A5x9se~_ zGDqzQp5>s9!l6ryJ$6ubEO=axKfpqj$AtB3onN(BL<2oXSeLxPW>e+;%qQzx&oQ6l zzsjx5i+Cjh zDS$F2d~8XfjCtM6_cbu}THggubt2OE<+0u4rQ}e&V#?2FKjy5Hl2d8~i?P__^`^>u z_ui~STSI?5VvoG{{joZNfbX{-&pRfPN!+^|i)~fVAK#S_e8^k43n*aYNvm1#=4oY9 zHLw9``(v_j#wLvvNmOCEl<$5B2ZZlPAN^T79Un%@H^Tb%%3IX|^Z2B5OwP>1BStK} z_P3mpM-LDBF`ao-879;?#<7Y&y<@OC_{WuZv7W1K)-9VV zi;df#DY`O|P*wOr;82OhqeIF-G0KGFUe6GxIgbk{)1izXbjVpqvZyc!l`5phH{5LP z;Y^K+Q^S<`{w0oA0^0=4c(#of^2iQ#l!7oFwoQ&Ay|_OlWqjsFtCTMg>lTVgiTr4( zav`CIVhmx{B{}NAZXL-mhVbi78AV3nDoO-Yz+^>4xN8Ql*WXHj`R$4O_quDI{!Swe zDw~e~ce(TP#Oao?oNkt~;%g<^gDTYJP3 zdWSsJ*T5z;*y7wwtIP~_!Q)9M^`C~{DqDE!UpX#zaF~=T9)Uz)S!+yoLKp*#FX1Fs zt`t*#etSZz&7zQ50iZm>hBJSfnGX9%lndSdhXI-(3#dgWZ`#)d(otxOYzG%%p zay76~MRM|udfH_ApouGUtvk3|tt^I3a6|ti;;-z!DY1rd`#ITqka6YmiRde^eB4<*pLwrplL7#f?l>!^}sEaL5RA zzUIpTL5Xf_XJzklzUO$hvKZh1cK(u3SvB~t`XOavx%k@bR~i#$aXWvlEYe~Pj2JoqDydAayuuxc85BpNi_%wrDFT~WTlwW z7^Z1^gmvxK^Q2X=J#Ha+XBbHN0sXlmxnkf(P0%Ksv^-yoWAU#x1`4VYlR9O|Xim#1 zu^RfME2-AIESM=^T1wXEP2_Ao^DmjtD^-3k5*3{6$_q)E+%L;2<;o28jnyZQI3(sm zP)v>3oA=21JIzx*F!kA!6Sj!mEQYEQ@PUv~D&{f4^Kd|z-bCcHrbKbEDe^}Lm}90d#_gu@raRu-zNc^uv>wlW|AgYn$tven`R(m8~M zz8;gykBhC1>Z$(N^vN}tP8ks{*0GYe)pRI+drIzJ(Yrc3p>0o@xmOfzzog?7K9nGz z(zX`@I@UF?sdpU+*5N{30E>Z^690@Ag{vJ6ohm+i3U3{4BWH$+WF5FuTa;QkT)Ux) z@&44PLsm9ouc|*~M-$nID@2MZKfgV-ZM0a%LKO^EDW5&{&N*T^#!=7>jyvPUGKP|> z1V5_8v=__BF6JpmmAg~LvVA8)7K2cOsaPkLKX^tl^nh8k%IX93$?Q=yekBHDxG>#fZp4xkzSjM<8V34YS=_r?xin1;QVijO%5n?%m zUaaLw8FWZ2zxj?3bd>9wAxc?hD5DB|sM2?;*g`%>RmIqYS0c7}mq^Rxlf}!WAm2u)} z{!l!UG_bWP4Av*-NHFoRRyGedwSQ^;w}~t(bSl5I#wUW1CC%& zk|?oqj}C36984kHJym?iCPokfd=%lb_@49Hl`w?>(wDo)v0o->9=gP|SHk631`~>@ zE-}6B6?sGJ7S*9WJuXAu&_<~Ls-JC7`+mIafAl>)71c9vjbrMT`f|mU zsTw*8V$;L>8Fp2 z^*pJpJjc&xXLmf*V-V-*lz{@;9^a&kQ|xJh(ge)(b2;L_<_#)ipzIb~WPLXLi7&;} zZ)f*DB5zr^OgmfKH2bw^>jy@CU{F#hk;*+!-bPn8MG9hvc93;h{o)VfsO$+4xQ(um z@}mmR|JF^@{g1L3ZeZ_q?kz7WktX=F^9p6%sYW@l!1Qpk7-d#!j6JeT$b&|&t#knk zOy4Fo7I*OKA~sV4`*^&J9JxZ5&{DE%5Gul@dWV<#2FHx)7lOC(uBbZ3cTuZ71(X@n9o0pLK^r%wCH%^cp2T&6JI9DzpIQ@yLpj>4Rr#<+N{Sa;~ox z>7X#A?PXI7pS8+el|K)}EU z9y7Ygmespn@TfuHsg{;3%mB;`NM>fRou58U4dE18cyJR`%u3NXG}2jno4M1TsH4Ag zc%Kr}3*<1?lfj|*AcduyyLexW;))1?6@j*U*H?FaGrzBpt@a$y4LPW)Fem#f_{i`~dgo`C`? zK5{so%~F=urA!f?n7G<~YX9EqFHjEmqoNrn1Ih&MtduLVo_8TV7 zg6TKm96b!iF8@jz_6Gx*2|>VL50H; zx2B8zQGG&4A$(RUbI$2xm5PWx?uruo$(iA~0Rm56jwf|1^-VAJ#DNR4A(K_)8ujPC z8*o>~WoR6R$FQmLvFO)zd(j3`Qlh-1s+6$pCmuT_A7g?CQxj6NPo0$uBL{}G_1jOr zpW{yW?Q8WZP>lWd+zuDq35QTANR*aW#@>}9=X&_b1%!%kuYRs^mho%8p|{Q=xq zNl0m{&wU`yoyZ}(43-&ml_%btBWL!TrwK9Yd;4U=eb%8XZ)qHP%5zUP4!7R-wimF; z|FGDT*IyAIGCoz3C;{|cXRTs}l1UJZ)j#PXm)RzxUr-@^a>W+0kDPA1u_;m@T&ZOR z5}Sk~*=Nsb&9y9EMd%FlGUb;!M=r@(YxEAUanzFq0onGKC-g@g(HN-vUaKjO9@Y+Z zpHD8`Cm&)p5DBJ<&Aqd}c$@rE5Ii0?#L8L37!r&do}5)Ehu>>f@r^Fa&&gqhzgS_R zg_$QKcgZM*$^(I*Yk5|_;4KgZl?i7t+IpeQQ`OF%%P9D^ z>-Oj|E+v2-wNp;>rW)E}PyVt^=6-%sn-DYl?c5ew;sp7VkziV7?#l<{n6Uw6aidG_ zeV4`YRdaOP$JstRw||^C&N&nqdL@RR6UUwZRJZKh!UXZA_g-V3!jU`oxHw)BZiQhn zYyyv0!o_i}%T@DugKXk&1~m^vQ)NGdX=(a%qSep+LjQ7xA*!tJh3tb4zmNgpUqtn^Fp0{=7#{2%71v z5u0{aF8QvT(pG`P9xSL9vH=M}>EpBKb&Zu1{u!--X%o(SWFI_wS6gh}fqgPN?kC+D zPz~ruQsh*o3Fu+`lDFoNHJgtG1XYa^^!&5#FvpTm@P82Uq7&upPf`^IgJb@)74Do~ zzv@F+nn&KssWNtGa`nJDuVt<*J`(Ha$Xak(|yY_=}+!%d6-hW7fT^?GeySE4Rtm7=5`%DMx+wa+GJWHw)yPugB>U z8v1;O3N0g^*RF&BkECPbImV}8Fi7Xe2IT!M{;LQckNa}v>LVYh|0)wwMa0Q=YiFwe zs-JD=&p6`d-|)NcVK5RU66(u#@0ux3m1<)1nRtKy$UfaCYaa8yUM)Au35ucOL5Y!5 zWxFjcs=c4zbe-E^+!0xqPvd6QJIr5FF7M}c%FK}W=TEFJ$G%`|rrPTBIZE7$G^3qy zcBMj^S5_)lY|tBJYQY>}+j+_8;afshx9#~8OXb*1z58a`-291`+?ASpHrPGj1r$5k<%t^)l(k$`0f1E1lgMB8Um)N`|SMX zMdEur7es7{F2S+JS${cF?c4l89po&gkyJ5A^S6}B##~|rwoxv>DpGVJt5-(I+4)(c z<>)GXdPp?@q{CMU#yTHPt-lPzgNpG69TVY+~9uZP(8P%R3F`|SlOJ48NN6OdqRn*VJrIi-H* z>Nb~uWvU!)X3>g|&t4EXW1YH?rW5F}Zvl_6a+!JiY+VJ$5SkUs6s`|~(lA|6w^U}f zGxL`u71|4)i4zMtn%7+}Y995k$SJH$poeK53&_$u;V}^kOP{^)e2Tl2%z_HKkrH1h zC$)G(Dds7cm@X%AtN?o&9{EQKqPYIo+_{geq!uU*jms5_0%d~L)6juL5I#2WUJ_(Ms8aNALt*x&>0Y|UfAPow2@ zuDB~6AjM)GcFHx6>!$=27R36C<*MK@Z6Hd)orE@#hSXp~#%t0l*OF)h?yC~^H2 zxg`0J`mfP;;Z0lQk~>c6;4e|1f`zx9mPyU)iy2saGm?&KYrAETJT7* z3x=kPt?V~c88`Usf{4q~V(qiUDCf5e;=^S9Uiyxe(gi~AC|RF`xk|ZA6_yKsJ0?eW z%+XfYVhiSV5L6ryQL@U!LE?;wBslJ{c_noXLv;N2ON(=owMSEnnQ|DOaaLF78%K zM^KUmR(mhX&DZS(eL#A1u3X$Mzq&;gT(?dxmal7qsWR%E9LxL`)jg)S!n9MVhHiMQ z%92UsRZt~N1T8I=gG)?qN{C3gd|B*$rk-w5Rmy~m&d0mC9DPH5&!)d!5(ij7R2Jh$ zm4C;If4ycB=I&O6MZD_vV_>0fa{zSg?1OT*1}Qtr&i>{y z9pYLRBF$%Q5sPlq`Fc>?#8^$jkc09GE^H&g#Ob07iQ@gHdfC?m|Kfp}&YOR|tS8=^ zmDoiu=gK~3E@>$m$0F|FJ@N7a%_B)``>POD=Wo+~0gG)fnteb%!H8Y~Gf2sl_C!ul zna_M)7thRe4up4;5AvBO2^UhnIW0RbGn;K%%HjnboC8Y=b;c!2l~~+qi`dA3Rk?u@ z7?}<3j33O4Y?&%7Yck!wOV0OBxCb;+I+Uj?@Ol<&4Sm6-ou)Z!cMV!JT(U zwvG>&BKCz;afriDS4%7|Ef%Y|xzw43JM=;9+i7Bz*(WqSY7%PglLe2|)?!VS-=js~ z)7iQKRjPJd_;Q?E(sq=#m`1v!`(7t_@(lq>m68q|+tTGD%z=nv73+xIoFzxrJFlAu zOnIs0WQOcdmer<|;ODcKj6N+!vORebiy5Xv62)J>y?U925_6KpWZn}G5-_I3@Ezhe z^3zp?Vru=$>C*pCh)vU8lO+TJla4v6`)C zB?ag{yRccYv#IlpP!`P;dx>*dY|rndy1s5@+P7V<$Qdl#VpGKwV9r_b8Xp|QdIWhz zW-s}$+EGg$i;>xH&A6t-U3=tHtxS?g#G>t`Q{sK@1SlLx`d@^cVmad9X>yoWWU1>U%u?=*q$y z8Cv&`2$ityr&^tMpJXd-w+RPuaJ*pvvql9yQ#GK%hmMO~EbOWh_y(~~K+eEGXlkSs zmP9!7$??CLb~{CdW#O=O;@$2h=^dr8>Benx$y?^_YwQY-XD`Slds3AIyBnVgeY9-5@Mx)+!>u&01i%g#i+8w>d=X#}{r1w^GTpgtFY1871U~zz zmfOS^y+`XMHQ@X0r>5@{Sxs-!Mn*I|SO6&K8mrozk_5rsC=0rq*E%)fsf8)>=7$bw zZ8h|#HWi6C+4rEnr~a%I`AqZ`ov^M-F@5UwEi#G?h%(dDL1=eME~#~OpAf5u=gXo8 z2kH_a_1l3}tKFv;y`pCYL1n_HZm2Jx9`=m-?kjL8_d~8MjvcLsP8!G37UlA$?q(BI z`(pHK_lPyRFd!ED*z-JTx{J2)k*yH(hcXs>R^oK7`*<YRv`BI(S@$B(3z{okQblIv6ba#|2FzFmT z_1kxpi+VO!@udh@{f|hOb_kb?I_ce66NO8E%@MCH{#7>*t^Ts+8{PcDGu38{>f8PH zvcPJ0{$F0wA%*DP_R{~7d$2k&tIrP9+9_M`(aHoE%GvhP9sA^@r{`7&ZlDoqz3(v- zJOh2mM!BqRhJ~h{E;R^MnQ}Pq;h>8$5(auWTHcb873wY!JMgG%dF@kR0d5Os(OrPS zd|m_FE{HOYiDL^@&1Wy0U&|evU}>uxrf#CVu6dj4uW2B2ubj5QWPi~}1IPEuhgmPY zNSmVyV!w-&4?kF6g{2x#`RwJ*I=D~1I7%}mw2|`tPIpkxqx!RkSWI}~iaU@YoWg;T zAf!Z#PZ{J@8ekAw%#q#yoS*~7lo(Vj>s0tduJqf>Um5Q%>@-s&ML`H(opuXZ4pTfd zVcBbY|UofKICE%JO$}-G)cs(+d%sm9|Aw&$@-!qY7sSwfc3B$^PX1DglR9e|f#j z5PEe(toqq@k#CPXixcju3Z?vZ(PKN^{O>R5_z0-pvS{*VH~;hLA@w25zUo4N8v7D@3)zNnRu!#OI73se;P?G;_a+>u>R=y@fj&~|txIc9c+LYoq! z^4&h0^&*02F%nGA+qYU-+<30227$fC#CN7^SC%NGD^^Z*TX2|zKa^d4_KKnkx5d{j zR3Bz|tSF0hr-dEURbWh&kN3G#|9zt-pgx38YPr+ypRbXIsS+z+Z12t^p&|b@A+_@P zCT{LKZPg#94_Z02i93Rk<3+U9JXYqOawoH5gjh@SSW%KHJ8`@Lg8JF^$|GCc1suJ{ ze9H1Zd*#RB?t-qd)r6uicFCtdN(;pYzzWvMsXGd^a(Noqv3z%E`Eb3or`@n(bvHTh zE0gAn620w}&5LB5_h#%?6);ahUN!`QA>JTMSd0_EZuw_7Y)B|n&cvd+1iH^~*DgMz+_C;YzR zs28AAXF!R#LOHwe4&ARC9`_14dr6_{7}d|Vi~oL3?A?8J&A?}`s(U7QjO{0kqeR2B zd3$GK)o)@Z>DNZ97FGPqI+?*VLHSl&toYpsndhC^M6_+M8pC&v?6+k|;Rlc0d^wp< z0mV&~XZq(ly?NIQPzs#iekL{7N#rbkC4``I1cPPU^Bp6q&>00iJf+&46MN3(Idy!n2{hp3> z&0~!(;0oT4ia|L|Slc|)T~>I9&SDkRNVgWbop|yR3C8&?XHMkxr{yWFnLPU^2mK=nauv$JFY?Y0uJ5GXP9khAK%d3(VSI7l8#))mWi zY$SP$Sa)wh@Oo0oCBamO)Vip2ahLa~9egP9aHR7PbJM)gKwy3RYPXodL|v+& zF(v-C!|g|&O~tfIL(i?_{Ef{e{!rBr`&pLTuXwN;y{b}F2^0pore=kOps$bY=%zh% z&0fL!m6zSZdDp4&x{^edvR&2*a0ICWbeR72vh@XrCo_>^pr4EnKFfkTpp-t87PvDH>KmJ`FQmI+2CR5&Robq9a*Igdc=kmI{2-J}weD_XUbP3U-`;Th zUbpd;xAbx;C4Bb!uPVBK!P*v@n$B@Oxo&=WXP*pU4@>}$(}C`9bB~V_%HdS`F0%U; zvdwp9*i%<)d-hECb(E;21UOWQbKQSsR-l+_vFk5HcmMd>bJ80k5}X^IiQ32kWx@^h z_gcSld4oR;jtvF1T>H*o^g}8B2op@5=B`5=XK5lTu{b*suYCHrOpG~t_7u0ZexQKXVqeJvmPEkdm~>D z$qoPACLFNG?~&pjE`U%){g9HI=??EPNlEYsH?I>{emo~xrBREBK;KvH9A@uTmB`ao zI7ff^%j4>JmHNTs^=8gVhNFm7Kg5aA%#CLYD+n+o!DDriSngc`(AsXgF2aq^KBJ9{6u-Tx-BCB?p$CBARoT?2 zmg6ImFTylWh190O@oopw@+x3Df;RqiR-B9bNzbAY4Uakd+^*ybucQP7s&qIl9wwQG zvU-^Uw(0RL&S7q}gF*c$G5(x$xJvgw#$ubY0?y${=QJ==qVECcz_r1@>AQ_?UfrMc z9}PNK zSF!7ho}e3{(dC;a&hxaFt8RF9Vn=sLv9`Ez<)+=qVj7#m)K|J7<@FRXjdXOFPjyNV zK8ttW;?w965L?;Bd5hiwo2o|Wv!5Lk=5{@`LZ3p?HhuQ-FtAE2jT7bP5R``}1AFDQFlfH&Ej3Y*%oE22dunpY4<((pX`EU^?k%B{@WH zd{tW=a8wy~+@sLKnvl+b>idV zZ}q1Dg3sREn)5-tA_7At$+kDghlyu6{SyQS`uOe5&vtNQEBv|yIH>Y{u{-6~hU!wE z3M)$ND|BPG>_Vb9cG&!5H#eU4^8Xln*SM;R?|&Rc#Vd+p;uS+fK~#)P#qjKz%~2G* zu|*?MQHs=}BrP>k5k*Chn%9IvQMsrnN@Q7H&fdq;C@eGzv9$1$W~HH+rk3`9ubF)~ zyuSS&{2%NGXPud~-s`ew&t=Vw3CybQeEhIO!v$<}Q6+@b&-pG*{}Bh10D~#zQiV?E zTS9_KIE{>M%qfvgC62S9jAKC;^`*sjy8GDAVQ&*)%fQ_^I@u9~TXWzO*^KX8=;i2x zVG~fwSl_7~a|FF7hmiuu&d~*qAnfFda=LP7-%G|IER&L05H)otZio67NKLV-x-;44 z=(Ipw(?q49oaLC(Y4z8F2TFmkvCt80#SsQhd+cQH>{}#XUj|sZ0$+H5fWV)il5!k!4C;_Hcb=T%W zj)334a=i_^n`St+4z>%X{sev3Yl+MgiA$s{9#!2nA9;=#0+u;_yDKHmD8Xz`3A`_v zE~=UXZ;GKk=jR^b?>K2BqOldvb(@mf`u z=r|(+P-5>%$7^%iiW2$4Q9sV+%ZFSRpXi_HMc%zS($V7UJmDx|)7^7JjA--N)l@R_ z?wNMh18aPQ1Cl^ElHq8vw1;rRRkHf-6*a6F3n67F0gX{2Cxb;`m_XXP5s%!ao{p&_ zUJ(0$mH#&Zoh#PpJ6b%T7SpQc2G%$%E9K$x?Z_{???)&!6a^^L9{>6%;pSzOr~)ZX zf)x!rmZ7N(huj%{j!`?S#PpA_SgvKh@f(f|Az-jPszg^9*Kn%nZD)g$`x0}sU@wc# zCX_>o^iap}#}?A80+5p~<1Z=C;KNr19zxsP?O_Hbm5_mCiM0hRw&4hpU=sc;cJv*j z3pW71rslQ7#oimd#HvP1a%Jvs6~@0vumcY1%3NE9r3l075D+BO8`q8HP#>DgMk&uL z+0hp%yd(ugtD4s)$@mPr^avPC04eGf#~7RsLrj4bC1US8hJ6v{e8oWM=b;}!!va?N;j=cKcD0j8|L_!qAjRgq#=%@i~!F*?D~IzVQa3FY##{2Zx1a-5A)C^7Pqk&7ejfk6gc-XMSb z0}LcmL69Yert2-Sm;wy)CCrN)YrKQQJmxx0m=_aaLNH8#Sm~F%@5efphR+hg-%+T4 z0sWwzRl@35gyf4HQ`k=}W(Z)R}=8|$T#<-VMCYTE4^1Ti_Uii#cwEU9CEicCl-;Z^M19%)R zXDhInl3M4vBhhRH&i+AtIpEE26=YnSWf3M6ObV6hs(>MVbl{gSbBqS~G`e_H z2eBR)YGfklB#z#gr$bvf9Lr$BFZn$^jZ2sgYYANt>zbP1Hqqi2e6H;jY0#%0l8OYfb!Ed7|%+IwumSO{06~v+x2y;@5k1#0^f8OQoF>Bop2C5>a1)gDS z*=P5uz64hA^bunrj`Rm88Tc)WAP_zVZ=z#dczgjbLlTYo97V2&I6T~3C|cfa79 z5aYm4AG#b>@N>Aa04b~aPMiL>#5eQxWf(=1SipyrOAn39a>|vA1j6Nq#xJCC+_Ib4sB!M$eQ6%9%mP3sq&4&h$71)!y~g(>)|khiJS zoc&~2{ona|8n#lye3I#&7V{lV;0Dx}NcRjZFz#dgVgo@S-Sc#saX+h%lVDZ%-~zU2 z%xxjQOknq8cmu4P5>u(%qe9*Qd*4Ew-`hQHF0kcT^Q00{V&sm;?d2U%H=AU(Gy z@ra`(a%nT2;jm}S5w;XBVH4nh6o`EWCz7Lc)g8K|3c`kDwgwCSBo;P7iA_GnDMMa> zA_xJ&H*456(}d6%RkFN|;_bbtkvE~QtknnO)W~)S2U%s`5l1W60u`#P z!$||U><1D|9P5kN3god}1=1=Cm_2I`v)X6-3Fs&_Qr+7q!1%KNV8sp4AqgdL9Bzj( z^|spYx$w|Pz;Vg>;9(QG+{$uv2(c^e;*D1Vv z$9oxLZnhPEfuT`l)I;M5B$Pp<)ON3yW0cgNQQWI;Qr^czCRCRZv^P3i`2+(A9aW-8 zxxL8|Mme@35DxGGVL^&fo_J4WGf2Xm3%mu^=ZFU$lvs41?ZORYHV}XVgp?xgar!Uk z8wFsC!?*{MIALJ&3kXYmK#&v6sD>&_FL4i?xKskMlCU9&d-NYHctnX3MGv_L66;|c z84i0_S8xwpLQRCSIv!v8h|ikjv96l!zb{u4B;n|6hxhec;sruAh?8Q!uS|R$$ z0@z+`L}`YNY;b`YL1-KgPpG$Ve(&k#ywxL_rj;*%RRr*+BNOT|z0LG=+*G62g+pa6 z_Y7D`z)Y*xRVYWPBwL; zklNQTR$qs%w;NO+IeB1GErhSPVFT|S`rwJ3TuWu#}a3J>?5R8fgXj047W5TUlRxUh%JNKlc5Hfc(?G|63=JMhR`tda*tG<0}W9)(fGj&@1Q zL`cCVDDh+s8!S(omq-i4eAy68W0Nj60#?Yp*ifXdTyFn6OSH=|K{h?g+7LL?fLF9AFH>u2{=Pe1KghEWPgNii|rQCBPKA zobcAqt>D7yvI=gdbz3e^D3kVh6y{ibTHZ4PEMWR^7TbyGV-z=W{1olzhHF*xVdOHQ z+`08^r#V{;9*}Zxt0NG{Yt2Q&CTzOD$1z7AOsT~KhlF08#k7q5POGEJ^}9^_d#*Fa zAolxBY!_yWVLsWQ6#lT4?ZS~%s4v@(!n?uxK`iM(G3mMep}~$c9IytY64?HB{toZ1 zTSzQ*(KU5{`}>Y|9W;UCs__0TFZKr-CE_9MvA?UYqdoSmK?=1HsM2J;l7ea5RB1vG z0?Hjd5k)?jUvEGL+TT0c=r6;`3OGurq2K>RC~FE`U<(szV4{6Rq>n&~ z65!FllC}I$-cLY+4gsE zC?fPA1EEBiJ3EW9DSnt@!|t=QPwpb+=%Et-AzY^tf``OmO<>QX?*MDj17Ow&_AAE7 zsjvqmfiOOXw|-jQDr*zaAM@6)18F|0NFJ{Qvjym};a~YS>|dU1L}4-ms>^WLzr5HO zjm6k1AeBsCs^wi?$tT#P9FV>`n|F~3hcc4)FTKu|m{<8gLExC@$-8WcafQ_W%w%H> zCJj|Sex;85aNFf>~v1wZw4s&7>fbje`p=IpsF@#7}899P5= z4oCshYvHUF=08ZeR5Ia#Ub7t$*myw%lmg+Yt%{06L`{pK%ALgNIMgV}Q`0Up57B$(ykBaRX>k`MSNI)a|BxA*=)`@_nJji=~e8T<#j`6)f` zoOJ2}9Nog073bvur93hAK(7#`2e$0yqdi46g8qLA-LENr81EBwS>pfda$v||Wd^c; z;5F%}12eK6UP$-23IkPH=h+~2qsY4P7dBlMqMtw&=pw`6zz%=If#cSrfCKRX9^12-0;>}afP-=$yF2kO(P2`cpi64jmHwFg%RweUl?P#LEEembzVu5`i#Rq5Ggl~1k5w)5^k9>bEJy^1 zCguFueJoGvz=3Kg(QB-6`C^o7y0oZO7H@$sA=v{8qDsf>Y!tTcKy_)tqRu&NCytdM zrr?1Rz2bDz6rLj-VCr9?e1UEt@sK5OtQ8i@oE;KW=@M-mMFvqmsznLqo~*E6#N0S| zm`I;YW7DvPgZk1QMT5_X5-4tJi-QH?%`KUz2C)N^S^SKLF3%M`9mY<>gd_#Jpv0&` z)&qBQlt~^SumaW^%HdxV$M7(H0FoDo;3kxdEqC;LL8T>5G6A4RR_gsQS|jD)3L+fS z8zKV$KTKd_{PZAsRwofhszt*qSgd)of-aa2f5;{xZ|im)kLY>I`ny*}z(}!Ah4VLB z)QT)Xk^q6R1CDZZ%w#6)gQV^R}!8Tz=0I*G_>pYBN?A6EzQ=0(xA1Q_Y%3-wpZguU) z{?7V4*yT%Aq?EnB%I_asbD9uR@?+VS_v+n%b49V=Eajw$7$ucP;$*IcvK zMW5#?=g^+s1`b)}%VYXlbb6_ZG|-7yHW0fwQ{kwNLdxmctS8?6k~;5~cV^wN#0ix2 zAlCB}^^4{ViG=)nqi~;?JV!am2aoxc*RWPmcW$mz_ z>jG8@zo6R#P$sZT-oxHBx7wo$cpNQKZaCIErGW4st6yGk;J$g+;jTs30$5`l2%^`; z`b{_HgU~i(3`p+=Gr!w!31F}qnBJbx8kl#bg$Tl5>8u&TvJARJ2{$}+XH9VQBJ63t zK>wG=!A5r0u>MHA=zb{ggGC*pSYx~% zz#zSU@Tm;P0!$85H~@=PJ=nUIHNoeEbP>Ki*wT|-z)^#+mW+>sZ920I3`J2~y5Zns zN0|G9cPU^@vE!21FpNE_oL{knU1QkS!6I9Zm_kY*w*ARti=_Godl$2(;{ph!RR3VV zFy@ZlIPnk`JE){kMuZp?0WEZyUf@_cCf^nO2l1WrM8M%+6S|KZpLgC)8v~ge#mB+% z1;%d{fBIL3!@;k89J|jg5+5AlsDtCP*g%}c*bzMNKne~J9*EZi^-Tk*;^-3A8}_(L zAB%(k(Kar{as>TT;(=-?@mw{tAi1T)`6B<9urR{-9sB2WFa>;6Syp6(;RSLbU7XEU z;3G~vQiU#WU1Oavkx6a<0aaEXVY6`vPd=2BB?__Y2n`!mfn8BzZ8@8U#HA1r+yoD- znonCOPXvt;B|h+Uyl4|cNYmgbvUBa?Fvo zM&3>-4fJgg`#g7+Xr&~uU#=LBVbY$I!vk$?(Z?@gj|4%Nz^t|_u@CDlP9J;}##$nA8ud*h|2vkoT)|z<#UXbe))EW4q+A)jK!>_tbG%~C zNyB>}xHEQ48R>G=A_a|hlz6xjKV;w@YI}+|!0T}vy5IqiHfMN) zLpR8L^r-3~T%5QXdE6b%(9?A^NZNWHFTM*bF-yd8b;6vlR!t(Lmhqa_%4voj0 zB}T6#MVRo=6n7<|e(KvH9Nb$NxsQrVM;+Q@cRU-PB>FI@2IxB@9Ahw9-UK)R41~45 zY>{~s1FC_r?V4VKTe{t(*pcd?Jexfatpg#IRrYMv`{GUv6esmjqM%mqhs>RhuqjG` z$0w(hckva=cfPlWzQ5u~`{HY=VwTX)u+zxvONC7()2|ZsA&}tWQQlb(!|oR1fl^>v zenx-#NVMPqJ|NV@IpSLm6}m{V=kK!AcgKoe4&bB2PbtcKuI~%{f~&JKv1S^h<5#IfanD@GW4KrXKj@tS!6RLvd)~>nlF(w4(i~I2 zL9iIK9jO+#DIT_M{B_p^K=B}EB%=$5;)De}cs4s@JaV-`#Vo=22y?KBkP;%Kuo&Z` zbBib>!@v|(#^$i|xCakE$k0|0jhAp@rPpS!F)s}F?(tp;f7`%U9n5WZq$ zQ#>HFV65A%fnv}sw0+-qlcNvrp-1`G! z*af_V042q~KP1MW{%IL_$PyC*l)2`*X_Tlk&0qQ6>=&kzw(n01VZYnZTrW6@}Vn5Ja=k)RW+rpLfw;GF{ zfz&oT!8kNvvCu_I$@VptYdKCgK+5(=5Lyc8O&xBaR+842VjA={U6{2-9+*dbum|SZ=|Z`Z#szE;raK@-27gH_cXqX(x3Gu6 zQ4(>+h(&f7j3fP0;$O%vBgV);vP58_@;73gc;KS7oon?w&WSR@-Oc6Ybo0gl&~fY`!rnc2LgN$h9?d)FkeU(0}n_U5jPB5 zBAhJ@2-bpOrR9SNrHS-K#)>efgyQt5YRPPG)wjUW0sUQe9lwf~^ zC-x2J<>N;Qq@WrIw~w)x5mN0yC_^JXY^j%*avB3(8|*>q58Jn| zkd4R9fy6@)j@&uP{>I*WXe*cw)1S^}e`CKXRUxEdm;c#nY&|yGX)(=$>tc*P*xE%% zNvx%LlM3Y-EQ7h3pJ6sl$|k_A(xg+=texYC51W`;M+jz3m&_1 zg4IQI;Q(Fo*o}L@^UWnaQKbkQrV0=rb=rP4&(oL)SE4>KjkbK9?rTgmzxyE}A<>rK zB7JznS4#<))b{HqYL$*Fya{{I zT}gU?JWpDxzYD7~cd;X+#Ppo<ZBDMliol*XEVn60DHmHw|QEeQgg> zq>s_?UNe_2{jrndy15JnJ{lW5J;LyTa;8nKY)H1dF$v=ah!hPQW1CZyG|a*Qy3}^m zhuJIuX%>(|kClx%dtLbpqXO`dV#m0-8KV(n#1!B(HWt2(vd$FRW)fj?s0RzfLaHfN zV{yIj8qZ-Hl4(;bdjTh=%-htOb%$?qNJ>1c9s|EXq{Q(&+OeCM{sBX(rn1?8)Uaoe zssW2ju}Rn}H#hfz;GqzYq?{yn9wR;*VjZHbZ0^BQKI{_@N}`aCdad_ecDLR@g1x@+ z6uXBrnTRQ*Xlx!|9~Ql+H)$%G3$_$_#oJ^?H-XdGvWxCY+O^J3ZLMs@3_sQvy{gS``gbW0RkeWa;pa{Hd1kLFZ7U1Z4Z+z=aKybh7!lR zsw~Snxo#?T*;pHEEJgAYl@Lrfuc)wun{R{Ac1sH%%P7~ht;V+bZc;K{?x5gbdJ#32 z*>Ju7uX!SdfS!%J=6k^VC?QRVtR3!-WaBy<+uNQpRxRpX_l;uPk>xtF?=oRiVW6E| z0@(FGrZ@veV>xXGv7}OY9*QjSO^z`VTQb3v9xKbecGnn-WDWv`M`L+k*6N#)%waQI zZMFwV2i^X+P6U5PiqkzmC9!TYZCx6>p`HyfaohYQK-24Z?k|_*e4SUzNo& zGdj7{->>RZ{tiWI#e z)UV@lC@Iu_2MNn{zz%&Jq;xlTlnIL+dbQL@LW_$Z5_<86*-97l)*TTa@4pvdOu+V1 zQ;Nn)-f3=sh_5=~zyreO6#d88%c7Bnqcry6m$k-7tO8OA7)WCuH4igJ;go<9curLL z5L=lRqW@9H^b;>HdoWI&H9e=Xk3SAFlF(1i2aiO2#b}`#YH(uqH_gFBU#ync}5!k#Sa+{XMRstR@BR< zW97}m*!bWZ!ckEezp$!K^VrC9i^Mwz_!>L68Y^dL40NXn&)th=z0vZ6AUapk!^GHm%(#n2V5X~<3c}8Ua2zy+@>v1qjjF7~X3H#x< zLZc~;2S$BqkBejDj8SklAr;~9$+pexvdgnq8D zn5gp0FU*pQgduVRJg2d1841c9=#dWscz}R|O!uvi6IIHH>93z>=nV$GNj*ntvELf| z9{(`g-Np1qMG9|QB0gy{q;Age<38}JFD?3nol5;$(Q%VEc@_nlT(Sq>;4v{Ib@53d^;q zFRR?FykbAK;9X%LA@+7-oTX--ydc!SjWhngLAE}ANSofq0Xh}Pw(J@IylX>#eQqC9J%IAeH-TPiOWL=GaJV2|4OsK)LcHN~AhV{Da{4Hv9ZQtFeD@ z`$s!$5ziBU>rM zFo6t&N8@e}JoIXev&kNU&|rbx=!9)I?EpOl+H!Y4Kjn=TToA5OmBzasvgKIgO9T&5 zqKVgawhS%IZB!xUcoUy!B@gQ3U!fdt(yK_XLZbwq2v>5C-)nU@j0!T%_k}m@8)^*0 z@+6Q7tMg`PpmXJfyRZjup69FofS@C*!zNbV;>lA+6nX~0A(?s>d9%x&L!7a$anG`N z7K)o#qnu1V8-*Krf6W(tDS#Wv+vpdJk(kDIb6TwJf@sTT*TCD#+d2GLHHJK7AfcSM zXBw+Uj~><{##Y|8c(E}P-+fZ94DT)XE=@4{j!B~*1kB3Y-yCGwTCabK>!!RcyU_b2 z9>8Jc9S#&IA+7`}jd#eoaDrwPps7^f2a8jA=7q^%I*$AN;eDbpR-32<2paE*F3Sdl z19YLs$~(v7=5uUw#lO-GUFJ9Eo#ER}m=0S(uIzfPv*L|qbU>G`?6&Hz{sS6msuD%I z_|8gD`pHC4sa!V<-k+E6z7*SiVV2Sj{SOqEp6mXEuQ3#zJmQf`;qW-d9_x@u4o68G zk2g!uFJcUI8}U&>G1mPTX2f94lS;rIs1jOjyo&B(-C7fL?Lj1a4f(`K=-Q(Hq|LU`aeuI6wIO2`6akhwK2oI{zfnz z(6>MtfZ#`Q>D#An6z~Cw2gzE}Vo&{%#s|!Ji6qdY@~0knpu{~F;mM*JBSpEOz{rwE zNacKG06IowAc)rZz@>+I2OMD!r1V(%z^6Rf(*5yHduV*{_9HxC?puOTM!JL!w)0-* zggeAq`Jl@AC&q(moq<9F!+5}3Tg84;paqXk^9%*uVn~qo7<@O|U}$AfU)pr&npz%! z5l*TM2aSh)n9X}!mHi-;()iG1oP&lrr4o2w7--m_=DbH6nQJWxHF3Nrc)$h3)XFWb zaMKlVG=a7-qxS!YFvi=m2YH_mNm8P@MGLX)!6tb8kkBp73ytx1f2TdHoU5~Wz@EEK zzi6C)iJRbmcHI%el?&C%Rt&4JI&mmP-o{XK&Hx6oa<%b&-eINdz+A4jF6AAP2MQd} z$I6-81!E}o6T+1wN#jhvaN-R#235}HB3#A$^6?4r?u!!Po#yc#gPVyM6V49*Xg+@; zYPr+d8V^5>gGd{Rja#yH7~Z3j_d;U;H;^PNkNB%O@3kpM@PJLMe0W8Q{T~>abg_|N zM)WFDmZAwJ0|~JsahiCq53f@FFt|bEg7C>Wm*<9Gk2fMQTmdNQ%7{;@jae9sfFOB{ z{%0%ifrA#nP)dnfbjf%X*~cB7jnbHge#%RjAR-DOlWKywjTKoC@xD(c>M6N z4=^bUU=rzs@m|A_Brn9Gl*T8Z?I^~S34X|TZhFLBo;f^Gq}7Z?Hnl8x|0c zxNecgCisPTM3J^r7G2V}V4|uH`V{{PVgqy013gDFvVcA#CWhIOLjupy z6V2l>(s~B7fmA{;P1v!?XyMAFw(?mGBG^|r7{z%YJD>HLjqUi-$H_zE^FOokH9J0{ zz2>y_M&omjjumGj5L2P;T+H_DKmr7QNF%?sFo<2mUgCV15Ra8FL}o}iS~y4%=LhlB z=jItI2F+FAVWY5Iij6(I^&hULZ|OR3CCc{)@tOiUN^Fl*CgQUODbj>X&u1tdJijD2 zQZ*}2n>62e8U51|*}_=)vdKQiNHb1_`peLaKZ7nDj6;u=FL$pr9>aVgRRIo-uWWi( z&p|`qQ8EQ#vXA~QGVQ5^F!CzveU^?R?Z8lKyK+T^F%~JB_*Vj3nH_2j$5@PblnJK0 zLyQq}6N%)p@@Tpdj&xJAgq5$nzE%GPtANl&I%?(l&c+x(NAyS@>z>)l-x;x5OhUq@ z8ee@k*zm#XN;?1pIE}AKImWt;C=wQfSSw$XU!gz18&xM|jpL#IKXV2i>Z1y-m+yf2 z(>j~lE=B5>kq?9Vghk`Fno|AOqsN4iK~TBvcBTHSYYqj3fBp4e%~f-tweq!1JL^~R zd9^tWwE3Vve{XsZB1j{zAJkll$3TZ|G7a-V&_x2v zc^IlpHK&lkN8>rz!We_WI~dAlE9bx^V}VS5lIrig7j3M@K~E(RE2`wS+{&B1--eEE zWZg+@ep^5NA@X4~^k9dOP4j^x!(4qs6_hBdv3HFcsz3@oR=$V(o!Dmgcb=&d&G-6O z=+#j}SumL5iRSydUa}v>(oYVmpbiMH$Jq4?^C%IFvi?Efu|_DeTp@uTE8myurgwFp zElw!wPEra%@NTr0ic1~`zushwlufWy?og_mu^!jnp*~q&0@Gm_o zubk@5Bff4&|H_c6M4U!w^}eQ+ORs~+I8h~vtX`Rs#UpT9G=7*&(f^2m)nQ=rt;)ZP zeyFd;t|M5B9xJbY;y&MT;t1`2kPXU>M`?WD_<`aDLJ5uka;AU}-`|1Mr^m{F`Q3*P zM^6dBh=<01ZM}(AVB!uxB+_3u<*?i47DP8u0>kP`9Gi$Avcz@IBkbsi^2e|8?*87M3vR^vB{BiOenQ6&we@tXAe%#g=@$o8Y==WFZ>%sZe8sju;y z4gJ`+NGm{n$@FGm82c9M9+`E7zV567-3`+~V7fle-UA}*2)FiCv#;>cBIttYtvfNS z0!wK`fT1;BiyOz!PHHZ^pGw-^{-m0fVk#VBWk}r)skAId@U;9xMN6dAPj;PQz*Gd>bD1-^4yg6gpk0y15@? zI@&_wLC{sV7CCxza{&`bHMP+@jBmUy@1lS{n(FaqI4^X&AiBkZ;AV|6thj^;^@%Zf zRF>-Qt|=rCsw3>Tk(uEP2i5Ivu)U_ZFPsA}Vv zuknq4%VTJTSoeeeiXVa>stez$jq`l@^Oz!l=Y@gP#*wwGE#{M{gaFoLZGn-w@-I;W z27)fny4h(G5>4F+SD77QiRwhaqe zf6T*>a%JS(wn{Yn9lLBuEFP=cw%-xf5%W?MFhXqG1$R$m%y?MWT&QgqS12RRo}i?r^l+cuW8Q4V&b24K?xA<*mUPMHGtF9$3C;^ z9g*lqB_v_xd_~23Tlac&=$N1%M#q`r17_6J4)d;A{4kY8CWL6K+Trz+mdDK@AQVK2 zi^(h;;XoxsoOXDWt5N}NbLjt|)O{z0r%zsu)?STOPnp$9!ywaZ6LpMrNEKPVic)h?BBYyehQY)*@HdvZNngzcrN z5Q|4syFD4p7GWNjs^C%8ZvBoJqp;{_df%#c3ywA-T}c!mydJ~GU~QgwpbB^-iqLP510}n~?#ks|eaWZL;GaS0_Niyun zy=dq3t#6pS{u9QPU`W-hs&9<1J`p1vh^0qUeG{{l5~QucYclA3`8?x?6MKgS}iGT+sjwc%vKlIUgi=^gxcH?dM{GO;4cObD@|19)b=SB=nveBA7c~4-=_X?YZ1j z{~Yr$C6EFmf$(0ur5H#dlB$64VTI+J`Wte4ev1=|u$z=hfR9yuqE{*NM`HkyG8~== zPGY^$fzL+?ltKyLE5;P$(h{&T()Nk5rK}fv(&Pq!v#S32H7q$P)M+tQ_21{g=3sjm zB*-fMXS0-2EIyJhAZn`rrEKLpgf?IkWALaxY>B|7b|e<1G_{xeY-0q{tVjwr@5rw@PsldI!d`Hpne*0v_O_slDHdXW{r*=#bKpnD$r2_r!e@ll1=GOkO#3bZRc^z^L5kdni`0dfG*}oE0drH zUb|pSkuRY{+Nb>`qcO5O2$(3*r>KU%jNSz>kfd>HpM_q`SH?MMtEqiTD)k&7ty3#LurXAgG9s-W6u6#wxz;N+HK=E?y?n_lLKgu&NUdaEkdo?r_k)Jl zu-33TKuKUhxQaB_++inh1nIeKH+FLXsRS1ENtm83_nONpeV_5<5onYkfh4JF-_glN zj2u`=V0~ZR#0qhvG#F9|P3^m+IS)sCl*o6^s`gz!pN9v@;|m}Oglu2-b;A)Z=zWW` zxcW_N;z18ed=SsWv4UrlR-&kn2Vh9PV0Y!hWIij3$_r1qOny zs{Kx;pFm=ptF`R+VXzT^<^0?7H3g4*gIFspgP}Ny9jW$jcZ{_`8-$>hy7Ygil(obT z2oekEn%aN+5u-8uLM2cQgraQL8oNoU1h9Z`JjQ4&7ayq#m|`T}+B-#jUl1cG@nf9* zTVzt#Y1{v125-FaDPkyX8r&_L1>q$mU7|?a;DMLyWf<^6ti%zlrSry78oc<3(Qt93u&ETg+|y`; zBUDnQZ>?(Z@&Ka|Hqw{LaM08NtqPcihK-ypstjl!!y++!t5a?O_IXAkM*y}UqE#Ib z8mn)S8AQZqq&lEFft{$A7Of80bC*w>X@{vj=%T41SmX-)xw}|h5mG`vy21l7dyXGcY)H8q z4-8-H;|0pQc*U;5`y$2$ zHk`+!<;L`7tT>>2)>6jVr1Gl;mGceHuw=qu^Uap;^N5kTF;cOr>w1t}SF7 z-9QwVNC)}FGKSk2yW`CU7EK+*=jj^`O?F0*RULHZK8wa^jkFaw29;l6cC1AwLR(?v z!MI`fQ#7zq&R1;khuQjhnFKDy4#t6XW6ZBQs%h%rlRm5!E`TMAp&AHPLF_W1lP*9D z9GFyzZ&52+ONd5^Un5uz#vUYA@^}=j6EaE<s=)dIgU>-|0TPXqknI7rvjRNL!_A{@C>c5)@q@apbeFkIG*Wql!0i>Gx%&Vn* ze53D#2_aTf!+vSbmcx}%vMn5^4jXqt&#AW#qz;?x!zN>S0re%YVXN*MtFb2$K9R}| zTN`Su3Ed_LAX?R784>ImbL*c_ZdgVF8;LfVa6my#wLIx(^gvRp4Fre=;ps?yEm|r3 zpb}Qq619~b!3l@Z)GYCQywO08U*zjy+18m2L(V3OOE*|BZQE5IZy}@5ax8^)#mRB7 zg~{|_9J_%@CNPz{SXQ56cd(1I#HovAO)a}~x=73@N?_ktvOlnfLrC!e>CFswbH^yB z)iqT~^3^wD{sWpy<&>9>=^JIxnS{;pdLe)ThlH+t8_Mos2R8x310D|wS(;iRv=s(o zZE}r<7y$!=tirnb8Vm65*I8XTyO>Y>JVTTyBVepw1*4V@C8Ryr;u`i1zUp8o)#ttK zdL_nlnbMV3mE((i4t?!96Y@b<*f&?|u4M+(`-k~r94Cc1l@{aE7qbd;eIMX7m9K+X zbXiO(AiV9ysvC+EbtH7Y{|pO2nkR}&ZFy}hD?$rHZU7cdRWVt9&A*!nOJNT+p_2XD zV!E)pBz!a5_%pDOB1pDl;bAe1u}4A|!6STP8asj}*aj)7wC5~*Mi`%n0UlWlk5vsf zvREnxymdIje~qvw<8uP0q@bpT-?7=JA}y&dKCC@)=d^r9O|ifD^Eu|hMkr-ftwHfz z!$egb59{lp?9=TcA4Y0xEr`?C<0>Bl23=5Pe>}^L+(#&-O^3gp%vNE|0t^YMrVjro zj%8pbuLRIh4TR(FECZ>pR6>Xye)1Um6icm9U<%8l#OH3j8PcQb2qW4gv7G4<&N#KI zBSyO$tDEf?Mg|W}9T9Pt<>4H7)R#C$U^3$qd~k>$Y&rthL+4>W85m4Lj62KgC_Y#O zA3UZevRurtqKd>ZBEB=r{Y%7HSQ$Yd5pQR?vbm5Z9I-e}v0=nlh2RHTl-N|Md?8cL zOeq^8ly%j0&{cJWo?%?Xs}4}6*a$bCh&eVgkmQkjO<9jv#}8@35x+e&zJH}gl#sgo z4kxJmPsu2%EbsP4J zfgwFsb>!0(Y%&f)M{()fk$c=&*zf7i5~@0?vyYLAy@0T|RBqIg(qZpms9{>ns*bvO zLGKI2beK@sbX4tny_b2YyMP|q*qyCHnwHH8eKamS^~41z5KG^FO^w9N_q#|&M{$WX z5|h;HaDhQbXR95#+K)}Z=^jKVBdbT+*iVkQg^!Lx~`k9#tLPypVk& zZ*iAF7j=FRk9ul{tA!aoAzWXB-WJ6=s(=SJ_MQCvEQyq2M@K}nPjJ2iOlT6GgOq<7 z*DEo#hJAc@g76DUfycBW_HoQFE?q{yG?smg3xiD0p~OXBV>L$eC{B-69la-)oy6CJ zO5g$XW8r-2ta7Imt2*YnSYssS*j!aU&*f9I`jBGufXCNysA6%30|;jm_*9&aiz-t6 zF*64lBVc4`D&u4Hx4wMp^WEv?p$Aod^WanE)Si5!#>_7@M%3HLrjEWjh)-R0ww{!m z7x>gI+w0-D=f+yJeF`v5hk+2x5Y3i68mz3wQQbfL$Odp<9;?UB;z*7COPuyoZK72%g z`l#}IC7t=O`LYTKUghiq**}#ysKkr-ie)-OdtB$S?8S;t zUG^AT8_OfH7ELAaK_rCMfdd&(73<6m3PZ1TG=Z90BkjdB3P zCgLH1ZA)S=VxA5Jscp1>jUI{&04gDvM)ywAhhP~DKcusxlY{j1`lp;l=f<;{NRESI zW{G=)48Lbai{U-YsHqyd*00NjAu3^2C-e)pZ^n0D$9pSqObEKBD{`GswjUG1F6qPS zB`m8GX3S$rm|%t!v&uWA`fu{=Faigr*_ZTmbkD(1;+TNG)LQiUsR|xVov?f>TZ<&{ zg*ax^pMXtR)tPPlwOq7?-~qz_Li8~h^5BQbS#2 z5H?k^kT)g}1{n?$x4E$pykf8?NkWOoBlwG0N&`dbxrvh)Pn5T*nN=>F($`}Nyu=wF z6Orn*4jZ?KAgn&g%}rm2jgBrJSWH2pjw>7{{#MP_!M8vu#ZC$gQx0Gi5Ckc9QeR)? zAchHbR-ZJjRM~~S)pbZG4RmK~k!L`<;89gUK+u^MQ`Lz-r1159#f>>6T2m*Ts9`52 z%R>@`k)Q2_(+RKMb0P(g8_Ddlx%vjC<3Mm{mp_|Hs?(#Y&u%PaRxB?NFes?0&whK- zvJ4S)8|q&rtH(AfwxpUzatmP3e&NUd!e<^oB#visy4Y7Z@t{t**mjjHZc%SqDp7P@LVm${fL z{`W+DbR=z`+k2Q>ew2BifUm00?cd5RIJ^@`Nn@)zc}yCc4ZjdURGGZGnzc7i;skt6 zoxE)jd;90^0(z7nWIbdXWCDaV;pB^4rETH`W2G#9(L= zdM7E*)}LUG3y4sr;#Ic+2OjXiK^&9npPn5zwSql{>>nxt=$blpp9c@Tn@t4Cbn*f_ zU+wWiJ+WJSX%{CIm&(N*KdHQj8x2}Ic})Ia2w&j-xUiUDIt5+Yx3R>DACl>mmf>uh znPmbKTGc77{aNmBw}n^`G0QJez281h8puWGmB<9R^#-64Ta(D%;G&dRb!a zeI*N0Odddr6B09VJv1ax73lKNosY(be4EqR(>_RHZ=aW!luKPOh4gk*z2$}YNzIK+ z9DYLvLN!f|?^eUwV{n%0Z0O_t(^z|SH^>brVO8Uo6zP>1H&B&C8k@#{QfOr2lSBl7 z1CNicC_iEG0RNiy_}?+U>g+%fFhHuQ)5n(THRkdp;9J${MZs)!6Yi=q{m15P6-GGt zm;9osGm4HV$>wUc;4$O-9OZxbs#6I;Nbrm_o`hI}F0`F-HIcPP71LtiaW$CD#Bwc_ zkWA48&o&Qc1rJr7fsfz9cc8K1=R8BOKAZ@>azTki9e>m8hd3#Laam5;FM1yB?c zRyEjSKh+CWkkI2+Irha$WG)NoV^v@DiDvV$gawh(l`mT2*d#Nf3d(8fi?0^xv(bekivb4_n(MP%NByYk zi)(V&*8BAiG6L}vcFr{C0qM=h4Rh{{4hH@K8tt1vDtm@3vyX-icEr60z zW)1hYm!pec1tUWrO`V1Lk7JE}MXZCUsk0YE+b@|DM6fGLxLx3@u)1P{8w6qYD>W6{&zmdp|B4csB=%KxELZmL5d?wsm2QdLhHo|)%AlL$@6Fz< zp9?n!YxCL2J$IEDTFVx|`v65&)6_Z7r&unSr_NRh0?T4=Ak~XXl#xB=@KbC)j-zu` z!OYd0^{*$GGd_W>#i?&lo&16-@wse)dHNiPR&@@vO)(d^QR4qa$OvL7?YEOiauiBr zEN15<2I;vu8CmQcPR<7odLZRq5_=WNf+Q9X2!D@duV8t>B!I_%3AOAMbKfhZXlk-& zB%6%B0~ts#O@90^TZ{f_9gbwnCbkaK9d%-pN88w%>GDQ;p>1+pF`Hwi&q=1!gV=h6 zHljz`BYDCZwlVz>{g5_IP6%N(WHOOh5o5`*UTht{gj7Z1m{iKvHR~jBWRkYY&j&D@ z{Ug_locy9cTZhj)@sK!>9kvcbb6_x~V1awBJRZj^@v1*tjTuFhkc3yiIK|g?O?GMv zDJx6anxg!Ag!P-)2IR8ArY6(*SJ(!O3G^nV)NIa{|FyRBlv~)NIRxGpJYMrZ z!mOCIrz*lgujLlA?&z-rm?@=OHeVAXQ{Pf4=#pQ?TNx~po@JG}&unE~u^;(roz=^u z`I?LJa96=}?%Nlbj}$4H&OLL9J%tI?I$(2aAM&}+APUr3OV+%0gr4mrzs?N=E z<8StS;F?33Hz2r1$6kWap`KMMsgOsOsyhtL+AMl%c-lF|TC+UsfXz1A@ky zI`>KpU-hM&#TUUp_rXKH>V1(HOiVR(UawNV>cx5!%x~b7)pwDtPrx8WQ|HgLGY>e5 zaNx133-*WeAsf7e2~kQ@7nEl4!55CZ1|th@gz|86L%$IF=FwV~WlmK9n5r%uRLWO! zKiAjv&EYkCu>0R4q=Zf1Y?{PpFKFg`O*Qq+6%~9Ly0m2grW3G~VcC52(7CQEDQqzx z4JlAeDwkq!&SxFEA^PipuBnUM$MUz{xatb+#XE{vj+P^ogBdk-ahtn5uJb!$Tq&w7 z3C6AElbg7tESYkjFGgqsB~{bZ)UDOZQlv68fWiL`kzq{pf*9Xp@DQ230%Axfm4s^)K> z6}N|(Oy4ZzFU;5Jhm5hM4}BRf8Wl)|8&Yqb=m?T;`%yUpjW*E9`tpR;~B=On0V+Dq7n zgAKWWP3}aMq)U7qcKMUV1Yy}uFW$e|9f4G6yKMI{-XDvPghPm3hMb9h*xHo|dkA35 zz3*}ykfH!K@zK=f&!+ICMajbI&_`3#E3UB&Oqrm*RBna4Kiee%d*L!rHD%zAduT;>o=SIr#6+jta-K{J4%O2$cc2J;^PMvtnlnitJmW62jdWOI?x z`JE*K`#y~G4JH97dg?&lbn*@SK5ub{Uuq94lt8ZK~f^dH=FtF~}%A~H| z@5Z}*-$cBGLjBc75cmE5C(=bqS@Ttmq2ts8HyKia>YAao+=|;wuEM5Ztf^};zHOxb zEqFwcD>qC`WO+CM$E-3|-LSNPPh8?Jj4ZU>aMqniq{KNz7P@!)}{^aGP*!yB2N$~s%(u?awlLbECre~1xyI(EavccpH8peeVI>qv@u^T>_$4#_ zlCcrX3pNCd0G9b_ih;w#2$+PP)g;YmgVY2aRfK`EIvru%a9b836+E(h<5*Wr+)<1P zDOqoL9Nmvbnv@ev-yN1{WVcyDKP1z4PhTVcm%y^ScpA%5f(U|XcDGZ;GVIiW zktL4ouqNDJ2_;_4j5xbZ(c^)a+FkjA_^(1qm{wve#xAFQN-a%0VBL$cd*D z%3Q5O_MS7wMBH(hiYjH~{p`Y0VNWlkUkB1r){l)1GVUouQ4`V`H0M$23{D+6# z$B4$A2PjUDRn7h(+<41O)c{kgx_wxh(FePI^5H0$QB${%tu&VA{41)U1PEJe*a%m` zG6)wclwC;GAgc?DZLdjJcA~Fc=j`qOcq;ADVKsqS)g72&9e^W;8WN>A+N zvB{PWCBg?;{%0<739&n@zKSLHfs==(?ihYVX(7);kp|lFVvf=hDH3HeJ_f2eO{4Yq zuuG3{0Gw6LY38e!#9kJ@MJY|qX;*4l^K0F@7zmFiSiZ;YlyxS|@vT&rU=f3mf`_W+ z6h$hl&DDEhpd5XxGJNpQq6(@(%9V1Z@1HxI1XQ_o-M-D7b%OUGCESf|!5(iNOaTsr zJ6`s+t{y!I|AgBQnrGUh%1Cu*XJ%h@eUH;~nz}Qj*4T@DZQ=o|S=F6S#@hG)*g%vJ zICc&#wLgxvHE1fsVdwI23(N$Ur35DVKX@u}-(i69GsyHTSDXeK;~zfYKzCW{Eiz)R(TzYq80Q zMrh|d4U}i8Rcu&os#7j+TAb0%+}k9W=8ZeUwxd3@C41mD`5E4ZFJ3X|K_g?_Q5bO{EbV=_3fJ zf)aPUjAyah1Psy*`P^H1xwkz21Vr4go}Ohyp>d=*g~sF5`~%mGL!ZrbwjU^QG|uRO znQt?aaTWSswMru}wSg(n;->G`^NqsSVrkAmt^^dcPc{nCJ%`1}NGRcT*!cfAd;2h( zsy6T+0ANJ%<_NtvQX16h$hfQXVShnUqRRDay-~CrMOF2{o0> z>@7-0ROBtg5UG?JPl{5%&suArIp_QMUf1`!et+yg&N}z|z1G@muXV5cE@=0(d5anK zm;aEenqASzSe*tSu0P?f;5^EglsevJs{_ky-axGnWtmeG-Px5kY7fFRES*xaAQGfU zdFYa(r4$~Cy!g(q!c5lthHO)!sU}LT){T{Jd#-lTV4eN?d4*RGkpGnXmA?8^$t<4T z|1DmPC}v0Y%J0{?Irh8K_MkV;tH#*A(2Lz$6hLe}h$zsPoE&&A`UU((Zid`tGp1O< z-Z)zBUg)2i8%f_dHo?{9#I)$fVsHF+Tj}Mz#j139v*nc1q3p6?LJRTcMMq1AFn<)< zS)1x|!@9d8qsPme?F-eN94!{0J`Z)@7FKfykyk;}Fl^s^d6qoh^n25j@h>zu^!w@J zvu05P3T1c_M+e>~y+sw5RavX$zY}jT7Q<>lEdSrKMCVm9+Oyj2z#2x=FmkN=DKD^| zE1Afl=}=arWq$v1w$zZpc8Bwe%H(B9Pr5c@TQ{6vW#<1~-DT};`4<8H_|~WKMUtE4 zw2Kg3b97zl!;FT#jBF#=+g)PanZ$lo3;X5mQ1?Dwg*{fzrAZ* zX*Q`#P~Qr+x_+@*kTXl$)IzK#bHk$O1?%e9rvwU+LvukmN-eBeUpk)sG1S*INXMVB zF|dG`D}LDS@XqB$CPA$3$p+<}jBye;J3PP-zv1S6?i50J5!)*F&KG%sn(RqK!;Y7C zxIzt%u&9@?Vo{Uz6M3hLBq01GBEN+rs1PlcPw&L2}W}H(r<}_MHh_*M8%H&-;lVbAne;U zN2Ue#`|A%&^nbg0vJ`7&r89&Sv>cJ`=PFVIAtf^s6PM zM?Ps~wp-kT7kUSDo#2t-j_GXp=G7D`4eV<`4D2=tOpD zpSNe?=o1Qz3G7n=4*pFkK!)`K580<=zmDPxS^|=>5TUS zOENR%ZPwZe{wg@|%B&<=3tOK?D6ry|9kPn~R5$R10E4Q0hwCROXgRJIdwuX#RUr3kg{=%21b_1+K(kLgsv=^&S?R_n& z@LlSD_V)pOCsF$*BuzP8De;3AH5X|~l4_4ba(-wu8c3Dy@v_h+EY%IIs;_e!`>$6+ zqZKzNOX*_;{&yQ8z3 zCl6c|*poP{YTAlMdK zAL5W42rF3EjLD@B`L9CJI8RGz`iGpCUi|XnT=!F3)6ZJHXZu2TC%Y_I z+$xvW;;6fw3%T0+unkG?HpQ;j*OZ&HD1Av*Sxf?*YW^UUj6Un7oQ$uCZThM7CD^!_ zKv3{Clq3B=&Xjllnjh|CjwrGNt;fh~!jk6fDRI}^%|FX!`0Xd%b(|ETnAIik^O=ES zMgt9|C(G3_6TN=y!qH(Y#MQ~k-W3}TXq#HmN9`$*H!ic)kB#R>T(sM}c~Y64U@ezX z#GM!On%6|D>HaaQYn=NbM_7gaOG`$R)qFiu4!T$_*9}i|UEavrKxiD2F%(ir;$`9Y zkD!b@TjBLs9x47T6K3Ri4ps+TOB<3~)tvs#nfjoipO8#i9OKoy^5p)RPbj_GHV<^a@qe+hdGO9|=l9dL3^tp~SGX zzJ0K~m;l{tBaSL~YNj{1U6vtg3`AyhRUOg6xAu6Xq=Tx-Z6B?iG0-Diw;e+OH!@f^ zJY2fZOFw3`wf1Ki>8qKt}Y( zMtAahjlI~+9|mIfb9wH!go+jd!4s6ni>3rFj_lGp^4OfwUZ0wo#{1A9A^QW9z1|3> z*%}&8-o<$XvG{|nZ-es0-XbsU{SA7rPfsX8c`Bhuo*Zl(t#zTHO|d-5p)dsX6O=jq za#XEbZLl$PLh=-;uupN@crO;i&>?yH{S;Npo{U%n%_-bg(uppm4%vpQ9q#dye zz8sD`ef+fC?_VD@Y@g1WZLU%Sp-~|EzYv_g$aFq=bcZp1nLB;GN__kg&C+)5c{eAj zng{Kr2}BRcoYgbDG4_PlLOe}bvus{Th;1c(?r^gF!*a9`0~wV00g`8N!%oWq;gHN9 zzgm8sW7EGHhy|o?WY1A=xa%h*&(i5?P^KvXd+>y0!3vQ-?2TpHKNenbTq;?gg0F27 zl!dqKP&MTu&B5w&9*q~tbX1){Pz1A`fAP(U?qd`FFuf2OAz8fXjH<~kEQngsm*~+z z5IFtRP6mrfzV`@Y|5SL&SxgCxM{Y9}*sWma<+Oi^TE0Ko8OE}G>4|t%v+MFOm!K@= z2)r&402s)!eTnPomE@uWdMaf7ma*=kjbgAgq`W^`ovFUwTTt3Ec8(h!sti;7guUb&_x z8(mf`pP*_kvG?YIARRyRN3A83!YCnmEqAl3`O8n?hM>IKbV-T6%%Zs(r7jNGowHY{FGn#>#CFEesFBH)rIOj%AUj zYulFLtObyrlw(L%3Ld9gWb#R5o=-#a>AQ>F@0c!1p@BXj`D{m;)TJ;Dwy;D$Z&BZ?TRPW37%rcaT)t7S zZ}nL@UER;L$dq6 z3a|Ffwkne$x_fthcX#qyBPn(b%Ko3@yfL?gvhj*Ka zzDx^t;PyEBX}7Q|gF@hfSA%#5{not-XK;^Q|ZDj*v z?M(O0**acbwxFxzICx~Y`^aE3AKN~5_`W)_j&H&$h>AbgOAgZ>XJxqUiXrg@k{SMO$5zdhxI0FCH&nI1`2hjKUq=5XET67B$ zS|T?y2$91Y=!cZC8%{Dw>joe~^1}_u8!EYcuKO4b!{vMOT0e}oenUup)K*K=%_mQ2FL_BUa8hAqT+obfs@3?_~_2bANO*YHvSIu=OYq8&dpCqmcsCG0h2(EOthRE4YA-VkoPw%G0)FuZ2DC1NflzgOE><-b zmY7ZlYZ~&4RE^h9nJxv5psL$qT4325zZ#+z<$sgh*ZDftJ@fn_Gi?=WqV@JwsOpqN zZRn3ERqv&BlEpQo0)Nb@dT*!6Kz|>G#*k`IbG52RL?-y!1R-_7>~X3_mvrMwq@|%? zlH6~ryI6?Wsf*P;O~046Oe{P>)o9BOul@A(hQoZJ>2`3CYN;Sh#@3);~XbfgVsbu+1fi_OT9Pg~l{KAa%R|C`=AZ}tThGRXiHG(DTtw$z>xNd`ZXxu(>NT%gO`aj6 zDa)+=f*C`q&Emd2R$r} zwq)Tg;(A9Zg3&l!wdH*~-k+r5;;8dzw4=7*LaUq>jtu^!`l%pwnxqRxq#9nU#|9# zNkMZ|$2}z_ncQjCk^%(;l}K*kaJowUPHg|29sa5DKCFRgoaUY(;!d-*a?IJ{o_Y95 zv)bCLCGFC>T(%NQTm^9{pXRkjchc`lYoM-sa;0IUQjMd!yqf6MrRcIo!LHOA;Qev| zM--*F0pUWbYsEVEK3>3DQVUUDEOC4UXb8|kDmA`Dnm4&3V#02hp9p;5pC%YdFaI{P zv^jAo+H+-^Xd0Vt3GC(gL}_4sny_(=Hn4sim45AH)tG?x0=R$%#IG~F)~x8)*l`i$tpv&~pLU0pQyfr#hMt45H% zveJv0a@>e+Q2Ji5q7>>8<^F$^C&)OHB8yhquoGjTe(dzVuy0CuREAVvc4~_8+$k&u zw~*>LV1m4G@Mww~2ZLGY&YaXGj`Wb9hd`5gwY1sRHD%N=;3Dkju?LqxE?sij% zECHNzH^yoX;GXXXQ{aV+-bNNwwd);s<^v1MI4oR#c>t) z?p4Mw20|{*^al06T@MW{7!?hAZ1e^V``Mu2EQ%n+>~??SrG-!I0Lo|-`N_XP}6XlZS<}guru;HaX`5y`wb%)XdxBM%kye`d(3EnH8l^4OT0l`wMRK? zpn>zcd({aC!lqWafw#tc1IXQz&j7M)55#zbK7KMX_OpUXUS?;TWW^92bj5V^Gz1)2t83<59YVdp0yg{cML{J7FPWA>x z#gM2WZL4`h37*uxH7G-1TLyY=mn#+~JGhXh4CW^p1`^sy}yB3LpNRLedB13j(LmXVoyWcY3?QVmqUSPR`u}Ltju<&YP4d0$4?F8#=+pRRnRnHL z7a1C$g;XB3NduB47tZ~r`7g&tn*4O?8Ur*Sc#VmQ{)xw^R9aPC!?H^Bd_aB?*>0I)Mz`R zQJgmskH|Tk5WJ6M5CNfv~sh%_MwSp`rNB^x#sIsukTQM=4ptIo}cRVxNU6YWik5lSZ{RS z`Nqg)x))Nc)EnKaURZQTjlQChH@f$~q8hq)_eQgK0YUvZ>e}OpUJk3RLfdhJYV4*n z-q84|T1t%;Qsd&Ld$mc$(3bD)hPt0#|6{y2lz0aX0gaHFF!ZF9y;A7ZVI9SjtFCXi z&btv;@~?#$dtkD6TcPR^n=6Jz3(~Vyew4{?>ovoU7Fhew>%%I#;;$2vNr+M+w5LCC0 zo#a-KJ)0h6iJsSR%e*C?bNZNZ6zXTIoBqi4?x550!wPoukCom;76P!eqv6)k^Sv90 z5zvAeXWyQ>%xm|Q{Rl8}O#Ww{yyQoA4c3C)_DYgBiIt;9G2LMjZU`mW{KR;5%;7?b^(=42YrmOS0s4g0jI&d`{E<^5Hk~&4g!jUOdn0lbTyfmH^Q)2Jz8+FDf1eh3 zkU%cFgSBbFe>1%?l#1*f?zl74j=JaA{aPM2P(fv~cLTQuO6{tTS~htb-TUaLh;D8A zQ1NN^zSGCF>iVJl&4-oV7+!0O!bBhXc(pf*oIB`Y+0Gg|)9X0q8{;_|L+WAr$4#8g zYpWaeXSJE>UG;ozLlin8HEW=FSNRD+j0qo|R9^Z%!7az|9MXo=V;98qk0llk87!Aa zWQI55)TW3WkDlyMTI{cbfC{Nc9t2{_h_D<%^~lr9ysMc|AgHCK9?o@f-qk0@M!f$h z?}q%>6Ld-l>+DBAp6-oUP(Px6AsaH=ercjZ8ugQ_9=o;DTM%asQmt}Nb9}v$gF`r2 z6FzI(XKpVpec^GlFtTzyv$v>J z7gE?8w)!)_Z7ltd{eH1*pE$bFTf{9rEM{pu+cDR@f8!c+v{aUVz-Yf@wY5mF29?p}JF5l#HMuB?)) z7Tq@AUHWmRF(Dlv3y9atedMIclQRK#!&;bw##4WUa=e!r9&ZE?ulp%hOz3pmYwr{PN<|Sl(VFt31<86wzKQ-7yWmreO^XoeI zeLl``rREe;YwKrtb>6gb(x41iYoCdAKe~ReIa)lWZTjx=RJVM8ZG(cdLTdF#>E5E< zmxW^{Lh6IUZ8C?5e+1DFb!hsmcURWfV^)ra%X@ch@#cSEZ)(y7f?(A{Ky+r<&C5#)M zpxU(~P1U%-UNbQU+WYZw_29lsO%E#6qjL9aTy*^1Y<5{U?0N2tEa2c!bE&3#;ocV0 zWWk)#`iE^ev@S_KEqgV%mE*wDd{zCd$%kU)IB-p+ypd>vjqUt#VDwm7>E~uB(`?U! zICN^aYWBHJjAqzYv`-R`_idH7hhNW-zoODMtHbZ)%b%RzmLeEXA$6D{kY|ayK`}cj zkHoE(KiOqrGSH1db);jioPB4hmeNXkq*IZcU3**)K>axCNQ+fc;^*xF3Y4jf?PMoyD%UjxArT>YlJ!G>tCbPsx<8iC_lB()i)TSXGDU(yE3NTtD7VTuwol z=+A9()MVbbh4{tL__-oW)#zJGJIZQ%^4=Y)#@!~tfpz7{C(C6&Cq+P5N1d$PtZH@Y zAMx!k6EozSYIF6!mc}oXuGu){MbiKp6>~nHDH|X9Q=2cNOgH>G1fm>sXnPoo{XB{F zX+gHW8)fcCsmf{X&R3s`A51b>S9!QQDaB z6pqUBHxO;ZoDQlnuSS08=l@n9=bqWJZTj8Xp4QpF%{&)ait5^gJVAAO`Y;t6v&Kxp zKv5v)T)FD`+kLu3thLzbpX1a8XHpCeAcE?5F7VHoA4PIz3|D`U7;R-%!01v<3--tLv9h_J-OR8wrZf_l zb7VBN{o_nbciD8rY{~l#7nIB9>+Xzn?KA9-7V}zzpmo%l&UNJDUb#jNh&t-bOIfmc zq^Y~2dq7B?d2610L{<&l=o5Wuh7`@TIjN2MXFi@TE7^jk!)aMU>g+YMo7seS)OaQ6ZAZGUc+C_^4$4X95?T| zV+P{p*|L=EFMwN=zb_jtr#QqwC@bl?PCMMI$cuj}Y@ol>R>>*8J4W4aWqDQwmDw9K{J9hOE6d08s1EjNzBNI@s&lMYhBjnqtLm-Og%mUwGa8(lE%RmWvRkSq6sKnE z7+KhHe3W2q(_~@(m8Jp0(D+xL%pciR%b=g2Q*+-+ne)`vs_3P40&l|J0pkTQoCD)YW=7U(ksv?wUAi*=%Kq64eMg$OWxUlYlxszcib>{ z5rKXT9?dD})a!U!)!;Y|JuK0B51x>BiT?4~0x|QPyi3>#_|~TNW=)oNxc~x$et;ld z;jzWfnVyVIsi)O5Reohc@r_Kw5TXc(+CvVFLc{grI`vO+%e^RggI(5ZhePd@xKU<9)eXOm?G$6#=;#ua0T3f|ZO`4g8wZ&T^r?Up4A>~ch{M5Sg(u`(&?}LKH;p|o5Jk(}8D90WhXeC*c7e9~ z1)LIpz}SzV42dZvoX2!JOXKDBveFN*Gm>7g=d_$6$`yio4uQtjN+}MWiP+ zE_5p8Br%GmbSYzx3%7Sr^|@SJ2wQ`4;jk@U3)mv1CEB>jO4W?mK!oxe&MThw=utW3 zr$8_!Z2Z^fM-mdB-L=)EfL?wGSnM>H8PAsX|#c(Wgk~EQB0eF1~R7XBg2rkhY_p^C%k`i zR}_0_ZL^&wTpX`mH*&h-H2Hag7sGbP@-Rx%{2gA*HN(uNJ<^7p^BW45c`>tFnlC;J z@xKjm+?#Jk^s&~(XdAa?mTF#VTwtvp*E7zm&G7>G*86dPPf&3rcP+E(J5Jo+#j0+5 zN83WvpsZQ56|(n>6Q(DFHr#18yhPPGK3?PdG`7s}Y8Lh}AKs_*@QC}Yj#rafdRjRn zN3(J9s`m8G#tm?yidoSd6?b!0v2?gN@m}>ht;`3nF=6xCCsq90KN&fY1(eM*-Mt)O z9)qZ*ad4XZE|EJ8=~6UO*6jLN701mstsD<1e@=8y{MsY%4G^ZGd996J-S2HuGt(`b zw<~hbung2hkv8ZwAG5`)d$);9wEr`!IVo z+vb0b^%|rvjLfGkVq@L2$4pWIYq1uqcgUYCC3O#iFxAxV?(RO}FY>T!$Z7FgxvHo3 zM21*=Lab_Wt-bROO2~<)GHr{oleCkpqvDh1saiMO8tJ$#HsyJ>8k$XeD|-CH+fOtUj6TPL?Bw$ zPF3-@pA7fHkdyG%Xs=#=-N?cy;q6?nE+3kRZO2^8rn|k`B+wb)N8LCrx5VsVbDno#mm7vmzf4*k3_Ch9*o>22IAS( zs)?U0%5X`1V54m6S0geRB$6Jd$%Lo`^@-HP_;|nF_+O-7Yl-9Md$qYl0teea5^tn5 zeC#L_I-8;;Z8dO}9Af`;0{zr-Ih5vIz~*o%v!&6cb&I3wv0Zr*BHH#1gOXq5)nIwf zXtF4+X6*E07Mox}Yq8eVXQ&viGbDs(?N)ozygCm}G(-_B^EkKE9xd9gofKOnyIBVnVnQCu zymd-Z4Y^AUxYf4Jr^~#qo0S>rgAGLE({4^t;2zCi2bUxaVrcty$ zKpf8%VHzK7C+VQm@siOZgz;eutsrI12INRO$|0qGfEct{di&v~7UH^8xrERrjBLkC z$MKnx=I6_?5L1pz0@H=&!UK(IBAsF;8cB`SJ5KK|Eu*i=cbru&mqlOe?D*mqN%=pB zF4-sjxFnDeUZHe+HBqwub$Pfuz~${_auNO17ad(z&ya3ZVTPqI={s|z8wIv}2*>I8 z-ch;MPYO`Rjx!Cb#M$*-m?$*f+ad$m5bqFw=-+V@MxO)F#o zr(O}8Zpo5sT32}=8s#}cx!W)e8p!vV&+MQ(E_5LJ<1Fc83tL%NcKo?mMssbCe{GlQ z`145-{}!ip)Zq;G252m&(^(q-Oq0Hk-WcW*ayr!PvsCz7?z!k#2!V`K+MRDY3h>B5_`oww-#P5SgE9jBMoSB(ZX(y*{X+ zGM!EoxERI<9zU#dok)u}jCf1mw~o_k+%QR@vbA;;G-$XbRz}o(GD@(Cb!0@$ME$Q1 zarbVy#!nz@?9plJOzFo_i59F(`?k})>m+dK;xJJ}zkjt{e)!gK0|w%Hdh_QGg^2>e zcjc4xa|F{5qCeb822v3Pf*J~xC8wnY+WHNkEZZhIbbi?Chlb)bX-NVTc`@=M_$I&w)1f=?d5TZZtlrdcI(Arv} zr{X1m_aj_REs;wJzm3RoE?q`o4=p+m#RfT=&?u1xl6Q`HqB0^_=L^zhfS=~vaOs>d zP6pawQY&fa%topntDJ)HgwlEBY*mjh94#pi)dz==3pz4hU`w*>hP8JZNf8c++?CRV zi{t$8iH3H|B^TckCC8-6s$SG)xkKSZ89^|I9zX~NjR`xY&z5mvqEx@RqrNnZBoBw} zooQ0f`qoF88Ye9un;pgF(W9!~G?M{S+ZCcSljZtzX2Z-nsxt-Y+C(SJ?L2di+(2#@ z?FOsMgA`7Tc4g<88B*Jau#yt`a+b6Cgz&`H`KdVZX1@{Th9?VU5H+X(_leF;mK&I* zd;_7&ym-msqd8)+c}wI=F!FHI3WLUeJtWDR>Is)3}4S=|~(H>{LAVw`#D zX^k;5d%*$Xf;9~q@EirD@~mGp8W3xSNitbiv^{uWyJn&!-|?Wa7!bpq&g;%dka8Xo zi+va`S2HIUGSk5c8muiF5iS;SR9U86$*loc`doJ8$_NfYH3W^Ie7RCakZVu(F}TpM zbF7SFbwPvQwZEAzIb7c0U*BTiFd1ChH%g9gGbR527<0#~$xxe<$wxVnDLwsqz?Mtp z6lsxGY6L?L>`}Q{TBMAPNP20_Oi>Xol-{hnPnuydz%^^Cb=EyC&B&I-56k7!<`blk z|9Z6$T!bA!miH>rFQpX6wF^QK(JyVaO&mhat1NbDo0F1ELNeU|4T^-e&D9WEef=Ou z`xVj}Cn1N=mP|(>@;Yf}1A$v_Q?|)O|D4d$xMP(x zVUHegU)$+Rq}Tnmq9o;V&lL%EjV?%vU?iDs8F@$1@Jx{m-oGHqrt@Y=XL3|Uh`umi zhL&`Qvd7}daxHJ6|ZU2dK0~>eBUu{{7 zomegHqplx2m;RC_V=vnhg+c+gE_TV{`-QTIBfc1Al*66lolDvKY{>^6GWZbZsDk)w z4`U!2Go4FmNDJ%>_Ydm5R+}VQ_uC3WMs(fKjX)i(oUum=^$D+IQG=A0XiB4*GTe4t zO?bSM!pT`Iu}<1zMwgTer%BJ~Oui|NPD(3ZQXgWAG$ELyAiNArX*ov{t6vn>cDR!g zJI)=-Sv26*4JqwnC6+8S$HM(Ig>#@$tjCZ+qd86r8B@km9$KUD1f7(AJ5}P%<|IH1 zM?=?CN#@<)qo98Oc@p&BK%meNm?W|OM>9~MkkUVcjc(of3-xwbO5Pc%$DI+tt@l$1 zTbs_xkC2l9y$FVM9nl;LeqarM~;}Wdsu)VpO40(0D5e$e- z8cHPTzKVWps%%Pn;Kn3Vr#opVQ0`;aw%4GwUts%miFCyt2ir zuEe*#EB~|8y^@VgtsGDgWf~u}yM9;>8rEh?CtKxC)1jfbL`HCk;S<#jr)5OOMIrjwfFWPo)5Y#7UKR*twe7;v>1yI}6T-7p7XlzI zOHS@bgGK2=ZvAl)&xM>W?KjG}rX$1ZQ@Qwx>2fjgxK)1X(mP9X`8e|pMAhM;ajIVQ zw?~&TtYMrr{R_h{zAn@jRFQgG*-jT0M(zBSqcLHZCks`*K(=<2HUV;Qvn`L!j0oHH zt7Z5_!7y6{J60^CDzdb4wx4$SPGr>9*CQtEa&)7Niqs^hwlOuz+H$r=|F8z?+Hjj> z`067^rqi`cjJoEAWJANsQ8~rCKdzh38)`ymoUWXV4D~NefkKVtG4oV)igi>;+La94 zXW_!|u!41EukQh_%Iml-=mee1Zq8LLh?Uf$gGO!ReRX6phr)~ojYb*bCK;;MO^+Kn z7U|KDnn>kh!jbu*#it>dCokBPcDliFQgb)U*JQ!d5C{guP1EEoSGxIOZJIhMN!46j zZX`9zrLv<~ll>sg*2t0iT#RZ+dW$MuQvbV6&Ted{t!_Ka<@q~GU*=9?X?TUw?ZG1V zg(sU=1++VtKe^F+aKj}gP6;|xF8-iGPP3t^t&Va*r(4_E@=A0Cp>FN6N~ayN^^%O} z-I!s1dv?3#WvbdGrow|YQ1{+DRJEnIM*2thvDLhJ*A*L^7HD0%{gkJw z`D5SsrTbKl^QqheOD!$ck3U(ZMUPC5-M66&C-2}}BX`H9r#*A>Zgsh0UxrGIerff% zIJ5sT0zs=#dQej*)sAY-CFJzrB(bxkGr^mG$ zC8z&;5u5h7E7?6rk>CJ=SsHh*cTaA9LL121qsIeza+dqqSWEW+*gmsE)o%Hb=~7mX zkEg3zoB^Z0Rj$WjHq6ikaI0JozC1&0|7cxI$L+CQ++TX!7daN}ai|0?zl8-0Iz4_D z$>-GwxG%@KB)5_U$tn8jG2Qul)=!oW7ycZWO&T5v(%@@|Y^NvT zgh87b-KyV{ucV78Orza^ZxHO2Gt$>zC|X_Km@j#J;30<(K^V?e9O;2#$8FC~b7j0= zC&04(Y^CJ;S0RCdD2uCcgzzGR6+NxtdsBgmi|sRrGwT(}(@F;E&c zF?!e)z3mRVA=7%_cHFd!x5?#x-VLK|T9-l@9BtFI%d?~l*G!hsVfGp>-SVZat^4K+ z)_sYLq2v_K_5~w6YAB}(n9%1kY@Q7BPeqM^(#B1coBZ{yeY9w0llVE+M=P(Y8=qJ?-vUk`sAXQiFTiG#TfA0b8PTrpZV@l^hK~P?1gi4=Zai zb_1^`n!$JFa3}49&2lySQh;mmvz;`LWeA;Yg79MD7yjp)ol6L2Lze2aaNC0E-S#<%H?|0y4ywx>jTu?q)5 z-=@2gB$D_$iH|X!KbkqiPI{k~ICC zlyIS66SXL5TjXZ$crU6Fj7`YqtdlhajX|f^p!~qG#9@I)P#+V(CHsU7WVSrTxCMfS z;0nnk|5&ND2PkY)_9tDkrhz^*Tr(^1HB(<(AlB${+lw>47pJBMZo{{G_59I`#rFU_ z2T_ei17&Br@YM*o^?t8OlVt=)Hr>P1MX%dWN-lZfB5t6bc(X-^`!@`Cx?XHl)n-ux zgfA(hVMqTCS`Xh151a`cB{;t+T&yntJ{s6b0475Dl0Fb4pOSM}(?AzQe`2@9CWuR> zk_Zj=gp47=%r_z3fyrPDH>b567KJFreK*`1?l7?ZEMGp0)QhHJTe@s}uWECLhA%TD zV0WE-1ASE~0Y2U%f)U;RXCyHRD0_EEChM`mbUIkmP(C4W=IMVM^=YKx@HuHt8Ww)| zf*s3ozvyUVt{{x)-$~$&k`7Uh`nHpl5mta6+8#jsk`^c>|0&>FABdjpAWhim<%cgR zJI(|AOaq_+@kfcIlMYuK2(*yXn;Odpk@6I-EYuLa;~GgkuYxr+7}0xk0CDgddz;ks zkKWCTrHl^=@U0xZ*_Uem*&{|U!=*PnS7k&pA*H4fbb7Z;mhvZU8cEP-XqzZKqW3L( zCry)!cq4;i?a}+9S<;&CeJv@C2-dcmlyNI9($FzYzQ}#R$YF`59G4$>HsstG_3kmGP;m8KM=m79$F$5_nXV3nk~vbmL?lcMO85DO@+%H z1aKb>pL1<0*)CBn|Vh?CMc}Bi!X$~KK-#&XhaDeO{S`HqRd%BuzXU0Vr zA7ViV=|s3`L6!Q?O_tAWHKZ_&ZE}nlG^QNi4a+Bpifrr-cY4286xc;x7A>iD6wA@l zx3)x?@bx<4v4s>dVR3F?7oR+^^$o=3jRStzJVrv7Rf}aaGcLx_bZGcsj%3?&Q(v(6 z=E)(-yJ8Fdpxj3rWh4VxqtE~f;e_pY%|~c_R4zv_GE(|PH}902*e&)A1Paw$`;gZO z88i)i`w3f}k@*8I1cC4F-yy6V4O`>o3vMiH8lcfo)=0{uD_i&8mLwC>O*(Dg*M7ipLl+R@*93DmkPqw@vV>Y`)DbfQdONH=%iOm3!JfKsx?Hm zlU}PlP(eV*BDnB`obiqS@-fB1A?kDKbV5E^a5T&oh>Uc1N52|T+GdaQl3Gp)K+rBo(!0-+Kx7LV6dIn1 z9zf~6W911ZM^x8@heNbbHgb!?R}K_nHy-1KQ)nTO)WO--TE1YxB-zgQTtSsz*kroF zF98Z+Xbd_h+w0sI)`f=QX_7-i5Ky$mXc#4HWY&f^jqCcB6mUlX2_)AI`D2inAhz}#4HG8F7sG8b4=d>{ngN<_ct_%E1rp2a63Q+``R2Vo$;A4K3`crpGiJz&&LfpfoS2MA4JWwueiEzha`auqkpT6PJl}6zi=%nA@NKO;VQ;1-e=&TizflnalTb($A z&xqXdxj^IbWI2M70oTuXC;iz>`J!Ic$*SX|KNllk`>Rjrpv(MX`7HXimcC$~Y;T^b zO{fV2v80+@P?;alL{FLVvUIaNQo7GL+v>8gL=OB>FYE?TC`OqdxkH0sR3q@BBN5Wy z;{1gdFkG~xK+w=N`g|_^)!9<^ca9+n7a+L3Fwn6jl@XnWx2ws)oJiUtC;iqIY zXf4LH^=_*4;`^l((HZW#VW+gRXD9ZRlupoMI&Tj&)pmvGyT$V1xsF;WU;Ph~r8DnJ z{`Dn&|Ag$a*BNvJ+HUMFSzM`FZf9I*Y>JVx2iHaQAWn&A2u*0>B|=|KeW|B;W#dz2pDzd}`NpJ_H+8Te*{{MThlxrw{A(5U*&!eL46DIQV-cOS995X?#dzy92ffDN@c( zFSgJR4Q*JVunhEx!j_VT^KxxAEg)#<8Yg%BHQx;N0`1#ADV!DH7mTCjpyATp^7Z(( znudO8;7IS&fvY0U?vt7*pWe7DiY-h0ZTlkGJe@xMisdW+EXi7|zsRv;W=Fwx%RaJB zHu)iERYN^zCI%pC5gmF8U>=+j&A75u2TWpVtGt>-Oj)s)`g{ZM8WpPa!KJ!T>I7< z=*A>T@mH``(n)#JlV!0+feu9Pm?w7<>jXjHmCVUsu-5jwB}DJbkXxC?t63w%_Reau zku@#^eFNRkp&ux))M%hk_dbPl!o74lxPU^<*!6y)9xK?~4AChN_}7<|i+#OW)A6rw z!o@4(2Ck}T!7R#usIW~DYTtwqT^u9x2|Um!gH3mMeU9A4Jf{1HZFuphEaqJqfq470 z^dPLjHxM*RR!MdC#QTz=Es|rz2jM8)2+HBjl8)C-F>QfH1F@r%IDFJrrvbL#XUTqU zqiePn;^aK}JSz~z<@8P&#AX)|zH%I0SJ+55Uzb0Zh+<2=5bs-}Bxl@D?FDUQ&_Yh% z8t0^6-yx<;CFqXZmz2v>Y>0%V(f5KR2}G`A15xXYWN>2$aH~sSb_Dt&16-`6eJ`vp z7n1=(3q~U-4R*`tx3$p5(U0Twjo&9%aC{97{Qwc4CI#&awUju%sMZ+Z$=uGHsL+d#jE*&O=$-A)^2)IBrh5clbO=5Mue9CtEUe-az`2~Y8v_@Gk zTRADxY`PY|}t4~3P3%b5DI>-d|72CN~B}s&lQ`0tf80X!(GHTs>lGTBC~O~IEuV2e3KZYS zkB*jIq;S)tk_VK+ObPHQsvAH9<>_j2Io-i;0OA>}PM#|*m=&Gf(fuS9L3F=?P|+rQ zHWh|vy1p-slLR{MBIH1Hy#MNHxtBRv(js2>L`noF(@*+8S0JwvY#0XI~GI!cDK8*s612Hm~nIEyWJ~aGTDe14TG$=;9pBO7&aOr!n6&)H}KkiDNZwP8SIQ#DjvYUMexLCFs zb(hI3PToP$PteI={>UJX4RGs*jF@=ISh`mK(2wI}5OlQrn9ZC|L&(Xf*+}-V?7@US zQT8o&Bb4T1AsS4SHf;WB8t9VkWH1MP-SA=kLqD*MPn3Lr3I+|L2v)?Ajds*>-K8>` zrOS>;nR{roIxc-Lv7al@0HSrF42WdVg=q2|xrP}gA>2PQI_3txI((~M<=6oKjLyk& zIh~Ha;fv1c%G~HROh(6LvXP9S8luc-%N2kx4!s*?kFLc!+iUo+4Z%83k*`@e7h+S; z5G;L;WUGfws~2L?vLPobs3e1R&;~xuag^;r8KW{}!|^wbv#oLz z#I0aqvbuGbTcqpV=Ju9v;U%awyj0Ku+F=!l5lgs{d zNtg=|73*XhRa7+uXh0mUBm4c$9oz8h2^mT9EcEcT{qJ-cNyJQ~;irv~zh+aIC^SyR z%P=Z{fbW~|pQEyiYbI4(`ZcJOHX}=nO=%1|{Tg5PE7-8n@(Rga z0oRY?^cz|%Ygrip?h8hoziE=)&+>! zS<;t`u?1FrAnvKA+Q)Y_7F%O{%hA9J_Je%ofOvAUtf;>|+yD)-HZ4CB$$Refdm&T$ zu;wqc;{^z|^*eE{4j12ZPji8cTsLiDTdKdb`f-qY==A4qQ>=~4v2yh1GeNA8wWQfj ze>SPRFY2}PA{hJrd5U6~urz=}H6*Q* zYZ2ehns5t^PaG0@kDt~od_N`0@6N*)(9>Z2eucAXK$0hLxC8Z;bOp^}|Hb%Srh-_FSM zO}{rN2!?XMuaY7Qt{Z9iPE@PQs^Ts}PN2>fng8o2nyr3x!!&uxKbyBGhm%!4A7_Q( z2_-YSpl^xfM}hhnhZAG$%-CGEm0R-^|2Xq zgAI*7@p8K#bz|k|SFDn6NH*G98kwtA>~B?BHG)py-f^X0^L?uYvnVsCxZAnOwVbYP z`e}d!OG|7&x9SIOiIpNYb^+I}M6l~}R2*qFY0!`31a8lfcOy5shC2b0XRV5fxAf7v;T9oyPLOHzBEhlbVqhJCczFP0zk}}pTHm&+=GgVSl z!YL|6-?fM~_t*^_G?y~sPNFp#N zaGsaG88^s~$?BpT zok(|(Vbjz_biy7r)`fkb-)DBAu%5&*%ETf?yJoC%c zRF%-qZvKG+#62adG3!&^PZNwT_s&oW^g@2vZYfVD$$}T6l2^!_4l8c)xb zcl?4<)*ftkymQFB=6xE^Zj%M|AhAo_lhEVNrlnlMD)r$Vk3 z$`}3f32t>8JgJ3)#z%#+*pE@90UB?TU5w8aUl$-sb7f(RP*_qLR*zF{vQ3bUZlqz| zWEJ<336SwS-G)@vxQC6Yu@IXls8-+J66M=ZSt$Pdc4Ht5vE_tnwd3DrWSa)nWrdeL z?kTgBWMm@Pp?zK!iCLR6vT1&2wY|7yY3v=RnnyZ6C_fI9SK_BeabYiKUyVIc zCj4c#ie+|SGSOWk*>P!vpVL}Ugu^aDhjV%Zn6Yq|v8KU930ik)mW|9pscS*qofs#-Qc za1>8vS~D6TN<;Tb)pC@H+|ekGqk812q_6F{DGfmMovvE?aj>=_BTps#kPw-J6v628 z1N*dM`a!TK;@$U%P%UJFrPI+6nx)!(oEfDY`wlO6jWad{1uj<~Rf!iijZ7Dc5X~2U zX%dApccsk7o`CBI6fOpTMI8w}Nw3kut0~^emi)|nQXwrrnQ4nj#N;wfCINO(l!bRmaeWQ=^5m87F zydNb8JBsB5=;~?965TvOo?{)PxmY)Rwpr%4xg<*c_vWd@`xYAYt)yGlNihe@;OhsQ zZYz``--JHVnR3uSzP1hPJE>+1tCH`b{-@QXoJ&__(BOeaStlvwL;`T1=%@RnJo9TU zhkk(gbhea#eQy{A1AR6{%84#TD2+mcdqOq1YpDq(hA0iBT`gxLQya)CN9x7-8=lqR ze*f5ZMqN13Rz$JmW!nnX_~EY&7fW>iFgf^1)M=zTTp^38Xu!YvL6?dh@-Fv*B61um zk%Q5h(lYZ3y$kZ|n*7ODjvdQX;$wj*&mF{bc7>yfqV0D_<%KgFqI&XIyHzVctA-Wq zr!(>bp&CA0hZpWXulmM6O)o4pV}IXD)zS~YEi**FKOx`QC5u)N1N|^XzO#wqG#%I; znxv9R6sL*WhSPDf?t_J4Z9(~NnrebQe72xe){%o$wMQuJC?F2)b8ld8ypRSOHA;Qe zCjW4l1{OP>tGiUx*#B{&YU*!=g960&CsZQK9z3TXX#BK7B@#%kxfr4jE7%wPh+V&- zgS^N~iyzu*pq!bZTHM(r;!3AZeK}Oyq)oJ4%3&|~5Jgj~ba5K(lr@A%Ktn$$_s={P zx7l7nwD$PtsER9z8kG(yhSyI1B#Q01bk#08g^CliM76y08ZEXJ%(;-faG?vsJvrz& ztukey|B5zTOv5LwB5rV6Y>`8p6KY9$LL|y7R&h_K=^sAP*di6nl{;;*1Ostl2bEaQ zTp{uyT9?bgNM;UbG@h+maigqCTPJar5Qbv}vs~IRkfZnMoMuTXu4~ozKBC9&Q0={m*-#VB0=j#bwQod}%p+?;j+KrY{E&S2U7C#Z`4h5j|tF zEG{rRJ(?2`9ZOUavBf@H8u+}ALiR}&zVXdBhLY^ls zC&G64Bq_SAmua98l(eHNnWTu?QD(fbjj!79n)gw#N5&Z%s&-ju=(kuU-O`L-Mmb-iImJmj!EUJ2|LcdZy{PRNnRuqCo=fD+&)2$ z6DD3-)e9foAuq6CMRi~OCs)Xe(FszV852~CuGfa;(4wDIEjU~P#W(VED`cU~DeKdi zTSrPG>r-v@v8u(7v%+o|?l?q;mG(UtW(&luB-M^PnrrA%g~ko@x~s;|F9>r1;@Pn( zzVW9~o|`vOwTxWH))1Xk0{f@hggi977AuF?5YhI)zli?Eba{(oQr&TR#yieSMJ&H= zi0We-Qe_d@#e91}gKd{X(KjHRjZ5TZ?lt-l`1Xx$s_m_PHA~+fe9p%C3z9OU%X{@z zLdvVU!)Ykw*pVoUzUdTJjs|WDzchDql<1rJ79)>yWV$$-F7r8T(=;H8U|+6O@t5_A zBFakoC41Y)=fdUlmsltwsTK@L$?UVhKa8aj8o#cSqD%YgY0Dq``}y)@10Vqtf`X&C zuYT~D*Gob5Q@ZBcFiel}v{%(%?b|8k-UzW%yExqmvyI{$kP8jh1$ zBUxVYZ$KbN$jM~OwTQSJ{W~f-gw6y1}V5z5Nb!5kR zxRXh-!S^mc8y+4A)}D#&_-Tz|mCJOx%QAc4!{?GUNtTnS1+jGlGzh<4HrAx1@=^L8 zS4kxQ_4_(RLzVL9?3Tt+5T;>liM$nAYomS+n=0LY2zwuhA$ck>zN;2nKWKYnp1j7! zT~tH5yw*Hbv(yg|S2a?NZ$2Hx<>iU;ZY1ak&z((nd)b&8tsrdgnj=g7s7~V-=dbB* z+HV^J{M*0KxbiX#Y z~TBsu1B(XlsXQaP&6SVo=mab8x}Y(xZgWtqH&8gYQ(xPj8HkOu++coGU%Z z?b-nxgQ0QA8PKqiYV_8FW{becF>q3$SM8=RP53?vhMa+%S8nZl(403IIR?rJS0CA& zf{Se!8t3MsEgjc-ZlJTIv@sKk7Twr%;EP+_b7bQyM0BJ6!29aB`COL+Ut1n7OD4Iu z5Mc(2PxOa#?*7|P8eJ@n4=Ms@NO=gjPjpjEX*;4!CV*mSWRckUA1cJa!9rxU%~jR> z%yEWBRM?WVi%o(~R%tca^zZ-Ls39y$c4EBzZhdRz$Znk{zq9RvP(H-vMJA#O zmUi4`6ZZNCcf4t^2Fjkg*gX&l)CoG-TrvB~KbwbB$jP2@(%p4yq%g0OU6AJPvIjCg z+qXu$l}8=pIip~->$pQgw7CHt`w`O0RV%s_nD z?p^2p$hkWB*0)Tmgw&D!z*b- zxsAi*_wBDl8R(CF@-5}nAZQIV=(;%fSMIG9gh%C|TjSlsWbxKL+2}IpuFdWtE<9)m z&_d3jh0EN-9Jio{wdtUXmZ(1KF4x1uYCCA|O7|dH@b-o~%%Fvn+|TKBS~+8}K?CAc zb56W94a;Tu1ounEi`Ldc42en#Oo1hnTA)eBL92Za~|^byS}&Wl{EE6S7aXxg%la$e9r5 z9%$6tIJ-=DsT?YiY~}C~#Vn1S&Ka_mlK`~UkN$kgb@!359}ZUioOjN-dl?=D;mIIp z=kfoqy|0a{s@VRgNKGx1La&O2Qdtz4g@tw_&z$Wk3>0OHg5o2FiK3z?8lrFz6*+)B z$djU^C@P8)S>D8CFS`~gX@;qkrIrtsY3aR|b+zAj*6hRKbN#>lzq@|iFPJm4)^|P3 z?AbGG));5}esvAO!M`3gPOI@p87k!NNaF}j#n+<=gtKWOz5R>}(3s@HI6@w)F)rc0 z0uVI|e50w|X?Ho|%EzV!HzkQLFv6g{N%V1dqX}c7B#Iik7V>-*-K)j&5Uf<#jQfFh zwCDq}U}vO_8Tbg+5dT83wat&$rAsLVh3K)yd7Q+M!nw&oDfrd_Ynwgo*^?P*Fz)wg zW8PU0tw9#gOV&2;s3a>)Gz3v5r+$5bJCFjlJKXqrD94_g(CZqF&&=!%++^sz85Dw9 zpJdBThW2WvkRfW8qi7&z;>#}gJ1wVaA*EH~ICc_&K|VHZsC%e!2tEpw%9KMNsxvP7 zyz3-7L>oH0S)4p|PGV3>9(ppsxQL4s9B2mCx;FG{U*ig{@~U@=Kg|7_aSAc4QtPbK zVX)kV!Bj=5C=us_>8#*RZG~kv@5R`1uB=NjxKnes&1COrhBO7T;=4Y%z)X=ri zxv}DB6)0@t*nt&+FMe=w8Sa@RrnF9HIs7G`{n;yza7>LwN;`ZCW^7&6G6M@YJQ}x$ zWnOdI<^NTqC{Z*F2$P_cDTnWk6`%Z}4j2NyO&gAt-cPX5Eo(5+;h)8bPjD2$G!$z5 z)D1Mg=K}bLu8o+vhrCs)bl`Fe(}w@zCK}5Bl7h(xO@0j$4FQ~5PZ1q4tjTEjEs=XR zTYtpDXyeCaR!bc;rV2_AQU}anFTg)oxDn;e#t?M_yMjJqS&aBj#lx5dyWFf@aBn2+ zA|Fi)>su!NdO}@!$w0lP#r(+ikuJDhCwu6v9Z5q*s6il~0Co6*xu zMX8#M&Zj3{ZQ{ss2BzQdZS+9ThNMb3y^FWujp+t{Rh}G%ypr936oQ#GLOM~qj@?`X zn;z3dJ&#;rX<6&~bJ2#kO6)9~*fbsK=i3bFA^$Leb;v<(eRYH=gng`IxwXyT=eq2o z$9ouk-)Qy4r^nqC4cOw8G!#+1flfa52jhTnHf^MPH)>;jjV~Bakv=x+6+0EZex0R- zPIN82gAcWTfb*X!#|`grklJ@y$Kx2|7-PA}9#A53&4C5XeY z3ka%?9DaYiNJe-RG$w)R+DIH$IB5n@G0|{Jpl46c=d4Zj5?HuVO3&Wt$d=<8qulF^ z+em=nfGHL9=Yq)R#gVKJHq@v=g8XsSf^;dCWmL~b;}&{0?4o>ZR6>%0%ulX(hf$;Y zi7&tQ%9tAajy=uL4IF%ytfX>4?Kx%dwj z0H6!yAEUFcicAEhfu&i1lvKxhuVSJs{^)&}<@DSuf0d7>js7`aoWq**VMSv&3f#rH z*2@eLJ(EQGBYU_8IO$r%gE`{ot_ByNnWjYy^b;$F*(DCi+NMQ}!@)lEeXx|ZP=JF< z>8_~)3Z`8ahj2s$)s?i7y@RN0_&TS?P-964aUH$VMKtfA_#8*N9pJ*UM4U^ol%ktL zK}y<)okE;ink9cVYaH_t%P{>i?E<1WLi8n8zd+P%f_9AEV$Spmcx@JFqS29 zWFfWr{4#IoZkFy4frIqFnZZw>qic~bxYHvqJSTyv8CwJJ$?{18pgph-QrFw)3=0pjBO^-PpC78#Z!Qs*vmxzwFaZeFGVkM(lRP$I=*YnSGI ztYn2l?b8+TOLT997aE$SL+^K^yzYT*4d4(RoVbaSl7mk*e$Y<_{v3eW|=XqwJFEolP7hdAJ4OIiXv)9a*xI$VVCz`|z{wiQYjiGj;3F{yD)s*F5gGGvY zjNq_@J~qLZ9zwhW{!mUg=7wD?RLf9-DAcvF{@p~LIsb@}9yAvHCC}VQRW#NX8t)*X z(0!@{p+@*kk&9H=_*Ef|>=tG%`zPOMpbEzN_*nbw`v><1MjF-8pPtitGYtp`8VgHB zGG-a4Pg(0e-t@d zXbje(5Ye5ELAu^Ko{qbEp7P3`lvgb%Geq@XEY6Q@>tqWroE|F*%p3rUXkxV}z#V1y z!}QZVUZTL9DxlD&MSboe3SLrM;HsdaleG)JUL`O*nl`ShCqnM#xM+;SA(BLRp((Ad zje8(b)GcW5(nr)EKGrsv2c-ZyYTG>cmPJ#dagIXl}U0eGXVabX$V8 z&6-fBaG-JdlE}v`BB-wXV;nYxZzHznu&V&K{$UMP*1%)a#v@7fZAAXTKa}OhFT5;H zmp<)mf*Odo{2Jm*4`S%!Kki`d@@GpbeO;SSd_&lA1G`iZYS^?1o8rYj3`4uiKo5A z;nrF3qQ9-CqW9}9U@KYc+T>t=>f8R`Tu>_q=}F7V#9{6}Mt7HLsqOSAaE zJp9GLCIhLjy7^p*KPkG-I(V*nohUAAV#&{3sDc-P#xHT=C?rjBia)uhuh@&@KhU^j zt7(%528j=_Ac6vAsL99#{taT#&_r>Wj6IxvIF?n90%%^>qOp+K^|jyFE@-A}Q^FFA z-0r2U(+jf38_+P<@I;kR@dH4ilwZluJfCa zp~s|uW4))iuM?@R-C3|K#(GcTLdiuLYU-dUF&e8q5<1hEKHg3nE$_L$v!{L-EH2=V zL->Z$`qZOt;sO>!WdWMlw5g{Siwg+O2T}Rhv^PVmt?olBWp(92u`y+q{YZF;HEnsn zkFHJI=Pv57LITm{v=#+OJ+K#HCz1v$DDHsC99$*=1EOthosa0=vRe;pKeV^RH zucQ=d%GhE3tj{1~25v1iuWQrq^%7^z2^z~X9Y@U0jqbqxw!0K=dW4%edpw*Al!D~- zS-3$+(qJyr7akR7v9t$*$_uAwxr+~Des>8seF<(J#TF+xDCk8pooAX7Ts9pL z`^@CwitVlp@nM8=DX`VFnIRKtHIg)ehRGJGDLzC3E=fb-I3mP3EJUEaGUbfB7K?K# zH@4K8uFZ&TC(i7COKMDFK|xWJ_z(#W09_T#zCWCn9yr1NQ7;uV`@uk3y2)Gi21R4$ zt#Au2`{9qG!;x3*U0uX{=_O=J%nSV32iEk_q1BJ3m41-j`B(&vpkDllI^@7a1IAk zS~O8QogZ0h9N1fDy$>}|D@>ao?QiVOye3gvaX(L(mY5Z8eDdRUT;b^^9~7)=79Sz- z034*mHZ2k7_&&nke28;TJn0|1sq_!E{DVSWTOdlsd**Q$6xg%{(~epnZddIx<$@WP ztUVDN=ca&Z+JbrR)`zjx+TA(DUpOby+IikvJO-gq*A^~}wswB~h($JG91E8;SReSa z&edPuT2Wzru&z-4suZ+%UO1Kf$CqsjAJMfWoLA4r_LIq0*B1416Q%JSX|Eh+(R+4r zAC_HAL!kze5x$F67PzJI!bLc?mua4ig-!-*$*=6ObjQX}Ti7;54k*bPe)a#tFb>fr z(c#)+iyQU+qTKqfatGAFr^a#f!wOh*Ep?zbt$8rgg7y{*7tC`=oytE{CtNbjZf)C1 zE$g$AmwXT@-bW-8Fetm&aRFO-uQ*rFwx{o=^599VAQnF*H{R+yHq<#i*EM^7H_AVg zCkIERvD8slt$wlUv@r{3Uz9|%-G=a<)+uQQI|WjC^a&n=7)R<`Rn|8W`C~*WD0N|w z^$mPBb$9k#`)58h6W{8nuDIC0tE0I~2lE^kwRCOiV*ylfpf?kBNDo>XkWPz#dfD|# zvJY^x7QK_~>gwrdgQx%@5pu50O2GAK1@hjXoOh?agmol_K%0rh;%1Bs59dMYjI6P8Q3d4$oaY*Vj)HUtne^1&~KAN_&LnKXFsO}_ZX;&81Q2K8=AGU_&bZzB#`IHlFV`ym#SzsJ|m$I zrR2;$fi(HbIM+Kn^SOAMwr)1Rl31siL;YylttTzANxf81=CDYL^_lJ*wrpDF@EV%x zH__$%naMXP8HZRQn91e%Vsf$a1angKa8Qbyz?z? z&LJoBb~{?u{4{d`7T{=ZqU4yJ44pNW`Q<@M!j~sds!=Ml(LrzA7|GJIp)&udqNHO7 z<*!a)FVWPzWNCS&)6Bp6&|A^j?1fC!(J6-#v7HYSs3s1td`c|c$4!{1W8z?1VZKcv zkFGfa1TFbAkb$X!zn`EbNYZo)G-~C?h~wFEO2qm)+N+N22#lsh$Oa6JNka|S9C`;z zJ$KU8Zyo!>DHqX$QXhCw(On&bOUl#K6v{Hk2r@k1pg>kr8Orgm2x*KScVLab2Tl=en2C9l7W z{R1il^xqO`5k4cYLpYWttIc9cUZF1h05Dz4y03;7VTA-5Qv+M(ZBJ`2FBOZ=II^ZE zQl{yaZ0xKy4w{O|glq!6f<~uyG^MVMltm%UYIB)J7V^#3O6ysc0GfX$j(tqg7#={2 z?jP^!9$ByWP@H=$D@XxL@S-Gqnxnb`mNm|u#$nyE);TlCivNbDJ6Y-ZF+n7NVf&JWV!Jb%e+o*>`(V#<+MkwrXVp&v;_D3lLgF(I1 zgR(vRDD^AVU;LS9FXYT{N301LrAnKzp9{pq5R`#&`AtOui*Bg>-x*EkOvcEITW!qgZ;OvArN{yV?s$h+v1+TP9K%QOp5(qiMg1z+PY*NOT@xv$caUo`k&PRrqcMJgx7v4Z;DPs>Z8(Xd+g7@Q31(BODZ!IZ5hN=_6-~ z8^wJyTVBRs7F7CEG7>9phL5pp=ODX!6lO{nP{97NDk6+l;VfW_PFG=LdC5h!w+7a_ zwyIqsEx|?SZcb@eJ&{fe#_Z$qG(}EGR&AL`x#kHfxD}dojG`&<(^~ktqVcq#xw9YS znIq_+#tW6S0Fx_+QrZO|mCoF*P`+2+u-+^$Wu zYVHs0K>&vuKYP%e{BmdauxYD)N~gG&3)xSV#?HpmM6V)NVT#mPZpQ|SM<|a(stRUb zqBIBxT_{;{$9YgR@}HXGqe*QyNDS-7uYNfank{&$F4PKbXuu#!c=lh`2;c%kvl* zSV1g)-jmG~g8+yY9C`Diu`;1aeW*T{pXN_%9uASvRYBgd97=g4*gsW%YS#a8$2B6PDFy-3$ z9b+l5b3iLD{VQnr(5~F04$BbD{1>Z8pYS>>2*%R2{J;Yg`rEUzK#iyQV{+)tOFLY$ z(pf~CCE?U%_!2e)8*253J`|)f zwVKdBcc-x2%T5IW*r$Os#0)!OqN`88L?duM8`@Itu=>;i8sXKJai9=fE_R?1-T5plu=o-GXX0#k(*0rL`jg<8ue~BxXDw-8aBUAF&>DZLTT@xr9 zFD+>P^53rE(#0~Ovkn6i3tVrPj1z_1{a zTFfhXCa|*0G`f1TG?aYMfSe|QQSV9!eD)AD1d|bMd_4<%z_Y1DX6%QCq+G0W5KVo7HT{dOd}8q2rjC>6wj!j z*#4K;P>i&AhlA2TZk5`m_(}}r^qwK->&leHAGV|P-{-p&RGe5&;SJkag~|*|5-AKi zmB3&*u>EHxb$^*Z0j1_qaJ!vx6gV!R27Ckshte&L2%E2UnxW+NgESV01n{d8t|UC1 z#y%e@+big6rY0I!j()-uMdniSuSVlWg;ufd$!_I<3W?5br;gS zGV1(vI_m>8Y+C7?VU%}~uNF|Ul+Ha*xyN{eTPdg%ADS_yZJF&I8Y^i^qBh9#K2Qyx zx>ovpFpcSQ(Piv4k491P1*=QzrMMu8Fd?>IDpW#!$(GnXOd#-~O@Y_d>Eib)=Qk!3@#(7K4!#+qI)S zDF-=aWdVe?K;Ql<5^sU9@twVvE7 zp?TCO2%{-iJXsbt!`cOL#uc24b^r%P2wf}dT}gSts=p{h{ZVCo4tL>wq88kt)8TGb z-ye=zd#P_ETIY$YIq^{hq^5#?#UX)$B+=Qqs<3WtBjvxwvG}S< z+0hDPS>`qVSXFF4FBK)2p_obrurn2+Bu$+q<{Im|xEnTP1HKMF1rF5sGf}*ZFkm!M z#$H!hP06iyn%85$vY`J*lBJ0>*TcAi0}mLG(2RPZZx7(ekv<36o*BR;C zSRczBu~_Ot5wf=USqn1ao6%W`-L2rNLMGuy(W*n#F z5d8M|1o3h}y(Fk~y20ll6r^*%1t(qG@Nq0N9TnBJeC9-T!VQtTDWx{T z0yz|y4cq$Bk_|rmzGdhe&NGWWIxD8QaCm?`ceMvJ!}TB*0qgsi>V`0 zFr|P79-Fq|+gR(;f#>9+oqTNC#+AX;@yI3?zh2s<;^{=nxOj|}>?R8~HdY$H;m~GV zST032sR)XrWm}h9@T*c`1@?jB5dU!s&JX*7M(iF+YojsI5ZN0l(qE!iv9|RPX5(mv z0-L*-jdNN+sf4Qt2@rEI$AvB=Yt-;-v<7;3h+fW7ssb}dJ6^yg&R$vZZLIMXVrpc8 zBKozjWwz_Qn5JzS6-h&%9l*13;25H9%4wjbFSXpxfP$%Y)DG!S?gLv!x@l$%bws8T z$e}!WQ)3VMt*rBvf0#9j>Zse!23N1#T(X!l{Be z!Qy3PCj=Lzpv}3vjrWn1S=L~-mG?B#Llghv8Jm(NCdTSzrWi&Io3^dVm7>e0e4k@LXt-mZPhu)-_Iak<} zK&xw&6O(8bf=WgREWrIf0+4po{6&E?rc<3nj~M{#H-(;xiOLmK#3SE-$uZ zCDydcW$nm*WPs$NI&$S(ku*DV8!L!aSoz`}W1%_QQW}eQP*5F$sZ>(&<}O2>^CsIItd z8{#m2nRh@si$c2X*=sbno%+y(_-OL?2J-h9!&?-9kAnUgmM3v2PXbeHpX?;|B5oXi zm<7*xh`mTs4W!D0wk`Lkl&&4+a)-$lOM7a_Ds4`GOuezs(GCmu~UZ{}b{;2Bt z7o{!Mj4Z4uv@xGqE##Q)3PtiSnG$SlB;Gniap4_59LTYHm$1R zGCgd4jk^c9prA3{*nq{*ddRX_YOLzVtJE%QJo_o*sQT$VwS(h=gJQe=H9;$T@DWu7 zYt=gE66@srQ=s@X9iN&WBs|tDQT$fNZMUA zCc))F+ryFE1ZnYFe0}?V*!GP#rN-og94jl5jJt7k7-le4cs80=PFHyq6c$vz; z_5B_DzM&NeDuFK4D79l-(;#7fN2N#)+VNXCt>_oPE~RL|aqog#I#hoN)2e%h8GEtH zBDKyns$cLH51sx-x`KQ(t$Luh7=sOepj6$X8aL86^!iJpRBWr$E9t3GhWxNnYxLE# z(#0oOM1U+xW7UbF;*vWk0d`WUCvttMgER z&;+PY`DyjHQIy$>_Xy1P`s9HRVd_7M@*z4v1a|6n9aM%W^xr#~U>W@&z#Y*tDJA9Ym=^ zT^ztTc3P6i_ovTX(~X^vWr$Lk(rlt>JAILfqH8~=_@MD}qu9FsYo~%xFe_ASMMz?b zvp4Kat2P2L=Wo&J&Qeb?!OWG#(06q#r{y>q+uixn+WBdfk%RY3i?Mfgjzf(%orco2 zo!7Ar^Y#q(G5CnC?P?!Q%PXd_!+^C-+tu|JJs!SJj>&Rzgc{EuG~{ir_*FUXuE=m> zG=NF1qYyQ=)KGuiCoXZ+GmT0?{kyA!O6+vI_D4~FEHIyPBHgvgPi#G}$pYnLyLJSL zt!6Iy5^39ApJ7RD>M_v?+fVM&F1x-uPxeob@Svn>d=Xopi^Gx^oKCmvrXMZ)^tx++ zt!aN#Y;he_(zTinXT_G*$z5yij}}{TDh((D;S#9PIY4anZ8=e9(`u|uv;vz;kW~3t z&9Fdm;@otX2L0&v-gTkg!`9cTtF zwdvwxWU-PofW@Z0*WP*4%j(`c zh{Z=O6wKIP-xZsF@DJq<@5~rn{}A5$5(ij==z>nh+1{&o2zIb6iQSB|eN-fq5-!2F z-cz0PG-bIOWsJhoFHowX@15)A#<|P;n2QDh?XLU!+1b$UD zGXHko>h8x0mQ=wX$kOcIi?4TOSzgIF8emz4>S)iy)+;L$I$2~K8@Q+%+Y&liJE)sj zrK0sgZBBn9mg0j(;((He40>|JWs#1x3#gB6JLpM#ai+gm#{ISyzy>{??k$#nX6JDX zX!Sw;4>!_MTnd3dSo}f3I4!sYq18~I;u3X#CE2Icv 0 +MAG_AUTO < 21 +FWHM_IMAGE > 0.3 / 0.187 +FWHM_IMAGE < 1.5 / 0.187 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +[MASK:flag] +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + + +[MASK:star_selection] +# Star selection using the FWHM mode +MAG_AUTO > 18. +MAG_AUTO < 22. +FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 +FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 +FLAGS == 0 +IMAFLAGS_ISO == 0 + +[MASK:fwhm_mag_cut] +FWHM_IMAGE > 0 +FWHM_IMAGE < 40 +MAG_AUTO < 35 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +# Split the 'star_selection' sample into +# two random sub-samples with ratio 80/20 +[RAND_SPLIT:star_split] +RATIO = 20 +MASK = star_selection + +# The following selection is only used for plotting + +[PLOT:size_mag] +TYPE = plot +FORMAT = png +X_1 = FWHM_IMAGE{fwhm_mag_cut} +Y_1 = MAG_AUTO{fwhm_mag_cut} +X_2 = FWHM_IMAGE{star_selection} +Y_2 = MAG_AUTO{star_selection} +MARKER_1 = + +MARKER_2 = . +MARKERSIZE_1 = 3 +MARKERSIZE_2 = 3 +LABEL_1 = All +LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" +TITLE = "Stellar locus" +XLABEL = "FWHM (pix)" +YLABEL = Mag + +[PLOT:hist_mag_stars] +TYPE = hist +FORMAT = png +Y = MAG_AUTO{star_selection} +BIN = 20 +LABEL = "stars" +XLABEL = "Magnitude" +YLABEL = "Number" +TITLE = "Magnitude of stars" + +[PLOT:fwhm_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = FWHM_IMAGE{star_selection}*0.186 +MARKER = . +LABEL = "FWHM (arcsec)" +TITLE = "FWHM of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[PLOT:mag_star_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = MAG_AUTO{star_selection} +MARKER = . +LABEL = "Magnitude" +TITLE = "Magnitude of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[STAT:star_stat] +"Nb objects full cat" = len(FWHM_IMAGE) +"Nb objects not masked" = len(FWHM_IMAGE{flag}) +"Nb stars" = len(FWHM_IMAGE{star_selection}) +"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. +"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 +"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 +"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 +"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 +"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 From 1b22b9884d658affc2816e3b9ec38b169826a31f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:22:51 -0400 Subject: [PATCH 013/100] =?UTF-8?q?feat(orchestration):=20manifests=20beco?= =?UTF-8?q?me=20the=20DAG's=20currency=20=E2=80=94=20completeness=20check?= =?UTF-8?q?=20CLI=20+=20manifest-driven=20report=20(#848=20D2/D3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit completeness.py gains the 'check ' CLI every rule's shell line calls after shapepipe_run: counts products under $SP_RUN against the floor table, writes a byte-stable JSON manifest (per-runner found/expect/ floor/warn, log-scraped failure reasons, no wall-clock), exits nonzero iff a mandatory runner is below floor — manifest always written first. STAGE_DIR maps stages to the committed configs' fixed RUN_NAMEs; the ngmix entry env-expands ${SP_NGMIX_CHUNK} so chunk K checks chunk K's dir. run_report.py rewritten to read manifests instead of disk-scanning: per-tile / per-exposure tables with failure reasons, attrition aggregate (failed units excluded), tile-blocked-by-exposure join, 'not run' for missing manifests. CLI interface unchanged (--run-dir --index --status; --out/--limit added). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- workflow/scripts/completeness.py | 209 ++++++++++++++++++++- workflow/scripts/run_report.py | 313 ++++++++++++++++++++----------- 2 files changed, 409 insertions(+), 113 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index b2246abef..dd0836c66 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -8,12 +8,18 @@ its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) sits between ``floor`` and ``expect`` and is tolerated. -Two consumers share this table: - * ``sp_rule.py`` — after ``shapepipe_run``, counts products per runner and - exits nonzero if any mandatory runner is below its floor (``keep-going`` - then isolates that unit's cone; ``retries`` handle transient failures). - * ``run_report.py`` — disk-scans finished trees against ``expect`` to - enumerate shortfalls for the human, mid-run or after. +This file is also the ``check`` CLI — the second half of every rule's shell line +(PRD D2/D3):: + + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} \\ + && completeness.py check exp_mask {output} + +It counts the unit's products under ``$SP_RUN``, writes the manifest at +``{output}``, and exits nonzero iff a mandatory runner is below its floor. The +manifest is ALWAYS written first, failure included: it is the DAG's currency and +the only thing ``run_report.py`` reads, so a failed unit must still leave a +record of *why*. Manifests carry no wall-clock — identical on-disk state must +produce a byte-identical manifest, or the mtime rerun-trigger churns the cone. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -27,6 +33,13 @@ ``ls / | wc -l`` semantics (broken symlinks excluded by the caller). """ +import argparse +import json +import os +import re +import sys +from pathlib import Path + # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} COMPLETENESS = { # --- tile prepare (phase A) --- @@ -94,3 +107,187 @@ def check_floor(stage, run_dir): if not spec.get("warn", False) and n < spec["floor"]: ok = False return ok, details + + +# --- where a stage writes ------------------------------------------------- +# +# stage -> (level, run_sp_ dir under $SP_RUN/output/). These are the +# committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so +# the check never resolves a run-log. The ngmix entry interpolates the same env +# var its config does, so chunk K's check looks at chunk K's dir. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_detect_uc": ("tile", "run_sp_tile_Uc"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +# --- failure reasons ------------------------------------------------------ + +# Lines worth showing a human who asks "why is this runner short?". Deliberately +# crude: the point is a pointer into the logs, not a taxonomy (there is no error +# whitelist in this design — the count floor is the policy). +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 # logs are per-CCD; a handful is enough to characterise +MAX_TAIL_LINES = 120 # per file +MAX_REASONS = 3 # per runner + + +def _normalise(line: str) -> str: + """Collapse a log line to its shape, so 40 per-CCD copies dedupe to one.""" + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) # paths differ per CCD + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + """Best-effort, bounded: distinct error-looking lines from a runner's logs. + + Two sources, in order of usefulness: the runner's per-process worker logs + (``/logs/process-*.log`` — where the module's own exception lands), + and the stage's ``logs/log_sp.log`` (where ShapePipe records its error + tally). Sorted, truncated, deduped by shape — a manifest must stay + byte-stable for a given tree. + """ + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +# --- manifest ------------------------------------------------------------- + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + """Count, classify and (on shortfall) scrape. Returns (manifest, ok). + + Stages absent from the table fall back to the zero-output floor: any product + anywhere under the stage dir passes, nothing at all fails. + """ + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, + "level": level, + "unit": unit, + "run_dir": str(run_dir), + "stage_dir": str(stage_dir), + "runners": {}, + "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "floor": 1, + "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_floor(stage, stage_dir) + short = False + for runner, n, floor, expect, warn in details: + below = n < floor + if n < expect: + short = True + manifest["runners"][runner] = { + "found": n, "expect": expect, "floor": floor, "warn": warn, + "status": ("complete" if n >= expect else + "warn" if (warn or not below) else "below_floor"), + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "floor": floor, "expect": expect, + "warn": warn, "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def _unit_from_env(): + """``SP_UNIT_NUM`` carries the rules' dashed, dash-prefixed form + (``-210-282``, ``-2605805``); the manifest records the human ID.""" + raw = os.environ.get("SP_UNIT_NUM", "") + return raw.lstrip("-") or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="ShapePipe per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--run-dir", type=Path, default=None, + help="the unit's $SP_RUN (default: the env var)") + c.add_argument("--unit", default=None, help="override $SP_UNIT_NUM") + c.add_argument("--stage-dir", default=None, + help="override the run_sp_* subdir (default: the stage table)") + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_env() + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + args.manifest.parent.mkdir(parents=True, exist_ok=True) + args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} + print(f"[completeness] {runner}: {r['found']}/{r['expect']} " + f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) + print(f"[completeness] {args.stage} {unit}: {manifest['status']} " + f"-> {args.manifest}", file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 683433ecb..05ca92478 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -1,70 +1,169 @@ #!/usr/bin/env python3 -"""Standalone run report — NOT a DAG node. - -A report rule that declares all tiles' outputs as inputs is a descendant of every -job, so any single hard failure (under --keep-going) poisons its cone and the -report never runs — the exact scenario it exists for (finding 8). So this is a -plain script: it disk-scans the tiles/ and exp/ trees against the count-floor -table (completeness.py) and the index, and emits run_report.json. It runs -automatically via the Snakefile's onsuccess/onerror hooks, and by hand any time -(``sp report``), mid-run included. - -It distinguishes whole-exposure absence (all runners missing — a real gap or a -deletion/forest bug) from tolerated per-CCD attrition (counts between floor and -expect), so a future deletion bug cannot be silently absorbed (finding 5). - -Attrition is counted at file granularity, not unit granularity: each stage's -``products`` block aggregates found-vs-expected file counts per runner across -all passing units, with a per-unit shortfall map where files are missing. -``warn`` runners (psfex_interp) are exempt from *failing* a unit, not from -being *reported* — the P0 validation found 46/760 psfex_interp CCDs missing -(6%) behind a unit-level "zero attrition" claim, which is the failure mode -this granularity exists to prevent. +"""``sp report`` — the run's success/failure tables, read from the manifests. + +NOT a DAG node. A report rule that declared all tiles' outputs as inputs would be +a descendant of every job, so one hard failure under --keep-going would poison +its cone and the report would never run — the exact scenario it exists for. So it +is a plain script, runnable at any time including mid-run; the Snakefile's +onsuccess/onerror hooks call it so every invocation ends with one. + +It reads two things and nothing else (PRD D3): + + * the **index** (``run_index.sqlite``) — the units the run declared, and the + tile->exposure edges that let an exposure failure be blamed on the tiles it + blocks; + * the **manifests** — one per unit per stage, written by + ``completeness.py check``, carrying per-runner found/expect/floor and + log-scraped failure reasons. + +No disk scanning: counting products is the *check's* job, done once at the moment +the products were fresh. A unit with no manifest for a stage is "not run" — which +is a real and distinct answer from "ran and produced nothing". + +Manifests are discovered by glob (``tiles/**/manifests/*.json``), not by +constructed path: the unit stores are sharded (``tiles///``) and the +sharding depth is not this script's business. """ import argparse import json import sqlite3 import sys +from collections import defaultdict from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from completeness import COMPLETENESS, check_floor # noqa: E402 - -# stage -> (level, run_sp_ dir name) -STAGE_DIR = { - "tile_get_images": ("tile", "run_sp_tile_Git"), - "tile_uncompress": ("tile", "run_sp_tile_Uz"), - "tile_find_exposures": ("tile", "run_sp_tile_Fe"), - "exp_get_images": ("exp", "run_sp_exp_Gie"), - "exp_split": ("exp", "run_sp_exp_Sp"), - "exp_mask": ("exp", "run_sp_exp_Ma"), - "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), - "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), - "tile_mask": ("tile", "run_sp_tile_Ma"), - "tile_detect": ("tile", "run_sp_tile_Sx"), - "tile_vignets": ("tile", "run_sp_tile_PiViVi"), - "tile_merge_cats": ("tile", "run_sp_Ms"), - "tile_make_cat": ("tile", "run_sp_Mc"), -} - - -def classify(stage, run_dir): - """'absent' | 'under' | 'attrition' | 'complete' for one unit's stage.""" - if not run_dir.is_dir(): - return "absent", [] - ok, details = check_floor(stage, run_dir) - if not ok: - # all mandatory runners empty -> whole-unit absence; else below floor - mand = [d for d in details if not d[4]] - if mand and all(n == 0 for _, n, *_ in mand): - return "absent", details - return "under", details - # warn runners are exempt from FAILING a unit (check_floor), not from - # attrition reporting — psfex_interp is precisely the runner that attrits. - if any(n < expect for _, n, _, expect, _ in details): - return "attrition", details - return "complete", details +# Stage order per level — the report's column order, and the definition of +# "expected" (a declared unit with no manifest for one of these is not run). +TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_mask", "exp_psf"] + +STATUSES = ("complete", "warn", "failed", "not_run") + + +def load_manifests(run_dir: Path, sub: str) -> dict: + """``{unit: {stage: manifest}}`` for one store (``tiles`` or ``exp``). + + The unit key is the manifest dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the manifest's own + ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is + not the index's ``210.282``). The stage comes from the manifest body, never + the filename: ngmix chunks share a stage under per-chunk filenames, and they + collapse to one worst-case entry. + """ + out: dict = defaultdict(dict) + for path in sorted((run_dir / sub).glob("**/manifests/*.json")): + try: + m = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable manifest {path}: {exc}", file=sys.stderr) + continue + unit = path.parent.parent.name + stage = m.get("stage", path.stem) + prev = out[unit].get(stage) + # Worst status wins when several manifests share a stage (ngmix chunks). + rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 + if prev is None or rank(m) > rank(prev): + out[unit][stage] = m + return out + + +def shortfalls(m: dict) -> dict: + """``{runner: (found, expect, floor)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"], d["floor"]) + for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} + + +def reasons(m: dict) -> list: + """Flattened failure reasons, runner-tagged, for the report's why column.""" + out = [] + for f in m.get("failures", []): + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')} (floor {f['floor']})" + out += [f"{head}: {r}" for r in f["reasons"]] or [head] + return out + + +def tally_level(units, stages, manifests) -> dict: + """Per-stage counts + named unit lists, for one level.""" + per_stage = {} + for stage in stages: + t = {"complete": 0, "warn": [], "failed": [], "not_run": []} + agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) + for u in units: + m = manifests.get(u, {}).get(stage) + if m is None: + t["not_run"].append(u) + continue + status = m.get("status", "failed") + status = status if status in ("complete", "warn") else "failed" + if status == "complete": + t["complete"] += 1 + else: + t[status].append(u) + if status == "failed": + # Failed units are named above, never folded into the attrition + # aggregate: a whole-unit failure is not per-CCD attrition, and + # mixing them hides real deletion bugs behind a big denominator. + continue + for runner, d in m.get("runners", {}).items(): + a = agg[runner] + a["found"] += d["found"] + a["expect"] += d["expect"] + if d["found"] < d["expect"]: + a["by_unit"][u] = d["expect"] - d["found"] + for a in agg.values(): + if not a["by_unit"]: + del a["by_unit"] + t["products"] = dict(agg) + per_stage[stage] = t + return per_stage + + +def unit_rows(units, stages, manifests) -> list: + """One row per non-clean unit: its first bad stage, shortfalls, why.""" + rows = [] + for u in units: + got = manifests.get(u, {}) + bad = [s for s in stages + if got.get(s) is None or got[s].get("status") != "complete"] + if not bad: + continue + stage = bad[0] + m = got.get(stage) + rows.append({ + "unit": u, + "stage": stage, + "status": "not_run" if m is None else m.get("status", "failed"), + "shortfalls": shortfalls(m) if m else {}, + "reasons": reasons(m) if m else [], + "n_bad_stages": len(bad), + }) + return rows + + +def print_table(title, rows, limit=25): + print(f"\n{title} ({len(rows)} affected)") + if not rows: + print(" — none") + return + print(f" {'unit':<14} {'stage':<20} {'status':<8} why") + for r in rows[:limit]: + short = ", ".join(f"{k} {v[0]}/{v[1]}" for k, v in r["shortfalls"].items()) + why = (r["reasons"][0] if r["reasons"] else short) or "-" + print(f" {r['unit']:<14} {r['stage']:<20} {r['status']:<8} {why[:90]}") + if len(rows) > limit: + print(f" … and {len(rows) - limit} more (see the JSON report)") + + +def print_stage_table(title, per_stage, n_units): + print(f"\n{title} ({n_units} units declared)") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} attrition") + for stage, t in per_stage.items(): + att = [f"{r} {a['found']}/{a['expect']}" + for r, a in t["products"].items() if a["found"] < a["expect"]] + print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " + f"{len(t['failed']):>6} {len(t['not_run']):>8} {', '.join(att)[:60]}") def main() -> None: @@ -73,66 +172,66 @@ def main() -> None: p.add_argument("--index", required=True, type=Path) p.add_argument("--status", default="manual") p.add_argument("--out", type=Path, default=None) + p.add_argument("--limit", type=int, default=25, + help="rows per stdout table (the JSON report is complete)") args = p.parse_args() - tiles, exps = [], [] + tiles, exps, tile_exp = [], [], defaultdict(list) if args.index.exists(): con = sqlite3.connect(args.index) - tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles")] - exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures")] + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] + for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + tile_exp[tile_id].append(exp_id) con.close() + else: + print(f"[run_report] no index at {args.index} — reporting manifests only", + file=sys.stderr) + + tile_m = load_manifests(args.run_dir, "tiles") + exp_m = load_manifests(args.run_dir, "exp") + tiles = tiles or sorted(tile_m) + exps = exps or sorted(exp_m) missing_json = args.index.parent / "missing.json" missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] - report = {"status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), - "missing_tiles": missing, "stages": {}} - for stage, (level, prefix) in STAGE_DIR.items(): - if stage not in COMPLETENESS: - continue - units = tiles if level == "tile" else exps - sub = "tiles" if level == "tile" else "exp" - tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} - # File-level aggregate per runner over PASSING units (complete/attrition); - # under/absent units are enumerated by name above, not folded in here — - # mixing them would double-count whole-unit absence as attrition. - products = {r: {"found": 0, "expect": 0, "by_unit": {}} - for r in COMPLETENESS[stage]} - for u in units: - run_dir = args.run_dir / sub / u / "output" / prefix - verdict, details = classify(stage, run_dir) - if verdict in ("complete", "attrition"): - tally[verdict] += 1 - for runner, n, _, expect, _ in details: - agg = products[runner] - agg["found"] += n - agg["expect"] += expect - if n < expect: - agg["by_unit"][u] = expect - n - else: - tally[verdict].append(u) - for agg in products.values(): - if not agg["by_unit"]: - del agg["by_unit"] - tally["products"] = products - report["stages"][stage] = tally - - finals = [t for t in tiles - if (args.run_dir / "tiles" / t / f"final_cat-{t}.fits").exists()] - report["final_cats"] = {"present": len(finals), "of": len(tiles), - "missing": [t for t in tiles if t not in finals]} + report = { + "status": args.status, + "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m), + "tiles": unit_rows(tiles, TILE_STAGES, tile_m), + "exposures": unit_rows(exps, EXP_STAGES, exp_m), + } + + # Blame propagation: an incomplete exposure blocks every tile that reads it. + # Without this, a tile stalled at tile_vignets looks like its own failure. + bad_exp = {r["unit"] for r in report["exposures"]} + blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} + report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} + + done = report["tile_stages"]["tile_make_cat"]["complete"] + report["final_cats"] = {"present": done, "of": len(tiles)} out = args.out or (args.index.parent / "run_report.json") - out.write_text(json.dumps(report, indent=2)) - print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " - f"cats -> {out}", file=sys.stderr) - for stage, tally in report["stages"].items(): - for runner, agg in tally["products"].items(): - if agg["found"] < agg["expect"]: - pct = 100 * (agg["expect"] - agg["found"]) / agg["expect"] - print(f"[run_report] attrition {stage}/{runner}: " - f"{agg['found']}/{agg['expect']} files (-{pct:.1f}%) " - f"across {len(agg['by_unit'])} units", file=sys.stderr) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "")) + print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) + print_stage_table("TILES", report["tile_stages"], len(tiles)) + print_table("exposures not complete", report["exposures"], args.limit) + print_table("tiles not complete", report["tiles"], args.limit) + nb = report["tiles_blocked_by_exposures"] + if nb: + print(f"\ntiles waiting on incomplete exposures ({len(nb)})") + for t, e in list(nb.items())[:args.limit]: + print(f" {t:<14} {', '.join(e[:6])}" + + (f" (+{len(e) - 6})" if len(e) > 6 else "")) + print(f"\n[run_report] -> {out}") if __name__ == "__main__": From 83d26f2088447c5989060b6e5c4bac3b23c39c6d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:36:11 -0400 Subject: [PATCH 014/100] =?UTF-8?q?feat(orchestration):=20the=20rules=20re?= =?UTF-8?q?write=20=E2=80=94=20manifests=20as=20DAG=20currency,=20sharded?= =?UTF-8?q?=20stores,=20parse-time=20index,=20native=20scattergather=20(#8?= =?UTF-8?q?48=20D1=E2=80=93D5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sp_rule.py is deleted; its responsibilities dissolve into the design: config normalization -> the committed S2 configs; log sync -> dead (zero 'last:'); unit furniture -> inline shell (tile_numbers.txt, pseudo-Fe exp_numbers with the ORIGINAL exposure name, star-cat symlinks); count floor -> completeness.py check writing the manifest each rule declares as its only output. SP_UNIT_NUM carries the leading-dash dashed form; the rules do the transform. Stores shard to tiles/<2ch>// and exp/<2ch>//; build_index accumulates (no DROP TABLE) and is built at parse time of the compute invocation; bin/sp reduces to run (two invocations) + report + cancel. ngmix scatters via the scattergather knob with in-job closed ID ranges (scripts/ngmix_range.py); retries with attempt-scaled mem on transient- exposed rules; protected() dropped; temp(directory()) rides as a scoped secondary output on the vignette store and chunk dirs — the one exception to no-directory-outputs, so native temp() reclaims intra-tile bulk. Deviations proven by implementation: merge_sep_cats INPUT_DIR must stay relative (MergeSep re.sub's the first '1' in the path — absolute sharded paths break chunk discovery); failed jobs need keep-incomplete in the profile or Snakemake deletes the failure manifests sp report exists to read (-> S6). Dry-runs: prepare 13 jobs, compute 141 on the 210/211 quad (19 exposures, 8 ngmix chunks); --set-scatter ngmix=1 -> 113. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- workflow/Snakefile | 293 ++++++++---- workflow/bin/sp | 81 ++-- workflow/config.yaml | 14 +- .../config/cfis/config_merge_sep_cats.ini | 11 +- workflow/rules/exposure.smk | 108 +++-- workflow/rules/prepare.smk | 85 ++-- workflow/rules/tile.smk | 269 ++++++----- workflow/scripts/build_forest.py | 13 +- workflow/scripts/build_index.py | 61 ++- workflow/scripts/ngmix_range.py | 55 +++ workflow/scripts/sp_rule.py | 419 ------------------ 11 files changed, 644 insertions(+), 765 deletions(-) create mode 100644 workflow/scripts/ngmix_range.py delete mode 100644 workflow/scripts/sp_rule.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 5d93a1810..b707fdc17 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,24 +1,31 @@ """ShapePipe real-data orchestration — Snakemake workflow. -Design / rationale: CosmoStat/shapepipe#848 (the living PRD). - -Execution is honestly THREE static invocations, chained by ``workflow/bin/sp`` -so the UX stays one command (``sp prepare`` / ``sp run``): - - snakemake --profile profiles/nibi prepare_tiles # Git_vos + Uz + Fe (keep-going) - python workflow/scripts/build_index.py ... # PLAIN SCRIPT, not a DAG node - snakemake --profile profiles/nibi prepare_exposures # Gie_vos + star cats - snakemake --profile profiles/nibi # the main compute DAG - -The run index is loaded ONCE here at parse time into plain dicts; it is never a -rule input, so appending tiles and rebuilding it changes which jobs exist without -invalidating completed work. build_index.py is a plain script *between* -prepare_tiles and prepare_exposures (the exposure job set is data-derived from -the tiles' find_exposures output, so it cannot be scheduled in the same static -DAG that produces it). +Design / rationale: CosmoStat/shapepipe#848 (the living PRD), D1-D5. + +`sp run` is TWO snakemake invocations over this one Snakefile: + + SP_PHASE=prepare snakemake prepare_all_tiles # Git -> Uz -> Fe, per tile + SP_PHASE=compute snakemake all # everything else + +They are two because the exposure job set is *data-derived*: it comes from the +tiles' find_exposures output, and a Snakemake DAG is fixed at parse time. The +join between them is the tile<->exposure index, built HERE at parse time of +invocation 2 (build_index.build(), imported — there is no `sp index` verb) and +loaded into plain dicts. The index is never a rule input, so appending tiles and +rebuilding it changes which jobs exist without invalidating completed work; it +ACCUMULATES across invocations, so a later clean_exposure (S5) sees every +consuming tile of the whole campaign, not just this tile list. + +The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared +output is its MANIFEST (`/manifests/.json`), written by +`completeness.py check`. Product files are not declared — a missing CCD is often +legitimate, and at DR6 scale per-CCD declaration means millions of paths. """ +import hashlib +import os import sqlite3 +import sys from pathlib import Path # Resolved relative to THIS file, not the working directory: snakemake runs with @@ -32,16 +39,49 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) -CONFIG_SRC = Path(config["config_src"]) STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" -NGMIX_CHUNKS = config.get("ngmix_chunks", 8) +# The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the +# configs and the rules that set the env vars they interpolate are one artefact +# and must version together. Hence no `config_src` knob. +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" + +sys.path.insert(0, str(SCRIPTS)) +import build_index # noqa: E402 +from completeness import STAGE_DIR # noqa: E402 + +# `prepare` suppresses the missing-tile threshold (tiles are still being +# prepared) and the report hooks. Unset == compute (a bare hand-run of `all`). +PHASE = os.environ.get("SP_PHASE", "compute") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] -# --- parse-time index load (compute phase only) ---------------------------- +# --- ngmix scatter (D4) ---------------------------------------------------- +# Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one +# ngmix job per tile. We take the count and drive our own integer `chunk` +# wildcard rather than snakemake's `{scatteritem}` ("3-of-8") token, because the +# chunk number is not ours alone: it names the run dir +# (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and +# merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the +# first "1" — which only works for bare integers. +scattergather: + ngmix=int(config.get("ngmix_chunks", 8)) + +NGMIX_CHUNKS = workflow._scatter["ngmix"] + +# --- parse-time index build + load (D1) ------------------------------------ +# Invocation 2's parse IS the index build. Tiles whose find_exposures output is +# absent land in index/missing.json and are dropped from the DAG; the build is +# fatal only above SP_MISSING_THRESHOLD (and never in the prepare phase, where +# absent Fe output is the normal state). +if any(build_index.exp_list_path(RUN_DIR, t).exists() for t in TILES): + build_index.build( + TILES, RUN_DIR, INDEX_DB, + missing_threshold=(None if PHASE == "prepare" + else float(os.environ.get("SP_MISSING_THRESHOLD", "0.0")))) + # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. @@ -49,93 +89,186 @@ EXP, TILE_EXP = {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) - for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): - TILE_EXP.setdefault(tile, []).append(exp) + for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(_tile, []).append(_exp) _con.close() +# Tiles this run can actually compute: declared AND indexed. The index spans the +# campaign, so it is intersected with the declared list, not used as it. +TILES_READY = [t for t in TILES if t in TILE_EXP] + wildcard_constraints: - tile = r"\d{3}\.\d{3}", - exp = r"\d{6,7}", + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", chunk = r"\d+", -# --- shared wrapper invocation --------------------------------------------- -def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, - extra=""): - """Build the shell command that runs one unit through sp_rule.py. - - NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass - ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download - numbering must not be constrained. ``--threads {threads}`` is appended in - each rule's ``shell:`` string (NOT here: Snakemake formats the shell string - once, so a ``{threads}`` inside params.cmd would survive literally) and - becomes SMP_BATCH_SIZE (fork width == cpus_per_task). - """ - cmd = ( - f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " - f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} " - f"--script-hash {SCRIPT_HASH}" - ) - if not isolate: - cmd += " --no-isolate" - if exp_forest: - cmd += f" --exp-forest {exp_forest}" - if extra: - cmd += " " + extra - return cmd +# --- the sharded stores (D2) ---------------------------------------------- +# tiles/<2-char prefix>// and exp/<2-char prefix>// — no directory +# exceeds ~1k entries at full-UNIONS scale. Rules carry the shard as its own +# wildcard because an output pattern cannot compute it; every path the DAG uses +# is built by these helpers, so a mismatched (shard, id) pair is never requested. +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") def tile_dir(tile): - return RUN_DIR / "tiles" / tile + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" def exp_dir(exp): - return RUN_DIR / "exp" / exp - -# Content hash of the wrapper scripts, computed once at parse time. Exposed as a -# param on every rule (via sp_rule() and tile_exp_forest) so the default -# rerun-triggers' *params* trigger covers script edits: the `code` trigger only -# hashes the rule's own shell string, NOT external scripts it calls — without -# this, a wrapper/forest-script fix silently leaves stale outputs in place -# (bitten live: the forest-shard fix rebuilt nothing on rerun). -import hashlib as _hashlib -SCRIPT_HASH = _hashlib.md5(b"".join( - (SCRIPTS / n).read_bytes() - for n in ("sp_rule.py", "completeness.py", "build_forest.py"))).hexdigest()[:12] + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def final_cat(tile): + return f"{tile_dir(tile)}/final_cat-{tile}.fits" + +def unit_num(unit): + """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading + dash (tile ``210.282`` -> ``-210-282``; exposure ``2605805`` -> ``-2605805``). + The RULES do this transform; the configs just interpolate $SP_UNIT_NUM into + NUMBER_LIST (the `set_config_number_list` mechanism that replaced the retired + -e/--exclusive flag, #746).""" + return "-" + unit.replace(".", "-") + +# Content hash of completeness.py, computed once at parse time and carried as a +# param on every rule: the default rerun-triggers' `code` trigger hashes only the +# rule's own shell string, NOT external scripts it calls — without this, a fix to +# the count table silently leaves stale manifests in place (bitten live). Scoped +# to completeness.py alone, the one script every shell line runs; build_forest.py +# gets its own hash, on the forest rule only. +SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] +FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] + +# --- the shell every rule runs (D2) ---------------------------------------- + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1), + ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) + + +def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, + pre_run=()): + """The unit-furniture + environment prologue, as bash. + + Returned as a rule ``params`` value, NEVER inlined into the ``shell:`` + string: snakemake formats a shell string ONCE, so a ``{output}``/``{threads}`` + placeholder inside a params value would survive literally — and, conversely, + the literal ``${SP_NGMIX_CHUNK}`` braces this prologue needs would blow up + that formatting if they lived in the shell string. Params values are + substituted after formatting, so both hazards go away together. + + What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT + -e/--exclusive): + * ``output/`` and ``manifests/``; + * tile: ``tile_numbers.txt`` (dot format — what get_images reads); + * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the + ORIGINAL exposure name from the index (``2605805p``), so get_images + matches ``.fits.fz`` in the store — the bare base id matches + nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale + pre-fix file with the bare id; + * ``star_cat_exp`` / ``star_cat_tiles`` dir symlinks into the shared + pre-generated pool (the mask configs read them as INPUT_DIRs). + There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at + the committed config dir. + + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's + FileHandler raises on an existing run dir, and it is how a rerun never sees + stale products (D2: the job clears its run dir at start). + """ + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + _, subdir = STAGE_DIR[stage] + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_UNIT_NUM='{unit_num(unit)}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + # Also set via apptainer-args in the profile; kept here so a hand-run of + # this same line outside snakemake behaves identically. + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + for k, v in (env or {}).items(): + lines.append(f"export {k}='{v}'") + + if level == "tile": + lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + else: + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + src = STAR_CATS / sub + lines.append(f"[ -d '{src}' ] && ln -sfn '{src}' \"$SP_RUN/{name}\" || true") + + lines += list(pre_run) + lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] + return "\n".join(lines) + + +def sp_shell(stage, config_name): + """The rule's shell string: prologue, one shapepipe_run, one completeness check. + + ``{threads}`` and ``{output}`` are placeholders HERE and nowhere else (see + unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number + by construction (D4). + + The check runs even when shapepipe_run failed — the manifest is the only + thing `sp report` reads, so a failed unit must still leave a record of why. + """ + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}} || rc=1\n" + "exit $rc\n" + ) + include: "rules/prepare.smk" include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only aggregation targets are localrules (no-ops under the local executor, but -# they keep the compute chain group-compatible: no mid-chain localrules). -localrules: all, prepare_tiles, prepare_exposures +# Only the aggregation targets are localrules: a mid-chain localrule would break +# `group:` fusion of the compute chains (D4). +localrules: all, prepare_all_tiles rule all: input: - expand(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits"), tile=TILES) - -# Phase A.1 — per-tile static DAG (download + find_exposures). keep-going so -# tile failures are independent; build_index.py runs next as a plain script. -rule prepare_tiles: - input: - expand(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe"), tile=TILES) + [final_cat(t) for t in TILES_READY] -# Phase A.2 — per-exposure static DAG (download + star cats), parseable now the -# index exists. Star cats are re-keyed per unit (see prepare.smk). -rule prepare_exposures: +# Invocation 1 — the static per-tile DAG, known from the tile list alone. +# keep-going makes tile failures independent; the ones that lose their exposure +# list are dropped by the index build at invocation 2's parse. +rule prepare_all_tiles: input: - expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)) + [tile_manifest(t, "tile_find_exposures") for t in TILES] # --- report hooks ----------------------------------------------------------- # run_report is NOT a DAG node (a descendant of every job would be poisoned by # any hard failure — the exact case it exists for). It is a standalone script, -# emitted automatically at the end of every invocation, runnable any time. +# emitted automatically at the end of the COMPUTE invocation, runnable any time +# via `sp report`. def _report(status): shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " f"--index {INDEX_DB} --status {status} || true") -onsuccess: - _report("success") +if PHASE == "compute": + + onsuccess: + _report("success") -onerror: - _report("error") + onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp index d1f105548..1ead411a3 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,23 +1,23 @@ #!/usr/bin/env bash -# sp — the committed launcher for the ShapePipe Snakemake workflow. +# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). # -# Removes the shell-state dependencies the review flagged (finding 20): it loads -# the apptainer module (Snakemake resolves `apptainer` via PATH at job runtime) -# and runs the Snakemake that lives on the shared /project venv (must be on a -# shared FS — the executor re-invokes it inside jobs). One entry point, so a -# fresh tmux / a restart after a crash always launches with the right state. +# Two verbs, nothing else: # -# Usage: -# sp prepare # prepare_tiles -> build_index.py -> prepare_exposures -# sp prepare_tiles # phase A.1 only -# sp index # build_index.py only (plain script, threshold-gated) -# sp prepare_exposures # phase A.2 only -# sp run [ARGS...] # the main compute DAG -# sp rerun # forced recompute of protected/finished products -# sp cancel # scancel this workflow's jobs (name sweep) before --unlock -# sp clean-exposures [--apply] # index-driven exposure-store reclaim -# sp report # emit run_report.json now (mid-run ok) -# Extra args after the subcommand pass through to snakemake. +# sp run [ARGS...] bring the products on disk up to date with the tile list. +# Two snakemake invocations over one Snakefile: +# 1. PREPARE snakemake prepare_all_tiles +# 2. COMPUTE snakemake all <- its PARSE builds the index +# ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# sp report [ARGS...] emit run_report.json now (mid-run is fine). +# +# Anything else is passed straight through to snakemake with the same profile and +# state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). +# +# It also loads the apptainer module (snakemake resolves `apptainer` via PATH at +# job runtime) and activates the snakemake venv on the shared /project FS (the +# executor re-invokes it inside jobs, so it cannot live on a node-local path). +# One entry point, so a fresh tmux or a restart after a crash always launches +# with the right state. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ @@ -34,35 +34,40 @@ source "$VENV/bin/activate" # Minimal scalar reader for workflow/config.yaml (key: value, no nesting). cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" + # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO -# THE RUN on /scratch, never on /project (hard 27/27 TiB group quota killed a -# metadata write mid-run live). --directory only moves state: all data paths are +# THE RUN on /scratch, never on /project (a hard 27/27 TiB group quota killed a +# metadata write mid-run, live). --directory only moves state: all data paths are # absolute, and the Snakefile resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" -TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" + +# SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of +# declared tiles allowed to be missing their exposure list (default 0.0). +export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } -build_index() { - python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ - --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" -} -cmd="${1:-}"; shift || true +cmd="${1:-}" case "$cmd" in - prepare) sm prepare_tiles "$@"; build_index; sm prepare_exposures "$@" ;; - prepare_tiles) sm prepare_tiles "$@" ;; - index) build_index ;; - prepare_exposures) sm prepare_exposures "$@" ;; - run) sm "$@" ;; - rerun) sm --forcerun "$@" ;; + run) + shift + SP_PHASE=prepare sm prepare_all_tiles "$@" + SP_PHASE=compute sm all "$@" + ;; + report) + shift + python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + --status manual "$@" + ;; cancel) - run="${1:?usage: sp cancel }" + # Kept only because it is two lines: scancel this workflow's jobs by name + # before an --unlock. Not part of the design surface. + run="${2:?usage: sp cancel }" squeue --me --noheader --format='%i %j' \ | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel - echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; - clean-exposures) python "$SCRIPTS/clean_exposures.py" \ - --run-dir "$RUN_DIR" --index "$INDEX_DB" "$@" ;; - report) python "$SCRIPTS/run_report.py" \ - --run-dir "$RUN_DIR" --index "$INDEX_DB" --status manual "$@" ;; - *) echo "sp: unknown subcommand '${cmd:-}' (see the header of $0)"; exit 2 ;; + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" + ;; + *) + sm "$@" + ;; esac diff --git a/workflow/config.yaml b/workflow/config.yaml index 592ecf989..942199b14 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -13,16 +13,14 @@ tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif -# Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: -# /tiles/// and /exp/// +# Where products land ($SP_RUN root). The sharded per-unit stores live under here: +# /tiles/<2-char prefix>// and /exp/// run_dir: /scratch/cdaley/shapepipe-output/smk-p0 -# The canonical config chain: the committed, wrapper-free configs (#848 D2 config -# sweep) -- RUN_DATETIME=False, fixed INPUT_DIRs, NUMBER_LIST=$SP_UNIT_NUM baked -# in. example/cfis stays untouched for the bash flow; runs/p3-batch1/cfis's real -# divergences (real data paths, external star cats, SAVE_BATCH memory tuning) -# were folded in -- see the config-sweep report for the full list. -config_src: /home/cdaley/shapepipe-smk/workflow/config/cfis +# There is no config_src knob: the config chain is workflow/config/cfis, resolved +# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / +# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules +# are one artefact and must version together, so the dir is fixed by construction. # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). # Star catalogues for masking, pre-generated (network step done in prepare): diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index a033f28d5..dbaf12334 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -41,8 +41,15 @@ NUMBER_LIST = $SP_UNIT_NUM # NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner # derives the other chunks' paths itself from N_SPLIT_MAX and this pattern # (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the -# same fixed $SP_RUN/output. -INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +# same fixed output dir. +# +# RELATIVE ON PURPOSE (do not "fix" this to $SP_RUN/...): merge_sep_cats derives +# chunk k's path with re.sub("1", str(k), input_file, 1) -- it replaces the FIRST +# "1" in the path. An absolute path under the sharded store +# (.../tiles/21/210.282/...) has digits before "Ng1u", so chunk 2 would be looked +# for in a nonexistent directory. Every rule runs shapepipe_run with cwd=$SP_RUN, +# so "./output/..." resolves and "Ng1u" carries the first "1". +INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output # Output directory OUTPUT_DIR = $SP_RUN/output diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 82cdb4cd4..667906223 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,74 +1,90 @@ -"""Exposure chain — per exposure, keyed by exp_id (dedup is structural). +"""Exposure chain — per exposure, keyed by exp base id (dedup is structural). -Gie -> Sp -> Ma -> psf, each in the exposure's own work dir, chained by -ShapePipe's last:/run_sp_exp_* INPUT_DIR resolver. Completeness is the count -floor (completeness.py). + exp_get_images -> exp_split -> exp_mask -> exp_psf -NO temp() on these shared exposure directories: exposures overlap tiles by -construction (~7-10 tiles per exposure), so temp() would delete an exposure the -moment its forests are built and cascade destructive reruns across spatial -neighbours whenever a tile is appended (findings 4/5/6/19). The store is -persistent; space is reclaimed by the explicit index-driven `sp clean-exposures` -verb (scripts/clean_exposures.py), which deletes an exposure's intermediates -only once every consuming tile in the index has its final_cat present. +Each in the exposure's own sharded work dir, chained by manifests; every config +reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a +run log. There is no `prepare_exposures` aggregation target: these chains hang +off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). -NUMBER_LIST is injected only for exp_split (numbering scheme == exposure id); -NEVER for get_images (download) / exp_mask / exp_psf (per-CCD schemes where the -#746 startup validation would turn tolerated per-CCD attrition into a -whole-exposure hard failure). -""" +NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by +construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, +not over one invocation — reclamation here is clean_exposure's job (S5), driven +by the accumulating index. A temp() here would delete an exposure the moment +this invocation's readers finished and cascade destructive reruns across spatial +neighbours the next time a tile is appended. -EXP_OUT = str(RUN_DIR / "exp/{exp}/output") +NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); +never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering +would make the #746 startup validation turn tolerated per-CCD attrition into a +whole-exposure hard failure. That is now a property of the committed configs +(config_exp_Sp.ini has NUMBER_LIST = $SP_UNIT_NUM; Gie/Ma/psfex have none). +""" -# Symlink/download exposure image/weight/flag; reads the fabricated pseudo-Fe -# exp_numbers-000-000.txt the wrapper drops (via last:find_exposures_runner). rule exp_get_images: output: - directory(f"{EXP_OUT}/run_sp_exp_Gie") + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" params: - cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", - "exp", wc.exp, isolate=False, - extra=f"--exp-name {EXP[wc.exp]}") + pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, + exp_name=EXP[wc.exp]), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_get_images", "config_exp_Gie.ini") -# Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; -# persistent (see module docstring on temp()). +# Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the +# tiles' merge_headers reads). rule exp_split: input: - rules.exp_get_images.output + rules.exp_get_images.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_Sp") + manifest = f"{EXP_DIR}/manifests/exp_split.json" params: - cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) + pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_split", "config_exp_Sp.ini") -# Mask per-CCD. Star cats are NOT a per-unit input: the store is per-CCD -# (star_cat--.fits) and the mask config reads it as a DIR symlink -# (see prepare.smk) — a per-unit file input here has no producer and makes the -# DAG unbuildable the moment a new exposure is appended (caught live by the -# append-invariant test). A missing cat fails the mask count-floor loudly. rule exp_mask: input: - rules.exp_split.output, + rules.exp_split.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_Ma") + manifest = f"{EXP_DIR}/manifests/exp_mask.json" params: - cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", - "exp", wc.exp, isolate=False) + pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_mask", "config_exp_Ma.ini") # SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. -# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's +# :warn on psfex_interp_runner. rule exp_psf: input: - rules.exp_mask.output + rules.exp_mask.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_SxSePsfPi") + manifest = f"{EXP_DIR}/manifests/exp_psf.json" params: - cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, - isolate=False) + pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + retries: 2 + benchmark: + f"{EXP_DIR}/manifests/exp_psf.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 240 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_psf", "config_exp_psfex.ini") diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index d8698c826..92998e184 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -1,58 +1,67 @@ -"""Phase A — prepare: tile staging, find_exposures, per-unit star cats. +"""Invocation 1 — PREPARE: one static chain per tile. -Split into two static invocations with build_index.py (a plain script) between: - * prepare_tiles — tile_get_images -> tile_uncompress -> tile_find_exposures - * [build_index.py] — plain script, threshold-gated (not a DAG node) - * prepare_exposures — exp_get_images (exposure.smk) + per-unit star cats + tile_get_images -> tile_uncompress -> tile_find_exposures -Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). -Rules stay group-compatible: no mid-chain localrules, no run:/pipe outputs. +Known from the tile list alone, cheap, wide, idempotent. find_exposures parses +the tile FITS HISTORY header into ``exp_numbers--.txt`` — the +data-derived tile->exposure edge that invocation 2's parse aggregates into the +index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node +tier). + +Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. + +Star catalogues for masking are pre-generated offline (create_star_cat.py on a +networked login node) and consumed as DIRECTORIES: the mask configs read +``$SP_RUN/star_cat_{exp,tiles}`` with per-CCD numbering. Every rule's prologue +materialises the two dir symlinks; there is NO per-unit star-cat DAG node — the +store is pre-run input like the image store, and a missing cat fails the mask +stage's count floor loudly. """ -# get_images reads a one-line tile_numbers.txt the wrapper drops in $SP_RUN. -# NUMBER_LIST is NEVER injected here (download stage; nothing on disk to -# validate against — #746 would hard-fail). +# NUMBER_LIST is never set for get_images (download stage; nothing on disk to +# validate against — #746 would hard-fail the unit). The committed +# config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the +# config, not of an injection step. rule tile_get_images: output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Git")) + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" params: - cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", - "tile", wc.tile, isolate=False) + pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_get_images", "config_tile_Git.ini") rule tile_uncompress: input: - rules.tile_get_images.output + rules.tile_get_images.output.manifest output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Uz")) + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" params: - cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_uncompress", "config_tile_Uz.ini") -# find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the -# data-derived tile->exposure edge build_index.py aggregates (as a plain script). rule tile_find_exposures: input: - rules.tile_uncompress.output + rules.tile_uncompress.output.manifest output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe")) + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" params: - cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 shell: - "{params.cmd} --threads {threads}" - -# Star catalogues for masking are pre-generated offline (create_star_cat.py on a -# networked login node) and consumed as DIRECTORIES: the mask configs read -# INPUT_DIR $SP_RUN/star_cat_{exp,tiles} with per-CCD numbering (exp cats are -# star_cat--.fits, 40/exposure; tile cats star_cat--.fits) -# — the v2.0 mechanism. The wrapper materialises the two dir symlinks in every -# unit work dir (sp_rule.materialise_unit); there is NO per-unit star-cat DAG -# node: the store is pre-run input like the image store, and a missing cat fails -# the mask stage's count-floor loudly. (Earlier per-unit file rules linked names -# that don't exist — one file per exposure vs the store's 40 — and nothing read -# them: the configs only see the dir symlinks.) When star-cat GENERATION moves -# in-workflow (P2), it becomes real per-unit rules producing into the store. + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 04cc4c6c9..a0f048a59 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -1,151 +1,218 @@ -"""Tile post-chain — per tile: gather exposures, then detect/PSF/shape/catalogue. - -A per-tile exposure "forest" (symlink view of exactly this tile's exposures' -products) gives the tile modules a deterministic $SP_EXP to glob. But the forest -is only a runtime convenience, NEVER the DAG edge: every tile rule that reads -through $SP_EXP declares the underlying exposure directory() outputs as input via -an index dict lookup, so the last real reader is the last DAG consumer (findings -5/19). All rules are group-compatible (shell only, no mid-chain localrules). -""" +"""Tile post-chain — per tile: gather exposures, then detect / PSF / shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> tile_vignets -> tile_ngmix x N + -> tile_merge_cats -> tile_make_cat -TILE_OUT = str(RUN_DIR / "tiles/{tile}/output") -FOREST = str(RUN_DIR / "tiles/{tile}/exp_forest") +The DAG edge to the exposures is always the exposures' MANIFESTS, looked up +through the index (TILE_EXP). The per-tile exposure "forest" — a symlink view of +exactly this tile's exposures' products — exists only so the ShapePipe configs +have one deterministic ``$SP_EXP`` to glob; it is NEVER the edge. Its 2-char +shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires +``///output/run_sp_*`` into its glob, so a flat forest +makes every tile gather stage fail "No split_exp_runner output found". +All rules are group-compatible (shell only, no mid-chain localrules). + +Note there is no `tile_mask` rule: the committed config chain is the +"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask +run), and no tile-mask config was committed in the S2 sweep. Adding the masked +variant is a config + one rule, at the config selector the PRD describes. +""" -def _exp_out(wc, prefix): - return [str(exp_dir(e) / "output" / prefix) for e in TILE_EXP.get(wc.tile, [])] +def tile_exp(wc): + return TILE_EXP.get(wc.tile, []) -def tile_exp_split(wc): return _exp_out(wc, "run_sp_exp_Sp") -def tile_exp_mask(wc): return _exp_out(wc, "run_sp_exp_Ma") -def tile_exp_psf(wc): return _exp_out(wc, "run_sp_exp_SxSePsfPi") -def tile_exp_products(wc): - """All of this tile's exposures' product dirs — the forest's DAG inputs.""" - return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) +def exp_manifests(wc, stage): + return [exp_manifest(e, stage) for e in tile_exp(wc)] +def tile_exp_split(wc): return exp_manifests(wc, "exp_split") +def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") +def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) -# Build the per-tile symlink forest (exp_forest//output/run_sp_exp_* -> the -# real exposure products). Declaring the exposure dirs as input makes this the -# tile's wait-on-its-exposures edge; the forest itself is just the SP_EXP view. + +# Build the per-tile symlink forest. Declaring the exposure manifests as input +# makes this wait on its exposures; the forest itself is only the $SP_EXP view. +# Its output stays a directory() (it has no ShapePipe run dir and no manifest — +# it is not a shapepipe_run at all). rule tile_exp_forest: input: - tile_exp_products + tile_exp_all output: - directory(FOREST) + forest = directory(f"{TILE_DIR}/exp_forest") params: - # --forest {output} lives in the shell string: Snakemake formats shell - # ONCE — an {output} placeholder inside params.cmd survives literally - # (same trap as {threads}; all four forest jobs then race one './{output}'). - cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB} " - f"--script-hash {SCRIPT_HASH}") + cmd = lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB}"), + # build_forest.py's own content hash rides here and nowhere else. + script_hash = FOREST_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 shell: - # no --threads: build_forest.py is single-threaded symlinking - "{params.cmd} --forest {output}" + # --forest {output} lives in the shell string: snakemake formats shell + # ONCE, so an {output} placeholder inside params.cmd would survive + # literally and every forest job would race one './{output}'. + "{params.cmd} --forest {output.forest}" -# Merge single-exposure WCS headers into the tile-level sqlite log -# (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng -# consume via NUMBERING_SCHEME -000-000). Reads headers-*.npy through the forest, -# so the exposure split dirs are explicit inputs. +# Merge single-exposure WCS headers into the tile-level sqlite +# (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). +# Reads headers-*.npy through the forest -> the split manifests are the edge. rule tile_merge_headers: input: - forest=rules.tile_exp_forest.output, - split=tile_exp_split, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, output: - directory(f"{TILE_OUT}/run_sp_tile_Mh_exp") + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" params: - cmd=lambda wc: sp_rule("tile_merge_headers", "config_tile_Mh_exp.ini", - "tile", wc.tile, isolate=True, - exp_forest=FOREST.format(tile=wc.tile)) + pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" - -# Mask tiles (star cats: pre-run store, read via the wrapper's dir symlink). -rule tile_mask: - input: - git=rules.tile_get_images.output, - output: - directory(f"{TILE_OUT}/run_sp_tile_Ma") - params: - cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", - "tile", wc.tile, isolate=True) - shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") # SExtractor object detection on the tile. rule tile_detect: input: - mask=rules.tile_mask.output, - mh=rules.tile_merge_headers.output, + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, output: - directory(f"{TILE_OUT}/run_sp_tile_Sx") + manifest = f"{TILE_DIR}/manifests/tile_detect.json" params: - cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_detect", "config_tile_Sx.ini") -# PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF -# models + split through the forest — those dirs are explicit inputs). +# PSFEx interpolation to galaxies + vignet postage stamps: the last stage that +# reads exposure products, and the bulk intra-tile intermediate. +# +# The vignette store is declared as a SECOND, temp(directory()) output alongside +# the manifest. This is the ONE scoped exception to "no directory() outputs" +# (D5): the store is ~tens of GB per tile and must be reclaimed when its last +# intra-tile reader finishes, but it must not become DAG currency — so the +# manifest stays the edge, and the directory rides along purely so native temp() +# fires at the right moment. Its readers (tile_ngmix, tile_make_cat) declare +# BOTH. `--notemp` keeps it for debugging. rule tile_vignets: input: - sx=rules.tile_detect.output, - forest=rules.tile_exp_forest.output, - split=tile_exp_split, - psf=tile_exp_psf, + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + psf = tile_exp_psf, output: - directory(f"{TILE_OUT}/run_sp_tile_PiViVi") + manifest = f"{TILE_DIR}/manifests/tile_vignets.json", + store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), params: - cmd=lambda wc: sp_rule("tile_vignets", "config_tile_PiViVi_canfar_sx.ini", - "tile", wc.tile, isolate=True, - exp_forest=FOREST.format(tile=wc.tile)) + pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 16 + resources: + mem_mb = lambda wc, attempt: 32000 * attempt, + runtime = 240 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_vignets", "config_tile_PiViVi.ini") -# ngmix shape measurement — static N chunks. Each chunk computes its own closed -# object-ID range in-job from the tile's own sexcat (a declared input); chunks -# never rewrite the shared log (--no-log-sync) and read literal INPUT_DIRs. +# ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own +# CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params +# function cannot, because params evaluate before the sexcat exists. Closed, not +# open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour straggler's +# root cause (ngmix treats id_obj_max <= 0 as unbounded). +# +# Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and +# merge_sep_cats — DAG-serialised after all chunks — is the gather. rule tile_ngmix: input: - vignets=rules.tile_vignets.output, - sx=rules.tile_detect.output, + vignets = rules.tile_vignets.output.manifest, + store = rules.tile_vignets.output.store, + sx = rules.tile_detect.output.manifest, output: - directory(f"{TILE_OUT}/run_sp_tile_ngmix_Ng{{chunk}}u") + manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", + # temp(directory()) for the same reason as the vignette store above. + chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), params: - cmd=lambda wc: sp_rule( - "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, - extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), + pre = lambda wc: unit_pre( + "tile_ngmix", "tile", wc.tile, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + pre_run=[f'eval "$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS})"']), + script_hash = SCRIPT_HASH + threads: 4 + retries: 2 + benchmark: + f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 14000 * attempt, + runtime = 720 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + + +def ngmix_manifests(wc): + return [f"{tile_dir(wc.tile)}/manifests/tile_ngmix_{k}.json" + for k in range(1, NGMIX_CHUNKS + 1)] -def ngmix_chunks(wc): - return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" +def ngmix_chunkdirs(wc): + return [f"{tile_dir(wc.tile)}/output/run_sp_tile_ngmix_Ng{k}u" for k in range(1, NGMIX_CHUNKS + 1)] -# Merge the N chunk catalogues (DAG-serialized: the single per-tile log-sync -# point before make_cat). N_SPLIT_MAX == chunk count. +# The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's +# own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). rule tile_merge_cats: input: - ngmix_chunks + manifests = ngmix_manifests, + chunkdirs = ngmix_chunkdirs, output: - directory(f"{TILE_OUT}/run_sp_Ms") + manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" params: - cmd=lambda wc: sp_rule("tile_merge_cats", "config_merge_sep_cats_template.ini", - "tile", wc.tile, isolate=True, - extra=f"--n-split-max {NGMIX_CHUNKS}"), + pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, + env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") -# Build the final catalogue: the run's science product. protected() so a -# code/params-drift rerun cannot silently discard a finished catalogue — the -# recompute must go through `sp rerun` (--forcerun). +# The run's science product. make_cat also reads the vignette store's +# psfex_interp output, so it — not ngmix — is the store's last reader. +# +# No protected(): the full default rerun-triggers govern, and protected() only +# ever forced people through a `--forcerun` detour. rule tile_make_cat: input: - rules.tile_merge_cats.output + ms = rules.tile_merge_cats.output.manifest, + store = rules.tile_vignets.output.store, output: - protected(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits")) + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", params: - cmd=lambda wc: sp_rule("tile_make_cat", "config_make_cat_psfex_nosm.ini", - "tile", wc.tile, isolate=True, - extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), + pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + # Publish the catalogue next to the manifest: a real file, so it is a + # real declared output (and it persists — never temp()). + "{params.pre}\n" + "rc=0\n" + 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' + f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}} || rc=1\n" + "if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' + ' | head -1)" {output.final_cat}\n' + "fi\n" + "exit $rc\n" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index e945820e0..7ccaa308e 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -3,12 +3,12 @@ A plain script (not a run: block) so the tile chain stays group-compatible. Reads the tile's exposures from run_index.sqlite and symlinks each exposure's -``exp//output`` into ``///output`` by exact name (no -glob). The 2-digit ```` shard level is NOT cosmetic: ShapePipe's +``exp///output`` into ``///output`` by exact +name (no glob). The 2-char ```` shard level is NOT cosmetic: ShapePipe's ``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its $SP_EXP glob (``///output/run_sp_*/...``), so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -(The exposure STORE stays flat — only the module-facing view needs the shard.) +(The exposure STORE is sharded the same way, for the filesystem's sake.) The forest is a convenience view; the DAG edge to the exposures is declared in the rule's input (tile.smk), not here. """ @@ -24,11 +24,6 @@ def main() -> None: p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--index", required=True, type=Path) p.add_argument("--forest", required=True, type=Path) - p.add_argument("--script-hash", default=None, - help="ignored at runtime: carries the wrapper-scripts content " - "hash into the rule's params so the params rerun-trigger " - "covers script edits (the code trigger only hashes the " - "rule's own shell string)") args = p.parse_args() con = sqlite3.connect(args.index) @@ -38,7 +33,7 @@ def main() -> None: args.forest.mkdir(parents=True, exist_ok=True) for e in exps: - src = args.run_dir / "exp" / e / "output" + src = args.run_dir / "exp" / e[:2] / e / "output" dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) if dst.is_symlink() or dst.exists(): diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 7ba865d94..31fc6178b 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -5,7 +5,11 @@ at parse time into plain dicts, so extending a run's tile list changes which jobs exist without touching the mtime chain of completed work. -It records, per run: +It ACCUMULATES: tables are created if absent and rows upserted, never dropped +(D1). Successive invocations widen it, so a later ``clean_exposure`` sees every +consuming tile of the whole campaign rather than of the current tile list. + +It records: tiles(tile_id, ra_dir, n_exp, status) exposures(exp_id) -- deduplicated union over tiles @@ -14,20 +18,21 @@ The tile->exposure edges are *data-derived*: they are read from each tile's ``find_exposures`` output (``exp_numbers--.txt``), which ``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. -So this is a **plain script**, not a DAG node: it runs between the two static -invocations — after ``snakemake prepare_tiles`` (Git+Uz+Fe, keep-going) and -before ``snakemake prepare_exposures`` parses the now-existing index. It +So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME +by the Snakefile of the COMPUTE invocation, after the PREPARE invocation has +produced the tiles' find_exposures output. (There is no ``sp index`` verb; the +CLI below stays for hand-inspection.) It iterates the *declared* tile list and checks each tile's Fe output at its deterministic path (no globbing — O(tile-list) existence checks, respecting the no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in -``missing.json`` and the index is built over the rest. The script exits nonzero -only if the missing *fraction* exceeds ``--missing-threshold`` (default 0.0), -so a keep-going download storm that lost a few tiles does not cost the run. -There is NO ``--allow-missing`` flag (that Snakemake flag does not exist). +``missing.json`` and the index is built over the rest. The build fails only if +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the +check — what the PREPARE phase passes, where absent Fe output is normal), so a +keep-going download storm that lost a few tiles does not cost the run. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the -exposure-rule wildcard, matching the ``exp///`` forest layout. +exposure-rule wildcard, matching the sharded ``exp///`` store. """ import argparse @@ -55,8 +60,20 @@ def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: return pairs +def exp_list_path(run_dir: Path, tile_id: str) -> Path: + """This tile's find_exposures output, at its deterministic path. + + Sharded store (D2), fixed run dir (RUN_DATETIME=False) — an existence check, + never a glob (no ``ls`` at scale). + """ + idra, iddec = tile_id.split(".") + return (run_dir / "tiles" / tile_id[:2] / tile_id / "output" / + "run_sp_tile_Fe" / "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + + def build(tile_ids: list[str], run_dir: Path, db_path: Path, - missing_threshold: float = 0.0) -> dict: + missing_threshold: float | None = 0.0) -> dict: """Build the index over ``tile_ids``; return a summary dict. For each tile, check its ``exp_numbers--.txt`` at the tile's @@ -67,14 +84,14 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, """ db_path.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(db_path) + # No DROP: the index accumulates across invocations (D1). con.executescript( """ - DROP TABLE IF EXISTS tiles; - DROP TABLE IF EXISTS exposures; - DROP TABLE IF EXISTS tile_exposures; - CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); - CREATE TABLE exposures(exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); - CREATE TABLE tile_exposures( + CREATE TABLE IF NOT EXISTS tiles( + tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE IF NOT EXISTS exposures( + exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS tile_exposures( tile_id TEXT, exp_id TEXT, PRIMARY KEY (tile_id, exp_id)); """ @@ -84,30 +101,26 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: ra_dir = tile_id.split(".")[0] - idra, iddec = tile_id.split(".") - # Flat forest, deterministic run dir (RUN_DATETIME=False). - exp_file = (run_dir / "tiles" / tile_id / "output" / "run_sp_tile_Fe" / - "find_exposures_runner" / "output" / - f"exp_numbers-{idra}-{iddec}.txt") + exp_file = exp_list_path(run_dir, tile_id) if not exp_file.exists(): missing.append(tile_id) continue exp_pairs = read_exposure_list(exp_file) - con.execute("INSERT INTO tiles VALUES (?,?,?)", + con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", (tile_id, ra_dir, len(exp_pairs))) for exp_id, _name in exp_pairs: con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", (tile_id, exp_id)) all_exposures.update(exp_pairs) - con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + con.executemany("INSERT OR REPLACE INTO exposures VALUES (?,?)", sorted(all_exposures)) con.commit() con.close() (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) frac = len(missing) / len(tile_ids) if tile_ids else 0.0 - if frac > missing_threshold: + if missing_threshold is not None and frac > missing_threshold: raise SystemExit( f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py new file mode 100644 index 000000000..9e3058052 --- /dev/null +++ b/workflow/scripts/ngmix_range.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Print one ngmix chunk's closed object-ID range as bash exports. + +Run inside the chunk's shell, from the tile's own sexcat, because the range is +only knowable at execution time (PRD D4):: + + eval "$(ngmix_range.py --run-dir $SP_RUN --chunk 3 --n-chunks 8)" + # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 + +SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering +[1, N] processes every object exactly once. The first N-1 chunks get equal +shares; the last takes the remainder, with a CLOSED upper bound at n_obj — +never ID_OBJ_MAX = -1, which ngmix reads as unbounded and would double-count. +Each tile is sized from its OWN count (the bash monolith used the campaign +average and left the last chunk open). +""" + +import argparse +from pathlib import Path + + +def id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + hi = lo + base + (rem if k == n_chunks else 0) - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def object_count(run_dir: Path) -> int: + """NAXIS2 of the last HDU of this tile's sexcat (get_number_objects.py).""" + from astropy.io import fits + + cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not cats: + raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") + with fits.open(cats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--chunk", required=True, type=int) + p.add_argument("--n-chunks", required=True, type=int) + a = p.parse_args() + lo, hi = id_ranges(object_count(a.run_dir), a.n_chunks)[a.chunk - 1] + print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py deleted file mode 100644 index 8ee71c693..000000000 --- a/workflow/scripts/sp_rule.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/env python3 -"""Thin per-unit ShapePipe rule wrapper — the single entrypoint every rule calls. - -Under the one-allocation local executor + apptainer software-deployment, -Snakemake runs each rule's shell *inside* the container on the allocation's node. -So this script already runs in-container: it calls ``shapepipe_run`` directly -(never apptainer itself), which is why the workflow never hand-invokes apptainer. - -What it owns — only what is intrinsically ShapePipe's, nothing Snakemake already -does: - -1. **Per-unit work dir + v2.0 isolation furniture.** Each tile/exposure gets its - own ``$SP_RUN`` (``tiles//`` or ``exp//``). Isolation is by work-dir - *content*, the proven v2.0 mechanism — NOT ``-e/--exclusive``: - - a ``cfis`` symlink to the canonical config dir (so ``$SP_CONFIG`` / ``.mask`` - refs resolve); - - ``star_cat_exp`` / ``star_cat_tiles`` symlinks to the shared cat pool; - - for a tile: ``tile_numbers.txt`` = this tile ID (dot format, real data — - what get_images reads); - - for an exposure: a fabricated - ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-000-000.txt`` - = this exposure ID (v2.0 ``init_exp_work_dir`` pseudo-Fe), so the Gie - config's ``last:find_exposures_runner`` resolver finds exactly one unit. - -2. **Per-unit config copy.** Forces ``RUN_DATETIME = False`` (deterministic - ``output/run_sp_/`` the rule declares as its ``directory()`` output), - sets ``SMP_BATCH_SIZE = {threads}`` (fork width == cpus_per_task, so SLURM - packing density and true per-job process count are one number), and — only - when ``isolate`` (tile-scheme stages + exp split; NEVER get_images / exp_mask / - exp_psf, whose per-CCD/download numbering would make #746 turn tolerated - per-CCD attrition into a whole-exposure hard failure) — ``NUMBER_LIST = -``. - -3. **Log sync at START and end.** ``update_runs_log_file.py`` regenerates - ``log_run_sp.txt`` before ``shapepipe_run`` (after the pre-delete, so - resolution never sees a just-deleted dir) and again after. Skipped for ngmix - chunks: the log is already complete from the prior tile stages, and the N - concurrent chunk jobs share the tile work dir — a full log *rewrite* would - race, whereas ShapePipe's own single-line ``O_APPEND`` of each chunk's run is - atomic. ngmix chunk configs additionally carry *literal* INPUT_DIRs - (deterministic under RUN_DATETIME=False), so they need no log read at all. - -4. **Count-floor check** (``completeness.py`` — the ported bash ``complete_check`` - table, the single failure policy: no 3-class taxonomy, no whitelist). After - the run, count products per mandatory runner and exit nonzero below the floor. - ``keep-going`` isolates that unit's cone; per-CCD attrition between floor and - ``expect`` is tolerated. Stages absent from the table fall back to a - zero-output floor. - -Stage-specific verbs (grounded in ``runs/p3-batch1/job_p3_batch1.sh``): - -- ``--ngmix-chunk K --ngmix-nchunks N`` — read this tile's SExtractor object - count from its ``run_sp_tile_Sx`` output, split ``[1, N_obj]`` into N disjoint - closed ID ranges, and generate chunk K's config from - ``config_tile_Ng_template_batch.ini``. -- ``--n-split-max N`` — generate the merge config from - ``config_merge_sep_cats_template.ini`` with ``N_SPLIT_MAX = N``. -- ``--final-cat PATH`` — after ``make_cat``, copy the produced ``final_cat`` FITS - to PATH (the rule's declared, protected science-product output). - -Run by hand to reproduce exactly what a rule does (the escape hatch — it -pre-deletes the run dir, just like Snakemake's delete-before-rerun):: - - python sp_rule.py --stage exp_split --config config_exp_Sp.ini \\ - --unit 2243881 --level exp --run-dir $SP_RUN --config-src $SP_CONFIG \\ - --star-cats $STAR_CATS --threads 8 -""" - -import argparse -import os -import shutil -import subprocess -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from completeness import COMPLETENESS, check_floor # noqa: E402 - - -def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: - """Per-unit work dir in the forest: tiles/ or exp/. - - Flat (no IDra/prefix sharding): the shard existed only to keep the bash - layer's ``ls exp/`` bearable, but Snakemake never globs the forest — the - run index drives every path — so flattening keeps single-wildcard rules. - ``run`` level (aggregation stages) work directly in ``run_dir``. - """ - if level == "tile": - return run_dir / "tiles" / unit - if level == "exp": - base = unit[:-1] if unit[-1].isalpha() else unit - return run_dir / "exp" / base - return run_dir - - -def materialise_unit(work: Path, level: str, unit: str, config_src: Path, - star_cats: Path, exp_name: str | None = None) -> None: - """Create the v2.0 unit-isolation furniture in the work dir.""" - (work / "output").mkdir(parents=True, exist_ok=True) - - cfis = work / "cfis" # $SP_CONFIG -> canonical config dir (.mask refs resolve) - if not cfis.exists(): - cfis.symlink_to(config_src) - - for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): - link = work / name - if not link.exists() and (star_cats / sub).exists(): - link.symlink_to(star_cats / sub) - - if level == "tile": - (work / "tile_numbers.txt").write_text(unit + "\n") # dot format, real data - elif level == "exp": - fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" - fe.mkdir(parents=True, exist_ok=True) - exp_numbers = fe / "exp_numbers-000-000.txt" - # Written UNCONDITIONALLY (cheap, deterministic): an exists-guard once - # pinned a stale pre-fix file with the bare base id. Content is the - # ORIGINAL exposure name (with its 'p'-style suffix, from the index) — - # the on-disk store is .fits.fz; the bare base id matches nothing. - # v2.0 copies the name verbatim from the tile's Fe output; the index - # carries it for us. - exp_numbers.write_text((exp_name or unit) + "\n") - - -def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: - """Force RUN_DATETIME=False, SMP_BATCH_SIZE=threads, and (if isolating) - NUMBER_LIST=-. - - ``NUMBER_LIST`` uses ShapePipe's image-number convention (dots -> dashes), - the ``set_config_number_list`` mechanism that replaced the retired - ``-e/--exclusive`` flag (#746). - """ - lines = text.splitlines() - out, in_file, saw_datetime, saw_numberlist, saw_smp = [], False, False, False, False - number = "-" + unit.replace(".", "-") - for line in lines: - stripped = line.strip() - if stripped.startswith("[") and stripped.endswith("]"): - in_file = stripped.upper() == "[FILE]" - if stripped.upper().startswith("RUN_DATETIME"): - out.append("RUN_DATETIME = False") - saw_datetime = True - continue - if stripped.upper().startswith("SMP_BATCH_SIZE"): - out.append(f"SMP_BATCH_SIZE = {threads}") - saw_smp = True - continue - if isolate and stripped.upper().startswith("NUMBER_LIST"): - out.append(f"NUMBER_LIST = {number}") - saw_numberlist = True - continue - out.append(line) - if in_file and stripped.upper() == "[FILE]" and isolate and not saw_numberlist: - out.append(f"NUMBER_LIST = {number}") - saw_numberlist = True - if not saw_datetime: - # ShapePipe reads RUN_DATETIME from [DEFAULT]; inject after its header. - patched, done = [], False - for line in out: - patched.append(line) - if not done and line.strip().upper() == "[DEFAULT]": - patched.append("RUN_DATETIME = False") - done = True - out = patched if done else ["[DEFAULT]", "RUN_DATETIME = False"] + out - if not saw_smp: - # SMP_BATCH_SIZE lives in [JOB]; inject after its header. - patched, done = [], False - for line in out: - patched.append(line) - if not done and line.strip().upper() == "[JOB]": - patched.append(f"SMP_BATCH_SIZE = {threads}") - done = True - out = patched if done else out + ["[JOB]", f"SMP_BATCH_SIZE = {threads}"] - return "\n".join(out) + "\n" - - -def run_prefix_from_text(text: str, config_label: str) -> str: - """The RUN_NAME (== run_sp_) the module will write under.""" - for line in text.splitlines(): - s = line.strip() - if s.upper().startswith("RUN_NAME"): - return s.split("=", 1)[1].strip() - raise SystemExit(f"no RUN_NAME in {config_label}") - - -# --- ngmix chunking -------------------------------------------------------- - -# Literal, deterministic INPUT_DIRs for the ngmix chunk (RUN_DATETIME=False), so -# concurrent chunks need no log read at all (finding 18). -NGMIX_LITERAL_INPUT = ( - "./output/run_sp_tile_Sx/sextractor_runner/output, " - "./output/run_sp_tile_PiViVi/psfex_interp_runner/output, " - "./output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, " - "./output/run_sp_tile_Mh_exp/merge_headers_runner/output") - - -def ngmix_id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: - """Split ``[1, n_obj]`` into ``n_chunks`` disjoint *closed* ID ranges. - - SExtractor's ``NUMBER`` column (what ngmix reads as ``obj_id``) runs 1..N - contiguous, so covering ``[1, N]`` processes every object exactly once. The - first n_chunks-1 chunks get ``n_obj // n_chunks`` each; the last gets the - remainder — a *closed* upper bound at ``n_obj`` (never ``ID_OBJ_MAX = -1``: - ngmix treats ``id_obj_max <= 0`` as unbounded, which would double-count). - - Deviation from the nibi monolith (deliberate): the monolith sizes every - tile's chunks from the *average* object count across all tiles and leaves the - last chunk open (``-1``); here each tile is an independent job, so we size - from *this* tile's own count and close the last chunk. The merged catalogue - is identical either way (merge_sep_cats recombines all chunks, ngmix fits - each object independently) — this just removes the open-ended overflow. - """ - base, rem = divmod(n_obj, n_chunks) - ranges, lo = [], 1 - for k in range(1, n_chunks + 1): - size = base + (rem if k == n_chunks else 0) - hi = lo + size - 1 - ranges.append((lo, hi)) - lo = hi + 1 - return ranges - - -def tile_object_count(work: Path) -> int: - """Object count for this tile from its run_sp_tile_Sx SExtractor catalogue. - - Matches ``get_number_objects.py``: the ``NAXIS2`` of the catalogue's last - HDU. Per-unit isolation means exactly one ``sexcat*.fits`` lives here. - """ - from astropy.io import fits - - sexcats = sorted((work / "output" / "run_sp_tile_Sx").glob( - "sextractor_runner/output/sexcat*.fits")) - if not sexcats: - raise SystemExit( - f"[sp_rule] FATAL ngmix: no sexcat under {work}/output/run_sp_tile_Sx") - with fits.open(sexcats[0]) as hdul: - return int(hdul[-1].header["NAXIS2"]) - - -def apply_ngmix_subs(text: str, chunk: int, lo: int, hi: int) -> str: - """Fill the ngmix batch template for one chunk (the monolith's sed recipe) - plus literal INPUT_DIRs so the chunk needs no shared-log read.""" - text = text.replace("NgXu", f"Ng{chunk}u").replace("X_interp", "psfex_interp") - out = [] - for line in text.splitlines(): - s = line.strip() - if s.startswith("ID_OBJ_MIN"): - out.append(f"ID_OBJ_MIN = {lo}") - elif s.startswith("ID_OBJ_MAX"): - out.append(f"ID_OBJ_MAX = {hi}") - elif s.startswith("INPUT_DIR") and "sextractor_runner" in s: - out.append(f"INPUT_DIR = {NGMIX_LITERAL_INPUT}") - else: - out.append(line) - return "\n".join(out) - - -def apply_merge_subs(text: str, n_split_max: int) -> str: - """Fill N_SPLIT_MAX in the merge template (the monolith's sed recipe).""" - out = [] - for line in text.splitlines(): - if line.strip().startswith("N_SPLIT_MAX"): - out.append(f"N_SPLIT_MAX = {n_split_max}") - else: - out.append(line) - return "\n".join(out) - - -# --- in-job count floor ---------------------------------------------------- - -def count_floor(run_out: Path, stage: str) -> None: - """Fail loudly if any mandatory runner is below its ``floor`` (completeness.py). - - Stages absent from the table skip this floor (only the zero-output floor - applies to them). - """ - if stage not in COMPLETENESS: - return - ok, details = check_floor(stage, run_out) - for runner, n, floor, expect, warn in details: - tag = "warn" if warn else ("OK" if n >= floor else " <-- BELOW floor") - print(f"[sp_rule] {runner}: {n}/{expect} (floor {floor}) {tag}", - file=sys.stderr) - if not ok: - print(f"[sp_rule] FATAL {stage}: count floor breached in {run_out}", - file=sys.stderr) - sys.exit(1) - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--stage", required=True) - p.add_argument("--config", required=True, help="config filename in --config-src") - p.add_argument("--unit", required=True) - p.add_argument("--level", required=True, choices=["tile", "exp", "run"]) - p.add_argument("--run-dir", required=True, type=Path, help="$SP_RUN root") - p.add_argument("--config-src", required=True, type=Path, - help="dir holding the canonical configs") - p.add_argument("--star-cats", type=Path, default=None, - help="shared star-cat pool (tiles/ + exp/ subdirs)") - p.add_argument("--threads", type=int, default=1, - help="SMP_BATCH_SIZE == cpus_per_task (fork width)") - p.add_argument("--exp-forest", type=Path, default=None, - help="$SP_EXP: per-tile exposure forest (tile post-stages)") - p.add_argument("--exp-name", default=None, - help="original exposure name incl. suffix (e.g. 2605805p); " - "written into the fabricated exp_numbers list so " - "get_images finds .fits.fz in the store") - p.add_argument("--no-isolate", action="store_true", - help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") - p.add_argument("--ngmix-chunk", type=int, default=None, - help="1-indexed ngmix chunk K (with --ngmix-nchunks)") - p.add_argument("--ngmix-nchunks", type=int, default=None, - help="total ngmix chunks N (with --ngmix-chunk)") - p.add_argument("--n-split-max", type=int, default=None, - help="N_SPLIT_MAX for the merge_sep_cats template") - p.add_argument("--final-cat", type=Path, default=None, - help="copy the produced final_cat FITS here (make_cat)") - p.add_argument("--script-hash", default=None, - help="ignored at runtime: carries the wrapper-scripts content " - "hash into the rule's params so the params rerun-trigger " - "covers script edits (the code trigger only hashes the " - "rule's own shell string)") - args = p.parse_args() - - ngmix_mode = args.ngmix_chunk is not None - if ngmix_mode and args.ngmix_nchunks is None: - raise SystemExit("--ngmix-chunk requires --ngmix-nchunks") - - work = unit_work_dir(args.run_dir, args.level, args.unit) - materialise_unit(work, args.level, args.unit, args.config_src, - args.star_cats or args.run_dir, exp_name=args.exp_name) - - # Build the config copy. ngmix chunks and merge fill template placeholders - # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, - # [+ NUMBER_LIST]). - src_text = (args.config_src / args.config).read_text() - isolate = not args.no_isolate - if ngmix_mode: - n_obj = tile_object_count(work) - lo, hi = ngmix_id_ranges(n_obj, args.ngmix_nchunks)[args.ngmix_chunk - 1] - print(f"[sp_rule] ngmix chunk {args.ngmix_chunk}/{args.ngmix_nchunks}: " - f"n_obj={n_obj} -> ID_OBJ [{lo}, {hi}]", file=sys.stderr) - src_text = apply_ngmix_subs(src_text, args.ngmix_chunk, lo, hi) - cfg_name = f"config_tile_Ng{args.ngmix_chunk}u.ini" # distinct per chunk - elif args.n_split_max is not None: - src_text = apply_merge_subs(src_text, args.n_split_max) - cfg_name = args.config - else: - cfg_name = args.config - - cfg_mod_dir = work / "cfis_mod" - cfg_mod = cfg_mod_dir / cfg_name - cfg_text = normalize_config_text(src_text, args.unit, isolate, args.threads) - cfg_mod_dir.mkdir(parents=True, exist_ok=True) - cfg_mod.write_text(cfg_text) - - prefix = run_prefix_from_text(cfg_text, str(cfg_mod)) - run_out = work / "output" / prefix - - # Pre-delete the deterministic run dir (delete-before-rerun for the hand-run - # escape hatch; ShapePipe's FileHandler.mkdir raises on an existing run dir). - if run_out.exists(): - shutil.rmtree(run_out) - - env = dict(os.environ) - env["SP_RUN"] = str(work) - env["SP_CONFIG"] = str(work / "cfis") # canonical dir (.mask refs), not cfis_mod - if args.exp_forest: - env["SP_EXP"] = str(args.exp_forest) - # Thread caps also come via apptainer-args; keep here for the hand-run path. - env.setdefault("OMP_NUM_THREADS", "1") - env.setdefault("OPENBLAS_NUM_THREADS", "1") - env.setdefault("MKL_NUM_THREADS", "1") - env.setdefault("NUMEXPR_NUM_THREADS", "1") - env.setdefault("MALLOC_ARENA_MAX", "2") - env.setdefault("MALLOC_TRIM_THRESHOLD_", "0") - - # Log sync before the run (after pre-delete). Skipped for ngmix chunks: N - # concurrent chunks would race a full rewrite (literal INPUT_DIRs make it moot). - if not ngmix_mode: - subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) - - print(f"[sp_rule] stage={args.stage} unit={args.unit} work={work}", - file=sys.stderr) - rc = subprocess.call(["shapepipe_run", "-c", str(cfg_mod)], - cwd=str(work), env=env) - if rc != 0: - print(f"[sp_rule] WARN shapepipe_run exit {rc} — checking count floor", - file=sys.stderr) - - if not ngmix_mode: - subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) - - # Zero-output floor for stages without a count table; count floor otherwise. - if args.stage in COMPLETENESS: - count_floor(run_out, args.stage) - else: - produced = list(run_out.glob("**/output/*")) if run_out.exists() else [] - if not produced: - print(f"[sp_rule] FATAL {args.stage} {args.unit}: zero output in {run_out}", - file=sys.stderr) - sys.exit(1) - - # make_cat: publish the science product to its declared, protected path. - if args.final_cat is not None: - finals = sorted(run_out.glob("**/output/final_cat*.fits")) - if not finals: - print(f"[sp_rule] FATAL {args.stage} {args.unit}: no final_cat under {run_out}", - file=sys.stderr) - sys.exit(1) - args.final_cat.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(finals[0], args.final_cat) - print(f"[sp_rule] final_cat -> {args.final_cat}", file=sys.stderr) - - print(f"[sp_rule] OK {args.stage} {args.unit}", file=sys.stderr) - - -if __name__ == "__main__": - main() From b149e187618b4c086fc6637b4768a8ec4a2346a8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:55:48 -0400 Subject: [PATCH 015/100] fix(orchestration): harden the rewrite against its adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All eleven findings from the fresh-eyes review, repro-verified: keep-incomplete:true (failure manifests are the post-mortem record — Snakemake otherwise deletes exactly what sp report reads); sp run survives a partial prepare (the missing-threshold is the gate, not set -e); the parse-time index build runs only under SP_PHASE=compute, always evaluates the threshold there, and refuses a silent empty run (WorkflowError on zero ready tiles) while passthrough invocations (--unlock, --dag, targets) never build; profile set-threads/set-resources blocks deleted — they silently replaced the rules' attempt-scaled lambdas and fork widths; manifests write only on content change (a byte-identical rewrite churned the whole cone via mtime); per-tile edge refresh in build_index (stale tile→exposure edges are unrepresentable-shrink no more); blame join blocks on failed/missing only, never warn (else all-tiles-blocked-by-all- exposures at production attrition); ngmix range eval no longer swallows script failure; compute DAG gets a regeneration edge to tile_find_exposures; threshold check precedes any durable index write. Manifest 'unit' is the human ID (basename of SP_RUN); build_forest handles a real-dir dst; stale comments (N_SPLIT_MAX expansion, && check form, sp_rule/protected refs) corrected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- profiles/nibi/config.yaml | 71 ++++------------- workflow/Snakefile | 37 ++++++--- workflow/bin/sp | 26 +++++- .../config/cfis/config_merge_sep_cats.ini | 12 ++- workflow/rules/tile.smk | 19 ++++- workflow/scripts/build_forest.py | 9 ++- workflow/scripts/build_index.py | 79 +++++++++++++------ workflow/scripts/completeness.py | 48 +++++++---- workflow/scripts/run_report.py | 19 ++++- 9 files changed, 200 insertions(+), 120 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 07f68833a..029329aa0 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,67 +37,26 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true +# D3 DEPENDS ON THIS. Snakemake's default is to DELETE the declared outputs of a +# failed job — and this workflow's declared output IS the manifest, the only +# record of *why* a unit failed and the only thing `sp report` reads. Without +# keep-incomplete, every failed unit reads back as "not_run" and the post-mortem +# debris is gone. Safe here because each rule `rm -rf`s its own run dir at start, +# so a rerun never sees stale products. +keep-incomplete: true + # rerun-triggers left at the full Snakemake v9 default # {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. -# No override — the wrapper writes outputs only on change so mtimes move only -# when reality moves; protected() guards finished science products. +# No override — completeness.py writes the manifest only on change, so mtimes +# move only when reality moves. default-resources: mem_mb: 4000 runtime: 120 # minutes cpus_per_task: 1 -# Per-stage resources — measured footprints from the nibi P3 runs (#808). -# threads == cpus_per_task == ShapePipe SMP fork width (sp_rule sets -# SMP_BATCH_SIZE={threads}); mem_mb == threads x measured per-worker footprint. -# -# set-threads is REQUIRED for {threads} to resolve above 1: the local scheduler -# derives cpus_per_task from threads, NOT the reverse (verified snakemake 9.23 — -# set-resources cpus_per_task alone leaves {threads}==1, i.e. SMP_BATCH_SIZE=1, -# serial). Keep the two blocks in lockstep. -set-threads: - exp_split: 4 - exp_mask: 4 - exp_psf: 8 - tile_mask: 16 - tile_detect: 16 - tile_merge_headers: 8 - tile_vignets: 8 - tile_ngmix: 8 - tile_merge_cats: 4 - tile_make_cat: 4 - -set-resources: - exp_split: - cpus_per_task: 4 - mem_mb: 8000 - exp_mask: - cpus_per_task: 4 - mem_mb: 8000 - exp_psf: - cpus_per_task: 8 - mem_mb: 16000 - runtime: 240 - tile_mask: - cpus_per_task: 16 - mem_mb: 16000 - tile_detect: - cpus_per_task: 16 - mem_mb: 16000 - tile_merge_headers: - cpus_per_task: 8 - mem_mb: 8000 - tile_vignets: - cpus_per_task: 8 - mem_mb: 24000 - runtime: 240 - tile_ngmix: - cpus_per_task: 8 - mem_mb: 24000 - runtime: 360 - tile_merge_cats: - cpus_per_task: 4 - mem_mb: 8000 - tile_make_cat: - cpus_per_task: 4 - mem_mb: 8000 +# NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a +# rule's own values (verified snakemake 9.23), which would kill the +# attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +# ngmix thread count. The RULES own threads and resources; this profile only +# sizes the allocation and sets the defaults for rules that state nothing. diff --git a/workflow/Snakefile b/workflow/Snakefile index b707fdc17..7e9300c5f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -28,6 +28,8 @@ import sqlite3 import sys from pathlib import Path +from snakemake.exceptions import WorkflowError + # Resolved relative to THIS file, not the working directory: snakemake runs with # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project # (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). @@ -51,9 +53,12 @@ sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 from completeness import STAGE_DIR # noqa: E402 -# `prepare` suppresses the missing-tile threshold (tiles are still being -# prepared) and the report hooks. Unset == compute (a bare hand-run of `all`). -PHASE = os.environ.get("SP_PHASE", "compute") +# SP_PHASE is set by bin/sp on the two invocations of `sp run` and NOWHERE else. +# It gates the two parse-time side effects — the index build and the report +# hooks — so that a passthrough invocation (`sp --unlock`, `sp --dag`, `sp +# exp_psf ...`) never mutates durable state or dies on a threshold it was not +# asked about. Unset means "read the index, build nothing". +PHASE = os.environ.get("SP_PHASE", "") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] @@ -72,15 +77,18 @@ scattergather: NGMIX_CHUNKS = workflow._scatter["ngmix"] # --- parse-time index build + load (D1) ------------------------------------ -# Invocation 2's parse IS the index build. Tiles whose find_exposures output is -# absent land in index/missing.json and are dropped from the DAG; the build is -# fatal only above SP_MISSING_THRESHOLD (and never in the prepare phase, where -# absent Fe output is the normal state). -if any(build_index.exp_list_path(RUN_DIR, t).exists() for t in TILES): +# The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY +# there — no "some Fe output exists" guard. That guard used to make a +# totally-failed prepare produce an empty index, an empty DAG and a green exit 0; +# with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the +# SP_MISSING_THRESHOLD gate and fails loudly, as it should. +# +# In every other phase (prepare, or unset for a passthrough invocation) the parse +# builds NOTHING and only loads whatever index is already on disk. +if PHASE == "compute": build_index.build( TILES, RUN_DIR, INDEX_DB, - missing_threshold=(None if PHASE == "prepare" - else float(os.environ.get("SP_MISSING_THRESHOLD", "0.0")))) + missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches @@ -97,6 +105,15 @@ if INDEX_DB.exists(): # campaign, so it is intersected with the declared list, not used as it. TILES_READY = [t for t in TILES if t in TILE_EXP] +# A compute invocation with nothing to compute is never a success. Without this, +# an empty intersection yields `rule all` with no inputs, an empty DAG and exit +# 0 — the silent green run the threshold gate exists to prevent. +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} tiles " + f"are ready to compute (index {INDEX_DB}). Run the prepare phase first " + f"(`sp run`), or check {INDEX_DB.parent / 'missing.json'}.") + wildcard_constraints: tile = r"\d{3}\.\d{3}", exp = r"\d{6,7}", diff --git a/workflow/bin/sp b/workflow/bin/sp index 1ead411a3..3424584d1 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -51,8 +51,30 @@ cmd="${1:-}" case "$cmd" in run) shift - SP_PHASE=prepare sm prepare_all_tiles "$@" - SP_PHASE=compute sm all "$@" + # PREPARE failing is NOT fatal to the run: keep-going means a failed tile + # poisons only its own cone, and the tiles that lost their exposure list are + # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which + # the compute parse's index build enforces over the WHOLE tile list — so we + # record the failure and go on rather than letting `set -e` abort here. + prep_rc=0 + SP_PHASE=prepare sm prepare_all_tiles "$@" || prep_rc=$? + if [ "$prep_rc" -ne 0 ]; then + echo "" >&2 + echo "############################################################" >&2 + echo "## WARNING: the PREPARE phase exited $prep_rc." >&2 + echo "## Some tiles may be missing their exposure list and will be" >&2 + echo "## dropped from the compute DAG. Continuing to COMPUTE; the" >&2 + echo "## SP_MISSING_THRESHOLD gate (now $SP_MISSING_THRESHOLD) decides" >&2 + echo "## whether that is tolerable." >&2 + echo "############################################################" >&2 + echo "" >&2 + fi + + comp_rc=0 + SP_PHASE=compute sm all "$@" || comp_rc=$? + if [ "$prep_rc" -ne 0 ] || [ "$comp_rc" -ne 0 ]; then + exit "$([ "$comp_rc" -ne 0 ] && echo "$comp_rc" || echo "$prep_rc")" + fi ;; report) shift diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index dbaf12334..750fe1124 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -84,11 +84,9 @@ NUMBERING_SCHEME = -000-000 WARNING = always # Maximum number of separated catalogues per input. -# BLOCKER: merge_sep_cats_runner.py reads this with plain config.getint (not -# getexpanded) -- $NGMIX_N_CHUNKS would NOT expand here, unlike ID_OBJ_MIN/MAX -# in the ngmix module. Committed as a constant matching workflow/config.yaml's -# ngmix_chunks: 8; if that value changes, this line must be edited by hand -# (or the module gets the same getexpanded treatment ngmix_runner got). -# Chunk count comes from the workflow (single source of truth: the -# scattergather count); env-expanded by the module. +# merge_sep_cats_runner.py reads this with getexpanded (runner .py line ~31), so +# $NGMIX_N_CHUNKS DOES expand here, exactly like ID_OBJ_MIN/MAX in the ngmix +# module. The tile_merge_cats rule exports it from the workflow's scattergather +# chunk count, which is the single source of truth; this value must equal that +# count, so do not replace it with a literal. N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index a0f048a59..e894309b9 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -63,6 +63,12 @@ rule tile_merge_headers: input: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, + # config_tile_Mh_exp.ini reads run_sp_tile_Fe output, which the PREPARE + # phase produced. Declaring the Fe manifest gives the COMPUTE DAG a + # regeneration path for it instead of a silent dependency on a + # previous invocation (prepare.smk is included in every parse, so the + # rule exists here too). Normally a satisfied no-op. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" params: @@ -109,6 +115,9 @@ rule tile_vignets: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, psf = tile_exp_psf, + # config_tile_PiViVi.ini reads run_sp_tile_Fe output — same reason as + # tile_merge_headers above. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_vignets.json", store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), @@ -144,8 +153,14 @@ rule tile_ngmix: pre = lambda wc: unit_pre( "tile_ngmix", "tile", wc.tile, env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, - pre_run=[f'eval "$(python {SCRIPTS}/ngmix_range.py --run-dir ' - f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS})"']), + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing sexcat would fall + # through to shapepipe_run with an unset range and fail as something + # else. Capture, check, then eval — the range script fails as itself. + pre_run=[f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' + f'|| exit 1', + 'eval "$ngmix_range_out"']), script_hash = SCRIPT_HASH threads: 4 retries: 2 diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 7ccaa308e..427e9c647 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -14,6 +14,7 @@ """ import argparse +import shutil import sqlite3 from pathlib import Path @@ -36,8 +37,14 @@ def main() -> None: src = args.run_dir / "exp" / e[:2] / e / "output" dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree — unlink() + # raises IsADirectoryError on it and would kill the job. if dst.is_symlink() or dst.exists(): - dst.unlink() + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() dst.symlink_to(src) print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 31fc6178b..73d08609a 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -5,9 +5,19 @@ at parse time into plain dicts, so extending a run's tile list changes which jobs exist without touching the mtime chain of completed work. -It ACCUMULATES: tables are created if absent and rows upserted, never dropped -(D1). Successive invocations widen it, so a later ``clean_exposure`` sees every -consuming tile of the whole campaign rather than of the current tile list. +It ACCUMULATES ACROSS INVOCATIONS, but is AUTHORITATIVE FOR THE CURRENT TILE +LIST (D1). Precisely: + + * a tile in the current list that has find_exposures output has its ``tiles`` + row replaced and its ``tile_exposures`` edges DELETED AND REBUILT — the Fe + output is the truth, so a tile whose exposure list shrank or changed must + not keep stale edges to exposures it no longer reads; + * a tile NOT in the current list is left completely untouched, which is what + makes the index span the campaign and lets a later ``clean_exposure`` see + every consuming tile; + * ``exposures`` rows only ever accumulate (``INSERT OR IGNORE``). An exposure + no tile references any more is a harmless orphan: nothing reads that table + except by joining through ``tile_exposures``. It records: @@ -19,16 +29,20 @@ ``find_exposures`` output (``exp_numbers--.txt``), which ``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME -by the Snakefile of the COMPUTE invocation, after the PREPARE invocation has -produced the tiles' find_exposures output. (There is no ``sp index`` verb; the -CLI below stays for hand-inspection.) It +by the Snakefile of the COMPUTE invocation ONLY — ``SP_PHASE == "compute"``, +which ``bin/sp`` sets — after the PREPARE invocation has produced the tiles' +find_exposures output. No other parse builds anything: a prepare parse or a +passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is +already on disk. (There is no ``sp index`` verb; the CLI below stays for +hand-inspection.) It iterates the *declared* tile list and checks each tile's Fe output at its deterministic path (no globbing — O(tile-list) existence checks, respecting the no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in ``missing.json`` and the index is built over the rest. The build fails only if -the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the -check — what the PREPARE phase passes, where absent Fe output is normal), so a -keep-going download storm that lost a few tiles does not cost the run. +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the check +entirely), so a keep-going download storm that lost a few tiles does not cost the +run. The threshold is checked BEFORE anything is written, so a failed build +leaves the previous index and ``missing.json`` intact. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the @@ -81,7 +95,24 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, tile whose exposure list is missing is recorded in ``missing.json`` and the index is built over the rest (a bad tile costs that tile, not the run). The build is fatal only if the missing fraction exceeds ``missing_threshold``. + + ORDER MATTERS: the threshold is evaluated FIRST, from the missing set, and + the database + ``missing.json`` are written only if it passes. A build that + aborts must leave no trace — an aborted parse that had already mutated + durable state was the bug this ordering fixes. + + The write is idempotent, which is what makes it acceptable that the compute + parse runs it even under ``-n``: re-running over an unchanged tree produces + an identical database. """ + missing = [t for t in tile_ids if not exp_list_path(run_dir, t).exists()] + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if missing_threshold is not None and frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + db_path.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(db_path) # No DROP: the index accumulates across invocations (D1). @@ -97,34 +128,32 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, """ ) - missing = [] + missing_set = set(missing) all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: - ra_dir = tile_id.split(".")[0] - exp_file = exp_list_path(run_dir, tile_id) - if not exp_file.exists(): - missing.append(tile_id) + if tile_id in missing_set: continue - exp_pairs = read_exposure_list(exp_file) + ra_dir = tile_id.split(".")[0] + exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", (tile_id, ra_dir, len(exp_pairs))) - for exp_id, _name in exp_pairs: - con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", - (tile_id, exp_id)) + # Replace this tile's edge set wholesale. INSERT OR IGNORE alone only + # ever added, so a tile whose exposure list shrank kept edges to + # exposures it no longer reads — and those stale edges would block it in + # the report and pin those exposures against cleanup. + con.execute("DELETE FROM tile_exposures WHERE tile_id = ?", (tile_id,)) + con.executemany("INSERT INTO tile_exposures VALUES (?,?)", + [(tile_id, exp_id) for exp_id, _ in exp_pairs]) all_exposures.update(exp_pairs) - con.executemany("INSERT OR REPLACE INTO exposures VALUES (?,?)", + # Exposures accumulate: OR IGNORE, never REPLACE (the name never changes, + # and orphans left by a shrunken tile are harmless). + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", sorted(all_exposures)) con.commit() con.close() (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) - frac = len(missing) / len(tile_ids) if tile_ids else 0.0 - if missing_threshold is not None and frac > missing_threshold: - raise SystemExit( - f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " - f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " - f"Re-run prepare_tiles for them, or raise --missing-threshold.") return {"n_tiles": len(tile_ids) - len(missing), "n_exposures": len(all_exposures), "n_missing": len(missing)} diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index dd0836c66..1a8820c10 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -9,17 +9,24 @@ sits between ``floor`` and ``expect`` and is tolerated. This file is also the ``check`` CLI — the second half of every rule's shell line -(PRD D2/D3):: +(PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing +onto it, so the check runs — and the manifest is written — even when +``shapepipe_run`` failed:: - shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} \\ - && completeness.py check exp_mask {output} + rc=0 + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? + completeness.py check exp_mask {output} || rc=1 + exit $rc It counts the unit's products under ``$SP_RUN``, writes the manifest at ``{output}``, and exits nonzero iff a mandatory runner is below its floor. The -manifest is ALWAYS written first, failure included: it is the DAG's currency and -the only thing ``run_report.py`` reads, so a failed unit must still leave a -record of *why*. Manifests carry no wall-clock — identical on-disk state must -produce a byte-identical manifest, or the mtime rerun-trigger churns the cone. +manifest is ALWAYS written, failure included: it is the DAG's currency and the +only thing ``run_report.py`` reads, so a failed unit must still leave a record of +*why*. (The profile sets ``keep-incomplete`` so snakemake does not delete that +record on the way out.) Manifests carry no wall-clock, and are rewritten ONLY +when their content changes — identical on-disk state must leave a byte-identical +manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on +every unrelated ``--forcerun``. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -246,11 +253,15 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok -def _unit_from_env(): - """``SP_UNIT_NUM`` carries the rules' dashed, dash-prefixed form - (``-210-282``, ``-2605805``); the manifest records the human ID.""" - raw = os.environ.get("SP_UNIT_NUM", "") - return raw.lstrip("-") or "unknown" +def _unit_from_run_dir(run_dir): + """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). + + NOT ``SP_UNIT_NUM``, which carries ShapePipe's dashed numbering form + (``-210-282``) and would put ``210-282`` in the manifest — a key that joins + to nothing. ``run_report`` keys units on the store directory name, which is + exactly this basename, so the two now agree. + """ + return Path(str(run_dir)).name or "unknown" def main(argv=None) -> int: @@ -261,7 +272,8 @@ def main(argv=None) -> int: c.add_argument("manifest", type=Path) c.add_argument("--run-dir", type=Path, default=None, help="the unit's $SP_RUN (default: the env var)") - c.add_argument("--unit", default=None, help="override $SP_UNIT_NUM") + c.add_argument("--unit", default=None, + help="override the unit ID (default: basename of $SP_RUN)") c.add_argument("--stage-dir", default=None, help="override the run_sp_* subdir (default: the stage table)") args = p.parse_args(argv) @@ -271,11 +283,17 @@ def main(argv=None) -> int: print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", file=sys.stderr) return 2 - unit = args.unit or _unit_from_env() + unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) args.manifest.parent.mkdir(parents=True, exist_ok=True) - args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + # Write ONLY on change. An unconditional write moves the mtime on every run, + # and mtime is a rerun-trigger: a `--forcerun` of one upstream stage would + # then rewrite this manifest byte-identically and drag the whole downstream + # cone along with it. + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + if not args.manifest.exists() or args.manifest.read_text() != text: + args.manifest.write_text(text) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 05ca92478..1a8d128fd 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -206,9 +206,24 @@ def main() -> None: "exposures": unit_rows(exps, EXP_STAGES, exp_m), } - # Blame propagation: an incomplete exposure blocks every tile that reads it. + # Blame propagation: a BLOCKING exposure blocks every tile that reads it. # Without this, a tile stalled at tile_vignets looks like its own failure. - bad_exp = {r["unit"] for r in report["exposures"]} + # + # Blocking means "failed" or "never ran" — NOT "warn". Warn is the expected + # per-CCD attrition (setools rejecting a sparse CCD, psfex_interp short an + # epoch); it is present in essentially every exposure at production scale, so + # counting it here made every exposure block every tile and the table said + # nothing. + # Judged over ALL the exposure's stages, not just the first bad one, so an + # exposure that warns early and fails late still blocks. + def _blocks(unit): + for stage in EXP_STAGES: + m = exp_m.get(unit, {}).get(stage) + if m is None or m.get("status", "failed") == "failed": + return True + return False + + bad_exp = {e for e in exps if _blocks(e)} blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} From 12ecfc2b1c1cc4b9c88e60c38798420d392e5a89 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 23:13:06 -0400 Subject: [PATCH 016/100] feat(orchestration): clean_exposure becomes an in-DAG rule with tombstones (#848 D5, S5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposure-store reclamation moves from a standalone script you remember to run into the DAG itself. One `clean_exposure` job per exposure takes every consuming tile's `tile_vignets` manifest as `input:` — vignets is the last stage that reads exposure products — deletes the store, and leaves a `cleaned.json` tombstone as its output. Writer, then readers, then cleaner: DAG-ordered, race-free, rolling mid-run while ngmix still grinds. The consumer set comes from the accumulating index (new EXP_TILES inverse edge), so it spans the campaign, not one invocation. The tombstone / late-append interaction, which is the whole design problem here: * The job deletes the exposure's `manifests/` along with its `output/`. That is load-bearing. The manifests are the exposure rules' DECLARED outputs; had they survived, a tile appended after the clean would find the exposure chain "up to date" and run its vignets against a store that is no longer on disk. With them gone the DAG sees an unbuilt chain and regenerates it — the accepted cost of a late append, expressed as ordinary Snakemake bookkeeping rather than a special case. The tombstone deliberately does NOT stand in for those manifests: it is input to nothing, so it can never mask their absence. * Tiles already finished are not disturbed: Snakemake demands a missing intermediate only when something downstream of it must run. * `params.consumers` carries the consumer set, so growing it makes the tombstone stale under the default `params` rerun-trigger; the clean job then reruns after the new tile's vignets, against the enlarged set. Cleaned once per consumer set, not once per campaign. * Nothing is lost to the report: each manifest's content is copied verbatim into the tombstone before deletion. Scope guard: an exposure is only eligible when every consuming tile is either in this run's scope or already has its vignets on disk. Without that parse-time test, requesting a tombstone for an exposure shared with a later batch would drag that batch's whole tile chain into this DAG through the clean rule's input — scope expansion by cleanup. Ineligible exposures are deferred, never lost: the invocation that finishes their last consumer picks them up. Gated by `clean:` in workflow/config.yaml (default false for the 4-tile P0 run, where the exposure store is exactly what you want to inspect) and by SP_PHASE=compute, so prepare and passthrough parses schedule no deletions. Flipping it on later reclaims retroactively — the missing tombstones schedule exactly the outstanding clean jobs. Invariants held: no temp() on exposure-level outputs; the loud WorkflowError on zero ready tiles; keep-incomplete untouched. Dry-runs on the 210/211 quad (19 exposures): prepare 13; compute 145 with clean off (identical to b149e187 under the same disk state); compute 164 with clean on = 145 + 19 clean_exposure. Prepare and passthrough parses show no clean rules and leave the index untouched. Deviation from the gap-map wording: one wildcarded rule with an index-driven input function, not literally one generated rule object per exposure. The DAG is identical, and at DR6 scale ~20k rule objects is a parse cost with nothing bought. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- workflow/README.md | 173 ++++++++++++++++++---------- workflow/Snakefile | 54 ++++++++- workflow/config.yaml | 10 ++ workflow/rules/exposure.smk | 40 +++++++ workflow/scripts/clean_exposure.py | 82 +++++++++++++ workflow/scripts/clean_exposures.py | 64 ---------- 6 files changed, 293 insertions(+), 130 deletions(-) create mode 100644 workflow/scripts/clean_exposure.py delete mode 100644 workflow/scripts/clean_exposures.py diff --git a/workflow/README.md b/workflow/README.md index 4d8abd1f1..f42a76300 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -1,56 +1,88 @@ # ShapePipe Snakemake orchestration -Snakemake workflow that orchestrates real-data ShapePipe runs, replacing the +Snakemake workflow that orchestrates real-data ShapePipe runs. It replaces the `curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` -bash layers (and per-site sbatch reimplementations). **Module code is untouched**: -rules call `shapepipe_run -c ` on the existing config chains. Design and -rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +bash layers and the per-site sbatch reimplementations. **Module code is +untouched**: rules call `shapepipe_run -c ` on the existing config +chains. Design and rationale: +[CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) (the living PRD). -## Quick start (nibi) +Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is +unsupported: `sp` sets the state directory, the SLURM profile, and +`SP_PHASE`, which the Snakefile needs to build the tile/exposure index at +parse time. Running snakemake directly skips all of that. -**One-allocation mode.** `sbatch` one node, then run Snakemake with the *local* -scheduler inside it: it bin-packs the DAG into the allocation (no per-job -sbatch). Each job's shell runs `shapepipe_run` **inside the container** (the -profile's apptainer software-deployment — you never type `apptainer`). +## Quick start (nibi) ```bash -# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which is -# node-local; the executor re-invokes this python inside jobs). +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which +# is node-local; the SLURM executor re-invokes this python inside every job). uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate -uv pip install 'snakemake>=9,<10' +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, config_src, container, star_cats. +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats. -# The committed launcher loads apptainer/1.4.5 + the /project venv, so a fresh -# shell always has the right state. Inside your allocation: -workflow/bin/sp prepare # prepare_tiles -> build_index.py -> prepare_exposures -workflow/bin/sp run # the main compute DAG +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a +# fresh shell always has the right state. +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp cancel # scancel this workflow's jobs ``` -`sp` subcommands: `prepare` (chains all three prepare steps), `prepare_tiles`, -`index`, `prepare_exposures`, `run`, `rerun ` (forced recompute of -protected products), `cancel ` (scancel sweep before `--unlock`), -`clean-exposures [--apply]`, `report`. +Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, +queried 2026-07-30): `snakemake==9.23.1`, `snakemake-executor-plugin-slurm==2.7.1`. +Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The +v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became +plugins, and full `rerun-triggers` became the default. + +Anything other than `run`, `report`, `cancel` passes straight through to +snakemake with the workflow's profile and state dir — the escape hatch for +`sp --unlock`, `sp --dag`, `sp exp_psf ...`. + +## Execution: two static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, +so it cannot live in the same static DAG that produces it. `sp run` is +therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** — `snakemake prepare_all_tiles`: per-tile static DAG + (`Git_vos → Uz → Fe`), `keep-going` so tile failures are independent. A + nonzero exit here is not fatal to the run — tiles that lost their + exposure list are dropped at the compute parse — but it is a warning: + `SP_MISSING_THRESHOLD` (default 0.0) is the real gate. +2. **COMPUTE** — `snakemake all`: this invocation's *parse* builds the + tile↔exposure index (`build_index.py`, imported at parse time, not a DAG + node) and runs the full tile/exposure compute chain. The index + accumulates across invocations, so appending tiles later changes which + jobs exist without invalidating completed work. + +`sp run` chains both so the UX is one command; both exit codes are checked +and the run fails if either phase failed. -## Execution is three static invocations +## Execution mode: one SLURM job per rule -The exposure job set is data-derived from the tiles' `find_exposures` output, so -it cannot be scheduled in the same static DAG that produces it. Hence: +The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule +instance becomes its own SLURM job carrying that rule's own attempt-scaled +resources (`cpus_per_task = threads`, `mem_mb`, `runtime`) and runs inside +`apptainer exec` via the profile's software-deployment method — the workflow +never calls apptainer directly. Snakemake feeds the queue as jobs finish, so +the full campaign's job count never needs to queue at once, and multi-node +scaling is inherent. `jobs:` in the profile caps concurrent submissions at a +fraction of the cluster's per-user submit limit (queried, not invented — see +the comment in the profile file for the query and date). -1. `snakemake prepare_tiles` — per-tile static DAG (Git_vos → Uz → Fe), - `keep-going` so tile failures are independent. -2. `build_index.py` — a **plain script**, not a DAG node. It iterates the - declared tile list, checks each tile's Fe output at its deterministic path (no - glob), writes `run_index.sqlite` + `missing.json`, and exits nonzero only if - the missing *fraction* exceeds a threshold. (There is no `--allow-missing` - Snakemake flag; that mechanism does not exist.) -3. `snakemake prepare_exposures` — per-exposure static DAG (Gie_vos + per-unit - star cats), parseable now the index exists. -4. `snakemake` (main DAG) — the tile/exposure compute chain. +The profile intentionally sets no `set-resources` / `set-threads` overrides: +those replace a rule's own values wholesale, which would kill the +attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +ngmix thread count. Rules own their own resources; the profile only supplies +defaults for rules that state nothing. -`sp prepare` chains 1–3 so the UX stays one command. +`group:` labels that fuse short rules (uncompress, merges) into their chunky +neighbours (queue-latency amortization, per the PRD) are **not yet wired**: +they require labels in `workflow/rules/*.smk`, out of scope for this +profile-only pass. ## Layout @@ -58,7 +90,7 @@ it cannot be scheduled in the same static DAG that produces it. Hence: workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks config.yaml the run: tile list, paths, container, chunk count - bin/sp committed launcher (module load + /project venv + subcommands) + bin/sp committed launcher (module load + /project venv + run/report/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats exposure.smk per-exposure: get_images, split, mask, psf (no temp()) @@ -69,37 +101,50 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) - clean_exposures.py index-driven exposure-store reclaim (replaces temp()) -profiles/nibi/config.yaml local scheduler; apptainer SDM; resources; keep-going + clean_exposure.py ONE exposure's store + manifests -> tombstone (the clean_exposure rule) +profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` ## How it works -- **Completeness = the count floor.** Each stage declares its deterministic - `output/run_sp_/` directory as `directory()` output (the per-unit - config copy forces `RUN_DATETIME = False`, so the path is known at DAG time). - After the run, `sp_rule.py` counts products per mandatory runner against - `completeness.py`'s floor and exits nonzero below it; per-CCD attrition between - floor and `expect` is tolerated (the directory simply lacks that product). - This *is* the failure policy — no 3-class taxonomy, no error-signature - whitelist. `--keep-going` isolates a failure to its own cone. -- **Per-unit isolation is by work-dir content, not `-e/--exclusive`.** Every - tile/exposure runs its own `shapepipe_run` in `tiles//` or `exp//`, - with a `cfis` config symlink, `star_cat_{exp,tiles}` symlinks, a - `tile_numbers.txt` (tiles) or fabricated `exp_numbers-000-000.txt` pseudo-Fe - (exposures), and a config copy setting `RUN_DATETIME=False` and - `SMP_BATCH_SIZE={threads}`. `NUMBER_LIST=-` is injected **only** for - stages whose numbering scheme is the unit ID (tile-scheme stages + exp split), - never for get_images / exp_mask / exp_psf. -- **No `temp()` on shared exposure dirs.** Exposures overlap tiles, so temp() - would cascade destructive reruns when a tile is appended. The store is - persistent; `sp clean-exposures` reclaims space only for exposures all of - whose consuming tiles have a final_cat. -- **The index is parse-time data, never a rule input.** Appending tiles changes - which jobs exist without invalidating completed work. Star cats are re-keyed - per unit for the same reason. `final_cat` is `protected()`. +- **The atom is one rule == one `shapepipe_run` on one unit.** Its single + declared output is that unit's manifest + (`/manifests/.json`), not its product files — a missing + CCD is often legitimate, and at DR6 scale per-CCD declaration means + millions of paths. +- **Manifests are the DAG's currency.** `completeness.py check` writes the + manifest, and it is the only record of *why* a unit failed. That is why + the profile sets `keep-incomplete: true` — Snakemake's default deletes a + failed job's declared outputs, which would erase the manifest the report + needs. +- **Completeness is a count floor, not a taxonomy.** After a run, + `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition + between floor and `expect` is tolerated. No 3-class taxonomy, no + error-signature whitelist. `--keep-going` isolates a failure to its own + DAG cone. +- **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` + in `tiles/<2-char prefix>//` or `exp///`, with a `cfis` + config symlink, `star_cat_{exp,tiles}` symlinks, and a config copy setting + `RUN_DATETIME=False`. Configs are committed under `workflow/config/cfis/` + and version with the rules that set the env vars they interpolate — there + is no `config_src` knob, by construction. +- **The index is parse-time data, never a rule input.** Appending tiles + changes which jobs exist without invalidating completed work. Star cats + are re-keyed per unit for the same reason. `final_cat` is `protected()`. +- **Exposure products are not `temp()`.** Exposures overlap tiles, so + `temp()` would cascade destructive reruns when a tile is appended later. + Reclamation is the in-DAG `clean_exposure` rule instead: one job per + exposure, taking every consuming tile's `tile_vignets` manifest as input + (the campaign-wide consumer set comes from the accumulating index), which + deletes the store *and* the exposure's manifests and leaves a `cleaned.json` + tombstone. Deleting the manifests is what makes a late append correct: the + appended tile finds an unbuilt chain and regenerates it. The `clean:` flag + in `config.yaml` gates it; flipping it on later reclaims retroactively, + since the missing tombstones schedule exactly the outstanding clean jobs. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs - per-CCD attrition). It is a standalone script — a DAG report node would be - poisoned by the very failures it must enumerate — emitted automatically by the - `onsuccess`/`onerror` hooks and runnable any time via `sp report`. + per-CCD attrition). It runs standalone — a DAG report node would itself be + poisoned by the failures it must enumerate — and fires automatically from + the COMPUTE invocation's `onsuccess`/`onerror` hooks, or on demand via + `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile index 7e9300c5f..50dc75779 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -93,12 +93,15 @@ if PHASE == "compute": # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. -EXP, TILE_EXP = {}, {} +# EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure +# is keyed on (D5). +EXP, TILE_EXP, EXP_TILES = {}, {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(_tile, []).append(_exp) + EXP_TILES.setdefault(_exp, []).append(_tile) _con.close() # Tiles this run can actually compute: declared AND indexed. The index spans the @@ -162,6 +165,52 @@ def unit_num(unit): # gets its own hash, on the forest rule only. SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] +CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] + +# --- exposure reclamation (D5, S5) ----------------------------------------- +# `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no +# jobs at all (nothing requests a tombstone); flipping it on later reclaims +# RETROACTIVELY, because the exposures already cleaned are exactly the ones with +# a tombstone, so the missing tombstones schedule exactly the clean jobs. +# Only ever active under SP_PHASE=compute: the prepare and passthrough parses +# read no index of their own and must schedule no deletions. +CLEAN = bool(config.get("clean", False)) and PHASE == "compute" + + +def tombstone(exp): + """The clean_exposure output. Lives BESIDE manifests/, not inside it: the + clean job deletes manifests/ wholesale, and `sp report` scans it.""" + return f"{exp_dir(exp)}/cleaned.json" + + +def clean_consumers(exp): + """Every tile in the campaign that reads this exposure — the set whose + vignets must all exist before the store may go.""" + return sorted(EXP_TILES.get(exp, [])) + + +def clean_targets(): + """Which exposures this invocation may clean. + + An exposure is eligible only when every consuming tile is either in this + run's scope or has already produced its vignets on disk. Without that test, + requesting a tombstone for an exposure shared with a LATER batch would drag + that batch's whole tile chain into this DAG through the clean rule's input — + scope expansion by cleanup, which is not a trade anyone asked for. Ineligible + exposures are simply skipped; the invocation that finishes their last + consumer picks them up. Deferral, never loss. + """ + if not CLEAN: + return [] + ready = set(TILES_READY) + out = [] + for exp, tiles in EXP_TILES.items(): + if not tiles: + continue + if all(t in ready or Path(tile_manifest(t, "tile_vignets")).exists() + for t in tiles): + out.append(tombstone(exp)) + return sorted(out) # --- the shell every rule runs (D2) ---------------------------------------- @@ -264,7 +313,8 @@ localrules: all, prepare_all_tiles rule all: input: - [final_cat(t) for t in TILES_READY] + [final_cat(t) for t in TILES_READY], + clean_targets(), # Invocation 1 — the static per-tile DAG, known from the tile list alone. # keep-going makes tile failures independent; the ones that lose their exposure diff --git a/workflow/config.yaml b/workflow/config.yaml index 942199b14..ab06a459e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -32,6 +32,16 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # 60-day purge for anything that must survive. index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_exposure` job per exposure. It fires once every campaign tile that reads +# that exposure has its vignets, deletes the exposure's store AND its manifests, +# and leaves `cleaned.json` (which absorbs the manifests, so the report keeps +# them). Off here because P0 is a 4-tile debugging run where the exposure store +# is exactly what you want to inspect; on for any run big enough to care about +# disk. Flipping it on later reclaims retroactively — the missing tombstones +# schedule exactly the outstanding clean jobs. +clean: false + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 667906223..b752bbee7 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -88,3 +88,43 @@ rule exp_psf: runtime = 240 shell: sp_shell("exp_psf", "config_exp_psfex.ini") + + +# --- reclamation (D5) ------------------------------------------------------- +# The one exception to "no reclamation in this file": clean_exposure OWNS +# exposure-level deletion, and it is a real job, not temp() bookkeeping, because +# an exposure's consumer set closes over the CAMPAIGN. The index supplies that +# set (EXP_TILES, accumulated across invocations); the input is every consuming +# tile's tile_vignets manifest — vignets is the last stage that reads exposure +# products, everything after it reads tile-level files. +# +# Three properties make the late append behave (see clean_exposure.py): +# * the job deletes the exposure's manifests too, so a tile appended after the +# clean sees an unbuilt chain and regenerates it instead of running against +# an empty store. The tombstone deliberately does NOT stand in for those +# manifests — it is not an input to anything but itself. +# * a finished tile is not disturbed: Snakemake demands a missing intermediate +# only when something downstream of it must run. +# * params.consumers carries the consumer set, so growing it makes the +# tombstone stale under the default `params` rerun-trigger; the clean job +# reruns after the new tile's vignets, against the enlarged set. +# +# Not a localrule: at DR6 scale these are ~20k rmtrees and the local-cores cap +# would serialise them behind the aggregation targets. +rule clean_exposure: + input: + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp)] + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)), + script_hash = CLEAN_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py new file mode 100644 index 000000000..2d3d40cc5 --- /dev/null +++ b/workflow/scripts/clean_exposure.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Reclaim ONE exposure's store and leave a tombstone (PRD #848 D5, S5). + +Run as the shell of the in-DAG ``clean_exposure`` rule, never by hand: the rule's +``input:`` is every consuming tile's ``tile_vignets`` manifest, so by the time +this executes, every campaign tile that reads this exposure has already extracted +its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. + +What it deletes: the exposure's whole ``output/`` tree (the bulk store — +run_sp_exp_Gie/Sp/Ma/SxSePsfPi) AND its ``manifests/``. Deleting the manifests is +deliberate and load-bearing, not tidiness: + + * the manifests are the exposure rules' DECLARED outputs. If they survived, a + tile appended later would find the exposure chain "up to date" and run + tile_vignets against products that are no longer on disk. With them gone the + DAG sees the chain as unbuilt and regenerates it — the accepted cost of a + late append (D5), expressed as ordinary Snakemake bookkeeping rather than as + a special case. + * Snakemake only demands a missing intermediate when something downstream of it + needs to run, so tiles already finished are NOT rerun by their exposures' + manifests vanishing. + +Nothing is lost to the report: each manifest's content is copied verbatim into +the tombstone under ``manifests`` before deletion, so `sp report` can still say +what this exposure produced and why any runner was short. + +The tombstone records the consumer set it was cleaned against. The rule carries +that same set as a ``params`` value, so when the index grows a new consumer the +tombstone goes stale under the default ``params`` rerun-trigger and the clean job +is rescheduled after the new tile's vignets — the exposure is cleaned once per +consumer set, not once per campaign. +""" + +import argparse +import json +import shutil +import time +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path) + p.add_argument("--exp", required=True) + p.add_argument("--tombstone", required=True, type=Path) + p.add_argument("--consumers", default="", + help="comma-separated tile ids this exposure was cleaned against") + args = p.parse_args() + + consumers = [t for t in args.consumers.split(",") if t] + + # Absorb the manifests before they go: the tombstone becomes the exposure's + # surviving record. + manifests = {} + mdir = args.exp_dir / "manifests" + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + manifests[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + manifests[f.stem] = {"unreadable": str(exc)} + + removed = [] + for target in (args.exp_dir / "output", mdir): + if target.exists(): + shutil.rmtree(target) + removed.append(str(target)) + + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + args.tombstone.write_text(json.dumps({ + "exp": args.exp, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "consumers": consumers, + "removed": removed, + "manifests": manifests, + }, indent=2) + "\n") + print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " + f"{len(consumers)} consuming tile(s)") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposures.py b/workflow/scripts/clean_exposures.py deleted file mode 100644 index 4f46bf19b..000000000 --- a/workflow/scripts/clean_exposures.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -"""Index-driven exposure-store cleanup (`sp clean-exposures`). - -temp() is deliberately NOT used on the shared exposure directories (findings -4/5/6/19): they overlap tiles by construction, so temp() deletion would cascade -destructive reruns across spatial neighbours whenever a tile is appended. Instead -the store is persistent and space is reclaimed explicitly here. - -For each exposure in run_index.sqlite, delete its bulky intermediates -(run_sp_exp_Sp / run_sp_exp_Ma / run_sp_exp_SxSePsfPi) ONLY when every tile that -consumes it (tile_exposures) has its final_cat-.fits present across the whole -output root. An exposure still feeding an unfinished tile is left untouched. - -Caveat: if a later batch appends a tile that shares a deleted exposure, -Snakemake will regenerate that exposure — cleanup trades peak storage for -recompute. Run it between campaign phases, not mid-DAG. --dry-run by default. -""" - -import argparse -import shutil -import sqlite3 -import sys -from pathlib import Path - -PREFIXES = ("run_sp_exp_Sp", "run_sp_exp_Ma", "run_sp_exp_SxSePsfPi") - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--run-dir", required=True, type=Path) - p.add_argument("--index", required=True, type=Path) - p.add_argument("--apply", action="store_true", - help="actually delete (default: dry-run, list only)") - args = p.parse_args() - - con = sqlite3.connect(args.index) - edges = con.execute("SELECT exp_id, tile_id FROM tile_exposures").fetchall() - con.close() - - consumers: dict[str, list[str]] = {} - for exp_id, tile_id in edges: - consumers.setdefault(exp_id, []).append(tile_id) - - def tile_done(tile): - return (args.run_dir / "tiles" / tile / f"final_cat-{tile}.fits").exists() - - freed, held = 0, 0 - for exp_id, tiles in consumers.items(): - if not all(tile_done(t) for t in tiles): - held += 1 - continue - for prefix in PREFIXES: - d = args.run_dir / "exp" / exp_id / "output" / prefix - if d.is_dir(): - freed += 1 - print(f"{'DELETE' if args.apply else 'would delete'} {d}") - if args.apply: - shutil.rmtree(d) - print(f"[clean_exposures] {freed} dirs {'freed' if args.apply else 'reclaimable'}, " - f"{held} exposures held (consuming tile unfinished)", file=sys.stderr) - - -if __name__ == "__main__": - main() From e677f679f41b398d7630ce1aeee9494abbef9726 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 23:13:49 -0400 Subject: [PATCH 017/100] feat(orchestration): nibi profile goes slurm-executor (#848 D4, S6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The profile flips from the P0 one-allocation/local-scheduler mode to the PRD's executor: slurm. jobs: 800 = 80% of the queried MaxSubmitPU=1000 on def-mjhudson (sacctmgr, 2026-07-30). default-resources carry mem_mb, runtime, slurm_account: def-mjhudson; keep-incomplete stays (a failed job's manifest is its only post-mortem record). No set-resources/set-threads — rules own their attempt-scaled lambdas. Env pins recorded in the README: snakemake>=9,<10, snakemake-executor-plugin-slurm>=2.7,<3 (installed: 9.23.1 / 2.7.1). group: fusion deferred — it needs rule edits, tracked in the README. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 029329aa0..7819a0fdc 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -1,20 +1,29 @@ # Snakemake profile for the Nibi cluster (Digital Research Alliance). # -# ONE-ALLOCATION mode: sbatch one node up front, then run snakemake with the -# LOCAL scheduler inside it (no per-job sbatch; the local scheduler bin-packs the -# DAG into the allocation against `cores` + `resources: mem_mb`). This is the -# proven nibi path (job_p3_batch1) and the single honest first-pass mode; -# multi-node scaling = multiple tile-disjoint invocations, deferred. +# SLURM-EXECUTOR mode (PRD #848 D-profile): one SLURM job per rule instance, +# each carrying its rule's own attempt-scaled resources (cpus_per_task, +# mem_mb, runtime). Snakemake feeds the queue as jobs finish, so DR6's ~170k +# total jobs never queue at once, and multi-node scaling is inherent — this +# supersedes the earlier one-allocation/local-scheduler mode. # # Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). # software-deployment-method wraps every job's shell in `apptainer exec` — the # user never types apptainer; the container is set in the Snakefile (local .sif). -# Local scheduler (no `executor:` -> built-in local). Size to the allocation. -cores: 192 # node cores available to the DAG -local-cores: 4 # cap localrule/aggregation concurrency -resources: - mem_mb: 720000 # node RAM cap the local scheduler bin-packs under +executor: slurm + +# Per-user submit cap, queried 2026-07-30 on nibi: +# sacctmgr show assoc user=cdaley format=Account,MaxSubmitPU -P +# -> def-mjhudson_cpu 1000, def-mjhudson_gpu 1000 (MaxSubmitPU; no site-wide +# MaxSubmitJobs in `scontrol show config`, so the association limit governs) +# Set to ~80% of that (1000) so this workflow never starves other submissions +# under the same account. Re-query if the association limits change. +jobs: 800 + +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: def-mjhudson software-deployment-method: [apptainer] # Explicit container environment (finding 2/3): the apptainer SDM otherwise drops @@ -50,13 +59,13 @@ keep-incomplete: true # No override — completeness.py writes the manifest only on change, so mtimes # move only when reality moves. -default-resources: - mem_mb: 4000 - runtime: 120 # minutes - cpus_per_task: 1 - # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the # attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned # ngmix thread count. The RULES own threads and resources; this profile only -# sizes the allocation and sets the defaults for rules that state nothing. +# sets defaults for rules that state nothing (default-resources above). +# +# `group:` fusion of the short rules (uncompress, merges) into their chunky +# neighbours (PRD D-profile) is NOT set here — it requires labels in the rule +# files themselves, which is out of scope for this profile-only pass. Deferred +# to whichever slice next touches workflow/rules/*.smk. From 96c78cdb1fbb2f48c46b96f1f9c351738718ec60 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:08:07 -0400 Subject: [PATCH 018/100] fix(orchestration): stop the clean_exposure cascade; report reads tombstones (#848 D5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial review of the in-DAG clean_exposure design found eight problems. This fixes all eight. The central one is a rerun cascade. THE CASCADE, AND WHAT ACTUALLY CUT IT clean_exposure deletes an exposure's manifests on purpose: that is what makes a tile appended later rebuild the chain instead of running against an empty store. But those manifests are the tile side's inputs, and an exposure is read by ~7-10 tiles. So one rebuilt exposure made every other tile that reads it rerun, and that rerun rebuilt ITS exposures, spreading across the whole exposure-overlap connected component. On fixture t4 (four tiles chained by shared exposures, all complete, all exposures cleaned, one tile's catalogue deleted), asking for the one damaged tile scheduled all four tiles' chains. ancient() alone does NOT fix it, and the measurement says why. Snakemake suppresses the pending-producer propagation only when every shared file is ancient AND EXISTS (dag.py); reclamation has deleted these. Job counts with ancient() alone were identical to before. Three changes cut it, and t4 proves each: * a FINISHED tile (final_cat on disk) drops the exposure manifests that are gone from its input list, and holds the rest through ancient(). It has already extracted everything it will ever read from them, so the propagation has nowhere to go. Only missing manifests are dropped, so a campaign that has cleaned nothing declares exactly the edges it always did. * clean_exposure declares only IN-SCOPE consumers' vignets manifests. An out-of-scope consumer is checked for existence at parse time instead; declaring it pulled that finished tile's whole chain into the DAG. * the profile drops the `input` rerun-trigger. With it on, the conditional edge reads as "set of input files has changed" and reruns the finished tiles anyway — against a store that is gone. Nothing is lost: a changed exposure list still arrives through the find_exposures manifest's mtime, and the ngmix chunk count rides in params. t4, one damaged tile of four: 82 jobs before, 70 with the edge cut but the trigger on, 28 with all three — exactly the damaged tile's chain, its own two exposures, and the clean jobs. Same 28 whether the tile list is the one tile or all four. Appending a tile that shares a cleaned exposure still rebuilds that exposure and reschedules its clean, and touches no other tile (25 jobs). A complete campaign schedules nothing, clean on or off. The cost to know: --forcerun on a tile whose final_cat exists will not rebuild its reclaimed exposures. Delete the final_cat first. THE OTHER SEVEN * sp report read no tombstones, so every cleaned exposure came back "not_run" and blocked the complete tiles whose completion had authorised the deletion. It now reads the absorbed manifests out of cleaned.json, reports the unit as `cleaned` with its warn counts intact, and never blames it for a tile. Fixture t3: 3 exposures "not run" + 2 tiles blocked, before; 3 cleaned + 0 blocked, after. * clean_ignore_tiles: in config.yaml. A permanently-failed tile pinned its ~80 exposures forever; listed there, it leaves the consumer sets. Retrying it later rebuilds those chains from scratch, and the config says so. * the exp_psf benchmark tsv moved out of manifests/, which reclamation deletes. It is the measured-memory feed for mem_mb sizing (D4). * clean_exposure is a localrule. At DR6 scale it was ~20k sbatch submissions to run rmtree. * `--config clean=false` was truthy, so switching reclamation off switched it on. Config booleans now parse strings, and reject what they cannot read. * the tombstone is written, complete and atomically, BEFORE anything is deleted. The crash window now costs disk, never the record. * the false comments in clean_exposure.py and config.yaml are true again. Dry-runs only; no jobs were run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 22 +++++++-- workflow/README.md | 22 +++++++++ workflow/Snakefile | 71 +++++++++++++++++++++++++----- workflow/config.yaml | 25 ++++++++--- workflow/rules/exposure.smk | 26 +++++++++-- workflow/rules/tile.smk | 63 +++++++++++++++++++++++++- workflow/scripts/clean_exposure.py | 36 +++++++++++---- workflow/scripts/run_report.py | 69 ++++++++++++++++++++++++++--- 8 files changed, 294 insertions(+), 40 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 7819a0fdc..b79985e62 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -54,10 +54,24 @@ printshellcmds: true # so a rerun never sees stale products. keep-incomplete: true -# rerun-triggers left at the full Snakemake v9 default -# {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. -# No override — completeness.py writes the manifest only on change, so mtimes -# move only when reality moves. +# rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still +# propagate — completeness.py writes the manifest only on change, so mtimes move +# only when reality moves. +# +# `input` is dropped because reclamation needs the tile->exposure edge to be +# CONDITIONAL: a tile whose final_cat is on disk declares no exposure inputs, so +# a neighbour rebuilding a shared exposure cannot drag it along (see tile.smk). +# With the `input` trigger on, that same conditional reads as "set of input files +# has changed" and reruns every finished tile — against an exposure store that +# reclamation has already deleted. Measured on fixture t4: 70 jobs with the +# trigger, 28 without, for one damaged tile in a four-tile chain. +# +# Nothing this workflow relied on is lost. The two genuinely data-derived input +# sets are covered another way: a changed exposure list arrives through the +# (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides +# in params. And the Snakefile refuses to schedule reclamation while the trigger +# is enabled, so a bare snakemake invocation cannot quietly undo this. +rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the diff --git a/workflow/README.md b/workflow/README.md index f42a76300..c64047e7f 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -142,6 +142,28 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee appended tile finds an unbuilt chain and regenerates it. The `clean:` flag in `config.yaml` gates it; flipping it on later reclaims retroactively, since the missing tombstones schedule exactly the outstanding clean jobs. + The tombstone is written *before* anything is deleted, so a crash can cost + disk but never the record. +- **A finished tile declares no reclaimed exposures.** Deleting an exposure's + manifests would otherwise rerun every other tile that reads it, and those + reruns spread across the exposure-overlap component. So a tile whose + `final_cat` exists drops the exposure manifests that are gone from its input + list, and holds the rest through `ancient()`. This is why the profile runs + with `rerun-triggers: [mtime, params, code, software-env]`: the `input` + trigger reads that cut as a reason to rerun the very tiles it protects. + Know the consequence — `--forcerun` on a tile whose `final_cat` exists will + not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **A dead tile can be told to stop pinning exposures.** An exposure is + cleanable only once every consuming tile has its vignets, so one + permanently-failed tile holds its ~80 exposures for the life of the + campaign. List it under `clean_ignore_tiles:` in `config.yaml` and it leaves + the consumer sets. Retrying an ignored tile later is legal and expensive: + its exposure chains are gone and rebuild from scratch. +- **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the + absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its + per-runner counts and blocks no tile. The `exp_psf` benchmark tsv lives + beside `manifests/`, not inside it, so reclamation does not eat the + memory-sizing data. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs per-CCD attrition). It runs standalone — a DAG report node would itself be diff --git a/workflow/Snakefile b/workflow/Snakefile index 50dc75779..419e56cb7 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -107,6 +107,7 @@ if INDEX_DB.exists(): # Tiles this run can actually compute: declared AND indexed. The index spans the # campaign, so it is intersected with the declared list, not used as it. TILES_READY = [t for t in TILES if t in TILE_EXP] +READY_SET = set(TILES_READY) # A compute invocation with nothing to compute is never a success. Without this, # an empty intersection yields `rule all` with no inputs, an empty DAG and exit @@ -168,13 +169,50 @@ FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest( CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- + + +def flag(value, default=False): + """Truthiness for a config value that may arrive as a STRING. + + `--config clean=false` delivers the string "false", and every non-empty + string is truthy in Python — a plain bool() read that as ON and scheduled + the deletions the user had just switched off. YAML booleans pass through + unchanged; only strings are parsed, and an unparseable one is an error, not + a guess. + """ + if value is None: + return default + if isinstance(value, str): + v = value.strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off", ""): + return False + raise WorkflowError(f"Cannot read {value!r} as a boolean (use true/false).") + return bool(value) + + # `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no # jobs at all (nothing requests a tombstone); flipping it on later reclaims # RETROACTIVELY, because the exposures already cleaned are exactly the ones with # a tombstone, so the missing tombstones schedule exactly the clean jobs. # Only ever active under SP_PHASE=compute: the prepare and passthrough parses # read no index of their own and must schedule no deletions. -CLEAN = bool(config.get("clean", False)) and PHASE == "compute" +CLEAN = flag(config.get("clean", False)) and PHASE == "compute" + +# Reclamation REQUIRES the `input` rerun-trigger to be off (profiles/nibi sets +# the list; tile.smk explains why). The two are already tied together: CLEAN is +# gated on SP_PHASE=compute, which only workflow/bin/sp sets, and bin/sp always +# launches with that profile. A bare snakemake invocation without SP_PHASE +# schedules no clean job at all, so it cannot meet the incompatible combination. +# There is no runtime assertion because the trigger set is not readable at parse +# time (workflow.dag_settings is None until the DAG is built). + +# Tiles that must not pin an exposure's store. See config.yaml: a permanently +# failed tile otherwise holds every exposure it touches (~80) forever, because +# its vignets manifest will never exist and its exposures are therefore never +# eligible. Listing it here drops it from the consumer sets. +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) def tombstone(exp): @@ -185,8 +223,9 @@ def tombstone(exp): def clean_consumers(exp): """Every tile in the campaign that reads this exposure — the set whose - vignets must all exist before the store may go.""" - return sorted(EXP_TILES.get(exp, [])) + vignets must all exist before the store may go — minus the ignored tiles.""" + return sorted(t for t in EXP_TILES.get(exp, []) + if t not in CLEAN_IGNORE_TILES) def clean_targets(): @@ -199,15 +238,23 @@ def clean_targets(): scope expansion by cleanup, which is not a trade anyone asked for. Ineligible exposures are simply skipped; the invocation that finishes their last consumer picks them up. Deferral, never loss. + + Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in + `clean_ignore_tiles` neither gates eligibility nor appears in the job's + input — which is the whole point of that list. """ if not CLEAN: return [] - ready = set(TILES_READY) out = [] - for exp, tiles in EXP_TILES.items(): - if not tiles: + for exp, raw in EXP_TILES.items(): + if not raw: continue - if all(t in ready or Path(tile_manifest(t, "tile_vignets")).exists() + # An empty set AFTER filtering means every consumer is ignored: nothing + # is left that could ever read this exposure, so it is eligible now. + # That is the whole point of clean_ignore_tiles — all() of an empty set + # is True, and it is true here in the intended sense. + tiles = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() for t in tiles): out.append(tombstone(exp)) return sorted(out) @@ -307,9 +354,13 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only the aggregation targets are localrules: a mid-chain localrule would break -# `group:` fusion of the compute chains (D4). -localrules: all, prepare_all_tiles +# Only the aggregation targets and clean_exposure are localrules: a mid-chain +# localrule would break `group:` fusion of the compute chains (D4). +# clean_exposure is safe there because it is not in any chain — it hangs off +# `all` — and it is seconds of rmtree. As an sbatch job it would be ~20k +# scheduler submissions at DR6 scale to delete directories; the local-cores cap +# serialises them instead, which costs nothing at rmtree speed. +localrules: all, prepare_all_tiles, clean_exposure rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index ab06a459e..e52aa4457 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -35,13 +35,28 @@ index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, -# and leaves `cleaned.json` (which absorbs the manifests, so the report keeps -# them). Off here because P0 is a 4-tile debugging run where the exposure store -# is exactly what you want to inspect; on for any run big enough to care about -# disk. Flipping it on later reclaims retroactively — the missing tombstones -# schedule exactly the outstanding clean jobs. +# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them +# back out of the tombstone and reports the exposure as `cleaned`. Off here +# because P0 is a 4-tile debugging run where the exposure store is exactly what +# you want to inspect; on for any run big enough to care about disk. Flipping it +# on later reclaims retroactively — the missing tombstones schedule exactly the +# outstanding clean jobs. clean: false +# Tiles that may NOT pin an exposure store (default: empty). +# +# An exposure is eligible for cleaning only once EVERY consuming tile has its +# vignets. One permanently-failed tile therefore holds all ~80 exposures it +# touches for the life of the campaign. A tile listed here is dropped from the +# consumer sets, and its exposures become eligible. +# +# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its +# exposures' stores. If you later retry that tile, those exposure chains are +# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# exposure. That is correct, and expensive. Ignore a tile when you have decided +# it is dead, not while you are still debugging it. +clean_ignore_tiles: [] + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index b752bbee7..e04dd76fc 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -82,7 +82,11 @@ rule exp_psf: threads: 8 retries: 2 benchmark: - f"{EXP_DIR}/manifests/exp_psf.benchmark.tsv" + # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ + # wholesale, and this tsv is the measured-memory feed for mem_mb sizing + # (D4). Inside manifests/ it died with the first reclamation and took + # the campaign's only record of exp_psf's real footprint with it. + f"{EXP_DIR}/exp_psf.benchmark.tsv" resources: mem_mb = lambda wc, attempt: 16000 * attempt, runtime = 240 @@ -109,12 +113,26 @@ rule exp_psf: # tombstone stale under the default `params` rerun-trigger; the clean job # reruns after the new tile's vignets, against the enlarged set. # -# Not a localrule: at DR6 scale these are ~20k rmtrees and the local-cores cap -# would serialise them behind the aggregation targets. +# The tile side reads the exposure manifests through ancient() (see tile.smk), +# which is what keeps this deletion from rebuilding every neighbouring tile. +# This rule's OWN inputs are deliberately not ancient: a tile that really did +# rebuild its vignets must reschedule the cleans of the exposures it read. +# +# A localrule (declared in the Snakefile): it is an rmtree, not science, and one +# sbatch per exposure would be ~20k scheduler submissions at DR6 scale. Local +# execution serialises them under local-cores, which costs nothing at rmtree +# speed and never blocks the compute chains (this rule is in none of them). rule clean_exposure: input: + # ONLY the consumers this invocation may actually build. A consumer that + # is out of scope had its vignets manifest checked for existence at parse + # time (clean_targets' eligibility test) — declaring it here as well would + # pull that finished tile's whole chain into the DAG, where a rebuilt + # shared exposure then reruns it. That is how one damaged tile reached its + # spatial neighbours. In-scope consumers keep their edge: they may run in + # this DAG, so the clean must be ordered after them. lambda wc: [tile_manifest(t, "tile_vignets") - for t in clean_consumers(wc.exp)] + for t in clean_consumers(wc.exp) if t in READY_SET] output: tombstone = f"{EXP_DIR}/cleaned.json" params: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index e894309b9..8b665d1cf 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -23,8 +23,69 @@ variant is a config + one rule, at the config selector the PRD describes. def tile_exp(wc): return TILE_EXP.get(wc.tile, []) +# --- the tile->exposure edge, and why it is cut for finished tiles ---------- +# +# THE cascade fix. clean_exposure deletes the exposure's manifests on purpose: +# that is what makes a tile appended later rebuild the chain instead of running +# against an empty store. But those manifests are the tile side's inputs, and an +# exposure is read by ~7-10 tiles. So the moment ONE tile's chain rebuilt an +# exposure, every other tile reading it saw "input files updated by another job" +# and reran — and that rerun rebuilt ITS exposures, which reran THEIR other +# consumers, propagating across the whole exposure-overlap connected component. +# On fixture t4, asking for one damaged tile scheduled all four tiles' chains. +# +# Two mechanisms, and only the second one actually cuts it: +# +# 1. ancient() on every exposure-manifest edge. Correct on its own terms — a +# tile has no business rerunning because an exposure manifest is NEWER — and +# it is what keeps a pure-mtime disturbance (a re-touched manifest, a +# restored backup) from waking finished tiles. But ancient() governs +# TIMESTAMPS only. Snakemake propagates "my input is produced by a job that +# will run in this DAG" separately, and ancient does not suppress it +# (measured: t4 counts were identical with ancient alone). +# +# 2. Cutting the RECLAIMED edges of a FINISHED tile — the mechanism that works. +# A tile whose final_cat is on disk needs nothing further from its +# exposures: it has already extracted everything it will ever read. So for +# such a tile the input list drops the manifests that are GONE, and the +# propagation has nowhere to go. An UNfinished tile keeps its full edge set +# and therefore still drags in — and rebuilds — every exposure it needs, +# which is the accepted price of a late append, unchanged. +# +# Only the missing ones are dropped, never a manifest that still exists: a +# campaign that has cleaned nothing then declares exactly the edges it always +# did, and the cut cannot perturb it. +# +# The marker is final_cat, not the tile's own vignets manifest: on a tile whose +# catalogue was lost, the vignets manifest still exists while the vignette store +# (temp()) does not, so keying on vignets would cut the edge on exactly the tile +# that has to rerun, and run it against a deleted exposure store. +# +# THE CUT REQUIRES THE `input` RERUN-TRIGGER TO BE OFF (profiles/nibi sets the +# trigger list). Dropping an input is itself a change in the set of input files, +# which that trigger reads as a reason to rerun — reinstating the very cascade, +# now as "Set of input files has changed", and running finished tiles against a +# store that is gone. Measured on fixture t4, one damaged tile of four: 82 jobs +# with neither fix, 70 with the cut but the trigger on, 28 with both (= exactly +# the damaged tile's own chain, its two exposures, and the clean jobs). +# +# The cost, stated plainly: `--forcerun` on a tile whose final_cat exists will +# NOT rebuild its reclaimed exposures, because those edges are not in the DAG. +# Delete that tile's final_cat first and the whole chain comes back. +# +# What none of this weakens: clean_exposure's own inputs are neither ancient nor +# cut, so a tile that really did rebuild its vignets still reschedules the cleans +# of the exposures it read, and a grown consumer set still travels through +# params.consumers. +def tile_finished(tile): + return Path(final_cat(tile)).exists() + + def exp_manifests(wc, stage): - return [exp_manifest(e, stage) for e in tile_exp(wc)] + paths = [exp_manifest(e, stage) for e in tile_exp(wc)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] def tile_exp_split(wc): return exp_manifests(wc, "exp_split") def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 2d3d40cc5..e441fc3b5 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -21,8 +21,20 @@ manifests vanishing. Nothing is lost to the report: each manifest's content is copied verbatim into -the tombstone under ``manifests`` before deletion, so `sp report` can still say -what this exposure produced and why any runner was short. +the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned +exposure's record out of the tombstone — it reports the unit as ``cleaned``, +warn counts and shortfalls intact, instead of "not run". + +Order matters, and it is the reverse of the obvious one: the tombstone is +written FIRST, complete, and only then is anything deleted. A crash between the +two leaves a tombstone beside a store that is still there — the next invocation +treats the exposure as cleaned and only the disk is lost. Deleting first would +put the crash window where the manifests are already gone and the record that +replaces them was never written, and the report would be blind to that exposure +forever. + +The exp_psf benchmark tsv lives beside ``manifests/``, not inside it, so it +survives this job — it is the measured-memory feed for resource sizing (D4). The tombstone records the consumer set it was cleaned against. The rule carries that same set as a ``params`` value, so when the index grows a new consumer the @@ -60,20 +72,26 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - removed = [] - for target in (args.exp_dir / "output", mdir): - if target.exists(): - shutil.rmtree(target) - removed.append(str(target)) + targets = [t for t in (args.exp_dir / "output", mdir) if t.exists()] + # Tombstone first, complete, fsync'd — then delete. See the module docstring: + # the crash window has to sit where the data still exists, not where the + # record does not. args.tombstone.parent.mkdir(parents=True, exist_ok=True) - args.tombstone.write_text(json.dumps({ + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ "exp": args.exp, "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "consumers": consumers, - "removed": removed, + "removed": [str(t) for t in targets], "manifests": manifests, }, indent=2) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed = [] + for target in targets: + shutil.rmtree(target) + removed.append(str(target)) print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " f"{len(consumers)} consuming tile(s)") diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 1a8d128fd..43ce8dc62 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -16,6 +16,11 @@ ``completeness.py check``, carrying per-runner found/expect/floor and log-scraped failure reasons. +A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after +copying them into ``/cleaned.json``. That tombstone is read as the +unit's record and the unit is reported as **cleaned** — not "not run", and it +blocks no tile. + No disk scanning: counting products is the *check's* job, done once at the moment the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". @@ -69,6 +74,36 @@ def load_manifests(run_dir: Path, sub: str) -> dict: return out +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: + """Fill in reclaimed units from their ``cleaned.json``; return their ids. + + A cleaned exposure has NO ``manifests/`` — ``clean_exposure`` deleted it, + after copying every manifest verbatim into the tombstone. Read them back, or + the report inverts the truth exactly when reclamation works: the exposure + shows as "not run" and blocks the very tiles whose completion authorised the + deletion. + + Manifests on disk win if both exist — that is a re-built chain, and the + tombstone is then a stale record of the previous generation. + """ + cleaned = set() + for path in sorted((run_dir / sub).glob("**/cleaned.json")): + unit = path.parent.name + try: + tomb = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) + continue + if manifests.get(unit): + continue + for key, m in (tomb.get("manifests") or {}).items(): + if not isinstance(m, dict): + continue + manifests[unit][m.get("stage", key)] = m + cleaned.add(unit) + return cleaned + + def shortfalls(m: dict) -> dict: """``{runner: (found, expect, floor)}`` for every runner under expect.""" return {r: (d["found"], d["expect"], d["floor"]) @@ -84,17 +119,25 @@ def reasons(m: dict) -> list: return out -def tally_level(units, stages, manifests) -> dict: - """Per-stage counts + named unit lists, for one level.""" +def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: + """Per-stage counts + named unit lists, for one level. + + ``cleaned`` units are counted by the status their absorbed manifests carry + (complete or warn — attrition is preserved) and additionally listed under + ``cleaned``, so a reclaimed campaign reads as reclaimed rather than as a + campaign that never ran. + """ per_stage = {} for stage in stages: - t = {"complete": 0, "warn": [], "failed": [], "not_run": []} + t = {"complete": 0, "warn": [], "failed": [], "not_run": [], "cleaned": []} agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) for u in units: m = manifests.get(u, {}).get(stage) if m is None: t["not_run"].append(u) continue + if u in cleaned: + t["cleaned"].append(u) status = m.get("status", "failed") status = status if status in ("complete", "warn") else "failed" if status == "complete": @@ -158,12 +201,14 @@ def print_table(title, rows, limit=25): def print_stage_table(title, per_stage, n_units): print(f"\n{title} ({n_units} units declared)") - print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} attrition") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} " + f"{'cleaned':>8} attrition") for stage, t in per_stage.items(): att = [f"{r} {a['found']}/{a['expect']}" for r, a in t["products"].items() if a["found"] < a["expect"]] print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " - f"{len(t['failed']):>6} {len(t['not_run']):>8} {', '.join(att)[:60]}") + f"{len(t['failed']):>6} {len(t['not_run']):>8} " + f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") def main() -> None: @@ -190,6 +235,8 @@ def main() -> None: tile_m = load_manifests(args.run_dir, "tiles") exp_m = load_manifests(args.run_dir, "exp") + # Reclaimed exposures speak through their tombstones (D5, S5). + cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) tiles = tiles or sorted(tile_m) exps = exps or sorted(exp_m) @@ -201,7 +248,8 @@ def main() -> None: "n_tiles": len(tiles), "n_exposures": len(exps), "missing_tiles": missing, "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), - "exp_stages": tally_level(exps, EXP_STAGES, exp_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), + "cleaned_exposures": sorted(cleaned_exp), "tiles": unit_rows(tiles, TILE_STAGES, tile_m), "exposures": unit_rows(exps, EXP_STAGES, exp_m), } @@ -216,7 +264,13 @@ def main() -> None: # nothing. # Judged over ALL the exposure's stages, not just the first bad one, so an # exposure that warns early and fails late still blocks. + # A CLEANED exposure never blocks: its store is gone precisely because every + # consuming tile already had its vignets. Its absorbed manifests are read + # above, so a cleaned exposure that genuinely failed still shows in the + # tables — it just does not get to hold complete tiles hostage. def _blocks(unit): + if unit in cleaned_exp: + return False for stage in EXP_STAGES: m = exp_m.get(unit, {}).get(stage) if m is None or m.get("status", "failed") == "failed": @@ -235,7 +289,8 @@ def _blocks(unit): out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" - + (f" ({len(missing)} tiles missing exposure lists)" if missing else "")) + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "")) print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) print_stage_table("TILES", report["tile_stages"], len(tiles)) print_table("exposures not complete", report["exposures"], args.limit) From 4d28897550cc566b80ce5ea3b171ca5f75fb294a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:09:29 -0400 Subject: [PATCH 019/100] docs(orchestration): profile comment tells the truth about the clean/trigger gate Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index b79985e62..63997cb05 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -69,8 +69,10 @@ keep-incomplete: true # Nothing this workflow relied on is lost. The two genuinely data-derived input # sets are covered another way: a changed exposure list arrives through the # (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides -# in params. And the Snakefile refuses to schedule reclamation while the trigger -# is enabled, so a bare snakemake invocation cannot quietly undo this. +# in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute +# + this profile), so a bare snakemake invocation cannot quietly recombine +# reclamation with the `input` trigger; a runtime assertion is impossible +# (the trigger set is unreadable at parse time) — the gate is the launcher. rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a From 831a1273b369762c859622625bb2bbb557e5cbd9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:10:40 -0400 Subject: [PATCH 020/100] run: point the quad at a fresh smk-g3 run dir (old smk-p0 tree is unsharded, pre-manifest) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- workflow/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index e52aa4457..e06235f1a 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -8,14 +8,14 @@ # 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the # structural exposure dedup (the 196 quad had zero overlap; kept for the # append-invariant test). -tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt +tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # Where products land ($SP_RUN root). The sharded per-unit stores live under here: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-p0 +run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -30,7 +30,7 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and # report live with the run on /scratch until space is reclaimed. Mind the # 60-day purge for anything that must survive. -index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads From f82e7f46e93f95fb348bab05162fedfc509bb7ea Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:11:55 -0400 Subject: [PATCH 021/100] fix(orchestration): slurm_account is the association name (def-mjhudson_cpu) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executor plugin validates against sacctmgr associations, which on nibi carry the _cpu/_gpu suffix — the bare account that sbatch scripts accept is rejected at submission ('appears to be invalid'). First-contact finding from the first real slurm-executor run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 63997cb05..e4ad8e2c0 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -23,7 +23,7 @@ jobs: 800 default-resources: mem_mb: 2000 runtime: 120 # minutes - slurm_account: def-mjhudson + slurm_account: def-mjhudson_cpu software-deployment-method: [apptainer] # Explicit container environment (finding 2/3): the apptainer SDM otherwise drops From 5cfdc040cdc181ab6d10aa17458ccfd3dcc70eb5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 18:47:55 -0400 Subject: [PATCH 022/100] fix(orchestration): index build is the main compute parse's alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SLURM executor re-invokes snakemake inside every job, and that re-invocation inherits SP_PHASE=compute — so every job's parse re-ran build_index.build. Hundreds of concurrent sqlite writers on Lustre produced the 'database is locked' storm that killed the first real run (smk-g3, 2026-07-31): the top-level compute parse died first, and the compute jobs a later invocation submitted kept dying in their own parses. - gate the build on workflow.is_main_process as well as SP_PHASE; job parses now only LOAD the index - timeout=60 on every sqlite3.connect (Snakefile loader, build_index, run_report, build_forest): Lustre lock handoffs outlive the 5 s default even under pure reader traffic - config.yaml: the quad is 186/187 (real files); 210/211's tiles are broken symlinks into anaennis' moved processed_tiles tree, as are ~8.5k of the 10.3k staged tiles Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 13 +++++++++++-- workflow/config.yaml | 7 ++++--- workflow/scripts/build_forest.py | 2 +- workflow/scripts/build_index.py | 2 +- workflow/scripts/run_report.py | 2 +- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 419e56cb7..54d5a95a0 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -85,7 +85,14 @@ NGMIX_CHUNKS = workflow._scatter["ngmix"] # # In every other phase (prepare, or unset for a passthrough invocation) the parse # builds NOTHING and only loads whatever index is already on disk. -if PHASE == "compute": +# +# is_main_process matters as much as PHASE: the SLURM executor re-invokes +# snakemake inside every job, that re-invocation parses this file again, and it +# inherits SP_PHASE=compute from the submitting environment. Without the guard, +# every one of the run's jobs re-runs the build — hundreds of concurrent sqlite +# writers on Lustre, which is exactly the "database is locked" storm that killed +# the first real run (2026-07-31). Job parses only LOAD the index below. +if PHASE == "compute" and workflow.is_main_process: build_index.build( TILES, RUN_DIR, INDEX_DB, missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) @@ -97,7 +104,9 @@ if PHASE == "compute": # is keyed on (D5). EXP, TILE_EXP, EXP_TILES = {}, {}, {} if INDEX_DB.exists(): - _con = sqlite3.connect(INDEX_DB) + # timeout=60: Lustre lock handoffs are slow; the default 5 s trips on + # nothing more sinister than a reader in another job's parse. + _con = sqlite3.connect(INDEX_DB, timeout=60) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(_tile, []).append(_exp) diff --git a/workflow/config.yaml b/workflow/config.yaml index e06235f1a..d3b417149 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,9 +5,10 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -# 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the -# structural exposure dedup (the 196 quad had zero overlap; kept for the -# append-invariant test). +# 186/187 quad: 18 unique exposures with cross-tile reuse — exercises the +# structural exposure dedup. (The 210/211 quad used earlier is unusable: its +# tile images are symlinks into anaennis' moved processed_tiles tree. ~8.5k of +# the 10.3k staged tiles are such broken links; ~600 tiles are real files.) tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 427e9c647..f780531e1 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -27,7 +27,7 @@ def main() -> None: p.add_argument("--forest", required=True, type=Path) args = p.parse_args() - con = sqlite3.connect(args.index) + con = sqlite3.connect(args.index, timeout=60) exps = [r[0] for r in con.execute( "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] con.close() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 73d08609a..8dc9cb13a 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -114,7 +114,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, f"Re-run prepare_tiles for them, or raise --missing-threshold.") db_path.parent.mkdir(parents=True, exist_ok=True) - con = sqlite3.connect(db_path) + con = sqlite3.connect(db_path, timeout=60) # No DROP: the index accumulates across invocations (D1). con.executescript( """ diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 43ce8dc62..567a0c96b 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -223,7 +223,7 @@ def main() -> None: tiles, exps, tile_exp = [], [], defaultdict(list) if args.index.exists(): - con = sqlite3.connect(args.index) + con = sqlite3.connect(args.index, timeout=60) tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): From acfb91a0783799d76f0177b3b5986ba2a1ba70f8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 18:50:59 -0400 Subject: [PATCH 023/100] fix(orchestration): sp pins --snakefile; works from any cwd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An sbatch head job starts in the submission directory; snakemake's Snakefile discovery is cwd-relative and found nothing. The launcher already promised cwd-independence — now it delivers it. Co-Authored-By: Claude Fable 5 --- workflow/bin/sp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflow/bin/sp b/workflow/bin/sp index 3424584d1..ec563f032 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -45,7 +45,9 @@ STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # declared tiles allowed to be missing their exposure list (default 0.0). export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" -sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } +# --snakefile pins the workflow to this checkout: sp must work from any cwd +# (an sbatch head job starts in the submission directory, not the repo). +sm() { snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } cmd="${1:-}" case "$cmd" in From dedc1f0064c349b8b784a1982c8b26e70f3c8d40 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 18:55:03 -0400 Subject: [PATCH 024/100] docs(orchestration): a failed tile pins ~8 exposures, not ~80 P3 fixture measures 7.9 exposures/tile (50 tiles / 181 exposures, sp_p3_batch1-17716364.log). Co-Authored-By: Claude Fable 5 --- workflow/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index d3b417149..dfb3bb3a0 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -47,8 +47,8 @@ clean: false # Tiles that may NOT pin an exposure store (default: empty). # # An exposure is eligible for cleaning only once EVERY consuming tile has its -# vignets. One permanently-failed tile therefore holds all ~80 exposures it -# touches for the life of the campaign. A tile listed here is dropped from the +# vignets. One permanently-failed tile therefore holds all its exposures +# (~8 measured on P3: 7.9 exposures/tile) for the life of the campaign. A tile listed here is dropped from the # consumer sets, and its exposures become eligible. # # READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its From 2a24d3409f3652c152846d98f70e946a6685b5c0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 19:07:35 -0400 Subject: [PATCH 025/100] fix(orchestration): manifests are success-only DAG currency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A manifest was written on failure too, and `keep-incomplete: true` kept it. A head process that died without recording job failures therefore left failed stages looking complete, and a resume scheduled downstream work on top of them (exp_psf on an empty exp_mask output dir, live). completeness.py now forks on the result: success writes `.json` and removes a stale `.failed.json`; failure writes `.failed.json` and removes a stale `.json`. Exactly one exists, so `.json` means the stage succeeded. Both keep the write-only-on-change behaviour. The failed manifest is never a declared output, so snakemake neither tracks nor deletes it — which lets the profile drop `keep-incomplete` and get the wanted default back: a failed job's declared output is removed. run_report.py reads either name and takes the status from the manifest body, so manifests written under the old convention still report as failed. Co-Authored-By: Claude Fable 5 --- profiles/nibi/config.yaml | 14 +++---- workflow/README.md | 15 +++++--- workflow/Snakefile | 7 +++- workflow/scripts/clean_exposure.py | 8 +++- workflow/scripts/completeness.py | 59 +++++++++++++++++++++--------- workflow/scripts/run_report.py | 18 +++++++-- 6 files changed, 86 insertions(+), 35 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index e4ad8e2c0..f78bea7f7 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -46,13 +46,13 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true -# D3 DEPENDS ON THIS. Snakemake's default is to DELETE the declared outputs of a -# failed job — and this workflow's declared output IS the manifest, the only -# record of *why* a unit failed and the only thing `sp report` reads. Without -# keep-incomplete, every failed unit reads back as "not_run" and the post-mortem -# debris is gone. Safe here because each rule `rm -rf`s its own run dir at start, -# so a rerun never sees stale products. -keep-incomplete: true +# No `keep-incomplete` here (it was the pre-S7a way to preserve failed manifests +# as report evidence): the declared output IS the manifest, so letting snakemake +# delete a failed job's output is exactly the semantics this workflow wants — a +# `.json` on disk then means the stage succeeded, and a resume cannot +# schedule downstream work on top of a failure. The post-mortem evidence lives in +# `.failed.json`, which completeness.py writes on failure and which is +# never a declared output, so snakemake never touches it. # rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still # propagate — completeness.py writes the manifest only on change, so mtimes move diff --git a/workflow/README.md b/workflow/README.md index c64047e7f..2183f03e8 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -112,11 +112,16 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee (`/manifests/.json`), not its product files — a missing CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. -- **Manifests are the DAG's currency.** `completeness.py check` writes the - manifest, and it is the only record of *why* a unit failed. That is why - the profile sets `keep-incomplete: true` — Snakemake's default deletes a - failed job's declared outputs, which would erase the manifest the report - needs. +- **Manifests are the DAG's currency, and they are success-only.** + `completeness.py check` writes `.json` when the stage passed its count + floor, and `.failed.json` when it did not — removing the other file + either way. So `.json` on disk means "this stage succeeded", and a + resume after an unclean death cannot schedule downstream work on top of a + failure. The failed manifest carries the same content plus the scraped + reasons; it is never a declared output, so Snakemake never tracks or deletes + it, and `sp report` reads it when the success manifest is absent. The profile + runs *without* `keep-incomplete` for the same reason: deleting a failed job's + declared output is the wanted semantics. - **Completeness is a count floor, not a taxonomy.** After a run, `sp_rule.py` counts products per mandatory runner against `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition diff --git a/workflow/Snakefile b/workflow/Snakefile index 54d5a95a0..55aae67bf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -346,8 +346,11 @@ def sp_shell(stage, config_name): unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number by construction (D4). - The check runs even when shapepipe_run failed — the manifest is the only - thing `sp report` reads, so a failed unit must still leave a record of why. + The check runs even when shapepipe_run failed, so a failed unit still leaves + a record of why — but that record goes to `.failed.json`, and the + declared `.json` is removed if it was there. Success manifests are + success-only currency (completeness.py); the failed one is evidence for + `sp report` and is never a declared output. """ return ( "{params.pre}\n" diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index e441fc3b5..ce6fef80a 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -20,11 +20,17 @@ needs to run, so tiles already finished are NOT rerun by their exposures' manifests vanishing. -Nothing is lost to the report: each manifest's content is copied verbatim into +Nothing is lost to the report: every ``manifests/*.json`` is copied verbatim into the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned exposure's record out of the tombstone — it reports the unit as ``cleaned``, warn counts and shortfalls intact, instead of "not run". +The absorption is by GLOB, so it takes whatever is in ``manifests/``, keyed by +file stem; the report re-keys on each manifest's own ``stage`` field. A +``.failed.json`` is therefore carried through unremarkably — it should +never be there (an exposure with a failed stage has no complete vignets consumer +and so is not eligible for cleaning), but it costs nothing to be right about. + Order matters, and it is the reverse of the obvious one: the tombstone is written FIRST, complete, and only then is anything deleted. A crash between the two leaves a tombstone beside a store that is still there — the next invocation diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 1a8820c10..51c2521d8 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -18,15 +18,25 @@ completeness.py check exp_mask {output} || rc=1 exit $rc -It counts the unit's products under ``$SP_RUN``, writes the manifest at -``{output}``, and exits nonzero iff a mandatory runner is below its floor. The -manifest is ALWAYS written, failure included: it is the DAG's currency and the -only thing ``run_report.py`` reads, so a failed unit must still leave a record of -*why*. (The profile sets ``keep-incomplete`` so snakemake does not delete that -record on the way out.) Manifests carry no wall-clock, and are rewritten ONLY -when their content changes — identical on-disk state must leave a byte-identical -manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on -every unrelated ``--forcerun``. +It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory +runner is below its floor. WHERE it writes the record is the whole point: + + * success -> ``.json`` (the rule's declared output, the DAG's currency) + and any stale ``.failed.json`` is removed; + * failure -> ``.failed.json`` (same content, ``status: failed``) and any + stale ``.json`` is REMOVED. + +``.json`` therefore means "this stage succeeded", full stop — a resume can +never schedule a downstream stage on top of a failed one, not even after a head +process died without recording the failure. The failed manifest is never a +declared output of any rule: it is post-mortem evidence for ``run_report.py``, +which reads it when the success manifest is absent, and snakemake neither tracks +nor deletes it. + +Manifests carry no wall-clock, and either file is rewritten ONLY when its content +changes — identical on-disk state must leave a byte-identical manifest with an +UNMOVED mtime, or the mtime rerun-trigger churns the cone on every unrelated +``--forcerun``. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -253,6 +263,18 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok +def failed_path(manifest: Path) -> Path: + """``.json`` -> ``.failed.json``, in the same manifests/ dir.""" + return manifest.with_name(manifest.name[:-len(".json")] + ".failed.json") + + +def write_if_changed(path: Path, text: str) -> None: + """Write only when the bytes differ — see the module docstring on mtime.""" + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != text: + path.write_text(text) + + def _unit_from_run_dir(run_dir): """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). @@ -286,21 +308,24 @@ def main(argv=None) -> int: unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) - args.manifest.parent.mkdir(parents=True, exist_ok=True) - # Write ONLY on change. An unconditional write moves the mtime on every run, - # and mtime is a rerun-trigger: a `--forcerun` of one upstream stage would - # then rewrite this manifest byte-identically and drag the whole downstream - # cone along with it. text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" - if not args.manifest.exists() or args.manifest.read_text() != text: - args.manifest.write_text(text) + + # The success/failure fork. Exactly one of the two files exists afterwards, + # so the presence of .json IS the statement "this stage succeeded" and + # the DAG can never build on top of a failure. The removal of the stale + # counterpart is what makes a success->failure or failure->success transition + # complete rather than additive. + written = args.manifest if ok else failed_path(args.manifest) + stale = failed_path(args.manifest) if ok else args.manifest + write_if_changed(written, text) + stale.unlink(missing_ok=True) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} print(f"[completeness] {runner}: {r['found']}/{r['expect']} " f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) print(f"[completeness] {args.stage} {unit}: {manifest['status']} " - f"-> {args.manifest}", file=sys.stderr) + f"-> {written}", file=sys.stderr) for f in manifest["failures"]: for reason in f["reasons"]: print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 567a0c96b..69c2e4b5f 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -16,6 +16,13 @@ ``completeness.py check``, carrying per-runner found/expect/floor and log-scraped failure reasons. +A stage that failed leaves ``.failed.json`` instead of ``.json`` +(``completeness.py``): the success name is reserved for success, so the DAG +cannot build on a failure. Both names are read here, and the status comes from +the manifest BODY, so a stage is reported as failed either way — including +manifests written before that split, where the failure content sat under the +success name. + A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after copying them into ``/cleaned.json``. That tombstone is read as the unit's record and the unit is reported as **cleaned** — not "not run", and it @@ -42,7 +49,7 @@ TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", "tile_merge_headers", "tile_detect", "tile_vignets", "tile_ngmix", "tile_merge_cats", "tile_make_cat"] -EXP_STAGES = ["exp_get_images", "exp_split", "exp_mask", "exp_psf"] +EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] STATUSES = ("complete", "warn", "failed", "not_run") @@ -54,8 +61,13 @@ def load_manifests(run_dir: Path, sub: str) -> dict: agnostic, and the only form that joins to the index (the manifest's own ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is not the index's ``210.282``). The stage comes from the manifest body, never - the filename: ngmix chunks share a stage under per-chunk filenames, and they - collapse to one worst-case entry. + the filename: ngmix chunks share a stage under per-chunk filenames, and + ``.failed.json`` names the same stage as ``.json``. + + Several files can therefore map to one (unit, stage), and the WORST status + wins. That is what collapses the ngmix chunks to one entry, and what makes a + failed manifest speak even in the transient window where its success-named + counterpart has not yet been removed. """ out: dict = defaultdict(dict) for path in sorted((run_dir / sub).glob("**/manifests/*.json")): From a1d4dfac3501e6dd531ebf385e29632665d28e7e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 19:11:56 -0400 Subject: [PATCH 026/100] feat(orchestration): star-cat creation moves into the DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PRD assumed mask star catalogues were pre-generated offline. The pre-staged store covers 1 of the current quad's 18 exposures, and symlinking the whole store into a unit breaks the mask module outright: the file handler intersects the image numbers found across a config's INPUT_DIRs, and the store contributes 7240 foreign numbers, so the intersection is empty ("numbers ... do not intersect"). New rule `exp_star_cat`, one per exposure: it runs create_star_cat.py in the container against the exp_get_images symlink farm (one Vizier cone query over the focal plane, ~6 s), then builds a REAL per-unit `star_cat_exp/` of 40 per-CCD symlinks — exactly this exposure's numbers, so the intersection is right by construction and config_exp_Ma.ini needs no change. exp_mask takes its manifest as an input. It is a localrule, so the queries run serially in the head process (an sbatch job on a compute node, which has internet): CDS is protected at any campaign scale and the scheduler never sees a six-second job. The catalogue itself is cached run-independently under `star_cats:`, now the cache root on /scratch, and create_star_cat.py skips an existing cat, so reruns and later campaigns are free. unit_pre no longer materialises the star_cat_exp/star_cat_tiles store symlinks. tile.smk records what a future tile_mask needs on the tile side. Co-Authored-By: Claude Fable 5 --- workflow/README.md | 29 ++++--- workflow/Snakefile | 34 ++++---- workflow/config.yaml | 13 +++- workflow/config/cfis/config_exp_Ma.ini | 4 +- workflow/rules/exposure.smk | 104 ++++++++++++++++++++++++- workflow/rules/prepare.smk | 10 +-- workflow/rules/tile.smk | 8 ++ 7 files changed, 165 insertions(+), 37 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 2183f03e8..c286ebdcb 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,7 +22,8 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats. +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the +# star-catalogue cache root). # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -92,8 +93,8 @@ workflow/ config.yaml the run: tile list, paths, container, chunk count bin/sp committed launcher (module load + /project venv + run/report/cancel) rules/ - prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats - exposure.smk per-exposure: get_images, split, mask, psf (no temp()) + prepare.smk tile get_images/uncompress/find_exposures + exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) @@ -129,14 +130,22 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee error-signature whitelist. `--keep-going` isolates a failure to its own DAG cone. - **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` - in `tiles/<2-char prefix>//` or `exp///`, with a `cfis` - config symlink, `star_cat_{exp,tiles}` symlinks, and a config copy setting - `RUN_DATETIME=False`. Configs are committed under `workflow/config/cfis/` - and version with the rules that set the env vars they interpolate — there - is no `config_src` knob, by construction. + in `tiles/<2-char prefix>//` or `exp///`. Configs are + committed under `workflow/config/cfis/` and version with the rules that set + the env vars they interpolate — there is no `config_src` knob, and no per-unit + config symlink; `$SP_CONFIG` points straight at the committed directory. +- **Mask star catalogues are built in the DAG.** `exp_star_cat` runs one Vizier + cone query per exposure into the run-independent cache at `star_cats:`, then + fans it out into a real per-unit `star_cat_exp/` directory of 40 per-CCD + symlinks, which `exp_mask` consumes. The directory must be per-unit and real: + the file handler intersects the image numbers it finds across a config's + `INPUT_DIR`s, so a symlink to the whole cache contributes every other + exposure's numbers and the intersection comes out empty. It is a `localrule`, + so the queries run serially in the head process — CDS is never hammered, and + the scheduler never sees a six-second job. The cache makes reruns and later + campaigns free. - **The index is parse-time data, never a rule input.** Appending tiles - changes which jobs exist without invalidating completed work. Star cats - are re-keyed per unit for the same reason. `final_cat` is `protected()`. + changes which jobs exist without invalidating completed work. - **Exposure products are not `temp()`.** Exposures overlap tiles, so `temp()` would cascade destructive reruns when a tile is appended later. Reclamation is the in-DAG `clean_exposure` rule instead: one job per diff --git a/workflow/Snakefile b/workflow/Snakefile index 55aae67bf..2b2ec322e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -23,6 +23,7 @@ legitimate, and at DR6 scale per-CCD declaration means millions of paths. """ import hashlib +import json import os import sqlite3 import sys @@ -41,7 +42,13 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) +# Run-independent cache root for the mask star catalogues, keyed by exposure and +# filled by the exp_star_cat rule (config.yaml explains the placement). STAR_CATS = Path(config["star_cats"]) +# This checkout's own tree: exp_star_cat runs its scripts/python/create_star_cat.py +# against its src/, for the same reason CONFIG_DIR is the committed config dir — +# script, library and rule are one artefact and version together. +REPO_DIR = Path(workflow.basedir).parent INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the @@ -296,11 +303,10 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, ORIGINAL exposure name from the index (``2605805p``), so get_images matches ``.fits.fz`` in the store — the bare base id matches nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale - pre-fix file with the bare id; - * ``star_cat_exp`` / ``star_cat_tiles`` dir symlinks into the shared - pre-generated pool (the mask configs read them as INPUT_DIRs). + pre-fix file with the bare id. There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at - the committed config dir. + the committed config dir, and ``star_cat_exp`` is a real per-unit directory + built by the ``exp_star_cat`` rule, not a symlink into a shared pool. Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -330,10 +336,6 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, lines += [f'mkdir -p "{fe}"', f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] - for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): - src = STAR_CATS / sub - lines.append(f"[ -d '{src}' ] && ln -sfn '{src}' \"$SP_RUN/{name}\" || true") - lines += list(pre_run) lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] return "\n".join(lines) @@ -366,13 +368,15 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only the aggregation targets and clean_exposure are localrules: a mid-chain -# localrule would break `group:` fusion of the compute chains (D4). -# clean_exposure is safe there because it is not in any chain — it hangs off -# `all` — and it is seconds of rmtree. As an sbatch job it would be ~20k -# scheduler submissions at DR6 scale to delete directories; the local-cores cap -# serialises them instead, which costs nothing at rmtree speed. -localrules: all, prepare_all_tiles, clean_exposure +# The aggregation targets, clean_exposure and exp_star_cat run in the head +# process. clean_exposure is seconds of rmtree and hangs off `all`; exp_star_cat +# is ~6 s dominated by one CDS query, and running it locally serialises those +# queries (see exposure.smk). Both would otherwise be ~20k sbatch submissions at +# DR6 scale for work shorter than the scheduling latency. +# +# exp_star_cat is the one MID-CHAIN localrule, so it must stay out of any future +# `group:` label: a local job cannot be fused into a submitted group. +localrules: all, prepare_all_tiles, clean_exposure, exp_star_cat rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index dfb3bb3a0..2c5b3008f 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -24,8 +24,17 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # are one artefact and must version together, so the dir is fixed by construction. # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# Star catalogues for masking, pre-generated (network step done in prepare): -star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats + +# The mask star-catalogue CACHE root — run-independent, shared by every campaign. +# `exp_star_cat` writes /exp/star_cat-.fits, one GSC 2.3 cone query per +# exposure (~787 KiB, ~6 s), and skips a cat that is already there. Exposures +# recur across tiles and batches, so the cache is mostly hits. +# +# On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day +# purge therefore applies, and that is acceptable: a purged cache regenerates one +# query per exposure, and exp_star_cat is a localrule, so the head process runs +# those queries serially and never hammers CDS. +star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache # The run index. Design intent (finding 15) is durable products on /project — # BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini index 0b85fe86c..d5b521080 100644 --- a/workflow/config/cfis/config_exp_Ma.ini +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -63,7 +63,6 @@ INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_c NUMBERING_SCHEME = -0000000-0 # Input file patterns: image, weight, external flag, external star catalogue -# (folded from runs/p3-batch1/cfis: offline cluster, no online star-cat fetch) FILE_PATTERN = image, weight, flag, star_cat FILE_EXT = .fits, .fits, .fits, .fits @@ -76,7 +75,8 @@ USE_EXT_FLAG = True # External star catalogue flag, use external cat if True, # obtain from online catalogue if False -# (folded from p3-batch1: True, using the pre-staged star_cat_exp pool) +# True: the cat comes from $SP_RUN/star_cat_exp, the per-unit farm the +# exp_star_cat rule builds (40 per-CCD links to this exposure's one cat). USE_EXT_STAR = True # File name suffix for the output flag files (optional) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index e04dd76fc..da3e49e4f 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,6 +1,7 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -> exp_mask -> exp_psf + exp_get_images -> exp_split ---> exp_mask -> exp_psf + -> exp_star_cat -/ Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a @@ -36,6 +37,102 @@ rule exp_get_images: shell: sp_shell("exp_get_images", "config_exp_Gie.ini") +# --- mask star catalogues --------------------------------------------------- +# One GSC 2.3 (Vizier) cone query per exposure, covering the whole MegaCam focal +# plane, plus the 40 per-CCD symlinks the mask module's numbering scheme needs. +# +# It reads the exp_get_images output dir directly: that is a symlink farm of +# image-.fitsfz, exactly what create_star_cat.py's `-k exp` mode consumes — +# one multi-extension header read, one query, ~6 s measured. +# +# A LOCALRULE (declared in the Snakefile), so it runs in the head process — an +# sbatch job on a compute node, and nibi compute nodes have internet (verified by +# curl to vizier.cds.unistra.fr from a compute job). Local execution serialises +# the queries, which throttles CDS at any campaign scale for free, and it spares +# the scheduler one submission per exposure for six seconds of work. +# +# The CATALOGUE is cached run-independently under config `star_cats`, keyed by +# exposure. create_star_cat.py skips a cat that exists, so retries, reruns and +# later campaigns re-link an existing file and issue no query. +# +# The per-unit farm is a REAL directory holding exactly this exposure's 40 +# numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR +# and the file handler INTERSECTS the numbers found across INPUT_DIRs, so a +# symlink to a shared whole-store pool contributes every other exposure's numbers +# and the intersection is empty ("numbers ... do not intersect", live). +# +# Declared output is a manifest rather than a symlink or a directory(): it is +# written last, it is the one per-exposure file that is unique to this rule, it +# records what the farm points at, and it keeps the "one rule, one manifest" +# currency of every other rule — including being deleted by clean_exposure, so a +# reclaimed exposure rebuilds its farm from the cache at no query cost. + +# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE +# pointing at a path that does not exist inside the image, so requests is pointed +# at the bundle explicitly (proven in the p3-batch1 bash precedent). +STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" + + +def star_cat_cmd(exp): + """The whole rule body, as bash — carried as a params value, never inlined + in ``shell:``: it contains literal ``{}`` (the manifest JSON) and snakemake + formats a shell string once, which would consume those braces.""" + cache = f"{STAR_CATS}/exp" + cat = f"{cache}/star_cat-{exp}.fits" + work = exp_dir(exp) + farm = f"{work}/star_cat_exp" + images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" + manifest = exp_manifest(exp, "exp_star_cat") + body = json.dumps({ + "stage": "exp_star_cat", "level": "exp", "unit": exp, + "status": "complete", "cache": cat, "link_dir": farm, "n_links": 40, + }, indent=2, sort_keys=True) + return "\n".join([ + "set -euo pipefail", + f"mkdir -p '{cache}' '{farm}' '{work}/manifests'", + # Same binds and isolation as the profile's apptainer-args; the SDM does + # not wrap this rule (container: None below), because the farm loop and + # the manifest write belong to the host side of the job. + f"apptainer exec --cleanenv --home {Path.home()} --bind /project --bind /scratch" + f" --env PYTHONPATH={REPO_DIR}/src,REQUESTS_CA_BUNDLE={STAR_CAT_CA}" + f",SSL_CERT_FILE={STAR_CAT_CA},CURL_CA_BUNDLE={STAR_CAT_CA}" + f" '{config['container']}'" + f" python {REPO_DIR}/scripts/python/create_star_cat.py" + f" -i '{images}' -o '{cache}' -k exp", + f"test -s '{cat}'", + # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the + # one focal-plane catalogue (pattern from the p3-batch1 precedent). + f"for ccd in $(seq 0 39); do ln -sfn '{cat}' " + f"'{farm}/star_cat-{exp}-'\"$ccd\"'.fits'; done", + # Byte-stable, and written only after the links exist: an unconditional + # write would move the mtime, which is a rerun-trigger. + f"tmp='{manifest}.tmp'", + "cat > \"$tmp\" <<'SP_STAR_CAT_JSON'", + body, + "SP_STAR_CAT_JSON", + f"cmp -s \"$tmp\" '{manifest}' && rm -f \"$tmp\" || mv -f \"$tmp\" '{manifest}'", + ]) + + +rule exp_star_cat: + input: + rules.exp_get_images.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_star_cat.json" + params: + cmd = lambda wc: star_cat_cmd(wc.exp) + # No SDM wrapping: the rule calls apptainer itself, because only the query + # runs in the container (bin/sp has loaded the apptainer module). + container: + None + threads: 1 + retries: 2 + resources: + mem_mb = 2000, + runtime = 10 + shell: + "{params.cmd}" + # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: @@ -55,7 +152,10 @@ rule exp_split: rule exp_mask: input: - rules.exp_split.output.manifest + # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs + # and this exposure's own star_cat_exp farm. + rules.exp_split.output.manifest, + rules.exp_star_cat.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_mask.json" params: diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 92998e184..4b1e401a6 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -10,12 +10,10 @@ tier). Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. -Star catalogues for masking are pre-generated offline (create_star_cat.py on a -networked login node) and consumed as DIRECTORIES: the mask configs read -``$SP_RUN/star_cat_{exp,tiles}`` with per-CCD numbering. Every rule's prologue -materialises the two dir symlinks; there is NO per-unit star-cat DAG node — the -store is pre-run input like the image store, and a missing cat fails the mask -stage's count floor loudly. +Star catalogues for masking are NOT a prepare-phase concern and not pre-run +input: they are built in the compute DAG, one node per exposure +(``exp_star_cat`` in exposure.smk), into a run-independent cache. The tile side +has no star-cat node because it has no mask rule yet — see tile.smk. """ # NUMBER_LIST is never set for get_images (download stage; nothing on disk to diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8b665d1cf..d52b76e0a 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -18,6 +18,14 @@ Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask run), and no tile-mask config was committed in the S2 sweep. Adding the masked variant is a config + one rule, at the config selector the PRD describes. + +That rule also needs a tile-side analogue of ``exp_star_cat``: tile star cats key +on TILE id, so they are a separate cache namespace and a separate node, and the +earliest point it can run is after ``tile_uncompress`` (create_star_cat.py's +``-k tile`` mode reads the uncompressed tile image's primary header). The mask +config would then read a real per-unit ``$SP_RUN/star_cat_tiles`` directory, +built the same way and for the same reason (the file handler intersects numbers +across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). """ def tile_exp(wc): From 6a8e71a09d6a0e7d3f1f8cb729a140a5b347ac19 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 20:08:28 -0400 Subject: [PATCH 027/100] fix(orchestration): the verdict composes counts with shapepipe_run's rc completeness.py forked on its count verdict alone. A job whose floors were met but whose shapepipe_run exited nonzero therefore wrote the SUCCESS manifest and still exited nonzero. Snakemake then deleted that manifest as a failed job's output, the check had already unlinked any prior failed.json, and the failure left no trace on disk at all. The two statements are now composed. sp_shell passes `--job-rc "$rc"`; a nonzero rc forces the failure branch, records job_rc and a shapepipe_run failure entry that run_report surfaces, and the fork writes failed.json and exits 1 as usual. The rc is recorded only when nonzero, so every existing success manifest stays byte-identical and the mtime rerun-trigger does not churn. Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 14 +++++++++++++- workflow/scripts/completeness.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2b2ec322e..eb60f7c39 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -353,12 +353,24 @@ def sp_shell(stage, config_name): declared `.json` is removed if it was there. Success manifests are success-only currency (completeness.py); the failed one is evidence for `sp report` and is never a declared output. + + The verdict is COMPOSED, which is why `--job-rc "$rc"` is passed: the count + floors and shapepipe_run's exit status are two independent statements about + the same job, and the manifest must reflect both. Without it, a job whose + counts cleared their floors but whose shapepipe_run died (a late runner + raising after the counted ones wrote their files) wrote the SUCCESS manifest + and still exited nonzero — snakemake then deleted that manifest as a failed + job's output, no failed.json was written (the check thought it had won), and + any prior failed.json had already been unlinked. The failure left no trace at + all. With the rc composed in, the fork lands on the failure side and the + evidence survives. """ return ( "{params.pre}\n" "rc=0\n" f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" - f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}} || rc=1\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" + " --job-rc \"$rc\" || rc=1\n" "exit $rc\n" ) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 51c2521d8..138399af5 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -15,11 +15,14 @@ rc=0 shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} || rc=1 + completeness.py check exp_mask {output} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory -runner is below its floor. WHERE it writes the record is the whole point: +runner is below its floor OR ``--job-rc`` is nonzero — the verdict is COMPOSED of +the counts and shapepipe_run's own exit status, because a runner can raise after +the counted ones have written their files. WHERE it writes the record is the +whole point: * success -> ``.json`` (the rule's declared output, the DAG's currency) and any stale ``.failed.json`` is removed; @@ -298,6 +301,8 @@ def main(argv=None) -> int: help="override the unit ID (default: basename of $SP_RUN)") c.add_argument("--stage-dir", default=None, help="override the run_sp_* subdir (default: the stage table)") + c.add_argument("--job-rc", type=int, default=0, + help="shapepipe_run's exit status, composed into the verdict") args = p.parse_args(argv) run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) @@ -308,6 +313,29 @@ def main(argv=None) -> int: unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + + # The verdict is COMPOSED of two independent statements: the count floors + # (above) and shapepipe_run's own exit status (here). Counts alone are not + # enough — a runner can raise AFTER the counted runners have written their + # files, so the floors are met while the job died. That combination used to + # write the SUCCESS manifest, which snakemake then deleted as a failed job's + # output, leaving no manifest at all: failure invisible, and any earlier + # failed.json already unlinked by the success branch. An rc-driven override + # puts the fork on the failure side, so failed.json is written and kept. + # + # Recorded only when nonzero, which keeps every existing success manifest + # byte-identical (the mtime rerun-trigger reads those bytes). + if args.job_rc != 0: + ok = False + manifest["status"] = "failed" + manifest["job_rc"] = args.job_rc + manifest["failures"].append({ + "runner": "shapepipe_run", "found": 0, "floor": 1, "expect": 1, + "warn": False, + "reasons": [f"shapepipe_run exited {args.job_rc} " + f"(counts above floor)"], + }) + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" # The success/failure fork. Exactly one of the two files exists afterwards, From 734101201c2ff6ae9be94fe77bc8e708b8ac627b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 20:08:44 -0400 Subject: [PATCH 028/100] fix(orchestration): exp_star_cat stops trusting what it cannot see MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ways the star-catalogue stage lied about its own state, plus one comment that lied about the scheduler. The farm loop wrote into the shared pool. Legacy unit dirs carry star_cat_exp as a symlink into the old shared star-cat pool, `mkdir -p` no-ops on it, and the 40-link loop then followed the link and wrote this exposure's links into the pool (520 stray links found live). The rule now unlinks a symlinked farm before the mkdir. It never rm -rf's a real farm: that farm is its own output, and ln -sfn refreshes it in place. A killed cache write stayed poisonous forever. create_star_cat.py wrote the cache FITS in place, so an interrupted write left a truncated file that the `isfile -> continue` skip trusted for the rest of the campaign — and `test -s` passes on partial bytes. Both branches now write a hidden, PID-tagged temp with a .fits suffix (astropy picks its writer from the extension) and os.replace it into position; a leftover temp is removed in a finally. The dot-prefix keeps the temp out of the `star_cat*` globs. The manifest attested forever. It was the rule's only declared output, so a purged cache (/scratch, 60 days) or a deleted farm left the stage looking complete while exp_mask had nothing to read. The ccd-0 farm link is now declared alongside it. Snakemake's existence test follows symlinks, so a purge leaves that link dangling, the rule reruns, and it re-queries or re-links — the recovery config.yaml already promised. The script was outside the fingerprint. exp_star_cat's params held only paths, so an edit to create_star_cat.py never reran it. STAR_CAT_HASH follows the existing completeness/forest/clean pattern. The hash deliberately does NOT key the cache path: a rerun still takes the skip-if-exists branch, so a semantic change to the query needs the cache cleared by hand. config.yaml now says so. And the localrule comment claimed local execution serialises the queries. It does not: local jobs run in a pool sized by --local-cores (unset here, so the head process's CPU count, 5 as measured), which is the real concurrency cap. The localrule stays — 5-way is ordinary CDS politeness, and the alternative is ~20k sbatch submissions for six seconds of work each. Co-Authored-By: Claude Fable 5 --- scripts/python/create_star_cat.py | 27 +++++++++++++-- workflow/Snakefile | 8 +++++ workflow/config.yaml | 11 +++++-- workflow/rules/exposure.smk | 55 +++++++++++++++++++++++++------ 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py index 0e3586f88..404cee4ac 100755 --- a/scripts/python/create_star_cat.py +++ b/scripts/python/create_star_cat.py @@ -118,6 +118,29 @@ def query_vizier(ra, dec, radius_arcmin): return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) +def _write_atomic(table, output_dir, img_number, output_name): + """Write ``table`` to ``output_name`` atomically. + + The catalogue is a run-independent CACHE, and the caller's only test for a + cache hit is ``os.path.isfile``. An in-place ``table.write`` that is killed + part-way (job timeout, OOM, node failure) therefore leaves a truncated FITS + that every later run trusts forever — and ``test -s`` passes on partial + bytes. Writing to a temp and renaming makes the visible file all-or-nothing: + ``os.replace`` is atomic within a directory. + + The temp keeps a ``.fits`` suffix, because astropy picks the writer from the + extension. It is dot-prefixed and PID-tagged so it stays out of the + ``star_cat*`` globs the rules use, and two concurrent writers cannot collide. + """ + tmp = f"{output_dir}/.tmp-{os.getpid()}-star_cat{img_number}.fits" + try: + table.write(tmp, overwrite=True) + os.replace(tmp, output_name) + finally: + if os.path.exists(tmp): + os.remove(tmp) + + def main(input_dir, output_dir, kind): file_list = os.listdir(input_dir) @@ -140,7 +163,7 @@ def main(input_dir, output_dir, kind): f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" ) table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) + _write_atomic(table, output_dir, img_number, output_name) else: h = fits.getheader(fpath, 0) @@ -156,7 +179,7 @@ def main(input_dir, output_dir, kind): continue table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) + _write_atomic(table, output_dir, img_number, output_name) return 0 diff --git a/workflow/Snakefile b/workflow/Snakefile index eb60f7c39..1c8222271 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -183,6 +183,14 @@ def unit_num(unit): SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] +# Same argument for the star-cat query script, which lives in the repo's own +# scripts/python/ rather than workflow/scripts/: exp_star_cat's params otherwise +# fingerprint nothing but paths, so an edit to create_star_cat.py would never +# rerun the rule. The hash does NOT key the cache path — see config.yaml's +# star_cats block on clearing the cache after a semantic change. +STAR_CAT_HASH = hashlib.md5( + (REPO_DIR / "scripts" / "python" / "create_star_cat.py").read_bytes() +).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- diff --git a/workflow/config.yaml b/workflow/config.yaml index 2c5b3008f..55962297e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -30,10 +30,17 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # exposure (~787 KiB, ~6 s), and skips a cat that is already there. Exposures # recur across tiles and batches, so the cache is mostly hits. # +# THE CACHE IS NOT KEYED BY SCRIPT VERSION. A semantic change to +# scripts/python/create_star_cat.py (query radius, catalogue ID, column set) +# does rerun exp_star_cat — the script's hash is a rule param — but the rerun +# takes the skip-if-exists branch and only re-links the OLD catalogues. Clear +# this cache by hand when the change must reach the data. +# # On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day # purge therefore applies, and that is acceptable: a purged cache regenerates one -# query per exposure, and exp_star_cat is a localrule, so the head process runs -# those queries serially and never hammers CDS. +# query per exposure. exp_star_cat declares its ccd-0 farm link as an output, so +# a purge leaves that link dangling and the rule reruns, instead of trusting a +# manifest that outlived its data. star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache # The run index. Design intent (finding 15) is durable products on /project — diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index da3e49e4f..ee34f639a 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -47,9 +47,19 @@ rule exp_get_images: # # A LOCALRULE (declared in the Snakefile), so it runs in the head process — an # sbatch job on a compute node, and nibi compute nodes have internet (verified by -# curl to vizier.cds.unistra.fr from a compute job). Local execution serialises -# the queries, which throttles CDS at any campaign scale for free, and it spares -# the scheduler one submission per exposure for six seconds of work. +# curl to vizier.cds.unistra.fr from a compute job). Local execution does NOT +# serialise the queries — an earlier version of this comment claimed it did, and +# that was false. In cluster mode snakemake runs local jobs in a worker pool +# sized by `--local-cores`, and this rule takes one thread each, so as many CDS +# queries (and apptainer execs) run at once as that pool is wide. The nibi +# profile leaves --local-cores unset, so it is the head process's CPU count — +# 5 in the measured allocation. +# +# The localrule stays anyway. 5-way concurrency against Vizier is ordinary +# politeness, not abuse, and the alternative is one sbatch per exposure: ~20k +# submissions for six seconds of work each, which is exactly the sub-15-minute +# scheduler churn cluster policy asks us to bundle away. If CDS ever objects, +# --local-cores is the knob — it caps these queries directly. # # The CATALOGUE is cached run-independently under config `star_cats`, keyed by # exposure. create_star_cat.py skips a cat that exists, so retries, reruns and @@ -61,11 +71,23 @@ rule exp_get_images: # symlink to a shared whole-store pool contributes every other exposure's numbers # and the intersection is empty ("numbers ... do not intersect", live). # -# Declared output is a manifest rather than a symlink or a directory(): it is -# written last, it is the one per-exposure file that is unique to this rule, it -# records what the farm points at, and it keeps the "one rule, one manifest" -# currency of every other rule — including being deleted by clean_exposure, so a -# reclaimed exposure rebuilds its farm from the cache at no query cost. +# TWO declared outputs, and the second one is the point. +# +# The manifest keeps the "one rule, one manifest" currency of every other rule: +# written last, unique to this rule, a record of what the farm points at, and +# deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the +# cache at no query cost. +# +# But a manifest attests FOREVER, and the two things it attests to both live +# outside the unit's manifests/ dir: the cache FITS on /scratch (60-day purge) +# and the farm itself. Either can vanish under a manifest that still says +# "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is +# declared too — one link stands for all 40, they are created by the same loop +# in the same instant, and declaring 40 buys nothing. Snakemake's existence test +# is os.path.exists, which FOLLOWS symlinks and is therefore False for a link +# whose target the purge removed. A purged cache or a deleted farm makes the +# rule out of date, it reruns, and it re-queries or re-links as needed — the +# recovery config.yaml's star_cats comment promises, now actually wired up. # The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE # pointing at a path that does not exist inside the image, so requests is pointed @@ -89,6 +111,14 @@ def star_cat_cmd(exp): }, indent=2, sort_keys=True) return "\n".join([ "set -euo pipefail", + # LEGACY-SYMLINK HAZARD. Unit dirs built before this rule existed carry + # star_cat_exp as a SYMLINK into the old shared star-cat pool. `mkdir -p` + # is a no-op on an existing symlink-to-directory, so the 40-link loop + # below followed it and wrote this exposure's links INTO THE SHARED POOL + # (520 stray links found live). Replace the link — never `rm -rf` it, + # which would recurse into the pool, and never touch a real directory: + # a real farm is this rule's own output and `ln -sfn` refreshes it. + f"[ -L '{farm}' ] && rm -f '{farm}' || true", f"mkdir -p '{cache}' '{farm}' '{work}/manifests'", # Same binds and isolation as the profile's apptainer-args; the SDM does # not wrap this rule (container: None below), because the farm loop and @@ -118,9 +148,14 @@ rule exp_star_cat: input: rules.exp_get_images.output.manifest output: - manifest = f"{EXP_DIR}/manifests/exp_star_cat.json" + manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", + # The sentinel: ccd-0 of the 40-link farm (see above). + link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" params: - cmd = lambda wc: star_cat_cmd(wc.exp) + cmd = lambda wc: star_cat_cmd(wc.exp), + # create_star_cat.py is external to the shell string, so the `code` + # rerun-trigger does not see it — same reason SCRIPT_HASH exists. + script_hash = STAR_CAT_HASH # No SDM wrapping: the rule calls apptainer itself, because only the query # runs in the container (bin/sp has loaded the apptainer module). container: From cba14de8b9bd828c6f3d1ff56a6f16c9ed0e7c31 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 20:08:56 -0400 Subject: [PATCH 029/100] fix(orchestration): reclaim the star-cat farm, stop the home-quota leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clean_exposure targeted only output/ and manifests/, so the star_cat_exp link farm outlived the exp_star_cat manifest that owned it — 40 symlinks belonging to no rule. The farm and the legacy star_cat_tiles link are now targets too. This is consistency, not bytes: the catalogue stays in the run-independent cache, so rebuilding the farm is a relink and no query. Deletion is symlink-safe. Legacy unit dirs carry star_cat_exp as a link into the shared pool, and rmtree would recurse through it and empty that pool for every other exposure in the campaign. A symlink target is unlinked; only a real directory is rmtree'd. Existence is tested as `is_symlink() or exists()`, because exists() follows the link and would skip a dangling legacy one. Separately, astroquery pickled every Vizier response into $HOME/.astropy/cache/astroquery/Vizier at ~2 MB a query (10 queries, 10 pickles, observed live). The workflow already caches the result as a FITS on scratch and skips the query on a hit, so the pickle is a duplicate — and at campaign scale (~25k exposures) it is ~50 GB against a 50 GB home quota. query_vizier now passes cache=False. Co-Authored-By: Claude Fable 5 --- src/shapepipe/utilities/vizier.py | 10 +++++++++- workflow/scripts/clean_exposure.py | 28 ++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py index 1f4fd3472..0bf077a78 100644 --- a/src/shapepipe/utilities/vizier.py +++ b/src/shapepipe/utilities/vizier.py @@ -64,8 +64,16 @@ def query_vizier(ra, dec, radius_arcmin, cat_id): v = Vizier( row_limit=-1, timeout=timeout, vizier_server=server ) + # cache=False: astroquery otherwise pickles every HTTP response into + # $HOME/.astropy/cache/astroquery/Vizier, ~2 MB per query. The + # workflow already caches the RESULT as a FITS catalogue on scratch + # and skips the query when it hits, so the pickle is pure duplicate — + # and at campaign scale (~25k exposures) it is ~50 GB against a + # 50 GB home quota. Home is for source and config, not for a second + # copy of the survey. result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id + coord, radius=radius_arcmin * u.arcmin, catalog=cat_id, + cache=False, ) if len(result) > 0: print( diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index ce6fef80a..e8b575469 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,8 +7,19 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi) AND its ``manifests/``. Deleting the manifests is -deliberate and load-bearing, not tidiness: +run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/``, and its star-catalogue link +farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are +reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted +here like every other, so the exposure's chain must read as unbuilt, and 40 +symlinks left behind are a farm no rule now owns. The catalogue itself lives in +the run-independent cache, so rebuilding the farm costs a relink and no query. + +Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not +``rmtree``d. Legacy unit dirs carry ``star_cat_exp`` as a link into the old +shared pool, and an rmtree would recurse through it and delete the shared cache +for every other exposure in the campaign. + +Deleting the manifests is deliberate and load-bearing, not tidiness: * the manifests are the exposure rules' DECLARED outputs. If they survived, a tile appended later would find the exposure chain "up to date" and run @@ -78,7 +89,11 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - targets = [t for t in (args.exp_dir / "output", mdir) if t.exists()] + # is_symlink() first, and OR'd with exists(): exists() follows the link, so a + # dangling legacy star_cat_exp would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, + args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") + targets = [t for t in candidates if t.is_symlink() or t.exists()] # Tombstone first, complete, fsync'd — then delete. See the module docstring: # the crash window has to sit where the data still exists, not where the @@ -96,7 +111,12 @@ def main() -> None: removed = [] for target in targets: - shutil.rmtree(target) + # NEVER rmtree a symlink: star_cat_exp is a link into the shared pool in + # legacy unit dirs, and rmtree would follow it and empty that pool. + if target.is_symlink(): + target.unlink() + else: + shutil.rmtree(target) removed.append(str(target)) print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " f"{len(consumers)} consuming tile(s)") From 30567ac17ef7db2875b19484105d7d30418c0230 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 24 Aug 2026 16:55:22 -0400 Subject: [PATCH 030/100] =?UTF-8?q?run(smk-g3):=20flip=20clean=20on=20?= =?UTF-8?q?=E2=80=94=20quad=20validated,=20batch1=20append=20needs=20the?= =?UTF-8?q?=20room?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quad is proven equivalent to the bash baseline (validation verdict p0night1), so its exposure stores' inspection value is spent. The campaign now scales into the p3-batch1 tiles: 181 exposures un-cleaned would be ~1.2 TB against the 1 TiB scratch quota, so rolling reclamation (D5) is load-bearing from here on. Dry-run: exactly 18 clean_exposure jobs, no recompute. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014jAmAvmxJ3UCwYQrpHghkb --- workflow/config.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index 55962297e..3f7deab87 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -53,12 +53,14 @@ index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, # and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. Off here -# because P0 is a 4-tile debugging run where the exposure store is exactly what -# you want to inspect; on for any run big enough to care about disk. Flipping it -# on later reclaims retroactively — the missing tombstones schedule exactly the -# outstanding clean jobs. -clean: false +# back out of the tombstone and reports the exposure as `cleaned`. On here: +# the quad is validated against the bash baseline (validation verdict +# p0night1), so its exposure stores' inspection value is spent, and the +# campaign is scaling into the p3-batch1 tiles — 181 exposures un-cleaned +# would be ~1.2 TB against a 1 TiB scratch quota, so rolling reclamation is +# load-bearing from here on. Flipping it on reclaims retroactively — the +# missing tombstones schedule exactly the outstanding clean jobs. +clean: true # Tiles that may NOT pin an exposure store (default: empty). # From fadc27739af81e6249ce61452987268d19b67e8e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 24 Aug 2026 16:59:03 -0400 Subject: [PATCH 031/100] run(smk-g3): the campaign appends the first 30 batch1 tiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tiles4.txt grows from 4 to 34 (parse-time config; nothing completed is invalidated). 30 of p3-batch1's 50 tiles now, not all: 30 tiles dedup to 100 unique exposures — worst-case un-cleaned high-water ~660G against ~840G free scratch — while 40 would hit ~1.1T. The remaining 20 are the next append, exercising append-at-scale a second time with the first wave's exposures already reclaimed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014jAmAvmxJ3UCwYQrpHghkb --- workflow/config.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index 3f7deab87..94edd70c9 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,10 +5,14 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -# 186/187 quad: 18 unique exposures with cross-tile reuse — exercises the -# structural exposure dedup. (The 210/211 quad used earlier is unusable: its -# tile images are symlinks into anaennis' moved processed_tiles tree. ~8.5k of -# the 10.3k staged tiles are such broken links; ~600 tiles are real files.) +# The campaign grows by appending to this file — parse-time config, so +# completed work is never invalidated. Current contents: the 186/187 quad +# (18 exposures, complete and validated against the bash baseline), plus the +# first 30 tiles of the p3-batch1 list (100 unique exposures — chosen so the +# worst-case un-cleaned exposure high-water ~660G fits the scratch quota with +# margin; the remaining 20 batch1 tiles are the next append). (The 210/211 +# quad used earlier is unusable: its tile images are symlinks into anaennis' +# moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are broken.) tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). From 93c7b4e059b5fd2221ceca3b68b893d22ae86346 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 28 Aug 2026 09:54:37 -0400 Subject: [PATCH 032/100] Move failure evidence onto snakemake's native log: directive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each shapepipe_run rule now declares `log: /logs/.json`. `completeness.py check` writes its full verdict there on EVERY run — counts against floors, per-runner detail, scraped reasons, job_rc when nonzero — and writes the declared `manifests/.json` only when that verdict is a success. Snakemake deletes a failed job's declared output natively and never touches its log, so the two directives say exactly the two things the workflow needs, and the success/failure fork with its unlink of the stale counterpart is gone (`failed_path` deleted). Layout: `logs/` is a sibling of `manifests/`, and clean_exposure reclaims both. A log attesting "complete" for a reclaimed store would contradict the unbuilt chain the DAG must then see, and for a successful stage its content is byte-identical to the manifest, so the tombstone keeps absorbing manifests only. The benchmark tsv stays outside both dirs — it measures the job, it makes no claim about the store. run_report reads both dirs, worst status wins per (unit, stage): the manifest for success, the log for failure, and legacy `.failed.json` still parses as failed. Records without a `stage` key are skipped, so a stray JSON deeper in a unit tree is inert. tile_make_cat gains `--job-rc`, which it was missing: without it a job whose counts clear their floors but whose shapepipe_run died writes a "complete" log for a manifest snakemake is about to delete. exp_star_cat gets no log — it computes no verdict. The one guarantee this trades away is stated at the definition site: removing a previous success's manifest is now snakemake's job, so a head process SIGKILLed between a job's failure and that deletion leaves a stale success-named manifest. rerun-incomplete covers the DAG; the report's worst-status-wins covers the tables. Verified: `--lint` parses (the only remaining log-directive lints are the three rules deliberately without one); a `--dry-run` against the live state dir schedules 579 jobs, byte-identical in count and reasons to the same dry run with these changes stashed, so nothing here enlarges the pending rerun set; unit tests of the check (byte-stable manifest across a no-op rerun, log-only on count shortfall, log-only on nonzero job-rc), of run_report against a fixture mixing all three record conventions, and of clean_exposure reclaiming logs/ and still reporting the exposure as cleaned. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- profiles/nibi/config.yaml | 14 +++--- workflow/README.md | 32 ++++++------ workflow/Snakefile | 49 +++++++++++------- workflow/rules/exposure.smk | 12 +++++ workflow/rules/prepare.smk | 6 +++ workflow/rules/tile.smk | 23 +++++++-- workflow/scripts/clean_exposure.py | 27 ++++++---- workflow/scripts/completeness.py | 81 ++++++++++++++++-------------- workflow/scripts/run_report.py | 79 +++++++++++++++++------------ 9 files changed, 200 insertions(+), 123 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index f78bea7f7..8bc71a9d1 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -46,13 +46,13 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true -# No `keep-incomplete` here (it was the pre-S7a way to preserve failed manifests -# as report evidence): the declared output IS the manifest, so letting snakemake -# delete a failed job's output is exactly the semantics this workflow wants — a -# `.json` on disk then means the stage succeeded, and a resume cannot -# schedule downstream work on top of a failure. The post-mortem evidence lives in -# `.failed.json`, which completeness.py writes on failure and which is -# never a declared output, so snakemake never touches it. +# No `keep-incomplete` here. The declared output IS the manifest, so letting +# snakemake delete a failed job's output is exactly the semantics this workflow +# wants: a `.json` on disk means the stage succeeded, and a resume cannot +# schedule downstream work on top of a failure. The post-mortem evidence is the +# rule's `log:` (`/logs/.json`), which completeness.py writes on +# every run and which snakemake preserves through the failure that produced it — +# which is also what `show-failed-logs` above surfaces on the console. # rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still # propagate — completeness.py writes the manifest only on change, so mtimes move diff --git a/workflow/README.md b/workflow/README.md index c286ebdcb..1d219617e 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -102,7 +102,7 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) - clean_exposure.py ONE exposure's store + manifests -> tombstone (the clean_exposure rule) + clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` @@ -114,15 +114,16 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. - **Manifests are the DAG's currency, and they are success-only.** - `completeness.py check` writes `.json` when the stage passed its count - floor, and `.failed.json` when it did not — removing the other file - either way. So `.json` on disk means "this stage succeeded", and a + `completeness.py check` writes its full verdict — per-runner counts against + floors, scraped failure reasons, the `shapepipe_run` exit status when nonzero + — to the rule's `log:` (`/logs/.json`) on *every* run, and + additionally to the declared `.json` manifest only when that verdict is + a success. So `.json` on disk means "this stage succeeded", and a resume after an unclean death cannot schedule downstream work on top of a - failure. The failed manifest carries the same content plus the scraped - reasons; it is never a declared output, so Snakemake never tracks or deletes - it, and `sp report` reads it when the success manifest is absent. The profile - runs *without* `keep-incomplete` for the same reason: deleting a failed job's - declared output is the wanted semantics. + failure. Nothing unlinks anything: Snakemake deletes a failed job's declared + output natively and never touches its log, which is why the profile runs + *without* `keep-incomplete`. `sp report` reads both dirs — the manifest for + success, the log for failure — and a unit with neither ran nothing. - **Completeness is a count floor, not a taxonomy.** After a run, `sp_rule.py` counts products per mandatory runner against `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition @@ -151,8 +152,9 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee Reclamation is the in-DAG `clean_exposure` rule instead: one job per exposure, taking every consuming tile's `tile_vignets` manifest as input (the campaign-wide consumer set comes from the accumulating index), which - deletes the store *and* the exposure's manifests and leaves a `cleaned.json` - tombstone. Deleting the manifests is what makes a late append correct: the + deletes the store *and* the exposure's `manifests/` and `logs/`, and leaves a + `cleaned.json` tombstone. Deleting the manifests is what makes a late append + correct: the appended tile finds an unbuilt chain and regenerates it. The `clean:` flag in `config.yaml` gates it; flipping it on later reclaims retroactively, since the missing tombstones schedule exactly the outstanding clean jobs. @@ -175,9 +177,11 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee its exposure chains are gone and rebuild from scratch. - **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its - per-runner counts and blocks no tile. The `exp_psf` benchmark tsv lives - beside `manifests/`, not inside it, so reclamation does not eat the - memory-sizing data. + per-runner counts and blocks no tile. The logs go with the manifests — a log + claiming `complete` for a store that is gone would contradict the unbuilt + chain the DAG must now see, and its content duplicates the manifest anyway. + The `exp_psf` benchmark tsv lives beside both dirs, not inside either, so + reclamation does not eat the memory-sizing data. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs per-CCD attrition). It runs standalone — a DAG report node would itself be diff --git a/workflow/Snakefile b/workflow/Snakefile index 1c8222271..3b18fd1b8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -20,6 +20,13 @@ The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared output is its MANIFEST (`/manifests/.json`), written by `completeness.py check`. Product files are not declared — a missing CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. + +Failure evidence rides on the rule's `log:` (`/logs/.json`, +`unit_log` below): the check writes its full verdict there on every run, and the +manifest only when that verdict is a success. Snakemake deletes a failed job's +declared output and never touches its log, so the two directives say exactly the +two things this workflow needs — "this stage succeeded" and "here is what +happened". """ import hashlib @@ -240,8 +247,8 @@ CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) def tombstone(exp): - """The clean_exposure output. Lives BESIDE manifests/, not inside it: the - clean job deletes manifests/ wholesale, and `sp report` scans it.""" + """The clean_exposure output. Lives BESIDE manifests/ and logs/, not inside + either: the clean job deletes both wholesale, and `sp report` scans it.""" return f"{exp_dir(exp)}/cleaned.json" @@ -305,7 +312,12 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT -e/--exclusive): - * ``output/`` and ``manifests/``; + * ``output/``, ``manifests/`` and ``logs/`` — the last two are a SIBLING + pair, not one dir with two naming conventions: clean_exposure deletes + BOTH wholesale, because a log attesting "complete" for a store that has + been reclaimed contradicts the unbuilt chain the DAG must now see. (The + benchmark tsv is deliberately outside both: it measures the job, it does + not claim anything about the store, so reclamation must not eat it.) * tile: ``tile_numbers.txt`` (dot format — what get_images reads); * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the ORIGINAL exposure name from the index (``2605805p``), so get_images @@ -330,7 +342,7 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, # Also set via apptainer-args in the profile; kept here so a hand-run of # this same line outside snakemake behaves identically. f"export {_THREAD_CAPS}", - 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests"', + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', ] if forest: lines.append(f"export SP_EXP='{forest}'") @@ -352,33 +364,32 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, def sp_shell(stage, config_name): """The rule's shell string: prologue, one shapepipe_run, one completeness check. - ``{threads}`` and ``{output}`` are placeholders HERE and nowhere else (see - unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number - by construction (D4). + ``{threads}``, ``{output}`` and ``{log}`` are placeholders HERE and nowhere + else (see unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task + one number by construction (D4). The check runs even when shapepipe_run failed, so a failed unit still leaves - a record of why — but that record goes to `.failed.json`, and the - declared `.json` is removed if it was there. Success manifests are - success-only currency (completeness.py); the failed one is evidence for - `sp report` and is never a declared output. + a record of why: the verdict goes to the rule's `log:` on EVERY run, and to + the declared `.json` manifest only when it is a success. Snakemake + deletes a failed job's output and never touches its log, so the manifest + stays success-only currency and the log is the evidence `sp report` reads. The verdict is COMPOSED, which is why `--job-rc "$rc"` is passed: the count floors and shapepipe_run's exit status are two independent statements about - the same job, and the manifest must reflect both. Without it, a job whose + the same job, and the record must reflect both. Without it, a job whose counts cleared their floors but whose shapepipe_run died (a late runner - raising after the counted ones wrote their files) wrote the SUCCESS manifest - and still exited nonzero — snakemake then deleted that manifest as a failed - job's output, no failed.json was written (the check thought it had won), and - any prior failed.json had already been unlinked. The failure left no trace at - all. With the rc composed in, the fork lands on the failure side and the - evidence survives. + raising after the counted ones wrote their files) publishes a SUCCESS + manifest and still exits nonzero — snakemake deletes that manifest as a + failed job's output, and the log is left claiming "complete" for a stage + with nothing to show for it. With the rc composed in, both files agree that + the job failed. """ return ( "{params.pre}\n" "rc=0\n" f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" - " --job-rc \"$rc\" || rc=1\n" + " --log {log} --job-rc \"$rc\" || rc=1\n" "exit $rc\n" ) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index ee34f639a..f56653e6c 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -25,6 +25,8 @@ whole-exposure hard failure. That is now a property of the committed configs rule exp_get_images: output: manifest = f"{EXP_DIR}/manifests/exp_get_images.json" + log: + f"{EXP_DIR}/logs/exp_get_images.json" params: pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, exp_name=EXP[wc.exp]), @@ -151,6 +153,10 @@ rule exp_star_cat: manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", # The sentinel: ccd-0 of the 40-link farm (see above). link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" + # No `log:`. Every other rule's log carries a completeness VERDICT, and this + # rule computes none: it is not a shapepipe_run, it has no count floor, and + # under `set -euo pipefail` it either completes or aborts at the failing + # step. Snakemake's own captured job stderr is the evidence for that. params: cmd = lambda wc: star_cat_cmd(wc.exp), # create_star_cat.py is external to the shell string, so the `code` @@ -175,6 +181,8 @@ rule exp_split: rules.exp_get_images.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_split.json" + log: + f"{EXP_DIR}/logs/exp_split.json" params: pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), script_hash = SCRIPT_HASH @@ -193,6 +201,8 @@ rule exp_mask: rules.exp_star_cat.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_mask.json" + log: + f"{EXP_DIR}/logs/exp_mask.json" params: pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), script_hash = SCRIPT_HASH @@ -211,6 +221,8 @@ rule exp_psf: rules.exp_mask.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_psf.json" + log: + f"{EXP_DIR}/logs/exp_psf.json" params: pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), script_hash = SCRIPT_HASH diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 4b1e401a6..d4a8d1cdd 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -23,6 +23,8 @@ has no star-cat node because it has no mask rule yet — see tile.smk. rule tile_get_images: output: manifest = f"{TILE_DIR}/manifests/tile_get_images.json" + log: + f"{TILE_DIR}/logs/tile_get_images.json" params: pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -39,6 +41,8 @@ rule tile_uncompress: rules.tile_get_images.output.manifest output: manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" + log: + f"{TILE_DIR}/logs/tile_uncompress.json" params: pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -54,6 +58,8 @@ rule tile_find_exposures: rules.tile_uncompress.output.manifest output: manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" + log: + f"{TILE_DIR}/logs/tile_find_exposures.json" params: pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), script_hash = SCRIPT_HASH diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d52b76e0a..5f8dfacd0 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -140,6 +140,8 @@ rule tile_merge_headers: fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" + log: + f"{TILE_DIR}/logs/tile_merge_headers.json" params: pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, forest=forest_dir(wc.tile)), @@ -158,6 +160,8 @@ rule tile_detect: mh = rules.tile_merge_headers.output.manifest, output: manifest = f"{TILE_DIR}/manifests/tile_detect.json" + log: + f"{TILE_DIR}/logs/tile_detect.json" params: pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -190,6 +194,8 @@ rule tile_vignets: output: manifest = f"{TILE_DIR}/manifests/tile_vignets.json", store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), + log: + f"{TILE_DIR}/logs/tile_vignets.json" params: pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, forest=forest_dir(wc.tile)), @@ -218,6 +224,8 @@ rule tile_ngmix: manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", # temp(directory()) for the same reason as the vignette store above. chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), + log: + f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" params: pre = lambda wc: unit_pre( "tile_ngmix", "tile", wc.tile, @@ -258,6 +266,8 @@ rule tile_merge_cats: chunkdirs = ngmix_chunkdirs, output: manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" + log: + f"{TILE_DIR}/logs/tile_merge_cats.json" params: pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), @@ -281,6 +291,8 @@ rule tile_make_cat: output: manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", + log: + f"{TILE_DIR}/logs/tile_make_cat.json" params: pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -289,12 +301,17 @@ rule tile_make_cat: mem_mb = lambda wc, attempt: 16000 * attempt, runtime = 120 shell: - # Publish the catalogue next to the manifest: a real file, so it is a - # real declared output (and it persists — never temp()). + # sp_shell's body, plus the catalogue publish: a real file, so it is a + # real declared output (and it persists — never temp()). `--job-rc` is + # composed in for the same reason it is everywhere else (see sp_shell) — + # without it a job whose counts cleared their floors but whose + # shapepipe_run died writes a "complete" log for a manifest snakemake is + # about to delete. "{params.pre}\n" "rc=0\n" 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' - f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}} || rc=1\n" + f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}}" + ' --log {log} --job-rc "$rc" || rc=1\n' "if [ $rc -eq 0 ]; then\n" ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' ' | head -1)" {output.final_cat}\n' diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index e8b575469..e65c0dbcf 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,8 +7,8 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/``, and its star-catalogue link -farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are +run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its +star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted here like every other, so the exposure's chain must read as unbuilt, and 40 symlinks left behind are a farm no rule now owns. The catalogue itself lives in @@ -31,16 +31,25 @@ needs to run, so tiles already finished are NOT rerun by their exposures' manifests vanishing. +``logs/`` goes with them, and for the same reason rather than for bytes. Each +log holds the completeness verdict of one stage, written on every run and kept by +snakemake through failures; a log left behind would attest "complete" for a store +that is no longer there, contradicting the unbuilt chain the DAG must now see. +Its content for a successful stage is byte-identical to the manifest beside it, +so absorbing the logs into the tombstone would duplicate what the manifests +already carry — they are deleted, not copied. + Nothing is lost to the report: every ``manifests/*.json`` is copied verbatim into the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned exposure's record out of the tombstone — it reports the unit as ``cleaned``, warn counts and shortfalls intact, instead of "not run". The absorption is by GLOB, so it takes whatever is in ``manifests/``, keyed by -file stem; the report re-keys on each manifest's own ``stage`` field. A -``.failed.json`` is therefore carried through unremarkably — it should -never be there (an exposure with a failed stage has no complete vignets consumer -and so is not eligible for cleaning), but it costs nothing to be right about. +file stem; the report re-keys on each manifest's own ``stage`` field. A legacy +``.failed.json`` from the pre-``log:`` convention is therefore carried +through unremarkably — it should never be there (an exposure with a failed stage +has no complete vignets consumer and so is not eligible for cleaning), but it +costs nothing to be right about. Order matters, and it is the reverse of the obvious one: the tombstone is written FIRST, complete, and only then is anything deleted. A crash between the @@ -50,8 +59,8 @@ replaces them was never written, and the report would be blind to that exposure forever. -The exp_psf benchmark tsv lives beside ``manifests/``, not inside it, so it -survives this job — it is the measured-memory feed for resource sizing (D4). +The exp_psf benchmark tsv lives beside ``manifests/`` and ``logs/``, not inside +either, so it survives this job — it is the measured-memory feed for resource sizing (D4). The tombstone records the consumer set it was cleaned against. The rule carries that same set as a ``params`` value, so when the index grows a new consumer the @@ -91,7 +100,7 @@ def main() -> None: # is_symlink() first, and OR'd with exists(): exists() follows the link, so a # dangling legacy star_cat_exp would otherwise be skipped and survive. - candidates = (args.exp_dir / "output", mdir, + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs", args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") targets = [t for t in candidates if t.is_symlink() or t.exists()] diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 138399af5..58c1ee9eb 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -10,33 +10,41 @@ This file is also the ``check`` CLI — the second half of every rule's shell line (PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing -onto it, so the check runs — and the manifest is written — even when +onto it, so the check runs — and the verdict is recorded — even when ``shapepipe_run`` failed:: rc=0 shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} --job-rc "$rc" || rc=1 + completeness.py check exp_mask {output} --log {log} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory runner is below its floor OR ``--job-rc`` is nonzero — the verdict is COMPOSED of the counts and shapepipe_run's own exit status, because a runner can raise after -the counted ones have written their files. WHERE it writes the record is the -whole point: - - * success -> ``.json`` (the rule's declared output, the DAG's currency) - and any stale ``.failed.json`` is removed; - * failure -> ``.failed.json`` (same content, ``status: failed``) and any - stale ``.json`` is REMOVED. - -``.json`` therefore means "this stage succeeded", full stop — a resume can -never schedule a downstream stage on top of a failed one, not even after a head -process died without recording the failure. The failed manifest is never a -declared output of any rule: it is post-mortem evidence for ``run_report.py``, -which reads it when the success manifest is absent, and snakemake neither tracks -nor deletes it. - -Manifests carry no wall-clock, and either file is rewritten ONLY when its content +the counted ones have written their files. + +WHERE it writes the verdict is the whole point, and it is two files with two +different jobs: + + * the LOG (``--log``, the rule's snakemake ``log:``) gets the full verdict on + EVERY run, success or failure — counts against floors, per-runner detail, + scraped failure reasons, ``job_rc`` when nonzero. Snakemake never deletes a + log file, so it survives the failed job that wrote it and is the post-mortem + evidence ``run_report.py`` reads. + * the MANIFEST (the rule's declared ``output:``) gets that same verdict ONLY + when it is a success. Snakemake deletes a failed job's declared output + natively, so nothing here has to unlink anything. + +``.json`` therefore means "this stage succeeded": a resume cannot schedule +a downstream stage on top of a failed one. The removal of a PREVIOUS success's +manifest is snakemake's to do, not this script's, and the one gap that leaves is +a head process SIGKILLed between the job's failure and that deletion — a +success-named manifest then outlives the failure it no longer describes. The +profile's ``rerun-incomplete`` covers the DAG side, and ``run_report.py`` takes +the WORST status across a stage's log and manifest, so the report is right even +in that window. + +Neither file carries wall-clock, and each is rewritten ONLY when its content changes — identical on-disk state must leave a byte-identical manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on every unrelated ``--forcerun``. @@ -266,11 +274,6 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok -def failed_path(manifest: Path) -> Path: - """``.json`` -> ``.failed.json``, in the same manifests/ dir.""" - return manifest.with_name(manifest.name[:-len(".json")] + ".failed.json") - - def write_if_changed(path: Path, text: str) -> None: """Write only when the bytes differ — see the module docstring on mtime.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -295,6 +298,9 @@ def main(argv=None) -> int: c = sub.add_parser("check", help="count products, write the manifest") c.add_argument("stage") c.add_argument("manifest", type=Path) + c.add_argument("--log", type=Path, required=True, + help="the rule's log: path — the verdict is written here every " + "run, success or failure") c.add_argument("--run-dir", type=Path, default=None, help="the unit's $SP_RUN (default: the env var)") c.add_argument("--unit", default=None, @@ -317,11 +323,10 @@ def main(argv=None) -> int: # The verdict is COMPOSED of two independent statements: the count floors # (above) and shapepipe_run's own exit status (here). Counts alone are not # enough — a runner can raise AFTER the counted runners have written their - # files, so the floors are met while the job died. That combination used to - # write the SUCCESS manifest, which snakemake then deleted as a failed job's - # output, leaving no manifest at all: failure invisible, and any earlier - # failed.json already unlinked by the success branch. An rc-driven override - # puts the fork on the failure side, so failed.json is written and kept. + # files, so the floors are met while the job died. Without this, such a job + # publishes a SUCCESS manifest that snakemake then deletes as a failed job's + # output — and the log would claim "complete" for a stage with no manifest, + # which reads as a bookkeeping bug rather than as the failure it is. # # Recorded only when nonzero, which keeps every existing success manifest # byte-identical (the mtime rerun-trigger reads those bytes). @@ -338,22 +343,22 @@ def main(argv=None) -> int: text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" - # The success/failure fork. Exactly one of the two files exists afterwards, - # so the presence of .json IS the statement "this stage succeeded" and - # the DAG can never build on top of a failure. The removal of the stale - # counterpart is what makes a success->failure or failure->success transition - # complete rather than additive. - written = args.manifest if ok else failed_path(args.manifest) - stale = failed_path(args.manifest) if ok else args.manifest - write_if_changed(written, text) - stale.unlink(missing_ok=True) + # The log ALWAYS gets the verdict; the manifest gets it only on success. No + # unlink of anything: snakemake deletes a failed job's declared output, so + # the presence of .json IS the statement "this stage succeeded" and + # the DAG can never build on top of a failure. A failure->success transition + # publishes the manifest; a success->failure has the manifest removed for us, + # and the log is overwritten with the new verdict either way. + write_if_changed(args.log, text) + if ok: + write_if_changed(args.manifest, text) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} print(f"[completeness] {runner}: {r['found']}/{r['expect']} " f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) print(f"[completeness] {args.stage} {unit}: {manifest['status']} " - f"-> {written}", file=sys.stderr) + f"-> {args.log}" + (f" + {args.manifest}" if ok else ""), file=sys.stderr) for f in manifest["failures"]: for reason in f["reasons"]: print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 69c2e4b5f..22201f7e7 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -12,19 +12,21 @@ * the **index** (``run_index.sqlite``) — the units the run declared, and the tile->exposure edges that let an exposure failure be blamed on the tiles it blocks; - * the **manifests** — one per unit per stage, written by - ``completeness.py check``, carrying per-runner found/expect/floor and - log-scraped failure reasons. - -A stage that failed leaves ``.failed.json`` instead of ``.json`` -(``completeness.py``): the success name is reserved for success, so the DAG -cannot build on a failure. Both names are read here, and the status comes from -the manifest BODY, so a stage is reported as failed either way — including -manifests written before that split, where the failure content sat under the -success name. - -A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after -copying them into ``/cleaned.json``. That tombstone is read as the + * the **verdicts** written by ``completeness.py check`` — per-runner + found/expect/floor and log-scraped failure reasons — which land in two places + with two different lifetimes. Every run writes the unit's ``logs/.json`` + (the rule's snakemake ``log:``, which snakemake never deletes); a run whose + verdict is a success ADDITIONALLY writes ``manifests/.json``, the + rule's declared output, which snakemake deletes when the job fails. + +Both dirs are read here and the status comes from the file BODY, so a failed +stage speaks through its log while a successful one is corroborated by two +identical files. A unit with neither ran nothing. Also read, for continuity with +stores written before this convention: ``manifests/.failed.json``, which +is where failure evidence used to go. + +A reclaimed exposure has neither dir: ``clean_exposure`` deleted both, after +copying the manifests into ``/cleaned.json``. That tombstone is read as the unit's record and the unit is reported as **cleaned** — not "not run", and it blocks no tile. @@ -32,9 +34,9 @@ the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". -Manifests are discovered by glob (``tiles/**/manifests/*.json``), not by -constructed path: the unit stores are sharded (``tiles///``) and the -sharding depth is not this script's business. +Records are discovered by glob (``tiles/**/manifests/*.json`` and +``tiles/**/logs/*.json``), not by constructed path: the unit stores are sharded +(``tiles///``) and the sharding depth is not this script's business. """ import argparse @@ -55,31 +57,41 @@ def load_manifests(run_dir: Path, sub: str) -> dict: - """``{unit: {stage: manifest}}`` for one store (``tiles`` or ``exp``). + """``{unit: {stage: verdict}}`` for one store (``tiles`` or ``exp``). - The unit key is the manifest dir's *parent directory name* — shard-depth - agnostic, and the only form that joins to the index (the manifest's own + Reads BOTH of a unit's record dirs: ``manifests/`` (the rules' declared + outputs, success-only) and ``logs/`` (the rules' ``log:``, written every run + and never deleted by snakemake, so this is where a failure survives). + + The unit key is the record dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the record's own ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is - not the index's ``210.282``). The stage comes from the manifest body, never - the filename: ngmix chunks share a stage under per-chunk filenames, and - ``.failed.json`` names the same stage as ``.json``. - - Several files can therefore map to one (unit, stage), and the WORST status - wins. That is what collapses the ngmix chunks to one entry, and what makes a - failed manifest speak even in the transient window where its success-named - counterpart has not yet been removed. + not the index's ``210.282``). The stage comes from the body, never the + filename: ngmix chunks share a stage under per-chunk filenames, and a log + names the same stage as the manifest beside it. + + Several files therefore map to one (unit, stage), and the WORST status wins. + That is what collapses the ngmix chunks to one entry, and it is why a + successful stage's two byte-identical records cost nothing while a failure + always speaks. A body with no ``stage`` field is skipped: it is not one of + ours, which is what keeps a stray JSON deeper in the tree inert. """ out: dict = defaultdict(dict) - for path in sorted((run_dir / sub).glob("**/manifests/*.json")): + paths = sorted((run_dir / sub).glob("**/manifests/*.json")) \ + + sorted((run_dir / sub).glob("**/logs/*.json")) + for path in paths: try: m = json.loads(path.read_text()) except (OSError, json.JSONDecodeError) as exc: - print(f"[run_report] unreadable manifest {path}: {exc}", file=sys.stderr) + print(f"[run_report] unreadable record {path}: {exc}", file=sys.stderr) + continue + if not isinstance(m, dict) or "stage" not in m: continue unit = path.parent.parent.name - stage = m.get("stage", path.stem) + stage = m["stage"] prev = out[unit].get(stage) - # Worst status wins when several manifests share a stage (ngmix chunks). + # Worst status wins when several records share a stage (ngmix chunks; + # a stage's manifest and its identical log). rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 if prev is None or rank(m) > rank(prev): out[unit][stage] = m @@ -89,8 +101,9 @@ def load_manifests(run_dir: Path, sub: str) -> dict: def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: """Fill in reclaimed units from their ``cleaned.json``; return their ids. - A cleaned exposure has NO ``manifests/`` — ``clean_exposure`` deleted it, - after copying every manifest verbatim into the tombstone. Read them back, or + A cleaned exposure has neither ``manifests/`` nor ``logs/`` — + ``clean_exposure`` deleted both, after copying every manifest verbatim into + the tombstone (the logs duplicate them). Read them back, or the report inverts the truth exactly when reclamation works: the exposure shows as "not run" and blocks the very tiles whose completion authorised the deletion. From b2cfa3bd62121128cf783389ce8a9a4dfbade54d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 10:02:06 -0400 Subject: [PATCH 033/100] Fetch star catalogues by sky chunk, not by exposure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mask star catalogue's network cost was a function of the exposure count: one GSC 2.3 cone per exposure, ~25k queries at DR6 scale, re-fetching the same sky ~8 times over because exposures overlap that deep. Make it a function of sky area instead. `star_catalogue` fetches the tile list's footprint once, one query per NSIDE=32 HEALPix chunk, into a run-independent chunk store under the `star_cats` root; `exp_star_cat` keeps its DAG position and both its outputs but is now a pure local cut of those chunks to the exposure's focal-plane disc. ~16x fewer queries and ~4x less queried area, and the campaign's star catalogue becomes a durable product in its own right. Chunk-need comes from the tile list, the one thing known before any download, padded 2 deg for the exposure offsets the fixed tile grid admits. Chunks already on disk are skipped, so a campaign growing past its footprint fetches only what it adds and one growing within it fetches nothing; the tile list's hash rides in params so that growth actually reaches the DAG. A chunk the pad failed to cover is a loud failure in the cut, never a quietly under-masked exposure. fetch and cut live in one module because they must agree on which pixel holds which star, and one shared NSIDE cannot drift. Verified on exposure 2086324: the cut reproduces the catalogue the old per-exposure query produced exactly — 10936 stars, identical GSC2.3 id sets, identical columns. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- workflow/Snakefile | 43 +++-- workflow/config.yaml | 35 ++-- workflow/rules/exposure.smk | 170 ++++++++++------ workflow/rules/prepare.smk | 7 +- workflow/scripts/star_cats.py | 354 ++++++++++++++++++++++++++++++++++ 5 files changed, 515 insertions(+), 94 deletions(-) create mode 100644 workflow/scripts/star_cats.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 3b18fd1b8..352761501 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -49,12 +49,14 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) -# Run-independent cache root for the mask star catalogues, keyed by exposure and -# filled by the exp_star_cat rule (config.yaml explains the placement). +# Run-independent root for the mask star catalogues: the HEALPix chunk store the +# star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it +# (config.yaml explains the placement). STAR_CATS = Path(config["star_cats"]) -# This checkout's own tree: exp_star_cat runs its scripts/python/create_star_cat.py -# against its src/, for the same reason CONFIG_DIR is the committed config dir — -# script, library and rule are one artefact and version together. +# This checkout's own tree: the star-cat rules run in the container against its +# src/ (shapepipe.utilities.vizier and .cfis), for the same reason CONFIG_DIR is +# the committed config dir — script, library and rule are one artefact and +# version together. REPO_DIR = Path(workflow.basedir).parent INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" @@ -190,14 +192,14 @@ def unit_num(unit): SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] -# Same argument for the star-cat query script, which lives in the repo's own -# scripts/python/ rather than workflow/scripts/: exp_star_cat's params otherwise -# fingerprint nothing but paths, so an edit to create_star_cat.py would never -# rerun the rule. The hash does NOT key the cache path — see config.yaml's -# star_cats block on clearing the cache after a semantic change. +# Same argument for star_cats.py, which both star-cat rules call: their params +# otherwise fingerprint nothing but paths, so an edit to the chunking or the cut +# would never rerun them. ONE hash for both rules because it is one script — and +# that is also why fetch and cut live in one module (they must agree on which +# pixel holds which star). The hash does NOT key the store path — see +# config.yaml's star_cats block on clearing the store after a semantic change. STAR_CAT_HASH = hashlib.md5( - (REPO_DIR / "scripts" / "python" / "create_star_cat.py").read_bytes() -).hexdigest()[:12] + (SCRIPTS / "star_cats.py").read_bytes()).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- @@ -399,15 +401,16 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# The aggregation targets, clean_exposure and exp_star_cat run in the head -# process. clean_exposure is seconds of rmtree and hangs off `all`; exp_star_cat -# is ~6 s dominated by one CDS query, and running it locally serialises those -# queries (see exposure.smk). Both would otherwise be ~20k sbatch submissions at -# DR6 scale for work shorter than the scheduling latency. +# The aggregation targets, clean_exposure, star_catalogue and exp_star_cat run in +# the head process. clean_exposure is seconds of rmtree and hangs off `all`; +# exp_star_cat is seconds of local FITS work; both would otherwise be ~20k sbatch +# submissions at DR6 scale for work shorter than the scheduling latency. +# star_catalogue is one job either way, and local keeps its CDS concurrency the +# explicit number its thread pool sets (see exposure.smk). # -# exp_star_cat is the one MID-CHAIN localrule, so it must stay out of any future -# `group:` label: a local job cannot be fused into a submitted group. -localrules: all, prepare_all_tiles, clean_exposure, exp_star_cat +# star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out +# of any future `group:` label: a local job cannot be fused into a submitted group. +localrules: all, prepare_all_tiles, clean_exposure, star_catalogue, exp_star_cat rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index 94edd70c9..c81d4e45d 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -29,22 +29,31 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# The mask star-catalogue CACHE root — run-independent, shared by every campaign. -# `exp_star_cat` writes /exp/star_cat-.fits, one GSC 2.3 cone query per -# exposure (~787 KiB, ~6 s), and skips a cat that is already there. Exposures -# recur across tiles and batches, so the cache is mostly hits. +# The mask star-catalogue root — run-independent, shared by every campaign, and +# holding two things: +# /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 +# query per HEALPix chunk (~3.4 deg^2, ~25k rows), written by +# `star_catalogue` over the tile list's footprint and never fetched twice; +# /exp/star_cat-.fits the per-exposure cuts +# `exp_star_cat` makes from those chunks, with no network at all. +# Network therefore scales with SKY AREA, not exposure count: exposures overlap +# ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. +# Design intent is /project — this is a durable science product, not scratch +# work — blocked only by the quota below. # -# THE CACHE IS NOT KEYED BY SCRIPT VERSION. A semantic change to -# scripts/python/create_star_cat.py (query radius, catalogue ID, column set) -# does rerun exp_star_cat — the script's hash is a rule param — but the rerun -# takes the skip-if-exists branch and only re-links the OLD catalogues. Clear -# this cache by hand when the change must reach the data. +# THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to +# workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun +# both rules — the script's hash is a param on each — but a chunk already on disk +# is skipped and only re-cut. Clear the store by hand when the change must reach +# the data. Changing NSIDE or the catalogue ID is the exception: those name the +# directory, so a change there fetches into a new one beside the old. # # On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day -# purge therefore applies, and that is acceptable: a purged cache regenerates one -# query per exposure. exp_star_cat declares its ccd-0 farm link as an output, so -# a purge leaves that link dangling and the rule reruns, instead of trusting a -# manifest that outlived its data. +# purge therefore applies. A purged per-exposure cut is free to remake, and +# exp_star_cat declares its ccd-0 farm link as an output so a purge leaves that +# link dangling and the rule reruns rather than trusting a manifest that outlived +# its data. A purged CHUNK costs its query again: exp_star_cat fails loudly on a +# missing chunk, and re-running `star_catalogue` refetches it. star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache # The run index. Design intent (finding 15) is durable products on /project — diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index f56653e6c..c6fea1cae 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,7 +1,11 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split ---> exp_mask -> exp_psf - -> exp_star_cat -/ + exp_get_images -> exp_split -----> exp_mask -> exp_psf + -> exp_star_cat --/ + star_catalogue ---------------/ + +``star_catalogue`` is campaign-level, not per-exposure: one fetch of the whole +footprint's stars, which every exposure's ``exp_star_cat`` then cuts locally. Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a @@ -40,32 +44,98 @@ rule exp_get_images: sp_shell("exp_get_images", "config_exp_Gie.ini") # --- mask star catalogues --------------------------------------------------- -# One GSC 2.3 (Vizier) cone query per exposure, covering the whole MegaCam focal -# plane, plus the 40 per-CCD symlinks the mask module's numbering scheme needs. +# Two rules, and the split between them is the design: the NETWORK is a function +# of the campaign's sky area, the per-exposure catalogue is a local cut. +# +# `star_catalogue` fetches the footprint's GSC 2.3 stars once, one Vizier query +# per HEALPix chunk, into a run-independent chunk store under config `star_cats`. +# `exp_star_cat` then reads the chunks covering an exposure's focal plane and +# cuts them to it — no network at all. workflow/scripts/star_cats.py holds both +# halves and the geometry they must agree on; its docstring is the reference for +# the chunking and the padding. # -# It reads the exp_get_images output dir directly: that is a symlink farm of -# image-.fitsfz, exactly what create_star_cat.py's `-k exp` mode consumes — -# one multi-extension header read, one query, ~6 s measured. +# The arithmetic: exposures overlap ~7-10 deep and a tile's exposures all look at +# the same square degree, so the old one-cone-per-exposure design re-fetched the +# same sky ~8 times over. Chunked by sky, a full-UNIONS footprint is ~1.5k +# queries where the exposure count would have been ~25k, and a campaign that +# grows within the fetched footprint issues none. + +# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE +# pointing at a path that does not exist inside the image, so requests is pointed +# at the bundle explicitly (proven in the p3-batch1 bash precedent). +STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" + +# The rules run star_cats.py inside the container (healpy, astroquery, astropy) +# but call apptainer THEMSELVES rather than letting the SDM wrap them +# (`container: None` on both): the CA bundle above and the exposure rule's +# host-side farm loop both need the explicit exec. bin/sp has loaded the +# apptainer module. PYTHONPATH pins this checkout's src/ for +# shapepipe.utilities.{vizier,cfis} — the mirror-retry query and the tile-ID +# grid convention are library code, not copies. +def in_container(cmd, *, network=False): + ca = ("," + ",".join(f"{k}={STAR_CAT_CA}" for k in + ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", "CURL_CA_BUNDLE")) + if network else "") + return (f"apptainer exec --cleanenv --home {Path.home()}" + f" --bind /project --bind /scratch" + f" --env PYTHONPATH={REPO_DIR}/src{ca}" + f" '{config['container']}' {cmd}") + + +# The campaign's star catalogue: a first-class durable science product, keyed by +# sky rather than by run. Chunk-need is recomputed from the tile list on every +# run and only the missing chunks are fetched, so appending tiles costs exactly +# the chunks they add. # -# A LOCALRULE (declared in the Snakefile), so it runs in the head process — an -# sbatch job on a compute node, and nibi compute nodes have internet (verified by -# curl to vizier.cds.unistra.fr from a compute job). Local execution does NOT -# serialise the queries — an earlier version of this comment claimed it did, and -# that was false. In cluster mode snakemake runs local jobs in a worker pool -# sized by `--local-cores`, and this rule takes one thread each, so as many CDS -# queries (and apptainer execs) run at once as that pool is wide. The nibi -# profile leaves --local-cores unset, so it is the head process's CPU count — -# 5 in the measured allocation. +# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the +# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the +# per-exposure rule reached by accident through --local-cores, now an explicit +# number that does not scale with the head node's CPU count. # -# The localrule stays anyway. 5-way concurrency against Vizier is ordinary -# politeness, not abuse, and the alternative is one sbatch per exposure: ~20k -# submissions for six seconds of work each, which is exactly the sub-15-minute -# scheduler churn cluster policy asks us to bundle away. If CDS ever objects, -# --local-cores is the knob — it caps these queries directly. +# `tile_list_hash` is what makes the incremental behaviour visible to the DAG. +# The tile list is parse-time config, not a rule input (and the profile drops the +# `input` rerun-trigger anyway), so appending tiles would otherwise leave this +# rule up to date against a footprint that has grown. Hashing the list into a +# param reruns it, and the rerun fetches only what is new. +STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" + + +rule star_catalogue: + output: + manifest = STAR_CAT_MANIFEST + # No `log:` — see write_manifest() in star_cats.py: this rule runs no + # shapepipe_run and computes no completeness verdict, so there is nothing to + # split between a manifest and a log. Under `set -euo pipefail` it either + # completes or aborts, and snakemake's captured stderr is the evidence. + # `cmd` is a params value, so it is substituted AFTER the shell string is + # formatted: a `{output.manifest}` placeholder in here would survive + # literally (see unit_pre in the Snakefile). Hence the explicit path. + params: + cmd = in_container( + f"python {SCRIPTS}/star_cats.py fetch" + f" --tile-list '{config['tile_list']}' --store '{STAR_CATS}'" + f" --manifest '{STAR_CAT_MANIFEST}'", network=True), + tile_list_hash = hashlib.md5( + Path(config["tile_list"]).read_bytes()).hexdigest()[:12], + script_hash = STAR_CAT_HASH + container: + None + threads: 4 + retries: 2 + resources: + mem_mb = 4000, + runtime = 720 + shell: + "set -euo pipefail\n{params.cmd}" + + +# The per-exposure catalogue and the 40 per-CCD symlinks the mask module's +# numbering scheme needs. Local: one header read for the focal-plane footprint, +# a load of the chunks covering it, a radial cut. # -# The CATALOGUE is cached run-independently under config `star_cats`, keyed by -# exposure. create_star_cat.py skips a cat that exists, so retries, reruns and -# later campaigns re-link an existing file and issue no query. +# A LOCALRULE for the same reason as clean_exposure: seconds of work, and one +# sbatch per exposure would be ~20k submissions well under the 15-minute floor +# cluster policy asks us to bundle away. # # The per-unit farm is a REAL directory holding exactly this exposure's 40 # numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR @@ -78,38 +148,30 @@ rule exp_get_images: # The manifest keeps the "one rule, one manifest" currency of every other rule: # written last, unique to this rule, a record of what the farm points at, and # deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the -# cache at no query cost. +# chunk store at no network cost. # # But a manifest attests FOREVER, and the two things it attests to both live -# outside the unit's manifests/ dir: the cache FITS on /scratch (60-day purge) +# outside the unit's manifests/ dir: the cut catalogue on /scratch (60-day purge) # and the farm itself. Either can vanish under a manifest that still says # "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is # declared too — one link stands for all 40, they are created by the same loop # in the same instant, and declaring 40 buys nothing. Snakemake's existence test # is os.path.exists, which FOLLOWS symlinks and is therefore False for a link -# whose target the purge removed. A purged cache or a deleted farm makes the -# rule out of date, it reruns, and it re-queries or re-links as needed — the -# recovery config.yaml's star_cats comment promises, now actually wired up. - -# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE -# pointing at a path that does not exist inside the image, so requests is pointed -# at the bundle explicitly (proven in the p3-batch1 bash precedent). -STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" - - +# whose target the purge removed. A purged cut or a deleted farm makes the rule +# out of date, it reruns, and it re-cuts or re-links as needed. def star_cat_cmd(exp): """The whole rule body, as bash — carried as a params value, never inlined in ``shell:``: it contains literal ``{}`` (the manifest JSON) and snakemake formats a shell string once, which would consume those braces.""" - cache = f"{STAR_CATS}/exp" - cat = f"{cache}/star_cat-{exp}.fits" + cut_dir = f"{STAR_CATS}/exp" + cat = f"{cut_dir}/star_cat-{exp}.fits" work = exp_dir(exp) farm = f"{work}/star_cat_exp" images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" manifest = exp_manifest(exp, "exp_star_cat") body = json.dumps({ "stage": "exp_star_cat", "level": "exp", "unit": exp, - "status": "complete", "cache": cat, "link_dir": farm, "n_links": 40, + "status": "complete", "cat": cat, "link_dir": farm, "n_links": 40, }, indent=2, sort_keys=True) return "\n".join([ "set -euo pipefail", @@ -121,16 +183,10 @@ def star_cat_cmd(exp): # which would recurse into the pool, and never touch a real directory: # a real farm is this rule's own output and `ln -sfn` refreshes it. f"[ -L '{farm}' ] && rm -f '{farm}' || true", - f"mkdir -p '{cache}' '{farm}' '{work}/manifests'", - # Same binds and isolation as the profile's apptainer-args; the SDM does - # not wrap this rule (container: None below), because the farm loop and - # the manifest write belong to the host side of the job. - f"apptainer exec --cleanenv --home {Path.home()} --bind /project --bind /scratch" - f" --env PYTHONPATH={REPO_DIR}/src,REQUESTS_CA_BUNDLE={STAR_CAT_CA}" - f",SSL_CERT_FILE={STAR_CAT_CA},CURL_CA_BUNDLE={STAR_CAT_CA}" - f" '{config['container']}'" - f" python {REPO_DIR}/scripts/python/create_star_cat.py" - f" -i '{images}' -o '{cache}' -k exp", + f"mkdir -p '{cut_dir}' '{farm}' '{work}/manifests'", + in_container(f"python {SCRIPTS}/star_cats.py cut" + f" --images '{images}' --store '{STAR_CATS}'" + f" --out '{cat}'"), f"test -s '{cat}'", # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the # one focal-plane catalogue (pattern from the p3-batch1 precedent). @@ -148,28 +204,26 @@ def star_cat_cmd(exp): rule exp_star_cat: input: - rules.exp_get_images.output.manifest + rules.exp_get_images.output.manifest, + # The chunks this cut reads. star_cats.py fails loudly on a chunk that is + # missing anyway, but the edge is what makes the fetch happen first. + rules.star_catalogue.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", # The sentinel: ccd-0 of the 40-link farm (see above). link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" - # No `log:`. Every other rule's log carries a completeness VERDICT, and this - # rule computes none: it is not a shapepipe_run, it has no count floor, and - # under `set -euo pipefail` it either completes or aborts at the failing - # step. Snakemake's own captured job stderr is the evidence for that. + # No `log:`, for the same reason as star_catalogue above. params: cmd = lambda wc: star_cat_cmd(wc.exp), - # create_star_cat.py is external to the shell string, so the `code` + # star_cats.py is external to the shell string, so the `code` # rerun-trigger does not see it — same reason SCRIPT_HASH exists. script_hash = STAR_CAT_HASH - # No SDM wrapping: the rule calls apptainer itself, because only the query - # runs in the container (bin/sp has loaded the apptainer module). container: None threads: 1 retries: 2 resources: - mem_mb = 2000, + mem_mb = 4000, runtime = 10 shell: "{params.cmd}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index d4a8d1cdd..08ac0d66f 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -11,9 +11,10 @@ tier). Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. Star catalogues for masking are NOT a prepare-phase concern and not pre-run -input: they are built in the compute DAG, one node per exposure -(``exp_star_cat`` in exposure.smk), into a run-independent cache. The tile side -has no star-cat node because it has no mask rule yet — see tile.smk. +input: the compute DAG fetches the campaign footprint's stars once +(``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in +exposure.smk, into a run-independent store. The tile side has no star-cat node +because it has no mask rule yet — see tile.smk. """ # NUMBER_LIST is never set for get_images (download stage; nothing on disk to diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py new file mode 100644 index 000000000..56886cfa0 --- /dev/null +++ b/workflow/scripts/star_cats.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""The campaign's GSC 2.3 star catalogue, as a HEALPix-chunked sky store. + +Masking needs, for every exposure, the bright stars over its focal plane. The +sky does not change between exposures, so the network cost of that is a property +of the campaign's SKY AREA, not of its exposure count: exposures overlap each +other ~7-10 deep, and a tile's exposures all look at the same square degree. + +So the store is chunked by sky, not by exposure. One GSC 2.3 cone query per +HEALPix pixel of NSIDE=32, written run-independently under the ``star_cats`` +config root and never fetched twice. A campaign that grows past the fetched +footprint queries only the chunks its new tiles add; one that grows within it +queries nothing. + +Two numbers set the scale. A full-UNIONS footprint is ~1.5k chunks against ~25k +exposures, so the QUERY COUNT drops ~16x. The queried AREA drops ~4x: the old +design covered the footprint ~8 times over (that is just the exposure overlap +depth), the new one ~2 times, the 2x being the price of bounding a HEALPix +quadrilateral by the cone Vizier speaks (see ``pixel_cone``) — 5.6-9 deg^2 for a +3.36 deg^2 pixel, ~40-60k rows and ~3-4 MB per chunk. + +Two subcommands, one module, deliberately: ``fetch`` and ``cut`` must agree +EXACTLY on which pixel holds which star, and a shared NSIDE constant in one file +is the only version of that agreement which cannot drift. + + fetch --tile-list ... --store ... --manifest ... + The campaign side. Turns the tile list into the set of pixels its + exposures can possibly need, fetches the missing ones, writes a manifest. + + cut --images ... --store ... --out ... + The per-exposure side, purely local: read the focal-plane footprint from + the exposure's image headers, load the chunks covering it, deduplicate, + and cut to the focal-plane disc. Reproduces byte-for-byte the same sky + selection the old one-query-per-exposure cone did. + +Geometry, and why the fetch pad is what it is. Chunk-need is computed from the +TILE list rather than from exposure pointings, because tile IDs are the one thing +known before any download: a pointing center means reading a FITS header of an +image get_images has not fetched yet, and the DAG needs the chunk set at parse +time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so +each tile is a disc of half-diagonal 0.354 deg; find_exposures gives a tile every +exposure whose footprint covers it, and the MegaCam focal plane is a disc of +radius 0.73 deg (measured on the cached catalogues). An exposure center is +therefore at most 0.354 + 0.73 deg from the tile center, and its stars 0.73 deg +beyond that: 1.81 deg, padded to ``PAD_DEG`` = 2.0. The pad is a perimeter cost +— negligible for a contiguous campaign, and paid once. + +The pad is a bound, not a promise: ``cut`` verifies that every chunk covering the +exposure it was handed is on disk, and fails loudly if one is not. A missing +chunk means the geometry above is wrong, and that must not degrade quietly into +an under-masked exposure. +""" + +import argparse +import json +import os +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np +import healpy as hp +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.table import Table, vstack +from astropy.wcs import WCS + +# GSC 2.3. The same catalogue the mask module's own CDS path uses +# (mask.py: _CDS_cat_ID), so the store is a drop-in for it. +CAT_ID = "I/305/out" + +# NSIDE=32 -> 3.36 deg^2 per pixel, 12288 pixels over the sky. Chosen so one +# chunk is a couple of MegaCam focal planes: small enough that a Vizier query +# stays within a small multiple of the per-exposure queries this replaces, large +# enough that a full-UNIONS footprint is ~1.5k chunks rather than ~25k. The +# cone-vs-quadrilateral overhead is scale-free, so NSIDE trades query count +# against query size and nothing else. NESTED, so a chunk id is a hierarchical +# sky address and a future NSIDE change is a subdivision. +NSIDE = 32 +NEST = True + +# Angular padding on the disc used to select chunks (see the module docstring). +PAD_DEG = 2.0 + +# The MegaCam focal-plane disc, and the margin added to a pixel's own bounding +# cone. Both in degrees. +MARGIN_DEG = 0.02 + +# GSC 2.3's object id: the deduplication key where chunk cones overlap. +ID_COL = "GSC2.3" + + +# --- the chunk store -------------------------------------------------------- + + +def store_dir(store: Path) -> Path: + """Chunks live under the catalogue and resolution that produced them, so a + later NSIDE or catalogue change is a new directory beside the old one rather + than a silent reinterpretation of files already on disk.""" + return Path(store) / CAT_ID.replace("/", "_") / f"nside{NSIDE}" + + +def chunk_path(store: Path, ipix: int) -> Path: + return store_dir(store) / f"star_chunk-{ipix:06d}.fits" + + +def chunks_for_disc(ra_deg: float, dec_deg: float, radius_deg: float) -> list[int]: + """Every pixel that touches the disc, as sorted ids. + + ``inclusive=True`` makes this a conservative superset — the guarantee ``cut`` + relies on is that no star inside the disc lives in a pixel this omits. + """ + vec = hp.ang2vec(ra_deg, dec_deg, lonlat=True) + return sorted(int(i) for i in hp.query_disc( + NSIDE, vec, np.radians(radius_deg), inclusive=True, fact=4, nest=NEST)) + + +def pixel_cone(ipix: int) -> tuple[float, float, float]: + """(ra, dec, radius_arcmin) of a cone that CONTAINS pixel ``ipix``. + + Vizier speaks cones, HEALPix speaks quadrilaterals, so the query is the + pixel's bounding cone: its center, and the largest center-to-boundary + distance plus a margin. The cone spills over the pixel edges, which costs a + little duplication between neighbours and buys the containment ``cut`` + depends on. The duplicates are removed on read, by ``ID_COL``. + """ + ra_c, dec_c = hp.pix2ang(NSIDE, ipix, nest=NEST, lonlat=True) + ra_b, dec_b = hp.vec2ang(hp.boundaries(NSIDE, ipix, step=8, nest=NEST).T, + lonlat=True) + center = SkyCoord(ra_c * u.deg, dec_c * u.deg) + radius = center.separation(SkyCoord(ra_b * u.deg, dec_b * u.deg)).deg.max() + return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) + + +def write_atomic(table: Table, path: Path) -> None: + """Publish ``table`` at ``path`` all-or-nothing. + + The store's only cache test is ``Path.exists``, so a write killed part-way + (timeout, OOM, node failure) would otherwise leave a truncated FITS that + every later run trusts forever. The temp keeps the ``.fits`` suffix because + astropy picks its writer from the extension, and is dot-prefixed and + PID-tagged so it stays out of the ``star_chunk-*`` globs and two concurrent + writers cannot collide. + """ + tmp = path.parent / f".tmp-{os.getpid()}-{path.name}" + try: + table.write(tmp, overwrite=True) + os.replace(tmp, path) + finally: + if tmp.exists(): + tmp.unlink() + + +def read_chunks(store: Path, ipixels: list[int]) -> Table: + """Load and deduplicate the given chunks. + + A missing chunk is fatal (see the module docstring): it means the fetch + footprint did not cover this exposure, and an under-masked exposure is worse + than a failed job. + """ + missing = [i for i in ipixels if not chunk_path(store, i).exists()] + if missing: + raise SystemExit( + f"star chunk(s) {missing} not in {store_dir(store)}. The campaign's " + f"star_catalogue fetch did not cover this exposure — re-run it " + f"(and check that its tile list contains this exposure's tiles).") + + table = vstack([Table.read(chunk_path(store, i)) for i in ipixels], + metadata_conflicts="silent") + _, keep = np.unique(np.asarray(table[ID_COL]), return_index=True) + return table[np.sort(keep)] + + +# --- exposure footprint ----------------------------------------------------- + + +def _wcs(header) -> WCS: + """Build the WCS by hand, from the linear terms only. + + Same construction as ``scripts/python/create_star_cat.py``: it sidesteps + distortion-convention incompatibilities between headers and astropy, and a + focal-plane footprint needs nothing finer. + """ + w = WCS(naxis=2) + w.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] + try: + w.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] + except KeyError: + w.wcs.cunit = ["deg", "deg"] + w.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] + w.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] + w.wcs.cd = [[header["CD1_1"], header["CD1_2"]], + [header["CD2_1"], header["CD2_2"]]] + return w + + +def focal_plane_disc(image: Path, n_ccd: int = 40) -> tuple[float, float, float]: + """(ra, dec, radius_deg) of the disc covering all CCDs of one exposure.""" + centers, radii = [], [] + for ext in range(1, n_ccd + 1): + h = fits.getheader(image, ext) + w = _wcs(h) + (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( + [[h["NAXIS1"] / 2.0, h["NAXIS2"] / 2.0], [0, 0]], 1) + centers.append(SkyCoord(ra_c * u.deg, dec_c * u.deg)) + radii.append(centers[-1].separation( + SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg) + + ras = np.array([c.ra.deg for c in centers]) + decs = np.array([c.dec.deg for c in centers]) + center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) + seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg + return (float(center.ra.deg), float(center.dec.deg), + float(np.max(seps + np.array(radii)))) + + +def exposure_image(images_dir: Path) -> Path: + """The one multi-extension exposure image in a get_images output dir. + + That dir is a symlink farm holding ``image-.fitsfz`` plus its weight and + flag; only the image carries the 40 CCD WCSs. + """ + found = sorted(p for p in Path(images_dir).iterdir() if "image" in p.name) + if not found: + raise SystemExit(f"no image file in {images_dir}") + return found[0] + + +# --- fetch ------------------------------------------------------------------ + + +def campaign_chunks(tile_ids: list[str]) -> list[int]: + """Every chunk the campaign's exposures can need, from the tile list alone.""" + from shapepipe.utilities.cfis import get_tile_coord_from_nixy + + needed: set[int] = set() + for tile_id in tile_ids: + nix, niy = tile_id.split(".") + ra, dec = get_tile_coord_from_nixy(nix, niy) + needed.update(chunks_for_disc(ra.degree, dec.degree, PAD_DEG)) + return sorted(needed) + + +def fetch(args: argparse.Namespace) -> None: + from shapepipe.utilities.vizier import query_vizier + + tile_ids = [ln.strip() for ln in Path(args.tile_list).read_text().splitlines() + if ln.strip()] + needed = campaign_chunks(tile_ids) + out_dir = store_dir(args.store) + out_dir.mkdir(parents=True, exist_ok=True) + todo = [i for i in needed if not chunk_path(args.store, i).exists()] + print(f"star chunks: {len(needed)} needed for {len(tile_ids)} tiles, " + f"{len(todo)} to fetch -> {out_dir}", file=sys.stderr) + + def one(ipix: int) -> int: + ra, dec, radius_arcmin = pixel_cone(ipix) + table = query_vizier(ra, dec, radius_arcmin, CAT_ID) + write_atomic(table, chunk_path(args.store, ipix)) + print(f"chunk {ipix}: {len(table)} rows " + f"(ra={ra:.4f} dec={dec:.4f} r={radius_arcmin:.1f}')", + file=sys.stderr) + return len(table) + + # A handful of concurrent queries, never one per exposure: the same modest + # concurrency the per-exposure rule reached through --local-cores, now an + # explicit number instead of an accident of the head node's CPU count. + if todo: + with ThreadPoolExecutor(max_workers=args.workers) as pool: + list(pool.map(one, todo)) + + write_manifest(args, tile_ids, needed, len(todo)) + + +def write_manifest(args, tile_ids, needed, n_fetched) -> None: + """The rule's declared output. + + Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and + has no per-runner count floors, so there is nothing to compose and no + separate ``log:`` — under ``set -euo pipefail`` the job either completes or + aborts at the failing query, and snakemake's captured stderr is the evidence. + The manifest keeps the workflow's "one rule, one manifest" currency: written + last, and only when the content changed, so an unchanged campaign leaves the + mtime where it was rather than churning the `mtime` rerun-trigger. + """ + body = json.dumps({ + "stage": "star_catalogue", + "level": "campaign", + "status": "complete", + "catalogue": CAT_ID, + "nside": NSIDE, + "nest": NEST, + "pad_deg": PAD_DEG, + "store": str(store_dir(args.store)), + "n_tiles": len(tile_ids), + "n_chunks": len(needed), + "n_fetched": n_fetched, + "chunks": needed, + }, indent=2, sort_keys=True) + path = Path(args.manifest) + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != body: + path.write_text(body) + + +# --- cut -------------------------------------------------------------------- + + +def cut(args: argparse.Namespace) -> None: + image = exposure_image(args.images) + ra, dec, radius = focal_plane_disc(image) + ipixels = chunks_for_disc(ra, dec, radius) + table = read_chunks(args.store, ipixels) + + center = SkyCoord(ra * u.deg, dec * u.deg) + stars = SkyCoord(np.asarray(table["RAJ2000"]) * u.deg, + np.asarray(table["DEJ2000"]) * u.deg) + inside = table[center.separation(stars).deg <= radius] + + print(f"{image.name}: ra={ra:.4f} dec={dec:.4f} r={radius:.4f} deg, " + f"{len(ipixels)} chunks -> {len(inside)} stars", file=sys.stderr) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + write_atomic(inside, out) + + +# --- CLI -------------------------------------------------------------------- + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + + f = sub.add_parser("fetch", help="fetch the campaign footprint's chunks") + f.add_argument("--tile-list", required=True, type=Path) + f.add_argument("--store", required=True, type=Path) + f.add_argument("--manifest", required=True, type=Path) + f.add_argument("--workers", type=int, default=4) + f.set_defaults(func=fetch) + + c = sub.add_parser("cut", help="cut one exposure's catalogue from the store") + c.add_argument("--images", required=True, type=Path, + help="a get_images output dir holding image-.fitsfz") + c.add_argument("--store", required=True, type=Path) + c.add_argument("--out", required=True, type=Path) + c.set_defaults(func=cut) + + args = p.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() From f2cf65b6a61a2d4a043fdd2df1abf612ca343fe1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:29:09 -0400 Subject: [PATCH 034/100] =?UTF-8?q?feat(container):=20sp=20container=20?= =?UTF-8?q?=E2=80=94=20per-user=20image=20layers=20and=20resolution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image every job runs inside is a fixed /project .sif, shared by everyone and refreshed by hand. `sp container` gives each user their own copy and one resolution order that the CLI and the workflow both obey: a writable sandbox if they built one, else their cached SIF, else the `container:` path in workflow/config.yaml. An empty cache lands on that shared .sif, so nothing changes until a user opts in. pull and sandbox stage to a sibling path and swap it in, so an in-flight job never sees a half-written image and a failed rebuild leaves the working one intact. status reads the image's OCI labels and places its revision against this checkout's HEAD. exec runs a one-off command with the profile's binds. The script is stdlib-only: it runs on the bare host, outside the container. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- profiles/nibi/config.yaml | 7 +- workflow/README.md | 45 +++- workflow/Snakefile | 11 +- workflow/bin/sp | 14 +- workflow/scripts/container.py | 432 ++++++++++++++++++++++++++++++++++ 5 files changed, 503 insertions(+), 6 deletions(-) create mode 100644 workflow/scripts/container.py diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 8bc71a9d1..8987ab9bc 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -8,7 +8,12 @@ # # Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). # software-deployment-method wraps every job's shell in `apptainer exec` — the -# user never types apptainer; the container is set in the Snakefile (local .sif). +# user never types apptainer. WHICH image is resolved in the Snakefile through +# workflow/scripts/container.py: this user's writable sandbox if they built one, +# else their cached SIF (`sp container pull`), else the `container:` path in +# workflow/config.yaml — the shared /project .sif, and the default whenever the +# cache is empty. `sp container status` prints which layer is live. `sp container +# pull` needs the network: login node or salloc, never a batch job. executor: slurm diff --git a/workflow/README.md b/workflow/README.md index 1d219617e..52e8d58c3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -30,6 +30,7 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' workflow/bin/sp run # bring products on disk up to date with the tile list workflow/bin/sp report # emit run_report.json now (mid-run is fine) workflow/bin/sp cancel # scancel this workflow's jobs +workflow/bin/sp container status # which image the jobs will run ``` Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, @@ -38,10 +39,49 @@ Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became plugins, and full `rerun-triggers` became the default. -Anything other than `run`, `report`, `cancel` passes straight through to +Anything other than `run`, `report`, `container`, `cancel` passes straight through to snakemake with the workflow's profile and state dir — the escape hatch for `sp --unlock`, `sp --dag`, `sp exp_psf ...`. +## The container image + +`sp container` owns which image the jobs run inside. Two layers, and the second +only exists if you ask for one: + +* your **cached SIF** (`~/.cache/shapepipe/shapepipe.sif`, `SP_CACHE_DIR` or + `SP_CONTAINER` to move it) — a pristine pull of the published image, private + to you, so nobody else's refresh moves the ground under your running jobs; +* an optional **sandbox** (`~/.cache/shapepipe/sandbox/`, `SP_SANDBOX`) — the + same image unpacked writable, so a `pip install` into it sticks. The escape + hatch for work needing a package the image does not carry yet. + +The Snakefile's `container:` is that resolution, in one order shared by the CLI +and the workflow: **sandbox → cached SIF → the `container:` path in +`config.yaml`**. With an empty cache — the normal case — that lands on the +shared `/project` `.sif` the workflow has always used, so this changes nothing +until you opt in. + +```bash +sp container status # layers present, active one, revision vs HEAD +sp container pull # ghcr.io/cosmostat/shapepipe:develop-runtime +sp container pull --tag docker://... # some other image +sp container sandbox # unpack the SIF writable (opt-in) +sp container exec --writable pip install +sp container exec python -c 'import shapepipe' +sp container resolve # just the path the workflow will run +``` + +`status` reads the image's OCI labels and places its +`org.opencontainers.image.revision` against this checkout's HEAD: +in-sync / behind / ahead / diverged, or unknown when the image carries no label +or the commit was never fetched here. + +**`pull` needs the network.** Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an `salloc` allocation — never from a +batch job. `pull` and `sandbox` both stage to a sibling path and swap it in, so +an in-flight job never sees a half-written image and a failed rebuild leaves the +one you had intact. + ## Execution: two static invocations The exposure job set is data-derived from the tiles' `find_exposures` output, @@ -91,7 +131,7 @@ profile-only pass. workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks config.yaml the run: tile list, paths, container, chunk count - bin/sp committed launcher (module load + /project venv + run/report/cancel) + bin/sp committed launcher (module load + /project venv + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) @@ -102,6 +142,7 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) + container.py image layers + the resolution order behind `sp container` (stdlib-only) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` diff --git a/workflow/Snakefile b/workflow/Snakefile index 352761501..2d9f712de 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -44,8 +44,15 @@ from snakemake.exceptions import WorkflowError configfile: str(Path(workflow.snakefile).parent / "config.yaml") # Every job's shell runs inside this container (apptainer software-deployment in -# the profile); the user never types apptainer. -container: config["container"] +# the profile); the user never types apptainer. WHICH image is the one resolution +# order `sp container` exposes — this user's writable sandbox if they built one, +# else their cached SIF if they pulled one, else `container:` above. An empty +# cache therefore lands on exactly the shared /project image the workflow has +# always run, and a package installed into a sandbox reaches the jobs too. +sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import container as _container # noqa: E402 + +container: _container.resolve_image()[0] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) diff --git a/workflow/bin/sp b/workflow/bin/sp index ec563f032..72e80b783 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,7 +1,7 @@ #!/usr/bin/env bash # sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). # -# Two verbs, nothing else: +# Three verbs, nothing else: # # sp run [ARGS...] bring the products on disk up to date with the tile list. # Two snakemake invocations over one Snakefile: @@ -9,6 +9,12 @@ # 2. COMPUTE snakemake all <- its PARSE builds the index # ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. # sp report [ARGS...] emit run_report.json now (mid-run is fine). +# sp container VERB manage the image every job runs inside: pull, status, +# sandbox, exec, resolve. `sp container --help` documents +# the two layers and the resolution order (sandbox -> your +# cached SIF -> the `container:` path in config.yaml). +# `pull` needs the network: run it on a login node or in an +# salloc allocation, never from a batch job. # # Anything else is passed straight through to snakemake with the same profile and # state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). @@ -83,6 +89,12 @@ case "$cmd" in python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ --status manual "$@" ;; + container) + # The image layer, outside snakemake entirely: the script is stdlib-only so + # it runs on the bare host, and the apptainer module is already loaded above. + shift + python "$SCRIPTS/container.py" "$@" + ;; cancel) # Kept only because it is two lines: scancel this workflow's jobs by name # before an --unlock. Not part of the design surface. diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py new file mode 100644 index 000000000..0479b0f46 --- /dev/null +++ b/workflow/scripts/container.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Manage this user's copy of the ShapePipe container image. + +Two layers, and the second only exists when you ask for one: + +* the **SIF** (``~/.cache/shapepipe/shapepipe.sif``) -- a pristine, read-only + copy of the published image, pulled into your own cache. Per-user by + construction: one file, one owner, nobody else's refresh moves the ground + under a running job. +* an optional **sandbox** (``~/.cache/shapepipe/sandbox/``) -- the same image + unpacked into a writable directory, so a ``pip install`` inside it sticks. + The escape hatch for work that needs a package the image does not carry yet. + +Resolution order, shared by this CLI and by the workflow: **sandbox if it +exists, else the cached SIF if it exists, else the ``container:`` path in +workflow/config.yaml**. That last one is the current shared /project image, so +a checkout with an empty cache behaves exactly as it did before this verb +existed, and a package installed into your sandbox is there for your workflow +jobs too. + +Subcommands, exposed as ``sp container ``:: + + sp container pull # fetch the tag into the cache + sp container status # what is here, and how current it is + sp container sandbox # unpack the SIF into a writable dir + sp container exec # run something inside it + sp container exec --writable # ... with writes that persist + +``pull`` needs the network. Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an ``salloc`` allocation -- never +from a batch job. + +Deliberately **stdlib-only**: it runs on the bare host, outside the container, +where the science stack is not installed, and so must import without it. +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +# The published image. CI pushes one tag per branch, sanitized; `-runtime` is +# the slim variant the workflow runs. +CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" + +# The single source of truth for the fallback image: the workflow's own +# `container:` key, which is also what the Snakefile reads. Written down once, +# here, so the CLI and the workflow cannot disagree about the default. +CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" +CONFIG_KEY = "container" + +# ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto +# a filesystem with room), XDG_CACHE_HOME moves it with everything else. +CACHE_DIR = Path( + os.environ.get("SP_CACHE_DIR") + or Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "shapepipe" +).expanduser() + +# This user's pristine image. Override with ``SP_CONTAINER`` (absolute path). +DEFAULT_SIF = CACHE_DIR / "shapepipe.sif" + +# The optional writable unpacking of it. Override with ``SP_SANDBOX``. +DEFAULT_SANDBOX = CACHE_DIR / "sandbox" + +# Bind mounts for `exec`, matching the nibi profile's apptainer-args (the two +# cluster filesystems this workflow reads and writes, plus the home that holds +# ~/.ssl/cadcproxy.pem). Override wholesale with ``SP_APPTAINER_BINDS``. +DEFAULT_BINDS = "/project,/scratch,/home" + + +def configured_default(): + """Return the ``container:`` path from workflow/config.yaml, or ``None``. + + A deliberately minimal scalar read (the same one bin/sp does in sed): this + module is stdlib-only, so there is no yaml to import. + """ + try: + text = CONFIG_FILE.read_text() + except OSError: + return None + match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) + return match.group(1) if match else None + + +def local_sif(): + """Return this user's cached image path (may not exist yet).""" + override = os.environ.get("SP_CONTAINER") + return (Path(override) if override else DEFAULT_SIF).expanduser() + + +def local_sandbox(): + """Return this user's writable sandbox directory (may not exist).""" + override = os.environ.get("SP_SANDBOX") + return (Path(override) if override else DEFAULT_SANDBOX).expanduser() + + +def resolve_image(): + """Return ``(path, kind)`` for the image everything should run. + + ``kind`` is ``"sandbox"``, ``"sif"``, ``"configured"`` (the shared + /project image named in config.yaml -- the default when the cache is + empty) or ``"none"``. + """ + sandbox = local_sandbox() + if sandbox.is_dir(): + return str(sandbox), "sandbox" + sif = local_sif() + if sif.exists(): + return str(sif), "sif" + default = configured_default() + if default: + return default, "configured" + return "", "none" + + +def image_labels(image): + """Return the image's OCI labels as a dict, or ``{}`` if unreadable. + + Never raises: a missing file, a missing ``apptainer`` or a corrupt image + all mean "we don't know", which every caller treats as non-fatal. + """ + path = Path(image) + if not path.exists() or shutil.which("apptainer") is None: + return {} + try: + out = subprocess.run( + ["apptainer", "inspect", "--labels", str(path)], + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return {} + if out.returncode != 0: + return {} + labels = {} + for line in out.stdout.splitlines(): + key, sep, value = line.partition(":") + if sep: + labels[key.strip()] = value.strip() + return labels + + +def image_revision(image): + """Return the commit the image was built from, or ``None``.""" + return image_labels(image).get("org.opencontainers.image.revision") + + +def _require_apptainer(): + """Exit unless ``apptainer`` is on PATH (bin/sp loads the module).""" + if shutil.which("apptainer") is None: + sys.exit("apptainer is not on PATH (bin/sp loads apptainer/1.4.5)") + + +def _git(*args, cwd=None): + """Run git, returning stripped stdout or ``None`` on any failure.""" + try: + out = subprocess.run( + ["git", *args], capture_output=True, text=True, cwd=cwd, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + +def compare_revision(revision, repo=None): + """Place an image revision relative to this checkout's HEAD. + + One of ``"in-sync"``, ``"behind"`` (the image predates HEAD), ``"ahead"``, + ``"diverged"``, or ``"unknown"`` (no label, no git, or a commit this clone + has never fetched). + """ + if not revision: + return "unknown" + repo = repo or Path(__file__).resolve().parents[2] + head = _git("rev-parse", "HEAD", cwd=repo) + if head is None: + return "unknown" + if head == revision: + return "in-sync" + if _git("cat-file", "-e", f"{revision}^{{commit}}", cwd=repo) is None: + return "unknown" + if _git("merge-base", "--is-ancestor", revision, head, cwd=repo) is not None: + return "behind" + if _git("merge-base", "--is-ancestor", head, revision, cwd=repo) is not None: + return "ahead" + return "diverged" + + +def cmd_pull(args): + """Pull ``--tag`` into the cache, atomically.""" + _require_apptainer() + sif = local_sif() + sif.parent.mkdir(parents=True, exist_ok=True) + # Pull to a sibling temp name and rename: an atomic rename within one + # directory, so an in-flight job sees either the whole old image or the + # whole new one. Pulling in place leaves the file half-written for the many + # minutes the pull takes. Jobs already running hold the old inode open. + tmp = sif.with_name(sif.name + f".pull.{os.getpid()}") + print(f"pulling {args.tag}\n -> {sif}") + try: + subprocess.run( + ["apptainer", "pull", "--force", "--name", str(tmp), args.tag], check=True + ) + os.replace(tmp, sif) + except subprocess.CalledProcessError as exc: + tmp.unlink(missing_ok=True) + sys.exit(f"pull failed ({exc.returncode}); {sif} is unchanged") + except KeyboardInterrupt: + tmp.unlink(missing_ok=True) + raise + labels = image_labels(sif) + print(f"revision: {labels.get('org.opencontainers.image.revision', 'unknown')}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + return 0 + + +def cmd_sandbox(args): + """Unpack the image into a writable directory -- the opt-in escape hatch.""" + _require_apptainer() + sandbox = local_sandbox() + if sandbox.exists() and not args.force: + sys.exit( + f"sandbox already exists at {sandbox}\n" + "pass --force to discard it and rebuild from a clean image" + ) + source = args.source + if not source: + image, kind = resolve_image() + if kind == "sandbox": + # Rebuilding from the sandbox itself would just re-copy the drift. + image = str(local_sif()) if local_sif().exists() else ( + configured_default() or CONTAINER_URI + ) + source = image or CONTAINER_URI + sandbox.parent.mkdir(parents=True, exist_ok=True) + print(f"building sandbox from {source}\n -> {sandbox}") + # Build beside the target and swap it in, as `pull` does -- and for a + # sharper reason. A half-written .sif fails loudly, but a half-unpacked + # sandbox *directory* is still a directory, so resolve_image() would elect + # it and every job would silently run a broken tree. Staging also means a + # --force rebuild that fails leaves the sandbox you already had intact. + # + # `--fix-perms` so the tree can be deleted again later. No `--fakeroot`: an + # unprivileged build from an existing image goes through user namespaces, + # which is what the Alliance clusters provide. + staging = sandbox.with_name(f"{sandbox.name}.build.{os.getpid()}") + shutil.rmtree(staging, ignore_errors=True) + try: + subprocess.run( + ["apptainer", "build", "--sandbox", "--fix-perms", str(staging), source], + check=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"sandbox build failed ({exc.returncode}); {sandbox} is unchanged") + except (KeyboardInterrupt, OSError): + shutil.rmtree(staging, ignore_errors=True) + raise + + if sandbox.exists(): + print(f"replacing {sandbox}") + shutil.rmtree(sandbox, ignore_errors=True) + if sandbox.exists(): + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"could not remove {sandbox}; remove it by hand and retry") + os.replace(staging, sandbox) + print( + "\nthis sandbox now takes precedence over the SIF everywhere, including " + "workflow jobs.\ninstall into it with: sp container exec --writable pip " + "install \nreset to a clean image with: sp container pull && " + "sp container sandbox --force" + ) + return 0 + + +def cmd_status(args): + """Report which image layer is live, its revision, and how current it is.""" + sif = local_sif() + sandbox = local_sandbox() + active, kind = resolve_image() + + if sif.exists(): + print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") + else: + print(f"SIF: absent ({sif})") + if sandbox.is_dir(): + print(f"sandbox: {sandbox} (writable; may carry local modifications)") + else: + print("sandbox: none") + print(f"configured: {configured_default() or 'unset'} ({CONFIG_FILE})") + + if kind == "none": + print("\nactive: NONE -- no cached image and no container: in config.yaml") + print(f"run: sp container pull --tag {CONTAINER_URI}") + return 1 + + print(f"\nactive: {active} ({kind})") + if kind == "configured" and not Path(active).exists(): + print(" WARNING: that path does not exist on this host") + return 1 + + labels = image_labels(active) + revision = labels.get("org.opencontainers.image.revision") + source = "" + if revision is None and kind == "sandbox" and sif.exists(): + # Some sandbox trees do not carry the original labels through. The SIF + # beside it is the best remaining evidence of what it was built from -- + # a guess, so it is labelled as one rather than printed as fact. + revision = image_revision(sif) + if revision: + source = " (inferred from the SIF beside it, not read from the sandbox)" + print(f"revision: {revision or 'unknown'}{source}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + if kind == "sandbox": + print( + " (that revision is what the sandbox was built from; " + "anything\n installed into it since is in no label)" + ) + verdict = compare_revision(revision) + explain = { + "in-sync": "matches this checkout's HEAD", + "behind": "older than this checkout's HEAD -- pull to refresh", + "ahead": "newer than this checkout's HEAD", + "diverged": "on a different branch from this checkout", + "unknown": "cannot compare (no label, or a commit this clone lacks)", + }[verdict] + print(f"checkout: {verdict} ({explain})") + return 0 + + +def cmd_exec(args): + """Run a command inside the resolved image -- the one-off path.""" + _require_apptainer() + command = [a for a in args.command if a != "--"] + if not command: + sys.exit("nothing to run; pass a command after `exec`") + binds = args.bind or os.environ.get("SP_APPTAINER_BINDS", DEFAULT_BINDS) + + if args.writable: + # A SIF is a read-only filesystem, so `--writable` against one fails + # obscurely; only a sandbox takes writes. + sandbox = local_sandbox() + if not sandbox.is_dir(): + sys.exit( + f"--writable needs a sandbox, and there is none at {sandbox}\n" + "build one with: sp container sandbox" + ) + image, extra = str(sandbox), ["--writable"] + else: + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + extra = [] + + cmd = ["apptainer", "exec", *extra, "--cleanenv", "--bind", binds, image, *command] + return subprocess.run(cmd).returncode + + +def cmd_resolve(args): + """Print just the resolved image path -- what the Snakefile consumes.""" + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + print(image) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="sp container", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="subcommand", required=True) + + p_pull = sub.add_parser( + "pull", + help="fetch the image into the cache (login node or salloc: compute " + "nodes have no network)", + ) + p_pull.add_argument( + "--tag", default=CONTAINER_URI, help=f"image to pull (default: {CONTAINER_URI})" + ) + p_pull.set_defaults(func=cmd_pull) + + p_status = sub.add_parser( + "status", help="report which image layer is live and how current it is" + ) + p_status.set_defaults(func=cmd_status) + + p_sandbox = sub.add_parser( + "sandbox", help="unpack the image into a writable directory (opt-in)" + ) + p_sandbox.add_argument( + "--source", help="image to unpack (default: the cached SIF, or the config path)" + ) + p_sandbox.add_argument( + "--force", + action="store_true", + help="discard an existing sandbox and rebuild from a clean image", + ) + p_sandbox.set_defaults(func=cmd_sandbox) + + p_exec = sub.add_parser("exec", help="run a command inside the resolved image") + p_exec.add_argument("--bind", help=f"bind mounts (default: {DEFAULT_BINDS})") + p_exec.add_argument( + "--writable", + action="store_true", + help="run against the sandbox so writes (e.g. pip install) persist", + ) + p_exec.add_argument("command", nargs=argparse.REMAINDER) + p_exec.set_defaults(func=cmd_exec) + + p_resolve = sub.add_parser( + "resolve", help="print the resolved image path (what the workflow runs)" + ) + p_resolve.set_defaults(func=cmd_resolve) + + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) From 1ed2537b91bfe27707363dff4b4308112eb8c209 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:30:26 -0400 Subject: [PATCH 035/100] Fuse the short rules into per-unit group jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three group labels, one sbatch per unit instead of one per rule: tile_prep (the whole prepare chain), exp_short (exp_split + exp_mask) and tile_finish (tile_merge_cats + tile_make_cat). Every member's median is under three minutes, well below the 15-minute floor Alliance policy asks us to bundle away, and at DR6 scale the ungrouped form is tens of thousands of submissions for minutes of work each. Heavy rules stay standalone (exp_psf, tile_detect, tile_vignets, tile_ngmix), and exp_get_images stays out of exp_short because the exp_star_cat localrule sits between it and exp_mask — including it would make the group both a dependency and a dependent of that localrule. tile_merge_headers is short but stranded between a localrule and tile_detect, so it stays one job. Group resources compose per toposort level (GroupResources.basic_layered): non -additive resources max across levels, runtime sums. Attempt-scaled mem_mb and per-rule retries survive — GroupJob.attempt re-sets attempt on every member and GroupJob.restart_times is the max over them. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- profiles/nibi/config.yaml | 10 ++++++---- workflow/rules/exposure.smk | 17 +++++++++++++++++ workflow/rules/prepare.smk | 22 ++++++++++++++++++++++ workflow/rules/tile.smk | 17 ++++++++++++++++- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 8987ab9bc..65746786f 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -86,7 +86,9 @@ rerun-triggers: [mtime, params, code, software-env] # ngmix thread count. The RULES own threads and resources; this profile only # sets defaults for rules that state nothing (default-resources above). # -# `group:` fusion of the short rules (uncompress, merges) into their chunky -# neighbours (PRD D-profile) is NOT set here — it requires labels in the rule -# files themselves, which is out of scope for this profile-only pass. Deferred -# to whichever slice next touches workflow/rules/*.smk. +# `group:` fusion of the short rules (PRD D-profile) lives in the rule files, not +# here: the labels are `tile_prep` (prepare.smk), `exp_short` (exp_split + +# exp_mask) and `tile_finish` (tile_merge_cats + tile_make_cat), each documented +# in its file's docstring. Grouping composes resources per toposort level (max +# mem/threads, summed runtime for a linear chain) and leaves the attempt scaling +# above intact, which is the other reason set-resources must stay out of here. diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index c6fea1cae..e41920137 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -19,6 +19,21 @@ by the accumulating index. A temp() here would delete an exposure the moment this invocation's readers finished and cascade destructive reruns across spatial neighbours the next time a tile is appended. +GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — +one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under +the 15-minute floor Alliance policy asks us to bundle away. The composition +rules are in prepare.smk's docstring; this chain is linear too, so the group +asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. + +The two rules NOT in it are structural, not taste: + * exp_psf is heavy (16 GB, 4 h) and never fuses with a short rule; + * exp_get_images cannot join, because ``exp_star_cat`` — a LOCALRULE, and so + ungroupable — sits between it and exp_mask. Pulling get_images in would make + the group both a dependency and a dependent of exp_star_cat, i.e. a cycle. + Starting the group at exp_split leaves star_cat's inputs entirely upstream + of it, so the group has one clean external edge. +Different exposures share no DAG edge, so this is one group job per exposure. + NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering would make the #746 startup validation turn tolerated per-CCD attrition into a @@ -231,6 +246,7 @@ rule exp_star_cat: # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: + group: "exp_short" input: rules.exp_get_images.output.manifest output: @@ -248,6 +264,7 @@ rule exp_split: sp_shell("exp_split", "config_exp_Sp.ini") rule exp_mask: + group: "exp_short" input: # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs # and this exposure's own star_cat_exp farm. diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 08ac0d66f..96a0ae261 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -8,8 +8,27 @@ data-derived tile->exposure edge that invocation 2's parse aggregates into the index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). +All three rules carry ``group: "tile_prep"``, so one tile's whole chain is ONE +sbatch instead of three (medians 0:41 / ~0:40 / 0:15 — all far under the +15-minute floor Alliance policy asks us to bundle away, and at DR6 scale three +submissions per tile is a scheduler load out of all proportion to the work). +Group membership is per connected DAG component and distinct tiles share no +edge, so this is exactly one group job per tile, never a cross-tile bundle. Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. +Group resource composition (snakemake 9.23, ``GroupResources.basic_layered`` in +snakemake/resources.py): jobs are laid out per toposort level; within a level +non-additive resources (mem_mb, cpus) SUM — split into layers when a global +constraint is exceeded, the group's width being the widest layer — while the +additive resource ``runtime`` is maxed within a layer and SUMMED across layers. +This chain is strictly linear, one job per level, so the group asks for +max(mem_mb) = 8000*attempt, max(threads) = 4 and sum(runtime) = 150 min. +Attempt scaling survives grouping: ``GroupJob.attempt``'s setter clears the +cached group resources and re-sets ``attempt`` on every member (jobs.py), and +``GroupJob.restart_times`` is the max over members — so tile_get_images' +``retries: 2`` still governs. A retry re-runs the whole group, which is safe +because every rule ``rm -rf``s its own run dir at start. + Star catalogues for masking are NOT a prepare-phase concern and not pre-run input: the compute DAG fetches the campaign footprint's stars once (``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in @@ -22,6 +41,7 @@ because it has no mask rule yet — see tile.smk. # config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the # config, not of an injection step. rule tile_get_images: + group: "tile_prep" output: manifest = f"{TILE_DIR}/manifests/tile_get_images.json" log: @@ -38,6 +58,7 @@ rule tile_get_images: sp_shell("tile_get_images", "config_tile_Git.ini") rule tile_uncompress: + group: "tile_prep" input: rules.tile_get_images.output.manifest output: @@ -55,6 +76,7 @@ rule tile_uncompress: sp_shell("tile_uncompress", "config_tile_Uz.ini") rule tile_find_exposures: + group: "tile_prep" input: rules.tile_uncompress.output.manifest output: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 5f8dfacd0..32c5165f6 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -12,7 +12,20 @@ shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires ``///output/run_sp_*`` into its glob, so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -All rules are group-compatible (shell only, no mid-chain localrules). +All rules are group-compatible (shell only, no mid-chain localrules), but only +the tail pair is actually grouped: ``group: "tile_finish"`` on tile_merge_cats +(median 0:15) and tile_make_cat (1:34) — one sbatch per tile for two short jobs +of identical shape (16 GB, 8 threads, 120 min each; the group asks max mem_mb = +16000*attempt, max threads = 8, sum runtime = 240, per the composition rules in +prepare.smk's docstring). Distinct tiles share no edge, so it is one group job +per tile. + +Nothing else on the tile side may join a group. merge_cats' upstream is +tile_ngmix (hours) and make_cat is the terminus, so this pair is the whole +connected short region. tile_merge_headers (0:38) is short but STRANDED: its +upstream is the tile_exp_forest localrule and its only downstream is tile_detect +(16 GB), so any group containing it either fuses a localrule — impossible — or +pulls a heavy rule in. It stays one job per tile, deliberately. Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask @@ -261,6 +274,7 @@ def ngmix_chunkdirs(wc): # The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's # own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). rule tile_merge_cats: + group: "tile_finish" input: manifests = ngmix_manifests, chunkdirs = ngmix_chunkdirs, @@ -285,6 +299,7 @@ rule tile_merge_cats: # No protected(): the full default rerun-triggers govern, and protected() only # ever forced people through a `--forcerun` detour. rule tile_make_cat: + group: "tile_finish" input: ms = rules.tile_merge_cats.output.manifest, store = rules.tile_vignets.output.store, From aee48dc5364ded9518f8918ba41b381eda31a6a4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:42:58 -0400 Subject: [PATCH 036/100] Derive merge_sep_cats chunk paths from the run-directory name MergeSep.process derived chunk k's input with re.sub("1", str(k), path, 1) -- the first literal "1" ANYWHERE in the path. Under the sharded store (.../tiles/21/210.282/...) that lands in a parent directory, so every chunk after the first was looked for somewhere that does not exist, and the workflow config had to carry a relative INPUT_DIR to keep the first "1" inside "Ng1u". chunk_path() now substitutes inside the last "run_*" path component only, which is where the chunk number actually lives, and raises if there is none. The bash pipeline's relative paths are unchanged; the workflow config becomes a fixed $SP_RUN path like every other config. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- .../merge_sep_cats_package/merge_sep_cats.py | 46 +++++++++++++++++-- tests/unit/test_merge_sep_cats_paths.py | 40 ++++++++++++++++ .../config/cfis/config_merge_sep_cats.ini | 9 +--- 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_merge_sep_cats_paths.py diff --git a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py index 5ecdb6fb0..b0be88944 100644 --- a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py +++ b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py @@ -7,7 +7,6 @@ """ import os -import re import warnings import numpy as np @@ -16,6 +15,48 @@ from shapepipe.pipeline import file_io +def chunk_path(input_file, n): + """Chunk Path. + + Derive chunk ``n``'s input path from chunk 1's. + + The separate catalogues live in ShapePipe run directories whose names differ + only in the chunk number (``run_sp_tile_ngmix_Ng1u`` -> + ``run_sp_tile_ngmix_Ng2u``). The substitution is confined to that + run-directory component: replacing the first "1" found anywhere in the path + breaks for absolute paths whose parent directories carry digits, e.g. a + sharded store ``.../tiles/21/210.282/output/run_..._Ng1u/...``. + + Parameters + ---------- + input_file : str + Path to chunk 1's catalogue + n : int + Chunk number + + Returns + ------- + str + Path to chunk ``n``'s catalogue + + Raises + ------ + ValueError + If no run-directory component of the path carries a chunk number + + """ + parts = input_file.split(os.sep) + for idx in reversed(range(len(parts))): + if parts[idx].startswith("run_") and "1" in parts[idx]: + parts[idx] = parts[idx].replace("1", str(n), 1) + return os.sep.join(parts) + + raise ValueError( + f"Cannot derive chunk {n}'s path from '{input_file}': no 'run_*' " + + "directory component contains a chunk number '1'" + ) + + class MergeSep(object): """Merge Sep. @@ -79,8 +120,7 @@ def process(self): input_path_n = [] input_path_n.append(input_file) for n in range(2, self._n_split_max + 1): - res = re.sub("1", str(n), input_file, 1) - input_path_n.append(res) + input_path_n.append(chunk_path(input_file, n)) # Open first catalogue, read number of extensions and columns cat0 = file_io.FITSCatalogue(input_file, SEx_catalogue=True) diff --git a/tests/unit/test_merge_sep_cats_paths.py b/tests/unit/test_merge_sep_cats_paths.py new file mode 100644 index 000000000..754bc76dd --- /dev/null +++ b/tests/unit/test_merge_sep_cats_paths.py @@ -0,0 +1,40 @@ +"""Chunk-path derivation in the merge_sep_cats module.""" + +import pytest + +from shapepipe.modules.merge_sep_cats_package.merge_sep_cats import chunk_path + + +REL = "./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-210-282.fits" +ABS = ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng1u" + "/ngmix_runner/output/ngmix-210-282.fits" +) + + +def test_relative_path_unchanged_behaviour(): + """The bash pipeline's relative INPUT_DIR still resolves as before.""" + assert chunk_path(REL, 3) == ( + "./output/run_sp_tile_ngmix_Ng3u/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_sharded_absolute_path(): + """Digits in the sharded parent dirs and in the file number are untouched.""" + assert chunk_path(ABS, 2) == ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng2u" + "/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_double_digit_chunk(): + assert "Ng12u" in chunk_path(ABS, 12) + + +def test_chunk_one_is_identity(): + assert chunk_path(ABS, 1) == ABS + + +def test_no_run_directory_raises(): + with pytest.raises(ValueError, match="run_"): + chunk_path("/scratch/run/tiles/21/210.282/ngmix-210-282.fits", 2) diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index 750fe1124..9dda5cf0f 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -42,14 +42,7 @@ NUMBER_LIST = $SP_UNIT_NUM # derives the other chunks' paths itself from N_SPLIT_MAX and this pattern # (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the # same fixed output dir. -# -# RELATIVE ON PURPOSE (do not "fix" this to $SP_RUN/...): merge_sep_cats derives -# chunk k's path with re.sub("1", str(k), input_file, 1) -- it replaces the FIRST -# "1" in the path. An absolute path under the sharded store -# (.../tiles/21/210.282/...) has digits before "Ng1u", so chunk 2 would be looked -# for in a nonexistent directory. Every rule runs shapepipe_run with cwd=$SP_RUN, -# so "./output/..." resolves and "Ng1u" carries the first "1". -INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output # Output directory OUTPUT_DIR = $SP_RUN/output From d5c208e3a26cbac88e13824757ee68a48eb6af37 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:43:19 -0400 Subject: [PATCH 037/100] Fail loudly on an unset SP_PHASE and an unresolvable container Three silent degradations, all at parse time: * SP_PHASE unset. A bare `snakemake` parsed as a passthrough, built `rule all` from an index it never built, and exited 0 on an empty DAG. The Snakefile now requires SP_PHASE, and bin/sp's sm() supplies "passthrough" for the escape hatch (`sp --unlock`, `sp --dag`, `sp exp_psf ...`) so the three phases are named rather than inferred from absence. Submitted jobs re-parse under `sbatch --export=ALL`, so they carry the launching phase. * A container: key lost from config.yaml with an empty cache resolved to "", which passes dry-run and only fails on a node. Now a WorkflowError. * SP_CONTAINER pointing at a nonexistent path was silently ignored and the job ran the shared image instead of the one the user named. Now a ContainerError, reported (not raised) by `sp container status`, which is the verb you run when something is wrong. Also: `sp container exec` reads the nibi profile's apptainer-args instead of a hand-rolled --cleanenv, so a one-off gets the same PYTHONPATH pin, --home and binds that jobs do; and status's "behind" hint names the tag `pull` would fetch, which is develop-runtime regardless of the branch you are on. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- workflow/Snakefile | 42 +++++++++++++++---- workflow/bin/sp | 11 ++++- workflow/scripts/container.py | 78 ++++++++++++++++++++++++++++++++--- 3 files changed, 118 insertions(+), 13 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2d9f712de..c89ff6faf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -52,7 +52,21 @@ configfile: str(Path(workflow.snakefile).parent / "config.yaml") sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) import container as _container # noqa: E402 -container: _container.resolve_image()[0] +# Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to +# resolve at all (empty cache AND no `container:` key). The empty string the +# resolver returns for "none" is a valid-looking container directive that passes +# dry-run and only fails once jobs reach a node. +try: + _image, _kind = _container.resolve_image() +except _container.ContainerError as _exc: + raise WorkflowError(str(_exc)) +if _kind == "none": + raise WorkflowError( + f"No container image resolved: no sandbox, no cached SIF, and no " + f"'{_container.CONFIG_KEY}:' key in {_container.CONFIG_FILE}. " + f"Run `sp container pull`, or restore the key.") + +container: _image # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) @@ -76,12 +90,26 @@ sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 from completeness import STAGE_DIR # noqa: E402 -# SP_PHASE is set by bin/sp on the two invocations of `sp run` and NOWHERE else. -# It gates the two parse-time side effects — the index build and the report -# hooks — so that a passthrough invocation (`sp --unlock`, `sp --dag`, `sp -# exp_psf ...`) never mutates durable state or dies on a threshold it was not -# asked about. Unset means "read the index, build nothing". +# SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two +# invocations of `sp run`, `passthrough` on the escape hatch (`sp --unlock`, `sp +# --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index +# build and the report hooks — so that a passthrough parse never mutates durable +# state or dies on a threshold it was not asked about. +# +# Requiring it to be SET is the point of the check below: a bare `snakemake` +# would otherwise parse as a passthrough, find `rule all` gated on an index it +# never built, and exit 0 on an empty DAG. Jobs re-parse this file under the +# slurm executor and inherit the head process's environment, so a submitted job +# always carries the launching phase. PHASE = os.environ.get("SP_PHASE", "") +if not PHASE: + raise WorkflowError( + "SP_PHASE is not set — run the workflow through workflow/bin/sp " + "(`sp run`), which exports it. A bare snakemake invocation would " + "silently build an empty DAG.") +if PHASE not in ("prepare", "compute", "passthrough"): + raise WorkflowError( + f"SP_PHASE={PHASE!r} is not one of prepare, compute, passthrough.") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] @@ -93,7 +121,7 @@ with open(config["tile_list"]) as f: # chunk number is not ours alone: it names the run dir # (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and # merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the -# first "1" — which only works for bare integers. +# chunk number in that run-directory name — which only works for bare integers. scattergather: ngmix=int(config.get("ngmix_chunks", 8)) diff --git a/workflow/bin/sp b/workflow/bin/sp index 72e80b783..d5f7bfb5a 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -53,7 +53,16 @@ export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" # --snakefile pins the workflow to this checkout: sp must work from any cwd # (an sbatch head job starts in the submission directory, not the repo). -sm() { snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } +# +# SP_PHASE is REQUIRED by the Snakefile (a bare `snakemake` would build an empty +# DAG and exit 0). `sp run` sets prepare/compute on its two invocations; every +# other verb and the escape hatch fall through to `passthrough`, which parses the +# index without building it and schedules no side effects. +sm() { + SP_PHASE="${SP_PHASE:-passthrough}" \ + snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" \ + --directory "$STATE_DIR" "$@" +} cmd="${1:-}" case "$cmd" in diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py index 0479b0f46..bbd7e0bb9 100644 --- a/workflow/scripts/container.py +++ b/workflow/scripts/container.py @@ -37,11 +37,21 @@ import argparse import os import re +import shlex import shutil import subprocess import sys from pathlib import Path + +class ContainerError(Exception): + """A misconfiguration the caller must fix (bad override, no image at all). + + Raised rather than ``sys.exit``ed so the Snakefile, which imports this + module at parse time, can turn it into a WorkflowError instead of a + SystemExit. ``main`` below turns it back into a one-line CLI error. + """ + # The published image. CI pushes one tag per branch, sanitized; `-runtime` is # the slim variant the workflow runs. CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" @@ -52,6 +62,12 @@ CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" CONFIG_KEY = "container" +# The profile whose `apptainer-args:` every workflow job runs under. `exec` reads +# it at runtime rather than restating it, so a one-off `sp container exec` and a +# job see the same environment (the PYTHONPATH pin above all: a divergence there +# means the one-off imports a different src/ than the workflow does). +PROFILE_FILE = Path(__file__).resolve().parents[2] / "profiles" / "nibi" / "config.yaml" + # ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto # a filesystem with room), XDG_CACHE_HOME moves it with everything else. CACHE_DIR = Path( @@ -85,6 +101,20 @@ def configured_default(): return match.group(1) if match else None +def profile_apptainer_args(): + """Return the profile's ``apptainer-args`` as a token list, or ``[]``. + + Minimal scalar read again (stdlib-only); a missing or unreadable profile + yields ``[]``, which callers replace with their own defaults. + """ + try: + text = PROFILE_FILE.read_text() + except OSError: + return [] + match = re.search(r'^apptainer-args:[ \t]*"(.*)"[ \t]*$', text, re.MULTILINE) + return shlex.split(match.group(1)) if match else [] + + def local_sif(): """Return this user's cached image path (may not exist yet).""" override = os.environ.get("SP_CONTAINER") @@ -110,6 +140,14 @@ def resolve_image(): sif = local_sif() if sif.exists(): return str(sif), "sif" + # An override that resolves to nothing is a typo, never an intention: falling + # through to the shared image would run the job against something other than + # what the user named. + if os.environ.get("SP_CONTAINER"): + raise ContainerError( + f"SP_CONTAINER={os.environ['SP_CONTAINER']} does not exist " + f"(resolved to {sif}). Unset it, or point it at an image that does." + ) default = configured_default() if default: return default, "configured" @@ -281,7 +319,13 @@ def cmd_status(args): """Report which image layer is live, its revision, and how current it is.""" sif = local_sif() sandbox = local_sandbox() - active, kind = resolve_image() + # status is the verb you run WHEN something is wrong, so a broken override is + # reported here rather than raised. + try: + active, kind = resolve_image() + except ContainerError as exc: + print(f"active: NONE -- {exc}") + return 1 if sif.exists(): print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") @@ -323,7 +367,11 @@ def cmd_status(args): verdict = compare_revision(revision) explain = { "in-sync": "matches this checkout's HEAD", - "behind": "older than this checkout's HEAD -- pull to refresh", + "behind": ( + "older than this checkout's HEAD -- `sp container pull` fetches " + f"{CONTAINER_URI}, which is not built from this branch unless you " + "pass --tag" + ), "ahead": "newer than this checkout's HEAD", "diverged": "on a different branch from this checkout", "unknown": "cannot compare (no label, or a commit this clone lacks)", @@ -338,7 +386,24 @@ def cmd_exec(args): command = [a for a in args.command if a != "--"] if not command: sys.exit("nothing to run; pass a command after `exec`") - binds = args.bind or os.environ.get("SP_APPTAINER_BINDS", DEFAULT_BINDS) + + # Same environment the workflow's jobs get: the profile's apptainer-args + # verbatim (--cleanenv, the PYTHONPATH pin, --home, its binds). Explicit + # binds still win, and a profile that cannot be read falls back to the old + # standalone defaults. + env_args = profile_apptainer_args() + explicit_binds = args.bind or os.environ.get("SP_APPTAINER_BINDS") + if not env_args: + print( + f"warning: could not read apptainer-args from {PROFILE_FILE}; " + "falling back to --cleanenv and the default binds, which may not " + "match what jobs run under", + file=sys.stderr, + ) + env_args = ["--cleanenv"] + explicit_binds = explicit_binds or DEFAULT_BINDS + if explicit_binds: + env_args += ["--bind", explicit_binds] if args.writable: # A SIF is a read-only filesystem, so `--writable` against one fails @@ -356,7 +421,7 @@ def cmd_exec(args): sys.exit("no image resolved; run: sp container pull") extra = [] - cmd = ["apptainer", "exec", *extra, "--cleanenv", "--bind", binds, image, *command] + cmd = ["apptainer", "exec", *extra, *env_args, image, *command] return subprocess.run(cmd).returncode @@ -425,7 +490,10 @@ def build_parser(): def main(argv=None): args = build_parser().parse_args(argv) - return args.func(args) + try: + return args.func(args) + except ContainerError as exc: + sys.exit(str(exc)) if __name__ == "__main__": From 15c4e6c66b612bcf8ef8e97b8eb8dde7cdafef56 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:43:19 -0400 Subject: [PATCH 038/100] Group tile_exp_forest with tile_merge_headers The docstring called tile_exp_forest a localrule and closed the question on that premise; it is not one (Snakefile's localrules list does not name it), so the two short jobs at the head of the tile chain can fuse: one sbatch per tile asking max mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also consumes the forest, but a consumer outside the group is an ordinary edge on the group job and does not pull its 16 GB in. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- workflow/rules/tile.smk | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 32c5165f6..8fc230c29 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -12,20 +12,22 @@ shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires ``///output/run_sp_*`` into its glob, so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -All rules are group-compatible (shell only, no mid-chain localrules), but only -the tail pair is actually grouped: ``group: "tile_finish"`` on tile_merge_cats -(median 0:15) and tile_make_cat (1:34) — one sbatch per tile for two short jobs -of identical shape (16 GB, 8 threads, 120 min each; the group asks max mem_mb = -16000*attempt, max threads = 8, sum runtime = 240, per the composition rules in -prepare.smk's docstring). Distinct tiles share no edge, so it is one group job -per tile. +All rules are group-compatible (shell only, no mid-chain localrules), and the two +short regions are grouped, per the composition rules in prepare.smk's docstring. +Distinct tiles share no edge, so each is one group job per tile: -Nothing else on the tile side may join a group. merge_cats' upstream is -tile_ngmix (hours) and make_cat is the terminus, so this pair is the whole -connected short region. tile_merge_headers (0:38) is short but STRANDED: its -upstream is the tile_exp_forest localrule and its only downstream is tile_detect -(16 GB), so any group containing it either fuses a localrule — impossible — or -pulls a heavy rule in. It stays one job per tile, deliberately. +* ``group: "tile_gather"`` — tile_exp_forest (2 GB, 20 min) and + tile_merge_headers (median 0:38; 8 GB, 4 threads, 120 min). The group asks max + mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also + consumes the forest, but a consumer OUTSIDE the group is just an ordinary DAG + edge on the group job — it does not pull tile_detect (16 GB) in. +* ``group: "tile_finish"`` — tile_merge_cats (median 0:15) and tile_make_cat + (1:34), two short jobs of identical shape (16 GB, 8 threads, 120 min each; + max mem_mb = 16000*attempt, max threads = 8, sum runtime = 240). + +Nothing else joins either. The heavy middle (tile_detect, tile_vignets, +tile_ngmix) is where the wall time is, and fusing a short rule onto one of those +would reserve its footprint for the short job too. Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask @@ -119,6 +121,7 @@ def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_ex # Its output stays a directory() (it has no ShapePipe run dir and no manifest — # it is not a shapepipe_run at all). rule tile_exp_forest: + group: "tile_gather" input: tile_exp_all output: @@ -142,6 +145,7 @@ rule tile_exp_forest: # (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). # Reads headers-*.npy through the forest -> the split manifests are the edge. rule tile_merge_headers: + group: "tile_gather" input: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, From 109739db43c2640656603705776395421142cea4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:35:49 -0400 Subject: [PATCH 039/100] fix(psf): deterministic star train/validation split; align tile STAR_THRESH with the 80% convention The 80/20 star split drew from unseeded np.random, so the PSF star sample differed between identical runs; it is now a permutation seeded from the unit's file number (SEED_FROM_POSITION philosophy). The tile PSF-interp configs kept the pre-split STAR_THRESH=20 that fdc86553 raised to 22 in the validation config; align the science path on 22. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Jrs8TeTCccRMC45QVGKPeY (cherry picked from commit 1da350ced18b425bcfc635f56d080697e0a94e71) --- example/cfis/config_tile_PiViVi_canfar_sx.ini | 2 +- example/cfis/config_tile_PiViVi_canfar_uc.ini | 2 +- .../modules/setools_package/setools.py | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/example/cfis/config_tile_PiViVi_canfar_sx.ini b/example/cfis/config_tile_PiViVi_canfar_sx.ini index ca7efd9ec..991153143 100644 --- a/example/cfis/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis/config_tile_PiViVi_canfar_sx.ini @@ -79,7 +79,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 diff --git a/example/cfis/config_tile_PiViVi_canfar_uc.ini b/example/cfis/config_tile_PiViVi_canfar_uc.ini index d59af1c4a..b33361b6d 100644 --- a/example/cfis/config_tile_PiViVi_canfar_uc.ini +++ b/example/cfis/config_tile_PiViVi_canfar_uc.ini @@ -77,7 +77,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 diff --git a/src/shapepipe/modules/setools_package/setools.py b/src/shapepipe/modules/setools_package/setools.py index c430b52b1..33d8efa8b 100644 --- a/src/shapepipe/modules/setools_package/setools.py +++ b/src/shapepipe/modules/setools_package/setools.py @@ -658,13 +658,18 @@ def _make_rand_split(self): cat_size = len(np.where(mask)[0]) n_keep = int(np.ceil(cat_size * ratio)) - mask_ratio = [] - mask_left = list(range(0, cat_size)) - while len(mask_ratio) != n_keep: - idx = np.random.randint(0, len(mask_left)) - mask_ratio.append(mask_left.pop(idx)) - mask_ratio = np.array(mask_ratio) - mask_left = np.array(mask_left) + # Deterministic split, seeded from the unit's file number: the + # train/validation assignment is a pure function of the input + # catalogue, so the PSF star sample (and everything downstream + # of the PSF model) is reproducible run-to-run. An unseeded + # np.random here made the shear catalogue non-reproducible + # upstream of ngmix's own position seeding. + seed = int( + re.sub(r"\D", "", self._file_number_string) or 0 + ) % (2 ** 32) + perm = np.random.RandomState(seed).permutation(cat_size) + mask_ratio = perm[:n_keep] + mask_left = np.sort(perm[n_keep:]) self.rand_split[key]["mask"] = mask self.rand_split[key][f"ratio_{int(ratio * 100)}"] = mask_ratio self.rand_split[key][f"ratio_{100 - int(ratio * 100)}"] = mask_left From 90782098f17b138823d3635df9879f477f06db69 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 18:53:37 -0400 Subject: [PATCH 040/100] fix(configs): the workflow's own PiViVi config takes STAR_THRESH 22 too #873 aligned the tile PSF-interp threshold with the 80/20 split convention in example/cfis, the only place those configs exist on develop. This branch carries a committed fork of that chain under workflow/config/cfis (#848 D2), so the science path the Snakemake run actually reads kept the stale 20 after the cherry-pick. Mirror it. The general hazard: every science-config change landing against example/cfis must be mirrored here until the two chains are unified. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VUNNyG8tw6Mdbvo4SVNrhp --- workflow/config/cfis/config_tile_PiViVi.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini index 246ee18f9..22294213f 100644 --- a/workflow/config/cfis/config_tile_PiViVi.ini +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -83,7 +83,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 From 43987f6a8828835c1f2a0729cd0957f7d9b1df32 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 19:01:40 -0400 Subject: [PATCH 041/100] feat(orchestration): the persistent root D5 always described MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D5 has always specified two roots — a persistent one for the durable, low-volume products and a scratch one for bulk intermediates — and the implementation has had one, because def-mjhudson /project was hard-full at 27/27 TiB when the campaign started. Space has since been reclaimed, so the design can land. `products_dir:` is the persistent root. It defaults to `run_dir`, so a fixture or a smoke test still needs exactly one path; when set, it takes the tree that mirrors the scratch one shard for shard and holds the final catalogues, the index, missing.json and the report. The star-catalogue sky store moves there too: it is the one product in this workflow bought with something local compute cannot re-buy — ~1.5k catalogue-server queries at DR6 scale. The final catalogue is the case that matters most, and not only because it is the science product. `tile_finished()` keys on its path to cut a finished tile's exposure edges, so on scratch a purge would not merely lose catalogues: it would make every finished tile re-declare inputs against exposure stores reclamation had already deleted, which is the rerun avalanche the cut exists to prevent, arriving by a different door. Snakemake's own state stays on scratch — the one durable-looking thing that does. D5's parenthetical listed it under the persistent root; that was wrong on its own terms. It is one small, hot metadata file per output, rewritten every job, >170k files of churn at DR6 against a backed-up filesystem with a 1M-inode group quota, and it is reconstructible: losing it costs a re-parse, not a rerun. The comment in bin/sp now says so instead of blaming the quota that first exposed it. Also refreshed: the profile's claim that this branch never touches src/, which four commits ago stopped being true. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VUNNyG8tw6Mdbvo4SVNrhp --- profiles/nibi/config.yaml | 11 ++++-- workflow/Snakefile | 24 +++++++++++- workflow/bin/sp | 12 ++++-- workflow/config.yaml | 79 +++++++++++++++++++++++---------------- workflow/rules/tile.smk | 2 +- 5 files changed, 87 insertions(+), 41 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 65746786f..7b75b458f 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,12 +37,15 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp -# PYTHONPATH SETTLED CALL 3 — P0 pins THIS branch's src/ (identical to -# develop@97e16d50: orchestration commits never touch src/). +# PYTHONPATH SETTLED CALL 3 — pins THIS branch's src/, which is +# develop@97e16d50 plus the four commits that genuinely need +# module code: ngmix chunk fields + position-seeded RNG, +# merge_sep_cats chunk paths, the vizier star-cat helpers, +# and the cherry-picked #873 (seeded setools split). # NOT shapepipe-prod (drifted to a PR branch mid-run, and the # live p3-batch1 job reads it) and not the sif default -# (frozen pre-#843). Production later rebuilds the sif at the -# validated commit and DROPS this --env line. +# (frozen pre-#843, and pre-#873). Production later rebuilds +# the sif at the validated commit and DROPS this --env line. apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job diff --git a/workflow/Snakefile b/workflow/Snakefile index c89ff6faf..0f2541334 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -69,7 +69,16 @@ if _kind == "none": container: _image # --- paths ----------------------------------------------------------------- +# TWO ROOTS (D5). RUN_DIR is the scratch root: bulk intermediates, sized so a +# batch finishes inside the purge window. PRODUCTS_DIR is the persistent root +# for the durable, low-volume products — the final catalogues, the index, the +# report. Snakemake's own state is the one durable-looking thing that stays on +# scratch (bin/sp explains why: tens of thousands of small, hot metadata writes +# against a backed-up, file-count-limited filesystem). RUN_DIR = Path(config["run_dir"]) +# Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no +# second path: one root, exactly the pre-D5 layout. +PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) # Run-independent root for the mask star catalogues: the HEALPix chunk store the # star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it # (config.yaml explains the placement). @@ -191,6 +200,9 @@ wildcard_constraints: # is built by these helpers, so a mismatched (shard, id) pair is never requested. TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") +# The persistent root mirrors the scratch one, shard for shard, so the two trees +# read as the same campaign seen from two filesystems. +PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") def tile_dir(tile): return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" @@ -208,7 +220,17 @@ def forest_dir(tile): return f"{tile_dir(tile)}/exp_forest" def final_cat(tile): - return f"{tile_dir(tile)}/final_cat-{tile}.fits" + """The campaign's science product, and its tile-finished marker. + + It lives on the PERSISTENT root, not the scratch one, and both halves of + that sentence are load-bearing. As a product: it is what the campaign is + for, and a 60-day purge must not eat it. As a marker: tile_finished() keys + on this path to cut a finished tile's exposure edges (D5), so if a purge + could remove it, finished tiles would re-declare inputs against exposure + stores that reclamation deleted — the rerun avalanche the cut exists to + prevent, arriving by way of the purge instead. + """ + return f"{PRODUCTS_DIR}/tiles/{tile[:2]}/{tile}/final_cat-{tile}.fits" def unit_num(unit): """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading diff --git a/workflow/bin/sp b/workflow/bin/sp index d5f7bfb5a..921b95d8d 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -42,9 +42,15 @@ cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO -# THE RUN on /scratch, never on /project (a hard 27/27 TiB group quota killed a -# metadata write mid-run, live). --directory only moves state: all data paths are -# absolute, and the Snakefile resolves its own configfile. +# THE RUN on /scratch, never on the persistent root — the one exception to D5's +# "durable, low-volume products go on /project". It is neither: one small, hot +# metadata file per output, rewritten on every job, which at DR6 is >170k files +# of churn against a backed-up filesystem with a 1M-inode group quota. It is also +# reconstructible — losing it costs a re-parse, not a re-run. (The lesson arrived +# the hard way: a hard 27/27 TiB group quota killed a metadata write mid-run, +# live. The quota has since eased; the placement is right on its own merits.) +# --directory only moves state: all data paths are absolute, and the Snakefile +# resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of diff --git a/workflow/config.yaml b/workflow/config.yaml index c81d4e45d..e67b6867e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -6,21 +6,43 @@ # The tile list that scopes this run (one "IDra.IDdec" per line). # The campaign grows by appending to this file — parse-time config, so -# completed work is never invalidated. Current contents: the 186/187 quad -# (18 exposures, complete and validated against the bash baseline), plus the -# first 30 tiles of the p3-batch1 list (100 unique exposures — chosen so the -# worst-case un-cleaned exposure high-water ~660G fits the scratch quota with -# margin; the remaining 20 batch1 tiles are the next append). (The 210/211 -# quad used earlier is unusable: its tile images are symlinks into anaennis' -# moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are broken.) -tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt +# completed work is never invalidated. Current contents: the 34-tile smk-g4 +# benchmark set, identical to the tile list smk-g3 finished (the 186/187 quad +# plus the first 30 p3-batch1 tiles), so the two campaigns' catalogues are +# comparable object for object. It lives on the persistent root because it +# defines the campaign and must outlive the scratch purge. +# (The 210/211 quad used earlier is unusable: its tile images are symlinks into +# anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are +# broken.) +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g4/tiles34.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif -# Where products land ($SP_RUN root). The sharded per-unit stores live under here: +# THE TWO ROOTS (D5). +# +# run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk +# intermediates, sized so a batch finishes inside the 60-day purge window. The +# sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-g3 +run_dir: /scratch/cdaley/shapepipe-output/smk-g4 + +# products_dir is the PERSISTENT root: the durable, low-volume products — the +# final catalogues (/tiles///final_cat-.fits, +# mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB +# per tile, so a full DR6 campaign is a few hundred GB. +# +# The final catalogue is also the tile-finished MARKER that cuts a finished +# tile's exposure edges (D5), which is the second reason it cannot sit on +# scratch: a purge would not merely lose a product, it would make every finished +# tile re-declare inputs against exposure stores reclamation already deleted. +# +# Unset means "one root": products land under run_dir, exactly the pre-D5 +# layout, which is what a fixture or smoke test wants. +# +# Snakemake's own state is the one durable-looking thing that stays on scratch +# (-state; bin/sp explains why). +products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g4 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -38,8 +60,12 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # `exp_star_cat` makes from those chunks, with no network at all. # Network therefore scales with SKY AREA, not exposure count: exposures overlap # ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. -# Design intent is /project — this is a durable science product, not scratch -# work — blocked only by the quota below. +# +# On the PERSISTENT root: the sky store is a durable science product bought with +# ~1.5k catalogue-server queries at DR6 scale, and re-buying it after a scratch +# purge is the one cost in this workflow that cannot be paid with local compute. +# (It sat on scratch through smk-g3 only because def-mjhudson /project was then +# hard-full at 27/27 TiB.) # # THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to # workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun @@ -47,32 +73,21 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # is skipped and only re-cut. Clear the store by hand when the change must reach # the data. Changing NSIDE or the catalogue ID is the exception: those name the # directory, so a change there fetches into a new one beside the old. -# -# On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day -# purge therefore applies. A purged per-exposure cut is free to remake, and -# exp_star_cat declares its ccd-0 farm link as an output so a purge leaves that -# link dangling and the rule reruns rather than trusting a manifest that outlived -# its data. A purged CHUNK costs its query again: exp_star_cat fails loudly on a -# missing chunk, and re-running `star_catalogue` refetches it. -star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache +star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache -# The run index. Design intent (finding 15) is durable products on /project — -# BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and -# report live with the run on /scratch until space is reclaimed. Mind the -# 60-day purge for anything that must survive. -index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite +# The run index, and — sharing its directory — missing.json and run_report.json. +# On the persistent root with the catalogues (D5): the index is the record of +# which tile reads which exposure, so it is what a post-purge reconstruction +# would otherwise have to rebuild from tile headers. +index_db: /project/def-mjhudson/cdaley/sp-products/smk-g4/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, # and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. On here: -# the quad is validated against the bash baseline (validation verdict -# p0night1), so its exposure stores' inspection value is spent, and the -# campaign is scaling into the p3-batch1 tiles — 181 exposures un-cleaned -# would be ~1.2 TB against a 1 TiB scratch quota, so rolling reclamation is -# load-bearing from here on. Flipping it on reclaims retroactively — the -# missing tombstones schedule exactly the outstanding clean jobs. +# back out of the tombstone and reports the exposure as `cleaned`. On for smk-g4: +# reclamation is part of what this campaign measures, and at 34 tiles from +# scratch the un-cleaned exposure high-water is the run's binding disk constraint. clean: true # Tiles that may NOT pin an exposure store (default: empty). diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8fc230c29..2739a1932 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -309,7 +309,7 @@ rule tile_make_cat: store = rules.tile_vignets.output.store, output: manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", - final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", + final_cat = f"{PROD_TILE_DIR}/final_cat-{{tile}}.fits", log: f"{TILE_DIR}/logs/tile_make_cat.json" params: From a0311c28342ec72ec6f86f5710e0e7ad51dd3535 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 21:27:32 -0400 Subject: [PATCH 042/100] feat(orchestration): the shape chain becomes one node-local job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit said the pipeline was I/O-bound, not CPU-bound: tile_ngmix held four cores and used 0.12 of one, spending 97% of a 7h34m median waiting on small random reads from a 95%-full NFS filer. The eight chunks of a tile each re-read the same 5.6 GB vignette store, so a tile that wrote 5.6 GB read ~163 GB back. Every granite node has 3.5 TB of ext4 NVMe at /local, handed to a job as $SLURM_TMPDIR. The workflow could not reach it: apptainer bound only /project and /scratch, and --cleanenv strips the path. Both are fixed in the profile. Node-local storage dies with the job, so it can only hold something whose producer and consumers are the same job. Hence the fusion: tile_vignets, the eight tile_ngmix chunks, tile_merge_cats and tile_make_cat become one `tile_shape` group job, and the vignette store is written to the node's NVMe and never appears on scratch at all. That is ~190 GB of peak removed at 34 tiles, on a filesystem whose quota is what currently caps batch size. Measured on one chunk (job 20795277, tile 198.305): 1h07m against a 7h34m baseline, 6.75x, with the staging copy costing 14 seconds. TotalCPU was 41m48s over 67m of wall — 0.62 cores busy, below one — which settles that ngmix is single-threaded. `-b` is SMP batch size over the process list (job_handler.py:381), and a chunk is one process; the module has no internal parallelism. threads drops 4 -> 1 accordingly. Runtimes are re-declared from measured p99 rather than the old I/O-bound ceilings, because the group's composed runtime is what gates partitions: 20 + 120 + 10 + 15 = 165 min, under cpubase_bycore_b1's 180, so the fused job reaches the widest partition set plus backfill. Composition was verified from snakemake 9.23.1's source and against a real sbatch: 8 CPUs, 112000M, 02:45:00. tile_detect stays outside the group. It is a 16 GB SExtractor run the shape chain does not need co-scheduled, and its runtime has no room in that sum. The store is no longer a declared output, so a resume can re-plan the group without tile_vignets while still scheduling the chunks. TILE_VIGNET_REQUIRED fails loudly and prints the fix rather than letting that become a confusing missing-input error; temp()-ing the manifest instead would break clean_exposure, which keys reclamation on it. completeness.py is deliberately untouched — its --run-dir/--unit flags were already the right hooks, and its hash is a param on every rule, so editing it would have invalidated the 117 completed exposure chains. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VUNNyG8tw6Mdbvo4SVNrhp --- profiles/nibi/config.yaml | 43 ++- workflow/config/cfis/config_tile_Mc.ini | 6 +- .../config/cfis/config_tile_Ng_template.ini | 21 +- workflow/config/cfis/config_tile_PiViVi.ini | 20 +- workflow/rules/tile.smk | 290 ++++++++++++++++-- 5 files changed, 352 insertions(+), 28 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 7b75b458f..276cc0143 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,6 +37,47 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# --bind /local NODE-LOCAL NVME. $SLURM_TMPDIR on nibi is +# /local/scratch/..0, and the mount is /local +# (/dev/nvme0n1, ext4, 3.5 TB) -- binding /localscratch is +# NOT enough, the real path is under /local (verified, +# probe job 20794182). The fused `tile_shape` group job +# WRITES its 5.6 GB vignette store there and never puts it +# on scratch at all (workflow/rules/tile.smk); every other +# rule simply ignores the bind. Adding it does NOT +# invalidate anything: the `software-env` rerun trigger +# hashes only job.container_img_url (+ conda env, env +# modules) -- +# snakemake/persistence/__init__.py::_software_stack_hash, +# v9.23.1 -- and apptainer-args is not in that hash. +# --env SLURM_TMPDIR=$SLURM_TMPDIR +# HOW the in-container shell LEARNS its node-local dir. +# --cleanenv strips the real SLURM_TMPDIR, and the two +# alternatives do not work here: +# * {resources.tmpdir} is NOT $SLURM_TMPDIR. Snakemake +# defers that resource to the job and evaluates it as +# tempfile.gettempdir(); nibi's Slurm does not export +# TMPDIR, so it lands on /tmp -- which on a compute +# node is a 378 GB RAM-backed tmpfs whose pages are +# charged to the job's memory cgroup. Staging 6 GB +# there would eat the job's RAM, not use the NVMe. +# (Measured, job 20794656.) +# * APPTAINERENV_SLURM_TMPDIR would have to be exported +# by something that already knows the JOB's value; +# bin/sp runs on the login node, where it is unset. +# This works because snakemake splices apptainer-args +# verbatim into a command it runs with Popen(shell=True) +# (snakemake/deployment/singularity.py::shellcmd + +# shell.py), so the JOB's /bin/sh expands $SLURM_TMPDIR at +# submission-shell time, in the job, where the value is +# known. Verified in-container, not assumed. +# Rules outside the tile_shape group ignore the variable; +# on the login node (localrules) it expands to empty, +# which apptainer accepts. Every tile_shape member REFUSES +# TO RUN if what arrives is not a real directory +# (tile.smk), so a broken passthrough is a loud failure -- +# never a silent slide back onto NFS, and never a 5.6 GB +# write into the RAM-backed /tmp tmpfs. # PYTHONPATH SETTLED CALL 3 — pins THIS branch's src/, which is # develop@97e16d50 plus the four commits that genuinely need # module code: ngmix chunk fields + position-seeded RNG, @@ -46,7 +87,7 @@ software-deployment-method: [apptainer] # live p3-batch1 job reads it) and not the sif default # (frozen pre-#843, and pre-#873). Production later rebuilds # the sif at the validated commit and DROPS this --env line. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch --bind /local --env SLURM_TMPDIR=$SLURM_TMPDIR" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 502731431..149e4d9e8 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -56,7 +56,11 @@ TIMEOUT = 96:00:00 [MAKE_CAT_RUNNER] # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output +# The psfex_interp store is NODE-LOCAL: it is the vignette-store run this +# tile's fused group job wrote to $SLURM_TMPDIR (see config_tile_PiViVi.ini +# and workflow/rules/tile.smk). $NGMIX_VIGNET_DIR names it; the other two +# inputs and this run's own output stay on $SP_RUN. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output # Input file pattern(s), list of strings with length matching number of expected input file types # Cannot contain wild cards diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index 6e3e33292..65c769060 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -62,7 +62,24 @@ TIMEOUT = 96:00:00 # Model-fitting shapes with ngmix [NGMIX_RUNNER] -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output +# INPUT and OUTPUT are independent here, and that is what makes node-local +# staging possible: [FILE] OUTPUT_DIR stays $SP_RUN/output (scratch, durable +# for the life of the job's tile), while the two directories ngmix actually +# READS AT RANDOM -- the psfex_interp galaxy_psf store and the vignetmaker +# run_2 image/background/weight/flag vignet stores, together the ~5.6 GB +# per-tile vignette store that all 8 chunks re-read -- come from +# $NGMIX_VIGNET_DIR. +# +# $NGMIX_VIGNET_DIR is the tile's run_sp_tile_PiViVi directory. tile_ngmix +# (workflow/rules/tile.smk) copies that directory to the job's node-local +# NVMe ($SLURM_TMPDIR) and points this variable at the copy; the outputs +# below are unaffected and still land on scratch. Set it to +# $SP_RUN/output/run_sp_tile_PiViVi to read the store in place. +# +# It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset $NGMIX_VIGNET_DIR is +# a loud error rather than a silent read of the wrong tree. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers @@ -75,7 +92,7 @@ NUMBERING_SCHEME = -000-000 # 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for # every tile (missing file -> error, no per-tile fallback); omit the option # entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite +BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite # Number of objects to batch save during processing, optional. Omit or set # to -1 for no batch saving. diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini index 22294213f..2e1547f3e 100644 --- a/workflow/config/cfis/config_tile_PiViVi.ini +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -43,8 +43,24 @@ NUMBER_LIST = $SP_UNIT_NUM # Input directory, containing input files, single string or list of names INPUT_DIR = . -# Output directory -OUTPUT_DIR = $SP_RUN/output +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT ## ShapePipe job handling options diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 2739a1932..cabf2a211 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -21,13 +21,21 @@ Distinct tiles share no edge, so each is one group job per tile: mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also consumes the forest, but a consumer OUTSIDE the group is just an ordinary DAG edge on the group job — it does not pull tile_detect (16 GB) in. -* ``group: "tile_finish"`` — tile_merge_cats (median 0:15) and tile_make_cat - (1:34), two short jobs of identical shape (16 GB, 8 threads, 120 min each; - max mem_mb = 16000*attempt, max threads = 8, sum runtime = 240). +* ``group: TILE_GROUP`` ("tile_shape") — tile_vignets -> 8 x tile_ngmix -> + tile_merge_cats -> tile_make_cat, THE WHOLE SHAPE CHAIN AS ONE JOB. This is + not a latency optimisation; it is what lets the 5.6 GB vignette store live on + node-local NVMe and never touch /scratch (see the TILE_LOCAL block below). + Composition, verified against snakemake 9.23.1 and a real sbatch: + cpus = max over levels of summed siblings = max(8, 8x1, 8, 8) = 8; + mem_mb = max(32000, 8x14000, 16000, 16000) = 112000; + runtime = sum along the chain of each level's MAX = 20 + 120 + 10 + 15 = 165. + 165 min is under the 180 min ceiling of ``cpubase_bycore_b1``, so the fused + job reaches the widest partition set (plus cpubackfill) — which is why each + member's runtime is measured p99 plus margin and not the old ceiling. -Nothing else joins either. The heavy middle (tile_detect, tile_vignets, -tile_ngmix) is where the wall time is, and fusing a short rule onto one of those -would reserve its footprint for the short job too. +The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor +run that the shape chain does not need co-scheduled, and folding it in would add +its runtime to a sum that has no room. Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask @@ -43,6 +51,144 @@ built the same way and for the same reason (the file handler intersects numbers across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). """ +# --- the node-local tile root (the I/O + scratch fix) ---------------------- +# +# MEASURED PROBLEM. tile_ngmix was I/O-bound, not CPU-bound: median 7h34m +# elapsed against 52 min of CPU (3% efficiency, 0.12 of 4 reserved cores). +# Each chunk read ~20 GB at 0.78 MB/s and all 8 chunks of a tile re-read the +# SAME 5.6 GB vignette store -- ~163 GB of small random sqlite preads per tile +# against /scratch, which on nibi is NFS (VAST, 4.5 PB, 95% full). What hurts +# is per-read LATENCY, not bandwidth: the sequential local-vs-scratch gap is +# only ~3x and is NOT the argument for this change. +# +# THE FIX IS THE FUSE. tile_vignets -> 8 x tile_ngmix -> tile_merge_cats -> +# tile_make_cat run as ONE group job on ONE node, and the vignette store is +# WRITTEN to that node's NVMe and never lands on /scratch at all. So the store +# is not copied, it is simply never remote: zero staging cost, zero NFS random +# reads, and -- the reason this was chosen over per-chunk staging -- the +# per-tile scratch high-water drops by the whole 5.6 GB store (~190 GiB across +# 34 concurrent tiles). Scratch quota, not speed, is what caps batch size. +# +# WHAT STAYS ON SHARED STORAGE, and it is everything that is DAG currency: +# every manifest (the success sentinels), every verdict log, the ngmix chunk +# dirs merge_sep_cats gathers, and final_cat on the PERSISTENT root. Only the +# bulk intra-tile intermediate is node-local. That split is possible because +# ShapePipe's configs set input and output paths independently -- see +# config_tile_PiViVi.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and +# config_tile_Ng_template.ini / config_tile_Mc.ini ($NGMIX_VIGNET_DIR). +# +# WHY IT NEEDS THE GROUP. $SLURM_TMPDIR is per SLURM JOB. Unfused, tile_vignets +# and tile_ngmix are different jobs on different nodes and the store MUST be on +# shared storage. Fused, all members share one job, one node and one +# $SLURM_TMPDIR, so the store written by the first member is simply there for +# the rest. (Verified: group members see the same $SLURM_TMPDIR.) +# +# HOW $SLURM_TMPDIR GETS IN HERE. profiles/nibi passes --bind /local and +# --env SLURM_TMPDIR=$SLURM_TMPDIR through apptainer-args; --cleanenv strips +# the real one. See that file for why {resources.tmpdir} is NOT usable (it +# resolves to /tmp, a RAM-backed tmpfs) and why the expansion happens in the +# job. The guard below turns any breakage in that chain into a hard failure +# rather than a silent fall back onto NFS -- or, worse, a 5.6 GB write into a +# tmpfs charged to the job's memory cgroup. +# +# THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, +# not one chunk, because the store dies with the job. At ~1 h per fused tile +# that is a cheap trade for the scratch it buys. Noted, not engineered around. +# +# It lives in each rule's `pre_run`, i.e. in THAT RULE's params.pre. The shared +# prologue (unit_pre) and completeness.py are untouched by design: both are +# fingerprinted into every rule, and changing either would invalidate the whole +# campaign, including the 117 finished exposure chains. +TILE_LOCAL = r""" +if [ -z "${SLURM_TMPDIR:-}" ] || [ ! -d "${SLURM_TMPDIR}" ]; then + echo "tile_shape: node-local storage unavailable." >&2 + echo " SLURM_TMPDIR=${SLURM_TMPDIR:-} is not a directory in the container." >&2 + echo " profiles/nibi/config.yaml apptainer-args must carry BOTH" >&2 + echo " --bind /local and --env SLURM_TMPDIR=\$SLURM_TMPDIR" >&2 + exit 1 +fi +export SP_LOCAL="$SLURM_TMPDIR/sp-tile" +export SP_VIGNET_OUT="$SP_LOCAL/output" +export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" +mkdir -p "$SP_VIGNET_OUT" +""" + +# Every member of a group job must agree on a STRING resource or snakemake +# raises "Resource slurm_extra is a string but not all jobs in group require +# the same value" (resources.py::_is_string_resource). So the tmp-disk request +# is one constant on all four members, not a property of tile_ngmix alone. +# +# --tmp is a node SELECTION FLOOR, not a reservation: Slurm matches it against +# the node's configured TmpDisk (nibi: 1.67-12 TB, TmpFS=/local) and does not +# decrement it per job. It buys exactly one thing, and it is the thing worth +# having -- the fused job can no longer land on a node with no usable local +# disk, which is the one configuration in which the whole design fails. 16 GB +# against a measured 5.6 GB store leaves room for an unusually rich tile, and +# filters no nibi node today, so it costs no queue time. Resources are not a +# rerun trigger, so adding it invalidates nothing. +TILE_SLURM_EXTRA = "--tmp=16000" + +# One label for the whole shape chain. Composition, verified empirically +# against snakemake 9.23.1 (resources.py::GroupResources.basic_layered, and a +# real sbatch): within a toposort level siblings SUM their mem_mb and cores and +# MAX their runtime; across levels the sums are MAXed and the runtimes SUMMED. +# So the eight chunks are one wave -- 8 x 1 core, 8 x mem, ONE chunk's runtime +# -- and the group's declared runtime is the sum along the chain. That sum is +# what gates partitions on nibi, so each member's runtime below is measured +# p99 plus margin, NOT the old defensive ceiling. See the tile_ngmix docstring. +TILE_GROUP = "tile_shape" + +# Cleanup, on the LAST member only. $SLURM_TMPDIR is Slurm's to reclaim, but +# /local/scratch on nibi demonstrably carries stale directories from +# long-finished jobs, so the epilog is not on its own reliable. tile_make_cat is +# the last reader of the store, so its EXIT trap is the earliest moment the +# 5.6 GB can go. It must NOT be set by the earlier members: their shells exit +# while the store is still needed. Residual failure mode accepted: SIGKILL +# (OOM-kill, node death) skips the trap, and if the epilog also misses it the +# store leaks until the node drains -- 5.6 GB against a 3.3 TB disk. +TILE_CLEAN = r""" +trap 'rm -rf "$SP_LOCAL"' EXIT +""" + +# tile_vignets ONLY. unit_pre clears each stage's own run dir on the shared root +# ("the job clears its run dir at start" — ShapePipe's FileHandler raises on an +# existing one); the node-local root needs the same treatment, and only the rule +# that WRITES the store may do it. Putting this in TILE_LOCAL would have +# tile_ngmix delete the store it is about to read. +TILE_VIGNET_FRESH = r""" +rm -rf "$SP_VIGNET_OUT/run_sp_tile_PiViVi" +""" + +# THE FUSE'S ONE SHARP EDGE, made loud rather than mysterious. +# +# The vignette store is not a declared output any more, so snakemake cannot see +# it. tile_vignets' manifest CAN be up to date while the store does not exist on +# this node — concretely, after a fused group job dies part-way: the manifest is +# a success and survives, so a LATER invocation re-plans the group with only the +# surviving members and the store is simply not there. (An in-flight `retries:` +# resubmission is safe: it reruns the same member set, tile_vignets included.) +# +# Recovery is one line, and the message says it: delete the tile's +# tile_vignets.json and resume — that puts tile_vignets back in the group and +# the store comes back with it. The alternative fixes are worse: temp()-ing the +# vignets manifest would break clean_exposure, which keys reclamation +# eligibility on exactly that file. +TILE_VIGNET_REQUIRED = r""" +if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output" ]; then + echo "tile_shape: the node-local vignette store is missing." >&2 + echo " expected: $NGMIX_VIGNET_DIR" >&2 + echo " This means tile_vignets did NOT run in this group job — its manifest" >&2 + echo " was already satisfied, most likely because a previous fused job for" >&2 + echo " this tile died after tile_vignets succeeded. The store lives and dies" >&2 + echo " with the job, so it cannot be inherited." >&2 + echo " FIX: rm \"\$SP_RUN/manifests/tile_vignets.json\" and resume." >&2 + exit 1 +fi +""" + + + + def tile_exp(wc): return TILE_EXP.get(wc.tile, []) @@ -200,6 +346,7 @@ rule tile_detect: # fires at the right moment. Its readers (tile_ngmix, tile_make_cat) declare # BOTH. `--notemp` keeps it for debugging. rule tile_vignets: + group: TILE_GROUP input: sx = rules.tile_detect.output.manifest, forest = rules.tile_exp_forest.output.forest, @@ -209,20 +356,53 @@ rule tile_vignets: # tile_merge_headers above. fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: + # THE MANIFEST IS THE ONLY DECLARED OUTPUT NOW. The vignette store used + # to ride along as a second temp(directory()) output purely so native + # temp() would reclaim it; it lives on node-local storage under + # $SLURM_TMPDIR, whose path is not knowable at DAG time, so it cannot + # be declared at all — and does not need to be. It was never DAG + # currency (the manifest always was the edge), its readers are now all + # inside this group job, and Slurm reclaims it with the job. The one + # affordance lost: `--notemp` can no longer keep it for debugging. manifest = f"{TILE_DIR}/manifests/tile_vignets.json", - store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), log: f"{TILE_DIR}/logs/tile_vignets.json" params: pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, - forest=forest_dir(wc.tile)), + forest=forest_dir(wc.tile), + pre_run=[TILE_LOCAL, TILE_VIGNET_FRESH]), script_hash = SCRIPT_HASH - threads: 16 + # 8, not 16, for the same reason tile_ngmix is 1: `-b {threads}` is SMP + # batch size over input FILE SETS, and a tile is one set -- this run's own + # log says "Batch size: 16 / Total number of processes: 1". 16 was the + # widest member and therefore set the whole GROUP's cpus_per_task; at 8 the + # group asks exactly what the eight-chunk ngmix wave needs. Billing is + # unchanged either way (nibi is MAX_TRES and the 112 GB memory term is + # 28 core-equivalents, well above both), but the group now packs onto a + # node in 8 cores instead of 16. + threads: 8 resources: mem_mb = lambda wc, attempt: 32000 * attempt, - runtime = 240 + # Measured median 3m29s, max 5:25 (this branch: 2m38s on 198.305). + # 20 min is p99 plus a wide margin, and it is a term in the GROUP's + # runtime sum, so the old defensive 240 is not free any more. + runtime = 20, + slurm_extra = TILE_SLURM_EXTRA shell: - sp_shell("tile_vignets", "config_tile_PiViVi.ini") + # sp_shell's body, except that the completeness check is pointed at the + # NODE-LOCAL run root. completeness.py is unmodified: --run-dir is its + # own documented override for "$SP_RUN", and --unit keeps the manifest's + # unit field the tile ID instead of the basename of the local root (it + # would otherwise default to "sp-tile"). Passing --unit also keeps the + # manifest CONTENT independent of the ephemeral local path, which the + # mtime rerun-trigger depends on. + "{params.pre}\n" + "rc=0\n" + 'shapepipe_run -c "$SP_CONFIG/config_tile_PiViVi.ini" -b {threads} || rc=$?\n' + f"python {SCRIPTS}/completeness.py check tile_vignets {{output.manifest}}" + ' --run-dir "$SP_LOCAL" --unit {wildcards.tile}' + " --log {log} --job-rc \"$rc\" || rc=1\n" + "exit $rc\n" # ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own # CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params @@ -233,13 +413,21 @@ rule tile_vignets: # Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and # merge_sep_cats — DAG-serialised after all chunks — is the gather. rule tile_ngmix: + group: TILE_GROUP input: + # The manifest, and only the manifest. The vignette store is no longer a + # declared input because it is no longer a declared output: it is + # node-local, produced by tile_vignets earlier in THIS SAME group job + # (see TILE_LOCAL). The manifest was always the real edge. vignets = rules.tile_vignets.output.manifest, - store = rules.tile_vignets.output.store, sx = rules.tile_detect.output.manifest, output: manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", - # temp(directory()) for the same reason as the vignette store above. + # STAYS ON SCRATCH, unlike the vignette store. It is DAG currency: + # tile_merge_cats reads it, and merge_sep_cats derives chunks 2..N from + # chunk 1's path. It is also small (~300 KB/chunk), so it costs the + # scratch high-water nothing worth chasing. temp() still reclaims it + # once merge_cats has run. chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), log: f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" @@ -254,15 +442,65 @@ rule tile_ngmix: pre_run=[f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py --run-dir ' f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' f'|| exit 1', - 'eval "$ngmix_range_out"']), + 'eval "$ngmix_range_out"', + TILE_LOCAL, TILE_VIGNET_REQUIRED]), script_hash = SCRIPT_HASH - threads: 4 + # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE + # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over + # filehd.process_list, i.e. parallelism ACROSS INPUT FILE SETS. An ngmix + # chunk is one catalogue, so process_list has exactly one entry: every real + # log says "Batch size: 4 / Total number of processes: 1". There is no + # internal parallelism either (no multiprocessing/Pool/joblib/threading in + # ngmix_package/ngmix.py), and OMP/BLAS are pinned to 1 by both the prologue + # and the profile. The reserved cores 2-4 never had anything to run. + # + # Worth being precise about what this saves. nibi bills + # TRESBillingWeights=CPU=1000,Mem=250G under PriorityFlags=MAX_TRES, i.e. + # max(cores, mem_GB/4). At 14 GB the memory term alone is 3.5 core- + # equivalents, so 4 -> 1 core moves the reservation from 4.0 to 3.5, a 12% + # saving -- NOT 75%. The real core-hour win is the elapsed-time collapse + # from killing the NFS random reads, not this. + threads: 1 retries: 2 benchmark: f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" resources: + # 14000, UNCHANGED — and the scary-looking number that argued for + # raising it is an artefact, so read this before touching it. + # + # sacct MaxRSS for the timed chunk (job 20795277) was 12.92 GiB, 94.5% + # of the reservation. It is not process memory. nibi runs + # JobAcctGatherType=jobacct_gather/cgroup, so sacct's MaxRSS is the + # cgroup's memory.current, which under cgroup v2 CHARGES PAGE CACHE to + # the job. Snakemake's own benchmark for the same job, taken from psutil + # per-process counters, says max_rss = 1262 MB / max_pss = 1165 MB. The + # ~11.7 GiB difference is file-backed cache — the store written and then + # read back — and it is reclaimable, so it cannot OOM the job; the + # kernel evicts cache before it kills anything. + # + # So ngmix's real footprint is ~1.2 GB, and 14000 is ~11x that. It is + # almost certainly reducible, and cutting it is the single biggest + # lever left on this rule's cost (see the mem_mb note in the group + # composition above — memory, not cores, is what nibi bills the fused + # job for). But it wants its own measurement across several tiles, not + # a guess off one chunk, so it stays at the value the last campaign ran. + # Keeping it generous also keeps the 5.6 GB store resident in page + # cache, which is part of why the fused tile is fast. + # + # The eight chunks are SIBLINGS in the group's toposort, so snakemake + # SUMS their mem_mb: the group reserves 8 x this (112000 MiB). That is + # deliberate — it is what makes all eight run as one wave, and one wave + # is what makes the group's runtime ONE chunk's runtime instead of eight. mem_mb = lambda wc, attempt: 14000 * attempt, - runtime = 720 + # 120, not 720. MEASURED on tile 198.305 chunk 1 with the store + # node-local (job 20795277): ~76 min elapsed for 3547 objects, against a + # 7h34m median on NFS — the same ~50 min of TotalCPU either way, so the + # six-fold collapse is pure I/O. 120 is that plus ~60% margin for a + # richer tile. It is the dominant term in the group's runtime sum, so + # the old 12 h ceiling is not free any more: at 720 the fused job would + # declare 12h45m and drop out of every 3 h partition. + runtime = 120, + slurm_extra = TILE_SLURM_EXTRA shell: sp_shell("tile_ngmix", "config_tile_Ng_template.ini") @@ -278,7 +516,7 @@ def ngmix_chunkdirs(wc): # The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's # own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). rule tile_merge_cats: - group: "tile_finish" + group: TILE_GROUP input: manifests = ngmix_manifests, chunkdirs = ngmix_chunkdirs, @@ -293,7 +531,9 @@ rule tile_merge_cats: threads: 8 resources: mem_mb = lambda wc, attempt: 16000 * attempt, - runtime = 120 + # Measured median 0:15, max 0:42. A term in the group's runtime sum. + runtime = 10, + slurm_extra = TILE_SLURM_EXTRA shell: sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") @@ -303,22 +543,28 @@ rule tile_merge_cats: # No protected(): the full default rerun-triggers govern, and protected() only # ever forced people through a `--forcerun` detour. rule tile_make_cat: - group: "tile_finish" + group: TILE_GROUP input: + # No store input: it is node-local, written by tile_vignets in this same + # group job. make_cat reads its psfex_interp output through + # $NGMIX_VIGNET_DIR (config_tile_Mc.ini). ms = rules.tile_merge_cats.output.manifest, - store = rules.tile_vignets.output.store, output: manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", final_cat = f"{PROD_TILE_DIR}/final_cat-{{tile}}.fits", log: f"{TILE_DIR}/logs/tile_make_cat.json" params: - pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), + pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile, + pre_run=[TILE_LOCAL, TILE_VIGNET_REQUIRED, + TILE_CLEAN]), script_hash = SCRIPT_HASH threads: 8 resources: mem_mb = lambda wc, attempt: 16000 * attempt, - runtime = 120 + # Measured median 1:33, max 1:50. A term in the group's runtime sum. + runtime = 15, + slurm_extra = TILE_SLURM_EXTRA shell: # sp_shell's body, plus the catalogue publish: a real file, so it is a # real declared output (and it persists — never temp()). `--job-rc` is From fb28e7a6b6d8a724de13ee6ebd07f3458adff5c9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 21:35:21 -0400 Subject: [PATCH 043/100] fix(orchestration): derive the node-local path, do not try to pass it Campaign 20798193 died on every fused tile in eight minutes. snakemake ESCAPES the `$` when it splices apptainer-args into the job command, so `--env SLURM_TMPDIR=$SLURM_TMPDIR` delivered the literal eight characters and the container's SLURM_TMPDIR was the string `$SLURM_TMPDIR`. The guard caught it immediately and printed why, which is the only reason this cost minutes instead of a night. The earlier verification missed it because it exercised a hand-built `apptainer exec`, never snakemake's quoting. Nothing can carry the value across that boundary: `--env` escapes, and `APPTAINERENV_*` would have to be exported by the job wrapper, which belongs to snakemake. So stop communicating the path and derive it. `/local/scratch` on nibi is drwxrwxrwt root:root -- world-writable with the sticky bit, checked from inside the container in probe job 20798618 -- and the tile id is a wildcard snakemake substitutes at DAG time. `tile_local(tile)` therefore emits a concrete `/local/scratch/sp-` with no `$` left for anything to escape. Confirmed in the emitted shell: `export SP_LOCAL="/local/scratch/sp-186.306"`. One group job per tile, so the name cannot collide, and the sticky bit means no other user can remove it. What we lose is Slurm's own reclamation of $SLURM_TMPDIR: TILE_VIGNET_FRESH clears a stale directory on the next attempt for that tile, the trap in the last group member removes it on the way out, and a 24-hour sweep of our own leftovers catches what a hard kill leaves -- on a node shared with other people that last one is manners. Not /tmp: inside this container it is a 378 GB tmpfs, i.e. RAM billed to the job's memory cgroup, so a 5.6 GB store there would be paid for twice. The guard stays. Its job is now to catch a missing --bind rather than a missing variable, and either way the failure must be loud rather than a silent fall back onto NFS. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VUNNyG8tw6Mdbvo4SVNrhp --- profiles/nibi/config.yaml | 4 +- workflow/config/cfis/config_tile_Mc.ini | 2 +- workflow/rules/tile.smk | 84 ++++++++++++++++++------- 3 files changed, 63 insertions(+), 27 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 276cc0143..a0e292351 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -50,7 +50,7 @@ software-deployment-method: [apptainer] # modules) -- # snakemake/persistence/__init__.py::_software_stack_hash, # v9.23.1 -- and apptainer-args is not in that hash. -# --env SLURM_TMPDIR=$SLURM_TMPDIR +# # HOW the in-container shell LEARNS its node-local dir. # --cleanenv strips the real SLURM_TMPDIR, and the two # alternatives do not work here: @@ -87,7 +87,7 @@ software-deployment-method: [apptainer] # live p3-batch1 job reads it) and not the sif default # (frozen pre-#843, and pre-#873). Production later rebuilds # the sif at the validated commit and DROPS this --env line. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch --bind /local --env SLURM_TMPDIR=$SLURM_TMPDIR" +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch --bind /local" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 149e4d9e8..55daef12e 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -57,7 +57,7 @@ TIMEOUT = 96:00:00 # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN # The psfex_interp store is NODE-LOCAL: it is the vignette-store run this -# tile's fused group job wrote to $SLURM_TMPDIR (see config_tile_PiViVi.ini +# tile's fused group job wrote to node-local storage (see config_tile_PiViVi.ini # and workflow/rules/tile.smk). $NGMIX_VIGNET_DIR names it; the other two # inputs and this run's own output stay on $SP_RUN. INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index cabf2a211..83874ded3 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -24,7 +24,7 @@ Distinct tiles share no edge, so each is one group job per tile: * ``group: TILE_GROUP`` ("tile_shape") — tile_vignets -> 8 x tile_ngmix -> tile_merge_cats -> tile_make_cat, THE WHOLE SHAPE CHAIN AS ONE JOB. This is not a latency optimisation; it is what lets the 5.6 GB vignette store live on - node-local NVMe and never touch /scratch (see the TILE_LOCAL block below). + node-local NVMe and never touch /scratch (see tile_local() below). Composition, verified against snakemake 9.23.1 and a real sbatch: cpus = max over levels of summed siblings = max(8, 8x1, 8, 8) = 8; mem_mb = max(32000, 8x14000, 16000, 16000) = 112000; @@ -77,19 +77,23 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # config_tile_PiViVi.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and # config_tile_Ng_template.ini / config_tile_Mc.ini ($NGMIX_VIGNET_DIR). # -# WHY IT NEEDS THE GROUP. $SLURM_TMPDIR is per SLURM JOB. Unfused, tile_vignets -# and tile_ngmix are different jobs on different nodes and the store MUST be on -# shared storage. Fused, all members share one job, one node and one -# $SLURM_TMPDIR, so the store written by the first member is simply there for -# the rest. (Verified: group members see the same $SLURM_TMPDIR.) +# WHY IT NEEDS THE GROUP. The store is node-local, so it exists only on the +# machine that wrote it and only while that job lives. Unfused, tile_vignets and +# tile_ngmix are different jobs on (usually) different nodes, and the store would +# have to be on shared storage -- which is the whole problem. Fused, all members +# run in one allocation on one node, so the store the first member writes is +# simply there for the rest. (Verified live: group members share the directory.) # -# HOW $SLURM_TMPDIR GETS IN HERE. profiles/nibi passes --bind /local and -# --env SLURM_TMPDIR=$SLURM_TMPDIR through apptainer-args; --cleanenv strips -# the real one. See that file for why {resources.tmpdir} is NOT usable (it -# resolves to /tmp, a RAM-backed tmpfs) and why the expansion happens in the -# job. The guard below turns any breakage in that chain into a hard failure -# rather than a silent fall back onto NFS -- or, worse, a 5.6 GB write into a -# tmpfs charged to the job's memory cgroup. +# HOW THE PATH GETS IN HERE. Not through the environment. profiles/nibi passes +# --bind /local, and tile_local() below builds the path from the tile wildcard, +# which snakemake substitutes at DAG time. $SLURM_TMPDIR was the obvious choice +# and does not work: snakemake escapes the `$` when it splices apptainer-args +# into the job command, so the container gets the literal string and every fused +# job dies on the guard (campaign 20798193, cancelled after 8 minutes). The guard +# stays, because its job now is to catch a missing --bind rather than a missing +# variable -- and either way the failure must be loud, never a silent fall back +# onto NFS or into /tmp, which in this container is a tmpfs charged to the job's +# memory cgroup. # # THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, # not one chunk, because the store dies with the job. At ~1 h per fused tile @@ -99,19 +103,51 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # prologue (unit_pre) and completeness.py are untouched by design: both are # fingerprinted into every rule, and changing either would invalidate the whole # campaign, including the 117 finished exposure chains. -TILE_LOCAL = r""" -if [ -z "${SLURM_TMPDIR:-}" ] || [ ! -d "${SLURM_TMPDIR}" ]; then +def tile_local(tile): + """The node-local prologue, as bash, for one tile. + + A FUNCTION of the tile rather than a constant, because the path has to be + literal by the time apptainer sees it. `$SLURM_TMPDIR` cannot be used: + snakemake escapes the `$` when it splices `apptainer-args` into the job + command, so the container receives the eight characters `$SLURM_TMPDIR` + and every fused job dies (observed live, campaign 20798193). Passing it + through `--env` fails the same way, and `APPTAINERENV_*` would have to be + exported by the job wrapper, which is snakemake's and not ours. + + So the path is derived instead of communicated. `/local/scratch` on nibi is + `drwxrwxrwt root:root` -- world-writable with the sticky bit, verified in + the container (probe job 20798618) -- and the tile id is a wildcard + snakemake substitutes at DAG time, so the shell string carries a concrete + path with no `$` left for anything to escape. One group job per tile means + the name cannot collide; the sticky bit means nobody else can remove it. + + What we give up is Slurm's own cleanup of `$SLURM_TMPDIR`. TILE_VIGNET_FRESH + reclaims a stale directory on the next attempt for the same tile, the trap + in the last group member removes it on the way out, and the sweep below + catches what a hard kill leaves behind -- on a shared node that last one is + manners, not housekeeping. + + NOT `/tmp`: inside the container that is a 378 GB tmpfs, i.e. RAM charged to + the job's cgroup, so a 5.6 GB store there would be paid for twice. + """ + return f''' +if [ ! -d /local/scratch ]; then echo "tile_shape: node-local storage unavailable." >&2 - echo " SLURM_TMPDIR=${SLURM_TMPDIR:-} is not a directory in the container." >&2 - echo " profiles/nibi/config.yaml apptainer-args must carry BOTH" >&2 - echo " --bind /local and --env SLURM_TMPDIR=\$SLURM_TMPDIR" >&2 + echo " /local/scratch is not a directory inside the container." >&2 + echo " profiles/nibi/config.yaml apptainer-args must carry --bind /local" >&2 exit 1 fi -export SP_LOCAL="$SLURM_TMPDIR/sp-tile" +export SP_LOCAL="/local/scratch/sp-{tile}" export SP_VIGNET_OUT="$SP_LOCAL/output" export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" -mkdir -p "$SP_VIGNET_OUT" -""" +mkdir -p "$SP_VIGNET_OUT" || {{ + echo "tile_shape: cannot create $SP_LOCAL on this node." >&2 + exit 1 +}} +find /local/scratch -maxdepth 1 -name 'sp-*.*' -user "$(id -u)" -mmin +1440 \ + -exec rm -rf {{}} + 2>/dev/null || true +''' + # Every member of a group job must agree on a STRING resource or snakemake # raises "Resource slurm_extra is a string but not all jobs in group require @@ -370,7 +406,7 @@ rule tile_vignets: params: pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, forest=forest_dir(wc.tile), - pre_run=[TILE_LOCAL, TILE_VIGNET_FRESH]), + pre_run=[tile_local(wc.tile), TILE_VIGNET_FRESH]), script_hash = SCRIPT_HASH # 8, not 16, for the same reason tile_ngmix is 1: `-b {threads}` is SMP # batch size over input FILE SETS, and a tile is one set -- this run's own @@ -443,7 +479,7 @@ rule tile_ngmix: f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' f'|| exit 1', 'eval "$ngmix_range_out"', - TILE_LOCAL, TILE_VIGNET_REQUIRED]), + tile_local(wc.tile), TILE_VIGNET_REQUIRED]), script_hash = SCRIPT_HASH # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over @@ -556,7 +592,7 @@ rule tile_make_cat: f"{TILE_DIR}/logs/tile_make_cat.json" params: pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile, - pre_run=[TILE_LOCAL, TILE_VIGNET_REQUIRED, + pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, TILE_CLEAN]), script_hash = SCRIPT_HASH threads: 8 From 7b34c9e80f499e8de088ac370d4770b595e590d7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 22:04:41 -0400 Subject: [PATCH 044/100] docs(profile): the $SLURM_TMPDIR passthrough is disproven, say so The apptainer-args comment still described splicing "$SLURM_TMPDIR" into the args and letting the job's shell expand it as the working mechanism -- "verified in-container, not assumed". It is not the mechanism, and it does not work: snakemake escapes the `$` before the string reaches any shell, which killed every tile of run 20798193 in eight minutes and is why fb28e7a6 derives the path from the tile wildcard instead. Leaving the old rationale in place is a trap: it reads as a working design and invites the next reader to reach for it again. Replace it with the three alternatives that were actually tried, what each costs, and which one failed in production. Comment only -- apptainer-args itself is untouched, and the profile is not hashed by any rerun trigger. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EtcvvamkDNjyEhVBxCdfR6 --- profiles/nibi/config.yaml | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index a0e292351..0f20bd361 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -51,9 +51,13 @@ software-deployment-method: [apptainer] # snakemake/persistence/__init__.py::_software_stack_hash, # v9.23.1 -- and apptainer-args is not in that hash. # -# HOW the in-container shell LEARNS its node-local dir. -# --cleanenv strips the real SLURM_TMPDIR, and the two -# alternatives do not work here: +# HOW the in-container shell LEARNS its node-local dir: +# IT DOES NOT. It DERIVES the path from the tile wildcard +# (tile.smk::tile_local -> /local/scratch/sp-), and +# that is the only mechanism that survives contact with +# snakemake. Three alternatives were tried; all fail, and +# the third failed IN PRODUCTION, so read this before +# reaching for any of them again: # * {resources.tmpdir} is NOT $SLURM_TMPDIR. Snakemake # defers that resource to the job and evaluates it as # tempfile.gettempdir(); nibi's Slurm does not export @@ -65,19 +69,22 @@ software-deployment-method: [apptainer] # * APPTAINERENV_SLURM_TMPDIR would have to be exported # by something that already knows the JOB's value; # bin/sp runs on the login node, where it is unset. -# This works because snakemake splices apptainer-args -# verbatim into a command it runs with Popen(shell=True) -# (snakemake/deployment/singularity.py::shellcmd + -# shell.py), so the JOB's /bin/sh expands $SLURM_TMPDIR at -# submission-shell time, in the job, where the value is -# known. Verified in-container, not assumed. -# Rules outside the tile_shape group ignore the variable; -# on the login node (localrules) it expands to empty, -# which apptainer accepts. Every tile_shape member REFUSES -# TO RUN if what arrives is not a real directory -# (tile.smk), so a broken passthrough is a loud failure -- -# never a silent slide back onto NFS, and never a 5.6 GB -# write into the RAM-backed /tmp tmpfs. +# * SPLICING "$SLURM_TMPDIR" INTO apptainer-args AND +# LETTING THE JOB'S SHELL EXPAND IT DOES NOT WORK. +# This was the shipped design for exactly one campaign +# and it killed every tile of run 20798193 in eight +# minutes: snakemake ESCAPES the `$` before the string +# reaches any shell, so the container receives the +# literal characters and never a path. NOTHING can be +# carried into the container this way -- not a tmpdir, +# not anything else. The bind below is static, and the +# VALUE is derived job-side from a wildcard instead. +# Every tile_shape member still REFUSES TO RUN if the +# derived directory is not real (tile.smk), which is the +# only reason that campaign cost minutes and not a night: +# a broken node-local path is a loud failure, never a +# silent slide back onto NFS, and never a 5.6 GB write +# into the RAM-backed /tmp tmpfs. # PYTHONPATH SETTLED CALL 3 — pins THIS branch's src/, which is # develop@97e16d50 plus the four commits that genuinely need # module code: ngmix chunk fields + position-seeded RNG, From 62242066247d91bf8ce72cd19b346da27b4aa7b2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 22:07:00 -0400 Subject: [PATCH 045/100] fix(orchestration): tile_ngmix runtime scales with attempt A TIMEOUT on the fused group re-queued against the identical wall and failed identically -- three 165-minute allocations spent learning nothing. `runtime = 120 * attempt` leaves attempt 1 exactly as it was (group 165 min, inside cpubase_bycore_b1's 180) and gives a retry 285 min in b2. The margin is thinner than the old comment implied, which is why this matters now. Measured eight-wide on one node (job 20799387, tile 186.307, 4412 objects/chunk): 1.54 s/object steady-state against 1.29 s/object for the same rule alone, so concurrency costs ~19% and the chunk lands at ~113 min. The campaign's largest tile projects to ~120 -- the declaration exactly, nothing spare. Attempt 1 is unchanged, so the benchmark is not perturbed; `resources` is not a rerun trigger, so no completed work is invalidated. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EtcvvamkDNjyEhVBxCdfR6 --- workflow/rules/tile.smk | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 83874ded3..ca3187266 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -528,14 +528,28 @@ rule tile_ngmix: # deliberate — it is what makes all eight run as one wave, and one wave # is what makes the group's runtime ONE chunk's runtime instead of eight. mem_mb = lambda wc, attempt: 14000 * attempt, - # 120, not 720. MEASURED on tile 198.305 chunk 1 with the store - # node-local (job 20795277): ~76 min elapsed for 3547 objects, against a - # 7h34m median on NFS — the same ~50 min of TotalCPU either way, so the - # six-fold collapse is pure I/O. 120 is that plus ~60% margin for a - # richer tile. It is the dominant term in the group's runtime sum, so - # the old 12 h ceiling is not free any more: at 720 the fused job would - # declare 12h45m and drop out of every 3 h partition. - runtime = 120, + # 120 on the FIRST attempt, and ATTEMPT-SCALED after it. MEASURED + # two ways, and the second is why the margin is thinner than it looks: + # * alone (job 20795277, tile 198.305 chunk 1): ~76 min for 3547 + # objects = 1.29 s/object, against a 7h34m median on NFS -- the + # same ~50 min of TotalCPU either way, so the collapse is pure I/O. + # * EIGHT-WIDE on one node (job 20799387, tile 186.307, 4412 + # objects/chunk): 1.54 s/object steady-state, i.e. concurrency + # costs ~19%, and the chunk lands at ~113 min. The campaign's + # largest tile (198.306, 4678 objects/chunk) projects to ~120 -- + # exactly this number, with nothing left over. + # + # Inside a group SLURM enforces only the GROUP's wall (165 min), never + # a member's, so 120 is a budgeting term rather than a kill line and + # the worst tile still lands ~127 min inside 165. What is NOT safe is a + # flat retry: a group that TIMEOUTs re-queues against the identical + # wall and fails identically, burning three 165-minute allocations to + # learn nothing. Scaling with `attempt` keeps the happy path in + # cpubase_bycore_b1 (20+120+10+15 = 165 <= 180) and gives a retry real + # headroom (285 min, which is b2) instead of a rerun of the same + # failure. Attempt 1 is unchanged, so this does not perturb the + # benchmark -- it only makes the failure branch mean something. + runtime = lambda wc, attempt: 120 * attempt, slurm_extra = TILE_SLURM_EXTRA shell: sp_shell("tile_ngmix", "config_tile_Ng_template.ini") From b2163e0d49927ec2d97be47622df0fc077d00a3d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 01:33:33 -0400 Subject: [PATCH 046/100] perf(orchestration): stage the WCS sqlite node-local with the vignette store The fuse moved five of six SQLite stores onto node-local NVMe and bought 4.3x. It did not solve the I/O problem, and `read_bytes = 0` was the wrong instrument for concluding that it had -- NFS reads are RPCs and never reach the block layer, so psutil's counter cannot see them. Thread-state sampling closes the budget properly. 3,200 samples across the eight chunks of job 20799387 give 0.56 threads on CPU and 0.44 in uninterruptible I/O wait, R + D = 1.00, with 147 of 176 D-samples inside `rpc_wait_bit_killable`. Open descriptors name exactly one hot NFS file left: log_exp_headers-.sqlite, 11.3 MB, which ngmix reads once per object PER EPOCH for the WCS. Size was never the problem and is not the problem now; the operation count is, and on a network filesystem every operation is a round trip. So $SP_WCS_DIR joins $NGMIX_VIGNET_DIR: tile_local() copies the file beside the store it already builds, and config_tile_Ng_template.ini's fourth INPUT_DIR points at the copy instead of at Mh_exp on scratch. A copy and not a symlink, because a link resolves straight back to NFS. Sub-second for 11 MB, against ~44% of the elapsed time of the rule that is 99.3% of tile-side core-hours -- so ~1.8x on tile_ngmix, unmeasured until the next campaign runs it. tile_vignets (PiViVi) reads the same file and still reads it over NFS. It is 3.4 minutes of a ~1.7 hour group, so it is left alone deliberately rather than overlooked; the staged copy is already there if that changes. Orchestration only -- no science config changes, so nothing to mirror into example/cfis (cf. the config-fork hazard that left #873's STAR_THRESH on 20). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NYcxuAbZbQivC5UFtgecxH --- .../config/cfis/config_tile_Ng_template.ini | 11 ++++- workflow/rules/tile.smk | 42 ++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index 65c769060..f0e374b08 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -79,7 +79,16 @@ TIMEOUT = 96:00:00 # It is NOT optional: ShapePipe's config expansion is strict # (pipeline/config.py::_expandvars_strict), so an unset $NGMIX_VIGNET_DIR is # a loud error rather than a silent read of the wrong tree. -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output +# +# $SP_WCS_DIR is the fourth input and the same idea applied to the file the +# vignette store left behind. log_exp_headers-.sqlite is 11.3 MB and +# ngmix reads it once per object PER EPOCH for the WCS; on NFS that is a +# network round trip per read, and thread-state sampling of the fused job +# measured every chunk 44% blocked in `rpc_wait_bit_killable` on this one file +# with everything else already node-local. tile_local() copies it beside the +# store and points this variable at the copy. Set it to +# $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output to read in place. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index ca3187266..6e01e7f04 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -75,7 +75,16 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # bulk intra-tile intermediate is node-local. That split is possible because # ShapePipe's configs set input and output paths independently -- see # config_tile_PiViVi.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and -# config_tile_Ng_template.ini / config_tile_Mc.ini ($NGMIX_VIGNET_DIR). +# config_tile_Ng_template.ini / config_tile_Mc.ini ($NGMIX_VIGNET_DIR, +# $SP_WCS_DIR). +# +# TWO node-local stores, not one, and the second is small on purpose. The +# vignette store is ~5-8 GB of bulk; $SP_WCS_DIR holds a single 11.3 MB file, +# log_exp_headers-.sqlite, which ngmix reads once per object per epoch. +# Moving 5.6 GB bought 4.3x and left 44% of every chunk blocked in NFS RPC on +# that one file. What costs on a network filesystem is the operation, not the +# byte, and the two stores are the same fix applied at the two ends of that +# distribution. # # WHY IT NEEDS THE GROUP. The store is node-local, so it exists only on the # machine that wrote it and only while that job lives. Unfused, tile_vignets and @@ -129,6 +138,26 @@ def tile_local(tile): NOT `/tmp`: inside the container that is a 378 GB tmpfs, i.e. RAM charged to the job's cgroup, so a 5.6 GB store there would be paid for twice. + + TWO STORES, AND THE SECOND IS 11 MB. `$SP_WCS_DIR` holds one file, + `log_exp_headers-.sqlite`, and staging it is the rest of the same fix + rather than a refinement of it. Moving the 5.6 GB vignette store off NFS + bought 4.3x and did not solve the problem: thread-state sampling of the + fused job (3,200 samples across 8 chunks, campaign smk-g4 job 20799387) put + every chunk at 56% on CPU and 44% in `rpc_wait_bit_killable`, with 147 of + 176 D-state samples inside NFS RPC and exactly one hot NFS file still open. + ngmix reads that file once per object PER EPOCH for the WCS, so what costs + is the operation count and not the byte count -- a network round trip per + read. The two stores are the same intervention at the two ends of the size + distribution, and the small one is worth ~1.8x on the rule that is 99.3% of + tile-side core-hours. + + A copy, never a symlink: a link resolves straight back to NFS. Every member + of the group runs this prologue, so the copy is attempted up to ten times + per tile; `cp -u` makes the repeats free. `tile_merge_headers` is upstream of + the whole group, so the file exists by the time any member runs, and if it + does not then failing here is correct -- the alternative is ngmix silently + reading a different tree. """ return f''' if [ ! -d /local/scratch ]; then @@ -140,10 +169,19 @@ fi export SP_LOCAL="/local/scratch/sp-{tile}" export SP_VIGNET_OUT="$SP_LOCAL/output" export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" -mkdir -p "$SP_VIGNET_OUT" || {{ +export SP_WCS_DIR="$SP_LOCAL/wcs" +mkdir -p "$SP_VIGNET_OUT" "$SP_WCS_DIR" || {{ echo "tile_shape: cannot create $SP_LOCAL on this node." >&2 exit 1 }} +# the WCS store -- see the docstring; a copy, never a symlink +cp -u "$SP_RUN"/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers-*.sqlite \ + "$SP_WCS_DIR"/ || {{ + echo "tile_shape: no log_exp_headers-*.sqlite under" >&2 + echo " $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output" >&2 + echo " tile_merge_headers must have run before this group." >&2 + exit 1 +}} find /local/scratch -maxdepth 1 -name 'sp-*.*' -user "$(id -u)" -mmin +1440 \ -exec rm -rf {{}} + 2>/dev/null || true ''' From af772eadd996127b9cfdefa6dfe340dcd34bfaba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 02:22:27 -0400 Subject: [PATCH 047/100] perf(orchestration): tile_ngmix mem_mb 14000 -> 5000, measured across 31 tiles The rule comment asked for a measurement across several tiles before this number moved, and the smk-g4 campaign is it: cgroup high-water 27.40 GiB at worst and 22.14 GiB on average across 31 fused groups, against the 109.4 GiB the group was reserving. 5000 per chunk takes the group to 40000 MiB, a 1.43x margin over the worst tile observed and about 4x the anonymous footprint -- snakemake's psutil benchmark puts each chunk at ~1.25 GiB, so the eight chunks are ~10 GiB and everything above that is reclaimable page cache. This is the campaign's largest single cost saving, and it is a schedule change rather than a bookkeeping one. nibi bills max(cores, mem_GB/4), so at 112 GB the group billed 28 core-equivalents for 8 real cores -- measured live at 905 billed against 266 allocated, a factor of 3.40. At 40 GB it bills 10. The account's fairshare target is ~250 CE, which buys 9 tiles in flight at 28 and 25 at 10, and a DR6 campaign's wall clock is (tiles / tiles-in-flight) x per-tile elapsed. On the campaign's measured per-tile cost that is ~190 days against ~68. Not 4000, which is the arithmetic floor: it would take the group to 32000 where tile_vignets' own 32000 binds and the group finally bills its 8 real cores. That is a 1.17x margin over the worst tile measured, and the first thing to give under pressure would be the page cache holding the node-local store -- which is what makes the fused tile fast. Worth taking later, with a measurement of cache behaviour under pressure rather than on arithmetic. `resources` is not a rerun trigger, so no completed work is invalidated. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NYcxuAbZbQivC5UFtgecxH --- workflow/rules/tile.smk | 62 ++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 6e01e7f04..e4e475c8c 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -27,7 +27,11 @@ Distinct tiles share no edge, so each is one group job per tile: node-local NVMe and never touch /scratch (see tile_local() below). Composition, verified against snakemake 9.23.1 and a real sbatch: cpus = max over levels of summed siblings = max(8, 8x1, 8, 8) = 8; - mem_mb = max(32000, 8x14000, 16000, 16000) = 112000; + mem_mb = max(32000, 8x5000, 16000, 16000) = 40000, so nibi's + max(cores, mem_GB/4) bills the group 10 core-equivalents rather than the 28 + it billed at 8x14000 (see the mem_mb note on tile_ngmix for the 31-tile + measurement that sized it); tile_vignets' own 32000 is now the second term + and becomes binding if the chunks ever go below 4000; runtime = sum along the chain of each level's MAX = 20 + 120 + 10 + 15 = 165. 165 min is under the 180 min ceiling of ``cpubase_bycore_b1``, so the fused job reaches the widest partition set (plus cpubackfill) — which is why each @@ -539,33 +543,41 @@ rule tile_ngmix: benchmark: f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" resources: - # 14000, UNCHANGED — and the scary-looking number that argued for - # raising it is an artefact, so read this before touching it. + # 5000, down from 14000, and this is the campaign's largest single cost + # saving — but read what the number means before moving it again. # - # sacct MaxRSS for the timed chunk (job 20795277) was 12.92 GiB, 94.5% - # of the reservation. It is not process memory. nibi runs - # JobAcctGatherType=jobacct_gather/cgroup, so sacct's MaxRSS is the - # cgroup's memory.current, which under cgroup v2 CHARGES PAGE CACHE to - # the job. Snakemake's own benchmark for the same job, taken from psutil - # per-process counters, says max_rss = 1262 MB / max_pss = 1165 MB. The - # ~11.7 GiB difference is file-backed cache — the store written and then - # read back — and it is reclaimable, so it cannot OOM the job; the - # kernel evicts cache before it kills anything. + # sacct's MaxRSS here is NOT process memory. nibi runs + # JobAcctGatherType=jobacct_gather/cgroup, so it reports the cgroup's + # memory.current, which under cgroup v2 CHARGES PAGE CACHE to the job. + # Snakemake's own psutil benchmark for the same chunks says max_rss + # ~1.25 GiB each, so the eight chunks' real anonymous footprint is + # ~10 GiB and everything above that is file-backed cache — the store + # written and then read back. Cache is reclaimable: the kernel evicts it + # before it kills anything, so this reservation cannot OOM the job. What + # it can do is squeeze the cache that keeps the node-local store + # resident, which is part of why the fused tile is fast. # - # So ngmix's real footprint is ~1.2 GB, and 14000 is ~11x that. It is - # almost certainly reducible, and cutting it is the single biggest - # lever left on this rule's cost (see the mem_mb note in the group - # composition above — memory, not cores, is what nibi bills the fused - # job for). But it wants its own measurement across several tiles, not - # a guess off one chunk, so it stays at the value the last campaign ran. - # Keeping it generous also keeps the 5.6 GB store resident in page - # cache, which is part of why the fused tile is fast. + # MEASURED ACROSS 31 TILES (campaign smk-g4, 2026-08-30): cgroup + # high-water 27.40 GiB max, 22.14 GiB mean, against the 109.4 GiB the + # group was reserving. The eight chunks are SIBLINGS in the group's + # toposort so snakemake SUMS their mem_mb; at 5000 the group asks + # 40000 MiB, a 1.43x margin over the worst tile observed and ~4x the + # anonymous footprint, leaving ~29 GiB of cache for an 8.1 GiB store. # - # The eight chunks are SIBLINGS in the group's toposort, so snakemake - # SUMS their mem_mb: the group reserves 8 x this (112000 MiB). That is - # deliberate — it is what makes all eight run as one wave, and one wave - # is what makes the group's runtime ONE chunk's runtime instead of eight. - mem_mb = lambda wc, attempt: 14000 * attempt, + # WHY THIS IS THE BIG ONE. nibi bills max(cores, mem_GB/4), so at + # 112 GB the fused group billed 28 core-equivalents for 8 real cores — + # 3.4x, measured live at 905 billed against 266 allocated. At 40 GB it + # bills 10. That is not a saving so much as a schedule: the account's + # fairshare target is ~250 CE, which buys 9 tiles in flight at 28 and 25 + # at 10, and DR6's wall clock is (tiles / tiles-in-flight) x elapsed. + # + # 4000 is the floor and is NOT recommended yet: it would take the group + # to 32000, where tile_vignets' own 32000 becomes the binding term and + # the group finally bills its 8 real cores — but that is only a 1.17x + # margin over the worst tile measured, and the first thing to give would + # be the page cache holding the store. Take it only with a measurement + # of cache behaviour under pressure, not on the arithmetic alone. + mem_mb = lambda wc, attempt: 5000 * attempt, # 120 on the FIRST attempt, and ATTEMPT-SCALED after it. MEASURED # two ways, and the second is why the margin is thinner than it looks: # * alone (job 20795277, tile 198.305 chunk 1): ~76 min for 3547 From 7302a2d458f4c97106dffb8bed071a626e689c48 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 02:45:17 -0400 Subject: [PATCH 048/100] fix(orchestration): make the WCS staging atomic; correct two memory claims Adversarial review of the two preceding commits, against the code rather than against their messages. Three things came back. STAGING WAS NOT ATOMIC, and `cp -u` made that permanent. `cp` writes in place, so an interruption -- ENOSPC on /local/scratch, an OOM-kill (which skips the EXIT trap), node death -- leaves a truncated destination whose mtime is NEWER than the NFS source. `cp -u` then skips it forever, and nothing downstream revalidates: TILE_VIGNET_FRESH and TILE_VIGNET_REQUIRED both look only at the vignette store. A truncated WCS store reaches ngmix as wrong astrometry rather than as an error, which is the worst failure mode this change could have had. It is now an unconditional copy to a temp name plus `mv -f`, atomic within the filesystem, with the source spelled out by tile id instead of globbed so a missing file is one precise error. Unconditional also closes the race the eight concurrent chunks had on a shared destination in the tile_vignets-not-in-group recovery path; renaming over a file a sibling already has open is safe, since the open descriptor keeps the old, identical inode. Nine 11 MB copies per tile at job start is not a cost worth a staleness rule. THE MEMORY MARGIN WAS OVERSTATED. The previous comment took snakemake's psutil benchmark (~1.25 GiB per chunk) as the anonymous footprint and claimed ~4x headroom. This repo carries a second estimate that disagrees by 1.8x -- config_tile_Ng_template.ini's own SAVE_BATCH note puts a worker at ~2.3 GB peak from an A/B test -- and the psutil figure is sampled on a 30-second grid, which is exactly what a SAVE_BATCH = 250 flush sawtooth hides. Prefer the larger: ~18.4 GiB anonymous for eight chunks, so the margin is ~2.1x with ~20 GiB left for cache against an 8.1 GiB store. Still comfortable, and now honest. The 4000 floor's margin is 1.14x, not the 1.17x claimed (MiB/GiB slip). EDITING tile_local() INVALIDATES FINISHED TILES, and the consequences on a resume are destructive rather than merely expensive: the reruns are unsatisfiable because clean_exposure has reclaimed the exposure stores, and a failed group job's postprocess deletes every member's existing outputs -- including tile_make_cat's final_cat on the persistent root. The full mechanism is now in the TILE_LOCAL comment, with the `--rerun-triggers` escape hatch for the case where a resume is unavoidable. Nothing about the change is wrong; it just must land between campaigns. Also corrected: the prologue does not run in tile_merge_cats (no pre_run), and config_tile_Mc.ini reads $NGMIX_VIGNET_DIR but not $SP_WCS_DIR. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NYcxuAbZbQivC5UFtgecxH --- workflow/rules/tile.smk | 110 +++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 25 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index e4e475c8c..5215a2095 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -79,8 +79,8 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # bulk intra-tile intermediate is node-local. That split is possible because # ShapePipe's configs set input and output paths independently -- see # config_tile_PiViVi.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and -# config_tile_Ng_template.ini / config_tile_Mc.ini ($NGMIX_VIGNET_DIR, -# $SP_WCS_DIR). +# config_tile_Ng_template.ini and config_tile_Mc.ini ($NGMIX_VIGNET_DIR), and +# config_tile_Ng_template.ini alone for $SP_WCS_DIR. # # TWO node-local stores, not one, and the second is small on purpose. The # vignette store is ~5-8 GB of bulk; $SP_WCS_DIR holds a single 11.3 MB file, @@ -116,6 +116,33 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # prologue (unit_pre) and completeness.py are untouched by design: both are # fingerprinted into every rule, and changing either would invalidate the whole # campaign, including the 117 finished exposure chains. +# +# EDITING THIS FUNCTION INVALIDATES FINISHED TILES. READ THIS BEFORE YOU DO. +# The returned string lands in `params.pre`, `params` is an active rerun trigger +# in profiles/nibi/config.yaml, and `params.pre` is a non-derived param (its +# lambda takes only `wc`), so snakemake records it and compares it. Any edit +# here therefore reschedules tile_vignets, all eight tile_ngmix chunks and +# tile_make_cat for EVERY tile the campaign already finished. +# +# On a fresh campaign that is free. On a RESUME it is destructive, and the +# mechanism is worth spelling out because it is not obvious: +# * clean_exposure has already reclaimed the exposure stores those tiles read, +# and tile_finished() deliberately drops the manifests of a finished tile's +# exposures, so the rerun is UNSATISFIABLE -- PiViVi runs against dangling +# symlinks and the group fails; +# * a failed group job's postprocess(error=True) fans out over every member in +# every toposort level and removes each member's EXISTING outputs. One of +# those is tile_make_cat's final_cat on the persistent root. So the failure +# deletes the science product of a tile that was finished and correct; +# * with final_cat gone, tile_finished() flips and the tile re-declares its +# whole exposure edge set, rebuilding the reclaimed chains from VOS. That is +# the rerun avalanche D5 exists to prevent, arriving through the params +# trigger instead of through inputs. +# So: land changes to this function BETWEEN campaigns, never mid-campaign. If a +# resume is genuinely needed after an edit, run it once with +# `--rerun-triggers mtime code software-env`, accepting that clean_exposure's +# consumer-set staleness detection (which rides on params) is off for that +# invocation. def tile_local(tile): """The node-local prologue, as bash, for one tile. @@ -156,13 +183,31 @@ def tile_local(tile): distribution, and the small one is worth ~1.8x on the rule that is 99.3% of tile-side core-hours. - A copy, never a symlink: a link resolves straight back to NFS. Every member - of the group runs this prologue, so the copy is attempted up to ten times - per tile; `cp -u` makes the repeats free. `tile_merge_headers` is upstream of - the whole group, so the file exists by the time any member runs, and if it - does not then failing here is correct -- the alternative is ngmix silently - reading a different tree. + A copy, never a symlink: a link resolves straight back to NFS. The prologue + runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in + `tile_merge_cats`, which has no pre_run), so the staging is attempted nine + times per tile and eight of those are concurrent siblings in one toposort + level. It is written as copy-to-a-temp-name plus `mv -f`, which is atomic + within a filesystem, and it is UNCONDITIONAL -- no `cp -u`, no + already-there test. `cp` is not atomic, so an interrupted copy leaves a + truncated destination whose mtime is NEWER than the source; `-u` would then + skip it forever, and nothing downstream would catch it, because + TILE_VIGNET_FRESH and TILE_VIGNET_REQUIRED both look only at the vignette + store. A silently truncated WCS store reaches ngmix as wrong astrometry + rather than as an error, which is the worst failure mode available here. + Nine unconditional copies of 11 MB per tile, once at job start, is not a + cost worth reasoning about; a staleness rule would be. Renaming over a file + a sibling chunk already has open is safe -- the open descriptor keeps the + old inode, whose contents are identical. + + `tile_merge_headers` is upstream of the whole group, so the file exists by + the time any member runs; failing loudly if it does not is correct, because + the alternative is ngmix silently reading a different tree. """ + # `log_exp_headers--.sqlite`, named from the tile with the dot + # replaced -- spelled out rather than globbed so a missing file is one + # precise error instead of an empty expansion. + tile_num = tile.replace(".", "-") return f''' if [ ! -d /local/scratch ]; then echo "tile_shape: node-local storage unavailable." >&2 @@ -178,12 +223,17 @@ mkdir -p "$SP_VIGNET_OUT" "$SP_WCS_DIR" || {{ echo "tile_shape: cannot create $SP_LOCAL on this node." >&2 exit 1 }} -# the WCS store -- see the docstring; a copy, never a symlink -cp -u "$SP_RUN"/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers-*.sqlite \ - "$SP_WCS_DIR"/ || {{ - echo "tile_shape: no log_exp_headers-*.sqlite under" >&2 - echo " $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output" >&2 - echo " tile_merge_headers must have run before this group." >&2 +# the WCS store -- see the docstring. Copy to a temp name and rename, because +# `cp` is not atomic and the eight chunks share this destination. +sp_wcs_src="$SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers-{tile_num}.sqlite" +cp "$sp_wcs_src" "$SP_WCS_DIR/.log_exp_headers.$$" && \ +mv -f "$SP_WCS_DIR/.log_exp_headers.$$" \ + "$SP_WCS_DIR/log_exp_headers-{tile_num}.sqlite" || {{ + rm -f "$SP_WCS_DIR/.log_exp_headers.$$" + echo "tile_shape: could not stage the WCS store." >&2 + echo " source: $sp_wcs_src" >&2 + echo " dest: $SP_WCS_DIR" >&2 + echo " check that tile_merge_headers ran, and that /local/scratch has room." >&2 exit 1 }} find /local/scratch -maxdepth 1 -name 'sp-*.*' -user "$(id -u)" -mmin +1440 \ @@ -549,20 +599,30 @@ rule tile_ngmix: # sacct's MaxRSS here is NOT process memory. nibi runs # JobAcctGatherType=jobacct_gather/cgroup, so it reports the cgroup's # memory.current, which under cgroup v2 CHARGES PAGE CACHE to the job. - # Snakemake's own psutil benchmark for the same chunks says max_rss - # ~1.25 GiB each, so the eight chunks' real anonymous footprint is - # ~10 GiB and everything above that is file-backed cache — the store - # written and then read back. Cache is reclaimable: the kernel evicts it - # before it kills anything, so this reservation cannot OOM the job. What - # it can do is squeeze the cache that keeps the node-local store - # resident, which is part of why the fused tile is fast. + # Cache is reclaimable — the kernel evicts it before it kills anything — + # so the cgroup high-water is an upper bound on what the job NEEDS, not + # a floor. What a tight reservation can still do is squeeze the cache + # that keeps the node-local store resident, which is part of why the + # fused tile is fast. # # MEASURED ACROSS 31 TILES (campaign smk-g4, 2026-08-30): cgroup # high-water 27.40 GiB max, 22.14 GiB mean, against the 109.4 GiB the # group was reserving. The eight chunks are SIBLINGS in the group's # toposort so snakemake SUMS their mem_mb; at 5000 the group asks - # 40000 MiB, a 1.43x margin over the worst tile observed and ~4x the - # anonymous footprint, leaving ~29 GiB of cache for an 8.1 GiB store. + # 40000 MiB = 39.1 GiB, a 1.43x margin over the worst tile observed. + # + # The ANONYMOUS half of that, which is the part that can actually OOM, + # is the number to argue about, and this repo carries two estimates that + # disagree by 1.8x. Snakemake's psutil benchmark says ~1.25 GiB per + # chunk (~10 GiB for eight); config_tile_Ng_template.ini's own + # SAVE_BATCH note says ~2.3 GB per worker from an A/B test (job + # 17607877), i.e. ~18.4 GiB. Prefer the larger: snakemake samples RSS on + # a 30-second grid (BENCHMARK_INTERVAL), and a SAVE_BATCH = 250 flush + # cycle is exactly the sawtooth such a grid misses. So read the margin + # as ~2.1x over anonymous memory with ~20 GiB left for cache against an + # 8.1 GiB store — comfortable, but not the 4x a psutil-only reading + # would suggest. An OOM is also self-healing: mem_mb scales with + # attempt, so a retry asks 80000. # # WHY THIS IS THE BIG ONE. nibi bills max(cores, mem_GB/4), so at # 112 GB the fused group billed 28 core-equivalents for 8 real cores — @@ -572,8 +632,8 @@ rule tile_ngmix: # at 10, and DR6's wall clock is (tiles / tiles-in-flight) x elapsed. # # 4000 is the floor and is NOT recommended yet: it would take the group - # to 32000, where tile_vignets' own 32000 becomes the binding term and - # the group finally bills its 8 real cores — but that is only a 1.17x + # to 32000 MiB, where tile_vignets' own 32000 becomes the binding term + # and the group finally bills its 8 real cores — but that is a 1.14x # margin over the worst tile measured, and the first thing to give would # be the page cache holding the store. Take it only with a measurement # of cache behaviour under pressure, not on the arithmetic alone. From 234b2c3de0c9017681ad7a45ce8528716e968898 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:07:15 -0400 Subject: [PATCH 049/100] run(smk-g5): a one-tile paired control for the two unvalidated fixes smk-g4 landed two changes after its campaign finished and neither has ever run: the tile-level WCS sqlite staged node-local, and tile_ngmix's mem_mb cut 14000 -> 5000. Point the campaign roots at a fresh smk-g5 tree holding one tile, 186.307. That tile is the control because it ran ALONE as smk-g4 job 20799387 rather than inside the 34-tile campaign, so its baseline has no concurrency confound: 8063 s elapsed, chunk median 6813 s at 59% mean load, MaxRSS 26.6 GiB. The prediction, registered in job_head.sh before the run: load above 90%, chunk median near its own 4050 s of CPU, group near 83 min, MaxRSS unmoved (it is tile_vignets' peak, not ngmix's), and final_cat bit-identical to smk-g4's -- which #873's position-seeded split makes checkable for the first time. Fresh roots rather than a resume, because tile_local() changed and `params` is an active rerun trigger: pointing this at smk-g4 would delete the 34 finished tiles' catalogues rather than merely re-run them. clean is off -- ~54 GiB of exposure stores against 979 GiB free buys a re-measurement that re-runs only the shape chain. --- workflow/config.yaml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index e67b6867e..40bf1d096 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -6,15 +6,17 @@ # The tile list that scopes this run (one "IDra.IDdec" per line). # The campaign grows by appending to this file — parse-time config, so -# completed work is never invalidated. Current contents: the 34-tile smk-g4 -# benchmark set, identical to the tile list smk-g3 finished (the 186/187 quad -# plus the first 30 p3-batch1 tiles), so the two campaigns' catalogues are -# comparable object for object. It lives on the persistent root because it -# defines the campaign and must outlive the scratch purge. +# completed work is never invalidated. Current contents: smk-g5, ONE tile +# (186.307), the paired control for the two fixes landed after smk-g4 — the +# node-local WCS sqlite and tile_ngmix's mem_mb 14000 -> 5000. That tile ran +# ALONE as smk-g4 job 20799387 (elapsed 8063 s, chunk median 6813 s, chunk mean +# load 59%, MaxRSS 26.6 GiB), so the baseline carries no concurrency confound +# and the contrast is same tile, same solitude, two commits apart. +# The 34-tile smk-g4 set is preserved at smk-g4/tiles34.txt. # (The 210/211 quad used earlier is unusable: its tile images are symlinks into # anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are # broken.) -tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g4/tiles34.txt +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g5/tiles1.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif @@ -25,7 +27,7 @@ container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # intermediates, sized so a batch finishes inside the 60-day purge window. The # sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-g4 +run_dir: /scratch/cdaley/shapepipe-output/smk-g5 # products_dir is the PERSISTENT root: the durable, low-volume products — the # final catalogues (/tiles///final_cat-.fits, @@ -42,7 +44,7 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g4 # # Snakemake's own state is the one durable-looking thing that stays on scratch # (-state; bin/sp explains why). -products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g4 +products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g5 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -79,16 +81,17 @@ star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction # would otherwise have to rebuild from tile headers. -index_db: /project/def-mjhudson/cdaley/sp-products/smk-g4/index/run_index.sqlite +index_db: /project/def-mjhudson/cdaley/sp-products/smk-g5/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, # and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. On for smk-g4: -# reclamation is part of what this campaign measures, and at 34 tiles from -# scratch the un-cleaned exposure high-water is the run's binding disk constraint. -clean: true +# back out of the tombstone and reports the exposure as `cleaned`. OFF for smk-g5: +# this is a one-tile control, its ~8 exposure stores cost ~54 GiB against 979 GiB +# free, reclamation is already measured exactly by smk-g4, and keeping the stores +# means a re-measurement re-runs only the shape chain instead of the whole tile. +clean: false # Tiles that may NOT pin an exposure store (default: empty). # From a8ec7647f7c25ad1a9411295f0cbedd140c59f3e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:22:42 -0400 Subject: [PATCH 050/100] perf(orchestration): split ngmix chunks by epoch-weighted cost, not object count ngmix fits an object jointly across every exposure it lands on, so a chunk's cost is set by its (object, epoch) pairs, not its objects. The campaign measured 0.2714 CPU-s per pair with an intercept consistent with zero, plus ~0.05 CPU-s of per-object setup, and equal-object-count chunks of the SAME tile still varied 1.6x in cost, tracking their epoch counts at R^2 0.91-1.000. The eight chunks are siblings in the fused tile_shape group, which ends when its slowest member does, so that spread was pure wall clock: 7.7 hours across the 34-tile smk-g4 campaign, and on 196.307 it took the group to 97.4% of its wall limit. The epoch count is already on disk. tile_detect's SExtractor post-process writes one EPOCH_ extension per overlapping exposure into the same sexcat this script already opens -- n_obj rows of NUMBER/EXP_NAME/CCD_N, 529,470 B each -- and summing CCD_N >= 0 across them reproduces the final catalogue's N_EPOCH column exactly, checked row by row against 186.307's final_cat, all 35,298 of them. The extension count is the tile's exposure count, so they are found by name prefix. LDAC_OBJECTS is never read: at 10,662 B per row it is 376,347,276 B of the 380 MB file, more than the straggle this exists to remove. Weights are w_i = 1000*epochs_i + 184, the 184 being 0.05/0.2714 epochs of per-object setup -- it is what stops the 13 zero-epoch objects on 186.307 from weighing nothing. The split minimises the MAXIMUM chunk weight exactly (bisection on capacity, then a left-to-right fill), because the group waits for the slowest chunk and not the average one. On 186.307's real sexcat the old ranges give per-chunk epochs 12,783-16,881, max/median 1.174; the new ones 14,509-14,704, max/median 1.008. Predicted slowest-chunk cost 4802 -> 4181 CPU-s, -12.9%. Across all 34 sexcats the slowest chunk goes from 1.131x-1.627x its tile's median to 1.0000x-1.0002x, and the sum of slowest-chunk cost falls 132,875 -> 113,710 predicted CPU-s, 14.4%. The predictions track that tile's eight measured chunk CPU times at R^2 0.974, over-predicting by a uniform ~3.4%, and the residual has a named cause: N_EPOCH is GEOMETRIC, while ngmix drops epochs it cannot fit. NGMIX_N_EPOCH in the same final catalogue is lower for 2,203 of the 35,298 objects and never higher -- 113,947 pairs against 116,727, 2.4% -- and refit on it the law is 0.2681 CPU-s per pair at R^2 0.997, i.e. the campaign's 0.2714 is right and the geometric proxy is what is slightly off. The oracle cannot be had before ngmix runs and is barely worth having: splitting on it moves boundaries by up to 177 objects and improves the true slowest chunk by 1.07%. Chunk boundaries cannot change a result -- ngmix seeds its RNG from object position, which is boundary-invariant by construction, and SEED_FROM_POSITION is no longer even optional (ngmix_runner.py raises on False). The hazard this is designed against is that all eight chunks compute the ranges INDEPENDENTLY and must agree; a disagreement double-measures or drops objects and merge_sep_cats would concatenate the result without complaint. Hence integer arithmetic end to end, NUMBER = 1..N asserted rather than assumed, and no fallback: an unreadable EPOCH extension exits non-zero, because a fallback would apply only to the chunks that hit it. NGMIX_RANGE_HASH is the same hazard across TIME. tile_ngmix's only fingerprint was SCRIPT_HASH, which is completeness.py; params.pre carries the range script's INVOCATION, and the invocation is invariant to the script's body. So a tile resumed across an edit to the split would have kept the old ranges on its finished chunks and taken the new ones on its reruns -- some objects measured twice, others by nobody, the tile green. The hash joins SCRIPT/FOREST/CLEAN/ STAR_CAT in the Snakefile and lands on tile_ngmix alone. It also means this commit correctly invalidates in-flight ngmix chunks rather than mixing splits. tests/unit/test_ngmix_range.py pins the partition invariant with hypothesis over arbitrary epoch distributions, and checks the split against a brute-force minimax on small cases. It imports the script by path, so it needs neither the container nor the shapepipe package. n_obj < n_chunks is now defined instead of accidental: the surplus chunks get (n_obj + 1, n_obj), the canonical empty closed interval. The old equal-count split emitted (1, 0) there, and ngmix reads ID_OBJ_MAX = 0 as unbounded (ngmix.py:804-806) -- each empty chunk would have re-measured the entire tile. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- tests/unit/test_ngmix_range.py | 212 +++++++++++++++++++++++++++++++ workflow/Snakefile | 11 ++ workflow/rules/tile.smk | 10 +- workflow/scripts/ngmix_range.py | 216 +++++++++++++++++++++++++++++--- 4 files changed, 431 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_ngmix_range.py diff --git a/tests/unit/test_ngmix_range.py b/tests/unit/test_ngmix_range.py new file mode 100644 index 000000000..a96ba5cf3 --- /dev/null +++ b/tests/unit/test_ngmix_range.py @@ -0,0 +1,212 @@ +"""The ngmix chunk split tiles ``[1, n_obj]`` exactly. + +``workflow/scripts/ngmix_range.py`` is run INDEPENDENTLY by each of the +``n_chunks`` ngmix processes, which then trust each other to have computed the +same ranges. Nothing downstream cross-checks: merge_sep_cats concatenates the +chunk catalogues, so an overlap silently duplicates objects and a gap silently +drops them. The one invariant that catches both is that the ranges partition +``1..n_obj`` — that is what this module asserts, over the epoch-weight +distributions the splitter actually has to cope with. + +Deliberately container-free: the range function takes a plain sequence of epoch +counts, so nothing here imports astropy, numpy, or shapepipe. +""" + +import importlib.util +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "ngmix_range.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_ngmix_range", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +ngmix_range = _load() + + +def assert_tiles(ranges, n_obj, n_chunks): + """Assert the partition invariant, in the form each failure mode takes.""" + assert len(ranges) == n_chunks + assert ranges[0][0] == 1 + assert ranges[-1][1] == n_obj + for (lo, hi), (prev_lo, prev_hi) in zip(ranges[1:], ranges): + assert lo == prev_hi + 1 + assert hi >= lo - 1, "an empty range is (hi + 1, hi), never narrower" + assert prev_hi >= prev_lo - 1 + covered = [i for lo, hi in ranges for i in range(lo, hi + 1)] + assert covered == list(range(1, n_obj + 1)) + # ID_OBJ_MAX <= 0 is ngmix's "unbounded" sentinel: a chunk emitting it + # re-measures the whole tile instead of nothing. + assert all(hi >= 1 for _, hi in ranges) + + +# --------------------------------------------------------------------------- # +# The property +# --------------------------------------------------------------------------- # + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=400), + n_chunks=st.integers(min_value=1, max_value=16), + data=st.data(), +) +def test_ranges_tile_the_catalogue(n_obj, n_chunks, data): + """Any epoch distribution, any shape: the ranges still partition 1..N.""" + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=40), + min_size=n_obj, + max_size=n_obj, + ) + ) + assert_tiles(ngmix_range.id_ranges(epochs, n_chunks), n_obj, n_chunks) + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=60), + n_chunks=st.integers(min_value=1, max_value=8), + data=st.data(), +) +def test_ranges_are_deterministic(n_obj, n_chunks, data): + """Same input, same ranges — the eight processes share nothing else. + + Integer weights make the split a pure function of the epoch list, with no + float accumulation whose order could matter; this pins that. + """ + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=99), + min_size=n_obj, + max_size=n_obj, + ) + ) + first = ngmix_range.id_ranges(epochs, n_chunks) + assert first == ngmix_range.id_ranges(list(epochs), n_chunks) + assert all(isinstance(b, int) for lo, hi in first for b in (lo, hi)) + + +# --------------------------------------------------------------------------- # +# Degenerate shapes — each pins a decision, not just an absence of crash +# --------------------------------------------------------------------------- # + + +def test_single_chunk_takes_everything(): + """n_chunks == 1: one range over the whole catalogue.""" + assert ngmix_range.id_ranges([3] * 17, 1) == [(1, 17)] + + +def test_one_object_per_chunk(): + """n_obj == n_chunks: one object each, however lopsided the weights.""" + assert ngmix_range.id_ranges([0, 9, 1, 40], 4) == [ + (1, 1), (2, 2), (3, 3), (4, 4) + ] + + +def test_fewer_objects_than_chunks_pads_with_empty_ranges(): + """DOCUMENTED: the surplus chunks get ``(n_obj + 1, n_obj)``, not (1, 0). + + The old equal-count split gave every surplus chunk ``(1, 0)``, and ngmix + reads ``ID_OBJ_MAX = 0`` as unbounded — so each of them would have measured + the entire tile rather than nothing. + """ + assert ngmix_range.id_ranges([2, 5, 1], 6) == [ + (1, 1), (2, 2), (3, 3), (4, 3), (4, 3), (4, 3) + ] + assert_tiles(ngmix_range.id_ranges([2, 5, 1], 6), 3, 6) + + +def test_zero_objects_is_fatal(): + """No split is meaningful, and (1, 0) is ngmix's unbounded sentinel.""" + with pytest.raises(ValueError, match="zero objects"): + ngmix_range.id_ranges([], 8) + + +def test_zero_chunks_is_fatal(): + """There is no zeroth chunk to hand a range to.""" + with pytest.raises(ValueError, match="n_chunks"): + ngmix_range.id_ranges([1, 2, 3], 0) + + +def test_equal_weights_reproduce_an_equal_count_split(): + """Uniform epochs: chunk sizes differ by at most one object.""" + ranges = ngmix_range.id_ranges([3] * 1000, 8) + assert_tiles(ranges, 1000, 8) + sizes = [hi - lo + 1 for lo, hi in ranges] + assert max(sizes) - min(sizes) <= 1 + assert sum(sizes) == 1000 + + +def test_zero_epoch_objects_still_weigh_something(): + """ALPHA is why: without it a 0-epoch tail is free and lands in one chunk. + + Ninety-six zero-epoch objects and four 1-epoch ones. Weighing only epochs + would let a single chunk swallow all ninety-six. + """ + ranges = ngmix_range.id_ranges([0] * 96 + [1] * 4, 4) + assert_tiles(ranges, 100, 4) + assert max(hi - lo + 1 for lo, hi in ranges) < 96 + + +def test_one_enormously_heavy_object_is_isolated(): + """The heavy object gets a chunk to itself; the tail still tiles.""" + epochs = [1] * 20 + [10_000] + [1] * 20 + ranges = ngmix_range.id_ranges(epochs, 4) + assert_tiles(ranges, 41, 4) + assert (21, 21) in ranges + + +# --------------------------------------------------------------------------- # +# The split is the optimal one, not merely a legal one +# --------------------------------------------------------------------------- # + + +def _brute_force_min_max(weights, n_chunks): + """Smallest achievable maximum chunk weight, by exhaustive enumeration.""" + n = len(weights) + best = {} + + def solve(start, chunks): + if chunks == 1: + return sum(weights[start:]) + if (start, chunks) not in best: + best[(start, chunks)] = min( + max(sum(weights[start:cut]), solve(cut, chunks - 1)) + for cut in range(start + 1, n - chunks + 2) + ) + return best[(start, chunks)] + + return solve(0, n_chunks) + + +@settings(deadline=None, max_examples=40) +@given( + epochs=st.lists( + st.integers(min_value=0, max_value=12), min_size=4, max_size=14 + ), + n_chunks=st.integers(min_value=2, max_value=4), +) +def test_slowest_chunk_is_minimal(epochs, n_chunks): + """The objective is the slowest chunk — check it against brute force.""" + if len(epochs) < n_chunks: + return + weights = [ + ngmix_range.MILLI_EPOCH * e + ngmix_range.ALPHA_MILLI_EPOCHS + for e in epochs + ] + ranges = ngmix_range.id_ranges(epochs, n_chunks) + loads = [sum(weights[lo - 1:hi]) for lo, hi in ranges] + assert max(loads) == _brute_force_min_max(weights, n_chunks) diff --git a/workflow/Snakefile b/workflow/Snakefile index 0f2541334..9ee270824 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -257,6 +257,17 @@ CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdiges # config.yaml's star_cats block on clearing the store after a semantic change. STAR_CAT_HASH = hashlib.md5( (SCRIPTS / "star_cats.py").read_bytes()).hexdigest()[:12] +# ngmix_range.py earns a hash for a stronger reason than the others. What it +# emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks +# are a PARTITION of its object IDs: resume a tile across an edit to the split +# and the chunks that already succeeded keep the old ranges while the reruns +# take the new ones, so within one tile some objects are measured twice and +# others by nobody. merge_sep_cats concatenates whatever it is handed, so the +# tile completes green with a corrupt catalogue and no error anywhere. Same +# failure family as the script's own no-fallback rule, across time rather than +# across processes. +NGMIX_RANGE_HASH = hashlib.md5( + (SCRIPTS / "ngmix_range.py").read_bytes()).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 5215a2095..7d8ceaf39 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -572,7 +572,15 @@ rule tile_ngmix: f'|| exit 1', 'eval "$ngmix_range_out"', tile_local(wc.tile), TILE_VIGNET_REQUIRED]), - script_hash = SCRIPT_HASH + script_hash = SCRIPT_HASH, + # The one rule whose fingerprint has to cover ngmix_range.py. + # `pre_run` already puts the INVOCATION in params, but the invocation + # is invariant to the script's body — and what that body decides is a + # PARTITION. Resume a tile across an edit to the split without this and + # the already-done chunks keep the old ranges while the reruns take the + # new ones: some objects measured twice, others by nobody, the tile + # green, no error anywhere. + range_hash = NGMIX_RANGE_HASH # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over # filehd.process_list, i.e. parallelism ACROSS INPUT FILE SETS. An ngmix diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index 9e3058052..9ff24b3f7 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -8,46 +8,228 @@ # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering -[1, N] processes every object exactly once. The first N-1 chunks get equal -shares; the last takes the remainder, with a CLOSED upper bound at n_obj — -never ID_OBJ_MAX = -1, which ngmix reads as unbounded and would double-count. -Each tile is sized from its OWN count (the bash monolith used the campaign -average and left the last chunk open). +[1, N] processes every object exactly once. The bounds are CLOSED, never +ID_OBJ_MAX = -1 — ngmix treats ``id_obj_max <= 0`` as unbounded +(``ngmix_package/ngmix.py:804-806``), so an open-ended last chunk silently +re-measures the whole tile instead of its share; rule tile_ngmix carries the +straggler that taught this. + +CHUNKS ARE BALANCED BY EPOCH-WEIGHTED COST, NOT BY OBJECT COUNT. ngmix fits an +object jointly across every exposure it lands on, so its cost scales with +(object, epoch) pairs, not objects: the campaign measured 0.2714 CPU-s per +pair with an intercept consistent with zero, plus ~0.05 CPU-s per-object of +setup. The old equal-count split therefore produced chunks whose cost varied +1.6x WITHIN a single tile, tracking their epoch counts at R^2 0.91-1.000. Those +chunks are siblings in the fused tile_shape group job, which ends when its +SLOWEST member does, so the spread was pure wall clock: 7.7 hours over the +34-tile smk-g4 campaign, and on the worst tile (196.307) the slowest chunk ran +26.3 min past the median and took the group to 97.4% of its wall limit. + +Re-splitting all 34 of that campaign's sexcats: the old boundaries leave the +slowest chunk at 1.131x-1.627x its tile's median predicted cost (worst +200.302), the new ones at 1.0000x-1.0002x, and the sum over tiles of +slowest-chunk cost falls 132,875 -> 113,710 predicted CPU-s, 14.4%. + +Moving the boundaries is scientifically free: ngmix seeds its RNG from object +POSITION (SEED_FROM_POSITION), so which chunk an object falls in cannot change +its measurement. + +DETERMINISM HERE IS CORRECTNESS, NOT TIDINESS. The n_chunks chunks are separate +processes that each run this script independently and must derive the IDENTICAL +range set; if two of them disagree, objects are silently measured twice or +silently dropped and nothing downstream notices — merge_sep_cats concatenates +whatever it is given. Hence integer arithmetic end to end (weights scale to +milli-epochs so no float ever decides a boundary), and NO FALLBACK: if the +EPOCH extensions cannot be read this script exits non-zero. Such a fallback +would be the worst available behaviour precisely because it would apply only to +the chunks that hit the failure, shredding the tile's coverage instead of +failing it. """ import argparse from pathlib import Path +# Weights are integers in milli-epochs (one epoch = 1000) so every boundary is +# decided by exact integer comparison, identically on all n_chunks processes. +MILLI_EPOCH = 1000 + +# The per-object setup cost, expressed in epochs so one integer weight carries +# both terms of the measured cost law: 0.05 CPU-s of setup / 0.2714 CPU-s per +# (object, epoch) = 0.184 epoch-equivalents. It is what keeps the zero-epoch +# objects (13 of 35,298 on tile 186.307) from weighing nothing — they still +# cost ~6% of a typical object, and a chunk handed thousands of them for free +# would be a straggler of a new kind. Only the RATIO of the two costs moves a +# boundary, which makes this robust: refitting on 186.307's eight measured +# chunk CPU times with the setup term held fixed gives 0.2619 per GEOMETRIC +# epoch (see object_epochs for why that is below 0.2714), i.e. ALPHA 0.191, +# and every boundary on that tile then moves by at most one object. Zeroing +# ALPHA entirely moves them by at most 44. +ALPHA_MILLI_EPOCHS = 184 + + +def id_ranges(epochs, n_chunks: int) -> list[tuple[int, int]]: + """Split object IDs ``1..len(epochs)`` into ``n_chunks`` closed ranges. + + ``epochs[i]`` is object ``i + 1``'s geometric epoch count. The ranges are + CONTIGUOUS — ``NGMIX_ID_MIN``/``NGMIX_ID_MAX`` is an interval, not a set — + and tile ``[1, n_obj]`` exactly, so every object is measured once. + + The objective is the SLOWEST chunk, not the average one, because the + group job waits for it. So this minimises the maximum chunk weight + exactly: binary-search the smallest feasible capacity, then fill left to + right under it. Ties in that maximum break toward the earlier chunks, + which is arbitrary but fixed, and fixed is the property that matters. + + With ``n_obj < n_chunks`` the first ``n_obj`` chunks take one object each + and the remainder are EMPTY, written ``(n_obj + 1, n_obj)``: ``lo = hi+1`` + is the canonical empty closed interval and preserves the chain + ``ranges[k][0] == ranges[k-1][1] + 1``. Deliberately not ``(1, 0)``, which + is what the old equal-count split emitted here — ``ID_OBJ_MAX = 0`` is + ngmix's unbounded sentinel, so each empty chunk would have re-measured the + ENTIRE tile. + """ + if n_chunks < 1: + raise ValueError(f"n_chunks must be >= 1, got {n_chunks}") + n_obj = len(epochs) + if n_obj < 1: + raise ValueError("cannot split a catalogue of zero objects") -def id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: - base, rem = divmod(n_obj, n_chunks) - ranges, lo = [], 1 - for k in range(1, n_chunks + 1): - hi = lo + base + (rem if k == n_chunks else 0) - 1 - ranges.append((lo, hi)) - lo = hi + 1 + if n_obj < n_chunks: + return ([(k, k) for k in range(1, n_obj + 1)] + + [(n_obj + 1, n_obj)] * (n_chunks - n_obj)) + + weights = [MILLI_EPOCH * int(e) + ALPHA_MILLI_EPOCHS for e in epochs] + cap = _min_feasible_capacity(weights, n_chunks) + + ranges: list[tuple[int, int]] = [] + start, load = 0, 0 + for i, w in enumerate(weights): + # Close before object i when the current chunk holds something and + # either it cannot take i under `cap`, or the objects still to come + # (n_obj - i) no longer outnumber the chunks still to open — the second + # clause is what guarantees exactly n_chunks non-empty ranges when a + # heavy head would otherwise leave the tail with nothing to hold. + if start < i and (load + w > cap + or n_obj - i < n_chunks - len(ranges)): + ranges.append((start + 1, i)) + start, load = i, 0 + load += w + ranges.append((start + 1, n_obj)) return ranges -def object_count(run_dir: Path) -> int: - """NAXIS2 of the last HDU of this tile's sexcat (get_number_objects.py).""" +def _chunks_needed(weights: list[int], cap: int) -> int: + """Chunks a left-to-right fill uses when none may exceed ``cap``.""" + used, load = 1, 0 + for w in weights: + if load + w > cap: + used, load = used + 1, w + else: + load += w + return used + + +def _min_feasible_capacity(weights: list[int], n_chunks: int) -> int: + """Smallest ``cap`` that fits ``weights`` into ``n_chunks`` chunks. + + ``_chunks_needed`` is monotone non-increasing in ``cap``, so bisection on + the integers ``[max(weights), sum(weights)]`` lands on the exact optimum + without a float ever entering the comparison. + """ + lo, hi = max(weights), sum(weights) + while lo < hi: + mid = (lo + hi) // 2 + if _chunks_needed(weights, mid) <= n_chunks: + hi = mid + else: + lo = mid + 1 + return lo + + +def object_epochs(run_dir: Path): + """Per-object geometric epoch count, from this tile's own sexcat. + + tile_detect's SExtractor post-process (``MAKE_POST_PROCESS`` in + ``config_tile_Sx.ini``) writes one ``EPOCH_`` extension per exposure + overlapping the tile — so the extension COUNT is tile-specific and is + discovered by name, never assumed — each with ``n_obj`` rows in NUMBER + order and ``CCD_N < 0`` where the object misses that exposure. Summing + ``CCD_N >= 0`` across them reproduces the final catalogue's ``N_EPOCH`` + column exactly — checked row by row against 186.307's + ``run_sp_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, + 116,727 pairs, mean 3.31. The post-process is upstream of the whole + tile_shape group, so the extensions always exist by the time ngmix runs; + their absence is a broken tile, not a case to accommodate. + + N_EPOCH is a GEOMETRIC count and mildly over-states the work, because ngmix + drops epochs it cannot fit. The same catalogue's NGMIX_N_EPOCH is lower for + 2,203 of the 35,298 objects and never higher: 113,947 pairs against + 116,727, 2.4%. That gap is the whole of the ~3.4% by which this cost model + over-predicts the tile's measured chunk times — refit on NGMIX_N_EPOCH the + law is 0.2681 CPU-s per pair at R^2 0.997, against 0.2619 at R^2 0.973 on + the geometric count. The better number is unavailable before ngmix runs and + is not worth wanting anyway: splitting on it moves boundaries by up to 177 + objects and improves the true slowest chunk by 1.07%. + + Only the EPOCH extensions are read. ``LDAC_OBJECTS`` carries a 10 kB + VIGNET per row (376 MB on 186.307) and pulling it in would cost more than + the straggle this split exists to remove; the EPOCH tables are ~530 kB + each, and its NAXIS2 comes from the header alone as the row-count check. + """ + import numpy as np from astropy.io import fits cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( "sextractor_runner/output/sexcat*.fits")) if not cats: raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") - with fits.open(cats[0]) as hdul: - return int(hdul[-1].header["NAXIS2"]) + with fits.open(cats[0], memmap=True) as hdul: + epoch_hdus = [h for h in hdul if h.name.startswith("EPOCH")] + if not epoch_hdus: + raise SystemExit( + f"[ngmix_range] FATAL: no EPOCH extensions in {cats[0]}; " + "tile_detect's SExtractor post-process did not run. " + "Refusing to fall back to an equal-object split: it would " + "apply to only the chunks that saw this failure, and the " + "tile's objects would be double-measured or dropped." + ) + if "LDAC_OBJECTS" not in hdul: + raise SystemExit( + f"[ngmix_range] FATAL: no LDAC_OBJECTS in {cats[0]}" + ) + n_obj = int(hdul["LDAC_OBJECTS"].header["NAXIS2"]) + expected = np.arange(1, n_obj + 1, dtype=np.int64) + counts = np.zeros(n_obj, dtype=np.int64) + for hdu in epoch_hdus: + data = hdu.data + # NUMBER is asserted, not assumed: the whole scheme is an ID + # INTERVAL, so a permuted or gappy NUMBER column would make the + # weights describe different objects than the bounds select. + if len(data) != n_obj or not np.array_equal( + np.asarray(data["NUMBER"], dtype=np.int64), expected + ): + raise SystemExit( + f"[ngmix_range] FATAL: {cats[0]}[{hdu.name}] is not " + f"{n_obj} rows of NUMBER = 1..{n_obj} in order" + ) + counts += np.asarray(data["CCD_N"]) >= 0 + return counts def main() -> None: + """Emit this chunk's range as the two bash exports the config reads.""" p = argparse.ArgumentParser(description=__doc__) p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--chunk", required=True, type=int) p.add_argument("--n-chunks", required=True, type=int) a = p.parse_args() - lo, hi = id_ranges(object_count(a.run_dir), a.n_chunks)[a.chunk - 1] + # Explicit, because a negative --chunk would index from the end and hand + # this process some other chunk's range without any error. + if not 1 <= a.chunk <= a.n_chunks: + raise SystemExit( + f"[ngmix_range] FATAL: --chunk {a.chunk} outside 1..{a.n_chunks}" + ) + lo, hi = id_ranges(object_epochs(a.run_dir), a.n_chunks)[a.chunk - 1] print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") From 4f389ea570facea5cb1bad1f77ee4dd9364c1bb0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:30:25 -0400 Subject: [PATCH 051/100] feat(orchestration): reclaim a finished tile's scratch store clean_exposure reclaims exposure stores; nothing reclaimed tiles, so a FINISHED tile held 1.19 GiB across 137 inodes on scratch forever (measured on 186.307: run_sp_tile_Sx 745 MB, run_sp_tile_Uz 382 MB, run_sp_Mc 46 MB, run_sp_Ms 39 MB, run_sp_tile_Mh_exp 11 MB, plus 62 non-output inodes of 15,726 bytes). At DR6's 23,114 tiles that is 26.9 TiB against a 1 TiB quota and 3.17M inodes against a 1M one; the byte bound binds first, so until this rule exists no batch may exceed ~859 tiles. Deleting only output/ does not fix it either -- 62 inodes per tile is still 1.43M at DR6, over quota on its own -- so the whole directory goes. clean_tile is the symmetric rule: a localrule keyed on the tile's own final_cat on the persistent root, ordered after the tile so reclamation is ROLLING (the scratch high-water tracks tiles in flight, not tiles in the batch). Eligibility is trivial where the exposure case is hard: a tile's store has no consumer outside that tile, so there is no consumer set, no staleness to detect, and no edge to clean_exposure at all. Scope runs the other way instead -- TILES_READY only, because an out-of-scope tile's tombstone would drag its final_cat and the whole fused tile_shape group into the DAG, where the params trigger can rerun a finished tile against reclaimed exposure stores. FOUR SURVIVORS, and two of them were found by working out what else reads a tile's store rather than by looking at the tile: * manifests/tile_vignets.json -- clean_exposure keys campaign-wide eligibility on exactly this path. Removing it stranded all 7 fixture exposures (dry-run job counts: 7 clean_exposure jobs with it, 0 without), because every exposure had an out-of-scope consumer whose vignets manifest is the test. * manifests/tile_find_exposures.json -- rule prepare_all_tiles declares it for EVERY tile in the list, and the list accumulates. Without it the prepare parse schedules the whole tile_prep group per cleaned tile per invocation (measured on the fixture: 4 jobs where there should be none), and tile_uncompress writes 382 MB back into the store just emptied. * output/run_sp_tile_Fe/.../exp_numbers--.txt -- build_index reads it at the parse of EVERY compute invocation. Without it the tile counts as missing and SP_MISSING_THRESHOLD=0.0 makes the parse fatal: "Missing exposure lists for 1/2 tile(s) (fraction 0.500 > threshold 0.0)". Not a degradation, a stop -- the campaign cannot resume at all. * cleaned.json -- the tombstone, absorbing all 16 manifests verbatim plus the 8 ngmix benchmark rows, so sp report still reports the tile and D4's measured-memory feed is not lost with the store. Ten inodes and 12.9 KB survive per tile (231k inodes, ~300 MB at DR6). The three pre-existing survivors are checked BEFORE any deletion and the job refuses loudly if one is absent: a drift in those paths is invisible at deletion time and shows up much later as a campaign that will not parse. KEEPING tile_vignets.json DOES NOT TRIP TILE_VIGNET_REQUIRED, and the reason is what gets deleted rather than what gets kept -- worth stating because a cleaned tile is on its face the exact state that guard exists to catch. Forcing a cleaned tile back (final_cat deleted) reschedules tile_vignets inside the tile_shape group, so the store is rebuilt: 19 jobs, whole chain from tile_get_images. Snakemake's own reason is "Input files updated by another job: exp_forest, tile_find_exposures.json, tile_detect.json" -- structural propagation, NOT the mtime trigger, which matters because it is not in profiles/nibi's rerun-triggers and cannot be switched off there. The counterfactual pins the mechanism: with every upstream manifest restored and mtimes controlled (the "delete only output/" design), the same force schedules 8 x tile_ngmix + merge_cats + make_cat with tile_vignets ABSENT and every chunk trips the guard. So tile_detect.json must never join the survivor list. smk-g4's job_head.sh runbook sweep for this hazard already skips tiles with a final_cat, i.e. every tile this rule touches. Absorption is ADDITIVE over an existing tombstone. A naive re-absorption cut a 16-manifest tombstone down to the two surviving manifests on the second clean, found by running the fixture clean twice; script_hash reruns this job on any edit to clean_tile.py, so that rerun is not hypothetical. (clean_exposure.py has the same latent hole and is deliberately left alone here.) Deletion is a whitelist walk, not an rmtree with exceptions, and is symlink-safe for a sharper reason than the exposure case: exp_forest holds ~8 links into the shared exposure stores. Verified on the fixture that all 7 exposure stores keep their content across a clean, and that a dangling link left by an already-reclaimed exposure is unlinked rather than tripped over. clean_tiles is a NEW config key, default false, deliberately not `clean:`. Exposure reclamation costs a rebuild if a tile is appended later; tile reclamation destroys the per-tile audit trail and nothing rebuilds it -- sp_tilecost.py reads each tile's 380 MB sexcat from the scratch store to attribute the fused group job's cost, and that is how every mem_mb and runtime number in tile.smk was derived. Required for any batch over ~850 tiles, and it turns off per-tile cost attribution. The eight benchmark TSVs it also reads survive inside the tombstone but not at their on-disk paths; teaching that tool to fall back to cleaned.json is a named follow-up, not done here. run_report reads tile tombstones back the way it reads exposure ones, with one asymmetry: clean_tile cannot empty manifests/, so absorb_tombstones takes the survivor set explicitly -- otherwise the "records on disk mean a rebuilt chain" guard fires on every cleaned tile and a fully reclaimed campaign reports as one that ran two stages and stopped. Verified: 2/2 final cats, all nine stages complete, the reclaimed tile listed as cleaned. Verified by construction on a throwaway fixture (structural copy of 186.307, bulk files stubbed, now deleted); no SLURM job was submitted and neither smk-g4 nor smk-g5 was touched. Dry-run counts: clean_tiles=false -> 0 clean_tile jobs; true with one finished tile -> exactly 1, and tile_make_cat is NOT dragged back in; true with the tile unfinished -> tile_make_cat then clean_tile, in that order. A real clean of the fixture tile took it from 137 inodes to 10. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- workflow/Snakefile | 58 ++++++- workflow/config.yaml | 33 ++++ workflow/rules/tile.smk | 119 +++++++++++++ workflow/scripts/clean_tile.py | 306 +++++++++++++++++++++++++++++++++ workflow/scripts/run_report.py | 44 ++++- 5 files changed, 550 insertions(+), 10 deletions(-) create mode 100644 workflow/scripts/clean_tile.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 0f2541334..5d714f8e3 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -249,6 +249,8 @@ def unit_num(unit): SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] +CLEAN_TILE_HASH = hashlib.md5( + (SCRIPTS / "clean_tile.py").read_bytes()).hexdigest()[:12] # Same argument for star_cats.py, which both star-cat rules call: their params # otherwise fingerprint nothing but paths, so an edit to the chunking or the cut # would never rerun them. ONE hash for both rules because it is one script — and @@ -349,6 +351,51 @@ def clean_targets(): out.append(tombstone(exp)) return sorted(out) +# --- tile reclamation (D5) -------------------------------------------------- +# A SEPARATE FLAG from `clean:`, deliberately (config.yaml carries the full +# argument): exposure reclamation costs nothing but a rebuild if a tile is +# appended later, while tile reclamation destroys the per-tile AUDIT TRAIL that +# the campaign's cost model was derived from. They are not the same decision and +# must not be made by the same switch. Gated on SP_PHASE=compute exactly as +# CLEAN is, and read through flag() for the same reason (`--config +# clean_tiles=false` arrives as the string "false"). +CLEAN_TILES = flag(config.get("clean_tiles", False)) and PHASE == "compute" + + +def tile_tombstone(tile): + """The clean_tile output. Beside manifests/ and logs/, not inside either — + same placement and same reason as the exposure tombstone() above.""" + return f"{tile_dir(tile)}/cleaned.json" + + +def clean_tile_targets(): + """Which tiles this invocation may clean: the in-scope ones, and no test. + + The exposure case needs clean_targets()' eligibility test because an + exposure's consumer set closes over the campaign. A TILE's store has no + consumer outside that tile — tiles read exposures, nothing reads another + tile's store — so there is nothing to wait for and nothing to defer on. Every + ready tile is eligible the moment its own final_cat exists, and the rule's + input is exactly that. + + The restriction to TILES_READY is therefore not eligibility, it is scope + containment, and it runs the opposite way from the exposure case: an + OUT-OF-SCOPE tile's tombstone would drag that tile's final_cat, its + tile_make_cat job and the whole fused tile_shape group into this DAG, where + the `params` rerun-trigger can find a finished tile out of date and rerun it + against reclaimed exposure stores (see the TILE_LOCAL warning in tile.smk). + In scope the edge is free: `rule all` already requests the same final_cat. + + Nothing is lost by skipping an out-of-scope tile, because the tile list only + ever grows — a later invocation has it in scope and cleans it then. That is + also what makes flipping `clean_tiles` on RETROACTIVE: the tiles already + reclaimed are exactly the ones with a tombstone, so the missing tombstones + schedule exactly the missing cleans. + """ + if not CLEAN_TILES: + return [] + return sorted(tile_tombstone(t) for t in TILES_READY) + # --- the shell every rule runs (D2) ---------------------------------------- _THREAD_CAPS = " ".join( @@ -458,21 +505,24 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# The aggregation targets, clean_exposure, star_catalogue and exp_star_cat run in -# the head process. clean_exposure is seconds of rmtree and hangs off `all`; -# exp_star_cat is seconds of local FITS work; both would otherwise be ~20k sbatch +# The aggregation targets, clean_exposure, clean_tile, star_catalogue and +# exp_star_cat run in the head process. The two clean rules are seconds of rmtree +# and hang off `all`; exp_star_cat is seconds of local FITS work; all three would +# otherwise be ~20k (clean_exposure, exp_star_cat) or ~23k (clean_tile) sbatch # submissions at DR6 scale for work shorter than the scheduling latency. # star_catalogue is one job either way, and local keeps its CDS concurrency the # explicit number its thread pool sets (see exposure.smk). # # star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out # of any future `group:` label: a local job cannot be fused into a submitted group. -localrules: all, prepare_all_tiles, clean_exposure, star_catalogue, exp_star_cat +# The two clean rules are DAG leaves and have no such constraint. +localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat rule all: input: [final_cat(t) for t in TILES_READY], clean_targets(), + clean_tile_targets(), # Invocation 1 — the static per-tile DAG, known from the tile list alone. # keep-going makes tile failures independent; the ones that lose their exposure diff --git a/workflow/config.yaml b/workflow/config.yaml index 40bf1d096..fa84bdc1a 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -93,6 +93,39 @@ index_db: /project/def-mjhudson/cdaley/sp-products/smk-g5/index/run_index.sqlite # means a re-measurement re-runs only the shape chain instead of the whole tile. clean: false +# Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_tile` job per in-scope tile, ordered after that tile's own final_cat, so +# a tile's scratch store goes as soon as its chain lands rather than at the end +# of the batch. It deletes the tile's whole /tiles/// tree +# except four things another mechanism owns — `cleaned.json`, +# `manifests/tile_vignets.json` (clean_exposure's eligibility currency), +# `manifests/tile_find_exposures.json` (prepare_all_tiles' target) and the Fe +# exposure list under output/ (build_index re-reads it at every compute parse). +# clean_tile.py argues each one. +# +# A SEPARATE SWITCH FROM `clean:` ABOVE, AND NOT AN OVERSIGHT. Exposure +# reclamation is reversible in the only sense that matters: a tile appended later +# rebuilds the exposure chain from VOS, expensively but completely. TILE +# reclamation DESTROYS THE PER-TILE AUDIT TRAIL and nothing rebuilds it. +# sp-products/tools/sp_tilecost.py attributes the fused tile_shape group job's +# cost per tile by reading that tile's SExtractor catalogue out of the scratch +# store (NAXIS2 of the sexcat is the object count, the cost model's independent +# variable) — and that model is how every mem_mb and runtime number in tile.smk +# was derived. The sexcat is ~380 MB per tile and cannot be kept; the eight +# tile_ngmix benchmark rows are absorbed into the tombstone, so the measurements +# survive but not at the paths the tool reads. +# +# The honest one-line summary: REQUIRED FOR ANY BATCH OVER ~850 TILES, AND IT +# TURNS OFF PER-TILE COST ATTRIBUTION. The arithmetic: a finished tile leaves +# 1.19 GiB across 137 inodes (measured on 186.307), so a 1 TiB scratch quota is +# full at 859 tiles, and DR6's 23,114 tiles would want 26.9 TiB and 3.17M inodes +# against a 1M quota. Reclaimed, a tile costs 10 inodes and 12.9 KB — 231k inodes +# and ~300 MB for the whole of DR6. +# +# OFF here for the same reason `clean:` is: smk-g5 is a one-tile control, and its +# store is what a re-measurement would read. +clean_tiles: false + # Tiles that may NOT pin an exposure store (default: empty). # # An exposure is eligible for cleaning only once EVERY consuming tile has its diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 5215a2095..db8fd3c4e 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -742,3 +742,122 @@ rule tile_make_cat: ' | head -1)" {output.final_cat}\n' "fi\n" "exit $rc\n" + + +# --- tile reclamation (D5) -------------------------------------------------- +# The tile-side counterpart of clean_exposure, and the differences are the whole +# design. Read that rule's commentary in exposure.smk first; this one only says +# where the tile case departs from it. +# +# WHY IT EXISTS. clean_exposure reclaims exposure stores; nothing reclaimed +# tiles, so a FINISHED tile held 1.19 GiB across 137 inodes on scratch forever +# (measured on 186.307; the per-directory breakdown is in clean_tile.py). At +# DR6's 23,114 tiles that is 26.9 TiB against a 1 TiB quota and 3.17M inodes +# against a 1M one. BOTH BOUNDS BIND, and the byte one binds first: without this +# rule no batch may exceed ~859 tiles. The inode arithmetic is why the whole +# directory goes rather than just `output/` — the 62 non-output inodes per tile +# are still 1.43M at DR6, i.e. over quota on their own. +# +# ELIGIBILITY IS TRIVIAL, AND THAT IS THE POINT. An exposure's consumer set +# closes over the CAMPAIGN (~7-10 tiles read each one), which is why +# clean_targets() has to test every consumer and why clean_exposure carries the +# set in params so a late append makes the tombstone stale. A tile's store has +# NO consumer outside that tile: tiles read exposures, and nothing reads another +# tile's store. So there is no consumer set, no eligibility test, no staleness +# to detect — only an ordering edge on the tile's own final_cat. +# +# THE INPUT IS final_cat, ON THE PERSISTENT ROOT, AND NOT ancient(). It is +# already the campaign's designated tile-finished marker (see final_cat()'s +# docstring and tile_finished() above), and an ordinary input is what orders the +# clean after the tile — which is what makes reclamation ROLLING: within one +# invocation a tile is reclaimed as soon as its own chain lands, so the scratch +# high-water tracks the tiles in flight instead of the tiles in the batch. That +# is the whole reason the rule is worth having; ancient() would keep the +# dependency but drop the ordering, and the batch would peak at its full size. +# +# SCOPE: TILES_READY only, never every tile in the index. clean_targets() has to +# reason about eligibility; this one has to reason about the converse, and for a +# different reason. Requesting an OUT-OF-SCOPE tile's tombstone would put that +# tile's final_cat — hence its tile_make_cat job, hence the whole fused +# tile_shape group — into this DAG, where the `params` rerun-trigger can find it +# out of date (see the TILE_LOCAL warning above) and reschedule a finished tile +# against exposure stores clean_exposure has already reclaimed. In scope, that +# costs nothing: `rule all` requests the same final_cat anyway, so the clean adds +# an edge and no jobs. Deferral, never loss: the tile list accumulates, so every +# later invocation has the tile in scope and picks it up — which is also how +# flipping `clean_tiles` on later reclaims retroactively (the tiles already +# cleaned are exactly the ones with a tombstone). +# +# FOUR SURVIVORS, not one, and clean_tile.py argues each. Two of them are the +# corrections this rule's first design needed: `manifests/tile_find_exposures.json` +# (rule prepare_all_tiles declares it for EVERY tile in the list, and the list +# accumulates — delete it and the prepare group reruns per cleaned tile per +# invocation, writing uncompress's 382 MB back into the store) and the Fe +# exposure list under `output/` (build_index.exp_list_path re-reads it at EVERY +# compute parse; delete it and the default SP_MISSING_THRESHOLD of 0.0 makes the +# first `sp run` after the first reclamation a fatal parse). Ten inodes per tile +# survive in total, ~231k at DR6. +# +# THE SURVIVING tile_vignets.json DOES NOT TRIP TILE_VIGNET_REQUIRED, AND THE +# REASON IS WHAT GETS DELETED, NOT WHAT GETS KEPT. Read that guard above first. +# A cleaned tile is, on its face, exactly the state it exists to catch: a valid +# tile_vignets manifest over a node-local store that is not there (here because +# it never outlived its job, rather than because a fused group half-ran). +# smk-g4's job_head.sh carries a runbook sweep for that state, and it skips any +# tile with a final_cat — i.e. every tile this rule ever touches — so a cleaned +# tile is invisible to it either way. +# +# Force a cleaned tile back anyway (delete its final_cat) and tile_vignets IS +# rescheduled, so the store is rebuilt and no chunk trips the guard. Measured on +# the fixture: 19 jobs, the whole chain from tile_get_images, tile_vignets inside +# the tile_shape group. Snakemake names the mechanism itself -- +# reason: Input files updated by another job: .../exp_forest, +# .../manifests/tile_find_exposures.json, .../manifests/tile_detect.json +# -- and it is NOT the `mtime` trigger. It is the structural propagation that +# makes every dependent of a rerunning job rerun, the same edge the cascade +# commentary above says ancient() cannot suppress. Worth the distinction: that +# propagation is not in profiles/nibi's rerun-triggers list and so cannot be +# switched off there, where an mtime argument could be. +# +# So the guard stays silent only because tile_detect.json and exp_forest/ GO. +# Counterfactual on the same fixture, upstream manifests restored and mtimes +# controlled so nothing reruns on timestamp -- i.e. the "delete only output/" +# design -- schedules 8 x tile_ngmix + tile_merge_cats + tile_make_cat with +# tile_vignets ABSENT, and every chunk trips the guard. +# ADDING A MANIFEST TO THE SURVIVOR LIST IS THEREFORE NOT FREE: tile_vignets is +# safe because nothing upstream of it survives, and tile_detect.json in +# particular must never join the list. +# +# A LOCALRULE (declared in the Snakefile), same as clean_exposure: it is an +# rmtree, not science, and one sbatch per tile is ~23k scheduler submissions at +# DR6 scale for work shorter than the scheduling latency. Unlike exp_star_cat it +# is a DAG LEAF, so being local can never make it both a dependency and a +# dependent of a group and there is no cycle to avoid — no `group:` label here, +# and none possible. +# +# WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to +# tile_local() mid-campaign reruns finished tiles unsatisfiably because their +# EXPOSURE stores are reclaimed. With this rule on, the tile's own store is gone +# too, so the rerun has even less to stand on. The recommendation is unchanged — +# land those edits between campaigns — but the margin for being wrong is smaller. +rule clean_tile: + input: + # The tile-finished marker, on the persistent root. A lambda rather than + # the PROD_TILE_DIR pattern so there is exactly one definition of this + # path (final_cat() in the Snakefile), the same way clean_exposure keys + # off wildcards.exp alone. + lambda wc: final_cat(wc.tile) + output: + tombstone = f"{TILE_DIR}/cleaned.json" + params: + # clean_tile.py is external to the shell string, so the `code` + # rerun-trigger cannot see it — same reason SCRIPT_HASH exists. + script_hash = CLEAN_TILE_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_tile.py" + " --tile-dir $(dirname {output.tombstone}) --tile {wildcards.tile}" + " --tombstone {output.tombstone}" diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py new file mode 100644 index 000000000..3321b2845 --- /dev/null +++ b/workflow/scripts/clean_tile.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Reclaim ONE finished tile's scratch store and leave a tombstone (PRD #848 D5). + +Run as the shell of the in-DAG ``clean_tile`` rule, never by hand: the rule's +``input:`` is the tile's ``final_cat`` on the PERSISTENT root, so by the time +this executes the tile has published the only thing the campaign wanted from it. +A tile's scratch store has no reader outside that tile — tiles read exposures, +nothing reads another tile's store — so unlike the exposure case there is no +consumer set to close over and no eligibility test to make. Writer, then +cleaner, and the DAG edge is the whole ordering argument. + +WHAT IT DELETES: the tile's whole ``/tiles///`` directory. +Measured on 186.307 (smk-g4, a finished 34-tile-campaign tile): 1,279,231,196 +bytes across 137 inodes (71 regular files, 9 symlinks, 57 directories) — +``output/run_sp_tile_Sx`` 745 MB, ``run_sp_tile_Uz`` 382 MB, ``run_sp_Mc`` +46 MB, ``run_sp_Ms`` 39 MB, ``run_sp_tile_Mh_exp`` 11 MB, and a non-``output/`` +remainder of 62 inodes totalling 15,726 bytes. Deleting only ``output/`` is NOT +enough: 62 x 23,114 DR6 tiles is 1.43M inodes against a 1M quota, so the inode +bound binds on its own and the directory has to go as a whole. + +THE FOUR SURVIVORS, AND WHY EACH ONE IS NOT RESIDUE +--------------------------------------------------- +Everything kept here is currency some OTHER mechanism owns and re-reads long +after this tile is done. Nothing is kept for tidiness, and the three that +already exist are a CONTRACT: ``require_survivors`` checks all of them BEFORE +anything is deleted, because a silent drift in one of these paths would not show +up as a broken clean — it would show up much later as a campaign that cannot +resume. The fourth, ``cleaned.json``, is this job's own output and is the one +thing here that cannot be pre-checked; it is written first instead (see ORDER +below). + + 1. ``cleaned.json`` — this script's own tombstone, and the tile's surviving + record. It absorbs every manifest verbatim so ``sp report`` can still + report a reclaimed tile as ``cleaned`` rather than as "not run"; see + ``run_report.absorb_tombstones``. + + 2. ``manifests/tile_vignets.json`` — CLEAN_EXPOSURE'S CURRENCY, not ours. + Exposure reclamation keys campaign-wide eligibility on exactly this path: + ``clean_targets()`` tests it for out-of-scope consumers and ``rule + clean_exposure`` declares it as an input for in-scope ones. Deleting it + would silently strand every exposure shared with an out-of-scope tile + (~7-10 tiles read each exposure), and would fail an in-scope + ``clean_exposure`` job outright on a missing input if the tile finished + mid-invocation. It is also not stale in any dangerous sense: it attests + that ``tile_vignets`` succeeded, which stays true forever, and its product + — the vignette store — was node-local and died with its job, so it never + lived on scratch to be contradicted. + + 3. ``manifests/tile_find_exposures.json`` — the PREPARE invocation's target. + ``rule prepare_all_tiles`` declares this manifest for EVERY tile in the + list, and the tile list accumulates across the campaign, so a cleaned tile + is still demanded by every later ``sp run``. Delete it and the whole + ``tile_prep`` group (get_images -> uncompress -> find_exposures) reruns per + cleaned tile per invocation, which does not merely cost jobs: uncompress + writes 382 MB back into the store this job just emptied. + + 4. ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-*.txt`` — + the campaign's tile->exposure edge data, and the reason this is a survivor + rather than a manifest is that NOTHING re-derives it. ``build_index.build()`` + runs at the parse of EVERY compute invocation, over the whole declared tile + list, and checks this exact path (``build_index.exp_list_path``); a tile + whose file is gone counts as missing, and the default + ``SP_MISSING_THRESHOLD`` of 0.0 makes ONE missing tile a fatal parse. So + deleting it does not degrade the campaign, it stops it: the first `sp run` + after the first reclaimed tile cannot build a DAG at all. + Only the one file is kept — ``run_sp_tile_Fe``'s own ``logs/``, ``tmp/`` + and the runner's process log are ordinary residue and go. + +The rest of ``run_sp_tile_Fe``'s parents come along because a file cannot +outlive its directories: 10 inodes per cleaned tile in total (the tile dir, +``cleaned.json``, ``manifests/`` + 2 manifests, and the 4-deep Fe path + its +file), i.e. ~231k inodes at DR6 against the 1M scratch quota — versus 3.2M if +nothing were reclaimed and 1.4M if only ``output/`` were. + +WHAT IS LOST, STATED PLAINLY. The per-tile audit trail. ``sp_tilecost.py`` +attributes the fused ``tile_shape`` group job's cost per tile by reading the +tile's SExtractor catalogue (NAXIS2 of the sexcat = the object count, the cost +model's independent variable, ~380 MB and unkeepable) and the eight +``tile_ngmix_.benchmark.tsv`` files. The sexcat is gone for good; the +benchmark ROWS are absorbed into the tombstone under ``benchmarks``, because +they are two lines each and they are the measured-memory feed D4 sizes +``mem_mb`` from — exposure.smk moves ``exp_psf``'s benchmark outside +``manifests/`` for exactly this reason. They are absorbed rather than left on +disk (8 more inodes per tile = 185k at DR6 buys nothing a JSON blob does not), +so ``sp_tilecost.py`` will not find them at their old paths: the record +survives, the tool's current reader does not. FOLLOW-UP, deliberately not done +here: teach ``sp_tilecost.py`` to fall back to ``cleaned.json`` for a tile whose +benchmark TSVs are gone. Until it does, per-chunk cost attribution stops at the +first reclaimed tile even though the numbers are still on disk. + +DELETION IS SYMLINK-SAFE, and here that is not a nicety. ``exp_forest/`` is a +sharded view of ~8 symlinks pointing at the EXPOSURE stores, which are shared +with 7-10 other tiles each. ``shutil.rmtree`` unlinks symlinked entries rather +than recursing through them, and every entry is tested with +``is_symlink()`` BEFORE ``is_dir()`` — a clean that followed the forest would +delete a large slice of the campaign's exposure stores from a job whose whole +job description is "one finished tile". + +LOGS ARE DELETED, NOT ABSORBED, for the same reason ``clean_exposure`` deletes +them: on a finished tile every ``logs/.json`` is BYTE-IDENTICAL to the +``manifests/.json`` beside it (verified across all 16 stage records of +186.307), because completeness.py writes the same verdict to both and a tile +with a failed stage has no final_cat and so is never cleaned. Absorbing them +would duplicate the manifests the tombstone already carries. + +ORDER: tombstone FIRST, complete and atomically renamed, and only then any +deletion — the reverse of the obvious order, and the same choice +``clean_exposure`` makes. A crash between the two leaves a tombstone beside a +store that still exists: the next invocation treats the tile as cleaned and only +disk is lost. Deleting first would put the crash window where the record that +replaces the store was never written. + +There is no ``consumers`` field and no consumer-set staleness to detect, because +a tile has no consumers. What reruns this job is the ordinary machinery: the +``script_hash`` param carries this file's content hash (the ``code`` trigger +hashes only the rule's shell string, not the scripts it calls), and the ``mtime`` +trigger fires if final_cat is ever rewritten. A rerun over an already-pruned tree +deletes nothing — and, because absorption is ADDITIVE over the existing +tombstone, does not blank the record either. See ``previous_record``: a naive +re-absorption reduced a 16-manifest tombstone to the two surviving manifests, +found by re-running the fixture clean twice. +""" + +import argparse +import csv +import json +import shutil +import time +from pathlib import Path + + +def survivor_paths(tile_dir: Path, tile: str) -> dict: + """``{what it is: path}`` for the three PRE-EXISTING survivors. + + Three, not four: ``cleaned.json`` is this job's own output and does not + exist yet when this is called. + + The Fe path is spelled out rather than globbed and MUST agree with + ``build_index.exp_list_path`` — that function is what re-reads it at every + compute parse. It is not imported, because this script runs inside the + container as a job shell and stays stdlib-only and import-free like + ``clean_exposure.py``; ``require_survivors`` below turns a drift between the + two into a loud failure on the first tile instead of a fatal parse later. + """ + idra, iddec = tile.split(".") + return { + "clean_exposure eligibility (clean_targets / rule clean_exposure input)": + tile_dir / "manifests" / "tile_vignets.json", + "prepare_all_tiles target (rule prepare_all_tiles input)": + tile_dir / "manifests" / "tile_find_exposures.json", + "index build input (build_index.exp_list_path)": + tile_dir / "output" / "run_sp_tile_Fe" / "find_exposures_runner" + / "output" / f"exp_numbers-{idra}-{iddec}.txt", + } + + +def require_survivors(survivors: dict, tile: str) -> None: + """Abort before deleting anything if the survivor contract is not met. + + Fatal, not a warning. Each of these is read by a mechanism OUTSIDE this + tile, and each failure mode is silent at deletion time and loud much later: + a missing vignets manifest strands shared exposures, a missing Fe manifest + reruns the prepare chain, a missing exposure list makes the next compute + parse exit on the missing-tile threshold. Failing here costs one red job in + a keep-going run (clean_tile is a leaf, so it poisons no cone) and leaves + the store intact for inspection. + """ + missing = {k: p for k, p in survivors.items() if not p.exists()} + if missing: + lines = [f"[clean_tile] {tile}: refusing to reclaim — " + f"{len(missing)} survivor(s) not on disk:"] + lines += [f" {p}\n owned by: {k}" for k, p in missing.items()] + lines.append(" Nothing was deleted. Either this tile's store is not in " + "the state a finished tile should be in, or one of these " + "paths has moved and clean_tile.py has not followed it.") + raise SystemExit("\n".join(lines)) + + +def previous_record(tombstone: Path) -> tuple: + """``(manifests, benchmarks)`` from an existing tombstone, or two empties. + + THE ABSORPTION IS ADDITIVE, and this is why. A second clean of an + already-cleaned tile finds only the two surviving manifests on disk, so a + fresh absorption would overwrite a complete record with a two-entry one and + the tile's history would be gone — silently, and for good. That is not a + hypothetical rerun: the ``script_hash`` param reruns this job on any edit to + this file, and the ``mtime`` trigger reruns it if final_cat is ever + rewritten. So the previous record is the BASE and what is on disk is laid + over it: a rebuilt stage's fresh manifest still wins, and a reclaimed one + keeps the only copy that exists. + """ + if not tombstone.exists(): + return {}, {} + try: + old = json.loads(tombstone.read_text()) + except (OSError, json.JSONDecodeError): + return {}, {} # unreadable: start over rather than refuse + return (dict(old.get("manifests") or {}), + dict(old.get("benchmarks") or {})) + + +def absorb_manifests(mdir: Path) -> dict: + """Every ``manifests/*.json`` verbatim, keyed by file stem. + + By GLOB, so it takes whatever is there; ``run_report.py`` re-keys on each + body's own ``stage`` field. Same shape as ``clean_exposure``'s absorption, + including its tolerance for a legacy ``.failed.json``. + """ + out = {} + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + out[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + out[f.stem] = {"unreadable": str(exc)} + return out + + +def absorb_benchmarks(mdir: Path) -> dict: + """The ngmix chunks' benchmark rows, keyed by file stem. + + Two lines each (header + values), so this is a few hundred bytes for the + campaign's only per-chunk record of runtime, RSS and mean load — the feed D4 + sizes ``tile_ngmix``'s ``mem_mb`` from. Numbers are kept as strings: this is + an archive of what the TSV said, not a re-measurement. + """ + out = {} + if mdir.is_dir(): + for f in sorted(mdir.glob("*.benchmark.tsv")): + try: + rows = list(csv.DictReader(f.read_text().splitlines(), + delimiter="\t")) + except OSError as exc: + out[f.name] = {"unreadable": str(exc)} + continue + if rows: + out[f.name] = dict(rows[0]) + return out + + +def prune(root: Path, keep: set, removed: list) -> None: + """Delete everything under ``root`` except ``keep`` and the dirs leading to it. + + A whitelist walk rather than an rmtree-with-exceptions, so adding a survivor + is one line and can never be half-implemented: a path is kept iff it is a + survivor, recursed into iff it is a real directory on the way to one, and + deleted otherwise. + + ``is_symlink()`` is tested BEFORE ``is_dir()`` and before the recursion, + both for the reason ``clean_exposure`` gives (a dangling link is invisible to + ``exists()``) and for a sharper one here: ``exp_forest/`` holds ~8 links into + the shared exposure stores. + """ + ancestors = {p for k in keep for p in k.parents} + for entry in sorted(root.iterdir()): + if entry in keep: + continue + if entry in ancestors and not entry.is_symlink(): + prune(entry, keep, removed) + continue + if entry.is_symlink(): + entry.unlink() + elif entry.is_dir(): + shutil.rmtree(entry) # unlinks nested symlinks, never follows + else: + entry.unlink() + removed.append(str(entry)) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-dir", required=True, type=Path) + p.add_argument("--tile", required=True) + p.add_argument("--tombstone", required=True, type=Path) + args = p.parse_args() + + survivors = survivor_paths(args.tile_dir, args.tile) + require_survivors(survivors, args.tile) + + mdir = args.tile_dir / "manifests" + manifests, benchmarks = previous_record(args.tombstone) + manifests.update(absorb_manifests(mdir)) + benchmarks.update(absorb_benchmarks(mdir)) + + # Tombstone first, complete — then delete. See the module docstring: the + # crash window has to sit where the data still exists, not where the record + # does not. + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ + "tile": args.tile, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "kept": sorted(str(p) for p in survivors.values()), + "manifests": manifests, + "benchmarks": benchmarks, + }, indent=2, sort_keys=True) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed: list = [] + prune(args.tile_dir, set(survivors.values()) | {args.tombstone}, removed) + print(f"[clean_tile] {args.tile}: removed {len(removed)} path(s); kept " + f"{len(survivors)} survivor(s) + the tombstone") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 22201f7e7..c6ff9ad56 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -30,6 +30,12 @@ unit's record and the unit is reported as **cleaned** — not "not run", and it blocks no tile. +A reclaimed TILE is the same story with one asymmetry: ``clean_tile`` deletes +``logs/`` but cannot empty ``manifests/``, because two of those manifests are +currency other mechanisms read (SURVIVING_TILE_STAGES). So the "records on disk +mean a rebuilt chain" test takes that survivor set explicitly — see +``absorb_tombstones``. + No disk scanning: counting products is the *check's* job, done once at the moment the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". @@ -53,6 +59,12 @@ "tile_ngmix", "tile_merge_cats", "tile_make_cat"] EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] +# The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names +# the mechanism that owns each). Their presence is therefore NOT evidence that a +# tile's chain was rebuilt, which is the one thing absorb_tombstones has to know +# to read a reclaimed tile's record out of its tombstone. +SURVIVING_TILE_STAGES = frozenset({"tile_vignets", "tile_find_exposures"}) + STATUSES = ("complete", "warn", "failed", "not_run") @@ -98,7 +110,8 @@ def load_manifests(run_dir: Path, sub: str) -> dict: return out -def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, + survivors: frozenset = frozenset()) -> set: """Fill in reclaimed units from their ``cleaned.json``; return their ids. A cleaned exposure has neither ``manifests/`` nor ``logs/`` — @@ -110,6 +123,16 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: Manifests on disk win if both exist — that is a re-built chain, and the tombstone is then a stale record of the previous generation. + + ``survivors`` is what makes that test work for TILES. ``clean_tile`` cannot + empty ``manifests/`` the way ``clean_exposure`` does: two of the manifests + there are currency other mechanisms read (SURVIVING_TILE_STAGES below), so a + reclaimed tile ALWAYS has records on disk. Without this argument the + "manifests win" guard fires on every cleaned tile, the tombstone is ignored, + and a fully reclaimed campaign reports as one that ran two stages and + stopped. A unit counts as REBUILT — and keeps the guard — iff it has a record + for some stage that is not a survivor; the default empty set reproduces the + exposure test exactly. """ cleaned = set() for path in sorted((run_dir / sub).glob("**/cleaned.json")): @@ -119,12 +142,15 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: except (OSError, json.JSONDecodeError) as exc: print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) continue - if manifests.get(unit): + if any(s not in survivors for s in manifests.get(unit, {})): continue for key, m in (tomb.get("manifests") or {}).items(): if not isinstance(m, dict): continue - manifests[unit][m.get("stage", key)] = m + # setdefault, not assignment: the surviving on-disk records are the + # live ones and stay authoritative, even though the tombstone's + # copies of them are byte-identical today. + manifests[unit].setdefault(m.get("stage", key), m) cleaned.add(unit) return cleaned @@ -260,8 +286,12 @@ def main() -> None: tile_m = load_manifests(args.run_dir, "tiles") exp_m = load_manifests(args.run_dir, "exp") - # Reclaimed exposures speak through their tombstones (D5, S5). + # Reclaimed units speak through their tombstones (D5, S5). Tiles pass the + # survivor set: clean_tile leaves two manifests on disk on purpose, and + # without that argument they would read as a rebuilt chain. cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) + cleaned_tiles = absorb_tombstones(args.run_dir, "tiles", tile_m, + SURVIVING_TILE_STAGES) tiles = tiles or sorted(tile_m) exps = exps or sorted(exp_m) @@ -272,9 +302,10 @@ def main() -> None: "status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), "missing_tiles": missing, - "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m, cleaned_tiles), "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), "cleaned_exposures": sorted(cleaned_exp), + "cleaned_tiles": sorted(cleaned_tiles), "tiles": unit_rows(tiles, TILE_STAGES, tile_m), "exposures": unit_rows(exps, EXP_STAGES, exp_m), } @@ -315,7 +346,8 @@ def _blocks(unit): print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") - + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "")) + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "") + + (f" ({len(cleaned_tiles)} tiles reclaimed)" if cleaned_tiles else "")) print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) print_stage_table("TILES", report["tile_stages"], len(tiles)) print_table("exposures not complete", report["exposures"], args.limit) From f196fa122197e35379120dc7093d5c4dab81388f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 04:12:19 -0400 Subject: [PATCH 052/100] docs(orchestration): range_hash is safe to add only at a campaign boundary Adding NGMIX_RANGE_HASH protects a resume from a corrupt partition by forcing the chunks to rerun -- and on a root that already holds finished tiles, that rerun is destructive rather than merely expensive. The hash and the rule now say so where each is read. Snakemake compares the SET of recorded param values. smk-g4's metadata for a finished chunk (record_format_version 6) holds two, the SCRIPT_HASH digest and the pre string, so a third entry is new and the job replans with "Params have changed since last execution". Measured on snakemake 9.23.1 under profiles/nibi's trigger set, on a fixture mirroring the group with everything up to date: the eight chunks, tile_merge_cats and tile_make_cat reschedule; tile_vignets and tile_detect do NOT, because their manifests exist, so missing_output never queues them and nothing propagates down. A group replanned without tile_vignets is exactly the state TILE_VIGNET_REQUIRED exists to catch. Every chunk trips the guard, the group fails, and GroupJob.postprocess(error=True) fans out over every member and removes its existing outputs -- tile_make_cat's final_cat on the persistent root among them. The TILE_LOCAL warning does not already cover this, and the distinction is the reason the note is worth its length: tile_local() sits in the params.pre of tile_vignets, tile_ngmix AND tile_make_cat, so an edit there drags tile_vignets back in and the store is rebuilt. range_hash is on tile_ngmix ALONE. It is the one-rule fingerprint change that is dangerous, not the three-rule kind that warning describes, and the warning reads as though it covered both. Nor does job_head.sh's runbook sweep help: it skips any tile with a final_cat, which is exactly the set this breaks. clean_tiles inverts it. With reclamation ON the tombstoned tile has lost tile_detect.json, and the structural "Input files updated by another job" propagation puts tile_vignets back in the group, so the store is rebuilt and nothing trips. The shipped default clean_tiles: false is the dangerous configuration -- the opposite of how reclamation reads everywhere else in tile.smk. Recommendation is tile_local()'s, unchanged: land the hash, and every later edit to ngmix_range.py, at a campaign boundary on a fresh root, with the same `--rerun-triggers mtime code software-env` escape hatch and the same caveat if a resume is unavoidable. Separately, ngmix_range.py's docstring now carries the CROSS-TIME hazard beside the cross-process one it already argued. The hash covers resume across an edit; nothing covers an edit DURING flight, and a fused group holds its eight chunks open over the live checkout for hours (smk-g4: 6,236-7,762 s elapsed per chunk), each reading the file when its own shell starts. Reproduced live: an invocation four seconds after a rewrite returned chunk 5 of 186.307 as 17649..22060 where the other seven had been split 17129..21229 -- 520 objects orphaned, 831 double-measured -- while twelve sequential and thirty-six concurrent runs against a stable checkout gave the identical correct partition. Not a defect in the script; the same between-campaigns rule, binding harder because tile_local() fails loudly and this fails green. Comments only. No code, no behaviour, no rerun trigger moves. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- workflow/Snakefile | 10 ++++++++ workflow/rules/tile.smk | 43 +++++++++++++++++++++++++++++++++ workflow/scripts/ngmix_range.py | 14 +++++++++++ 3 files changed, 67 insertions(+) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2cbf44acf..80b4bea03 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -268,6 +268,16 @@ STAR_CAT_HASH = hashlib.md5( # tile completes green with a corrupt catalogue and no error anywhere. Same # failure family as the script's own no-fallback rule, across time rather than # across processes. +# +# ADDING it is itself a params change, and on a RESUME that change is +# destructive rather than merely expensive: it replans the fused group WITHOUT +# tile_vignets, so every chunk trips TILE_VIGNET_REQUIRED and the failed +# group's postprocess removes each member's existing outputs, final_cat +# included. The TILE_LOCAL warning does not cover this — tile_local() sits in +# three rules' params and drags tile_vignets back in, this sits in one — and +# clean_tiles: false, the shipped default, is the configuration it bites. +# tile.smk's range_hash comment carries the mechanism. Land this hash, and any +# later edit to the split, at a campaign boundary on a fresh root. NGMIX_RANGE_HASH = hashlib.md5( (SCRIPTS / "ngmix_range.py").read_bytes()).hexdigest()[:12] diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d10d46ff1..36a5af438 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -580,6 +580,49 @@ rule tile_ngmix: # the already-done chunks keep the old ranges while the reruns take the # new ones: some objects measured twice, others by nobody, the tile # green, no error anywhere. + # + # ADDING THIS PARAM IS ITSELF A PARAMS CHANGE, AND ON A RESUME IT DOES + # THE VERY THING IT PREVENTS, BY ANOTHER ROUTE. Snakemake compares the + # SET of recorded param values, and smk-g4's metadata for a finished + # chunk (record_format_version 6) holds two of them -- the SCRIPT_HASH + # digest and the pre string -- so a third entry is new and the job + # replans with "Params have changed since last execution". On snakemake + # 9.23.1 under profiles/nibi's trigger set, on a fixture mirroring this + # group with everything up to date, that reschedules the eight chunks, + # tile_merge_cats and tile_make_cat -- and NOT tile_vignets or + # tile_detect, whose manifests exist, so missing_output never queues + # them and nothing propagates down to them either. + # + # A group replanned without tile_vignets is precisely the state + # TILE_VIGNET_REQUIRED exists to catch. Every chunk trips the guard, + # the group fails, and GroupJob.postprocess(error=True) fans out over + # every member and removes its EXISTING outputs -- tile_make_cat's + # final_cat on the persistent root among them. So the cost of getting + # this wrong is a deleted science product, not a wasted hour. + # + # WHY THE TILE_LOCAL WARNING ABOVE DOES NOT ALREADY COVER IT, which is + # the whole reason this needs its own note: tile_local() sits in the + # params.pre of tile_vignets, tile_ngmix AND tile_make_cat, so an edit + # there drags tile_vignets back in and the store is rebuilt. + # range_hash is on tile_ngmix ALONE. Same family, opposite shape -- it + # is the ONE-rule fingerprint change that is dangerous, not the + # three-rule kind that warning describes, and reading it as covering + # both is the mistake to avoid. job_head.sh's runbook sweep for the + # guard state does not help either: it skips any tile with a final_cat, + # which is exactly the set this breaks. + # + # INVERTED BY clean_tiles, so read the default carefully. With + # reclamation ON the tombstoned tile has lost tile_detect.json, and the + # structural "Input files updated by another job" propagation puts + # tile_vignets back in the group (clean_tile below measures this) -- + # the store is rebuilt and nothing trips. The SHIPPED DEFAULT + # clean_tiles: false is therefore the dangerous configuration, which is + # the opposite of how reclamation reads everywhere else in this file. + # + # So: land this hash, and every later edit to ngmix_range.py, at a + # campaign boundary on a fresh root. Same rule as tile_local()'s, same + # escape hatch and same caveat if a resume is unavoidable -- + # `--rerun-triggers mtime code software-env`. range_hash = NGMIX_RANGE_HASH # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index 9ff24b3f7..fc9fea7b9 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -44,6 +44,20 @@ would be the worst available behaviour precisely because it would apply only to the chunks that hit the failure, shredding the tile's coverage instead of failing it. + +The cross-TIME case is not covered here and cannot be. NGMIX_RANGE_HASH +(Snakefile) makes a RESUME across an edit rerun the chunks rather than mix two +partitions, but a fused group holds its eight chunks open over the LIVE +checkout for hours — smk-g4 measured 6,236-7,762 s of elapsed per chunk — and +each chunk reads this file only when its own shell starts. An edit landed +mid-flight is therefore read by some chunks and not others. Seen once, live: an +invocation four seconds after a rewrite returned chunk 5 of 186.307 as +17649..22060 where the other seven had been split 17129..21229, orphaning 520 +objects and double-measuring 831; twelve sequential and thirty-six concurrent +runs against a stable checkout gave the identical correct partition. Nothing in +this script can catch that. The rule is the one tile_local() already states — +land edits between campaigns — and it binds harder here, because tile_local() +fails loudly while this fails green. """ import argparse From 731f8cc85fa2004950e989212337f6736f57e327 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 04:16:43 -0400 Subject: [PATCH 053/100] fix(report): reclamation must not edit a tile's verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the merged branch. It cleared the survivor list — swept workflow/ and sp-products/tools/ for every path built under tiles/// and found nothing missing — and returned four things. One is a real bug. A RECLAIMED TILE'S `warn` SILENTLY BECAME `complete`. All eight chunk manifests carry stage "tile_ngmix". load_manifests resolves that collision worst-status- wins; absorb_tombstones resolved it with setdefault over sorted stems, so only tile_ngmix_1 survived and a warn on any other chunk vanished the moment the tile was cleaned. Reproduced on the real 186.307 manifests with chunk 5 flipped: tile_ngmix ok 0 warn 1 before, ok 1 warn 0 after, and the tile left the "tiles not complete" table. The data was in the tombstone all along; only the reader dropped it. The rule is now one function, keep_worst(), used by both readers, and the tombstone's per-stem records are collapsed to one per stage BEFORE merging rather than after. End-to-end on the fixture: every stage's counts and per-runner products identical before and after reclamation. The other half of that finding does not reproduce as a reclamation bug. A stage's products aggregate comes from its one surviving record either way, so tile_ngmix attrition is counted over one chunk of eight — on disk as well as in the tombstone, before this change and after. That is the price of collapsing the chunks to one stage row; the tally_level docstring claimed the opposite and now states it, including the 8x over-statement of completeness it implies. THE SYMLINK COMMENT NAMED THE WRONG BLAST RADIUS. A finished tile holds nine links, not seven. The two unnamed ones are in run_sp_tile_Git/get_images_runner/ output/ and point into /project/def-mjhudson/unions-wl/tiles — 621 GB across 2,536 files of staged survey imaging on the backed-up, group-shared filesystem, put there by get_images RETRIEVE=symlink, so every tile in every campaign carries a pair. Behaviour was already correct, but the comment told the next person editing prune() that the worst case was a rebuildable scratch store. It also reasoned about prune's own is_symlink() test, which is not what protects either class: both are handed wholesale to shutil.rmtree, exp_forest/ as a top-level entry and run_sp_tile_Git/ inside output/, which prune descends only because the Fe survivor lives there. Said explicitly now. A PARTIALLY REBUILT CLEANED TILE LOSES ITS TOMBSTONE RECORD. Written down, not fixed, and the reason is that the obvious fix is wrong rather than long: filling per-stage from the tombstone would report the previous generation's "complete" for a stage whose manifest is absent because it is mid-rerun or failed. A stale complete is invisibly wrong where "not run" is visibly incomplete, and the same hazard reaches exposures campaign-wide. The all-or-nothing guard is a generation boundary and is worth more than the cosmetics. sp_costmodel.py joins sp_tilecost.py in the clean_tiles: doc — it reads the same sexcat (plus its EPOCH_k extensions) and the same benchmark TSVs. Their third artifact, run_sp_tile_ngmix_Ngu/, is NOT lost to this flag: tile_ngmix declares its chunk dir temp(), so snakemake reclaims it when tile_merge_cats runs and it is already absent from every finished tile. Two test modules, following tests/unit/test_ngmix_range.py's convention (import the script by path, container-free). test_run_report_tombstones pins the invariant that catches the whole class — the tombstone path and the on-disk path agree, stage for stage — parametrised over all eight chunks because the bug was invisible on chunk 1, the one stem first-key-wins happened to keep and the one anybody would test by hand. Against the pre-fix reader: 8 failed, 3 passed. test_clean_tile_prune pins symlink safety at the depths rmtree actually walks, because that failure mode is silent, immediate and unrecoverable. 16 new tests; tests/unit is 258 passed, 3 skipped. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- tests/unit/test_clean_tile_prune.py | 167 ++++++++++++++++++++++ tests/unit/test_run_report_tombstones.py | 171 +++++++++++++++++++++++ workflow/config.yaml | 21 ++- workflow/scripts/clean_tile.py | 38 +++-- workflow/scripts/run_report.py | 80 +++++++++-- 5 files changed, 448 insertions(+), 29 deletions(-) create mode 100644 tests/unit/test_clean_tile_prune.py create mode 100644 tests/unit/test_run_report_tombstones.py diff --git a/tests/unit/test_clean_tile_prune.py b/tests/unit/test_clean_tile_prune.py new file mode 100644 index 000000000..9830ff2d9 --- /dev/null +++ b/tests/unit/test_clean_tile_prune.py @@ -0,0 +1,167 @@ +"""``clean_tile`` deletes a tile's store without ever following a symlink out of it. + +``workflow/scripts/clean_tile.py`` reclaims a finished tile by deleting its whole +scratch directory bar four survivors. A finished tile holds NINE symlinks in two +classes, and the second is why this module exists: + + * ``exp_forest///output`` — 7 links into the exposure stores, each + shared with 7-10 other tiles; + * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` — + 2 links into ``/project/def-mjhudson/unions-wl/tiles``, 621 GB of staged + survey imaging across 2,536 files on the BACKED-UP, GROUP-SHARED filesystem. + +Both are handed wholesale to ``shutil.rmtree`` rather than caught by ``prune``'s +own ``is_symlink()`` test, so the safety argument rests on rmtree's semantics at +a depth ``prune`` never inspects. That is a fine thing to rely on and a bad thing +to leave unpinned: the failure mode is silent, immediate and unrecoverable, and +it would be introduced by an innocent-looking edit to ``prune``. + +Deliberately container-free — clean_tile.py is stdlib-only. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "clean_tile.py" + +TILE = "186.307" +SURVIVORS = ( + "cleaned.json", + "manifests/tile_vignets.json", + "manifests/tile_find_exposures.json", + "output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-186-307.txt", +) + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_clean_tile", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +clean_tile = _load() + + +@pytest.fixture +def store(tmp_path): + """A finished tile's store, plus the two off-store trees it links into. + + Mirrors the real layout of 186.307 (smk-g4) at the depths that matter: the + forest links sit three levels down, the Git links four, and ``prune`` + descends into ``output/`` only because the Fe survivor lives there. + """ + precious = {} + for name, rel in (("exposures", "exp/21/2114045/output"), + ("project_tiles", "unions-wl/tiles")): + d = tmp_path / name / rel + d.mkdir(parents=True) + (d / "DO_NOT_DELETE").write_text("survey data\n") + precious[name] = d + + tile_dir = tmp_path / "run" / "tiles" / TILE[:2] / TILE + for rel in SURVIVORS[1:]: # the tombstone is written by the job + p = tile_dir / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("{}" if rel.endswith(".json") else "2114045p\n") + + # bulk that must go + for rel in ("output/run_sp_tile_Sx/sextractor_runner/output/sexcat.fits", + "output/run_sp_tile_Uz/uncompress_fits_runner/output/image.fits", + "logs/tile_detect.json", "manifests/tile_detect.json", + "tile_numbers.txt"): + p = tile_dir / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("bulk") + + # class 1: the exposure forest (rmtree'd as a top-level entry) + forest = tile_dir / "exp_forest" / "21" / "2114045" + forest.mkdir(parents=True) + (forest / "output").symlink_to(precious["exposures"]) + + # class 2: the get_images links into /project (rmtree'd inside output/) + git = tile_dir / "output" / "run_sp_tile_Git" / "get_images_runner" / "output" + git.mkdir(parents=True) + (git / "CFIS_image-186-307.fits").symlink_to( + precious["project_tiles"] / "CFIS.186.307.r.fits") + (git / "CFIS_weight-186-307.fitsfz").symlink_to( + precious["project_tiles"] / "CFIS.186.307.r.weight.fits.fz") + + return tile_dir, precious + + +def test_prune_never_follows_a_symlink_out_of_the_store(store): + """The whole point: reclaiming one tile touches nothing outside that tile.""" + tile_dir, precious = store + before = {k: sorted(p.rglob("*")) for k, p in precious.items()} + + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + + for name, d in precious.items(): + assert d.is_dir(), f"{name} tree was removed" + assert sorted(d.rglob("*")) == before[name], f"{name} tree was modified" + assert (d / "DO_NOT_DELETE").read_text() == "survey data\n" + + +def test_prune_keeps_exactly_the_survivors(store): + """Ten inodes: the four survivors and the directories that carry them.""" + tile_dir, _ = store + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + left = sorted(str(p.relative_to(tile_dir)) for p in tile_dir.rglob("*")) + assert left == sorted([ + "manifests", "manifests/tile_vignets.json", + "manifests/tile_find_exposures.json", + "output", "output/run_sp_tile_Fe", + "output/run_sp_tile_Fe/find_exposures_runner", + "output/run_sp_tile_Fe/find_exposures_runner/output", + "output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-186-307.txt", + ]) + + +def test_prune_unlinks_a_dangling_link(store): + """A forest link whose exposure clean_exposure already reclaimed. + + ``exists()`` follows the link and is False for a dangling one, so an + exists()-first test would skip it and leave the link behind. + """ + tile_dir, precious = store + import shutil + shutil.rmtree(precious["exposures"]) + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + assert not (tile_dir / "exp_forest").exists() + + +def test_survivor_contract_refuses_before_deleting_anything(store): + """A missing survivor aborts with the store intact — it is a contract.""" + tile_dir, _ = store + (tile_dir / "manifests" / "tile_vignets.json").unlink() + survivors = clean_tile.survivor_paths(tile_dir, TILE) + with pytest.raises(SystemExit) as exc: + clean_tile.require_survivors(survivors, TILE) + assert "tile_vignets.json" in str(exc.value) + assert (tile_dir / "output" / "run_sp_tile_Sx").is_dir() + + +def test_absorption_is_additive_over_an_existing_tombstone(tmp_path): + """A second clean must not blank the record the first one saved. + + ``script_hash`` reruns this job on any edit to clean_tile.py, so a re-clean + over an already-pruned store is routine — and it finds only the surviving + manifests on disk. + """ + tomb = tmp_path / "cleaned.json" + tomb.write_text(json.dumps({ + "tile": TILE, + "manifests": {f"tile_ngmix_{k}": {"stage": "tile_ngmix"} for k in range(1, 9)}, + "benchmarks": {"tile_ngmix_1.benchmark.tsv": {"s": "6852.09"}}, + })) + manifests, benchmarks = clean_tile.previous_record(tomb) + assert len(manifests) == 8 + assert benchmarks["tile_ngmix_1.benchmark.tsv"]["s"] == "6852.09" diff --git a/tests/unit/test_run_report_tombstones.py b/tests/unit/test_run_report_tombstones.py new file mode 100644 index 000000000..6403d0233 --- /dev/null +++ b/tests/unit/test_run_report_tombstones.py @@ -0,0 +1,171 @@ +"""Reclamation must not change what ``sp report`` says about a unit. + +``workflow/scripts/run_report.py`` reads a unit's verdict from two places with +the same shape and different lifetimes: the records on disk +(``manifests/*.json`` + ``logs/*.json``), and — once ``clean_tile`` or +``clean_exposure`` has deleted those — the copies absorbed into the unit's +``cleaned.json``. Both readers face the same collision, because several files +map to one ``(unit, stage)``: a stage's manifest and its byte-identical log, and +the eight ngmix chunks, which all carry ``stage: "tile_ngmix"``. + +If the two readers resolve that collision differently, RECLAMATION SILENTLY +EDITS THE REPORT. It did: absorption used first-key-wins, so only +``tile_ngmix_1`` survived and a ``warn`` on any other chunk became ``complete`` +the moment the tile was cleaned — and the tile dropped out of the "tiles not +complete" table. This module pins the invariant that catches that whole class: +**the tombstone path and the on-disk path agree, stage for stage.** + +Deliberately container-free — run_report.py is stdlib-only, so nothing here +imports astropy, numpy, or shapepipe. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "run_report.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; bin/sp calls it by path" + spec = importlib.util.spec_from_file_location("_run_report", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +run_report = _load() + +TILE = "186.307" +NGMIX_CHUNKS = 8 + + +def _manifest(stage, status="complete", found=1, expect=1): + """One completeness verdict, in the shape completeness.py writes.""" + return { + "stage": stage, "level": "tile", "unit": TILE, "status": status, + "runners": {f"{stage}_runner": { + "found": found, "expect": expect, "floor": found, + "status": status, "warn": status == "warn"}}, + "failures": [], + } + + +def _records(ngmix_status=None): + """A finished tile's manifests, keyed by file stem as the store keys them. + + ``ngmix_status`` is ``{chunk_number: status}``; unlisted chunks are complete. + """ + out = {s: _manifest(s) for s in run_report.TILE_STAGES if s != "tile_ngmix"} + for k in range(1, NGMIX_CHUNKS + 1): + out[f"tile_ngmix_{k}"] = _manifest( + "tile_ngmix", (ngmix_status or {}).get(k, "complete")) + return out + + +def _cleaned_store(root, records, survivors=("tile_vignets", "tile_find_exposures")): + """A reclaimed tile store on disk: the survivors, and the tombstone. + + Exactly what ``clean_tile`` leaves behind — ``logs/`` gone, ``manifests/`` + holding only the manifests other mechanisms own, every record absorbed into + ``cleaned.json``. + """ + tile_dir = root / "tiles" / TILE[:2] / TILE + (tile_dir / "manifests").mkdir(parents=True) + for stem in survivors: + (tile_dir / "manifests" / f"{stem}.json").write_text( + json.dumps(records[stem])) + (tile_dir / "cleaned.json").write_text(json.dumps({ + "tile": TILE, "manifests": records, "benchmarks": {}})) + return tile_dir + + +def _live_store(root, records): + """The same tile BEFORE reclamation: every manifest, and its identical log.""" + tile_dir = root / "tiles" / TILE[:2] / TILE + for sub in ("manifests", "logs"): + (tile_dir / sub).mkdir(parents=True) + for stem, m in records.items(): + (tile_dir / sub / f"{stem}.json").write_text(json.dumps(m)) + return tile_dir + + +def _stage_status(run_dir, cleaned_survivors=None): + """``{stage: status}`` as the report would tally it, from a store on disk.""" + manifests = run_report.load_manifests(run_dir, "tiles") + if cleaned_survivors is not None: + run_report.absorb_tombstones(run_dir, "tiles", manifests, + cleaned_survivors) + return {s: m.get("status") for s, m in manifests[TILE].items()} + + +@pytest.mark.parametrize("chunk", range(1, NGMIX_CHUNKS + 1)) +def test_a_warning_chunk_survives_reclamation(tmp_path, chunk): + """A warn on ANY chunk reads the same before and after the tile is cleaned. + + Parametrised over all eight because the bug was invisible on chunk 1: that + is the stem first-key-wins happened to keep, so the one chunk anybody would + test by hand was the one chunk that worked. + """ + records = _records({chunk: "warn"}) + + live = tmp_path / "live" + _live_store(live, records) + before = _stage_status(live) + + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, records) + after = _stage_status(cleaned, run_report.SURVIVING_TILE_STAGES) + + assert before["tile_ngmix"] == "warn" + assert after == before + + +def test_a_failed_chunk_survives_reclamation(tmp_path): + """Worst-status-wins, not merely warn: failed beats warn beats complete.""" + records = _records({3: "warn", 6: "failed"}) + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, records) + after = _stage_status(cleaned, run_report.SURVIVING_TILE_STAGES) + assert after["tile_ngmix"] == "failed" + + +def test_reclaimed_tile_reports_every_stage(tmp_path): + """The survivors must not read as a rebuilt chain. + + ``clean_tile`` cannot empty ``manifests/`` — two manifests are currency + other mechanisms own — so without the survivor set the "records on disk mean + a rebuilt chain" guard fires and a reclaimed tile reports as one that ran + two stages and stopped. + """ + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, _records()) + manifests = run_report.load_manifests(cleaned, "tiles") + got = run_report.absorb_tombstones(cleaned, "tiles", manifests, + run_report.SURVIVING_TILE_STAGES) + assert got == {TILE} + assert sorted(manifests[TILE]) == sorted(run_report.TILE_STAGES) + + +def test_rebuilt_chain_keeps_the_guard(tmp_path): + """A unit with a NON-survivor record on disk is a rebuilt chain: skip it. + + This is the default (empty survivor set) that the exposure side uses, and it + must keep behaving exactly as it did before tiles needed an argument here. + """ + cleaned = tmp_path / "cleaned" + tile_dir = _cleaned_store(cleaned, _records()) + (tile_dir / "manifests" / "tile_detect.json").write_text( + json.dumps(_manifest("tile_detect", "failed"))) + + manifests = run_report.load_manifests(cleaned, "tiles") + got = run_report.absorb_tombstones(cleaned, "tiles", manifests, + run_report.SURVIVING_TILE_STAGES) + assert got == set() + assert manifests[TILE]["tile_detect"]["status"] == "failed" + assert "tile_ngmix" not in manifests[TILE] diff --git a/workflow/config.yaml b/workflow/config.yaml index fa84bdc1a..e4d6aa48d 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -107,13 +107,20 @@ clean: false # reclamation is reversible in the only sense that matters: a tile appended later # rebuilds the exposure chain from VOS, expensively but completely. TILE # reclamation DESTROYS THE PER-TILE AUDIT TRAIL and nothing rebuilds it. -# sp-products/tools/sp_tilecost.py attributes the fused tile_shape group job's -# cost per tile by reading that tile's SExtractor catalogue out of the scratch -# store (NAXIS2 of the sexcat is the object count, the cost model's independent -# variable) — and that model is how every mem_mb and runtime number in tile.smk -# was derived. The sexcat is ~380 MB per tile and cannot be kept; the eight -# tile_ngmix benchmark rows are absorbed into the tombstone, so the measurements -# survive but not at the paths the tool reads. +# TWO tools read it: sp-products/tools/sp_tilecost.py and sp_costmodel.py. Both +# attribute the fused tile_shape group job's cost per tile from that tile's +# scratch store — the SExtractor catalogue (NAXIS2 of the sexcat is the object +# count, the cost model's independent variable; sp_costmodel also reads its +# EPOCH_k extensions for the geometric epoch count) and the eight tile_ngmix +# benchmark TSVs — and that model is how every mem_mb and runtime number in +# tile.smk was derived. The sexcat is ~380 MB per tile and cannot be kept; the +# benchmark ROWS are absorbed into the tombstone, so the measurements survive +# but not at the paths the tools read. +# +# (The third artifact they read, run_sp_tile_ngmix_Ngu/, is NOT lost to this +# flag: tile_ngmix declares its chunk dir temp(), so snakemake reclaims it as +# soon as tile_merge_cats runs. It is already absent from every finished tile, +# clean_tiles or not.) # # The honest one-line summary: REQUIRED FOR ANY BATCH OVER ~850 TILES, AND IT # TURNS OFF PER-TILE COST ATTRIBUTION. The arithmetic: a finished tile leaves diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py index 3321b2845..1beeb904d 100644 --- a/workflow/scripts/clean_tile.py +++ b/workflow/scripts/clean_tile.py @@ -72,10 +72,12 @@ file), i.e. ~231k inodes at DR6 against the 1M scratch quota — versus 3.2M if nothing were reclaimed and 1.4M if only ``output/`` were. -WHAT IS LOST, STATED PLAINLY. The per-tile audit trail. ``sp_tilecost.py`` -attributes the fused ``tile_shape`` group job's cost per tile by reading the -tile's SExtractor catalogue (NAXIS2 of the sexcat = the object count, the cost -model's independent variable, ~380 MB and unkeepable) and the eight +WHAT IS LOST, STATED PLAINLY. The per-tile audit trail, for BOTH tools that +read it — ``sp_tilecost.py`` and ``sp_costmodel.py``. They attribute the fused +``tile_shape`` group job's cost per tile by reading the tile's SExtractor +catalogue (NAXIS2 of the sexcat = the object count, the cost model's independent +variable, ~380 MB and unkeepable; sp_costmodel also reads its ``EPOCH_k`` +extensions for the geometric epoch count) and the eight ``tile_ngmix_.benchmark.tsv`` files. The sexcat is gone for good; the benchmark ROWS are absorbed into the tombstone under ``benchmarks``, because they are two lines each and they are the measured-memory feed D4 sizes @@ -88,13 +90,27 @@ benchmark TSVs are gone. Until it does, per-chunk cost attribution stops at the first reclaimed tile even though the numbers are still on disk. -DELETION IS SYMLINK-SAFE, and here that is not a nicety. ``exp_forest/`` is a -sharded view of ~8 symlinks pointing at the EXPOSURE stores, which are shared -with 7-10 other tiles each. ``shutil.rmtree`` unlinks symlinked entries rather -than recursing through them, and every entry is tested with -``is_symlink()`` BEFORE ``is_dir()`` — a clean that followed the forest would -delete a large slice of the campaign's exposure stores from a job whose whole -job description is "one finished tile". +DELETION IS SYMLINK-SAFE, and here that is not a nicety. A finished tile holds +NINE symlinks in TWO classes, and the second is the one that matters: + + * ``exp_forest///output`` — 7 links into the EXPOSURE stores, + each shared with 7-10 other tiles. Rebuildable, but only by re-running those + chains from VOS. + * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` + — 2 links into ``/project/def-mjhudson/unions-wl/tiles``, the staged survey + imaging: 621 GB across 2,536 files, on the BACKED-UP, GROUP-SHARED + ``/project``, and not this campaign's to lose. get_images RETRIEVE=symlink + is what puts them there, so every tile in every campaign carries a pair. + +Both classes are handed WHOLESALE to ``shutil.rmtree`` — ``exp_forest/`` as a +top-level entry, ``run_sp_tile_Git/`` as one inside ``output/``, which ``prune`` +descends only because the Fe survivor lives there. So the safety rests on +rmtree's own semantics (it unlinks a symlinked entry rather than recursing +through it), NOT on ``prune``'s ``is_symlink()`` test, which fires only for a +link that is itself a direct entry of a level prune walks. Both were verified on +the fixture: every link unlinked, no target followed. Whoever edits ``prune`` +next should know that the worst case is not a scratch store they could rebuild — +it is a rmtree walking into half a terabyte of shared, backed-up survey data. LOGS ARE DELETED, NOT ABSORBED, for the same reason ``clean_exposure`` deletes them: on a finished tile every ``logs/.json`` is BYTE-IDENTICAL to the diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index c6ff9ad56..69bd4fb42 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -68,6 +68,40 @@ STATUSES = ("complete", "warn", "failed", "not_run") +def _rank(m: dict) -> int: + """How BAD a record is, as a position in STATUSES; unknown sorts worst.""" + return STATUSES.index(m["status"]) if m.get("status") in STATUSES else len(STATUSES) + + +def keep_worst(records: dict, stage: str, m: dict) -> None: + """Collapse the several records of one stage into the WORST of them. + + ONE rule, used by both readers in this module, and that is the point. + Several files map to one (unit, stage) either way: on disk, a stage's + manifest and its byte-identical log, plus the eight ngmix chunks, which all + carry ``stage: "tile_ngmix"`` under per-chunk filenames; inside a tombstone, + those same eight chunks again, keyed by file stem. + + absorb_tombstones used to resolve that collision by first-key-wins + (``setdefault`` over sorted stems), so only ``tile_ngmix_1`` survived and a + ``warn`` on any other chunk disappeared the moment the tile was reclaimed — + ``tile_ngmix ok 0 warn 1`` before the clean, ``ok 1 warn 0`` after, and the + tile silently left the "tiles not complete" table. The data was in the + tombstone the whole time; only the reader dropped it (found in review, + reproduced on 186.307 with chunk 5 flipped to warn). + + What this does NOT fix, because it is not a reclamation bug: a stage's + per-runner ``products`` aggregate is taken from the single surviving record, + so tile_ngmix attrition is counted over one chunk of eight. That is true + before and after a clean, identically — it is the price of collapsing the + chunks to one stage row, and it is the same on both paths by construction + now. + """ + prev = records.get(stage) + if prev is None or _rank(m) > _rank(prev): + records[stage] = m + + def load_manifests(run_dir: Path, sub: str) -> dict: """``{unit: {stage: verdict}}`` for one store (``tiles`` or ``exp``). @@ -100,13 +134,7 @@ def load_manifests(run_dir: Path, sub: str) -> dict: if not isinstance(m, dict) or "stage" not in m: continue unit = path.parent.parent.name - stage = m["stage"] - prev = out[unit].get(stage) - # Worst status wins when several records share a stage (ngmix chunks; - # a stage's manifest and its identical log). - rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 - if prev is None or rank(m) > rank(prev): - out[unit][stage] = m + keep_worst(out[unit], m["stage"], m) return out @@ -133,6 +161,22 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, stopped. A unit counts as REBUILT — and keeps the guard — iff it has a record for some stage that is not a survivor; the default empty set reproduces the exposure test exactly. + + KNOWN AND DELIBERATE: the guard is all-or-nothing, so a PARTIALLY rebuilt + cleaned tile loses its whole tombstone record. Force a rerun that restores + tile_detect..tile_make_cat but not the prepare stages (which only the + prepare invocation produces) and tile_get_images / tile_uncompress read + "not run" though they did run and nothing invalidated them. + + Not fixed, because the obvious fix is wrong rather than long. Filling + per-stage from the tombstone would, on a unit whose chain is rebuilding, + report the PREVIOUS generation's "complete" for a stage whose manifest is + absent precisely because it is mid-rerun or failed — a stale complete is + invisibly wrong where "not run" is visibly incomplete, and the same hazard + reaches exposures, where the mixing would be silent and campaign-wide. The + all-or-nothing guard is a generation boundary and is worth more than the + cosmetics. A correct fix needs a per-stage notion of which generation a + record belongs to, which nothing here records today. """ cleaned = set() for path in sorted((run_dir / sub).glob("**/cleaned.json")): @@ -144,13 +188,19 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, continue if any(s not in survivors for s in manifests.get(unit, {})): continue + # Collapse the tombstone's per-stem records to one per stage FIRST, + # by the same rule the on-disk path uses (keep_worst), and only then + # merge. Reversing those two steps is what lost a warning chunk. + absorbed: dict = {} for key, m in (tomb.get("manifests") or {}).items(): if not isinstance(m, dict): continue + keep_worst(absorbed, m.get("stage", key), m) + for stage, m in absorbed.items(): # setdefault, not assignment: the surviving on-disk records are the # live ones and stay authoritative, even though the tombstone's # copies of them are byte-identical today. - manifests[unit].setdefault(m.get("stage", key), m) + manifests[unit].setdefault(stage, m) cleaned.add(unit) return cleaned @@ -174,9 +224,17 @@ def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: """Per-stage counts + named unit lists, for one level. ``cleaned`` units are counted by the status their absorbed manifests carry - (complete or warn — attrition is preserved) and additionally listed under - ``cleaned``, so a reclaimed campaign reads as reclaimed rather than as a - campaign that never ran. + and additionally listed under ``cleaned``, so a reclaimed campaign reads as + reclaimed rather than as a campaign that never ran. The STATUS is preserved + exactly, including a warn on any one of the eight ngmix chunks (keep_worst + is what makes that true on the tombstone path as well as on disk). + + The per-runner ``products`` aggregate is NOT a per-chunk total, before or + after reclamation: a stage contributes the runners of its one surviving + record, so tile_ngmix attrition is counted over one chunk of eight. Reading + it as a whole-tile figure over-states completeness by 8x. That is a property + of collapsing the chunks to one stage row, not of cleaning, and the two + paths now agree on it. """ per_stage = {} for stage in stages: From cfe62ed1d4847cdbb38e4d522191e9ed607483c4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 04:17:07 -0400 Subject: [PATCH 054/100] docs(orchestration): say which half of the range_hash hazard was observed The comment asserted that a failed group job removes final_cat. The replanning half of that chain was reproduced; the deletion half never has been, here or in the tile_local warning it builds on. Two independent reviews failed to reach it for the same reason -- snakemake reports "Group jobs: inactive (local execution)", so no login-node fixture exercises the group path and only a submitted job against a finished tile can settle it. The mechanism stays as written, because reading it off snakemake's own cleanup path is what makes the hazard credible. What changes is that a comment which confines a whole class of change to campaign boundaries now says that its last step is derived, and points at the experiment that would make it measured -- designed on a tile whose final_cat is bit-reproducible from another campaign's independent copy, so the observation costs nothing to make. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- workflow/rules/tile.smk | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 36a5af438..3129621e0 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -594,12 +594,25 @@ rule tile_ngmix: # them and nothing propagates down to them either. # # A group replanned without tile_vignets is precisely the state - # TILE_VIGNET_REQUIRED exists to catch. Every chunk trips the guard, - # the group fails, and GroupJob.postprocess(error=True) fans out over - # every member and removes its EXISTING outputs -- tile_make_cat's + # TILE_VIGNET_REQUIRED exists to catch. Every chunk trips the guard and + # the group fails; from there GroupJob.postprocess(error=True) fans out + # over every member and removes its EXISTING outputs -- tile_make_cat's # final_cat on the persistent root among them. So the cost of getting # this wrong is a deleted science product, not a wasted hour. # + # THE REPLANNING IS OBSERVED; THAT LAST STEP IS DERIVED, and the + # difference should not quietly disappear from a comment that confines a + # whole class of change to campaign boundaries. The deletion is read off + # snakemake's own cleanup path (job_scheduler.py:564 -> + # jobs.py:1653-1673 -> jobs.py:942-965, and profiles/nibi sets no + # keep-incomplete) and has never been seen -- twice now, by independent + # reviews, for the same reason: snakemake reports "Group jobs: inactive + # (local execution)", so no login-node fixture reaches the group path at + # all and only a submitted job against a finished tile can settle it. + # sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md designs exactly + # that, on a tile whose final_cat is bit-reproducible from another + # campaign's independent copy, so the observation costs nothing to make. + # # WHY THE TILE_LOCAL WARNING ABOVE DOES NOT ALREADY COVER IT, which is # the whole reason this needs its own note: tile_local() sits in the # params.pre of tile_vignets, tile_ngmix AND tile_make_cat, so an edit From 503c0839404b470439d31268c4297484eec70576 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 04:18:51 -0400 Subject: [PATCH 055/100] docs(orchestration): mark the Snakefile half of the deletion claim as derived cfe62ed1 separated the observed half of the range_hash hazard from the derived half in tile.smk, but the NGMIX_RANGE_HASH comment in the Snakefile still asserted the deletion flatly: "the failed group's postprocess removes each member's existing outputs, final_cat included". A reader who stops at the hash definition -- which is where the other four hashes are argued, so a plausible place to stop -- would take the destruction as measured. Same split as tile.smk's, one clause: the replanning and the guard trip are observed, the removal is DERIVED from snakemake's cleanup path and has never been seen. Points at tile.smk for the source lines and at sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md for the run that would settle it, rather than restating either. A separate commit and not an amend of f196fa12: two commits now sit on top of it in this worktree, one of them another session's report fix, and rewriting them to fold in a comment is not a trade worth making. Comments only. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- workflow/Snakefile | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 80b4bea03..542f2e64e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -271,13 +271,16 @@ STAR_CAT_HASH = hashlib.md5( # # ADDING it is itself a params change, and on a RESUME that change is # destructive rather than merely expensive: it replans the fused group WITHOUT -# tile_vignets, so every chunk trips TILE_VIGNET_REQUIRED and the failed -# group's postprocess removes each member's existing outputs, final_cat -# included. The TILE_LOCAL warning does not cover this — tile_local() sits in -# three rules' params and drags tile_vignets back in, this sits in one — and -# clean_tiles: false, the shipped default, is the configuration it bites. -# tile.smk's range_hash comment carries the mechanism. Land this hash, and any -# later edit to the split, at a campaign boundary on a fresh root. +# tile_vignets, so every chunk trips TILE_VIGNET_REQUIRED and the group fails. +# That the failed group then removes each member's existing outputs, final_cat +# included, is DERIVED from snakemake's cleanup path and has never been +# observed — tile.smk's range_hash comment cites the source lines, and +# sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md designs the run that +# would settle it. The TILE_LOCAL warning does not cover this case at all: +# tile_local() sits in three rules' params and drags tile_vignets back in, +# this sits in one — and clean_tiles: false, the shipped default, is the +# configuration it bites. Land this hash, and any later edit to the split, at +# a campaign boundary on a fresh root. NGMIX_RANGE_HASH = hashlib.md5( (SCRIPTS / "ngmix_range.py").read_bytes()).hexdigest()[:12] From 4b96effad59db1bc97f7e61cd68880be33c81dc2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 05:13:29 -0400 Subject: [PATCH 056/100] run(smk-g6): point the campaign at a fresh 64-tile root The merge that precedes this adds range_hash to tile_ngmix's params, which on a root holding finished tiles replans the fused group WITHOUT tile_vignets -- the state TILE_VIGNET_REQUIRED catches, after which a failed group job removes every member's outputs, final_cat included. So the merge and this switch are one operation: leaving the config pointing at smk-g5, which now holds a finished tile, would ship a branch whose next `sp run` destroys it. smk-g6 is 64 tiles chosen by batch geometry (sp-products/smk-g6/batch.md), seeded on the 34 smk-g4 tiles so the campaign is a paired contrast at scale rather than a fresh unknown -- growing that seed set is nearly free, because it is sparse and the first 18 tiles added introduce no new exposure stores. 127 distinct stores, ~838 GiB projected peak with clean_tiles on. Both reclamation mechanisms are on: exercising clean_tile at scale is one of the things this campaign exists to do, and without it 64 tiles leave 77 GiB of residue that never goes away. Preconditions and the launch order are in sp-products/smk-g6/LAUNCH.md. --- workflow/config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index e4d6aa48d..8e4de8506 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -16,7 +16,7 @@ # (The 210/211 quad used earlier is unusable: its tile images are symlinks into # anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are # broken.) -tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g5/tiles1.txt +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif @@ -27,7 +27,7 @@ container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # intermediates, sized so a batch finishes inside the 60-day purge window. The # sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-g5 +run_dir: /scratch/cdaley/shapepipe-output/smk-g6 # products_dir is the PERSISTENT root: the durable, low-volume products — the # final catalogues (/tiles///final_cat-.fits, @@ -44,7 +44,7 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g5 # # Snakemake's own state is the one durable-looking thing that stays on scratch # (-state; bin/sp explains why). -products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g5 +products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -81,7 +81,7 @@ star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction # would otherwise have to rebuild from tile headers. -index_db: /project/def-mjhudson/cdaley/sp-products/smk-g5/index/run_index.sqlite +index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads @@ -91,7 +91,7 @@ index_db: /project/def-mjhudson/cdaley/sp-products/smk-g5/index/run_index.sqlite # this is a one-tile control, its ~8 exposure stores cost ~54 GiB against 979 GiB # free, reclamation is already measured exactly by smk-g4, and keeping the stores # means a re-measurement re-runs only the shape chain instead of the whole tile. -clean: false +clean: true # Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_tile` job per in-scope tile, ordered after that tile's own final_cat, so @@ -131,7 +131,7 @@ clean: false # # OFF here for the same reason `clean:` is: smk-g5 is a one-tile control, and its # store is what a re-measurement would read. -clean_tiles: false +clean_tiles: true # Tiles that may NOT pin an exposure store (default: empty). # From acaeb0b5370baa12d062ba325b700ea2ed485a86 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 05:18:01 -0400 Subject: [PATCH 057/100] docs(orchestration): the final_cat deletion is now observed, not derived Job 20818649 against smk-g5, 32 seconds. A params change on tile_ngmix alone replanned the fused group without tile_vignets -- the SLURM group label itself came back as tile_shape_tile_make_cat_tile_merge_cats_tile_ngmix -- every chunk tripped TILE_VIGNET_REQUIRED, the group failed, and final_cat-186.307.fits was gone from the persistent products root afterwards. The tile's directory was empty. Restored from a copy taken first; the catalogue is bit-reproducible from smk-g4's independent copy of the same tile, which is why smk-g5 was the safe target. This had been derived twice from snakemake's cleanup path and never seen, because "Group jobs: inactive (local execution)" puts the group path out of reach of any login-node fixture. It is the constraint that confines a whole class of change to campaign boundaries, so it was worth 32 seconds to stop guessing. One thing the run makes plain that the derivation did not: the missing-store guard is what turns this from a silently wrong catalogue into a failed job, and the failed job is exactly what deletes the science product. The guard is still the right trade. But on a fused group "fails loudly" is not a cheap outcome, which argues for never reaching the state rather than for softening the guard. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017gNJUBiHxeYVBPpsGxRVKL --- workflow/Snakefile | 10 +++++----- workflow/rules/tile.smk | 35 +++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 542f2e64e..4a0bdc307 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -272,11 +272,11 @@ STAR_CAT_HASH = hashlib.md5( # ADDING it is itself a params change, and on a RESUME that change is # destructive rather than merely expensive: it replans the fused group WITHOUT # tile_vignets, so every chunk trips TILE_VIGNET_REQUIRED and the group fails. -# That the failed group then removes each member's existing outputs, final_cat -# included, is DERIVED from snakemake's cleanup path and has never been -# observed — tile.smk's range_hash comment cites the source lines, and -# sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md designs the run that -# would settle it. The TILE_LOCAL warning does not cover this case at all: +# The failed group then removes each member's existing outputs, final_cat +# included. That was derived from snakemake's cleanup path for months; it is now +# OBSERVED — job 20818649 against smk-g5, 2026-08-30, 32 seconds, and the tile's +# directory on the persistent root was empty afterwards. tile.smk's range_hash +# comment carries the run. The TILE_LOCAL warning does not cover this case at all: # tile_local() sits in three rules' params and drags tile_vignets back in, # this sits in one — and clean_tiles: false, the shipped default, is the # configuration it bites. Land this hash, and any later edit to the split, at diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 3129621e0..805388960 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -600,18 +600,29 @@ rule tile_ngmix: # final_cat on the persistent root among them. So the cost of getting # this wrong is a deleted science product, not a wasted hour. # - # THE REPLANNING IS OBSERVED; THAT LAST STEP IS DERIVED, and the - # difference should not quietly disappear from a comment that confines a - # whole class of change to campaign boundaries. The deletion is read off - # snakemake's own cleanup path (job_scheduler.py:564 -> - # jobs.py:1653-1673 -> jobs.py:942-965, and profiles/nibi sets no - # keep-incomplete) and has never been seen -- twice now, by independent - # reviews, for the same reason: snakemake reports "Group jobs: inactive - # (local execution)", so no login-node fixture reaches the group path at - # all and only a submitted job against a finished tile can settle it. - # sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md designs exactly - # that, on a tile whose final_cat is bit-reproducible from another - # campaign's independent copy, so the observation costs nothing to make. + # ALL OF THIS IS OBSERVED, end to end, on a real campaign root -- SLURM + # job 20818649 against smk-g5, 2026-08-30. It had been derived twice from + # snakemake's cleanup path and never seen, because "Group jobs: inactive + # (local execution)" puts it out of reach of any login-node fixture. The + # run took 32 seconds: + # * the dry run scheduled tile_ngmix x8 + tile_merge_cats + + # tile_make_cat, reason "params have changed since last execution", + # with tile_vignets and tile_detect ABSENT -- the group's own SLURM + # label came back as tile_shape_tile_make_cat_tile_merge_cats_tile_ngmix, + # with no tile_vignets in it; + # * every chunk tripped the guard below; + # * the group FAILED, and final_cat-186.307.fits was GONE from + # /project afterwards. The tile's directory on the persistent root + # was empty. + # (Restored from a copy taken first; the design and the transcript are in + # sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md.) + # + # WORTH SITTING WITH: the guard below is what makes this loud rather than + # silent, and the loud failure is precisely what triggers the deletion. A + # guard that turns a wrong catalogue into a failed job is still the right + # trade -- but on a fused group, "fails loudly" is not a cheap outcome, + # and that is an argument for never reaching this state rather than for + # relaxing the guard. # # WHY THE TILE_LOCAL WARNING ABOVE DOES NOT ALREADY COVER IT, which is # the whole reason this needs its own note: tile_local() sits in the From fd913652c650a1bca6cd1c45aff55588367bfcdf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:04:35 -0400 Subject: [PATCH 058/100] refactor(smk): one script_hash helper, unit_pre derives its level, star-cats use the resolved image Five surgical de-duplications in the workflow layer, all behaviour-preserving: * the six copy-pasted md5(...)[:12] blocks collapse to one script_hash(name) helper and six one-line constants (identical digests); * unit_pre() drops its `level` parameter and takes it from STAGE_DIR, which already maps stage -> (level, subdir); the 13 call sites lose an argument they could only get wrong. The generated params.pre strings are unchanged; * exposure.smk's in_container() stops hand-building an apptainer line from config['container'] and uses the Snakefile's resolved _image plus container.profile_apptainer_args(). This removes a real divergence: star_catalogue and exp_star_cat were the ONLY rules that ignored a user's dev sandbox. It fails loudly if the profile is unreadable rather than silently dropping --cleanenv and the PYTHONPATH pin. REPO_DIR is now unused and goes with it; * tile_local() calls unit_num() instead of re-spelling the dot->dash convention inline (unit_num supplies the separating dash, so the literals lose theirs); the produced path is byte-identical, which matters because it lands in params.pre; * clean_targets()/clean_tile_targets() and the sqlite slurp are gated on workflow.is_main_process. Both fed `rule all` at module level and so ran on every one of the ~800 per-job re-parses, none of which can schedule `all`; the slurp read the whole campaign's tile_exposures there. Job parses now get the same three lookups (exp_name / tile_exposures / exp_consumers) backed by memoised single-row queries, ORDER BY rowid so the lists match the eager path's insertion order exactly. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- workflow/Snakefile | 108 +++++++++++++++++++++++++++++------- workflow/rules/exposure.smk | 46 ++++++++++----- workflow/rules/prepare.smk | 6 +- workflow/rules/tile.smk | 29 +++++----- 4 files changed, 136 insertions(+), 53 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 4a0bdc307..7326c768a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -29,6 +29,7 @@ two things this workflow needs — "this stage succeeded" and "here is what happened". """ +import functools import hashlib import json import os @@ -83,11 +84,6 @@ PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) # star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it # (config.yaml explains the placement). STAR_CATS = Path(config["star_cats"]) -# This checkout's own tree: the star-cat rules run in the container against its -# src/ (shapepipe.utilities.vizier and .cfis), for the same reason CONFIG_DIR is -# the committed config dir — script, library and rule are one artefact and -# version together. -REPO_DIR = Path(workflow.basedir).parent INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the @@ -162,8 +158,15 @@ if PHASE == "compute" and workflow.is_main_process: # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. # EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure # is keyed on (D5). +# +# SLURPED ONLY IN THE HEAD PROCESS. The executor re-parses this file inside every +# job (~800 per run), and each of those parses needs at most its own unit's edges +# — reading the whole campaign's tile_exposures table there is pure waste that +# grows with the campaign rather than with the job. Job parses get the same three +# lookups backed by memoised single-row queries instead (the table's primary key +# indexes both), so every input function returns exactly what it would have. EXP, TILE_EXP, EXP_TILES = {}, {}, {} -if INDEX_DB.exists(): +if INDEX_DB.exists() and workflow.is_main_process: # timeout=60: Lustre lock handoffs are slow; the default 5 s trips on # nothing more sinister than a reader in another job's parse. _con = sqlite3.connect(INDEX_DB, timeout=60) @@ -173,9 +176,59 @@ if INDEX_DB.exists(): EXP_TILES.setdefault(_exp, []).append(_tile) _con.close() + +if workflow.is_main_process: + + def exp_name(exp): + """The exposure's original name (get_images matches .fits.fz).""" + return EXP[exp] + + def tile_exposures(tile): + """This tile's exposure base-ids, sorted as the index stores them.""" + return TILE_EXP.get(tile, []) + + def exp_consumers(exp): + """Every tile in the campaign that reads this exposure (D5).""" + return EXP_TILES.get(exp, []) + +else: + # One connection for the whole parse, and lru_cache over it: a job parse asks + # about a handful of units, each of them several times (params, input, log). + _JOB_CON = sqlite3.connect(INDEX_DB, timeout=60) if INDEX_DB.exists() else None + + @functools.lru_cache(maxsize=None) + def exp_name(exp): + row = _JOB_CON.execute( + "SELECT name FROM exposures WHERE exp_id = ?", (exp,)).fetchone() + if row is None: + raise KeyError(exp) # same failure as EXP[exp] would give + return row[0] + + @functools.lru_cache(maxsize=None) + def tile_exposures(tile): + if _JOB_CON is None: + return [] + # ORDER BY rowid, not by exp_id: the eager slurp above is a table scan, + # so its lists are in INSERTION order, and the two paths must hand the + # DAG the same list and not merely the same set. + return [r[0] for r in _JOB_CON.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id = ? ORDER BY rowid", + (tile,))] + + @functools.lru_cache(maxsize=None) + def exp_consumers(exp): + if _JOB_CON is None: + return [] + # Unindexed on exp_id (the PK leads with tile_id), so this is a scan — + # harmless because the only caller, clean_consumers, is reachable from + # the head process alone (clean_exposure is a localrule). + return [r[0] for r in _JOB_CON.execute( + "SELECT tile_id FROM tile_exposures WHERE exp_id = ? ORDER BY rowid", + (exp,))] + # Tiles this run can actually compute: declared AND indexed. The index spans the # campaign, so it is intersected with the declared list, not used as it. -TILES_READY = [t for t in TILES if t in TILE_EXP] +TILES_READY = [t for t in TILES if tile_exposures(t)] READY_SET = set(TILES_READY) # A compute invocation with nothing to compute is never a success. Without this, @@ -246,19 +299,21 @@ def unit_num(unit): # the count table silently leaves stale manifests in place (bitten live). Scoped # to completeness.py alone, the one script every shell line runs; build_forest.py # gets its own hash, on the forest rule only. -SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] -FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] -CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] -CLEAN_TILE_HASH = hashlib.md5( - (SCRIPTS / "clean_tile.py").read_bytes()).hexdigest()[:12] +def script_hash(name): + """The 12-hex fingerprint of one script under workflow/scripts/.""" + return hashlib.md5((SCRIPTS / name).read_bytes()).hexdigest()[:12] + +SCRIPT_HASH = script_hash("completeness.py") +FOREST_HASH = script_hash("build_forest.py") +CLEAN_HASH = script_hash("clean_exposure.py") +CLEAN_TILE_HASH = script_hash("clean_tile.py") # Same argument for star_cats.py, which both star-cat rules call: their params # otherwise fingerprint nothing but paths, so an edit to the chunking or the cut # would never rerun them. ONE hash for both rules because it is one script — and # that is also why fetch and cut live in one module (they must agree on which # pixel holds which star). The hash does NOT key the store path — see # config.yaml's star_cats block on clearing the store after a semantic change. -STAR_CAT_HASH = hashlib.md5( - (SCRIPTS / "star_cats.py").read_bytes()).hexdigest()[:12] +STAR_CAT_HASH = script_hash("star_cats.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks # are a PARTITION of its object IDs: resume a tile across an edit to the split @@ -281,8 +336,7 @@ STAR_CAT_HASH = hashlib.md5( # this sits in one — and clean_tiles: false, the shipped default, is the # configuration it bites. Land this hash, and any later edit to the split, at # a campaign boundary on a fresh root. -NGMIX_RANGE_HASH = hashlib.md5( - (SCRIPTS / "ngmix_range.py").read_bytes()).hexdigest()[:12] +NGMIX_RANGE_HASH = script_hash("ngmix_range.py") # --- exposure reclamation (D5, S5) ----------------------------------------- @@ -340,7 +394,7 @@ def tombstone(exp): def clean_consumers(exp): """Every tile in the campaign that reads this exposure — the set whose vignets must all exist before the store may go — minus the ignored tiles.""" - return sorted(t for t in EXP_TILES.get(exp, []) + return sorted(t for t in exp_consumers(exp) if t not in CLEAN_IGNORE_TILES) @@ -358,8 +412,14 @@ def clean_targets(): Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in `clean_ignore_tiles` neither gates eligibility nor appears in the job's input — which is the whole point of that list. + + HEAD PROCESS ONLY. This feeds `rule all` at module level, so it runs on every + parse — including the ~800 per-job re-parses under the slurm executor, none of + which can ever schedule `all`. Each of those was stat()ing the whole + campaign's tile_vignets manifests for nothing. The empty list a job parse + returns costs it exactly the target it could not have used anyway. """ - if not CLEAN: + if not CLEAN or not workflow.is_main_process: return [] out = [] for exp, raw in EXP_TILES.items(): @@ -415,8 +475,10 @@ def clean_tile_targets(): also what makes flipping `clean_tiles` on RETROACTIVE: the tiles already reclaimed are exactly the ones with a tombstone, so the missing tombstones schedule exactly the missing cleans. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. """ - if not CLEAN_TILES: + if not CLEAN_TILES or not workflow.is_main_process: return [] return sorted(tile_tombstone(t) for t in TILES_READY) @@ -429,7 +491,7 @@ _THREAD_CAPS = " ".join( ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) -def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, +def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, pre_run=()): """The unit-furniture + environment prologue, as bash. @@ -462,8 +524,12 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, FileHandler raises on an existing run dir, and it is how a rerun never sees stale products (D2: the job clears its run dir at start). """ + # The stage table already knows both halves of "where this stage writes": + # its level and its run dir. Taking them from there rather than from an + # argument means a caller cannot disagree with completeness.py about which + # store a stage belongs to. + level, subdir = STAGE_DIR[stage] work = tile_dir(unit) if level == "tile" else exp_dir(unit) - _, subdir = STAGE_DIR[stage] lines = [ "set -euo pipefail", f"export SP_RUN='{work}'", diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index e41920137..fc7c3af7d 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -47,8 +47,8 @@ rule exp_get_images: log: f"{EXP_DIR}/logs/exp_get_images.json" params: - pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, - exp_name=EXP[wc.exp]), + pre = lambda wc: unit_pre("exp_get_images", wc.exp, + exp_name=exp_name(wc.exp)), script_hash = SCRIPT_HASH threads: 1 retries: 2 @@ -84,17 +84,33 @@ STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" # but call apptainer THEMSELVES rather than letting the SDM wrap them # (`container: None` on both): the CA bundle above and the exposure rule's # host-side farm loop both need the explicit exec. bin/sp has loaded the -# apptainer module. PYTHONPATH pins this checkout's src/ for -# shapepipe.utilities.{vizier,cfis} — the mirror-retry query and the tile-ID -# grid convention are library code, not copies. +# apptainer module. +# +# WHICH image and WHICH arguments are not this file's to decide, and hand-rolling +# them here was a real divergence: these were the only two rules that ignored a +# user's dev sandbox, because they read `config['container']` — the shared /project +# fallback — instead of the image the Snakefile resolved for everything else. +# `_image` is that resolution (sandbox -> cached SIF -> config), and the profile's +# own apptainer-args are the same string the SDM splices onto every other rule, +# PYTHONPATH pin for shapepipe.utilities.{vizier,cfis} included. +# container.profile_apptainer_args() exists precisely so this file can read them +# rather than restate them. Only the CA bundle is added on top, and only when the +# command touches the network. def in_container(cmd, *, network=False): - ca = ("," + ",".join(f"{k}={STAR_CAT_CA}" for k in - ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", "CURL_CA_BUNDLE")) - if network else "") - return (f"apptainer exec --cleanenv --home {Path.home()}" - f" --bind /project --bind /scratch" - f" --env PYTHONPATH={REPO_DIR}/src{ca}" - f" '{config['container']}' {cmd}") + args = list(_container.profile_apptainer_args()) + if not args: + # Silently falling back would run these two rules with no --cleanenv and + # no PYTHONPATH pin, i.e. against a different src/ than every other rule. + raise WorkflowError( + f"Could not read apptainer-args from {_container.PROFILE_FILE}; " + f"star_catalogue and exp_star_cat build their apptainer line from it.") + if network: + # The container's certifi bundle, one --env per variable (the profile's + # own PYTHONPATH entry uses the same one-assignment-per-flag form). + args += [a for k in ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", + "CURL_CA_BUNDLE") + for a in ("--env", f"{k}={STAR_CAT_CA}")] + return f"apptainer exec {' '.join(args)} '{_image}' {cmd}" # The campaign's star catalogue: a first-class durable science product, keyed by @@ -254,7 +270,7 @@ rule exp_split: log: f"{EXP_DIR}/logs/exp_split.json" params: - pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), + pre = lambda wc: unit_pre("exp_split", wc.exp), script_hash = SCRIPT_HASH threads: 8 resources: @@ -275,7 +291,7 @@ rule exp_mask: log: f"{EXP_DIR}/logs/exp_mask.json" params: - pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), + pre = lambda wc: unit_pre("exp_mask", wc.exp), script_hash = SCRIPT_HASH threads: 4 resources: @@ -295,7 +311,7 @@ rule exp_psf: log: f"{EXP_DIR}/logs/exp_psf.json" params: - pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), + pre = lambda wc: unit_pre("exp_psf", wc.exp), script_hash = SCRIPT_HASH threads: 8 retries: 2 diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 96a0ae261..6ca24f4e8 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -47,7 +47,7 @@ rule tile_get_images: log: f"{TILE_DIR}/logs/tile_get_images.json" params: - pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), + pre = lambda wc: unit_pre("tile_get_images", wc.tile), script_hash = SCRIPT_HASH threads: 1 retries: 2 @@ -66,7 +66,7 @@ rule tile_uncompress: log: f"{TILE_DIR}/logs/tile_uncompress.json" params: - pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), + pre = lambda wc: unit_pre("tile_uncompress", wc.tile), script_hash = SCRIPT_HASH threads: 4 resources: @@ -84,7 +84,7 @@ rule tile_find_exposures: log: f"{TILE_DIR}/logs/tile_find_exposures.json" params: - pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), + pre = lambda wc: unit_pre("tile_find_exposures", wc.tile), script_hash = SCRIPT_HASH threads: 1 resources: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 805388960..bb203ecdd 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -204,10 +204,12 @@ def tile_local(tile): the time any member runs; failing loudly if it does not is correct, because the alternative is ngmix silently reading a different tree. """ - # `log_exp_headers--.sqlite`, named from the tile with the dot - # replaced -- spelled out rather than globbed so a missing file is one - # precise error instead of an empty expansion. - tile_num = tile.replace(".", "-") + # `log_exp_headers--.sqlite`, named from the tile in ShapePipe's + # dashed image-number form -- spelled out rather than globbed so a missing + # file is one precise error instead of an empty expansion. unit_num() owns + # that convention and supplies the SEPARATING DASH itself, so there is none + # in the literals below. + tile_num = unit_num(tile) return f''' if [ ! -d /local/scratch ]; then echo "tile_shape: node-local storage unavailable." >&2 @@ -225,10 +227,10 @@ mkdir -p "$SP_VIGNET_OUT" "$SP_WCS_DIR" || {{ }} # the WCS store -- see the docstring. Copy to a temp name and rename, because # `cp` is not atomic and the eight chunks share this destination. -sp_wcs_src="$SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers-{tile_num}.sqlite" +sp_wcs_src="$SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers{tile_num}.sqlite" cp "$sp_wcs_src" "$SP_WCS_DIR/.log_exp_headers.$$" && \ mv -f "$SP_WCS_DIR/.log_exp_headers.$$" \ - "$SP_WCS_DIR/log_exp_headers-{tile_num}.sqlite" || {{ + "$SP_WCS_DIR/log_exp_headers{tile_num}.sqlite" || {{ rm -f "$SP_WCS_DIR/.log_exp_headers.$$" echo "tile_shape: could not stage the WCS store." >&2 echo " source: $sp_wcs_src" >&2 @@ -318,7 +320,7 @@ fi def tile_exp(wc): - return TILE_EXP.get(wc.tile, []) + return tile_exposures(wc.tile) # --- the tile->exposure edge, and why it is cut for finished tiles ---------- # @@ -434,7 +436,7 @@ rule tile_merge_headers: log: f"{TILE_DIR}/logs/tile_merge_headers.json" params: - pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, + pre = lambda wc: unit_pre("tile_merge_headers", wc.tile, forest=forest_dir(wc.tile)), script_hash = SCRIPT_HASH threads: 4 @@ -454,7 +456,7 @@ rule tile_detect: log: f"{TILE_DIR}/logs/tile_detect.json" params: - pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), + pre = lambda wc: unit_pre("tile_detect", wc.tile), script_hash = SCRIPT_HASH threads: 8 resources: @@ -496,7 +498,7 @@ rule tile_vignets: log: f"{TILE_DIR}/logs/tile_vignets.json" params: - pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, + pre = lambda wc: unit_pre("tile_vignets", wc.tile, forest=forest_dir(wc.tile), pre_run=[tile_local(wc.tile), TILE_VIGNET_FRESH]), script_hash = SCRIPT_HASH @@ -560,8 +562,7 @@ rule tile_ngmix: log: f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" params: - pre = lambda wc: unit_pre( - "tile_ngmix", "tile", wc.tile, + pre = lambda wc: unit_pre("tile_ngmix", wc.tile, env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, # Two steps, not `eval "$(...)"`: a command substitution inside eval # discards the script's exit status, so a missing sexcat would fall @@ -760,7 +761,7 @@ rule tile_merge_cats: log: f"{TILE_DIR}/logs/tile_merge_cats.json" params: - pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, + pre = lambda wc: unit_pre("tile_merge_cats", wc.tile, env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), script_hash = SCRIPT_HASH threads: 8 @@ -790,7 +791,7 @@ rule tile_make_cat: log: f"{TILE_DIR}/logs/tile_make_cat.json" params: - pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile, + pre = lambda wc: unit_pre("tile_make_cat", wc.tile, pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, TILE_CLEAN]), script_hash = SCRIPT_HASH From 9ca1800aa11679f68ed8d29248b606bafc5a3623 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:05:47 -0400 Subject: [PATCH 059/100] refactor(smk): sp_shell composes the two special tile bodies tile_vignets and tile_make_cat carried near-copies of sp_shell's rc-composition body, each with its own copy of the rationale comment. sp_shell now takes check_args (extra completeness-check flags) and post (bash between the check and the exit), and all three rules go through it. Verified byte-identical shell strings for both rules, which is what keeps the `code` rerun-trigger quiet. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- workflow/Snakefile | 15 +++++++++++++- workflow/rules/tile.smk | 46 ++++++++++++++--------------------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 7326c768a..3813c8b3d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -557,9 +557,20 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, return "\n".join(lines) -def sp_shell(stage, config_name): +def sp_shell(stage, config_name, *, check_args="", post=""): """The rule's shell string: prologue, one shapepipe_run, one completeness check. + EVERY rule that runs shapepipe_run goes through here, including the two that + need a little more than the plain body — they pass it rather than restating + it, so the rc composition below has exactly one definition: + * ``check_args`` — extra flags for the completeness check. tile_vignets + points it at the NODE-LOCAL run root (``--run-dir "$SP_LOCAL"``) and keeps + the manifest's unit field the tile ID rather than the local root's + basename (``--unit``), which also keeps the manifest content independent + of an ephemeral path the mtime rerun-trigger reads. + * ``post`` — bash between the check and the ``exit``. tile_make_cat + publishes its final_cat there, guarded on ``rc``. + ``{threads}``, ``{output}`` and ``{log}`` are placeholders HERE and nowhere else (see unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number by construction (D4). @@ -585,7 +596,9 @@ def sp_shell(stage, config_name): "rc=0\n" f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" + f"{check_args}" " --log {log} --job-rc \"$rc\" || rc=1\n" + f"{post}" "exit $rc\n" ) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index bb203ecdd..f0babc3bf 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -519,20 +519,13 @@ rule tile_vignets: runtime = 20, slurm_extra = TILE_SLURM_EXTRA shell: - # sp_shell's body, except that the completeness check is pointed at the - # NODE-LOCAL run root. completeness.py is unmodified: --run-dir is its - # own documented override for "$SP_RUN", and --unit keeps the manifest's - # unit field the tile ID instead of the basename of the local root (it - # would otherwise default to "sp-tile"). Passing --unit also keeps the - # manifest CONTENT independent of the ephemeral local path, which the - # mtime rerun-trigger depends on. - "{params.pre}\n" - "rc=0\n" - 'shapepipe_run -c "$SP_CONFIG/config_tile_PiViVi.ini" -b {threads} || rc=$?\n' - f"python {SCRIPTS}/completeness.py check tile_vignets {{output.manifest}}" - ' --run-dir "$SP_LOCAL" --unit {wildcards.tile}' - " --log {log} --job-rc \"$rc\" || rc=1\n" - "exit $rc\n" + # The completeness check is pointed at the NODE-LOCAL run root. + # completeness.py is unmodified: --run-dir is its own documented override + # for "$SP_RUN", and --unit keeps the manifest's unit field the tile ID + # instead of the basename of the local root (it would otherwise default + # to "sp-tile"). See sp_shell for the rest. + sp_shell("tile_vignets", "config_tile_PiViVi.ini", + check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') # ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own # CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params @@ -802,22 +795,15 @@ rule tile_make_cat: runtime = 15, slurm_extra = TILE_SLURM_EXTRA shell: - # sp_shell's body, plus the catalogue publish: a real file, so it is a - # real declared output (and it persists — never temp()). `--job-rc` is - # composed in for the same reason it is everywhere else (see sp_shell) — - # without it a job whose counts cleared their floors but whose - # shapepipe_run died writes a "complete" log for a manifest snakemake is - # about to delete. - "{params.pre}\n" - "rc=0\n" - 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' - f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}}" - ' --log {log} --job-rc "$rc" || rc=1\n' - "if [ $rc -eq 0 ]; then\n" - ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' - ' | head -1)" {output.final_cat}\n' - "fi\n" - "exit $rc\n" + # The plain body plus the catalogue publish: final_cat is a real file, so + # it is a real declared output (and it persists — never temp()). The + # publish is guarded on rc, so a job whose shapepipe_run died never + # publishes a catalogue for a manifest snakemake is about to delete. + sp_shell("tile_make_cat", "config_tile_Mc.ini", + post="if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner' + '/output/final_cat*.fits | head -1)" {output.final_cat}\n' + "fi\n") # --- tile reclamation (D5) -------------------------------------------------- From 8610c0a8177397f26eccd8ef02d2dfc6f53642b5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:07:19 -0400 Subject: [PATCH 060/100] perf(scripts): drop dead stage-table rows, bound the report's globs, stat less MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four small fixes in the scripts the rules call: * completeness.py: the COMPLETENESS/STAGE_DIR rows for `tile_mask` and the STAGE_DIR row for `tile_detect_uc` describe stages no rule declares (tile.smk:44 says why), so they are removed and named in the table comment as what the masked variant would add back. workflow/config/cfis/config_tile_Uc.ini goes with them — no rule reads it; the legacy bash reads example/cfis/, which keeps its copy. * completeness.py count_products: os.scandir with a follow-stat only on symlink entries, instead of a p.exists() stat on every file. The dead-symlink filter is the point of that test and its semantics are unchanged. * run_report.py: `**` recursive globs become fixed-depth `*/*/manifests/*.json`, `*/*/logs/*.json` and `*/*/cleaned.json`. The sharded layout is exactly two levels; `**` walked every unreclaimed output/ tree to find records that can only be at one depth. The docstring's "the sharding depth is not this script's business" claim goes with it. * run_report.py tally_level: the per-stage record's "complete" becomes a list of unit ids like the other four status keys, rather than a bare int. The two consumers take len(); the emitted JSON gains the complete-unit list. * star_cats.py focal_plane_disc: one fits.open pass over the 40 CCDs instead of 40 fits.getheader calls, each of which reopened the file and re-walked the HDU list. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- workflow/config/cfis/config_tile_Uc.ini | 92 ------------------------- workflow/scripts/completeness.py | 33 +++++++-- workflow/scripts/run_report.py | 31 +++++---- workflow/scripts/star_cats.py | 20 +++--- 4 files changed, 58 insertions(+), 118 deletions(-) delete mode 100644 workflow/config/cfis/config_tile_Uc.ini diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini deleted file mode 100644 index f2ff51e4a..000000000 --- a/workflow/config/cfis/config_tile_Uc.ini +++ /dev/null @@ -1,92 +0,0 @@ -# ShapePipe configuration file for tile object selection using -# an (external) catalogue - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Uc - -# Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = read_ext_sexcat_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the -# dashed tile ID (e.g. -210-282). -NUMBER_LIST = $SP_UNIT_NUM - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[READ_EXT_SEXCAT_RUNNER] - -# NOTE(blocker): run_sp_tile_Gic (external-catalog get_images) is not one of -# the 13 configs in this sweep -- no committed config produces it, so this -# path is written by naming-convention analogy, unverified. Uc is otherwise -# fully de-wrapper-ified; do not rely on this input until Gic exists. -INPUT_DIR = $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output - -FILE_PATTERN = CFIS_cat, CFIS_image, log_exp_headers - -FILE_EXT = .cat, .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -# Side length of the square postage stamp (vignet) extracted from the tile -# image, in pixels (must be odd). Default: 51 -VIGNET_SIZE = 51 - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = ALPHA_J2000,DELTA_J2000 - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 58c1ee9eb..cf67363db 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -95,7 +95,6 @@ # --- tile post --- "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, - "tile_mask": {"mask_runner": dict(expect=1, floor=1)}, "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, "tile_vignets": { "psfex_interp_runner": dict(expect=1, floor=1), @@ -112,13 +111,32 @@ def count_products(run_dir, runner, spec): - """Count files in ``run_dir//output[/]/`` (live links only).""" + """Count files in ``run_dir//output[/]/`` (live links only). + + scandir, not iterdir: the dirent already says whether an entry is a symlink, + so only the symlinks need the follow-stat that drops dead links. A plain + ``p.exists()`` per entry stats every one of them, and at DR6 scale this runs + once per runner per job over directories of tens to hundreds of files on a + network filesystem. + """ out = run_dir / runner / "output" if "subpath" in spec: out = out / spec["subpath"] if not out.is_dir(): return 0 - return sum(1 for p in out.iterdir() if p.exists()) # p.exists() drops dead links + n = 0 + try: + with os.scandir(out) as entries: + for e in entries: + # A dead symlink is the one thing that must not count (the bash + # `ls | wc -l` semantics this ports counted live files only), and + # a symlink is the only entry that can be dead — so it is the + # only one worth a follow-stat. + if not e.is_symlink() or os.path.exists(e.path): + n += 1 + except OSError: + return 0 + return n def check_floor(stage, run_dir): @@ -143,6 +161,13 @@ def check_floor(stage, run_dir): # committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so # the check never resolves a run-log. The ngmix entry interpolates the same env # var its config does, so chunk K's check looks at chunk K's dir. +# +# EVERY ENTRY HERE (and in COMPLETENESS above) HAS A RULE. The table used to +# carry two stages that did not: `tile_mask` (run_sp_tile_Ma, mask_runner 1/1) +# and `tile_detect_uc` (run_sp_tile_Uc). The committed config chain is the +# "sx_nomask" tile_detect variant and no tile-mask config was committed, so both +# were unreachable — tile.smk's docstring is where the masked variant is argued, +# and it is a config plus a rule plus these two rows, added back together. STAGE_DIR = { "tile_get_images": ("tile", "run_sp_tile_Git"), "tile_uncompress": ("tile", "run_sp_tile_Uz"), @@ -152,9 +177,7 @@ def check_floor(stage, run_dir): "exp_mask": ("exp", "run_sp_exp_Ma"), "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), - "tile_mask": ("tile", "run_sp_tile_Ma"), "tile_detect": ("tile", "run_sp_tile_Sx"), - "tile_detect_uc": ("tile", "run_sp_tile_Uc"), "tile_vignets": ("tile", "run_sp_tile_PiViVi"), "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), "tile_merge_cats": ("tile", "run_sp_Ms"), diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 69bd4fb42..7c8b01585 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -40,9 +40,13 @@ the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". -Records are discovered by glob (``tiles/**/manifests/*.json`` and -``tiles/**/logs/*.json``), not by constructed path: the unit stores are sharded -(``tiles///``) and the sharding depth is not this script's business. +Records are discovered by glob (``tiles/*/*/manifests/*.json`` and +``tiles/*/*/logs/*.json``), not by constructed path: units are found rather than +named, so a store holding a unit the index never heard of still reports. The +depth is FIXED at two, because the sharded layout is exactly two levels +(``tiles///``) — a ``**`` walked the whole tree, including every +``output/`` a run has not reclaimed yet, to find records that can only ever be +at one depth. """ import argparse @@ -123,8 +127,8 @@ def load_manifests(run_dir: Path, sub: str) -> dict: ours, which is what keeps a stray JSON deeper in the tree inert. """ out: dict = defaultdict(dict) - paths = sorted((run_dir / sub).glob("**/manifests/*.json")) \ - + sorted((run_dir / sub).glob("**/logs/*.json")) + paths = sorted((run_dir / sub).glob("*/*/manifests/*.json")) \ + + sorted((run_dir / sub).glob("*/*/logs/*.json")) for path in paths: try: m = json.loads(path.read_text()) @@ -179,7 +183,7 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, record belongs to, which nothing here records today. """ cleaned = set() - for path in sorted((run_dir / sub).glob("**/cleaned.json")): + for path in sorted((run_dir / sub).glob("*/*/cleaned.json")): unit = path.parent.name try: tomb = json.loads(path.read_text()) @@ -238,7 +242,11 @@ def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: """ per_stage = {} for stage in stages: - t = {"complete": 0, "warn": [], "failed": [], "not_run": [], "cleaned": []} + # All five status keys are unit-id LISTS, "complete" included: it used + # to be a bare int, which made it the one key a caller had to special- + # case. The emitted JSON gains the complete-unit list; the printed + # counts are len() of it. + t = {"complete": [], "warn": [], "failed": [], "not_run": [], "cleaned": []} agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) for u in units: m = manifests.get(u, {}).get(stage) @@ -249,10 +257,7 @@ def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: t["cleaned"].append(u) status = m.get("status", "failed") status = status if status in ("complete", "warn") else "failed" - if status == "complete": - t["complete"] += 1 - else: - t[status].append(u) + t[status].append(u) if status == "failed": # Failed units are named above, never folded into the attrition # aggregate: a whole-unit failure is not per-CCD attrition, and @@ -315,7 +320,7 @@ def print_stage_table(title, per_stage, n_units): for stage, t in per_stage.items(): att = [f"{r} {a['found']}/{a['expect']}" for r, a in t["products"].items() if a["found"] < a["expect"]] - print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " + print(f" {stage:<20} {len(t['complete']):>6} {len(t['warn']):>6} " f"{len(t['failed']):>6} {len(t['not_run']):>8} " f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") @@ -395,7 +400,7 @@ def _blocks(unit): blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} - done = report["tile_stages"]["tile_make_cat"]["complete"] + done = len(report["tile_stages"]["tile_make_cat"]["complete"]) report["final_cats"] = {"present": done, "of": len(tiles)} out = args.out or (args.index.parent / "run_report.json") diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py index 56886cfa0..50810b75f 100644 --- a/workflow/scripts/star_cats.py +++ b/workflow/scripts/star_cats.py @@ -197,15 +197,19 @@ def _wcs(header) -> WCS: def focal_plane_disc(image: Path, n_ccd: int = 40) -> tuple[float, float, float]: """(ra, dec, radius_deg) of the disc covering all CCDs of one exposure.""" + # ONE open for all 40 CCDs. `fits.getheader(image, ext)` opens the file, + # walks the HDU list to `ext` and closes again, so the loop cost 40 opens and + # O(n^2) header seeks over a compressed multi-extension exposure. centers, radii = [], [] - for ext in range(1, n_ccd + 1): - h = fits.getheader(image, ext) - w = _wcs(h) - (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( - [[h["NAXIS1"] / 2.0, h["NAXIS2"] / 2.0], [0, 0]], 1) - centers.append(SkyCoord(ra_c * u.deg, dec_c * u.deg)) - radii.append(centers[-1].separation( - SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg) + with fits.open(image) as hdul: + for ext in range(1, n_ccd + 1): + h = hdul[ext].header + w = _wcs(h) + (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( + [[h["NAXIS1"] / 2.0, h["NAXIS2"] / 2.0], [0, 0]], 1) + centers.append(SkyCoord(ra_c * u.deg, dec_c * u.deg)) + radii.append(centers[-1].separation( + SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg) ras = np.array([c.ra.deg for c in centers]) decs = np.array([c.dec.deg for c in centers]) From 725a8b090eef2a8f0dc17a581455075701572738 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:09:20 -0400 Subject: [PATCH 061/100] refactor(star-cats): one atomic write and one focal-plane geometry, in the library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workflow/scripts/star_cats.py and scripts/python/create_star_cat.py carried the same atomic FITS write twice (same docstring, same argument) and the same manual-CD WCS construction and focal-plane footprint twice — star_cats.py's docstring even said "Same construction as create_star_cat.py". Both pairs move into src/shapepipe/utilities, beside vizier.py and reached the same way (the PYTHONPATH pin the star-cat rules already run under): * file_io.write_atomic(table, path) — the create_star_cat signature loses output_dir and img_number, which only ever reconstructed the destination's own directory and basename; * focal_plane.get_wcs / ccd_center_and_radius / focal_plane_disc — the linear-terms-only WCS workaround is unchanged, and the shared disc does one fits.open pass instead of a getheader per CCD. create_star_cat's tile branch now calls ccd_center_and_radius rather than inlining it, and both branches convert the shared degrees to the arcmin Vizier wants. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- scripts/python/create_star_cat.py | 145 +++---------------------- src/shapepipe/utilities/file_io.py | 45 ++++++++ src/shapepipe/utilities/focal_plane.py | 92 ++++++++++++++++ workflow/scripts/star_cats.py | 75 ++----------- 4 files changed, 160 insertions(+), 197 deletions(-) create mode 100644 src/shapepipe/utilities/file_io.py create mode 100644 src/shapepipe/utilities/focal_plane.py diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py index 404cee4ac..05e3d3bb9 100755 --- a/scripts/python/create_star_cat.py +++ b/scripts/python/create_star_cat.py @@ -19,14 +19,10 @@ from cs_util import args as cs_args from cs_util import logging as cs_logging -import numpy as np - -from astropy.coordinates import SkyCoord -from astropy.wcs import WCS from astropy.io import fits -from astropy import units as u -from astropy.table import Table +from shapepipe.utilities.file_io import write_atomic +from shapepipe.utilities.focal_plane import ccd_center_and_radius, focal_plane_disc from shapepipe.utilities.vizier import query_vizier as _query_vizier @@ -34,113 +30,10 @@ CDS_CAT_ID = "I/305/out" -def _get_wcs(header): - """Get WCS. - - Compute the astropy WCS from header manually. - (The purpose of this is to avoid possible incompatibility on distortion - convention) - - Parameters - ---------- - header : astropy.header - Image header - - Returns - ------- - astropy.wcs.WCS - WCS object - - """ - final_wcs = WCS(naxis=2) - final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except: - final_wcs.wcs.cunit = ["deg", "deg"] - final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - final_wcs.wcs.cd = [ - [header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]], - ] - - return final_wcs - - -def _sphere_dist_arcmin(ra1, dec1, ra2, dec2): - """Compute angular distance between two sky positions in arcmin.""" - c1 = SkyCoord(ra=ra1 * u.deg, dec=dec1 * u.deg) - c2 = SkyCoord(ra=ra2 * u.deg, dec=dec2 * u.deg) - return c1.separation(c2).arcmin - - -def _ccd_center_and_radius(header): - """Return (ra, dec, radius_arcmin) for a single CCD.""" - w = _get_wcs(header) - nx, ny = header["NAXIS1"], header["NAXIS2"] - cx, cy = nx / 2.0, ny / 2.0 - ra_c, dec_c = w.all_pix2world([[cx, cy]], 1)[0] - # radius = half-diagonal to CCD corner - ra_corner, dec_corner = w.all_pix2world([[0, 0]], 1)[0] - radius = _sphere_dist_arcmin(ra_c, dec_c, ra_corner, dec_corner) - return ra_c, dec_c, radius - - -def _focal_plane_center_and_radius(f, n_ccd=40): - """Return (ra, dec, radius_arcmin) covering all CCDs of an exposure.""" - ras, decs, radii = [], [], [] - for ind in range(1, n_ccd + 1): - h = fits.getheader(f, ind) - ra, dec, r = _ccd_center_and_radius(h) - ras.append(ra) - decs.append(dec) - radii.append(r) - - ras = np.array(ras) - decs = np.array(decs) - radii = np.array(radii) - - ra_center = np.mean(ras) - dec_center = np.mean(decs) - - # Radius = max distance from focal plane center to any CCD center + that CCD's half-diagonal - dists = np.array([ - _sphere_dist_arcmin(ra_center, dec_center, ras[i], decs[i]) - for i in range(len(ras)) - ]) - radius = np.max(dists + radii) - - return ra_center, dec_center, radius - - def query_vizier(ra, dec, radius_arcmin): return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) -def _write_atomic(table, output_dir, img_number, output_name): - """Write ``table`` to ``output_name`` atomically. - - The catalogue is a run-independent CACHE, and the caller's only test for a - cache hit is ``os.path.isfile``. An in-place ``table.write`` that is killed - part-way (job timeout, OOM, node failure) therefore leaves a truncated FITS - that every later run trusts forever — and ``test -s`` passes on partial - bytes. Writing to a temp and renaming makes the visible file all-or-nothing: - ``os.replace`` is atomic within a directory. - - The temp keeps a ``.fits`` suffix, because astropy picks the writer from the - extension. It is dot-prefixed and PID-tagged so it stays out of the - ``star_cat*`` globs the rules use, and two concurrent writers cannot collide. - """ - tmp = f"{output_dir}/.tmp-{os.getpid()}-star_cat{img_number}.fits" - try: - table.write(tmp, overwrite=True) - os.replace(tmp, output_name) - finally: - if os.path.exists(tmp): - os.remove(tmp) - - def main(input_dir, output_dir, kind): file_list = os.listdir(input_dir) @@ -152,34 +45,24 @@ def main(input_dir, output_dir, kind): img_number = re.split("image", os.path.splitext(f)[0])[1] fpath = os.path.join(input_dir, f) + output_name = f"{output_dir}/star_cat{img_number}.fits" + if os.path.isfile(output_name): + continue + if kind == "exp": # One query covering the full MegaCam focal plane - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - ra, dec, radius = _focal_plane_center_and_radius(fpath) + ra, dec, radius_deg = focal_plane_disc(fpath) + radius = radius_deg * 60.0 print( f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" ) - table = query_vizier(ra, dec, radius) - _write_atomic(table, output_dir, img_number, output_name) - else: - h = fits.getheader(fpath, 0) - w = _get_wcs(h) - nx, ny = h["NAXIS1"], h["NAXIS2"] - cx, cy = nx / 2.0, ny / 2.0 - ra, dec = w.all_pix2world([[cx, cy]], 1)[0] - ra_corner, dec_corner = w.all_pix2world([[0, 0]], 1)[0] - radius = _sphere_dist_arcmin(ra, dec, ra_corner, dec_corner) - - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - table = query_vizier(ra, dec, radius) - _write_atomic(table, output_dir, img_number, output_name) + # A single image: its own centre and half-diagonal. + ra, dec, radius_deg = ccd_center_and_radius(fits.getheader(fpath, 0)) + radius = radius_deg * 60.0 + + table = query_vizier(ra, dec, radius) + write_atomic(table, output_name) return 0 diff --git a/src/shapepipe/utilities/file_io.py b/src/shapepipe/utilities/file_io.py new file mode 100644 index 000000000..8a0964aa0 --- /dev/null +++ b/src/shapepipe/utilities/file_io.py @@ -0,0 +1,45 @@ +"""FILE I/O UTILITIES. + +Small, dependency-light helpers for publishing files the rest of the pipeline +treats as a cache. + +:Author: consolidated from workflow/scripts/star_cats.py and + scripts/python/create_star_cat.py + +""" + +import os + + +def write_atomic(table, path): + """Publish ``table`` at ``path`` all-or-nothing. + + Every caller's only cache test is existence (``Path.exists`` / + ``os.path.isfile``), so a write killed part-way -- job timeout, OOM, node + failure -- would otherwise leave a truncated FITS that every later run + trusts forever, and ``test -s`` passes on partial bytes. Writing to a temp + and renaming makes the visible file all-or-nothing: ``os.replace`` is atomic + within a directory. + + The temp keeps the target's suffix, because astropy picks its writer from + the extension. It is dot-prefixed and PID-tagged so it stays out of the + ``star_chunk-*`` / ``star_cat*`` globs the rules use, and two concurrent + writers cannot collide. + + Parameters + ---------- + table : astropy.table.Table + Table to write + path : str or pathlib.Path + Destination path + + """ + path = os.fspath(path) + directory, name = os.path.split(path) + tmp = os.path.join(directory or ".", f".tmp-{os.getpid()}-{name}") + try: + table.write(tmp, overwrite=True) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.remove(tmp) diff --git a/src/shapepipe/utilities/focal_plane.py b/src/shapepipe/utilities/focal_plane.py new file mode 100644 index 000000000..32770bd06 --- /dev/null +++ b/src/shapepipe/utilities/focal_plane.py @@ -0,0 +1,92 @@ +"""FOCAL PLANE GEOMETRY. + +The sky footprint of a MegaCam exposure, read from its image headers. Both +star-catalogue producers need exactly this and used to carry their own copy of +it -- ``workflow/scripts/star_cats.py`` (the HEALPix chunk store's ``cut``) and +``scripts/python/create_star_cat.py`` (the one-cone-per-exposure path the store +replaced). They must agree on which sky an exposure covers, so there is one +definition of it. + +:Author: consolidated from the two star-catalogue scripts + +""" + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS + + +def get_wcs(header): + """Build the WCS by hand, from the linear terms only. + + Deliberately NOT ``WCS(header)``: it sidesteps distortion-convention + incompatibilities between these headers and astropy, and a footprint needs + nothing finer than the linear terms. + + Parameters + ---------- + header : astropy.io.fits.Header + Image header + + Returns + ------- + astropy.wcs.WCS + WCS object + + """ + final_wcs = WCS(naxis=2) + final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] + try: + final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] + except KeyError: + final_wcs.wcs.cunit = ["deg", "deg"] + final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] + final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] + final_wcs.wcs.cd = [ + [header["CD1_1"], header["CD1_2"]], + [header["CD2_1"], header["CD2_2"]], + ] + + return final_wcs + + +def ccd_center_and_radius(header): + """Return ``(ra_deg, dec_deg, radius_deg)`` for a single CCD. + + The radius is the half-diagonal: centre to the ``(0, 0)`` corner. + + """ + w = get_wcs(header) + (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( + [[header["NAXIS1"] / 2.0, header["NAXIS2"] / 2.0], [0, 0]], 1) + center = SkyCoord(ra_c * u.deg, dec_c * u.deg) + radius = center.separation(SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg + return float(ra_c), float(dec_c), float(radius) + + +def focal_plane_disc(image, n_ccd=40): + """Return ``(ra_deg, dec_deg, radius_deg)`` covering all CCDs of one exposure. + + The centre is the mean of the CCD centres; the radius is the largest + centre-to-CCD-centre distance plus that CCD's own half-diagonal. + + ONE ``fits.open`` for all the extensions: ``fits.getheader(image, ext)`` + opens the file, walks the HDU list to ``ext`` and closes again, so a loop + over it costs ``n_ccd`` opens and O(n^2) header seeks. + + """ + centers, radii = [], [] + with fits.open(image) as hdul: + for ext in range(1, n_ccd + 1): + ra_c, dec_c, radius = ccd_center_and_radius(hdul[ext].header) + centers.append((ra_c, dec_c)) + radii.append(radius) + + ras = np.array([c[0] for c in centers]) + decs = np.array([c[1] for c in centers]) + center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) + seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg + return (float(center.ra.deg), float(center.dec.deg), + float(np.max(seps + np.array(radii)))) diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py index 50810b75f..2407af71b 100644 --- a/workflow/scripts/star_cats.py +++ b/workflow/scripts/star_cats.py @@ -53,7 +53,6 @@ import argparse import json -import os import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -62,9 +61,12 @@ import healpy as hp from astropy import units as u from astropy.coordinates import SkyCoord -from astropy.io import fits from astropy.table import Table, vstack -from astropy.wcs import WCS + +# The PYTHONPATH pin in profiles/nibi puts this checkout's src/ on the path (see +# exposure.smk's in_container), the same way the vizier helper is reached below. +from shapepipe.utilities.file_io import write_atomic +from shapepipe.utilities.focal_plane import focal_plane_disc # GSC 2.3. The same catalogue the mask module's own CDS path uses # (mask.py: _CDS_cat_ID), so the store is a drop-in for it. @@ -133,25 +135,6 @@ def pixel_cone(ipix: int) -> tuple[float, float, float]: return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) -def write_atomic(table: Table, path: Path) -> None: - """Publish ``table`` at ``path`` all-or-nothing. - - The store's only cache test is ``Path.exists``, so a write killed part-way - (timeout, OOM, node failure) would otherwise leave a truncated FITS that - every later run trusts forever. The temp keeps the ``.fits`` suffix because - astropy picks its writer from the extension, and is dot-prefixed and - PID-tagged so it stays out of the ``star_chunk-*`` globs and two concurrent - writers cannot collide. - """ - tmp = path.parent / f".tmp-{os.getpid()}-{path.name}" - try: - table.write(tmp, overwrite=True) - os.replace(tmp, path) - finally: - if tmp.exists(): - tmp.unlink() - - def read_chunks(store: Path, ipixels: list[int]) -> Table: """Load and deduplicate the given chunks. @@ -173,50 +156,10 @@ def read_chunks(store: Path, ipixels: list[int]) -> Table: # --- exposure footprint ----------------------------------------------------- - - -def _wcs(header) -> WCS: - """Build the WCS by hand, from the linear terms only. - - Same construction as ``scripts/python/create_star_cat.py``: it sidesteps - distortion-convention incompatibilities between headers and astropy, and a - focal-plane footprint needs nothing finer. - """ - w = WCS(naxis=2) - w.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - w.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except KeyError: - w.wcs.cunit = ["deg", "deg"] - w.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - w.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - w.wcs.cd = [[header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]]] - return w - - -def focal_plane_disc(image: Path, n_ccd: int = 40) -> tuple[float, float, float]: - """(ra, dec, radius_deg) of the disc covering all CCDs of one exposure.""" - # ONE open for all 40 CCDs. `fits.getheader(image, ext)` opens the file, - # walks the HDU list to `ext` and closes again, so the loop cost 40 opens and - # O(n^2) header seeks over a compressed multi-extension exposure. - centers, radii = [], [] - with fits.open(image) as hdul: - for ext in range(1, n_ccd + 1): - h = hdul[ext].header - w = _wcs(h) - (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( - [[h["NAXIS1"] / 2.0, h["NAXIS2"] / 2.0], [0, 0]], 1) - centers.append(SkyCoord(ra_c * u.deg, dec_c * u.deg)) - radii.append(centers[-1].separation( - SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg) - - ras = np.array([c.ra.deg for c in centers]) - decs = np.array([c.dec.deg for c in centers]) - center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) - seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg - return (float(center.ra.deg), float(center.dec.deg), - float(np.max(seps + np.array(radii)))) +# The WCS construction and the focal-plane disc live in +# shapepipe.utilities.focal_plane, beside the vizier helper and imported the same +# way: create_star_cat.py needs exactly the same geometry, and the two must not +# be able to disagree about which sky an exposure covers. def exposure_image(images_dir: Path) -> Path: From b158cb98ea3e009b243a36fbee8ea3f00bdc49e6 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:10:06 -0400 Subject: [PATCH 062/100] chore(launcher): parse config.yaml with yaml, flag the profile's one non-portable line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bin/sp's cfg() was a sed pattern-match over YAML; the venv it activates two lines earlier ships PyYAML (snakemake depends on it), so it parses the file instead. Same values for the two keys it reads. profiles/nibi's `--home /home/cdaley` cannot be made portable in place: YAML cannot splice an env var, snakemake escapes a literal $ before any shell sees it, and --apptainer-args REPLACES the profile value rather than appending — so injecting from bin/sp would mean restating the whole string, which container.py reads from this file precisely so `sp container exec` and a job cannot diverge. Marked as the one line a new user must edit, with the argument for why it stays a line and not a mechanism. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- profiles/nibi/config.yaml | 15 +++++++++++++++ workflow/bin/sp | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 0f20bd361..97df987ef 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,6 +37,21 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# >>> EDIT THIS: THE ONE NON-PORTABLE LINE IN THIS FILE. <<< +# The apptainer-args string below hardcodes one user's home +# (--home /home/cdaley) and one checkout's src/ (the +# PYTHONPATH pin). A different user must edit both by hand. +# IT CANNOT BE FIXED HERE: YAML cannot splice an env var, and +# snakemake escapes a literal `$` before the string reaches +# any shell (see the $SLURM_TMPDIR post-mortem below, which +# cost a whole campaign). Nor can bin/sp inject it: snakemake's +# --apptainer-args REPLACES the profile value rather than +# appending to it, so the launcher would have to restate this +# whole string -- and workflow/scripts/container.py reads THIS +# LINE to give `sp container exec` the same environment jobs +# get, so a second definition is exactly the divergence that +# mechanism exists to prevent. One edited line beats two +# sources of truth. # --bind /local NODE-LOCAL NVME. $SLURM_TMPDIR on nibi is # /local/scratch/..0, and the mount is /local # (/dev/nvme0n1, ext4, 3.5 TB) -- binding /localscratch is diff --git a/workflow/bin/sp b/workflow/bin/sp index 921b95d8d..104efe1b0 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -37,8 +37,11 @@ module load apptainer/1.4.5 2>/dev/null || true # shellcheck disable=SC1091 source "$VENV/bin/activate" -# Minimal scalar reader for workflow/config.yaml (key: value, no nesting). -cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } +# Scalar reader for workflow/config.yaml. The venv is active by this point and +# snakemake depends on PyYAML, so this parses the file rather than pattern-matching +# it -- quotes, inline comments and nesting are the parser's problem, not ours. +cfg() { python -c 'import sys, yaml +print(yaml.safe_load(open(sys.argv[1])).get(sys.argv[2]) or "")' "$CONFIG" "$1"; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO From 07f799dbce42fc92b64c22472bc612268fcb550d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:11:28 -0400 Subject: [PATCH 063/100] workflow/config/cfis: symlink the 11 byte-identical data files to example/cfis Only the .ini files are a genuine fork (fixed INPUT_DIRs, NUMBER_LIST); the SExtractor/PSFEx parameter files and the mask_default data directory were byte-for-byte copies. Symlinks keep $SP_CONFIG resolution unchanged while making the single source of truth visible. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- workflow/config/cfis/config_onthefly.mask | 87 +----------- .../config/cfis/config_tile_onthefly.mask | 91 +----------- workflow/config/cfis/default.conv | 6 +- workflow/config/cfis/default.param | 69 +-------- workflow/config/cfis/default.psfex | 86 +---------- workflow/config/cfis/default_exp.sex | 134 +----------------- workflow/config/cfis/default_noimaflags.param | 66 +-------- workflow/config/cfis/default_tile.sex | 134 +----------------- workflow/config/cfis/final_cat.param | 115 +-------------- workflow/config/cfis/mask_default | 1 + .../mask_default/MEGAPRIME_star_i_13.8.reg | 24 ---- .../cfis/mask_default/Messier_catalog.npy | Bin 4209 -> 0 bytes .../mask_default/Messier_catalog_updated.fits | Bin 8640 -> 0 bytes workflow/config/cfis/mask_default/default.ww | 40 ------ .../config/cfis/mask_default/halo_mask.reg | 50 ------- .../config/cfis/mask_default/ngc_cat.fits | Bin 201600 -> 0 bytes workflow/config/cfis/star_selection.setools | 104 +------------- 17 files changed, 11 insertions(+), 996 deletions(-) mode change 100644 => 120000 workflow/config/cfis/config_onthefly.mask mode change 100644 => 120000 workflow/config/cfis/config_tile_onthefly.mask mode change 100644 => 120000 workflow/config/cfis/default.conv mode change 100644 => 120000 workflow/config/cfis/default.param mode change 100644 => 120000 workflow/config/cfis/default.psfex mode change 100644 => 120000 workflow/config/cfis/default_exp.sex mode change 100644 => 120000 workflow/config/cfis/default_noimaflags.param mode change 100644 => 120000 workflow/config/cfis/default_tile.sex mode change 100644 => 120000 workflow/config/cfis/final_cat.param create mode 120000 workflow/config/cfis/mask_default delete mode 100644 workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg delete mode 100644 workflow/config/cfis/mask_default/Messier_catalog.npy delete mode 100644 workflow/config/cfis/mask_default/Messier_catalog_updated.fits delete mode 100644 workflow/config/cfis/mask_default/default.ww delete mode 100644 workflow/config/cfis/mask_default/halo_mask.reg delete mode 100644 workflow/config/cfis/mask_default/ngc_cat.fits mode change 100644 => 120000 workflow/config/cfis/star_selection.setools diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask deleted file mode 100644 index 7c185c602..000000000 --- a/workflow/config/cfis/config_onthefly.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask new file mode 120000 index 000000000..9cd6a04b1 --- /dev/null +++ b/workflow/config/cfis/config_onthefly.mask @@ -0,0 +1 @@ +../../../example/cfis/config_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask deleted file mode 100644 index 18cf5db2d..000000000 --- a/workflow/config/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1,90 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_PIXEL_SCALE = 0.187 -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask new file mode 120000 index 000000000..4743a87f0 --- /dev/null +++ b/workflow/config/cfis/config_tile_onthefly.mask @@ -0,0 +1 @@ +../../../example/cfis/config_tile_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv deleted file mode 100644 index 2590b9cba..000000000 --- a/workflow/config/cfis/default.conv +++ /dev/null @@ -1,5 +0,0 @@ -CONV NORM -# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. -1 2 1 -2 4 2 -1 2 1 diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 120000 index 000000000..e755942e7 --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1 @@ +../../../example/cfis/default.conv \ No newline at end of file diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param deleted file mode 100644 index 09ad8405e..000000000 --- a/workflow/config/cfis/default.param +++ /dev/null @@ -1,68 +0,0 @@ -NUMBER #Running object number -EXT_NUMBER #FITS extension number - -FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] -FLUXERR_AUTO #RMS error for AUTO flux [count] -MAG_AUTO #Kron-like elliptical aperture magnitude [mag] -MAGERR_AUTO #RMS error for AUTO magnitude [mag] -FLUX_WIN #Gaussian-weighted flux [count] -FLUXERR_WIN #RMS error for WIN flux [count] -MAG_WIN #Gaussian-weighted magnitude [mag] -MAGERR_WIN #RMS error for MAG_WIN [mag] -FLUX_APER(1) -FLUXERR_APER(1) - -FLUX_RADIUS #Fraction-of-light radii [pixel] - -SNR_WIN #Gaussian-weighted SNR - -BACKGROUND #Background at centroid position [count] -THRESHOLD #Detection threshold above background [count] - -X_IMAGE #Object position along x [pixel] -Y_IMAGE #Object position along y [pixel] - -X_WORLD #Barycenter position along world x axis [deg] -Y_WORLD #Barycenter position along world y axis [deg] - -X2_IMAGE #Variance along x [pixel**2] -Y2_IMAGE #Variance along y [pixel**2] -XY_IMAGE #Covariance between x and y [pixel**2] -ERRX2_IMAGE #Variance of position along x [pixel**2] -ERRY2_IMAGE #Variance of position along y [pixel**2] -ERRXY_IMAGE #Covariance of position between x and y [pixel**2] - -XWIN_IMAGE #Windowed position estimate along x [pixel] -YWIN_IMAGE #Windowed position estimate along y [pixel] - -XWIN_WORLD #Windowed position along world x axis [deg] -YWIN_WORLD #Windowed position along world y axis [deg] - -X2WIN_IMAGE #Windowed variance along x [pixel**2] -Y2WIN_IMAGE #Windowed variance along y [pixel**2] -XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] -ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] -ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] -ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] - -MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] -MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] - -FLAGS #Extraction flags -FLAGS_WIN #Flags for WINdowed parameters - -# The following flag requires a flag image -IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! - -FWHM_IMAGE #FWHM assuming a gaussian core [pixel] -FWHM_WORLD #FWHM assuming a gaussian core [deg] -ELONGATION #A_IMAGE/B_IMAGE -ELLIPTICITY #1 - B_IMAGE/A_IMAGE - -VIGNET(51,51) #Pixel data around detection [count] - -# For GaaP photometry -A_WORLD -B_WORLD -THETA_J2000 - diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 120000 index 000000000..bb469cf56 --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1 @@ +../../../example/cfis/default.param \ No newline at end of file diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex deleted file mode 100644 index a9d1a906c..000000000 --- a/workflow/config/cfis/default.psfex +++ /dev/null @@ -1,85 +0,0 @@ -# Default configuration file for PSFEx 3.17.1 -# EB 2017-11-30 -# - -#-------------------------------- PSF model ---------------------------------- - -BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE -BASIS_NUMBER 20 # Basis number or parameter -BASIS_NAME basis.fits # Basis filename (FITS data-cube) -BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter -NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT - # or PCA_COMMON -NEWBASIS_NUMBER 8 # Number of new basis vectors -PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) -PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units -PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values -PSF_SIZE 51,51 # Image size of the PSF model -PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? -MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON - -#------------------------- Point source measurements ------------------------- - -CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering -PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. -PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error - -#----------------------------- PSF variability ------------------------------- - -PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params -PSFVAR_GROUPS 1,1 # Group tag for each context key -PSFVAR_DEGREES 2 # Polynom degree for each group -PSFVAR_NSNAP 9 # Number of PSF snapshots per axis -HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON -STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE - -#----------------------------- Sample selection ------------------------------ - -SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? - -BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? -BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed - -#----------------------- PSF homogeneisation kernel -------------------------- - -HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE -HOMOBASIS_NUMBER 10 # Kernel basis number or parameter -HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter -HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF -HOMOKERNEL_DIR # Where to write kernels (empty=same as input) -HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels - -#----------------------------- Output catalogs ------------------------------- - -OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC - -#------------------------------- Check-plots ---------------------------------- - -CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, - # JPEG, AQT, PDF or SVG -CHECKPLOT_RES 0 # Check-plot resolution (0 = default) -CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? -CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS -CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS - # or NONE -CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi - -#------------------------------ Check-Images --------------------------------- - -# Note: Check-image types can be set the ShapePipe config file, psfex_runner section -####### -CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS - # or MOFFAT,-MOFFAT,-SYMMETRICAL -#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits - # Check-image filenames -#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? - -#----------------------------- Miscellaneous --------------------------------- - -PSF_SUFFIX .psf # Filename extension for output PSF filename -VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL -WRITE_XML N # Write XML file (Y/N)? - -NTHREADS 1 # Number of simultaneous threads for - # the SMP version of PSFEx - # 0 = automatic diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 120000 index 000000000..501adb090 --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1 @@ +../../../example/cfis/default.psfex \ No newline at end of file diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex deleted file mode 100644 index b87275ecb..000000000 --- a/workflow/config/cfis/default_exp.sex +++ /dev/null @@ -1,133 +0,0 @@ -# Default configuration file for SExtractor 2.19.5 -# EB 2017-11-30 -# - -#-------------------------------- Catalog ------------------------------------ - -CATALOG_TYPE FITS_LDAC - -PARAMETERS_NAME default.param - -#------------------------------- Extraction ---------------------------------- - -DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) -DETECT_MINAREA 5 # min. # of pixels above threshold -DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) -THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) - # or ABSOLUTE (in ADUs) -DETECT_THRESH 1.5 # or , in mag.arcsec-2 -ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 - -FILTER Y # apply filter for detection (Y or N)? -FILTER_NAME default.conv -FILTER_THRESH # Threshold[s] for retina filtering - -DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds -DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending - -CLEAN Y # Clean spurious detections? (Y or N)? -CLEAN_PARAM 1.0 # Cleaning efficiency - -MASK_TYPE CORRECT # type of detection MASKing: can be one of - # NONE, BLANK or CORRECT - -#-------------------------------- WEIGHTing ---------------------------------- - -WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, - # MAP_RMS, MAP_VAR or MAP_WEIGHT -RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? -WEIGHT_IMAGE weight.fits # weight-map filename -WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) -WEIGHT_THRESH # weight threshold[s] for bad pixels - -#-------------------------------- FLAGging ----------------------------------- - -FLAG_IMAGE flag.fits # filename for an input FLAG-image -FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX - # or MOST - -#------------------------------ Photometry ----------------------------------- - -PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels -PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , -PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , - # -PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures - # for MAG_AUTO and MAG_PETRO -PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS - -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) - -MAG_ZEROPOINT 30.0 # magnitude zero-point -MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) - -GAIN_KEY GAIN # keyword for detector gain in e-/ADU -PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) - -#------------------------- Star/Galaxy Separation ---------------------------- - -SEEING_FWHM 0.6 # stellar FWHM in arcsec -STARNNW_NAME default.nnw - -#------------------------------ Background ----------------------------------- - -BACK_TYPE AUTO # AUTO or MANUAL -BACK_VALUE 0.0 # Default background value in MANUAL mode -BACK_SIZE 64 # Background mesh: or , -BACK_FILTERSIZE 3 # Background filter: or , - -BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL -BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus -BACK_FILTTHRESH 0.0 # Threshold above which the background- - # map filter operates - -#------------------------------ Check Image ---------------------------------- - -####### -## AG : This parameter is set in pipeline config file. -####### -# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND -# can be NONE, BACKGROUND, BACKGROUND_RMS, - # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, - # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, - # or APERTURES -# CHECKIMAGE_NAME check.fits,back.fits -# Filename for the check-image - -#--------------------- Memory (change with caution!) ------------------------- - -MEMORY_OBJSTACK 3000 # number of objects in stack -MEMORY_PIXSTACK 300000 # number of pixels in stack -MEMORY_BUFSIZE 1024 # number of lines in buffer - -#------------------------------- ASSOCiation --------------------------------- - -ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate -ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) -ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] -ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD -ASSOC_RADIUS 2.0 # cross-matching radius (pixels) -ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, - # MAG_MEAN, SUM, MAG_SUM, MIN or MAX -ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED - -#----------------------------- Miscellaneous --------------------------------- - -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -HEADER_SUFFIX .head # Filename extension for additional headers -WRITE_XML N # Write XML file (Y/N)? - -NTHREADS 1 # 1 single thread - -FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? -INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation -INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation -INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL - -#--------------------------- Experimental Stuff ----------------------------- - -#PSF_NAME default.psf # File containing the PSF model -#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously -#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, - # RINGS-HARMONICS or GAUSS-LAGUERRE -#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 120000 index 000000000..4fcfdabc2 --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1 @@ +../../../example/cfis/default_exp.sex \ No newline at end of file diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param deleted file mode 100644 index b251f5b74..000000000 --- a/workflow/config/cfis/default_noimaflags.param +++ /dev/null @@ -1,65 +0,0 @@ -NUMBER #Running object number -EXT_NUMBER #FITS extension number - -FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] -FLUXERR_AUTO #RMS error for AUTO flux [count] -MAG_AUTO #Kron-like elliptical aperture magnitude [mag] -MAGERR_AUTO #RMS error for AUTO magnitude [mag] -FLUX_WIN #Gaussian-weighted flux [count] -FLUXERR_WIN #RMS error for WIN flux [count] -MAG_WIN #Gaussian-weighted magnitude [mag] -MAGERR_WIN #RMS error for MAG_WIN [mag] -FLUX_APER(1) -FLUXERR_APER(1) - -FLUX_RADIUS #Fraction-of-light radii [pixel] - -SNR_WIN #Gaussian-weighted SNR - -BACKGROUND #Background at centroid position [count] -THRESHOLD #Detection threshold above background [count] - -X_IMAGE #Object position along x [pixel] -Y_IMAGE #Object position along y [pixel] - -X_WORLD #Barycenter position along world x axis [deg] -Y_WORLD #Barycenter position along world y axis [deg] - -X2_IMAGE #Variance along x [pixel**2] -Y2_IMAGE #Variance along y [pixel**2] -XY_IMAGE #Covariance between x and y [pixel**2] -ERRX2_IMAGE #Variance of position along x [pixel**2] -ERRY2_IMAGE #Variance of position along y [pixel**2] -ERRXY_IMAGE #Covariance of position between x and y [pixel**2] - -XWIN_IMAGE #Windowed position estimate along x [pixel] -YWIN_IMAGE #Windowed position estimate along y [pixel] - -XWIN_WORLD #Windowed position along world x axis [deg] -YWIN_WORLD #Windowed position along world y axis [deg] - -X2WIN_IMAGE #Windowed variance along x [pixel**2] -Y2WIN_IMAGE #Windowed variance along y [pixel**2] -XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] -ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] -ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] -ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] - -MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] -MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] - -FLAGS #Extraction flags -FLAGS_WIN #Flags for WINdowed parameters - -FWHM_IMAGE #FWHM assuming a gaussian core [pixel] -FWHM_WORLD #FWHM assuming a gaussian core [deg] -ELONGATION #A_IMAGE/B_IMAGE -ELLIPTICITY #1 - B_IMAGE/A_IMAGE - -VIGNET(51,51) #Pixel data around detection [count] - -# For GaaP photometry -A_WORLD -B_WORLD -THETA_J2000 - diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 120000 index 000000000..e2bf3398c --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1 @@ +../../../example/cfis/default_noimaflags.param \ No newline at end of file diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex deleted file mode 100644 index ff3b25213..000000000 --- a/workflow/config/cfis/default_tile.sex +++ /dev/null @@ -1,133 +0,0 @@ -# Default configuration file for SExtractor 2.19.5 -# EB 2017-11-30 -# - -#-------------------------------- Catalog ------------------------------------ - -CATALOG_TYPE FITS_LDAC - -PARAMETERS_NAME default.param - -#------------------------------- Extraction ---------------------------------- - -DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) -DETECT_MINAREA 5 # min. # of pixels above threshold -DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) -THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) - # or ABSOLUTE (in ADUs) -DETECT_THRESH 1.5 # or , in mag.arcsec-2 -ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 - -FILTER Y # apply filter for detection (Y or N)? -FILTER_NAME default.conv -FILTER_THRESH # Threshold[s] for retina filtering - -DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds -DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending - -CLEAN Y # Clean spurious detections? (Y or N)? -CLEAN_PARAM 1.0 # Cleaning efficiency - -MASK_TYPE CORRECT # type of detection MASKing: can be one of - # NONE, BLANK or CORRECT - -#-------------------------------- WEIGHTing ---------------------------------- - -WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, - # MAP_RMS, MAP_VAR or MAP_WEIGHT -RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? -WEIGHT_IMAGE weight.fits # weight-map filename -WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) -WEIGHT_THRESH # weight threshold[s] for bad pixels - -#-------------------------------- FLAGging ----------------------------------- - -FLAG_IMAGE flag.fits # filename for an input FLAG-image -FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX - # or MOST - -#------------------------------ Photometry ----------------------------------- - -PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels -PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , -PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , - # -PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures - # for MAG_AUTO and MAG_PETRO -PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS - -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) - -MAG_ZEROPOINT 30.0 # magnitude zero-point -MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) - -GAIN_KEY GAIN # keyword for detector gain in e-/ADU -PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) - -#------------------------- Star/Galaxy Separation ---------------------------- - -SEEING_FWHM 0.6 # stellar FWHM in arcsec -STARNNW_NAME default.nnw - -#------------------------------ Background ----------------------------------- - -BACK_TYPE MANUAL # AUTO or MANUAL -BACK_VALUE 0.0 # Default background value in MANUAL mode -BACK_SIZE 64 # Background mesh: or , -BACK_FILTERSIZE 3 # Background filter: or , - -BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL -BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus -BACK_FILTTHRESH 0.0 # Threshold above which the background- - # map filter operates - -#------------------------------ Check Image ---------------------------------- - -####### -## AG : This parameter is set in pipeline config file. -####### -# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND -# can be NONE, BACKGROUND, BACKGROUND_RMS, - # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, - # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, - # or APERTURES -# CHECKIMAGE_NAME check.fits,back.fits -# Filename for the check-image - -#--------------------- Memory (change with caution!) ------------------------- - -MEMORY_OBJSTACK 3000 # number of objects in stack -MEMORY_PIXSTACK 300000 # number of pixels in stack -MEMORY_BUFSIZE 1024 # number of lines in buffer - -#------------------------------- ASSOCiation --------------------------------- - -ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate -ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) -ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] -ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD -ASSOC_RADIUS 2.0 # cross-matching radius (pixels) -ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, - # MAG_MEAN, SUM, MAG_SUM, MIN or MAX -ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED - -#----------------------------- Miscellaneous --------------------------------- - -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -HEADER_SUFFIX .head # Filename extension for additional headers -WRITE_XML N # Write XML file (Y/N)? - -NTHREADS 1 # 1 single thread - -FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? -INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation -INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation -INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL - -#--------------------------- Experimental Stuff ----------------------------- - -#PSF_NAME default.psf # File containing the PSF model -#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously -#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, - # RINGS-HARMONICS or GAUSS-LAGUERRE -#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 120000 index 000000000..5a2469c92 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1 @@ +../../../example/cfis/default_tile.sex \ No newline at end of file diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param deleted file mode 100644 index 8ddfebd37..000000000 --- a/workflow/config/cfis/final_cat.param +++ /dev/null @@ -1,114 +0,0 @@ -# coordinates -XWIN_WORLD -YWIN_WORLD - -# tile ID, for plot of tile-dependent additive bias. -# Can maybe be removed. -TILE_ID - -# flags -FLAGS -IMAFLAGS_ISO -NGMIX_MCAL_FLAGS - -# PSF ellipticity (original image PSF) -NGMIX_G1_PSF_ORIG_NOSHEAR -NGMIX_G2_PSF_ORIG_NOSHEAR - -# spread class -#SPREAD_CLASS - -# spread model flag and error -#SPREAD_MODEL -#SPREADERR_MODEL - -# Number of epochs (exposures) -N_EPOCH -NGMIX_N_EPOCH - -## Shape measurement outputs -## Ngmix: model fitting - -# galaxy ellipticity -NGMIX_G1_1M -NGMIX_G2_1M -NGMIX_G1_1P -NGMIX_G2_1P -NGMIX_G1_2M -NGMIX_G2_2M -NGMIX_G1_2P -NGMIX_G2_2P -NGMIX_G1_NOSHEAR -NGMIX_G2_NOSHEAR -#NGMIX_G1_ERR_1M -#NGMIX_G2_ERR_1M -#NGMIX_G1_ERR_1P -#NGMIX_G2_ERR_1P -#NGMIX_G1_ERR_2M -#NGMIX_G2_ERR_2M -#NGMIX_G1_ERR_2P -#NGMIX_G2_ERR_2P -NGMIX_G1_ERR_NOSHEAR -NGMIX_G2_ERR_NOSHEAR - -# flags -NGMIX_FLAGS_1M -NGMIX_FLAGS_1P -NGMIX_FLAGS_2M -NGMIX_FLAGS_2P -NGMIX_FLAGS_NOSHEAR - -# size and error -NGMIX_T_1M -NGMIX_T_1P -NGMIX_T_2M -NGMIX_T_2P -NGMIX_T_NOSHEAR -NGMIX_T_ERR_1M -NGMIX_T_ERR_1P -NGMIX_T_ERR_2M -NGMIX_T_ERR_2P -NGMIX_T_ERR_NOSHEAR -NGMIX_T_PSF_RECONV_1M -NGMIX_T_PSF_RECONV_1P -NGMIX_T_PSF_RECONV_2M -NGMIX_T_PSF_RECONV_2P -NGMIX_T_PSF_RECONV_NOSHEAR - -# flux and error -NGMIX_FLUX_1M -NGMIX_FLUX_1P -NGMIX_FLUX_2M -NGMIX_FLUX_2P -NGMIX_FLUX_NOSHEAR -NGMIX_FLUX_ERR_1M -NGMIX_FLUX_ERR_1P -NGMIX_FLUX_ERR_2M -NGMIX_FLUX_ERR_2P -NGMIX_FLUX_ERR_NOSHEAR - -# magnitudes -MAG_AUTO -MAGERR_AUTO -MAG_WIN -MAGERR_WIN -FLUX_AUTO -FLUXERR_AUTO -FLUX_APER -FLUXERR_APER -FLUX_RADIUS - -# SNR from SExtractor -SNR_WIN - -FWHM_IMAGE -FWHM_WORLD - -# PSF size measured on original image -NGMIX_T_PSF_ORIG_NOSHEAR - -# PSF size measured on reconvolved image -# NGMIX_T_PSF_RECONV_NOSHEAR - -# ngmix moment failure flag -NGMIX_MOM_FAIL diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 120000 index 000000000..93e6e1f20 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1 @@ +../../../example/cfis/final_cat.param \ No newline at end of file diff --git a/workflow/config/cfis/mask_default b/workflow/config/cfis/mask_default new file mode 120000 index 000000000..0970152ab --- /dev/null +++ b/workflow/config/cfis/mask_default @@ -0,0 +1 @@ +../../../example/cfis/mask_default \ No newline at end of file diff --git a/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg deleted file mode 100644 index 4e4164aaf..000000000 --- a/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg +++ /dev/null @@ -1,24 +0,0 @@ --11.5 68 --6 186.5 -7 188 -10 64.5 -31 55 -50 38.5 -56.5 11.5 -188 8 -192 -4 -59.5 -11.5 -45 -33 -13.5 -64 -5 -154 --6 -155 --11 -64.5 --40 -44.5 --51.5 -30.5 --62.5 -22.5 --68 -9.5 --177 -2 --176 3 --78 12.5 --67.5 14.5 --38.5 50 diff --git a/workflow/config/cfis/mask_default/Messier_catalog.npy b/workflow/config/cfis/mask_default/Messier_catalog.npy deleted file mode 100644 index ef07eb032b08de4fcf508418524692e27f92ad75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4209 zcmb7{dvKIj6~H$S0z_V+Av_zF=Wf!F0D%A@VJ{fCEf+Q}Ge}K(fi%99nrA zDunWq77DdgDz+6YRTR<6s3U8&j-3Igj&%@bimd}oA2U?zlbKdM_uP9w(m(ve`$y*c zzCFM5y62pGb0i#Dx@=XB?^WN{Ks?dak_puMR|f+dThsACg+H)u`P4vJg+CbRYTc6H z4~^zMo@mjpb;;T0Jrn)RYq*17Tj|%eB$n*RU?!Sg*O7@Qw1q{{wyp$y?@C2G6Z9UO zJ*|SwZ1?}atFSw_CB3P=b8}@&M!PQkRQogQ9WJxR1Qh>397tE&bDwclCSdu1(AH6 z4(J=nqclLjNNy$y{Uf>I4k$$U+;$icOckc`Be|`!t%iZDp)MQ_Fa8fL{-b;tQzLp( z1zK|XR4X~i+6ssD2bS^mr21)@q|-L|e`)L2lj@%utcLWY`f8xj0Yf~cz08Nmysu}I z@{^g-7~8uRih{w2Z!HWBN-@I6ONt@pAt6OgeuG|8G~c*H>$1v0xQBsaDf~%ZQ5pK+ zj9#xu(RJZAt&dbL!Y~#qk)qmYi%L-!8?ToXsYiGI;Jwj}Di2{ei;a+?!V|0AWsRl> zym+Os9#lTUNERCeCHYt zJUw6aL%5fLQZM+Z%ud_-6Iwr&qT_?AH)5Y0Q~eRfvY5YD%-O+3yrf8%k}`2kQ5lS5 zF$!?}vw9tm%8M`a(@7E=S+VZ$>vz-ugb6G*QHqIP%nRl)Kw>5JofNeih%kx8CQH%J ziDe7S6Z!TxZ4=cXgaC_C$oH}-GAe=*UV1+mP=gW5Sgf4nY({85j5bP5FwOf$skxcg z`ew}uH3Z>41}X@U&gz10ej=di1-q4`b^hq(-Jdoa+e?F#{-Xl{Y#R$_Fm@dV%C0vB0w`Q0Pu=9N?r10aa z1fiP6W{_UL)7z_%e`NK-A8(x7p+->k!%PNhq!_BzUb;+rXK_@tft|MwtqK197itW` zEEbzBMTsX?^IN<3T|d4!Y5&KoR4Kw77ON%ms|DMaqS5$4ebivn6}t8M9V&xgsj&!i z8K@&*oT+IkM%@MMhIoJxJ1e4yYTsCB3-YHfhMU}t3w%~Us)i{I@1A>5M z-PR@B7L@6soDj-%^Uc>N)_z{GGL!si)iAfUj; z5chgC? z3g;oLX0bJ-x6I~7j4@k7dKc!~37<-tWQGvdGVllib}C9S%nNnqSzCL_=S?yN!a4@l z6EN0g|6g7*%bn5qnc^)}C2BrGlz|ul+q+-jSP-A>&68b=j-t$X$xCiGfE6@H@}#1%&`}s!}C4|3xRHS%ncEW8jNY^fzTg z8JXpd^5&~XtPx=|i*1pj&>|L+d5Nq!9g?sS0L0-4w@4|Wsp3qZx=0U6~aLVzCyqRm{F~3&hHc>Jfya3>+iC-sHUVhk$^KK6XW4(DKwegl8D|DgmX=&0@1}8gR!+><@H9 z+WAAZ9^qLAjuT*e#Csq>?XndVKW$FmD8h3Le2oCVEeJOk2$!g6{6`KDwc_*QyqTuQPCx0DJ5Z>D8!r%S;}ikXzqWafEL$@B#sK&c5wX z9YclwMt!znzPT4B5ME^9B?1bpOTA8Mbw^hpRnUe7=8}*^_$C9V2=Fdu-YK1(=oR%* zt|gU1IL*MfJ`;oX=kB5NT zfba?f-uzXxoJl@$H4aq@J{Kw+tpw#gl=-AJ5(Csd;(*Y{{>v*ks<&9 diff --git a/workflow/config/cfis/mask_default/Messier_catalog_updated.fits b/workflow/config/cfis/mask_default/Messier_catalog_updated.fits deleted file mode 100644 index 6a9f00096170565aeb7fab3429c2256ce56f3533..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8640 zcmeHL4NO&K82bxk?Xcg61^5lKgn(d%puWZoviN(qOUV;yoAdgbQ9S23R5C zGJc%mUqCidTB!L;nwUE8!IV-Emzk_l)0xbfKNvc7R`!19<}PlRM&quncFy*U!+YNM z`JV52-?M$^Ch6l6Vxt8?D|on!!gOJNj?I!|&(0GZIf71aOcL@OCY#x0H+#s(5YrnI z^eKJtg(9EHZZ{PQW|PAtI0|zm&n)h!0aEly_^(HOFCm{TKRZLR3ptjD%+_qlmS@ee z<+(|E-9@JuqYY7g@ELn0_VZRD>u0W-raGWHpgN#BpgN#BpgN#B@ZWcUA19Oa@dm9B z$n*vyu-NFpUb~tuWLRw`d!gVkWn@W$RN!zOPJ5+1di$@QpJ)8@^Rsdu5B7Y14xWX4 zS&}W&v4Ee)%@0rF0_OL1DSR_L^MwZY!H4;MT?t}*vH{QW{rFzUwCCjK3b`h`DO++# zcF*4nulyY!*XREJQ20EvxYZhU`smoGB+vRn9^rG1jKwNtnfqonipd7O(vLv1l&P%K zd;Z)D_w!ESQ~KdWJ`L|jpvgWz+iDYpf$N*;MLuI{LbS_2yUFFz!1U?j6XRTb_*Jc3 z1LNcR9`<9t7x=h77oYd^x%!)DEt2Npr~knA%k{bVyr<7KFN(fYZ|HOJc~4)6GS3UG zHswc^f&1s;18>Zq`uoOTE~2Q~q|Mla|%g za-h=JE%N?j_uw*?bnuLd%4718+1H`mV&Nd-x*iGk9UB7p&}C4$PlbH^_CmVeV&Nd- z=89JNK<*p#3k)C#Ou1FXUhvhhA8^?Mtb-}voMsd9Z@}Xk4L!{R2OH5KblZ)dP$=zM!+^*|xtz<#td_j+GQ1-&6ZGat%$AEx3wsX?S2kT*O z(P(-PbvTh(obuFT5Z{>yK3MWT-;4Ig4*6`)85)KG1QE-xCN>W;=W>HMh*)Vk)ja6M z@syJyu$7@FVSU62dK5DdMBK3RxExb`isC~K0*QVOCn9^$ zbp(OdhJ(xyr{#o-Fcz?5&!N=z7J3m$omj$lnD%Km2LvbsqXagOsg}PA-bQ25LIe>V zY_+_lgL<|@I z07LL-iY?$qV<2@4R4q?dbnI3#Dr+G=U=KWr-9NxdBlb3dj*Wz&SaOD!L_8P^mIHNs zdT16GHv7tAi0Nvf^(agkZEeVSNc<=n#-hoUN=9c0WOjG+Cb?dqlKPu$YDEYqwBwk(z^jfXBY2yG-{#>NlWyeBp( z@5@r2p?zSM!|o1OUcz)9@V|8#^oBHrL)mTU;l)flYcOm;Vdr^9f&XuELZ^<7!rdZ> z7d$Tc4;qc|FXe?ZLlMI-okBI`F@A;7=F| BE!zM9 diff --git a/workflow/config/cfis/mask_default/default.ww b/workflow/config/cfis/mask_default/default.ww deleted file mode 100644 index c2797f904..000000000 --- a/workflow/config/cfis/mask_default/default.ww +++ /dev/null @@ -1,40 +0,0 @@ -#--------------------------------- Weights ------------------------------------ - -WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) - -WEIGHT_MIN 0. # Pixel below those thresholds will be flagged -WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged -WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels - -#---------------------------------- Flags ------------------------------------- - -FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) - -FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels -FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs -FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits - -#---------------------------------- Polygons ---------------------------------- - -POLY_NAMES "" # Filename(s) of input DS9 regions -POLY_OUTFLAGS # FLAG values for polygon masks -POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks -POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? - -#---------------------------------- Output ------------------------------------ - -OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename -OUTFLAG_NAME flag.fits # Output FLAG-map filename - -#----------------------------- Miscellaneous --------------------------------- - -GETAREA N # Compute area for flags and weights (Y/N)? -GETAREA_WEIGHT 0.0 # Weight threshold for area computation -GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area -MEMORY_BUFSIZE 256 # Buffer size in lines -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -WRITE_XML N # Write XML file (Y/N)? -XML_NAME ww.xml # Filename for XML output -XSL_URL file:///usr/local/share/weightwatcher/ww.xsl - # Filename for XSL style-sheet -NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/workflow/config/cfis/mask_default/halo_mask.reg b/workflow/config/cfis/mask_default/halo_mask.reg deleted file mode 100644 index c44f25167..000000000 --- a/workflow/config/cfis/mask_default/halo_mask.reg +++ /dev/null @@ -1,50 +0,0 @@ - 274.66813 -1.25966 - 272.54579 32.47406 - 266.21222 65.67579 - 255.76731 97.82190 - 241.37579 128.40544 - 223.26462 156.94408 - 201.71942 182.98775 - 177.07997 206.12573 - 149.73486 225.99312 - 120.11532 242.27660 - 88.68848 254.71937 - 55.94996 263.12519 - 22.41606 267.36151 - -11.38436 267.36151 - -44.91826 263.12519 - -77.65678 254.71937 --109.08362 242.27660 --138.70315 225.99312 --166.04827 206.12573 --190.68772 182.98775 --212.23292 156.94408 --230.34409 128.40544 --244.73561 97.82190 --255.18052 65.67579 --261.51409 32.47406 --263.63643 -1.25966 --261.51409 -34.99339 --255.18052 -68.19511 --244.73561 -100.34123 --230.34409 -130.92476 --212.23292 -159.46341 --190.68772 -185.50708 --166.04827 -208.64506 --138.70315 -228.51245 --109.08362 -244.79593 - -77.65678 -257.23870 - -44.91826 -265.64452 - -11.38436 -269.88084 - 22.41606 -269.88084 - 55.94996 -265.64452 - 88.68848 -257.23870 - 120.11532 -244.79593 - 149.73486 -228.51245 - 177.07997 -208.64506 - 201.71942 -185.50708 - 223.26462 -159.46341 - 241.37579 -130.92476 - 255.76731 -100.34123 - 266.21222 -68.19511 - 272.54579 -34.99339 diff --git a/workflow/config/cfis/mask_default/ngc_cat.fits b/workflow/config/cfis/mask_default/ngc_cat.fits deleted file mode 100644 index f51546da7fc88ae1ca919a3c13882fe1a7df6768..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 201600 zcmeGF2~?NW_CJoJC^(`d4mctzj+u&vn#6MsJc>gOJW`TaIFXu~17=1VYN#lVC{9`A zfF^3U*7~iL>ptto=A6A> zdpOfR16KEE`t~vd0zw1+=3oDSCjy4PGB*B|31cP(B)k&PwR?XnU}D1HvBL&W81^^i zL6Y73_w7F5Kk(}aeuF1W7(69l*x-c00SQya4gY&CZfb8apu2@1{}x{x@Ebd6%#h&| z0$z!CaUMBl_}GafUl}{m&EzltWe(_X_Kx@u{QCc;;Q#PHnc4ra>_5f+vA{nT_{ReO zSl}NE{9}QCEbxy7{=aVlnoe5X`}7VCXezt+?k{97v*};7dLm%R$gzVbObJLBJY@9n zfZ?wt=*j6{iu|Ad`Ty$tx!wQ&^JkuqxBdJ1bK4N`8$EpNO9>-rK0eGf6AwW7|MZXf zwfh@>ZQHi*(EdO0L;3&ouWxvt=-!ye{}0v+FHLx5(zt+egC`6gGdyAVguh=ebo>9s zukHU~Kj!!MT-<2;ckOQWim?73Ux)vLpAMP$k;6w1`_Ekahe!AB&i-gRZ1_to=dbbS z7W_Z_FXqSo_%HGcCVw;?JYndVkz)e_{`dIW{}=i7kLhb_`%M_EZS=px*R@a7GnyY( z)%@>&$B*#2{4w;uz>n~0e*Zmu+P@P=zCQd#EdBp?`Vv0P@4tsn#|z_&`ETIU{Qi6R zI&eI{JaQ~2RsQ$()BMDLBmVx$ZXxzixrB3$GW6Q(0ez6|7tpwsPev zE9_$Zz2elE+^{4Mk1&5CT)25il=*6ESZeMuO}OIgE6V;pk$!lZtK9R-la(jFLG{Ce zgM&?@R`xVmw#zfjG&lE{X_QTFpo;r{VpIYT4-XH^@R^e$4$l1-Ges54DBmR2@g6(K z08_?cRP44*R{!E(VYY_{OED^5k1^7gTU-U;EThuiC|UO7iAJ^HUxETctHr1ZQx=%Kcc z>e6aXiYz-~7y%<~Eu(H>f;j#D?66GD)H3QnxkmhJSO7Bx5rmtE)MtC1F!n(?CNwx6 z;tccqBy1<3L#$~8U8p0z+%$}G(2~t@Ljv7JmJ`7|UZzeZ5xO&1b%ycChkTEkj*#08u z1Gi0Q8in>3eKv%}$H!}6b9P-&&wXxi37FF|7NCiRz)1zJ){t5u)GmYsGGQu}SpIc} z+EXEvLmO30W5q_H_DtR9YJ`@t_>XjPZTfC%JWq3p#cMppwecNDq$VsG(^*_wn#fha z)HIeL=&r$0umx2Efq1;Ph{dwS(gFj)v=LVUeA7rfGL);JKIvl_OP3rL*Sg+(pRn$*MJnDbjw~neYf%DumDycd@N&)SDq-I(u$P>({Ll>$u(-1_>-aM z64G_!)`wXOI=OASZgW;xbaXUV2`1ledNEDxO4&kfMYost6MV#u?>}Z;n8*9hLb21= zmuwLVf@NeOaCbtijrhQ48d>r2VwZn8cYAr1P!uV4Rj$FVgtp*uJVfk5UkD6Zx%baz zJ96glRDy`zjm=k!9l5ZYyS-$Ej(6nyVQf>NwTx|Gy0iVWVK;Q|U>RGUIO51R_b`Et zydApiG)Hi&NH!l@A( zO!(x~Y4u{Sr`&$|!YCBI8<}p;efo8Vy&gSGyf%(1pUvGVWI1sCEHIs*7+BJ^!xPi^yWs?kwXT2H@Q-yKogI z{OOlyufM#Bbs^}Y%xBxA^d2rOj170uDT(r4dFs#ZHw}VLRfIR9@aB=544n)l%2&Lu zPLEq}OhMa+2`o_#eNdfl+l+~@M=n))giyX`E!AUMPeuywo1((zEY(W<$=y~a!a-E5 zc}{sfJH?2@phe1w3bD$hiy(&F$z`8CnqVixkC|^XXUAn9*lk1qQqlM z4y#=`dq{mEb3}!vUvc2F)&1vh6qk++A;*}glV3rKn0^Af7(b%z3;%6?Mi1~H=*+af=O{HQ zX$d`9*~>RgXeRVxl=9&FRPrFN^Tq2o!cwnyFb z@nM5Hdoxnjw%$|wjcCl2NGz(DqF%=+L+Xxzh^DANI8s%s(}yEjS9#({*hsV}*g_+% z7ZwyY3=N2+;Il-%S$Re<)CWWQEK$Gx1tapCpP9fV>h}#XI--LuK#3$$uD+EK7K`5U zJeb-{fVM-r&U43DeN1^`(a{5g_I`a#zAX)#nusMDPH82B4(to`9x^iE7cd8~e-|5o!87jd<{ax_ zY>LL4Zi$)d&xc)(XmaYbIQr))HZtqdNG^2ZzU$*|$W6YSp*F>SY{bVu}_=hRQ~-uHgymaniOq8qk1M2iPwFW{DQR z`N$^K284(Duad8l`Yo5Alcl5Da|HRXT-8^!7`Hy>5Y{h1Nxm^f%e8@GzgHeR3ihx> ztG*#(f3*SJIYO)@n&%!<2fS~SE;^E1qGQ{SUJ4?0IJ6#HEPmTjm3c7J)=h;Nj`%2% z0vt0(sr{JqXQB#+L(B9wYJcp<-1b?@<7iQ#eLrU&NfZu`f8YF|N8}0agIKx8cU^EI zKW<@o06JG`m*ioOTbIv%0nrle1I8GKR{!Y2VTq0vuZ!ZXGu`13;=f8w!Hf+kbrg0S zbUNpI48Q~o3R+fY(`cr6!Ek7sTK0J@-{5L3IjC!8p z`#;jTMKScJ+h0&$^g1RkAfyVYO6WJy>i4+D+!rz-tf^2T(|Q_RF+*uZVvRa|M46l* zcb8*eJ$EX+q09Ry@#Gr2Efx};blbOtv@*o0l-lG1?S}A1`QpK*Ta1)-3GeK!`eEFM z#dQM`!?K-P*B)b6t|VaHo;V^))opITsKnS3kr`v`Ow9Un!IXLQXm(3g7}NRT#98z> zS}W(b`8BvdWnFr9_O`#rkmLmu`V)^{_4nFE$3xsbOs`(Y>@zi#I~@9=qu)KggFD8a zX_BhSVS4 z*Qj|!VNG}wa|if@BES^=e@IhNYdo0;J1QEJyeLfa0GkYLis(PmRn($4*2IGJmEHMIesT(7)XvvquC6tF#(z=(k#z zSF8;C`kQ0PFFYF;uZ9KL*U=4sjY!TQH%vgke)IVR9->*f3H>hH-=RZ+Z*|a3#MJt+ zdwp3fBw6B7LaR9S*q^6dNW;ZM@c8z_imnD^ijm9DseNc-2sn*&WcD<*FYgm}WiMhn zY4T+Id(2=-IVOybj&>G3e?Y_`)-iE4#jDR%u=A%JWaV-hM}m*j|6UOrnIn1f;7Gdw z10P&Td`vO<)4O&7G9>t|Zu^r96LM~Ux11ftQeIo#$DUo#k2`uMOj(zob8l)h4hNPp zwMHAULoYS7J>Hmf$o^uW7rC;80O&uw)09mJv6h&c;A4MLn853(0wN?W3dsrIv6jWA zQUgjHTkOmXv8glB?e?wYyXq}0iu`piWr^vr&2yfa@hwLoGfjGDuTu%YAXa;S=6jJ! zR^G^s(1To=^ih%Xcc~>v>M{ZFO)$G=g*$9{%70VuAF6igpKhW#TH{6a}MW8N&tYNZiFeF zi)8(@0X*RBB&2gr@0Dc`QYB0+F()#`SoB9ai$yg{q?AjMWyTG3$5;xcaHTP8gC5$q zDR-h|={YA{Vl6QjQ(DhbyWMSW?zvRy^Ftc1_~J<0SxDQ?vzKrW1NAL2KV-5jlhmE6 z=%AZdr;k&m<5@OxDp_p)%|o)x+s!$WVfCS6am89T^02HNw6(+n_@zw3vxHRhNNrv0 zRK8Ww-5M6OE>?@%40J^ym|{8m>nA4-5A7&REb4bzdQ24SyHottWzz+9fgb6oIbb|u?_g)9qafvV zZ)e2)W<1`q=N6aR?TpC#gbbvarv2H|C^NqXG1LZHda$Q)VEsMj0W6kCYn<)WTn zz`!?_SiVt+I}m%r<)~#?Cfr0K(!Qd<3Sdg`&9 z<48l{x5U^KD116Kf~pqeF? zVryW~s+Qb5Y_iAFuQQxM>B_Ck@=M9a24tQH80^|tq+w#TWLdNrhi)97Dc(lzA~lH< zOu8RiwY9V3J*XF3j6L`EAWviV(@VLD0lq2LTw5<@Kx~pr{Z$i#otM`waN~IA**1pN z{U#F%D1zQy<0(CT*STZ;UH?^L%nh9`1yhv3)cpA;j__PtS3wGT^b*}$vKumPt&yd_ ziFU;ps^le#w_=;~1Rly+BBN6)lA!5vtp>F6ISVktb@OOJ|?bgn+ z;cw4!n`3P=dR|vMv73M&wD;F0-!&$O*xdDTiT4_3tNPX7VIJ&;wU|3pi~Ep)p@byFkE)v%m}`E+|(U-O)ig%lEsYu~Qj z66cgdNI@id4;E{Y<#@xDH$Y%r)=vyJwm$c^yUP0BGr}fxooB?LPfjIAZTL1vmf3#A zfMUAJhEu0y*)xG0Fw7%!$bIQmS0~)rV(X7y&w24mt~>pc`C^(Z3tfl@K1*Z{36y0| zDh`-TGU0|kvBnlmN5PPwn<6tONS57S)>Q>1uv)8%IUPW0T{5r7$g)`QLL|XQmEt3^ z%<<3N_So>V5H*qQp%P$fiH*_GvaDAgwJ7>bv2kD@S++qbW(vmOacrZgakZG+t*%n` zu4unexJzuru;!idfJ$hqZ^G!|-L9%z%BF^qqW6|xjnQzFLE66GBUG0CX@R>B+JyOf zEp&b47q-XylMczUCFx@(Y+7DHRL6V_QZ)2Uzg@`r3K4|g>f!bMW_64Sm`akP3P`#4 zv4TGJPE%U!>b&Lqv$u%`=n!BWEoJle^|GA*8SYR(L=~*FYoQ8$pbrFWmDItK*oLE+ zu<>89P9+dlLJI5wi=9h#Ds{~xN0IsU$68_y7+}Y{k|OMKw zu=L6V(o|x)JvusPBla3W&~DgKD@T?>XF$O40eZCz)fh(;;06u-0|c(m>Al=Zqnasp zbV(QOkb|WXFrg)~Zv^K|C|R1Lge7)j_F3Aaki{|~`^hyad9=cCXKtO}vM* zwk&o>?F7dM6L_H14QThg>mn<8JNI;4BKzWTr}watEVdUR-Tiz*&I6;64aD}?b!&~< z9UX0~MDs8R*q#M5q}Qaj959UIkt%zxt(Rrb-(lt0Q9ECYQcHARN~{0jwqpA`yE|=C zgC_!J&xkrXW51Z?9<=sQk_kgnfu4<9Vqe|nvNDFHO3;Oo?(3hZ3?xKSL5L>abRm_ku{(-^8h$Y9K`71mC@MkhL11DIlgz!ven~B?0%%O zF2(etoF#39S|aDbc~U*-WU;L&Ei{mV}aIOD9VRF775(bS=sjW_5UyvHmK2UH0c>b*$mQ|TD@mR!O{1U5hOGPf5d91NKZ9)}DMFty2*IXG4jQ0Xm3 zs!QOIO%vOL&!M2*0od?Vr|zGA+Pesrun*@s)gx2fCj8``y|VJ8*WGE#PgdlpJzz>m@mb<< z$YlE(PRsnTwN69MLN{Ec*R4s#g%R$J^u>tb2s zh|X(6A4~je_BLa7Nd}&!oWOQV5LTW5rJ3dh2g~v}ngCt2qfVT;ul68wMJ1R=o?R$? zOU_)eawlT;y5)=3zYAE--@S8hPBC1p))`fZGBcJ@;tNjfdp5rRtVz;{E zb8mB%0y5BPEM|6m-<;N_TK&Sv6ww!3&mfSIEm3fxjOB7C?Ec#-T+e3$g zDGI;N5Fgw4>h zLEw^KQmMpu*uZ=Wn{5zDpC!J3ElQkf*@ax7Ln?nvXUD4a6lI;rCP0ca69v9S$dt3F& zG%uuy9;A$@Am z;yZg8DT{r?4c{RfSkW55lL?I@!ABXG80G4zf+c>$QLm~9+=on02aHj5j6!Z=tnKM_ z`S$eB>bi9)Lb{{s_JPb4I80F_cFU^M|8yfQiinU^-yG{|gyEtHi;=1r#a8OsmL-1J zzTP;3yyXozicNSLBTzZaq%+}LxWW{tvB6!g?gZ)?=mU>=*Jah!KT?a*%Y>p2j>xKK z^_cBJHZ96MCaXTVhIyb0l)D@!Qjz?Fw%TH6a7wLOx8d%VeSUA0^3?~IbRTpEJ%80R zySS;NnkCMRJg%O>{tFDG-B9%Ve$f+E;58kmKOs+7^->oStGlf;7@lW|KWPR`?zF^D zJ!^?~KJs>jC75Ezd*F%ZjdTQ^Kh00i%^W4`%`w=ONu)e4I<*h0tpPwTUtDx>2lzWxK@+$m30wRI+og=kY; zxEv>|b$FejGq5wiT~M=eP6?jZJ9-qlbZ5jzLCO42}N;0f4 zqa`k$xGc*e5dlYQ92a52GF7gU6zzr!J7SDo^_H=-J28%98;xCFd9H9U#aZk$4>_z4 z95U0hgCm_G2lf##Duohaw==}%0gL<+E-qoaB^dJ{C`J*`U0i}2YT_6Ue$cvH_{iH{ zaN`Zm)4@nM+BC>h6=SM!1KI+dB`(!Kz7T8VT$m6JH^rse39`(;V%f+n_7XBiRo*|& zJwK{h;%xRbXDH4p_-j|frgow7MHE}QmqLl#lT|V{Fo9Czc*G6=e#A;#LdwMV#UAVz z00WMVLYAFAnh72hK^L$3*;_F1;kN|NdQ|L{RSDv=My7kAc?J7oWfycZP@g_iT)~cn z|L|}M7#;jqkm>OTflLUrmMCsJP25IK`XThI6NANaWF=9Bh^Dxj8822;i8at(j1~+iaEnlm{5G;vN+IGxWdvB*RW6c`?$xbBhW~% zZAcaOa5R#{0y==boNOP&egme~TK!*JCW|8Mo5E`pc;JCn*#nygwrh;K^lTgFrRK+} zMbgk_iR&Gs!~>sB>};R~0Sn&qyPl<;A|hJiMs&3J6Ip!lAkn6{4y3PReH)At;A4qj z-`T6G;-qWPF%M>X3jyQNM+Yt} z2BstG95{w!!1#d-4W`4g50*vOmjpsQ;kf-~kn}IVh35`nYKg1ueVo|`mbn}9ZTKZA zY!3%NOS$vdVQ~wg4NY~+zJu=bJkmcvNx;CgLa3^bBWFOWW9+tHqN@KxDIPs~k#ctn zR*9d_-{BqxOigk7OrZMc&9SVUKOwz?ZJ&zHLMAZMdpMTVA3MCzgKT1odoU0P)DeW{ zl7a4ecqrfOX|7gjiTnFn$#y+Lc>rdyH=x{{AN4|$n99-_W7HcsQv+Stx3cu6I%;aE zMlPh`(xXYL_@#0UCO}HKcz`4CDQlLwIwebbV0BWb&X1g7g~lMH1&fP6en<>ied#eO zR19D7>ci$CJz`dgCop|N3DUbBeK;_oce`GZNgI6 z@DIcg{jfx-Jrz_S5{TqEmVzqA@4mhqQb}YFPhI^|B}T?x(va&hZhLrcD^`9bOHn9@Dl-;3Rj_P25BUDX)T_!7 z@dH-#K_GySDa&F*Hv}sjvMHLxQ%4_oQgU zu5y(pMsZZ7oH0h6MXwED+Q^SoDGzJCi(~UJpGN9^cU8`ludQb2Nu+K0pgb{RP%nZ` zU$`v)cxN#J-35tdVBXj~SRh8bk(Nt2t~y}Cnu`)_k8gU|l3(&(zpVVWr`OpR8Dd1opm9R?MdtobK z(ArkKea>m}wmz_&N>VDX-fwS5!W|4L@GR;3>Kgf4wTmJE*$sTAtc>kiuQP$XjASEM zdA5yg3TK~pHRQ@?7OG4%4-m)=-DQ_DJAuWrwpF}^+OYkK zyG2z&BJ+Km$UcNAek;<}H?B}N!(ugFwW5^rrR_K@Q(1}=C<;oBHnIA?*ljAntt0Tz0rOZu zp?J{iDfS#B0T|Borrn;*+vrdbOuOcZ0_cJW(pCHpjglegKOjN7^05WQYHCS04l{y> zPq8|TbF8SZy=NY>P8HcdW zL)#?A@otciZ0E;E7ZSxZapG3IF%4*MaRsype6xXp{?bpj1&X7Eg*%sWWEmpl*ih3MVrxF5~V)8u#DQJMwe(}Rp zvKewpFfy+zWsPp5*8oQ(IE>Es;6KWxCT=`xR2MoYjvgY;><%fL_m{5mh!}3GzQ#iAE1JJod zt<9n0$Hl&^3tPR$TiN!kbKg+F(AQnotQF-{MPkMaz&t3%YQ0=bEo-1JqA;+UpWjmN zB1uik0hlRk;cin3Qa7-L_FV1S-m?7Vn(Qc>Tv;3ElRe?v0_|H%)^2-Qdj9+s$0;+d z-7!b}k~opqbKrv#(T8MdjJKn~lyTG^vr&4UxXe5N-ID%sg~A`_9a2$(dH6p_6B|!F z&p2RHl<=9MS_OSzpgs}7^z#(yIXIH7p2WZ&Asp*0Jzv<%eIYZgxuJp6t`=uD39TjT z^e&c-cDH35?3Y>(g47cj=fF@mu{v12HhR4;hm=1dt$$^stnkJU9NJu^_MXY&<380G z80%8|-85AR-39SLi74s6c$!nUbuph*VH~w~20Qgy58}4Zj;i~0uq@lB8f%+M%GF6o zRMl}WfL3d)t^X$x>aChO13WpHkfosHYr()K?-mzO;l}=yC57awjQ?8$|L0pJ!qBFSJuNWb6MQ= zq7opQvVPz7Mm6MucLIm5@(7O&sj3a;Qz)*(p~2I8#jj6Ru^q8LSHcw9 zww;!Y+g06nTg|OdB zvb1m(m>%#neDqOKjkJDloKYFGVB!JBC^0w7{ssX9hPp2dh|iOCk+-~z{CWVzN5Go# zvTo=;_ARP)mkrv*h{qSb$%BPWJnHtp;0(msFNzOl+MwTM;{ zErUF|Gj#B=WFss_f~v&2JA(iuM7F`V1zI^6X!u>_g_|V^I+~J$^X0hf#IXX;sfK| z&Tgn8pFV@>`3=_;F7V44;_Ql7dA`n$YE%(NG;dFKx9mo5rpfAqmQj4s2P5PBSg9Sq z@F)hmnz9j&oRvB~j-{~rfh|Jq9XKbAfYC7)ICPY%j&TkWwD$vRC#dmQi@|GT6Y$8p zAWI?E!H^6MLOw3D;ADln$~M3K{bJ7W5i6T;1U^VpWpLq=q_8U+4LWVSgYA}TAl#s^ zY}7DPRT^@ib@5>6$*E7T!Cf!Z*T5QOT@Tydc%<<*8W82_WFu^1%)sRh6xXex5$1){ zam|E~vT}_O4x@2{l1c!aC4)*YlmSQ)5>k``p;DX-K*|Iqw6hxr2CAB9>Oe^)pxlAe z>H@|}+Q02eQP`wqEjt|@>_Z++YF1EFP(0Tqwh6YcOBY2l4_FLUc1NqyS8^$6=?j-l zwr*6Du+fexI_R1_z!7gu6j7Xv1Yj>G8}XPsoCo7f65A9z&5d!H#fC0PWFV9{_+ZIl z>MqEPC{cC2n66LUY4w9zWU28;bU)N3XtD3Xsd&!-RrDmU$%{cMH6nsbu*I58J*`ro ztKhEEEFxQ$-{!}jW3f$NU2|{=QsDrlaWu^+G?rqKNCYSczkC(ryn*t)qubn_I!66gpVwL6)~b_+}IHfbEq>z!7!0gR-8F>s8I6a6uP2Pj>o#flse zjlkUr0^nP+B~rgp7;gzW+q7k8Z()^c6bexEh0B&96@-QUl*FQnDVyI*P!iMgO0WlO z+bl9gE&Q2p`Rd?rj{CPS&3d?JVagVnc{!&b1=i9fT4ATT8BXjH(o8bpBZQV{mB+Bo zPC}frE-kaJtK|0jun+j4N}nS+8!&OkZ?$r*PI-#{xDQmKizVA;`x-Wm69c+dt}X7G z+Iyqigw_d(GN|7=GBVAkP1$-xiVVUD_Y!TxW$VShGN_9_^9_5L5>j&31w5dRk)VTs zL%x+ThXE)ZFpqx_EBa#ho=PxMT7dlkOvlQMiNSbFd4J zXc92!1H%8cMC%1J)Uq0Fc-t6NV8X{@?3*w$A!V^`ho>HV2k%Vy>##Is+p?)@9QI8~ z3W%tJL2KNTYsk?$VA?K>weRScD0GQ-Et8Gskm`i$I)d6^Gqwo^VCbQxw8IgaCb;?d zkPy5xLmWBx4X5@BXmZuAZgVvr#~LA0M{?UX>s7qIO{%4||MaeGh1(0zLkC@lj!`lg zcZqFC;lS1d5Y+QxdZBJq#;WaWv6+Yc_9%* z-Vgb`k9-{a`!`^9%>%7r%#i!;0l3r5eAyD~>|WpjGg`7^%5m8oxg4qzOnp#CoQG*X z@eOuEE+Kt#eS})xI+NG?6RAXpf>ml9kivvyBvW?4?b>lTg9fI$MTJahrD}h)f_$q@ z7@;ENd+8Nf>;n<5^Bnw z^4ureOc-0NUdPq|iDl(NaJ}?(lz50VBrQX|j%PNYsgALb6Vr0$;~MmNS8E8Fm#mD8 zF@LFo8>s@j<~v~^e3tCoF-JB<+Jy)JX37wx)2HBwW;GBBsHY3LkfOrA|CoCR5KY;! z;cj&tcOwBzcNd*1gxdeY1)&mPmh6P<$MNn%i>B<Bq#ZYhDiiad5(3o! zcO!RhK2%lK52R?nbiu&`|JlPC7`w7_+smpJ&eYrhQec5DmEzQt&+2mjkxIIBR*~vh z$x#hm;)k@Hp1o&q&H&Xc8H%uMv}CH=4WZqS%i1e)*n~UTKv#=-{gUZ&!&A5`>W_dq z4Mx{b5B1gZg8tsJK=4x_`-IzGC*5^K!F4NUx2hjy3A z=L|3G&?LFqeiyuFFbCZQ8Hly*a{aI!iYXivBW+FDrT+!{Ev!9If~pObVVzdVCU{7r zga=9-Id05hj35VfFQE19@?FoFe&Y?`>*5mer7;Tct6-GnUqQY#;^}ni+4q zc4!#KBPGHnR}ii6oCERTDo_77MzzF&c0#(6;`Hfr4eUl3)$xNa5%zRX<%1^>Vn9GC zs2>%hd@xu%EYbO`2q$@dXNn+gJMn-`ENLQeo8WRW>T4sL z3CK*ms}I&Q^l&^0F%5GV6xSufH>IfYxbHz#{3$-n`=@0PF0w%}id9Q?ZImyYVrR94 zV9JQn_w72pXRy^lLH$KKIn=u#t~V& zNCv&79|XyzDv>yX*JQHI-e*Te%0SsiaVGllR$C-8W~+CfFDPLlO<;aLx5a9QL6SU`%3tR9~D# zpb{X$>gd;3z$euZFbpgbhe^E=$!;FaZrPs?E^s%oZdjbH?lH?ajKKo=K1)U}ub`HF zvS0|Bm)1AIK z`gj8~?TQzd-nzd`P?hs!kI0%8oaC2=leRj>dVCOTC&Pp`8W7N0vS*$BYBOGt$ptV? zcyG5;?XPD9dTvT_s;zyQ+^Bi|S%C1No)%*1II0>;|*u8^ftKh{Sy zA=;9?angP>W^BX*%9*k^)?%B{<1W%}Fl9fyQ}H{_^TRLHBBErJw~$TyC$gj19#K(; z(OuNyNQM;fz_mRe94R7QP|cKm=~1F5*BBpaf+hQ2sG}r?lbz5dlft2Io&8R!#v_?& zDz&J-SeoCNH<+&xbfwO*&z&5{Z)G`d9w=eSn7l-}xbb|R(IW!D@0Yk!^kGE?=0Tl-l^Um3VRPxB*#4C}t4vG-W8i(5z>@vnNLLf* zq;T)R@ew^N!hRR2PC(ZkY}D3`PWLY9JjabEi$zD?QgnR~#dRb{zdzJo`)zBsIx~$v zdR?u)(wSEl;A6>WdnVhTqanjuWJXg)<9rZ3J%Asyfu8+3Sk=REQ%KMTdiMAIsvdH% z&_l<1^xNr9Rh+q`7Im0$AZ2oQ-gOqsz@Ed47@tj^YyblVFT~_W%4 z>~{7GBYiG(yjttdDVg#S9s~Z|tEOS5N%VqpH^}M(GUL_8CoA&Uz$FG$4RI@%df9JGzqW5rYH%IA6$Zj&*syXt!EY@&pt7Ysu%aeo4clss%171M8-% zB{-x9kva;W$J-ryLu`8efqDptMZ2x+bR)5PG#&W)EqfV`6%Y@I1=CZf)mNAXo`){r z143dSr*iIF#`kd4apEy>#(lMHdLxQc@(l<$ny?#FIH1%4GjRHDwKU`jZiFCOav-MG zJ4-AE(O~+0y5sX=DPAKR#55MS!Cphc%ytFtz#)O^UfM+pN$t7Vb}?${`jJK&;uLvY zQx0ksmy?ML4@s!Po*UHKFDDc2mTU^K&^9N{X%nvx*zKhFh{Xi{lnI z(!PWTO56xmTd{`&9J)`A#kTRTnWMPZE+Ab7RlKft+lAZ+S2B*@eC+ha>$o!j!ICeS z4eaH30F;;lhb3dNPgnydW#bVr_)Pi2)bZ-MW7jx7H1y)oT!{*XT(9A54?sd_@d;C@AP!4jLM?!*WeN_mb>Nn*3<{NY^pN& z0tl~kWaupRkt&1HeeT6>6Y7&qEE$)Vs6N00T(AYTIa3bCl6Xl&ebFG7#+Si+CfhS` zIIh4IL64M(D?Xy?Vb==`sUd?0w$ZC%XiS0>5FjNv)oF*TNH<_H@PTrbj@fVNJgzP= zc=di&tz=3IEa0&!K~>u~iH0Nk!sU?qJ?$BAC4MN8g6#RSs&8^q#b%4)c+|Ag-M9fk z5_tT3iAQieqRWum6;v)>)gWzCN&O+6y`9JK#@$Xphd!1ZHZeoZ#72nWGGYACeB%df zq?V|UDl*V{rDSVyC29N8h&t+3%uonAl(XbWY%G_$vzn!_fnNHtP*uiuEkx>OI-*0g zeH4p#)Te5u9EamkDNVQWJctP+njcqHfg#g1+j>;Wk>jd7a^u0eAzO0jD@U9;$X3lakkSAi7HZ`p^rklEsy0ae$Z|h)gVZf!*&Y@ zx~Y#!?4#C|v|ETa<){sJ)q2n0*)QN@$q`8YgtV`>nn)NVyxsIhdE=aeUV zp9vH2%Go{~6}17KtBh}vMOUgM+N>PMQgF++U*8ZmGT@tX^a4NUEi9Tz7p^h^<4biU zBJhK@>4a59syed84@->0bCywPQ7d6}=8=F{uk=`Z?oX2_EGOZp*HM!nMAasoc&V0Z zhhyRoc}$#}(9?{3VFxJPHd^zb^)3U+3%=|%mz9{9#xNO5L&>c<3# zw&cX~@y_GJ!-Ws}pq=EG1oS}z@WU2FP$n^*_)WAkVB)9luJ5$~4^htVJWZG->XvQ{)@wbefwIUBWe?7pRFJ^ zmwfwbz!)bAPuh`zdQrRmzl)vJKGB(py)_iq7JDr|&k0L!!(%1fY01er0g$LC-#WBs z?i(u8)0L5mK|Fv;Bw{}1`I2z^<&CN#j_0(}q7(Yl7_!)uQ{$aD#Az_b+iGO7*FJ6F zBw~xAz=br?qtF?A)sK~9u~V@EzTd*1*FZrOFw>Axw6N*tuiKm@r@!SRPU32p2Rw%_ zTux6uBu=i~#S{3IbcXfKR>#y&$n*lKHu9S|Q1+Ai^0_6GlE#EZ<2?g-jd+0Yj-Rm- z^V5ffH};%!;;;Y010Ud9a``iz<(3J5Ng++cv^XwwABWTpDv`7*R4w>kzaRs&mV6V< z^abpR)3z~I^QN4BBGRbc{#jnlqsre9X5b~?ws@!JJb*zLFs-r1d9i*|ntG8LLHHro ziQH7j-F?nzIL&F_;Ct>rl3157dpe1MyWGbb=M0a=ji7F#F6?1biJ3D3)l_V@k%15a zrkrsuRlR{HL&(SseddJwDiLWHh$Oq3a_02QDiIqYAZUxt#5yVwIoNn01rJNkDCO%6 z%-JJ6;q2oiWyX?)jwo}IZ3?@Z@~xKFbGjfa35)9{_Ev|xIbpa?S%Slovq$8}={wr< z=`fg}yPWyheW!IvK1kzugfOs{6Zq0x?h8@Ek|}d;$tANs;K72$F6s!vwSnB@f&gIm zJ)NQ07=pE^4>4tO%l)e1vdUx}-H?-;p0i)VqM3j}-UtE z>yHx#I#?(eC1)X#xdA*NK^y3;7mqjtOYUeg(^;*Wt5gukruZy5tNJ08S~9-C4G?=c z-f4y_B2Y}1nB8yM!MF7jD!N0NjkEmC@!JA?I|=FRL7l@|lniUkV|H9&*dWhQ9H-D2 zNS7Tle#5y70;a{PY3dt1?MqVFm9tLfsqezw53|ZSl|93JaqB1-_5f4x7<1a*H+wk8 zDcf{bYdOvFxcyWj)|9il2?r^4?i^X{?5_^nv^t>@ zn($2ndo8Z~2f!~-5Xyaf%YF~XaDYK~O37Fh#g;r}n@Xel>|YY>bqFccr#M5EKaSa1 z2wbSHwVmT#!F~@-9lzC)JjZK|y#RwJO6ahBMB6!W9`-yOJAs4}9;tD5O41U8c(6U@ ztiEn1m%P&pK9-!b`>s6)&z<52t;-yoLn)8*4w+Dn?J>8Ek9c>reo_}`O*toKsM8w# z$9Y$Y*?AGp^CO1wDI_m4;and-yZel9Io44Hgg&u&C}0c^7A$ryrcUo7Z;4PKv8J5c z+t2P*c_X`lot^SYwi*Ck@LR3^-1*~eYr-$w1w$V&eT2vHLi=q@SaVUu<`Ty|?eq(E zsEKKd&CMuO%duL6gc7Fj^i<2k_R#D#6xA#_FL;#N5d0}OF;;)xAIDV9l6f+lFtyq! z;}U|OEXE~L{Vo{4eA}8N=sqa5YGF<#%;`wEOtMF6^dVIqz0^tsuCB7bx&0(swuL^Z z6^TtfwlU{t?3|s4O;O5{^XnI@b+7m3oIy2eb18_^nJGg>RdfbWu8*8QsXMxA7D{Ta3jnXcwr;Asi~fKmcaS1?>*2atmYKL(+nfbE@2; zH(8fnC@TJY;` zdp%ylq7q5umxWc*>@xbkSV@V}F;0`h16;y`OBf#ZN?b&tDq5GC+ngqNPa5?#j)jZH z*q-Qd$;hnSqQ(*SRvd026LOVB=3V=%zaE@k)VtXJ0tsfSqH$D8ciLUmZ!>_0Dd#77 zI5RT}36u_)MTJFn8*~P+mTpmtKCkoibS#=tT(_tN-}P}u;{lA5uG!YYe}}&KXtLcD zlRSXZ+Ag{kuO96Gfql!`F6ws92}7GB&oN=~FGcn~Jgi7!*|$qtc-vjk_DL*0Q!Z%| zZHsx=Ne_*5iHxzkVqdHhq!dsev?TJD-393%;(;obT#}HjCSflLztx@6;;o*Jx6-fj zXdLaZC_+g31IG;tx<@RQ*1T2Ni4r$Jv?**wxp{L>U z`BH2x;sTPtQvCRRLSAu_Xza_bOB0j zk7e5nRXgN^NEa=(V5ofuyIut#KoX>U+()&=-;p7v!9+-J7->I?GXm}shceVuyvbC} zB_;i*V6|8-WLyMk*hA}*j=MQ%a|9&QHkX@g2e$mu|v16z50 zp_6c^A&JyRUU|EMlYlGk{;s6`${&s>4@``5(M*}g%G1U6gOa%=N&(oDZPbIcy9_WS zKHcSty(vytQ|r=;EVi;R%c+5}60=vG#$H~rk25duK2M#v#H#1B)GtUM!^paKST%0F z`UOvfu5`7i)n@{YOn4vKY8)V_30Tfj3GnDHSK^QTj75SOKWN{sE4PUF+NT@JA3jmTJUeS-JOm_E8}?3h1wuGvzx=Bb@1%HuLO)OROE1 zsNTdfAs#qjMoX?Q<72;%#9c1LLP1Ne#g|BPS7$%$F*tI3&7fAwHPtyGO4(t0%p~JXmbzq*OJe1KqZXI55SB63$_J)52Jh0m3*Y;6TVNbZmqpm5^=> z&sVow=@+Qr7fWvZ5Pw*rWI=;crp$bAnj=5E?Y8MAY{s6%on{XJ!v`KyH#$9WM*^PL zZGYpOI!@11y;uq}eXmE6^X{&8B$7&)a`TC4cGdJYA^^bffl#n2=L(()y#@k4F#R;z zncebB!+;cgrrcCD#F>p%<3mDLMu@uh?)Uh)WOlJ>`8H=>YMfXE9$Y006V`H`zteJu zz7}$8Q}I*|j+-XwDp7K4M8wlxc)SL`CB`7cd4#URUjQP4O@#NK&vNDlXE3k~!m$<8 zrt~$Q^!G5xQBiUWvgaTCsnhywv8}@!s9E6+U9&AqzMp%{nb-Hthi`_;&6m@iq*Z;n z5pq~=35;=Gz>DOF9&N&{SH`PUtO{=EIEC1mqnz>{g}hdW2`#x5f7+;TEWgP_e7ei6 zb8b1kpX_gVK)HAdx(|jNR$EePa~=l3EV&)edSAr}L~<0wn(~9qp-zvI85=w5gRSG8 z9>~B$f^I-N_6-$bcvJ+%$!ezDQJkjmJSQ_{DIZ3r$oLrxT{AXQ?i`k_zUefHoeeDA z<*voP>cHRv_v~WxYjvFUU34}OAz{e_{dUW@KWxJWCte5{ zJ$*DYg^?_|8*9Qk7;jMm|AQN{&)j$BH3(v)3~U$jg*|SqaaY-Mag|tx=dm7E*@Nbh zi}@Jp(-$T8Cao76kgAB&1XJ$)Dn_is)6D^{u-rX7+IbVtNS;1@3O5k}-~SS7)Y6nK{W5Ucqm7{Gx+cjF)7 zMp^63I}9EEwd6+*au>MSM$l2plKafN&g)He?*OKz{P;nvI@zEHdoGhgYJY1S3BV5C z6ulz>N8y}OAIy_;!GnPvFvh5*80QdhdQx+sS97%#(;nM};~+e@9dleN!Br08l2HL( zi%v%e%eNN}Vm~{+79u!D{Aop`)5veGq!wR^>>4|l)Lt_&hz_V3ARJD#`jf`L%V zk_X?r>?B>PX_RI%h|OCv1ii6-$0Twx?4H<6Hs4A(QM;&?H)J!h;X3;1}j`WL%;8_)C3O5|%gRk>}2- z@3zil-{z9G$9yl#i9-S@<5a@br&x}|GjQOcL+WUFupA#dftj+Ak8Wrny)Z+;59CTf z&uk;ThVwI$IxK&{yTw^E7P2lks1N#LM3g#JcLF^xO!`>zC?+GDaCr+q&=)1YxE7)o zZ_8vH5N*k0WB1Ck1w5hAemOQFP?kk%4Tksh-lypSvs_{6}&4 z=^HF0k{~6H#eUaEokYM8p%+V8QAc{!`H_lKd_Wi6;r#}04|*Y_{3)bzahPLv2OV@E z0BQROXBtk+K{0LQGcEhbte0zXBlI9w9&ge>cEBY88=k{w%G^F_PSU*1%v2LbA9Iqj zlG$?*YsnLRJ!E?ve>v}p*aqJSi0Pbo4}i1^dnqrC3{ojQj{8uVAR`5%rhEOy2#CvL`NEBUp4Z{l;Ux z>`{UL@~Cb=Un9G5ZdqjxOV;INO(A>WXf@db@GW_=dn?&vRvveKY{I-Mv8n@@Cb=T{ z>t{TjW~R<-GxU>-3Z;BiKLiGSOnGvDoQ!;1|J5DD83>=(k&$>L5+aEYq&(hPR>EEZ zF%6~|%f}x+i^NH{c$g4`aQVaRZ8Bq^7hk%XLaw|T5F+39`hcxoK!p5yp0e%N7uhe+ zxVt>rvX4l|g@;zU3q^_aJQ=w|-%ZdsR<083acU2+wO>wVt`QsXzya#(M)+;0r|g28 zI)oHeELm7CMNS|66#EvwvE;WakBbPr2253e6eaS7_^99v1;6f-zcV~kf1K6_5AvQV ze+)Y!=MLHBT1a%0r=Ca?+c#%X&=DU?{)or-=8nx2rF0YPE(`mPai-ulSR8P`rk4CJ z_L!51ds=Xk2KIeetn(W3V}z7>oSqph4n~}0S53A5o|YOMRp_&Rjtc z?S>zErHj2dVMZnJS@QI2w^VIp1PVZ4ZGYHvPI`xpWxv2`mOS-ag!;bo3bMMvOuxIA z;JgtrpY6do9x3r7?l*Wp&u^z|i~Wesz#I3Yk^r5>{x~;OdXJw*47Dyr6%r(#qvKwR z#eV<(Drc_8heiNOtR$r0FDi8AKIO#?D4zWC{jEGFCHvuyW_MW>xKJGWc`gOMZc#

4h9YH;BjCK1aktyi9r? zzQt$C^H#pBd$rgdmOtYKs(xsJeBM`jmHUSF9~8T`SZDG*>9{Lg_Q0t9wl6z zA^Qy*Nt%Y{5|0aS$I1RBmwVU^KaDx<%*C-2s7@mgO4Lmk_e(aYSjr<+F8vlEmw10g z7zjFaX?NI}SLFydpbU1yzY{Kw_my>#=LZHdk|i%rik5Ye%qY--8zq0SGUW4kK?cUr zv3@ZtS>)p~Z6*u^aH#T0w)hrPIMk;x%959QB#0Tm`A{fm-(IPaCkOrBf-MH3DX$Ek zCSSl_BarGUSMbc&ko<=)E}8P`xb<>;$vqtQ%hd=UIpi;YFi2iqNcQ2=1j@o@p~ zurBy2hIQFVt}HH-EeF4H#ErDrD^Cu=+0i5zD40rI!Of)wd!r2C(7F^4xFv_+nky0T zp~M$ia>(_n?l>*RP=J)YJJzq>A1^H&(|U;GYTOvv0PjoygKiC1)7OZDnDdjxSnSm` zodxD~T!N((KXqN!#NrGlq$=p7Upi+LW3Mq6eqmj%-boQhO8zPWN~y68J$I`P#3(vInkocS5YUBK3>W^AE=#=OrEh1~*(e?ey#1m&d3~ z*0%QyQSw_|qPRHL*a;l4Cu!SPUiaxKpKbS-I}+Fa6(kZ%GHBrlwinEpV7zKtzf5*`fw+PX=I&f@ts<0}Jp-Br^LKt+FJEwcfC2POsVGV{hcA28zYX4)Au~#C{)4CwvgLTaF@eUZsl*2A(%P( z>y~ZFo4EKn0+)L6TW!-@7;wJAB@!|+9unAg@E*jW*k^#NV)h_-}eLE%2N2CsS z5439&Uf|KPEqOYMP3L&wDIvCUVi!=Zl&8Z)FQlrjm*bAM*UiWfNI?T3*;i!L<6?QWccHeFt* zIxKcrBek=&)h|p6cFI_$Oahp$^5?cjqJ$j@B&dd%F8|z-D7LTyUnlmponk$UD!|rl zUwll?zgM@0X{HK?y*=Q7_tn$_-PGZeWpA4m?0uCHiU(jY9V!3J-Qcw3>Tih8QGez5 z2`_mjz+mk0*MiaFAhRS0M*Y7gtQD`(Nh^zi1z@|1oN=U&4#FPVgnx@_v8MFVy7>5; z@hNYfXsfSR_W0|Gqn;x7))DHb%fFrr_H1<9i$W^@dgG*L1D&*zg2l4r-w&tlYoFCh zOVP?T7@r`YY0*wQN>`~}H9~w!_9&1lXQj(phE6No0X*7f$=c!t;?sbsb-iV6`DC$` z^wCH#5mdWppIE^#P#H)w{kO?Dv6W376w?ITkB2p1yML`VGLo`vKavn<9@FDO6WV`s z2z(ivk7d{+ALzEq)OVb7tZ|)f`+XZIJ~YRG zU0`!(IhhA0>meLc(gBv~yuky8hp-1W&9WQ5S}Irlx21(Wm4Ya@d9L@Rf9MXy_WNO>XVIr27sJ8!yAYf@DE?<(N1tUkxTRE_Wcf!qTk~jeQ@r?~;07%P zNtC#C0^3Hbl^)v2c7sQY#0RlD|Cmcyzgv95OdQy{$F&=*ND&(ls|be2x?-_`c_ju? zM3k!x_C~Y#SHuU0U%K5eEJExZd!t6GaWo=V)+!EhdeD{#NVgj|J}uV_D80C-VmFRR zkOgOkXwQ|=8tleBi)F#_)k+rwtTE}N3a&TfFNAM}Av&W6u-U$oW=-|iD>2{;qWV!028XcAzX~TXKJd0V}@dGK<;BiZ|wUK+# z{%!-Z8$TuF>gwNgl@eWITZPy_2AK>TMK#-Q_|hCPW#t6DWuuukYFaDPNZ*^SN@%j+NNdht7FHyU$PbYML>53#ulX`=_{iIMa{U{LTOaO8a!ckjWVoP|9` z9Q6kD4YhJ`C0HPky|Qp-vgHR4kBXqi%(T|^oT4L$YX6JCv}s-TSY1LRZG3~DY;)wK zR$t?2e7z9s$PWWo)kIJ(d$(BUUfqEyglj^bDXdC@!8F1qO~%RV?ro-dXrzr#UXa7N zm_u>(1II&?#X9Z*U#!x2QKc9(y+%YKh2Lnq@pob3+)qoi)wTLfZrLiYwpQy;W0q=_ zP2yV0CM3@bbw{vv6HYOv6-O!anby$cWU`$8>v(-90H#!VcaKb4Rjs_HklJ<=&vFQx z)gP)efbjk-nYQtm`Xi-Gd#*HI6WkK;_`ge(#mck?X8D*>=&S zlg78Q{GyRxKIkc8PKf!`j}rTYCzZ=^)HhvDld8j>QFM+T0>1Zs9c*iAZ_B zLS9b>{~!q39{vL=Wv55B`vN!1_HPmC>Bc}nF%4I~^=~@Ub3Iv#;;2IKsuEi~quCeB zfM8106mcD))Iy}eI9~`s{j%(U*2S_V zXV74%oHY`{IoYzZTy(n&%nrX#v|c(9Sr|O!qxLM8mX|utsri zJOiZsAc$(&Do7_pG*tpf(nMAaUnf1rZaQqOyp!{Vh&6txXVZY&i$s6=Acey)y=jqk z2Yb={VX6c?aa1hj)J-|S431^{L}R`U7fS@*xK%#O+7_ytF2}zfbf-`(W#L~b=cg5_5s8V-}@`TD|u&kDtA9;7S74M@qdn|?Q4&ib)NkApl) zO0x%M$$S6)R$C0Tl;9TDhfSB)8K`OOZc*_123-R9l!zT63Q9UFhAz|CYC#U7OQbwF zPqrXC5MoUfHqBZh3RZktS0Zp`g_&Cqc_Z6eW zEi1)pmO2q@da0%}S*UtLw@SsB5+CjpAMwprB`_g84g`yjJV$l^0dxqX&N}Nj74DXx z%0KZk=F4YwX8;jkOJsigVVH9l6HWVE zn9ty7c3*%T`EpKOx7GBuG-ux5Yt=x&0vJ--tQ1=xouvDZJVj_)b;io(Xp;)5e2rtk zO7TmzdHKs!$zRJ2?7HLocC%9_72kUs@@igg{uD z;BWTiEUS_M8hZumR8^K8*lM0k_-wQ4gVZO9Dlf0Ks_3;Ln7|F}5G{jP9o+>XPeBjt z94~|1vyy;=M?WE(bNp6Gp&C5C8RyKTb3{`UQh~&J6k8NC+|jrl$O6Eb-#*X*1ALg) zBTNosPkpuWtqI(~(4CGmX@pKD7No4s%W|^o%k6#YkF*1)R){Ov^HexAu;yPB$raNq z9k^AhmuhA&@jmj6PE>2G9>9az%jUFHlQWlal4AwG=%+XgA#mSz@$->qw3HIXBhXtZ zHmuQk6x|Zu0Fh5Vlnm2-X4%dA@0QI7?e%yLi$Q2`NaVlKPZJLSI zEB+vl5uP_DY~DUbJVtgP^ibX(V>iEHrtE1(C&rb{Ck4wMydhKLi}mKu&yty~LbRhW zLAu?%UyRtu`47d^;E0PAPqFBx2r#BZ-#y}~O|NN!sWK>8JjWWna<)bq)ONqj8@N%g z85ro+K2gNZF4RvEuuq-@^y~QZ(kxd^7s)_+!A-q6dPH_#wQAXmK~Il#zLodQ~^WmiU9G@lh11PVGPoGRR+{LJhP;$ zP>pby8|i&?Qa-1-ogky=Ps;(8q2puA1aDZg-!HCi*g=7FoQGNXk3w6IVj47T zr{`JL=oJp#DFsDkI*ZSlCk;`b5}(eKQ7=r<-v%^>fajtPn)EoD(4tY8Oyhl117r76u+5om~%NZu;GN_bx@XP8!oEL_w(1e)*=66T_V%BpBZY)#Gttgq)gpR9DeBTJX&tT>XOn&S@SkX73e= ztf3&mL}81&YsFx)N1?3|+v1u%p0C{drq~4Ncc)k_-OmZo$3R$VH6cHLhA-AzPB|^- zpR2O?uj+Er?H2cCJE4 zZHtWubUHdD7!$VGo8y$SJFBYTsC2s(+14HzW?stEgqEur2WLK@b1beKf#+r z#)Av^7UCGY7HWec6JlWkOC30f!Si~KcmD*1_Hp>Vx1Zy2p1@( z=;FUY1h`A`2FIyDXGw*5LxxgWb}*YM8*er515pL0vkFBtN1MQ)zPS*X@U~}-9t=h~ zMd-q?7=|X+TRgtVyPmU7${t!ut9QnUZJctALb-A!r4=*2C^kV$e7$9>U(Sfy;@fm5 zjU2_2260n@8W#Ut){sc;3o z_08MGn}ihgO$XbW<*+x4{;hLm>uC|sRs^NG=`LE|zSr47|50|ainCiRGdELQZT;h&E8MIayHsmv&5BcnJ0Q{V(VBIs zrSJf@Yt!0fv2@f+x?|J21as)<_FF&KQ}Zeng{^;_^X@E`!_Y%j0qIXm-o12~ngg|gKnf9R%etp0g`vyO=-|1~(;9LkidF2wn? zO}E>85hho$uTF6zHaIRv3} zl87x-wf|KjsnB|!9CtD95ed7^?~!s9-KTB`eoT69OZf9@e=)s&Q{8`X?@OlDQT|WO~BCjc1EoA>?Kbd*==5}6+1{th51aZ zx4E&>S?g!z(oo z!(&R0jK3|_*Ti7Ti;n$y7J%ZeO^Xl8{-Y+37&zNu#PFOC`OVFMwKw(z_ae72s%KyJeHmjSCfdtLQ0vkN60%I zCrq5D-C)GtcF6NQd*b}pEpcyzx|FA@7$eW$o!aTwHma?pP|CJLHbjaH`qMh3H=S{| zae~N$6iOlGtvP}>F|{kfGSUwHu~t4YZMnYl?lL{(tao!EQGzN-lN~>%k^EOF2;tEL z$0pkve<=0S?a=S`$`#SKiw`(-G(?pMZCWgT;O0^}>X)dH3cXM$AD#H7z!v)WtsU~` zO3zUytEz+%TYkv-nvW;NO(cgL%Jep3V>t>bxe`xwR@$@B!pX^Y( zNIbp#OS_)RD%Eo#7ioZOi2{6VUd(*^XGU~uk`GykjREWha z5K-K;hS1jzicmK z+Q`pta1IvztFP-DT|Te1mT_jHuF9rL=LGkos(eOnKRsF&t!}M*ZNoHok#i->4E*7$ zKXsOK<9tt`!MwYXJBOblI009$*E)YvgUeP?X*tS7Cx}66 zuWOI$SkK4w{E8{REW6!(rSb{#o|oenU8UWMt?~&LL-Ps#JY}G^YtP6AT-||(X>)B~ zDix;~OsEpMDhlfpUMLn-=bLHhNU`m<_b(7JW~8KC0nx0^6FLVmOny-f}hz4omc6k$1Q5Y)GRAJKGf;{jcgtotOL)*PAMJ`h>RU$)6nZ{vb zP9UR)f!^-REpqw2nHEM?aYhxU8b_FW#1W_PNw?cQkS}j~^IbK+HO_APSCJUwUJW%& zzn$m2&U3dxs9&Xd?3wADb#G25XkGq$+B1Y9e!j2EXM%fQ>jRc3pCu@P81lS}Ar7-k?mFQl`~s%zuK` zpv2}WSGJ2v^rUdlvmE8L)!VfSbT;smAR?5ZkuqSV$8xWihHI-Q1$%~bof5^AJ1H@~ zNZe1>Y9uJMF!c|0{$@)r*Efb~*LAeBfrrl!Y246m)gkddsRotB4AYV`vf<$Y>JNn$ zrbXvu!~Z;07clL}nNiEbRvA<&*HWfdcv5*|8IdM%+jlu6@1mnetck+KI3kr%A`3%*|dGb z0MDn~6Ibd34#FcRWl9T^vm64t{By=B5&tOlO_$T2yprEN{;|TLLV_xz13l+>lM4*0 zRi)eQcle3(2PW%Y3dRsPYnkk(2Zj#>HtZVnI%SmaOW2+v%M`MSV?On>?e;fhdYiIa zt(3z^NO?nusSJ!TpK*45=)6w3{{0JjFacw#Y@Y6%Azd;kl}*y^u(lV(Kc~0pO)ij9 zqA=Teh_M3_T;(2G=;R><*rwYGYY{0TNn@P{FkK}qW`kJ8DJ1?dCE6d%cTzcCsX~fs zS$242ptvI75B-VGP=v6z0%YPrV^bq`_H^%3XM~!Us-}*Szf17)!jy&TDjY`IVPy7M z+Hj@rP~ZXdD^EJ7Nt;+ol_1S@>~1G@%!eAf78|x?mNN_-9*~x(a0r{Y#aY6ciCj!* zaEyy|mL$#9eldilhn)a!qbuboMU{tRosn*)c=SP+6De}dT|X-+u02+)bgtm#VxXF~ zANJ84r-YvFA`f1vlB>&p)7NftbTj>Bl(&ctJ3p$_Xr|-O$_~$&3uH#hO=o0>1--Pz zv@YQc;UaE6rU*uC_@D}L73aqNP4fVlh!e}0dI8&5y~Ffux#rc@8V8an(ff!sg3VY& zy2|~W?9`HPD+(N@#M}xQ!6T6Tq2esv4(Gx34$tn<+QJyZeU;#7pzLXyY50SIKz~95 z!bTc26hMxaPDW_ zau-#!lnz{G={EO1-EOs!JA9WQr@5z0JjyQ-vyaF%H@>5nvZ$72cUVm#V@`)v(7+X&LD{;lMD!^N<9J(S-)c5O0y7~ zZr2OCuL0%BMHiy?0`F_;aNk%z(EkpT;b0 zkL*J4t*;zV8@l=;tlgpHu(R?%PwB%g+Ak4ZcFVZ7r*xcZrX9XJ;(Ym3R$a#y!TGTY zUPGd|3I2#d<798Sc-2Zd zjVxy{bgLYL2d95vsECL?K2to)6Ml;!%vUiM!D`@0-YE((M$WQ3^03u$4wGUF*B%dy zb6(}Rhq7s|YGNHzg*Z;y00j&JV(Srh70J0kCnQzyQsRzLvh!L;Yg?juY}+rVoHAD- z4I#ZkEFWR%E8SEZX?Kh|=WM_BJ^dxjQ|fo*&_Dz3Nbq0NUOKMbL@0w6gOYwj%Vh1yS&@5ZnV|$=SXi9pT=^e zPz9#7TfC8EOM>9Wn03?}b?t(>{-e{_LUD-HM-(?7i_T9SkOhx))XoNNmffjSq!`8O z`FfaYq@Bhki=Rlv@C4pBx^T;;-^K=dQsl4ganB-e-`ZK)4Tgt^^Y&*tTZ0KTuufOR zd;8^@i(4>83aLw4^5M&aX|8g1ry=LOEtyv)vHlUILaI|}j<>~Uo3*11kHS)KOQDag zxyqH3B9sjXP@0hH^cVSHIfPXP0j61Y=MHnc&HrhrqS#dF-0h%T#M!TMl+z_TU9rg9 z@)mtX)x{wedIvl=LLY2PP@YpIR7x2o#SpHE_r^c)rv5f)dvtD_EOW-4(*1`~?gJsJ zIMa&+RVCf-OkV4U9z3Q)%9NN*vfHbcY8;pVDS4UR7FXY-1{1FO_aF2IowRiX88;mF z6Q%#%s2g&Iil9!2~X{t2~_JZNq|lp3gv?4(#?ueqfGRg)2vO9vUXT4{ur*{GCo#cq2bA)y~!??7VH3 zNZ?bDWLhOv`tEVAA`uuEl=o3Tez_*_hjIgi{*#@ny4|L~K%m9he{A^{hh4bq4TP_K0 zGC(Bxt=3XJ%IcBr_Q>gswv?k_3b5RG=Ra(7#9=~RqI1Go=NO3=fNf%|^OhLr8~QQ+ ztBC)L$E$P1Hqx;8nb4*LM^>kEvtCItetF~o-v6|&A$R_KtFyxWSWpcUe!tW6K8F|b zFGlk5bDsBE{-M5!0ou{`-$NK-j)jP*$I-^+bP&Va<`WXwHt4Ldvfmn?r z^6Ex%#H9v`3|}ZljrmYRFW2hxCA@_y{Gk*?+o4OuD{Q1@!8A|nLXwpRcbw5-X)Km{ zQmo<>lB!aot4y0I+kSC_CWLFIK?@x9!nRVMU%DO19?n%fK)=g`6jIt`JI97<&$;$! zf57<65RoF@~6B2CB$8&atIZLtn3P zj8AYr;^R4=P6?YJ_Ue4+8!YC*x5hv`FaM2PsUw*x@VFM}Y=SE2s1M=R?andJ(Ym%D zIc>RftZb-)ZTj`dF@D|{26^BxA=SBZgg54#e(l#pa^&PC-tJ`PP-0;W4)p@+zBlw^ z*x4Fs)ft(x*4zm|EG2eVILVXB^#=z@|BLX$Zt)Q(W+-l2ROGd5z0*lq20`R#=wI%C zH-ljh^<6hqR$H5ynukN+2h-Xu!oy)frL8tlm)JtN{@FF!>KbWOvqI-x9@$YmbbLf{ zccl%7Yn3bcVW8g|d0Th;Q}r-L+o(x#vaNN=z$o(j9A_W>sgh!N1Xnv1BoFrkdYQ6E z)ZJUHjm3W|P$nFrmPLs1T+4bLXJ;#I zT#YnCBh5?nhP?ZY?t=^=X|gw%YXr(3`Km-0uI;VjhCj-wA7X#MV0}%VJpM327j-_- z`kLT}ptK!hcL|#*erH-8?(<6&ei_OPA`yOiva_kX`r zuYVX_f=4(Be4;6CtllLu&`G#p&iwRuriNb`~1YkRkiWHuAy?|d}4B?gS z&H?UuDyzdI-R|;UnpnXRycl?Rls%$@4~eTk4@gybE#Cn&XBgM_E)Zirb?FYw^ z?P8L7h}J|43>s%_ExIagBJENLd~AQSS0CCz=Bg6X|k5X*z@GiL=B*o{`GxCO)EfEEnHF zfT8&!M?ZbUxq3;YlA?a;cGvmUvVdca7V4L1d#tW?1lb$-uW5wQcf<>om;}XK9(M}y z5Gxa|6qNgAgUsNnTP{_Mwr2`tMhAUb&a~UEZEEG*ht0it_#ieYP2T31G^561Pv$$z z#y_QdDHx;vg*chv|EQLNUm#o#J$lbkab4j%dWR-RSNSSbHfsG~UGs=$zw%pph8%(} zMl1Jovg}Lx0;ODuCe$pE{W!D;Z1atZb`FUE9{5xQZT0B#h2l7`x#A?#ZAE`xD{f}X zm;Wlu1MJI4k;4myWxn}K^okT`IXCQ(;8uBujSwDRtb*XtRX%4)jqeL3Tq&PUu->X1 zp+CS`D)^&+8zCF5KcNR`#vb*;vU-6iC?BhR>y|hx8oz%*6O6^aohL%wkGL`N8vr~AwOps*X1q(>)It)wweE~I)Z1uKwTTBiD+)T! zVCEc@Q^4U*6~m)Tuyx9Wr0HS0lISUb`Aax{F(s@Bk?*dM?9#weCy1_OkIM6HFLWgx z%w+dfH&Y_JP%Mr6P0#qWJ-X&(ilv+l^w2y&I@0d?;$)dUQFm1~`2K^0A1SAG)jd=`0?wnu_i?xi`-3y=JvhhLOJ zY$a*V-HE2QN4M59MMpx4Ka_7M(JxSRWI+U57+~EdjgVarbkOZZGvzp@_FVR-7 z$KzP6=QoC5$}dK2@d9u1b@6pQe>YEr^BwCjiic*}z2QnRb8dh2MK=ro-8l|)inmJC zcuY&_-ha2q-=!D1jib6}$H`pI($?_^DH1m^K+7cxgS2uNszqC#MZ~x2mu`3OzF$a= zHc>+TAf!Z!o*c7K7Srl?AGTKXy!4z>_laA?H#Ad97<+UdbJUtfLNc;y7!$^PG1__h z{b|}jI4jHU{&A7aVV6fq(du`4@d|$imcDYTvXYwlbdcc~b#a><| zrbB>LNNOw?x>bJJAWl(bv9AGjFCXFTVHXg^RH-aGreUQxo}rIen0~$cuScCXxx|VD zGal>y?@4Fv@HKV7VtO19m8^|}(%2*BRYt*Gr`Lr;%&imTE#W70d&y9qi@CKx-a;Z{ zKU$?G)Y}G&IWWR`g7*na4D@t6W=_j@xYI1KxEj~NV{4cwVn<>2_IAm4ZTON zS~=yXPZS0dLG_t_=;saHIjC-3yT{N3ak#R)?vvDG*gF^%Tdlq z8N5M!$KkkqbTNdEX-+rytdrKZN5_LsH~Jue85})EP8KI8=`}WEakaHb5L(9+wjVZC9Nc48E06+j(J*0cr%5jg~q*u^TkPHLWR0@L~+;I^#}(m#7XuE6;o}op2J7URbF#18pcRDS}Bfjur2>0k9vgi z5#k8Si%m;5c-ygxuYyi9jX7KE6!lr`8-Kv`i?h~SoDflB0Y{ZDFIaEU<0>BE z+NQgTy!6TX<7t4AfUCtPAJ=tTJsT|%$2jDR3DplC^+M{4(cX|eeP!2_=*cYU8^6`s zFJ($>z5U7_Cn~*j$;D{-{L=Hz15&Wxpkf^pAm!fz@0|1Biz45CRL`-&)^sLCpft|z z`RHCL()DX!97;nm(|^R~L|Cid`^r)!v>RgiWG1d#HBz=6ZJ*=gQ& z5&At00~r}qVZss(Y|~2dHWSTU2%x3PS7+p{O_%$CL1;40yN>t8Kx$$vmZWBlhOZH6 zKrdIG>(!!>T)N=6Rt~FCsQ97N=mq{(q)9amGOr%vLQ4& zB36_5TS-AVn6{1(qe++>Os9nRAh6t#eOIv<3*ic-ZLcne#F1U^>s84z1-)0B04wFq zxtgigrI)Of*KRA*YZo(giF#5Z&RSDB_?VCP4G(VnwC1d}G9lp7kuuq7%^`3!b&+=1 z>U3kx0P`6W_Ue`5bYp?23Ie6T5xCN6Ko*G*pBs8{!LWejYG`VLuGf_@)@aV+=MmZ< zMasy@RtCo!m4Wy{SiM!w$T+BPE$ZW9y>2AIEJ-xh!-FbFnM^*&|2(Cm5VZeG{X0U% zb2rY>bJHMQ<(9415RO>6IFRz(WEuI;KUx>Vbmal@97uNool-D;e!qBrbGm8?rvK>m zOiSVA@O8ck2Y5VvR(Nl^OLtp_$BTuseX9w2C~1HdkwJzOT`*Lo((T^OxnIe0y3)l+ zc`I8y%q?FJ;ebsL`$L?(-hHu8SLyXvn9OodK^vz3P%!U2ZpZL-xltB4DVCiwJD>%G;R*~fn%4d(z`>cbqyB{@~NVM z^=7bnEf<7Ti5gvkbJ7tlZmPQ(+dJZ{;2A)DR>4$h6W~mASFYd#(|UwND3GVLb%sY&SKS?|0*Nfc5n6yI=<0Q z#h5Y@CDOaU(;4%Zq025+B`&&DP8a`J ze!g&s3k(pCkc*N3nsA5Iav{-^{$s8@cvv2a;RRkM_lF{*^;R_ z7+h`p9(D#kqPIAW3F8Ls7bRriR_Yrb^$5drMG2tzujx?YQV)pd$%g_`We=d&lX6?I zY{`?T3Ws5O(;ng3_l7o*vB%@1oEA(pOD+N%S?C0q`M%5a;nU6#H}k?S6%KJ`(tI&F zqaC+#o@mEDAq-Otjyo2KkdeQu`sVW*cV~<@_=yj-#i~?K7nh&r454MCt#M`C2NT5W zF*-AtiJ&+(x%yvvek^YMUZ;QG64fE8pKZroh!ZC&?y76Zan(CTCDTa$>++~qg5F^= z0|E4K!{Mj1*3BeoQZ@w(CgcHwvpnuwiaoGemffd&BYADwSNZ~~A+*d*9ZSnDM+#P> zMEk(ht1+Q6a)xq4pJka&dsgxk!B6v8x6kRoWCQ~lV11&NJCUq>s}c~??LM67?!YVV zKsQ$JbNh5Tk0p7f3ngHBUA!E|sc;Wp5B+KEb9<3<6In(SQw_b(n0;dZngtdjO*8E? zXMw!-p&5Ejx< zv@?lK(2|P=;@InB+?-ZCjQZ6Jw2j^RU71Z48(s;6!u{$e7t#Y%{5TXba`Xg zJ$h&j(vfzb(>&a``##0Pe1ZA~otAgPNaC-f6=P0|L9X2 zDYkK7NZErQMrQkI57%z6mI=$gcg_<#dJpr(X}aB)z}*q@gN9BO)Mv=D#`#CVc5%$f z5jzGq*Mu@{kMqmD?Q3W0vtQa`edh&=SIUMektVeJ&R-~AS@?}6fD|0jbL5kvTt^U$ zv+Ta}CyQ6gPiTS`+n0-dubilR_}%W?b-D~Lj?z8ME?s3nt<3I|tj`zvtN7?^hsoPs z3fIuF8U#+czFhjKPkjhUljZoUbjoE_3PMVSoKVtP$CxJ6+a&MH;@=x=9$z)3FCqD-e1**p~wqsuc3oL-FjR24%g2=iP=J`}3B0TUpc zk*rLp62^0D4|}iTqO@WPIK=YeMJ$WMHNKBU-y;*`B+><0v|EGYy;$*il{TNN%a;{0 zxrKSm1;%LW?Dn=}7f|siSBToU#{EwV;}>4d7%^T?jsuluNFT>=SY1QSapummNcWge7f}Gvz9~J z_aM~wD8KXzh>_E0>+2A%#lG4jZwt|l(0up%E{~J9H?29E?3+EWtjGB887BCxbnkM+JyartI2m^F6EUei}edRD;{T|`y%vy zZRUu_Z#$&br$oA4A3FCdH}NG@%2ACH{SS%9nKcHXT$Kja_l($eSDU)9?ALF**kux= z7z6bib57pLIt*?wU0=W9i$o$L1C=oaxBwelD2J0dUoq9L>^CA)lyba*KNMrcX6==2 z`W?{9!G|gn^F=A=r7(^w<+f^3${MVyVk~xdttjF;Lydt>m3uQqH@+_lhbi$uq~|?O z{QE&zuhuU5EsYo5IVGQA0vE!PND;$jCyZ<&sNeEJv59+p${r0@aZ6 zM1(VuaPYvC67X0P>rCVt0TN8}=*Pl;R_8t1w=ky4`pKdjpI3@2a1gi$`~};Spj4va zQL;g--QPtMjFfld#bT~8Dx|=t$~TcR^Xq%Ga^WhH>+|^Hh;vQQ!+IKNN>nZsyIHEJ z0eYE+UU67N^W>${#Q-}^k~h9UilBkTlc4Rc1-;@DcphW$Z)sMp@^utm?(F!f`PMabY%n#@n6j-NYbJ<*DtmF=Gd14}O#=-YFZ;T{`!@)7`c1XuCk$$x zsQpr-d5p=FIoFvNrDHUXhYyM`++3o zzm(y*63v4gRqcY!abzQw%#%MR_EW}D3W78}*@@sH8yJj{`%kZwf~y%-SWHXlpA)g~ zQLZ5=BO`?>^$6L)`&O8<3MPX3-y0$1U$cEJiYgBZ@e&u66fpQeSUO64FyZaG!0lg8 z+ZWEsiPLXUD@N*vkvElEpPOrprg`+wKIGX<=jic`a{9l%K~5-WbqRsPlcPCqT7)a} zRXFtjWRx7!z`X2~s{*F~%W0yB8{GV1BB=lM1>ytJ(dAKvA5}gOa>CdfbrhCp9Pk*y zw-V!+HrL6 zR1Tvjp-Z`HQ3J>lkh&s8TMVngbOujoxL-m)&2)OH=Rx-Of$dssc99&*qAWL`)c(9; zxS9-H^d+xRr1 zi-Hf~g$U1icg@jAd1stl@xo`i1f?j!*^Qgm-J{z}khTY@TW{mV&p6-G-he&l#0icB zP~VlZuGs0rM;r;J-6r&lk$!DY)va6&_^8(NCy6FOsj6U)iVY(5!8>$EU!##8*zG*U zF^gU3Vt{>f!Lhi3iTPaUReQvlKjvLx^dvxf@agOGuOesw>7?%Ne3!ytkPfJAWF6t8>};Pa6B?FUU-B-G(j`MxNeDQ=dAdX$L4bqM ze6+lqEv#~wdX!BQ8pL_}c6w5Wl!jivO29m4W1q$P&|;LX(upZ?o^DiByp1IF6tRM;kr@c0Pi&h!J|w=&psr+os~~k6CeI3MWJq4lEB3a&~>xYpV`^Ls(EGCm#Ax$5@Ciaeu7%@X@o{ zl>np4{RhPXCK7oU{qjhROu0VG=gKK|0^f!=nP-F2w5WuIXT+QAi+Ox)KY@fo-v*fX z)iJ>syFTHbJ)+`lwl9*g*ov*ZKcF4_*QI@I&ho<=D}Ta%b}ZLtFMv`Z02(z6tGw63$y*+B_1rjT0Y|2n>t6 zQAmF656O6dwcJMd>=~gh2`KfA33u!h|Jit~ZVZ#Kb+q`1%Yo&-a2Pl$K}_OyEf`!L zd)JB@j?mQjdYA++_SkBcvakutIljEmpg?~GgVJ#RyXR|V<4lghAdJm z90`i)z_4@s@_9XMDc!hMe_)r%;$McN{MQ8Tz^)vBA)Pj$RJD=z!0wHl$JwDlq#~r- z18**nE4bHK?o)1HWQ_A5mwWPkUCzJ>2j#pcpH(w)Rf;Ovlf`vpUVw*LnHYF)rnC0P zTlA6|B>*;mvh!5iXLYY_2ut>h$Dhj5210#!W*d|W96Yu)O09J#t47MU{L~94Zr5$UOtq-`C3atslA}EQ z*NplG?w%!{U{F&h<*jJ3mL)aUFYtJCp|j|pQF^FABtqb7#(=2TE%Z=db_i^vZ{T~o z{vy(Yk@8mDKyede13%dyZ++rJ6{E^Gwmqmpp?Hp7N>xD;y0D7lNwiuzH)e~);M zORQ+Bng=ER8*5GGNPwymuEY)sYAH5zM3euT5`!89i%qXxS0}cfltE-$`#1KHx;?Ey z{-aW>r)^VG4AMc(_lg4MrUgWDxt4M{jzR3IQPJQUCTs^nFL_^0&@aT9t^(;?-@`oCrAtiFV*tEQZ);3%N z>zQda<~p{*VF=Ns;yE+DH=V(tp376ecei{EumP1Kma97oDMm)hpkzn+X>)_dWQrGgGlk;nhbyl? zDMxZ75zo88ZWiJ>cc@^1P2J)g=HuxPuySqFDW~OCIBFN20YrdJ^Ap8f-av}_VULLw zVw0Ow#UPztsY}!W|={Dsam&RwtxVmjDC^ACw)_quJK z3c5jg=VZz$pSbdOFKWk>;oYq51lW#*BSdbEXskn-XPQOVKB7%Wz%J;!FsFWg54l(HDmcPw{) z;n^zwYnu9?dftCU$fa3DN~Gze2hHUm#zSUZ3iwDV%aKd(eOd=hzLxU(BH5dy0Lp~n z8pr-3vBph@YwW=dh&b*@6i@>A@HlYD8$}8xg#+Ix0tpQb4>w>AHgawupEL$`wLLY@ ziX$P}Y~S>L5INoA9-5&`7#`K-l=G&&N$6#4OL$76M43Ir3%sii^q;)v5vD}%3??%9A4WgJAHqfR^Q-wZ>Jpj z>v-!c{9*_^IrSVXSLmUf1ygicXT7egK!osJn9QxwI|;5E&ie@u`7RXFT?$8HvqIVW zx#gDKQk%B9*Tag?B|!u0xKsSV(MUuZBPVtV7C&&R z6CS1!CiW0c8}`;HZd{qz`>d10QQ~rdX#*wpA0hgDQKXqdL`q^oY6+VuOMRi8XeU^e zoDB8&qA+n_tjuC59I>iX0{Yl$+4rMQ^&+GJcKvqoJv(PH3|TM|OqU)Ox4Y@+s0QI? zo`_}US%Q>2<;wbmoA$_20UdooXWNMrcFL=cWNE9HX^TB_(izE}0Hqu_V0w49xXwN1 zX>_@Bf^4(QydpvsJU4fXRYjVw^{8)fupx8Dlqh}l%|IW2R?Hwph60wak=BEL?>Uj_ zZVZ_cE3)N{5&nAOZKM>8mUnPEj6d86+E^gIoBX&YAR44E##wKIG#eb@n!wvWInB@N zrfvwkj>tQUx+!g4&%F^N@7xsavj-M?MF`0aZVx3gG}4!MI!W9+$pyM53|St$Zy6t8 zM7ja<{y5qAqZGZCLkW2B4B+!bAyw202f*H{l@D_1s{A6b&kl((goDy0UrRY2A;R6G zURp}x4~N7}cTChnC*$lxN9E%4-&+bg)nHl`FDl8|r*N1m^$4dptxqZ!io1bZk159r z_L1H~0pkbsA12FSwx5xrE=}QEwp+@0hqns#jVmt{ip+@_IzbymaQuEj4&$Is2_-x_ zEPo4@7Ms%wn65IUQ?j?uRVjM?BVR!u@?e3tFG<~1m2hRCAwg?pDrcSMQ$_P2nWB97 zp80kAghP0P>RJv^T|pJVQDyuR>npyMQP^XbYPUl;NB;{qmlPpSm*{!aNoQUO2DeJb zmU0;DLHySxbS3ROpH~lrT&>IH>JNz>A^qJO5ZcH?Xr?J&w$REMT?VGf6vw2}H`vO+VTVGRoPY*$D@`)&|-)5jh8EazDc~oP@zhf=$k87xaZg~ z<4Ai51J=1Lh5J**XghSTe4yCmW%1M6jy@?KCKojSRT`(;L$a%#NS0hnr~(fN51zCR zyXnOZj~qWa_`U#(5^6F5uzFQ)J}KJ|?Ij*2PHVK0d2p=ySgZyGWWb{ zKyEk`owIgg50IKZdB`)d;umIGs)X^&YimXB_E5cYRiKf+eZ*SNtwpN1&fc~~F8=sz z-F@UCrDv=+*f><)Hzs^%i%dx#Bi8WEuF`HeQeeHzrCuCu8qkm<+pYcPz_RPguZyj> zCY7sCyh4kSj~9zOnorX0rA&1OLw;`L*la$cx~uKU3)b70u1eM~QGMFl$+09bxME35 zlfm_4i&mMSeS0>=deJ@djv2A~e-~ID-aAp+8hiYAr?s6g5Q@1mcA}DJX3d5Sr2uxe zRGcMSV>!SK;jg{cOaFYMUFp%`a4ud{lYNd7Dw>gUK0+L0XBB&xCYIEcgzQYb(bR-x zQllf*OC(lQB{YttfEe-P@P>L6JX=9e>afyDX2z#Vpi7pW)IQL9nZ6LHrejMAKV-ei zBGY=G#gfQ^_ak`&0i}NMXt703ySbole45m2j+1N-#~D{9#hn&Ea(h8lF*p*6tk+09 z0aG)3P3pN^CNWzJ0!NwZl#+N}KB>txUvG&Nxzf6WgOBrlgYTpPTjjKy_FPh>%^bOw zv85kU%2g9f8oSUcV0sTj(;AY-9M}V@-1gdO&7(%!^mg*mvV{dG zSGgH2t(VD@=K-lnG}4>H6i71ELkX=1P5jl(>Jb z`1L8B#oP2_No0;YnbpeoiMH*e$0DuoNoiMwfecbQgCAZB2Zr18Cq1`N+{^r>fKJJX z5uw47R-tjiQ zRMID!3vK-4xdQalRh5qF@@c>N>C#jI2#8;ut2=|2ttPbJxXfcS4evJ1`Q%@=hmFup% zR=Gh%^GG{s=OMXxN230Ec(ii#V-C3(GXREIke2VZ&d{+@#TDxfkhihdSOX5KAcZqm zPcsMdyC|ie)lVi{Ue>i$m2wU2WWHGPp{3)4YDhU>Y<4+D%5Nr zIwUCHQsQ!^NfnFalc%QY2r|kAwUoEbTB!VLJ}1eYL#-csp3^&cMas9y1EQ^;cxnMq zhH28}VVN9rm^c0YP!VJy8tBPetzVe(pqOz~G8w3bvSgA+c;h`vWLA1313%TH;20IS z$iezu*=yP`9OU4025E5X`8?~uf2El#^(^mZ}d zy)r`;c(5lP<)+A}(aoctE3+4hRZ9`SPh&A-c;k1U6DL&Q)n$@->dT1H@H?l+xCtnNA%i2fs5*o6uC5 zTOoR|4F+t}p(I}$D<7&bDX9#Ptg|w4@)6}E70K9xt)pY4u+LzWV`-)fMvGOiZ&1@F zMFiNKmeRl90g*+L$9P@h!5DEL>$nhHU6y5vv7d(M@j6nd^4J#7?>viG1`a=6WmTN# zckZjN$1kXlO&=)`53#zTN|-A7X<~G@+jO%1H+8IYhj%1Xz!0SD(h}0>k92kkCw7~ zzX;Dhp-pI%tJi<*jTLvHe%3{?^|WPAYaYX%IK+Y$=w$~zZ>=)d2MGt1JFwf@%Z^ou zZ$&Y=ELOB*gpz}D<+=fV5-&zMGt@BBbtT)^4e4_DuiKE1M>q{RDVsrOzz-?MT8jRA zGWB{P)gb)mto0#VM@VsTe3LBFIQ*n|*eD6^H1yMVVKed z$CIa~lDShL`FqXpOkP@yuH?)7Xb%;A> zU@(3eI_Sc_^Ne7X8~ja`Gvf7Cn>A82#vXrccYbwuc?^%?hpj1`yNCe?)nJ;EFS3qz z)2$NG>Gn{|Pu$FMSPtqVniBELWdi9OmZDtA|F+n5xz3v`{9i!{l%vXZ8$?5{GUXb3 zz?8>Z{n!*%(AR5Sa*B0k4vPB-nbyDWeWe<)X{JMS&WRibSC~&Rg~!ATB8X&BvmqE9 z(`H(KPR!69ERqmA>ySW#=t{TMSXY^SR$O8xXNgY*hSIeMm|Y1h@dQOvDI@#y7)(z@Mxw5OGGwzy@2ilduD<4$Tt;r zW2vF!gUus{NP_P$$IunEB97*v#O7%pY;d=Bd1Po44oP&9SyDrSiS?muw8xO^MnMN0 z5%f;JNaJ}|iYvw--4rdZH0MuMz(9Cyr??{Z4P^_pmIL9vwT>TYc@e4FKNfp^k7zULTANZ041K6lvPj0P9;lz*G{E+r7K!fdCqvk`T{Hl?(uFGU`0!|IC)OR1 zqQJr9dRNahBZm{M*b4&^DRh^%#*hsi`Aaf zC5HZUM&xo^5M7kRslsOc5|$bGuW5wC*aRKGhY#4sV#C_3^qloStb3Rst^Vb}hBZ$R z*KxGE3?A@->8U_(vgchI$aq40Vd2NU}AKMj0M(T>C=>N&G&X#J;JC2>rYNo!qBwrVL6E+kZehSGF|Yn z8?!}o9va1KrXL$NH`((O$#0Z$5Ye`ta?`U#8}2p2)O=rt%}x+^)6pwuS7~7P1?U4* zC6qAg-?K=Z9g?a0LWn50sMb2iGQ)9n!DxfdgIO@#9J{g86L@p5vEpr!-|E;armI zX4p@A#bJ(kD{V~`Mg~XHFVu}bhp}>XgtgBM6JtnWPWbJ%#iBPx&R6XvWe9h8t8{!B zBc~)y7T?tN)VknXs$3pHDLnR5$k-BFn4W*w_Xou%Y(OYiCMYAPw3sE1a8|*!2Lv{k z5=n}P9;zC;w5<@IR-G63(>(mO`j_(8n^f?=4SStlH!qhKO-e7Xf zn1-AZT`2~VSl|k*9;8cjTObk$%L24DghAs(=Ekc;7&gs!)wfRZ-pLFT$uJ#8RzTV< zQrsx~b-7%#@RBUeDR*q}_T|`X6uMw}s@#$5ZP~`8f-+3WI2|3M+lm=Ur%c{!J<0() z>}i^5%E4sI!=R=Nu^AdiPO%8%YPo_QVwi3aDdfz+TIz#=l-WzHtsit$)-s`;vSe23 zJA81!p!yJO`tVNKgu{aT;o4($tr*K>WEUm!RN$sO9x6_;8kpc4C#O6cZ><>6T8}P_ zl;^ity|^2xc=+imJ7!v6bCRjV*RoS?JiMGH;><`tR zt&Cyj^LuA{@D!)a z(xYHXQ6+h+x9h1DTDd3<{gm)V|9+n~a=0!LS>Ww<<#r|3bo9gfRCv3#Oj1luoDN?$ z(@R=A-8>AiekVOIGS$d4y~9X5Sr zdpcvFX@24m*P+UAwvn=`&>OufSeM9E6RY7@gvyZxMH+`;IypghV`hQZl#u|&u4s@o zQV&Oo5eOXf9L=Dn$`_}@7iEj&@&0 zzu=ALB3u4NV2_TrelPx0k1aLQ)GHQQHu)WLOy>v>QXq5>>#KFyr7iY!n5U9#jr`%N z|6qY^IO-)WHc$0I!wYLY-!eW`IE-?eD)+t2nJXnWLj!v*&Uzr*T*zIozH`HuMaYJ| zwu;%fLE{*{Y0f*vT$!;jG9|#Vr9ez2vmgvjBOJbUiI~D7LK0H2DJ3}lvytwz24H}L zu)C4fn-8BlGf|*g)bKYCiA3|-jSGxyYl%#Xf&fMpO1yDET-)hM#Z*ax2aB?4WGUjm zF7$c?r%x@4@uI*33p(d-eLIXrTe;bZLj8{#lD=CTFWHh3Y;CKrEHre zC#*5YfsELCggu#Z0wIN_<~x@fxY8T*uX*v)K!5*&xBFdf^i13sxiZe%eYQ^2;6~va z1#-Ome3mJ3Vyj5XYNfRe(H0}<)_mPp#~ARj$B%Qw5FYyhx*FJ|+o_>Dz0p4}*2Awj zrETg!6Jw}u43s+hq_-;<*r{*6*~7VKu{fr)9`)%e!@q1KZ$EaoZnp-V7OMSwKeeYH?WKqlMI0{`Z54zMz)%%*UDa?Oe6ZR)50Mi78)F_u3Aj%u;)+N?! zZSUUdIl*2udMGSFf1=nM^LeR`WK(5oBk?smc*5#mU9Xv%a zRaTUW-gl^w>R8X$eqmke={GzUB2^Viyu@)2PE;vExVBjNWZ7cc6x9+GffBzJS}Qp8 zoIwO>T~dGDCnoUC=CqvZ{y_Ry!oDxun;4Wzx2=|=Jma`{5l59MUE)vX`|N5cVEkZu zpin-3V{biTLo^0zbyDZlQl;SW{|GzxxT>o3 z|5KJ0sYzmLmM9vAk=a;z$;)1Aa})(bIijMdc%!`F6-6@+;-=Ag@7Jw_A(b`8O~%S0xOOm!}37@}x2tR28H=6D5vu79W&m*kk=O+aboT z@!Ww9j-QpQYO!y{Ib&I*Ry;5=JT}COG8U@T+~cBl_O1wLOoD!8*L*CjcWXIgIg?j* z)j%7Mi_hmw*M8}sL#hU0+iCGRx8iY<8Gc#Y(#7Ybg8{aO^rr;3H7Rx!Q+_^MgcZ82 zSI^c=sRl47B3Q;2#m4|!q>WsDIrt&V`dfjHKD}*gagOx!2Nw5qc|S(9W5iY$@AHEP zi&X1LCj~>#m2MsJ&BX1hxQAo!7TNmAz5;s)p}oI1OWs5-dCx$2?oh6%;#ua}SKG@s z#ZGOqYAC-LXV)-LX}awG${}3>1Ht2B2d9n6Ijp)av?XEP-3;Ir4{YMMMP#|#y8HD? zib6}3&nv{|q{2?ZK>Yp}p(00ozHFP;#gw2$2`^DT3R2~KmiRkE1wT@@Diex&Q=Ng_ zCsa%|Fj4P-Gmv2q7(7Rn*OEoPDD_G647BK+Txs$rs~!R#4_$WKK3}1mX+RtKM7T&V zh*Atq716l36vOCQl96)TLGciMGR4jJR&?Cy%*1O*QNXarLtUKK+?~ZZ28U>ORLm{@ zR!f0LZ(GEdI}N?t*Hi;X&q${shZYqN7(?i_(`obFC<|Rw1Au9tG^ZhVR?$`ggYd{^ z(c#sh;uh>-c%+|kmT)GI{~9BUUNKH&^S!FL3Vy027hB6&yFh|hBB9iY;oN6lD3Zmj zEVnrenK333sUk+ZQG?9W-|zw0E2rJ4apr_srgp=qG^Yv2PJ*GZ6ay1`JV&fg2bd`_ z{G1ie`LQ%(BzR;dI^pymTTOf*g=c5NIf+naX5tWfh0BI)#x^xWrxaUExfJ~PhqCIv zu%-!fGo8BhK^GB=w*SkfGb2T_!KFGZt2Fe2fO9iz7yO~%Q)NN26VB_DILY&VNt|;J zo>S0w>wwwXUgu>|RVuX`c#(}m1G+y&AE0l`abihQ4Gz!y>1SNY_%sXy5~e_{9^uGC>@pWuu2jST#p{_S8SLat3L^aZ$-xNreL*gw&hqPAj%J z`LAd7oks<0(%Q(GTH6B^PA`_^tCia^UVJmZmG1cs)5A@i-lZ`*3Mqv>+$-Wb?g2N% zruYWJduPOTw>_vGRigGh#oJfJ=WGk2m|~3jH3&zttj?p%t8Tax9@|r7mlkGc1Rjde zOb$qYNB2@j?9XQ$i|qx9dt$%3AZk^Fhk9Jwmi4+gXT6)`h=#`rA--TZs&HU6aGX3Q zzF=Y%Mra!X7laB~f<}VZ#K;;M%aElKDG)K?8yl@yF7;whRjs!zJ05b|wBJ%~LfNUA z+eWlfP0aXK#$*PM;Rde?I4Xo3M`8g#9X;^vys9LqAG$mk z>)ga(MBPmYDqSQGOH`b;s~a+DUYwyOiI@3`$tWQzxE)7*6(2CQE1Wnw6#g)7kVBK5 zp)6IPm{+B5EoaD|S6Eb0>QklNUh(qGD$#;r&r8a;vfV~enN?Q(DUkPV6DPT&sVv5i z5<{XL(rbmh507C7oQ=EyQkM>8xAMMxEZjMk`i-_JK?h-4ksDR_5gn(1gU1~QW&d{P zl(S7}%OxG%&g65<0~ka~^gZr&<~bmMc}5;H$4R{QsRc<&DsjJgvp!5cttcOgk^+?Q z@T$zIaQe|2$_y|Cy^i?&_fuNRMU8aS74ZcN|DJLn9TiP-=ofSq_&^vFFTSDy@rNn} zfzRsaKfSNjhp{3Y6`!AMt@JQ4Ci5-@A0xSIZ4*I=&AZg&5T~-#cxGuf&=dae{WM zbTL)d2yu!GcJT1LzlH*Pev0tyz#myEcxfl!v&kc-FJ|#7l($+A2FJ7GVMtu-q>YSCg zOnXjmf|hBqCyzO!NOu|ln5ptZs*}Nfb`=3*O8h-ioWN0@w(#H}*U6VMbpL^92w#+o zQA{-`?zN~Rh2jK@e;zPOFgs~P&d?wx+^s{qGD4gnTO%Spdwg_7Msisp^J<)Oko?3} zoyW*CGN-~%us){z!VjijH4!IBTcmgx-+sMXobU!D#$x2kI#K;v7A8DLXTTBx#g%eC z+rBf_$zCLNi^@~pw;N3j9^lZTGGRa&$Y$587c+~VbQvvld|WvoPVvG6#nlfUH=GeC zzHXsI3dR8Y*9CEsMf)(*UBKg}W6Fej{sNIIt?LIr2Zt&F4oduyhaG#?8BdN~Fc{_R zn2kY~eGf!>RcbJe%?%cN=Phv-Olwt$3fe8Ycy0f`r)6a6o2tGSr~fUMkq4J*0u}>U z1Q*!JmjkeF23W%m;uKdiszMFOCW+T6?jvb%oxavfNK`}H!5y7nY3eXk8bj!wx$g^}(1C{uOFLtWbBU&|bO~syx6KwO%t-{L zF~AIF4LU^Qu?D|!OZXQPL%*f!18Q$42ue9cNE6?2?*!whYKR?q zRvaghASgZMmK|^=^Fqu;ni!?1viMN&60hxfQXr7>@C%+bQ#cHdC0oRo#26TqJ1LQO zNqpI5Z74n<%$+JudfTLwKy0wkxsE$DibtlF@=A_4$$%s&gd_gOsqWn`*m}%wO7P(6 zN&G^6#TcgB!(?w~kG7aW`tLbT$6pRuVb}x4l&C@YJ=*C+?sU%{K0E%p1gGOM^E?ua zsq*1yagzLWC}x_6{pm)L3G@<72+?oH-xlrs{P4z5Xrs%!U1Z;$M>TZqm-t(Sb2o`e zI7??1cjB#%Zr2OXX%iYA^^OO3vF11%N5v_z@saVtpXn`?az>Z)VdA8>`G{zM{gNt9 zF=-B>T$si&w8HgO{!?tCQ!-4Szu@+KFJARfiVsXj#JKH-9oJ0Zfwrjuw_ch#GNPHr z-yZ4KC2c?@${Fa76a@dnf*BG#LeKWj^(?6cA!uOn4U@&0NA^|wHa_#H+b+!H8#g=> z!h-)`ac4RaWN?t%_RR06LTwHlPhD|uXU_r*#{2P0wz#*C_GzZNioI&JT>Xzu+NMpF z`tb!(&W|_W5eiFqhzMENJF*HmO7u+?Uofp!O!HJ|$7jbnVH};E4Pm!xFYy!h$s5Q? zon&GR0$01fJW;F-6wnr{L8t*eCtb$;^Yv?Z49S)wy~LXa>7gUeDK0DUhZUxQameP3 zmk-tuGLOoEj6e9p^o8*UGo5$6S3`BI$Gb&hh<9EaKIn3wKoRs{qD0$+H>ZPM+V?yy zh4b29bAgOMOq|v<;iztMAV&dllKR1v{2v3o1tP=rmu!3 z2-6Na?_hOWf}DkIPK7|QBKRp_(k%L#WDlFo8B>VpFw;a6lI zX7l`EOz0auzQ?C*3^lxJplLhAH(ZB;p^0Q48+NI`*HIWukL%lBDsL+stxKQ}V(&?j z?O3+0R*s#bAG~8pRHMXer^F8o;gax+F%Sp*&vN8l0fR`1cPeCL*BkX&$nUo%Nk3*)>-tG=z@f zZXDa3E$A(4D|_?|Pjl~kWO{YL^h`SA#xa5gl+m_l(rWiU(yx_Z4}(rA-j#D=6Svy~L zA`iR*X4KCpbvAICvPEc2*Rw7OxY)6JE2LacPA~t*oZL~N36JH8?!9EN^M{tgdk##r z9B;1xZ7DJ3nA6Y8*p{h*wK*-jdJ#kkpwHgpEa2jWQm#ZZWgRAuP8xz|sRO?MV@K$kx}rf#$|&t%jwBiNpA<^&5^L8FRC7r_L1@+t8& ziS8|vv^}2NA%5Y4q0%;}OEBdr_43rz)g>x(odpdCtMx+l1IPI=C!awt^_3(D7dJZ# zz3XyXO0S!aIbU(n0EVg*B{%@Fz*`_{s%?5U!|Fu(#y}`=do@aR{z(<3zA14!Rs8UH zGu;|6qu=h;y}eVwOIOchLd2rW&++n}mX)G6F;=BZa6tE0PVDlBX=1(Z3W(nr7;S}6 zqQv&tCeRM1mRA^y)v_a@cPGn1W6fmgPQu&7^SC>h~&%bL)Nb zjqZaqj$W(gxb@!uRXNExs@MAQZoP4fbqND380PkPe!2ccBMGJ)fBB7(qXK56Y)W)m zd0!x7!ndQH?S0Q%-=bWZ#!;h0`BW!?6e@Y9z4+{2+ls9#q^3ZEX+XVdl=!s0lf|{0 zG}JfH*B6L?d2@Ab^wJ1}5)b2?!0F$Qwwg3`+HbFreJ&$CS95z(bO^($NE3@j;g! zIpS7QVnXo5PK|IrpO&UisuZghmB2Iiw{d0vW`ZtGIVzz=n2h7}@os>j93=*4b$^8s z8=yQM*}MTk5Bc}E+ejckB@e8IhLcpJ?W#z^VZOlC5)u0%OgVqdJ2 zoYh53fe5e~1Tx0nasCGth3Z@PKX~-MZjXCiM(a@T0FPjn7{UEpr5tvp$~)slMdd%W z6hq*y#y_uJ|48_MS2`Y>A9(B-dw7_hp zglWR9r=7MeA<;}tC!KJZEHsSyRTB%S!0r7|z_~E9N;eOr`0Uc8BJb3FkYCf72Q$uBuUG_&GPO!8}o3vQBK)V#9Zc5ng&vs!`&uT23pNg2OZN zzstq1Rr-{y0oJ?uQ8AJ=0|ksyN^DjzpK(wg&NjgO`<%bfC-a8_M~O-2oF8{|)k_ss zy2`^x#ZZ#Hs1i+;=X!Sv6C?R(@rNg6O^jP-_6+?sg$E@bJ}M)4=v;|K5==#cjG$F| zVikcAN>F3)qVTNRt~z7Y#+8atrmOsAv=zyaRzc?nkHS6jR(fsz>)E5~sB<Ye6Ja3b7xam0?lE z*SO(A9kJXyWn!vC#97@hK3N?OeIDKD-oQr-re=|;&(O`HJxfS&K!*s^xAsW?4f+GD zfT4cbE;-DbiDxR$^@-gh9tIS8nE2?^AWVjFQ@CL$=xP#LCCf18`@lB7?bG&@NE}(I zW6UrO-yyraI#k<38>mlqeWz&e*P<4Z@|3oH28D}97>))IO9`LdXF$AIVqS4Kp6heR zDcOZ{^b}W&De*_;`V1)&UAdV84)vo1nJ2rN@0|I}_ZcDdH3H0M{L&}mn0SO6?byPE zRG;)+V$u3!?H2=V;w~|u`ZW`fzLepdBCqvDpl{U{Ta;g_G8`&`Q7#Z$den2aaTI3* z2Qs0l3WY--5^%}#^Gc*=vFYQ*`=Ush zM>@-hv!O$J5RRTH{X3Vx5NQIY&#@h)pLy$+M!7(&?6UKmg~e58fYo0P$XmQ6S0urs ze}Wv~CGpa_B!(51Rx%ac8p4rSKhden>5eLNp#(fK^W`{hP;EA&Pk1aymQU0z*T-Kd z<+l?j?RBm7YxDsbzY6Wd`bWiZZ~Hn=g;e5Q2dwY7bOk7769`O3zGEg1Y-L8Do%rA` z>-3Z@)r~OGx2<#`hiqztqfFT&@sSGa+m69%MRIHb_Aauf+S|?u}>75edWeZ)xt0JSe4bU^RHW6fbjr`&toRvkw#u^&g4hr{o;(H1<|ykEEAYyB#NVt&X6?f6sAm)~5qzer~ zR;R>}Z1FirJ6yHcic8i?HZhb5jgkLV;9QB0R*lfOA?Zw^)A39*-3WmX4}TLmX39{F z)Bt}O9)@t~UD?i&$dJ;$Ow42=mBzm1TqS;nS{(gz;@N)})JN$1BkCHk@{d@k`t zJwgF`Z@VuU${J47>G@3W(APQZS}%9eiwF%BkG^ekTrqq~^(I%}wmW1qrn6u$uI#&X zv$X`z2SUpA&FUh4BIzCGGf~)=w2AdMKBEU{EtJLju01Ax^pftP4^m3v#EGei_tv7bQOC3bI(QVR!J3jj4zVI$;7!8DS5iYq>XYo7p73%)ApSX;z!=p!9ePV zlw;ZAytj~P2poA|Jb1l^u6aCqFw>oQ=8`zi(Y4KCVafKR&Bj}G_~2aS@hC&MAI(<0 z3Cl-&C&-vuyLGHLRn6nk0U0vp*uDCiiD6oK*16c-WPs3OAN_2LbJ5&uG)N!)Zin+L zqrNJ1DO1Wly0@9g=Rm3=1eJ1+`p3&*u|1Uf%If&`FK6YZ?@jVws`%_ji!0>)tQjb) z8yssd$@@7{s2qin{C0B8UgzSZbJ}ydL~?^m)|1TUr}N=6RZ5P?PLm?lL_%pym4~97 zpPS6oUk{_p`HoUOCx#RcdQC!>YaVRHJ{Ewl-7Cp;c0kWm*JsK7O2x@2C5s?GNWlmg)K~*OMWIG?`AT> ze-&d&(Dl_`V(D-&z>-hPDeh;XC)!XhGgFpq*Dc!s>sPyd(BU9gLyTNoOL=RZY})kd z3(7F9LD);u|4X-M9L9uuj?3nK&6J^7)jTnc6sqKf1+RF=f$$9m>TyV}B=a@JO_$Rzx|?hqzf3C^rb~Rj zSx)8~1$0lBz?|R(a=KC9*dsYmE{o}LtFshX2D_sna7q-c!Wsh8AQ;w|OEm~cS zQM7+9gH13fSHP4JY}B_$EQ+gNirw$oaG92JPzRk>uKzt9gRhunN`)5Xn2Jt&qe{nV zK&$`V4%w0$A&ZE@G{v;vtC!^X7cW*jyMM3K)^i`+b_``U=|0A2l zx9s%z(bgDQ0mHmqjSs$>+mr3(5JbYbbn^6;Vz{DwC`Y+pN9pQFf0Nwbuf9<1k(yuxCx?JFY+N1xmqF`TUjU}OgNr{ejzCnV*p|$P*`~}(Q z$UNP$jkepfWFubQ0F(*-8U{*9?BLd6XJof&**-gEP>g$n^>p=uNPpgZyoJpr#2Tb2 zZFUA9K9!)ALp1jIF;h0UsGmpjbXmDhPDVK}c!W!dvI#NPGlVpyPqcN(40}xQr!<-> zoBgGSh7K0Lo$^qsypz=)Y@v({uun7HyBC;GG-m8=r|@1<<4UvCsV$b$C0e#JI6S8F z!o{uEwbIHZsi5o6OC-fzZ>(;&{bxtW`5)=0^OPBpvS6QR?!CxHm40^ruB+v;{387Y z(#}o^#EQnuZuO!vZ;@FnhBibw!{hy( zq7kcK3WrgSL<`r4FVcZqrmHZqcYQF$D-qlxZy}#ST}tF>MT9|GNDW^uux_s4x=wggq1IL~NGXB*qy0_F! z`<0xP8>akV@rSDBxBcx)MJByAe<*z*bl)qwaZp&{$kRy6yU4V$pNJ}EYZc03eyf>C z;T|9MFeUs;lSQMg=7CQbQ=$f8S*mDM{e+*-_P>|moF^#&UQ-sM$_uH%os5*QhZ$|9 zMx7SnESJL2n9%=R9V__bIiht=bGBO&&blnd3 zzNB(}49B=(cd|^6x=x1_jA2?9E7Px=sACaT;km@P}CLI;m;Y$861{zSS ziF+eQz0yLiObsh_JMmRlRq~Xh2F#5T0kSuML74zxH3$pqiva5ml<=4?P8R_dRh0L& z`UBqWVjbqioZX=wX281v>+rWdlnd05*0u+{5D*(U(p(m5Km$&s2meg0S0WZHemk{k zg7eEgW2y(NsZArC&&ZswkirL~XHrU6kvj~pdD{MYoorTYo_xSSlz4BKC}$}P#Y`g{ z@Y*SP>$ayC30BxyoqH_*cLkA&0)rDCpk zY%QQl)FhCcU@pD|l(AT9gG(aUEFgM>mV3ki`cs8eLr+aU6zto5m2T=_foW<~u(x+f zK>Iee`8n~|M&|ClDbcn_JnuabYY2}OJ4KuzQf>f#Z#&g@K!o4*j1p;P7O4rx#LWhn z5^MjD68Bb!OWb=6qlv)-1N9y)o@YhzB1{7s*!ViuFLf7crZ9%bxCHALIs+xv;J9at zyo+ONTd_J)(B=1Tt~KL{>S;pi!+T|Yq)#fs;_8Rk)rr=5uDT&Xm7+wG zj%7# zj>*SvcT~`s?jkk&jC?HePdXe7^o_M-#I=`<1~!TkH*hR^w?T^7uj6HdGh@_T%xF7s z!x{I!z$5xdGg$ogz;=G ztp_9*l}!hZzY_d`b*W0!->RjIj1#r`pVe&+&)N3CrU|mScgH(V8ED{RIo7FQZ*6sL zv4QOGog+^#5=?vye7vK${_{23x9~u@w~ojuw=@Yg51&1-4_A-!3QS;$# zs$*>6tWxVILK_LjrUMtoTR$`7Qv{=nERwfzDK8N6?Lc3Yye(;=0_IgI++(S2V-!=R z=#-{3lN%qbuS6h*Qvr^$178tdkt^CKODJ4cwjVl-yNIBh@$twT^eb{c%`m zq=eY@$ui>djOzZhrj+tjagam4M3B*C&<(q+d1M->j*r19assCqv6vA%h-YV7b74y1 z;ODmo)sM7JG2Gw}&mMn7I;c^qHHYgiA2Wn&r~~GsgR(9+>;f1FNg)07sI2QHN5ZZ! zjZU+2=?g2dm{xA!>20zB51%PuMr>TH^(E&|m4UDtRUX-FC3~5kVeGR9_25N&@5G&! zGRVGQ^=Bn-IymBViAN)?pkqU~<2Ewk6UJk?SK4t|L3pCI1h zK9J%8FvPZtl+6z=4NU?dOz0r|q_b2djB+KT_aA(vlRh2=4%ELu+GcLpy~S|DAYED@ zhVP8j(@3fzWk`{%*QlPhSm1vXY5;q_Kn!J!yA`BH>A_I_ zkWz!NFJ88M@uG!d>gTry-?uH;m-ib0WqR$w31LnO-DePOwJrm{m?MksyIqGAqG3u{ zwz=b8JwPi_AFV+b3*;>$t_w9&m~ub+J?|=H2c_-c8>8e692mN4MOt_RH~1*=bF<4oevO?X;Hb#OJ=tdcbN3dShSH>uDz0h6{hQ*`H4IGUOkmBvB840f9!81L641TR^oT*nThd1!m2*R8*Q6}h20 z(Oyu8=Ehyrv<%;#hVt35$@1dPrNh>!bHbPwSu6KlKdz{=Ndwrs>s(8BO?6h|B-AAOc>5#T*GUA~F z`ul<;aFEWy_wl!Sb)!nzW5}|p;xOkO6m%_i2#-NMw(B9aeqww(2d{oGacpgd59BEN;N+KO9$iDX+otYBMQ! z4$~cqanzfqrQE7}InOV@q{`dkwpVx3LmQ-sa~u`LTI%PshhD!!-o|bd^*sZ?SB*S0xmY%(8&^DF z43C$)h>LI6(Izy?i9J%b8>T=Rn+`o(VeKG^1{ZSGxQ;4s?+{z)J{41(1%au?#izPe zH;~hJAS=OuUuiE|0Oo#n0%D{A1#0md`gUE$2)bCcD4alL&}E< zVsl!5?H90M_0Ph@v&_VSZv2uqF+*;A`&|`ZreCkop$rY2kzJMzS54h$o9;XAEajbM zP@2whDA%#OFlnZQ0hWH>Hfxyog0dD{qq`XT{CJrtKhrIHx7K!Vq3rV7bD_XRIUeci z@?z6#l;cT|E*xkFscQQ^JH30Pbqn(bR5wOWf2Y7HX27}?&!HeCnpMa;WGZP0QcBTn zm8^4~x?!2_wtx>|ak|rh^QSRXfiYr#EtYNB2FLqK+unBi&1dDkcj^6XPyN_jdB?Kh zy4%uK(qBEce>dG0B29^5z4tg|xy=jcrN=D?_PlsyL9vZF}rfL z8-m(EKWE5P_SN~rtHRZcJHFeYP%23<9Z+93>%3ogD2B&?Gx84RQT(BtMTsHNvKgyi zO4~fG?N=9^EVAL?BoEk6qn)*+>8VW#6r_sF(G@P$U@g_uvDiOOi$O_`Xl)VO+aC5q zEwPFl{=txehLZr}d2hiTNq&1cPmuQ_rHm?J2;DcTkQy9uB=Je zbKH8K&HxOi#E3r8&JNt5fKd%8V^+&61oJwSQGWz4fIP#=t`cxi3Si-NWR|BVCy*+p_O* zkrIaR#}XO)V%>YZ)D&O=`jITzmS+H!E{KM}GjILwD^uq4aKuE4pBNeNUxOnfGf~{Y zbfua=x}bH@uj@W}t5PmwZL`v)%R8&xdlO?B$I={LywGldF;Xta$)4W`Ef&#!J0mh) z`qt_+fyVn8mFt{^SMJeFt2EQ5xw0o)ScH@b?Tp8>#JT+@?HO2l+oQ%FcUOG*j0T30 z{PxIZo1Nh^9?%vGQ&L7oop#i*69wH6TJLg(lZXRQiU4#{sPuekyLOcJZAQ`@*>k~e zWeWw2+qPjHdw3a^6&7^4xLMBQr9A4ZpKXuGKP8{Gmuo3VLM*v%MtOrg1MKBgXSugm zkfeBwWOd{p_%;t7rogxq%W&xo~F@ zO$>&{b0cqza`H;wRG>VYeqZ3sX9TO%*HSX}x0k)xenK%tM3+B8&v>y&4qDnodoG}m zW|SV1gD~OlP*{#)H*y%g)KwlMXGe&CykYWO8RgnvlD)|6;~5#G?Nem0A5DMyf0THs zT)fRCfl7b{wQ@5vWskgL)dm_>NA%c0-s0Nma#7hMV@jzkNeQ2iqq0sv6QA77xwaKTn2p-_@*`tPa z>G2_lIkrMTV@mJ@4_8zbQ)Bf}9gD^Gq91j@lqnpezTD1|X^(ddcU;B2V`e==MmPrH;!G#&mL11IKj91*L;z_9?cF$#J92-NK087$RJ_zf< zlj7=dof>Z=cJDFK_wBVxITH?}_SF}Cy(3me+gYWud1Jrs#^FPi3kR*%oG*+EeZ@wR z4rbnsuZEPfQ7%esAaJpTvDoMbGn@|IZBozz{d53sgAd z98vGQe!Hco)}?=(9Fd%^7#gdOsk2j#Xl<0csJe?$?C}l%p-c5wy#~|K*}J;)fPo%M zm1!gYG5ANnlk9Y)5eAWBO!#ZM9D4m{)m27w4khKzcQnE1(rTaV&)s_u7|M;zl|yPT z)gvWcV$76MXAF5ZGhm7dC^7Y{9254m4$G}dxiL8$*CnlBn|D9H%9(cfeH5KOnTvlC3%9O9|y)pFL*h(H_MtRe-@XpfSBp%e%R@ zhe#7+WBQfKyLrY-RnbVt&dhfL9BNf1@|2XZ^&_2*?42kaM(p%tCxb@}nGKVzk@PT0RN<%W(E%R6hYUoCM>+)WW;|9F^6l828)ciB zx2WQt8)^_-#c^^P+~J5PjOOs#7!Z^Sr8uG==A5x9se~_ zGDqzQp5>s9!l6ryJ$6ubEO=axKfpqj$AtB3onN(BL<2oXSeLxPW>e+;%qQzx&oQ6l zzsjx5i+Cjh zDS$F2d~8XfjCtM6_cbu}THggubt2OE<+0u4rQ}e&V#?2FKjy5Hl2d8~i?P__^`^>u z_ui~STSI?5VvoG{{joZNfbX{-&pRfPN!+^|i)~fVAK#S_e8^k43n*aYNvm1#=4oY9 zHLw9``(v_j#wLvvNmOCEl<$5B2ZZlPAN^T79Un%@H^Tb%%3IX|^Z2B5OwP>1BStK} z_P3mpM-LDBF`ao-879;?#<7Y&y<@OC_{WuZv7W1K)-9VV zi;df#DY`O|P*wOr;82OhqeIF-G0KGFUe6GxIgbk{)1izXbjVpqvZyc!l`5phH{5LP z;Y^K+Q^S<`{w0oA0^0=4c(#of^2iQ#l!7oFwoQ&Ay|_OlWqjsFtCTMg>lTVgiTr4( zav`CIVhmx{B{}NAZXL-mhVbi78AV3nDoO-Yz+^>4xN8Ql*WXHj`R$4O_quDI{!Swe zDw~e~ce(TP#Oao?oNkt~;%g<^gDTYJP3 zdWSsJ*T5z;*y7wwtIP~_!Q)9M^`C~{DqDE!UpX#zaF~=T9)Uz)S!+yoLKp*#FX1Fs zt`t*#etSZz&7zQ50iZm>hBJSfnGX9%lndSdhXI-(3#dgWZ`#)d(otxOYzG%%p zay76~MRM|udfH_ApouGUtvk3|tt^I3a6|ti;;-z!DY1rd`#ITqka6YmiRde^eB4<*pLwrplL7#f?l>!^}sEaL5RA zzUIpTL5Xf_XJzklzUO$hvKZh1cK(u3SvB~t`XOavx%k@bR~i#$aXWvlEYe~Pj2JoqDydAayuuxc85BpNi_%wrDFT~WTlwW z7^Z1^gmvxK^Q2X=J#Ha+XBbHN0sXlmxnkf(P0%Ksv^-yoWAU#x1`4VYlR9O|Xim#1 zu^RfME2-AIESM=^T1wXEP2_Ao^DmjtD^-3k5*3{6$_q)E+%L;2<;o28jnyZQI3(sm zP)v>3oA=21JIzx*F!kA!6Sj!mEQYEQ@PUv~D&{f4^Kd|z-bCcHrbKbEDe^}Lm}90d#_gu@raRu-zNc^uv>wlW|AgYn$tven`R(m8~M zz8;gykBhC1>Z$(N^vN}tP8ks{*0GYe)pRI+drIzJ(Yrc3p>0o@xmOfzzog?7K9nGz z(zX`@I@UF?sdpU+*5N{30E>Z^690@Ag{vJ6ohm+i3U3{4BWH$+WF5FuTa;QkT)Ux) z@&44PLsm9ouc|*~M-$nID@2MZKfgV-ZM0a%LKO^EDW5&{&N*T^#!=7>jyvPUGKP|> z1V5_8v=__BF6JpmmAg~LvVA8)7K2cOsaPkLKX^tl^nh8k%IX93$?Q=yekBHDxG>#fZp4xkzSjM<8V34YS=_r?xin1;QVijO%5n?%m zUaaLw8FWZ2zxj?3bd>9wAxc?hD5DB|sM2?;*g`%>RmIqYS0c7}mq^Rxlf}!WAm2u)} z{!l!UG_bWP4Av*-NHFoRRyGedwSQ^;w}~t(bSl5I#wUW1CC%& zk|?oqj}C36984kHJym?iCPokfd=%lb_@49Hl`w?>(wDo)v0o->9=gP|SHk631`~>@ zE-}6B6?sGJ7S*9WJuXAu&_<~Ls-JC7`+mIafAl>)71c9vjbrMT`f|mU zsTw*8V$;L>8Fp2 z^*pJpJjc&xXLmf*V-V-*lz{@;9^a&kQ|xJh(ge)(b2;L_<_#)ipzIb~WPLXLi7&;} zZ)f*DB5zr^OgmfKH2bw^>jy@CU{F#hk;*+!-bPn8MG9hvc93;h{o)VfsO$+4xQ(um z@}mmR|JF^@{g1L3ZeZ_q?kz7WktX=F^9p6%sYW@l!1Qpk7-d#!j6JeT$b&|&t#knk zOy4Fo7I*OKA~sV4`*^&J9JxZ5&{DE%5Gul@dWV<#2FHx)7lOC(uBbZ3cTuZ71(X@n9o0pLK^r%wCH%^cp2T&6JI9DzpIQ@yLpj>4Rr#<+N{Sa;~ox z>7X#A?PXI7pS8+el|K)}EU z9y7Ygmespn@TfuHsg{;3%mB;`NM>fRou58U4dE18cyJR`%u3NXG}2jno4M1TsH4Ag zc%Kr}3*<1?lfj|*AcduyyLexW;))1?6@j*U*H?FaGrzBpt@a$y4LPW)Fem#f_{i`~dgo`C`? zK5{so%~F=urA!f?n7G<~YX9EqFHjEmqoNrn1Ih&MtduLVo_8TV7 zg6TKm96b!iF8@jz_6Gx*2|>VL50H; zx2B8zQGG&4A$(RUbI$2xm5PWx?uruo$(iA~0Rm56jwf|1^-VAJ#DNR4A(K_)8ujPC z8*o>~WoR6R$FQmLvFO)zd(j3`Qlh-1s+6$pCmuT_A7g?CQxj6NPo0$uBL{}G_1jOr zpW{yW?Q8WZP>lWd+zuDq35QTANR*aW#@>}9=X&_b1%!%kuYRs^mho%8p|{Q=xq zNl0m{&wU`yoyZ}(43-&ml_%btBWL!TrwK9Yd;4U=eb%8XZ)qHP%5zUP4!7R-wimF; z|FGDT*IyAIGCoz3C;{|cXRTs}l1UJZ)j#PXm)RzxUr-@^a>W+0kDPA1u_;m@T&ZOR z5}Sk~*=Nsb&9y9EMd%FlGUb;!M=r@(YxEAUanzFq0onGKC-g@g(HN-vUaKjO9@Y+Z zpHD8`Cm&)p5DBJ<&Aqd}c$@rE5Ii0?#L8L37!r&do}5)Ehu>>f@r^Fa&&gqhzgS_R zg_$QKcgZM*$^(I*Yk5|_;4KgZl?i7t+IpeQQ`OF%%P9D^ z>-Oj|E+v2-wNp;>rW)E}PyVt^=6-%sn-DYl?c5ew;sp7VkziV7?#l<{n6Uw6aidG_ zeV4`YRdaOP$JstRw||^C&N&nqdL@RR6UUwZRJZKh!UXZA_g-V3!jU`oxHw)BZiQhn zYyyv0!o_i}%T@DugKXk&1~m^vQ)NGdX=(a%qSep+LjQ7xA*!tJh3tb4zmNgpUqtn^Fp0{=7#{2%71v z5u0{aF8QvT(pG`P9xSL9vH=M}>EpBKb&Zu1{u!--X%o(SWFI_wS6gh}fqgPN?kC+D zPz~ruQsh*o3Fu+`lDFoNHJgtG1XYa^^!&5#FvpTm@P82Uq7&upPf`^IgJb@)74Do~ zzv@F+nn&KssWNtGa`nJDuVt<*J`(Ha$Xak(|yY_=}+!%d6-hW7fT^?GeySE4Rtm7=5`%DMx+wa+GJWHw)yPugB>U z8v1;O3N0g^*RF&BkECPbImV}8Fi7Xe2IT!M{;LQckNa}v>LVYh|0)wwMa0Q=YiFwe zs-JD=&p6`d-|)NcVK5RU66(u#@0ux3m1<)1nRtKy$UfaCYaa8yUM)Au35ucOL5Y!5 zWxFjcs=c4zbe-E^+!0xqPvd6QJIr5FF7M}c%FK}W=TEFJ$G%`|rrPTBIZE7$G^3qy zcBMj^S5_)lY|tBJYQY>}+j+_8;afshx9#~8OXb*1z58a`-291`+?ASpHrPGj1r$5k<%t^)l(k$`0f1E1lgMB8Um)N`|SMX zMdEur7es7{F2S+JS${cF?c4l89po&gkyJ5A^S6}B##~|rwoxv>DpGVJt5-(I+4)(c z<>)GXdPp?@q{CMU#yTHPt-lPzgNpG69TVY+~9uZP(8P%R3F`|SlOJ48NN6OdqRn*VJrIi-H* z>Nb~uWvU!)X3>g|&t4EXW1YH?rW5F}Zvl_6a+!JiY+VJ$5SkUs6s`|~(lA|6w^U}f zGxL`u71|4)i4zMtn%7+}Y995k$SJH$poeK53&_$u;V}^kOP{^)e2Tl2%z_HKkrH1h zC$)G(Dds7cm@X%AtN?o&9{EQKqPYIo+_{geq!uU*jms5_0%d~L)6juL5I#2WUJ_(Ms8aNALt*x&>0Y|UfAPow2@ zuDB~6AjM)GcFHx6>!$=27R36C<*MK@Z6Hd)orE@#hSXp~#%t0l*OF)h?yC~^H2 zxg`0J`mfP;;Z0lQk~>c6;4e|1f`zx9mPyU)iy2saGm?&KYrAETJT7* z3x=kPt?V~c88`Usf{4q~V(qiUDCf5e;=^S9Uiyxe(gi~AC|RF`xk|ZA6_yKsJ0?eW z%+XfYVhiSV5L6ryQL@U!LE?;wBslJ{c_noXLv;N2ON(=owMSEnnQ|DOaaLF78%K zM^KUmR(mhX&DZS(eL#A1u3X$Mzq&;gT(?dxmal7qsWR%E9LxL`)jg)S!n9MVhHiMQ z%92UsRZt~N1T8I=gG)?qN{C3gd|B*$rk-w5Rmy~m&d0mC9DPH5&!)d!5(ij7R2Jh$ zm4C;If4ycB=I&O6MZD_vV_>0fa{zSg?1OT*1}Qtr&i>{y z9pYLRBF$%Q5sPlq`Fc>?#8^$jkc09GE^H&g#Ob07iQ@gHdfC?m|Kfp}&YOR|tS8=^ zmDoiu=gK~3E@>$m$0F|FJ@N7a%_B)``>POD=Wo+~0gG)fnteb%!H8Y~Gf2sl_C!ul zna_M)7thRe4up4;5AvBO2^UhnIW0RbGn;K%%HjnboC8Y=b;c!2l~~+qi`dA3Rk?u@ z7?}<3j33O4Y?&%7Yck!wOV0OBxCb;+I+Uj?@Ol<&4Sm6-ou)Z!cMV!JT(U zwvG>&BKCz;afriDS4%7|Ef%Y|xzw43JM=;9+i7Bz*(WqSY7%PglLe2|)?!VS-=js~ z)7iQKRjPJd_;Q?E(sq=#m`1v!`(7t_@(lq>m68q|+tTGD%z=nv73+xIoFzxrJFlAu zOnIs0WQOcdmer<|;ODcKj6N+!vORebiy5Xv62)J>y?U925_6KpWZn}G5-_I3@Ezhe z^3zp?Vru=$>C*pCh)vU8lO+TJla4v6`)C zB?ag{yRccYv#IlpP!`P;dx>*dY|rndy1s5@+P7V<$Qdl#VpGKwV9r_b8Xp|QdIWhz zW-s}$+EGg$i;>xH&A6t-U3=tHtxS?g#G>t`Q{sK@1SlLx`d@^cVmad9X>yoWWU1>U%u?=*q$y z8Cv&`2$ityr&^tMpJXd-w+RPuaJ*pvvql9yQ#GK%hmMO~EbOWh_y(~~K+eEGXlkSs zmP9!7$??CLb~{CdW#O=O;@$2h=^dr8>Benx$y?^_YwQY-XD`Slds3AIyBnVgeY9-5@Mx)+!>u&01i%g#i+8w>d=X#}{r1w^GTpgtFY1871U~zz zmfOS^y+`XMHQ@X0r>5@{Sxs-!Mn*I|SO6&K8mrozk_5rsC=0rq*E%)fsf8)>=7$bw zZ8h|#HWi6C+4rEnr~a%I`AqZ`ov^M-F@5UwEi#G?h%(dDL1=eME~#~OpAf5u=gXo8 z2kH_a_1l3}tKFv;y`pCYL1n_HZm2Jx9`=m-?kjL8_d~8MjvcLsP8!G37UlA$?q(BI z`(pHK_lPyRFd!ED*z-JTx{J2)k*yH(hcXs>R^oK7`*<YRv`BI(S@$B(3z{okQblIv6ba#|2FzFmT z_1kxpi+VO!@udh@{f|hOb_kb?I_ce66NO8E%@MCH{#7>*t^Ts+8{PcDGu38{>f8PH zvcPJ0{$F0wA%*DP_R{~7d$2k&tIrP9+9_M`(aHoE%GvhP9sA^@r{`7&ZlDoqz3(v- zJOh2mM!BqRhJ~h{E;R^MnQ}Pq;h>8$5(auWTHcb873wY!JMgG%dF@kR0d5Os(OrPS zd|m_FE{HOYiDL^@&1Wy0U&|evU}>uxrf#CVu6dj4uW2B2ubj5QWPi~}1IPEuhgmPY zNSmVyV!w-&4?kF6g{2x#`RwJ*I=D~1I7%}mw2|`tPIpkxqx!RkSWI}~iaU@YoWg;T zAf!Z#PZ{J@8ekAw%#q#yoS*~7lo(Vj>s0tduJqf>Um5Q%>@-s&ML`H(opuXZ4pTfd zVcBbY|UofKICE%JO$}-G)cs(+d%sm9|Aw&$@-!qY7sSwfc3B$^PX1DglR9e|f#j z5PEe(toqq@k#CPXixcju3Z?vZ(PKN^{O>R5_z0-pvS{*VH~;hLA@w25zUo4N8v7D@3)zNnRu!#OI73se;P?G;_a+>u>R=y@fj&~|txIc9c+LYoq! z^4&h0^&*02F%nGA+qYU-+<30227$fC#CN7^SC%NGD^^Z*TX2|zKa^d4_KKnkx5d{j zR3Bz|tSF0hr-dEURbWh&kN3G#|9zt-pgx38YPr+ypRbXIsS+z+Z12t^p&|b@A+_@P zCT{LKZPg#94_Z02i93Rk<3+U9JXYqOawoH5gjh@SSW%KHJ8`@Lg8JF^$|GCc1suJ{ ze9H1Zd*#RB?t-qd)r6uicFCtdN(;pYzzWvMsXGd^a(Noqv3z%E`Eb3or`@n(bvHTh zE0gAn620w}&5LB5_h#%?6);ahUN!`QA>JTMSd0_EZuw_7Y)B|n&cvd+1iH^~*DgMz+_C;YzR zs28AAXF!R#LOHwe4&ARC9`_14dr6_{7}d|Vi~oL3?A?8J&A?}`s(U7QjO{0kqeR2B zd3$GK)o)@Z>DNZ97FGPqI+?*VLHSl&toYpsndhC^M6_+M8pC&v?6+k|;Rlc0d^wp< z0mV&~XZq(ly?NIQPzs#iekL{7N#rbkC4``I1cPPU^Bp6q&>00iJf+&46MN3(Idy!n2{hp3> z&0~!(;0oT4ia|L|Slc|)T~>I9&SDkRNVgWbop|yR3C8&?XHMkxr{yWFnLPU^2mK=nauv$JFY?Y0uJ5GXP9khAK%d3(VSI7l8#))mWi zY$SP$Sa)wh@Oo0oCBamO)Vip2ahLa~9egP9aHR7PbJM)gKwy3RYPXodL|v+& zF(v-C!|g|&O~tfIL(i?_{Ef{e{!rBr`&pLTuXwN;y{b}F2^0pore=kOps$bY=%zh% z&0fL!m6zSZdDp4&x{^edvR&2*a0ICWbeR72vh@XrCo_>^pr4EnKFfkTpp-t87PvDH>KmJ`FQmI+2CR5&Robq9a*Igdc=kmI{2-J}weD_XUbP3U-`;Th zUbpd;xAbx;C4Bb!uPVBK!P*v@n$B@Oxo&=WXP*pU4@>}$(}C`9bB~V_%HdS`F0%U; zvdwp9*i%<)d-hECb(E;21UOWQbKQSsR-l+_vFk5HcmMd>bJ80k5}X^IiQ32kWx@^h z_gcSld4oR;jtvF1T>H*o^g}8B2op@5=B`5=XK5lTu{b*suYCHrOpG~t_7u0ZexQKXVqeJvmPEkdm~>D z$qoPACLFNG?~&pjE`U%){g9HI=??EPNlEYsH?I>{emo~xrBREBK;KvH9A@uTmB`ao zI7ff^%j4>JmHNTs^=8gVhNFm7Kg5aA%#CLYD+n+o!DDriSngc`(AsXgF2aq^KBJ9{6u-Tx-BCB?p$CBARoT?2 zmg6ImFTylWh190O@oopw@+x3Df;RqiR-B9bNzbAY4Uakd+^*ybucQP7s&qIl9wwQG zvU-^Uw(0RL&S7q}gF*c$G5(x$xJvgw#$ubY0?y${=QJ==qVECcz_r1@>AQ_?UfrMc z9}PNK zSF!7ho}e3{(dC;a&hxaFt8RF9Vn=sLv9`Ez<)+=qVj7#m)K|J7<@FRXjdXOFPjyNV zK8ttW;?w965L?;Bd5hiwo2o|Wv!5Lk=5{@`LZ3p?HhuQ-FtAE2jT7bP5R``}1AFDQFlfH&Ej3Y*%oE22dunpY4<((pX`EU^?k%B{@WH zd{tW=a8wy~+@sLKnvl+b>idV zZ}q1Dg3sREn)5-tA_7At$+kDghlyu6{SyQS`uOe5&vtNQEBv|yIH>Y{u{-6~hU!wE z3M)$ND|BPG>_Vb9cG&!5H#eU4^8Xln*SM;R?|&Rc#Vd+p;uS+fK~#)P#qjKz%~2G* zu|*?MQHs=}BrP>k5k*Chn%9IvQMsrnN@Q7H&fdq;C@eGzv9$1$W~HH+rk3`9ubF)~ zyuSS&{2%NGXPud~-s`ew&t=Vw3CybQeEhIO!v$<}Q6+@b&-pG*{}Bh10D~#zQiV?E zTS9_KIE{>M%qfvgC62S9jAKC;^`*sjy8GDAVQ&*)%fQ_^I@u9~TXWzO*^KX8=;i2x zVG~fwSl_7~a|FF7hmiuu&d~*qAnfFda=LP7-%G|IER&L05H)otZio67NKLV-x-;44 z=(Ipw(?q49oaLC(Y4z8F2TFmkvCt80#SsQhd+cQH>{}#XUj|sZ0$+H5fWV)il5!k!4C;_Hcb=T%W zj)334a=i_^n`St+4z>%X{sev3Yl+MgiA$s{9#!2nA9;=#0+u;_yDKHmD8Xz`3A`_v zE~=UXZ;GKk=jR^b?>K2BqOldvb(@mf`u z=r|(+P-5>%$7^%iiW2$4Q9sV+%ZFSRpXi_HMc%zS($V7UJmDx|)7^7JjA--N)l@R_ z?wNMh18aPQ1Cl^ElHq8vw1;rRRkHf-6*a6F3n67F0gX{2Cxb;`m_XXP5s%!ao{p&_ zUJ(0$mH#&Zoh#PpJ6b%T7SpQc2G%$%E9K$x?Z_{???)&!6a^^L9{>6%;pSzOr~)ZX zf)x!rmZ7N(huj%{j!`?S#PpA_SgvKh@f(f|Az-jPszg^9*Kn%nZD)g$`x0}sU@wc# zCX_>o^iap}#}?A80+5p~<1Z=C;KNr19zxsP?O_Hbm5_mCiM0hRw&4hpU=sc;cJv*j z3pW71rslQ7#oimd#HvP1a%Jvs6~@0vumcY1%3NE9r3l075D+BO8`q8HP#>DgMk&uL z+0hp%yd(ugtD4s)$@mPr^avPC04eGf#~7RsLrj4bC1US8hJ6v{e8oWM=b;}!!va?N;j=cKcD0j8|L_!qAjRgq#=%@i~!F*?D~IzVQa3FY##{2Zx1a-5A)C^7Pqk&7ejfk6gc-XMSb z0}LcmL69Yert2-Sm;wy)CCrN)YrKQQJmxx0m=_aaLNH8#Sm~F%@5efphR+hg-%+T4 z0sWwzRl@35gyf4HQ`k=}W(Z)R}=8|$T#<-VMCYTE4^1Ti_Uii#cwEU9CEicCl-;Z^M19%)R zXDhInl3M4vBhhRH&i+AtIpEE26=YnSWf3M6ObV6hs(>MVbl{gSbBqS~G`e_H z2eBR)YGfklB#z#gr$bvf9Lr$BFZn$^jZ2sgYYANt>zbP1Hqqi2e6H;jY0#%0l8OYfb!Ed7|%+IwumSO{06~v+x2y;@5k1#0^f8OQoF>Bop2C5>a1)gDS z*=P5uz64hA^bunrj`Rm88Tc)WAP_zVZ=z#dczgjbLlTYo97V2&I6T~3C|cfa79 z5aYm4AG#b>@N>Aa04b~aPMiL>#5eQxWf(=1SipyrOAn39a>|vA1j6Nq#xJCC+_Ib4sB!M$eQ6%9%mP3sq&4&h$71)!y~g(>)|khiJS zoc&~2{ona|8n#lye3I#&7V{lV;0Dx}NcRjZFz#dgVgo@S-Sc#saX+h%lVDZ%-~zU2 z%xxjQOknq8cmu4P5>u(%qe9*Qd*4Ew-`hQHF0kcT^Q00{V&sm;?d2U%H=AU(Gy z@ra`(a%nT2;jm}S5w;XBVH4nh6o`EWCz7Lc)g8K|3c`kDwgwCSBo;P7iA_GnDMMa> zA_xJ&H*456(}d6%RkFN|;_bbtkvE~QtknnO)W~)S2U%s`5l1W60u`#P z!$||U><1D|9P5kN3god}1=1=Cm_2I`v)X6-3Fs&_Qr+7q!1%KNV8sp4AqgdL9Bzj( z^|spYx$w|Pz;Vg>;9(QG+{$uv2(c^e;*D1Vv z$9oxLZnhPEfuT`l)I;M5B$Pp<)ON3yW0cgNQQWI;Qr^czCRCRZv^P3i`2+(A9aW-8 zxxL8|Mme@35DxGGVL^&fo_J4WGf2Xm3%mu^=ZFU$lvs41?ZORYHV}XVgp?xgar!Uk z8wFsC!?*{MIALJ&3kXYmK#&v6sD>&_FL4i?xKskMlCU9&d-NYHctnX3MGv_L66;|c z84i0_S8xwpLQRCSIv!v8h|ikjv96l!zb{u4B;n|6hxhec;sruAh?8Q!uS|R$$ z0@z+`L}`YNY;b`YL1-KgPpG$Ve(&k#ywxL_rj;*%RRr*+BNOT|z0LG=+*G62g+pa6 z_Y7D`z)Y*xRVYWPBwL; zklNQTR$qs%w;NO+IeB1GErhSPVFT|S`rwJ3TuWu#}a3J>?5R8fgXj047W5TUlRxUh%JNKlc5Hfc(?G|63=JMhR`tda*tG<0}W9)(fGj&@1Q zL`cCVDDh+s8!S(omq-i4eAy68W0Nj60#?Yp*ifXdTyFn6OSH=|K{h?g+7LL?fLF9AFH>u2{=Pe1KghEWPgNii|rQCBPKA zobcAqt>D7yvI=gdbz3e^D3kVh6y{ibTHZ4PEMWR^7TbyGV-z=W{1olzhHF*xVdOHQ z+`08^r#V{;9*}Zxt0NG{Yt2Q&CTzOD$1z7AOsT~KhlF08#k7q5POGEJ^}9^_d#*Fa zAolxBY!_yWVLsWQ6#lT4?ZS~%s4v@(!n?uxK`iM(G3mMep}~$c9IytY64?HB{toZ1 zTSzQ*(KU5{`}>Y|9W;UCs__0TFZKr-CE_9MvA?UYqdoSmK?=1HsM2J;l7ea5RB1vG z0?Hjd5k)?jUvEGL+TT0c=r6;`3OGurq2K>RC~FE`U<(szV4{6Rq>n&~ z65!FllC}I$-cLY+4gsE zC?fPA1EEBiJ3EW9DSnt@!|t=QPwpb+=%Et-AzY^tf``OmO<>QX?*MDj17Ow&_AAE7 zsjvqmfiOOXw|-jQDr*zaAM@6)18F|0NFJ{Qvjym};a~YS>|dU1L}4-ms>^WLzr5HO zjm6k1AeBsCs^wi?$tT#P9FV>`n|F~3hcc4)FTKu|m{<8gLExC@$-8WcafQ_W%w%H> zCJj|Sex;85aNFf>~v1wZw4s&7>fbje`p=IpsF@#7}899P5= z4oCshYvHUF=08ZeR5Ia#Ub7t$*myw%lmg+Yt%{06L`{pK%ALgNIMgV}Q`0Up57B$(ykBaRX>k`MSNI)a|BxA*=)`@_nJji=~e8T<#j`6)f` zoOJ2}9Nog073bvur93hAK(7#`2e$0yqdi46g8qLA-LENr81EBwS>pfda$v||Wd^c; z;5F%}12eK6UP$-23IkPH=h+~2qsY4P7dBlMqMtw&=pw`6zz%=If#cSrfCKRX9^12-0;>}afP-=$yF2kO(P2`cpi64jmHwFg%RweUl?P#LEEembzVu5`i#Rq5Ggl~1k5w)5^k9>bEJy^1 zCguFueJoGvz=3Kg(QB-6`C^o7y0oZO7H@$sA=v{8qDsf>Y!tTcKy_)tqRu&NCytdM zrr?1Rz2bDz6rLj-VCr9?e1UEt@sK5OtQ8i@oE;KW=@M-mMFvqmsznLqo~*E6#N0S| zm`I;YW7DvPgZk1QMT5_X5-4tJi-QH?%`KUz2C)N^S^SKLF3%M`9mY<>gd_#Jpv0&` z)&qBQlt~^SumaW^%HdxV$M7(H0FoDo;3kxdEqC;LL8T>5G6A4RR_gsQS|jD)3L+fS z8zKV$KTKd_{PZAsRwofhszt*qSgd)of-aa2f5;{xZ|im)kLY>I`ny*}z(}!Ah4VLB z)QT)Xk^q6R1CDZZ%w#6)gQV^R}!8Tz=0I*G_>pYBN?A6EzQ=0(xA1Q_Y%3-wpZguU) z{?7V4*yT%Aq?EnB%I_asbD9uR@?+VS_v+n%b49V=Eajw$7$ucP;$*IcvK zMW5#?=g^+s1`b)}%VYXlbb6_ZG|-7yHW0fwQ{kwNLdxmctS8?6k~;5~cV^wN#0ix2 zAlCB}^^4{ViG=)nqi~;?JV!am2aoxc*RWPmcW$mz_ z>jG8@zo6R#P$sZT-oxHBx7wo$cpNQKZaCIErGW4st6yGk;J$g+;jTs30$5`l2%^`; z`b{_HgU~i(3`p+=Gr!w!31F}qnBJbx8kl#bg$Tl5>8u&TvJARJ2{$}+XH9VQBJ63t zK>wG=!A5r0u>MHA=zb{ggGC*pSYx~% zz#zSU@Tm;P0!$85H~@=PJ=nUIHNoeEbP>Ki*wT|-z)^#+mW+>sZ920I3`J2~y5Zns zN0|G9cPU^@vE!21FpNE_oL{knU1QkS!6I9Zm_kY*w*ARti=_Godl$2(;{ph!RR3VV zFy@ZlIPnk`JE){kMuZp?0WEZyUf@_cCf^nO2l1WrM8M%+6S|KZpLgC)8v~ge#mB+% z1;%d{fBIL3!@;k89J|jg5+5AlsDtCP*g%}c*bzMNKne~J9*EZi^-Tk*;^-3A8}_(L zAB%(k(Kar{as>TT;(=-?@mw{tAi1T)`6B<9urR{-9sB2WFa>;6Syp6(;RSLbU7XEU z;3G~vQiU#WU1Oavkx6a<0aaEXVY6`vPd=2BB?__Y2n`!mfn8BzZ8@8U#HA1r+yoD- znonCOPXvt;B|h+Uyl4|cNYmgbvUBa?Fvo zM&3>-4fJgg`#g7+Xr&~uU#=LBVbY$I!vk$?(Z?@gj|4%Nz^t|_u@CDlP9J;}##$nA8ud*h|2vkoT)|z<#UXbe))EW4q+A)jK!>_tbG%~C zNyB>}xHEQ48R>G=A_a|hlz6xjKV;w@YI}+|!0T}vy5IqiHfMN) zLpR8L^r-3~T%5QXdE6b%(9?A^NZNWHFTM*bF-yd8b;6vlR!t(Lmhqa_%4voj0 zB}T6#MVRo=6n7<|e(KvH9Nb$NxsQrVM;+Q@cRU-PB>FI@2IxB@9Ahw9-UK)R41~45 zY>{~s1FC_r?V4VKTe{t(*pcd?Jexfatpg#IRrYMv`{GUv6esmjqM%mqhs>RhuqjG` z$0w(hckva=cfPlWzQ5u~`{HY=VwTX)u+zxvONC7()2|ZsA&}tWQQlb(!|oR1fl^>v zenx-#NVMPqJ|NV@IpSLm6}m{V=kK!AcgKoe4&bB2PbtcKuI~%{f~&JKv1S^h<5#IfanD@GW4KrXKj@tS!6RLvd)~>nlF(w4(i~I2 zL9iIK9jO+#DIT_M{B_p^K=B}EB%=$5;)De}cs4s@JaV-`#Vo=22y?KBkP;%Kuo&Z` zbBib>!@v|(#^$i|xCakE$k0|0jhAp@rPpS!F)s}F?(tp;f7`%U9n5WZq$ zQ#>HFV65A%fnv}sw0+-qlcNvrp-1`G! z*af_V042q~KP1MW{%IL_$PyC*l)2`*X_Tlk&0qQ6>=&kzw(n01VZYnZTrW6@}Vn5Ja=k)RW+rpLfw;GF{ zfz&oT!8kNvvCu_I$@VptYdKCgK+5(=5Lyc8O&xBaR+842VjA={U6{2-9+*dbum|SZ=|Z`Z#szE;raK@-27gH_cXqX(x3Gu6 zQ4(>+h(&f7j3fP0;$O%vBgV);vP58_@;73gc;KS7oon?w&WSR@-Oc6Ybo0gl&~fY`!rnc2LgN$h9?d)FkeU(0}n_U5jPB5 zBAhJ@2-bpOrR9SNrHS-K#)>efgyQt5YRPPG)wjUW0sUQe9lwf~^ zC-x2J<>N;Qq@WrIw~w)x5mN0yC_^JXY^j%*avB3(8|*>q58Jn| zkd4R9fy6@)j@&uP{>I*WXe*cw)1S^}e`CKXRUxEdm;c#nY&|yGX)(=$>tc*P*xE%% zNvx%LlM3Y-EQ7h3pJ6sl$|k_A(xg+=texYC51W`;M+jz3m&_1 zg4IQI;Q(Fo*o}L@^UWnaQKbkQrV0=rb=rP4&(oL)SE4>KjkbK9?rTgmzxyE}A<>rK zB7JznS4#<))b{HqYL$*Fya{{I zT}gU?JWpDxzYD7~cd;X+#Ppo<ZBDMliol*XEVn60DHmHw|QEeQgg> zq>s_?UNe_2{jrndy15JnJ{lW5J;LyTa;8nKY)H1dF$v=ah!hPQW1CZyG|a*Qy3}^m zhuJIuX%>(|kClx%dtLbpqXO`dV#m0-8KV(n#1!B(HWt2(vd$FRW)fj?s0RzfLaHfN zV{yIj8qZ-Hl4(;bdjTh=%-htOb%$?qNJ>1c9s|EXq{Q(&+OeCM{sBX(rn1?8)Uaoe zssW2ju}Rn}H#hfz;GqzYq?{yn9wR;*VjZHbZ0^BQKI{_@N}`aCdad_ecDLR@g1x@+ z6uXBrnTRQ*Xlx!|9~Ql+H)$%G3$_$_#oJ^?H-XdGvWxCY+O^J3ZLMs@3_sQvy{gS``gbW0RkeWa;pa{Hd1kLFZ7U1Z4Z+z=aKybh7!lR zsw~Snxo#?T*;pHEEJgAYl@Lrfuc)wun{R{Ac1sH%%P7~ht;V+bZc;K{?x5gbdJ#32 z*>Ju7uX!SdfS!%J=6k^VC?QRVtR3!-WaBy<+uNQpRxRpX_l;uPk>xtF?=oRiVW6E| z0@(FGrZ@veV>xXGv7}OY9*QjSO^z`VTQb3v9xKbecGnn-WDWv`M`L+k*6N#)%waQI zZMFwV2i^X+P6U5PiqkzmC9!TYZCx6>p`HyfaohYQK-24Z?k|_*e4SUzNo& zGdj7{->>RZ{tiWI#e z)UV@lC@Iu_2MNn{zz%&Jq;xlTlnIL+dbQL@LW_$Z5_<86*-97l)*TTa@4pvdOu+V1 zQ;Nn)-f3=sh_5=~zyreO6#d88%c7Bnqcry6m$k-7tO8OA7)WCuH4igJ;go<9curLL z5L=lRqW@9H^b;>HdoWI&H9e=Xk3SAFlF(1i2aiO2#b}`#YH(uqH_gFBU#ync}5!k#Sa+{XMRstR@BR< zW97}m*!bWZ!ckEezp$!K^VrC9i^Mwz_!>L68Y^dL40NXn&)th=z0vZ6AUapk!^GHm%(#n2V5X~<3c}8Ua2zy+@>v1qjjF7~X3H#x< zLZc~;2S$BqkBejDj8SklAr;~9$+pexvdgnq8D zn5gp0FU*pQgduVRJg2d1841c9=#dWscz}R|O!uvi6IIHH>93z>=nV$GNj*ntvELf| z9{(`g-Np1qMG9|QB0gy{q;Age<38}JFD?3nol5;$(Q%VEc@_nlT(Sq>;4v{Ib@53d^;q zFRR?FykbAK;9X%LA@+7-oTX--ydc!SjWhngLAE}ANSofq0Xh}Pw(J@IylX>#eQqC9J%IAeH-TPiOWL=GaJV2|4OsK)LcHN~AhV{Da{4Hv9ZQtFeD@ z`$s!$5ziBU>rM zFo6t&N8@e}JoIXev&kNU&|rbx=!9)I?EpOl+H!Y4Kjn=TToA5OmBzasvgKIgO9T&5 zqKVgawhS%IZB!xUcoUy!B@gQ3U!fdt(yK_XLZbwq2v>5C-)nU@j0!T%_k}m@8)^*0 z@+6Q7tMg`PpmXJfyRZjup69FofS@C*!zNbV;>lA+6nX~0A(?s>d9%x&L!7a$anG`N z7K)o#qnu1V8-*Krf6W(tDS#Wv+vpdJk(kDIb6TwJf@sTT*TCD#+d2GLHHJK7AfcSM zXBw+Uj~><{##Y|8c(E}P-+fZ94DT)XE=@4{j!B~*1kB3Y-yCGwTCabK>!!RcyU_b2 z9>8Jc9S#&IA+7`}jd#eoaDrwPps7^f2a8jA=7q^%I*$AN;eDbpR-32<2paE*F3Sdl z19YLs$~(v7=5uUw#lO-GUFJ9Eo#ER}m=0S(uIzfPv*L|qbU>G`?6&Hz{sS6msuD%I z_|8gD`pHC4sa!V<-k+E6z7*SiVV2Sj{SOqEp6mXEuQ3#zJmQf`;qW-d9_x@u4o68G zk2g!uFJcUI8}U&>G1mPTX2f94lS;rIs1jOjyo&B(-C7fL?Lj1a4f(`K=-Q(Hq|LU`aeuI6wIO2`6akhwK2oI{zfnz z(6>MtfZ#`Q>D#An6z~Cw2gzE}Vo&{%#s|!Ji6qdY@~0knpu{~F;mM*JBSpEOz{rwE zNacKG06IowAc)rZz@>+I2OMD!r1V(%z^6Rf(*5yHduV*{_9HxC?puOTM!JL!w)0-* zggeAq`Jl@AC&q(moq<9F!+5}3Tg84;paqXk^9%*uVn~qo7<@O|U}$AfU)pr&npz%! z5l*TM2aSh)n9X}!mHi-;()iG1oP&lrr4o2w7--m_=DbH6nQJWxHF3Nrc)$h3)XFWb zaMKlVG=a7-qxS!YFvi=m2YH_mNm8P@MGLX)!6tb8kkBp73ytx1f2TdHoU5~Wz@EEK zzi6C)iJRbmcHI%el?&C%Rt&4JI&mmP-o{XK&Hx6oa<%b&-eINdz+A4jF6AAP2MQd} z$I6-81!E}o6T+1wN#jhvaN-R#235}HB3#A$^6?4r?u!!Po#yc#gPVyM6V49*Xg+@; zYPr+d8V^5>gGd{Rja#yH7~Z3j_d;U;H;^PNkNB%O@3kpM@PJLMe0W8Q{T~>abg_|N zM)WFDmZAwJ0|~JsahiCq53f@FFt|bEg7C>Wm*<9Gk2fMQTmdNQ%7{;@jae9sfFOB{ z{%0%ifrA#nP)dnfbjf%X*~cB7jnbHge#%RjAR-DOlWKywjTKoC@xD(c>M6N z4=^bUU=rzs@m|A_Brn9Gl*T8Z?I^~S34X|TZhFLBo;f^Gq}7Z?Hnl8x|0c zxNecgCisPTM3J^r7G2V}V4|uH`V{{PVgqy013gDFvVcA#CWhIOLjupy z6V2l>(s~B7fmA{;P1v!?XyMAFw(?mGBG^|r7{z%YJD>HLjqUi-$H_zE^FOokH9J0{ zz2>y_M&omjjumGj5L2P;T+H_DKmr7QNF%?sFo<2mUgCV15Ra8FL}o}iS~y4%=LhlB z=jItI2F+FAVWY5Iij6(I^&hULZ|OR3CCc{)@tOiUN^Fl*CgQUODbj>X&u1tdJijD2 zQZ*}2n>62e8U51|*}_=)vdKQiNHb1_`peLaKZ7nDj6;u=FL$pr9>aVgRRIo-uWWi( z&p|`qQ8EQ#vXA~QGVQ5^F!CzveU^?R?Z8lKyK+T^F%~JB_*Vj3nH_2j$5@PblnJK0 zLyQq}6N%)p@@Tpdj&xJAgq5$nzE%GPtANl&I%?(l&c+x(NAyS@>z>)l-x;x5OhUq@ z8ee@k*zm#XN;?1pIE}AKImWt;C=wQfSSw$XU!gz18&xM|jpL#IKXV2i>Z1y-m+yf2 z(>j~lE=B5>kq?9Vghk`Fno|AOqsN4iK~TBvcBTHSYYqj3fBp4e%~f-tweq!1JL^~R zd9^tWwE3Vve{XsZB1j{zAJkll$3TZ|G7a-V&_x2v zc^IlpHK&lkN8>rz!We_WI~dAlE9bx^V}VS5lIrig7j3M@K~E(RE2`wS+{&B1--eEE zWZg+@ep^5NA@X4~^k9dOP4j^x!(4qs6_hBdv3HFcsz3@oR=$V(o!Dmgcb=&d&G-6O z=+#j}SumL5iRSydUa}v>(oYVmpbiMH$Jq4?^C%IFvi?Efu|_DeTp@uTE8myurgwFp zElw!wPEra%@NTr0ic1~`zushwlufWy?og_mu^!jnp*~q&0@Gm_o zubk@5Bff4&|H_c6M4U!w^}eQ+ORs~+I8h~vtX`Rs#UpT9G=7*&(f^2m)nQ=rt;)ZP zeyFd;t|M5B9xJbY;y&MT;t1`2kPXU>M`?WD_<`aDLJ5uka;AU}-`|1Mr^m{F`Q3*P zM^6dBh=<01ZM}(AVB!uxB+_3u<*?i47DP8u0>kP`9Gi$Avcz@IBkbsi^2e|8?*87M3vR^vB{BiOenQ6&we@tXAe%#g=@$o8Y==WFZ>%sZe8sju;y z4gJ`+NGm{n$@FGm82c9M9+`E7zV567-3`+~V7fle-UA}*2)FiCv#;>cBIttYtvfNS z0!wK`fT1;BiyOz!PHHZ^pGw-^{-m0fVk#VBWk}r)skAId@U;9xMN6dAPj;PQz*Gd>bD1-^4yg6gpk0y15@? zI@&_wLC{sV7CCxza{&`bHMP+@jBmUy@1lS{n(FaqI4^X&AiBkZ;AV|6thj^;^@%Zf zRF>-Qt|=rCsw3>Tk(uEP2i5Ivu)U_ZFPsA}Vv zuknq4%VTJTSoeeeiXVa>stez$jq`l@^Oz!l=Y@gP#*wwGE#{M{gaFoLZGn-w@-I;W z27)fny4h(G5>4F+SD77QiRwhaqe zf6T*>a%JS(wn{Yn9lLBuEFP=cw%-xf5%W?MFhXqG1$R$m%y?MWT&QgqS12RRo}i?r^l+cuW8Q4V&b24K?xA<*mUPMHGtF9$3C;^ z9g*lqB_v_xd_~23Tlac&=$N1%M#q`r17_6J4)d;A{4kY8CWL6K+Trz+mdDK@AQVK2 zi^(h;;XoxsoOXDWt5N}NbLjt|)O{z0r%zsu)?STOPnp$9!ywaZ6LpMrNEKPVic)h?BBYyehQY)*@HdvZNngzcrN z5Q|4syFD4p7GWNjs^C%8ZvBoJqp;{_df%#c3ywA-T}c!mydJ~GU~QgwpbB^-iqLP510}n~?#ks|eaWZL;GaS0_Niyun zy=dq3t#6pS{u9QPU`W-hs&9<1J`p1vh^0qUeG{{l5~QucYclA3`8?x?6MKgS}iGT+sjwc%vKlIUgi=^gxcH?dM{GO;4cObD@|19)b=SB=nveBA7c~4-=_X?YZ1j z{~Yr$C6EFmf$(0ur5H#dlB$64VTI+J`Wte4ev1=|u$z=hfR9yuqE{*NM`HkyG8~== zPGY^$fzL+?ltKyLE5;P$(h{&T()Nk5rK}fv(&Pq!v#S32H7q$P)M+tQ_21{g=3sjm zB*-fMXS0-2EIyJhAZn`rrEKLpgf?IkWALaxY>B|7b|e<1G_{xeY-0q{tVjwr@5rw@PsldI!d`Hpne*0v_O_slDHdXW{r*=#bKpnD$r2_r!e@ll1=GOkO#3bZRc^z^L5kdni`0dfG*}oE0drH zUb|pSkuRY{+Nb>`qcO5O2$(3*r>KU%jNSz>kfd>HpM_q`SH?MMtEqiTD)k&7ty3#LurXAgG9s-W6u6#wxz;N+HK=E?y?n_lLKgu&NUdaEkdo?r_k)Jl zu-33TKuKUhxQaB_++inh1nIeKH+FLXsRS1ENtm83_nONpeV_5<5onYkfh4JF-_glN zj2u`=V0~ZR#0qhvG#F9|P3^m+IS)sCl*o6^s`gz!pN9v@;|m}Oglu2-b;A)Z=zWW` zxcW_N;z18ed=SsWv4UrlR-&kn2Vh9PV0Y!hWIij3$_r1qOny zs{Kx;pFm=ptF`R+VXzT^<^0?7H3g4*gIFspgP}Ny9jW$jcZ{_`8-$>hy7Ygil(obT z2oekEn%aN+5u-8uLM2cQgraQL8oNoU1h9Z`JjQ4&7ayq#m|`T}+B-#jUl1cG@nf9* zTVzt#Y1{v125-FaDPkyX8r&_L1>q$mU7|?a;DMLyWf<^6ti%zlrSry78oc<3(Qt93u&ETg+|y`; zBUDnQZ>?(Z@&Ka|Hqw{LaM08NtqPcihK-ypstjl!!y++!t5a?O_IXAkM*y}UqE#Ib z8mn)S8AQZqq&lEFft{$A7Of80bC*w>X@{vj=%T41SmX-)xw}|h5mG`vy21l7dyXGcY)H8q z4-8-H;|0pQc*U;5`y$2$ zHk`+!<;L`7tT>>2)>6jVr1Gl;mGceHuw=qu^Uap;^N5kTF;cOr>w1t}SF7 z-9QwVNC)}FGKSk2yW`CU7EK+*=jj^`O?F0*RULHZK8wa^jkFaw29;l6cC1AwLR(?v z!MI`fQ#7zq&R1;khuQjhnFKDy4#t6XW6ZBQs%h%rlRm5!E`TMAp&AHPLF_W1lP*9D z9GFyzZ&52+ONd5^Un5uz#vUYA@^}=j6EaE<s=)dIgU>-|0TPXqknI7rvjRNL!_A{@C>c5)@q@apbeFkIG*Wql!0i>Gx%&Vn* ze53D#2_aTf!+vSbmcx}%vMn5^4jXqt&#AW#qz;?x!zN>S0re%YVXN*MtFb2$K9R}| zTN`Su3Ed_LAX?R784>ImbL*c_ZdgVF8;LfVa6my#wLIx(^gvRp4Fre=;ps?yEm|r3 zpb}Qq619~b!3l@Z)GYCQywO08U*zjy+18m2L(V3OOE*|BZQE5IZy}@5ax8^)#mRB7 zg~{|_9J_%@CNPz{SXQ56cd(1I#HovAO)a}~x=73@N?_ktvOlnfLrC!e>CFswbH^yB z)iqT~^3^wD{sWpy<&>9>=^JIxnS{;pdLe)ThlH+t8_Mos2R8x310D|wS(;iRv=s(o zZE}r<7y$!=tirnb8Vm65*I8XTyO>Y>JVTTyBVepw1*4V@C8Ryr;u`i1zUp8o)#ttK zdL_nlnbMV3mE((i4t?!96Y@b<*f&?|u4M+(`-k~r94Cc1l@{aE7qbd;eIMX7m9K+X zbXiO(AiV9ysvC+EbtH7Y{|pO2nkR}&ZFy}hD?$rHZU7cdRWVt9&A*!nOJNT+p_2XD zV!E)pBz!a5_%pDOB1pDl;bAe1u}4A|!6STP8asj}*aj)7wC5~*Mi`%n0UlWlk5vsf zvREnxymdIje~qvw<8uP0q@bpT-?7=JA}y&dKCC@)=d^r9O|ifD^Eu|hMkr-ftwHfz z!$egb59{lp?9=TcA4Y0xEr`?C<0>Bl23=5Pe>}^L+(#&-O^3gp%vNE|0t^YMrVjro zj%8pbuLRIh4TR(FECZ>pR6>Xye)1Um6icm9U<%8l#OH3j8PcQb2qW4gv7G4<&N#KI zBSyO$tDEf?Mg|W}9T9Pt<>4H7)R#C$U^3$qd~k>$Y&rthL+4>W85m4Lj62KgC_Y#O zA3UZevRurtqKd>ZBEB=r{Y%7HSQ$Yd5pQR?vbm5Z9I-e}v0=nlh2RHTl-N|Md?8cL zOeq^8ly%j0&{cJWo?%?Xs}4}6*a$bCh&eVgkmQkjO<9jv#}8@35x+e&zJH}gl#sgo z4kxJmPsu2%EbsP4J zfgwFsb>!0(Y%&f)M{()fk$c=&*zf7i5~@0?vyYLAy@0T|RBqIg(qZpms9{>ns*bvO zLGKI2beK@sbX4tny_b2YyMP|q*qyCHnwHH8eKamS^~41z5KG^FO^w9N_q#|&M{$WX z5|h;HaDhQbXR95#+K)}Z=^jKVBdbT+*iVkQg^!Lx~`k9#tLPypVk& zZ*iAF7j=FRk9ul{tA!aoAzWXB-WJ6=s(=SJ_MQCvEQyq2M@K}nPjJ2iOlT6GgOq<7 z*DEo#hJAc@g76DUfycBW_HoQFE?q{yG?smg3xiD0p~OXBV>L$eC{B-69la-)oy6CJ zO5g$XW8r-2ta7Imt2*YnSYssS*j!aU&*f9I`jBGufXCNysA6%30|;jm_*9&aiz-t6 zF*64lBVc4`D&u4Hx4wMp^WEv?p$Aod^WanE)Si5!#>_7@M%3HLrjEWjh)-R0ww{!m z7x>gI+w0-D=f+yJeF`v5hk+2x5Y3i68mz3wQQbfL$Odp<9;?UB;z*7COPuyoZK72%g z`l#}IC7t=O`LYTKUghiq**}#ysKkr-ie)-OdtB$S?8S;t zUG^AT8_OfH7ELAaK_rCMfdd&(73<6m3PZ1TG=Z90BkjdB3P zCgLH1ZA)S=VxA5Jscp1>jUI{&04gDvM)ywAhhP~DKcusxlY{j1`lp;l=f<;{NRESI zW{G=)48Lbai{U-YsHqyd*00NjAu3^2C-e)pZ^n0D$9pSqObEKBD{`GswjUG1F6qPS zB`m8GX3S$rm|%t!v&uWA`fu{=Faigr*_ZTmbkD(1;+TNG)LQiUsR|xVov?f>TZ<&{ zg*ax^pMXtR)tPPlwOq7?-~qz_Li8~h^5BQbS#2 z5H?k^kT)g}1{n?$x4E$pykf8?NkWOoBlwG0N&`dbxrvh)Pn5T*nN=>F($`}Nyu=wF z6Orn*4jZ?KAgn&g%}rm2jgBrJSWH2pjw>7{{#MP_!M8vu#ZC$gQx0Gi5Ckc9QeR)? zAchHbR-ZJjRM~~S)pbZG4RmK~k!L`<;89gUK+u^MQ`Lz-r1159#f>>6T2m*Ts9`52 z%R>@`k)Q2_(+RKMb0P(g8_Ddlx%vjC<3Mm{mp_|Hs?(#Y&u%PaRxB?NFes?0&whK- zvJ4S)8|q&rtH(AfwxpUzatmP3e&NUd!e<^oB#visy4Y7Z@t{t**mjjHZc%SqDp7P@LVm${fL z{`W+DbR=z`+k2Q>ew2BifUm00?cd5RIJ^@`Nn@)zc}yCc4ZjdURGGZGnzc7i;skt6 zoxE)jd;90^0(z7nWIbdXWCDaV;pB^4rETH`W2G#9(L= zdM7E*)}LUG3y4sr;#Ic+2OjXiK^&9npPn5zwSql{>>nxt=$blpp9c@Tn@t4Cbn*f_ zU+wWiJ+WJSX%{CIm&(N*KdHQj8x2}Ic})Ia2w&j-xUiUDIt5+Yx3R>DACl>mmf>uh znPmbKTGc77{aNmBw}n^`G0QJez281h8puWGmB<9R^#-64Ta(D%;G&dRb!a zeI*N0Odddr6B09VJv1ax73lKNosY(be4EqR(>_RHZ=aW!luKPOh4gk*z2$}YNzIK+ z9DYLvLN!f|?^eUwV{n%0Z0O_t(^z|SH^>brVO8Uo6zP>1H&B&C8k@#{QfOr2lSBl7 z1CNicC_iEG0RNiy_}?+U>g+%fFhHuQ)5n(THRkdp;9J${MZs)!6Yi=q{m15P6-GGt zm;9osGm4HV$>wUc;4$O-9OZxbs#6I;Nbrm_o`hI}F0`F-HIcPP71LtiaW$CD#Bwc_ zkWA48&o&Qc1rJr7fsfz9cc8K1=R8BOKAZ@>azTki9e>m8hd3#Laam5;FM1yB?c zRyEjSKh+CWkkI2+Irha$WG)NoV^v@DiDvV$gawh(l`mT2*d#Nf3d(8fi?0^xv(bekivb4_n(MP%NByYk zi)(V&*8BAiG6L}vcFr{C0qM=h4Rh{{4hH@K8tt1vDtm@3vyX-icEr60z zW)1hYm!pec1tUWrO`V1Lk7JE}MXZCUsk0YE+b@|DM6fGLxLx3@u)1P{8w6qYD>W6{&zmdp|B4csB=%KxELZmL5d?wsm2QdLhHo|)%AlL$@6Fz< zp9?n!YxCL2J$IEDTFVx|`v65&)6_Z7r&unSr_NRh0?T4=Ak~XXl#xB=@KbC)j-zu` z!OYd0^{*$GGd_W>#i?&lo&16-@wse)dHNiPR&@@vO)(d^QR4qa$OvL7?YEOiauiBr zEN15<2I;vu8CmQcPR<7odLZRq5_=WNf+Q9X2!D@duV8t>B!I_%3AOAMbKfhZXlk-& zB%6%B0~ts#O@90^TZ{f_9gbwnCbkaK9d%-pN88w%>GDQ;p>1+pF`Hwi&q=1!gV=h6 zHljz`BYDCZwlVz>{g5_IP6%N(WHOOh5o5`*UTht{gj7Z1m{iKvHR~jBWRkYY&j&D@ z{Ug_locy9cTZhj)@sK!>9kvcbb6_x~V1awBJRZj^@v1*tjTuFhkc3yiIK|g?O?GMv zDJx6anxg!Ag!P-)2IR8ArY6(*SJ(!O3G^nV)NIa{|FyRBlv~)NIRxGpJYMrZ z!mOCIrz*lgujLlA?&z-rm?@=OHeVAXQ{Pf4=#pQ?TNx~po@JG}&unE~u^;(roz=^u z`I?LJa96=}?%Nlbj}$4H&OLL9J%tI?I$(2aAM&}+APUr3OV+%0gr4mrzs?N=E z<8StS;F?33Hz2r1$6kWap`KMMsgOsOsyhtL+AMl%c-lF|TC+UsfXz1A@ky zI`>KpU-hM&#TUUp_rXKH>V1(HOiVR(UawNV>cx5!%x~b7)pwDtPrx8WQ|HgLGY>e5 zaNx133-*WeAsf7e2~kQ@7nEl4!55CZ1|th@gz|86L%$IF=FwV~WlmK9n5r%uRLWO! zKiAjv&EYkCu>0R4q=Zf1Y?{PpFKFg`O*Qq+6%~9Ly0m2grW3G~VcC52(7CQEDQqzx z4JlAeDwkq!&SxFEA^PipuBnUM$MUz{xatb+#XE{vj+P^ogBdk-ahtn5uJb!$Tq&w7 z3C6AElbg7tESYkjFGgqsB~{bZ)UDOZQlv68fWiL`kzq{pf*9Xp@DQ230%Axfm4s^)K> z6}N|(Oy4ZzFU;5Jhm5hM4}BRf8Wl)|8&Yqb=m?T;`%yUpjW*E9`tpR;~B=On0V+Dq7n zgAKWWP3}aMq)U7qcKMUV1Yy}uFW$e|9f4G6yKMI{-XDvPghPm3hMb9h*xHo|dkA35 zz3*}ykfH!K@zK=f&!+ICMajbI&_`3#E3UB&Oqrm*RBna4Kiee%d*L!rHD%zAduT;>o=SIr#6+jta-K{J4%O2$cc2J;^PMvtnlnitJmW62jdWOI?x z`JE*K`#y~G4JH97dg?&lbn*@SK5ub{Uuq94lt8ZK~f^dH=FtF~}%A~H| z@5Z}*-$cBGLjBc75cmE5C(=bqS@Ttmq2ts8HyKia>YAao+=|;wuEM5Ztf^};zHOxb zEqFwcD>qC`WO+CM$E-3|-LSNPPh8?Jj4ZU>aMqniq{KNz7P@!)}{^aGP*!yB2N$~s%(u?awlLbECre~1xyI(EavccpH8peeVI>qv@u^T>_$4#_ zlCcrX3pNCd0G9b_ih;w#2$+PP)g;YmgVY2aRfK`EIvru%a9b836+E(h<5*Wr+)<1P zDOqoL9Nmvbnv@ev-yN1{WVcyDKP1z4PhTVcm%y^ScpA%5f(U|XcDGZ;GVIiW zktL4ouqNDJ2_;_4j5xbZ(c^)a+FkjA_^(1qm{wve#xAFQN-a%0VBL$cd*D z%3Q5O_MS7wMBH(hiYjH~{p`Y0VNWlkUkB1r){l)1GVUouQ4`V`H0M$23{D+6# z$B4$A2PjUDRn7h(+<41O)c{kgx_wxh(FePI^5H0$QB${%tu&VA{41)U1PEJe*a%m` zG6)wclwC;GAgc?DZLdjJcA~Fc=j`qOcq;ADVKsqS)g72&9e^W;8WN>A+N zvB{PWCBg?;{%0<739&n@zKSLHfs==(?ihYVX(7);kp|lFVvf=hDH3HeJ_f2eO{4Yq zuuG3{0Gw6LY38e!#9kJ@MJY|qX;*4l^K0F@7zmFiSiZ;YlyxS|@vT&rU=f3mf`_W+ z6h$hl&DDEhpd5XxGJNpQq6(@(%9V1Z@1HxI1XQ_o-M-D7b%OUGCESf|!5(iNOaTsr zJ6`s+t{y!I|AgBQnrGUh%1Cu*XJ%h@eUH;~nz}Qj*4T@DZQ=o|S=F6S#@hG)*g%vJ zICc&#wLgxvHE1fsVdwI23(N$Ur35DVKX@u}-(i69GsyHTSDXeK;~zfYKzCW{Eiz)R(TzYq80Q zMrh|d4U}i8Rcu&os#7j+TAb0%+}k9W=8ZeUwxd3@C41mD`5E4ZFJ3X|K_g?_Q5bO{EbV=_3fJ zf)aPUjAyah1Psy*`P^H1xwkz21Vr4go}Ohyp>d=*g~sF5`~%mGL!ZrbwjU^QG|uRO znQt?aaTWSswMru}wSg(n;->G`^NqsSVrkAmt^^dcPc{nCJ%`1}NGRcT*!cfAd;2h( zsy6T+0ANJ%<_NtvQX16h$hfQXVShnUqRRDay-~CrMOF2{o0> z>@7-0ROBtg5UG?JPl{5%&suArIp_QMUf1`!et+yg&N}z|z1G@muXV5cE@=0(d5anK zm;aEenqASzSe*tSu0P?f;5^EglsevJs{_ky-axGnWtmeG-Px5kY7fFRES*xaAQGfU zdFYa(r4$~Cy!g(q!c5lthHO)!sU}LT){T{Jd#-lTV4eN?d4*RGkpGnXmA?8^$t<4T z|1DmPC}v0Y%J0{?Irh8K_MkV;tH#*A(2Lz$6hLe}h$zsPoE&&A`UU((Zid`tGp1O< z-Z)zBUg)2i8%f_dHo?{9#I)$fVsHF+Tj}Mz#j139v*nc1q3p6?LJRTcMMq1AFn<)< zS)1x|!@9d8qsPme?F-eN94!{0J`Z)@7FKfykyk;}Fl^s^d6qoh^n25j@h>zu^!w@J zvu05P3T1c_M+e>~y+sw5RavX$zY}jT7Q<>lEdSrKMCVm9+Oyj2z#2x=FmkN=DKD^| zE1Afl=}=arWq$v1w$zZpc8Bwe%H(B9Pr5c@TQ{6vW#<1~-DT};`4<8H_|~WKMUtE4 zw2Kg3b97zl!;FT#jBF#=+g)PanZ$lo3;X5mQ1?Dwg*{fzrAZ* zX*Q`#P~Qr+x_+@*kTXl$)IzK#bHk$O1?%e9rvwU+LvukmN-eBeUpk)sG1S*INXMVB zF|dG`D}LDS@XqB$CPA$3$p+<}jBye;J3PP-zv1S6?i50J5!)*F&KG%sn(RqK!;Y7C zxIzt%u&9@?Vo{Uz6M3hLBq01GBEN+rs1PlcPw&L2}W}H(r<}_MHh_*M8%H&-;lVbAne;U zN2Ue#`|A%&^nbg0vJ`7&r89&Sv>cJ`=PFVIAtf^s6PM zM?Ps~wp-kT7kUSDo#2t-j_GXp=G7D`4eV<`4D2=tOpD zpSNe?=o1Qz3G7n=4*pFkK!)`K580<=zmDPxS^|=>5TUS zOENR%ZPwZe{wg@|%B&<=3tOK?D6ry|9kPn~R5$R10E4Q0hwCROXgRJIdwuX#RUr3kg{=%21b_1+K(kLgsv=^&S?R_n& z@LlSD_V)pOCsF$*BuzP8De;3AH5X|~l4_4ba(-wu8c3Dy@v_h+EY%IIs;_e!`>$6+ zqZKzNOX*_;{&yQ8z3 zCl6c|*poP{YTAlMdK zAL5W42rF3EjLD@B`L9CJI8RGz`iGpCUi|XnT=!F3)6ZJHXZu2TC%Y_I z+$xvW;;6fw3%T0+unkG?HpQ;j*OZ&HD1Av*Sxf?*YW^UUj6Un7oQ$uCZThM7CD^!_ zKv3{Clq3B=&Xjllnjh|CjwrGNt;fh~!jk6fDRI}^%|FX!`0Xd%b(|ETnAIik^O=ES zMgt9|C(G3_6TN=y!qH(Y#MQ~k-W3}TXq#HmN9`$*H!ic)kB#R>T(sM}c~Y64U@ezX z#GM!On%6|D>HaaQYn=NbM_7gaOG`$R)qFiu4!T$_*9}i|UEavrKxiD2F%(ir;$`9Y zkD!b@TjBLs9x47T6K3Ri4ps+TOB<3~)tvs#nfjoipO8#i9OKoy^5p)RPbj_GHV<^a@qe+hdGO9|=l9dL3^tp~SGX zzJ0K~m;l{tBaSL~YNj{1U6vtg3`AyhRUOg6xAu6Xq=Tx-Z6B?iG0-Diw;e+OH!@f^ zJY2fZOFw3`wf1Ki>8qKt}Y( zMtAahjlI~+9|mIfb9wH!go+jd!4s6ni>3rFj_lGp^4OfwUZ0wo#{1A9A^QW9z1|3> z*%}&8-o<$XvG{|nZ-es0-XbsU{SA7rPfsX8c`Bhuo*Zl(t#zTHO|d-5p)dsX6O=jq za#XEbZLl$PLh=-;uupN@crO;i&>?yH{S;Npo{U%n%_-bg(uppm4%vpQ9q#dye zz8sD`ef+fC?_VD@Y@g1WZLU%Sp-~|EzYv_g$aFq=bcZp1nLB;GN__kg&C+)5c{eAj zng{Kr2}BRcoYgbDG4_PlLOe}bvus{Th;1c(?r^gF!*a9`0~wV00g`8N!%oWq;gHN9 zzgm8sW7EGHhy|o?WY1A=xa%h*&(i5?P^KvXd+>y0!3vQ-?2TpHKNenbTq;?gg0F27 zl!dqKP&MTu&B5w&9*q~tbX1){Pz1A`fAP(U?qd`FFuf2OAz8fXjH<~kEQngsm*~+z z5IFtRP6mrfzV`@Y|5SL&SxgCxM{Y9}*sWma<+Oi^TE0Ko8OE}G>4|t%v+MFOm!K@= z2)r&402s)!eTnPomE@uWdMaf7ma*=kjbgAgq`W^`ovFUwTTt3Ec8(h!sti;7guUb&_x z8(mf`pP*_kvG?YIARRyRN3A83!YCnmEqAl3`O8n?hM>IKbV-T6%%Zs(r7jNGowHY{FGn#>#CFEesFBH)rIOj%AUj zYulFLtObyrlw(L%3Ld9gWb#R5o=-#a>AQ>F@0c!1p@BXj`D{m;)TJ;Dwy;D$Z&BZ?TRPW37%rcaT)t7S zZ}nL@UER;L$dq6 z3a|Ffwkne$x_fthcX#qyBPn(b%Ko3@yfL?gvhj*Ka zzDx^t;PyEBX}7Q|gF@hfSA%#5{not-XK;^Q|ZDj*v z?M(O0**acbwxFxzICx~Y`^aE3AKN~5_`W)_j&H&$h>AbgOAgZ>XJxqUiXrg@k{SMO$5zdhxI0FCH&nI1`2hjKUq=5XET67B$ zS|T?y2$91Y=!cZC8%{Dw>joe~^1}_u8!EYcuKO4b!{vMOT0e}oenUup)K*K=%_mQ2FL_BUa8hAqT+obfs@3?_~_2bANO*YHvSIu=OYq8&dpCqmcsCG0h2(EOthRE4YA-VkoPw%G0)FuZ2DC1NflzgOE><-b zmY7ZlYZ~&4RE^h9nJxv5psL$qT4325zZ#+z<$sgh*ZDftJ@fn_Gi?=WqV@JwsOpqN zZRn3ERqv&BlEpQo0)Nb@dT*!6Kz|>G#*k`IbG52RL?-y!1R-_7>~X3_mvrMwq@|%? zlH6~ryI6?Wsf*P;O~046Oe{P>)o9BOul@A(hQoZJ>2`3CYN;Sh#@3);~XbfgVsbu+1fi_OT9Pg~l{KAa%R|C`=AZ}tThGRXiHG(DTtw$z>xNd`ZXxu(>NT%gO`aj6 zDa)+=f*C`q&Emd2R$r} zwq)Tg;(A9Zg3&l!wdH*~-k+r5;;8dzw4=7*LaUq>jtu^!`l%pwnxqRxq#9nU#|9# zNkMZ|$2}z_ncQjCk^%(;l}K*kaJowUPHg|29sa5DKCFRgoaUY(;!d-*a?IJ{o_Y95 zv)bCLCGFC>T(%NQTm^9{pXRkjchc`lYoM-sa;0IUQjMd!yqf6MrRcIo!LHOA;Qev| zM--*F0pUWbYsEVEK3>3DQVUUDEOC4UXb8|kDmA`Dnm4&3V#02hp9p;5pC%YdFaI{P zv^jAo+H+-^Xd0Vt3GC(gL}_4sny_(=Hn4sim45AH)tG?x0=R$%#IG~F)~x8)*l`i$tpv&~pLU0pQyfr#hMt45H% zveJv0a@>e+Q2Ji5q7>>8<^F$^C&)OHB8yhquoGjTe(dzVuy0CuREAVvc4~_8+$k&u zw~*>LV1m4G@Mww~2ZLGY&YaXGj`Wb9hd`5gwY1sRHD%N=;3Dkju?LqxE?sij% zECHNzH^yoX;GXXXQ{aV+-bNNwwd);s<^v1MI4oR#c>t) z?p4Mw20|{*^al06T@MW{7!?hAZ1e^V``Mu2EQ%n+>~??SrG-!I0Lo|-`N_XP}6XlZS<}guru;HaX`5y`wb%)XdxBM%kye`d(3EnH8l^4OT0l`wMRK? zpn>zcd({aC!lqWafw#tc1IXQz&j7M)55#zbK7KMX_OpUXUS?;TWW^92bj5V^Gz1)2t83<59YVdp0yg{cML{J7FPWA>x z#gM2WZL4`h37*uxH7G-1TLyY=mn#+~JGhXh4CW^p1`^sy}yB3LpNRLedB13j(LmXVoyWcY3?QVmqUSPR`u}Ltju<&YP4d0$4?F8#=+pRRnRnHL z7a1C$g;XB3NduB47tZ~r`7g&tn*4O?8Ur*Sc#VmQ{)xw^R9aPC!?H^Bd_aB?*>0I)Mz`R zQJgmskH|Tk5WJ6M5CNfv~sh%_MwSp`rNB^x#sIsukTQM=4ptIo}cRVxNU6YWik5lSZ{RS z`Nqg)x))Nc)EnKaURZQTjlQChH@f$~q8hq)_eQgK0YUvZ>e}OpUJk3RLfdhJYV4*n z-q84|T1t%;Qsd&Ld$mc$(3bD)hPt0#|6{y2lz0aX0gaHFF!ZF9y;A7ZVI9SjtFCXi z&btv;@~?#$dtkD6TcPR^n=6Jz3(~Vyew4{?>ovoU7Fhew>%%I#;;$2vNr+M+w5LCC0 zo#a-KJ)0h6iJsSR%e*C?bNZNZ6zXTIoBqi4?x550!wPoukCom;76P!eqv6)k^Sv90 z5zvAeXWyQ>%xm|Q{Rl8}O#Ww{yyQoA4c3C)_DYgBiIt;9G2LMjZU`mW{KR;5%;7?b^(=42YrmOS0s4g0jI&d`{E<^5Hk~&4g!jUOdn0lbTyfmH^Q)2Jz8+FDf1eh3 zkU%cFgSBbFe>1%?l#1*f?zl74j=JaA{aPM2P(fv~cLTQuO6{tTS~htb-TUaLh;D8A zQ1NN^zSGCF>iVJl&4-oV7+!0O!bBhXc(pf*oIB`Y+0Gg|)9X0q8{;_|L+WAr$4#8g zYpWaeXSJE>UG;ozLlin8HEW=FSNRD+j0qo|R9^Z%!7az|9MXo=V;98qk0llk87!Aa zWQI55)TW3WkDlyMTI{cbfC{Nc9t2{_h_D<%^~lr9ysMc|AgHCK9?o@f-qk0@M!f$h z?}q%>6Ld-l>+DBAp6-oUP(Px6AsaH=ercjZ8ugQ_9=o;DTM%asQmt}Nb9}v$gF`r2 z6FzI(XKpVpec^GlFtTzyv$v>J z7gE?8w)!)_Z7ltd{eH1*pE$bFTf{9rEM{pu+cDR@f8!c+v{aUVz-Yf@wY5mF29?p}JF5l#HMuB?)) z7Tq@AUHWmRF(Dlv3y9atedMIclQRK#!&;bw##4WUa=e!r9&ZE?ulp%hOz3pmYwr{PN<|Sl(VFt31<86wzKQ-7yWmreO^XoeI zeLl``rREe;YwKrtb>6gb(x41iYoCdAKe~ReIa)lWZTjx=RJVM8ZG(cdLTdF#>E5E< zmxW^{Lh6IUZ8C?5e+1DFb!hsmcURWfV^)ra%X@ch@#cSEZ)(y7f?(A{Ky+r<&C5#)M zpxU(~P1U%-UNbQU+WYZw_29lsO%E#6qjL9aTy*^1Y<5{U?0N2tEa2c!bE&3#;ocV0 zWWk)#`iE^ev@S_KEqgV%mE*wDd{zCd$%kU)IB-p+ypd>vjqUt#VDwm7>E~uB(`?U! zICN^aYWBHJjAqzYv`-R`_idH7hhNW-zoODMtHbZ)%b%RzmLeEXA$6D{kY|ayK`}cj zkHoE(KiOqrGSH1db);jioPB4hmeNXkq*IZcU3**)K>axCNQ+fc;^*xF3Y4jf?PMoyD%UjxArT>YlJ!G>tCbPsx<8iC_lB()i)TSXGDU(yE3NTtD7VTuwol z=+A9()MVbbh4{tL__-oW)#zJGJIZQ%^4=Y)#@!~tfpz7{C(C6&Cq+P5N1d$PtZH@Y zAMx!k6EozSYIF6!mc}oXuGu){MbiKp6>~nHDH|X9Q=2cNOgH>G1fm>sXnPoo{XB{F zX+gHW8)fcCsmf{X&R3s`A51b>S9!QQDaB z6pqUBHxO;ZoDQlnuSS08=l@n9=bqWJZTj8Xp4QpF%{&)ait5^gJVAAO`Y;t6v&Kxp zKv5v)T)FD`+kLu3thLzbpX1a8XHpCeAcE?5F7VHoA4PIz3|D`U7;R-%!01v<3--tLv9h_J-OR8wrZf_l zb7VBN{o_nbciD8rY{~l#7nIB9>+Xzn?KA9-7V}zzpmo%l&UNJDUb#jNh&t-bOIfmc zq^Y~2dq7B?d2610L{<&l=o5Wuh7`@TIjN2MXFi@TE7^jk!)aMU>g+YMo7seS)OaQ6ZAZGUc+C_^4$4X95?T| zV+P{p*|L=EFMwN=zb_jtr#QqwC@bl?PCMMI$cuj}Y@ol>R>>*8J4W4aWqDQwmDw9K{J9hOE6d08s1EjNzBNI@s&lMYhBjnqtLm-Og%mUwGa8(lE%RmWvRkSq6sKnE z7+KhHe3W2q(_~@(m8Jp0(D+xL%pciR%b=g2Q*+-+ne)`vs_3P40&l|J0pkTQoCD)YW=7U(ksv?wUAi*=%Kq64eMg$OWxUlYlxszcib>{ z5rKXT9?dD})a!U!)!;Y|JuK0B51x>BiT?4~0x|QPyi3>#_|~TNW=)oNxc~x$et;ld z;jzWfnVyVIsi)O5Reohc@r_Kw5TXc(+CvVFLc{grI`vO+%e^RggI(5ZhePd@xKU<9)eXOm?G$6#=;#ua0T3f|ZO`4g8wZ&T^r?Up4A>~ch{M5Sg(u`(&?}LKH;p|o5Jk(}8D90WhXeC*c7e9~ z1)LIpz}SzV42dZvoX2!JOXKDBveFN*Gm>7g=d_$6$`yio4uQtjN+}MWiP+ zE_5p8Br%GmbSYzx3%7Sr^|@SJ2wQ`4;jk@U3)mv1CEB>jO4W?mK!oxe&MThw=utW3 zr$8_!Z2Z^fM-mdB-L=)EfL?wGSnM>H8PAsX|#c(Wgk~EQB0eF1~R7XBg2rkhY_p^C%k`i zR}_0_ZL^&wTpX`mH*&h-H2Hag7sGbP@-Rx%{2gA*HN(uNJ<^7p^BW45c`>tFnlC;J z@xKjm+?#Jk^s&~(XdAa?mTF#VTwtvp*E7zm&G7>G*86dPPf&3rcP+E(J5Jo+#j0+5 zN83WvpsZQ56|(n>6Q(DFHr#18yhPPGK3?PdG`7s}Y8Lh}AKs_*@QC}Yj#rafdRjRn zN3(J9s`m8G#tm?yidoSd6?b!0v2?gN@m}>ht;`3nF=6xCCsq90KN&fY1(eM*-Mt)O z9)qZ*ad4XZE|EJ8=~6UO*6jLN701mstsD<1e@=8y{MsY%4G^ZGd996J-S2HuGt(`b zw<~hbung2hkv8ZwAG5`)d$);9wEr`!IVo z+vb0b^%|rvjLfGkVq@L2$4pWIYq1uqcgUYCC3O#iFxAxV?(RO}FY>T!$Z7FgxvHo3 zM21*=Lab_Wt-bROO2~<)GHr{oleCkpqvDh1saiMO8tJ$#HsyJ>8k$XeD|-CH+fOtUj6TPL?Bw$ zPF3-@pA7fHkdyG%Xs=#=-N?cy;q6?nE+3kRZO2^8rn|k`B+wb)N8LCrx5VsVbDno#mm7vmzf4*k3_Ch9*o>22IAS( zs)?U0%5X`1V54m6S0geRB$6Jd$%Lo`^@-HP_;|nF_+O-7Yl-9Md$qYl0teea5^tn5 zeC#L_I-8;;Z8dO}9Af`;0{zr-Ih5vIz~*o%v!&6cb&I3wv0Zr*BHH#1gOXq5)nIwf zXtF4+X6*E07Mox}Yq8eVXQ&viGbDs(?N)ozygCm}G(-_B^EkKE9xd9gofKOnyIBVnVnQCu zymd-Z4Y^AUxYf4Jr^~#qo0S>rgAGLE({4^t;2zCi2bUxaVrcty$ zKpf8%VHzK7C+VQm@siOZgz;eutsrI12INRO$|0qGfEct{di&v~7UH^8xrERrjBLkC z$MKnx=I6_?5L1pz0@H=&!UK(IBAsF;8cB`SJ5KK|Eu*i=cbru&mqlOe?D*mqN%=pB zF4-sjxFnDeUZHe+HBqwub$Pfuz~${_auNO17ad(z&ya3ZVTPqI={s|z8wIv}2*>I8 z-ch;MPYO`Rjx!Cb#M$*-m?$*f+ad$m5bqFw=-+V@MxO)F#o zr(O}8Zpo5sT32}=8s#}cx!W)e8p!vV&+MQ(E_5LJ<1Fc83tL%NcKo?mMssbCe{GlQ z`145-{}!ip)Zq;G252m&(^(q-Oq0Hk-WcW*ayr!PvsCz7?z!k#2!V`K+MRDY3h>B5_`oww-#P5SgE9jBMoSB(ZX(y*{X+ zGM!EoxERI<9zU#dok)u}jCf1mw~o_k+%QR@vbA;;G-$XbRz}o(GD@(Cb!0@$ME$Q1 zarbVy#!nz@?9plJOzFo_i59F(`?k})>m+dK;xJJ}zkjt{e)!gK0|w%Hdh_QGg^2>e zcjc4xa|F{5qCeb822v3Pf*J~xC8wnY+WHNkEZZhIbbi?Chlb)bX-NVTc`@=M_$I&w)1f=?d5TZZtlrdcI(Arv} zr{X1m_aj_REs;wJzm3RoE?q`o4=p+m#RfT=&?u1xl6Q`HqB0^_=L^zhfS=~vaOs>d zP6pawQY&fa%topntDJ)HgwlEBY*mjh94#pi)dz==3pz4hU`w*>hP8JZNf8c++?CRV zi{t$8iH3H|B^TckCC8-6s$SG)xkKSZ89^|I9zX~NjR`xY&z5mvqEx@RqrNnZBoBw} zooQ0f`qoF88Ye9un;pgF(W9!~G?M{S+ZCcSljZtzX2Z-nsxt-Y+C(SJ?L2di+(2#@ z?FOsMgA`7Tc4g<88B*Jau#yt`a+b6Cgz&`H`KdVZX1@{Th9?VU5H+X(_leF;mK&I* zd;_7&ym-msqd8)+c}wI=F!FHI3WLUeJtWDR>Is)3}4S=|~(H>{LAVw`#D zX^k;5d%*$Xf;9~q@EirD@~mGp8W3xSNitbiv^{uWyJn&!-|?Wa7!bpq&g;%dka8Xo zi+va`S2HIUGSk5c8muiF5iS;SR9U86$*loc`doJ8$_NfYH3W^Ie7RCakZVu(F}TpM zbF7SFbwPvQwZEAzIb7c0U*BTiFd1ChH%g9gGbR527<0#~$xxe<$wxVnDLwsqz?Mtp z6lsxGY6L?L>`}Q{TBMAPNP20_Oi>Xol-{hnPnuydz%^^Cb=EyC&B&I-56k7!<`blk z|9Z6$T!bA!miH>rFQpX6wF^QK(JyVaO&mhat1NbDo0F1ELNeU|4T^-e&D9WEef=Ou z`xVj}Cn1N=mP|(>@;Yf}1A$v_Q?|)O|D4d$xMP(x zVUHegU)$+Rq}Tnmq9o;V&lL%EjV?%vU?iDs8F@$1@Jx{m-oGHqrt@Y=XL3|Uh`umi zhL&`Qvd7}daxHJ6|ZU2dK0~>eBUu{{7 zomegHqplx2m;RC_V=vnhg+c+gE_TV{`-QTIBfc1Al*66lolDvKY{>^6GWZbZsDk)w z4`U!2Go4FmNDJ%>_Ydm5R+}VQ_uC3WMs(fKjX)i(oUum=^$D+IQG=A0XiB4*GTe4t zO?bSM!pT`Iu}<1zMwgTer%BJ~Oui|NPD(3ZQXgWAG$ELyAiNArX*ov{t6vn>cDR!g zJI)=-Sv26*4JqwnC6+8S$HM(Ig>#@$tjCZ+qd86r8B@km9$KUD1f7(AJ5}P%<|IH1 zM?=?CN#@<)qo98Oc@p&BK%meNm?W|OM>9~MkkUVcjc(of3-xwbO5Pc%$DI+tt@l$1 zTbs_xkC2l9y$FVM9nl;LeqarM~;}Wdsu)VpO40(0D5e$e- z8cHPTzKVWps%%Pn;Kn3Vr#opVQ0`;aw%4GwUts%miFCyt2ir zuEe*#EB~|8y^@VgtsGDgWf~u}yM9;>8rEh?CtKxC)1jfbL`HCk;S<#jr)5OOMIrjwfFWPo)5Y#7UKR*twe7;v>1yI}6T-7p7XlzI zOHS@bgGK2=ZvAl)&xM>W?KjG}rX$1ZQ@Qwx>2fjgxK)1X(mP9X`8e|pMAhM;ajIVQ zw?~&TtYMrr{R_h{zAn@jRFQgG*-jT0M(zBSqcLHZCks`*K(=<2HUV;Qvn`L!j0oHH zt7Z5_!7y6{J60^CDzdb4wx4$SPGr>9*CQtEa&)7Niqs^hwlOuz+H$r=|F8z?+Hjj> z`067^rqi`cjJoEAWJANsQ8~rCKdzh38)`ymoUWXV4D~NefkKVtG4oV)igi>;+La94 zXW_!|u!41EukQh_%Iml-=mee1Zq8LLh?Uf$gGO!ReRX6phr)~ojYb*bCK;;MO^+Kn z7U|KDnn>kh!jbu*#it>dCokBPcDliFQgb)U*JQ!d5C{guP1EEoSGxIOZJIhMN!46j zZX`9zrLv<~ll>sg*2t0iT#RZ+dW$MuQvbV6&Ted{t!_Ka<@q~GU*=9?X?TUw?ZG1V zg(sU=1++VtKe^F+aKj}gP6;|xF8-iGPP3t^t&Va*r(4_E@=A0Cp>FN6N~ayN^^%O} z-I!s1dv?3#WvbdGrow|YQ1{+DRJEnIM*2thvDLhJ*A*L^7HD0%{gkJw z`D5SsrTbKl^QqheOD!$ck3U(ZMUPC5-M66&C-2}}BX`H9r#*A>Zgsh0UxrGIerff% zIJ5sT0zs=#dQej*)sAY-CFJzrB(bxkGr^mG$ zC8z&;5u5h7E7?6rk>CJ=SsHh*cTaA9LL121qsIeza+dqqSWEW+*gmsE)o%Hb=~7mX zkEg3zoB^Z0Rj$WjHq6ikaI0JozC1&0|7cxI$L+CQ++TX!7daN}ai|0?zl8-0Iz4_D z$>-GwxG%@KB)5_U$tn8jG2Qul)=!oW7ycZWO&T5v(%@@|Y^NvT zgh87b-KyV{ucV78Orza^ZxHO2Gt$>zC|X_Km@j#J;30<(K^V?e9O;2#$8FC~b7j0= zC&04(Y^CJ;S0RCdD2uCcgzzGR6+NxtdsBgmi|sRrGwT(}(@F;E&c zF?!e)z3mRVA=7%_cHFd!x5?#x-VLK|T9-l@9BtFI%d?~l*G!hsVfGp>-SVZat^4K+ z)_sYLq2v_K_5~w6YAB}(n9%1kY@Q7BPeqM^(#B1coBZ{yeY9w0llVE+M=P(Y8=qJ?-vUk`sAXQiFTiG#TfA0b8PTrpZV@l^hK~P?1gi4=Zai zb_1^`n!$JFa3}49&2lySQh;mmvz;`LWeA;Yg79MD7yjp)ol6L2Lze2aaNC0E-S#<%H?|0y4ywx>jTu?q)5 z-=@2gB$D_$iH|X!KbkqiPI{k~ICC zlyIS66SXL5TjXZ$crU6Fj7`YqtdlhajX|f^p!~qG#9@I)P#+V(CHsU7WVSrTxCMfS z;0nnk|5&ND2PkY)_9tDkrhz^*Tr(^1HB(<(AlB${+lw>47pJBMZo{{G_59I`#rFU_ z2T_ei17&Br@YM*o^?t8OlVt=)Hr>P1MX%dWN-lZfB5t6bc(X-^`!@`Cx?XHl)n-ux zgfA(hVMqTCS`Xh151a`cB{;t+T&yntJ{s6b0475Dl0Fb4pOSM}(?AzQe`2@9CWuR> zk_Zj=gp47=%r_z3fyrPDH>b567KJFreK*`1?l7?ZEMGp0)QhHJTe@s}uWECLhA%TD zV0WE-1ASE~0Y2U%f)U;RXCyHRD0_EEChM`mbUIkmP(C4W=IMVM^=YKx@HuHt8Ww)| zf*s3ozvyUVt{{x)-$~$&k`7Uh`nHpl5mta6+8#jsk`^c>|0&>FABdjpAWhim<%cgR zJI(|AOaq_+@kfcIlMYuK2(*yXn;Odpk@6I-EYuLa;~GgkuYxr+7}0xk0CDgddz;ks zkKWCTrHl^=@U0xZ*_Uem*&{|U!=*PnS7k&pA*H4fbb7Z;mhvZU8cEP-XqzZKqW3L( zCry)!cq4;i?a}+9S<;&CeJv@C2-dcmlyNI9($FzYzQ}#R$YF`59G4$>HsstG_3kmGP;m8KM=m79$F$5_nXV3nk~vbmL?lcMO85DO@+%H z1aKb>pL1<0*)CBn|Vh?CMc}Bi!X$~KK-#&XhaDeO{S`HqRd%BuzXU0Vr zA7ViV=|s3`L6!Q?O_tAWHKZ_&ZE}nlG^QNi4a+Bpifrr-cY4286xc;x7A>iD6wA@l zx3)x?@bx<4v4s>dVR3F?7oR+^^$o=3jRStzJVrv7Rf}aaGcLx_bZGcsj%3?&Q(v(6 z=E)(-yJ8Fdpxj3rWh4VxqtE~f;e_pY%|~c_R4zv_GE(|PH}902*e&)A1Paw$`;gZO z88i)i`w3f}k@*8I1cC4F-yy6V4O`>o3vMiH8lcfo)=0{uD_i&8mLwC>O*(Dg*M7ipLl+R@*93DmkPqw@vV>Y`)DbfQdONH=%iOm3!JfKsx?Hm zlU}PlP(eV*BDnB`obiqS@-fB1A?kDKbV5E^a5T&oh>Uc1N52|T+GdaQl3Gp)K+rBo(!0-+Kx7LV6dIn1 z9zf~6W911ZM^x8@heNbbHgb!?R}K_nHy-1KQ)nTO)WO--TE1YxB-zgQTtSsz*kroF zF98Z+Xbd_h+w0sI)`f=QX_7-i5Ky$mXc#4HWY&f^jqCcB6mUlX2_)AI`D2inAhz}#4HG8F7sG8b4=d>{ngN<_ct_%E1rp2a63Q+``R2Vo$;A4K3`crpGiJz&&LfpfoS2MA4JWwueiEzha`auqkpT6PJl}6zi=%nA@NKO;VQ;1-e=&TizflnalTb($A z&xqXdxj^IbWI2M70oTuXC;iz>`J!Ic$*SX|KNllk`>Rjrpv(MX`7HXimcC$~Y;T^b zO{fV2v80+@P?;alL{FLVvUIaNQo7GL+v>8gL=OB>FYE?TC`OqdxkH0sR3q@BBN5Wy z;{1gdFkG~xK+w=N`g|_^)!9<^ca9+n7a+L3Fwn6jl@XnWx2ws)oJiUtC;iqIY zXf4LH^=_*4;`^l((HZW#VW+gRXD9ZRlupoMI&Tj&)pmvGyT$V1xsF;WU;Ph~r8DnJ z{`Dn&|Ag$a*BNvJ+HUMFSzM`FZf9I*Y>JVx2iHaQAWn&A2u*0>B|=|KeW|B;W#dz2pDzd}`NpJ_H+8Te*{{MThlxrw{A(5U*&!eL46DIQV-cOS995X?#dzy92ffDN@c( zFSgJR4Q*JVunhEx!j_VT^KxxAEg)#<8Yg%BHQx;N0`1#ADV!DH7mTCjpyATp^7Z(( znudO8;7IS&fvY0U?vt7*pWe7DiY-h0ZTlkGJe@xMisdW+EXi7|zsRv;W=Fwx%RaJB zHu)iERYN^zCI%pC5gmF8U>=+j&A75u2TWpVtGt>-Oj)s)`g{ZM8WpPa!KJ!T>I7< z=*A>T@mH``(n)#JlV!0+feu9Pm?w7<>jXjHmCVUsu-5jwB}DJbkXxC?t63w%_Reau zku@#^eFNRkp&ux))M%hk_dbPl!o74lxPU^<*!6y)9xK?~4AChN_}7<|i+#OW)A6rw z!o@4(2Ck}T!7R#usIW~DYTtwqT^u9x2|Um!gH3mMeU9A4Jf{1HZFuphEaqJqfq470 z^dPLjHxM*RR!MdC#QTz=Es|rz2jM8)2+HBjl8)C-F>QfH1F@r%IDFJrrvbL#XUTqU zqiePn;^aK}JSz~z<@8P&#AX)|zH%I0SJ+55Uzb0Zh+<2=5bs-}Bxl@D?FDUQ&_Yh% z8t0^6-yx<;CFqXZmz2v>Y>0%V(f5KR2}G`A15xXYWN>2$aH~sSb_Dt&16-`6eJ`vp z7n1=(3q~U-4R*`tx3$p5(U0Twjo&9%aC{97{Qwc4CI#&awUju%sMZ+Z$=uGHsL+d#jE*&O=$-A)^2)IBrh5clbO=5Mue9CtEUe-az`2~Y8v_@Gk zTRADxY`PY|}t4~3P3%b5DI>-d|72CN~B}s&lQ`0tf80X!(GHTs>lGTBC~O~IEuV2e3KZYS zkB*jIq;S)tk_VK+ObPHQsvAH9<>_j2Io-i;0OA>}PM#|*m=&Gf(fuS9L3F=?P|+rQ zHWh|vy1p-slLR{MBIH1Hy#MNHxtBRv(js2>L`noF(@*+8S0JwvY#0XI~GI!cDK8*s612Hm~nIEyWJ~aGTDe14TG$=;9pBO7&aOr!n6&)H}KkiDNZwP8SIQ#DjvYUMexLCFs zb(hI3PToP$PteI={>UJX4RGs*jF@=ISh`mK(2wI}5OlQrn9ZC|L&(Xf*+}-V?7@US zQT8o&Bb4T1AsS4SHf;WB8t9VkWH1MP-SA=kLqD*MPn3Lr3I+|L2v)?Ajds*>-K8>` zrOS>;nR{roIxc-Lv7al@0HSrF42WdVg=q2|xrP}gA>2PQI_3txI((~M<=6oKjLyk& zIh~Ha;fv1c%G~HROh(6LvXP9S8luc-%N2kx4!s*?kFLc!+iUo+4Z%83k*`@e7h+S; z5G;L;WUGfws~2L?vLPobs3e1R&;~xuag^;r8KW{}!|^wbv#oLz z#I0aqvbuGbTcqpV=Ju9v;U%awyj0Ku+F=!l5lgs{d zNtg=|73*XhRa7+uXh0mUBm4c$9oz8h2^mT9EcEcT{qJ-cNyJQ~;irv~zh+aIC^SyR z%P=Z{fbW~|pQEyiYbI4(`ZcJOHX}=nO=%1|{Tg5PE7-8n@(Rga z0oRY?^cz|%Ygrip?h8hoziE=)&+>! zS<;t`u?1FrAnvKA+Q)Y_7F%O{%hA9J_Je%ofOvAUtf;>|+yD)-HZ4CB$$Refdm&T$ zu;wqc;{^z|^*eE{4j12ZPji8cTsLiDTdKdb`f-qY==A4qQ>=~4v2yh1GeNA8wWQfj ze>SPRFY2}PA{hJrd5U6~urz=}H6*Q* zYZ2ehns5t^PaG0@kDt~od_N`0@6N*)(9>Z2eucAXK$0hLxC8Z;bOp^}|Hb%Srh-_FSM zO}{rN2!?XMuaY7Qt{Z9iPE@PQs^Ts}PN2>fng8o2nyr3x!!&uxKbyBGhm%!4A7_Q( z2_-YSpl^xfM}hhnhZAG$%-CGEm0R-^|2Xq zgAI*7@p8K#bz|k|SFDn6NH*G98kwtA>~B?BHG)py-f^X0^L?uYvnVsCxZAnOwVbYP z`e}d!OG|7&x9SIOiIpNYb^+I}M6l~}R2*qFY0!`31a8lfcOy5shC2b0XRV5fxAf7v;T9oyPLOHzBEhlbVqhJCczFP0zk}}pTHm&+=GgVSl z!YL|6-?fM~_t*^_G?y~sPNFp#N zaGsaG88^s~$?BpT zok(|(Vbjz_biy7r)`fkb-)DBAu%5&*%ETf?yJoC%c zRF%-qZvKG+#62adG3!&^PZNwT_s&oW^g@2vZYfVD$$}T6l2^!_4l8c)xb zcl?4<)*ftkymQFB=6xE^Zj%M|AhAo_lhEVNrlnlMD)r$Vk3 z$`}3f32t>8JgJ3)#z%#+*pE@90UB?TU5w8aUl$-sb7f(RP*_qLR*zF{vQ3bUZlqz| zWEJ<336SwS-G)@vxQC6Yu@IXls8-+J66M=ZSt$Pdc4Ht5vE_tnwd3DrWSa)nWrdeL z?kTgBWMm@Pp?zK!iCLR6vT1&2wY|7yY3v=RnnyZ6C_fI9SK_BeabYiKUyVIc zCj4c#ie+|SGSOWk*>P!vpVL}Ugu^aDhjV%Zn6Yq|v8KU930ik)mW|9pscS*qofs#-Qc za1>8vS~D6TN<;Tb)pC@H+|ekGqk812q_6F{DGfmMovvE?aj>=_BTps#kPw-J6v628 z1N*dM`a!TK;@$U%P%UJFrPI+6nx)!(oEfDY`wlO6jWad{1uj<~Rf!iijZ7Dc5X~2U zX%dApccsk7o`CBI6fOpTMI8w}Nw3kut0~^emi)|nQXwrrnQ4nj#N;wfCINO(l!bRmaeWQ=^5m87F zydNb8JBsB5=;~?965TvOo?{)PxmY)Rwpr%4xg<*c_vWd@`xYAYt)yGlNihe@;OhsQ zZYz``--JHVnR3uSzP1hPJE>+1tCH`b{-@QXoJ&__(BOeaStlvwL;`T1=%@RnJo9TU zhkk(gbhea#eQy{A1AR6{%84#TD2+mcdqOq1YpDq(hA0iBT`gxLQya)CN9x7-8=lqR ze*f5ZMqN13Rz$JmW!nnX_~EY&7fW>iFgf^1)M=zTTp^38Xu!YvL6?dh@-Fv*B61um zk%Q5h(lYZ3y$kZ|n*7ODjvdQX;$wj*&mF{bc7>yfqV0D_<%KgFqI&XIyHzVctA-Wq zr!(>bp&CA0hZpWXulmM6O)o4pV}IXD)zS~YEi**FKOx`QC5u)N1N|^XzO#wqG#%I; znxv9R6sL*WhSPDf?t_J4Z9(~NnrebQe72xe){%o$wMQuJC?F2)b8ld8ypRSOHA;Qe zCjW4l1{OP>tGiUx*#B{&YU*!=g960&CsZQK9z3TXX#BK7B@#%kxfr4jE7%wPh+V&- zgS^N~iyzu*pq!bZTHM(r;!3AZeK}Oyq)oJ4%3&|~5Jgj~ba5K(lr@A%Ktn$$_s={P zx7l7nwD$PtsER9z8kG(yhSyI1B#Q01bk#08g^CliM76y08ZEXJ%(;-faG?vsJvrz& ztukey|B5zTOv5LwB5rV6Y>`8p6KY9$LL|y7R&h_K=^sAP*di6nl{;;*1Ostl2bEaQ zTp{uyT9?bgNM;UbG@h+maigqCTPJar5Qbv}vs~IRkfZnMoMuTXu4~ozKBC9&Q0={m*-#VB0=j#bwQod}%p+?;j+KrY{E&S2U7C#Z`4h5j|tF zEG{rRJ(?2`9ZOUavBf@H8u+}ALiR}&zVXdBhLY^ls zC&G64Bq_SAmua98l(eHNnWTu?QD(fbjj!79n)gw#N5&Z%s&-ju=(kuU-O`L-Mmb-iImJmj!EUJ2|LcdZy{PRNnRuqCo=fD+&)2$ z6DD3-)e9foAuq6CMRi~OCs)Xe(FszV852~CuGfa;(4wDIEjU~P#W(VED`cU~DeKdi zTSrPG>r-v@v8u(7v%+o|?l?q;mG(UtW(&luB-M^PnrrA%g~ko@x~s;|F9>r1;@Pn( zzVW9~o|`vOwTxWH))1Xk0{f@hggi977AuF?5YhI)zli?Eba{(oQr&TR#yieSMJ&H= zi0We-Qe_d@#e91}gKd{X(KjHRjZ5TZ?lt-l`1Xx$s_m_PHA~+fe9p%C3z9OU%X{@z zLdvVU!)Ykw*pVoUzUdTJjs|WDzchDql<1rJ79)>yWV$$-F7r8T(=;H8U|+6O@t5_A zBFakoC41Y)=fdUlmsltwsTK@L$?UVhKa8aj8o#cSqD%YgY0Dq``}y)@10Vqtf`X&C zuYT~D*Gob5Q@ZBcFiel}v{%(%?b|8k-UzW%yExqmvyI{$kP8jh1$ zBUxVYZ$KbN$jM~OwTQSJ{W~f-gw6y1}V5z5Nb!5kR zxRXh-!S^mc8y+4A)}D#&_-Tz|mCJOx%QAc4!{?GUNtTnS1+jGlGzh<4HrAx1@=^L8 zS4kxQ_4_(RLzVL9?3Tt+5T;>liM$nAYomS+n=0LY2zwuhA$ck>zN;2nKWKYnp1j7! zT~tH5yw*Hbv(yg|S2a?NZ$2Hx<>iU;ZY1ak&z((nd)b&8tsrdgnj=g7s7~V-=dbB* z+HV^J{M*0KxbiX#Y z~TBsu1B(XlsXQaP&6SVo=mab8x}Y(xZgWtqH&8gYQ(xPj8HkOu++coGU%Z z?b-nxgQ0QA8PKqiYV_8FW{becF>q3$SM8=RP53?vhMa+%S8nZl(403IIR?rJS0CA& zf{Se!8t3MsEgjc-ZlJTIv@sKk7Twr%;EP+_b7bQyM0BJ6!29aB`COL+Ut1n7OD4Iu z5Mc(2PxOa#?*7|P8eJ@n4=Ms@NO=gjPjpjEX*;4!CV*mSWRckUA1cJa!9rxU%~jR> z%yEWBRM?WVi%o(~R%tca^zZ-Ls39y$c4EBzZhdRz$Znk{zq9RvP(H-vMJA#O zmUi4`6ZZNCcf4t^2Fjkg*gX&l)CoG-TrvB~KbwbB$jP2@(%p4yq%g0OU6AJPvIjCg z+qXu$l}8=pIip~->$pQgw7CHt`w`O0RV%s_nD z?p^2p$hkWB*0)Tmgw&D!z*b- zxsAi*_wBDl8R(CF@-5}nAZQIV=(;%fSMIG9gh%C|TjSlsWbxKL+2}IpuFdWtE<9)m z&_d3jh0EN-9Jio{wdtUXmZ(1KF4x1uYCCA|O7|dH@b-o~%%Fvn+|TKBS~+8}K?CAc zb56W94a;Tu1ounEi`Ldc42en#Oo1hnTA)eBL92Za~|^byS}&Wl{EE6S7aXxg%la$e9r5 z9%$6tIJ-=DsT?YiY~}C~#Vn1S&Ka_mlK`~UkN$kgb@!359}ZUioOjN-dl?=D;mIIp z=kfoqy|0a{s@VRgNKGx1La&O2Qdtz4g@tw_&z$Wk3>0OHg5o2FiK3z?8lrFz6*+)B z$djU^C@P8)S>D8CFS`~gX@;qkrIrtsY3aR|b+zAj*6hRKbN#>lzq@|iFPJm4)^|P3 z?AbGG));5}esvAO!M`3gPOI@p87k!NNaF}j#n+<=gtKWOz5R>}(3s@HI6@w)F)rc0 z0uVI|e50w|X?Ho|%EzV!HzkQLFv6g{N%V1dqX}c7B#Iik7V>-*-K)j&5Uf<#jQfFh zwCDq}U}vO_8Tbg+5dT83wat&$rAsLVh3K)yd7Q+M!nw&oDfrd_Ynwgo*^?P*Fz)wg zW8PU0tw9#gOV&2;s3a>)Gz3v5r+$5bJCFjlJKXqrD94_g(CZqF&&=!%++^sz85Dw9 zpJdBThW2WvkRfW8qi7&z;>#}gJ1wVaA*EH~ICc_&K|VHZsC%e!2tEpw%9KMNsxvP7 zyz3-7L>oH0S)4p|PGV3>9(ppsxQL4s9B2mCx;FG{U*ig{@~U@=Kg|7_aSAc4QtPbK zVX)kV!Bj=5C=us_>8#*RZG~kv@5R`1uB=NjxKnes&1COrhBO7T;=4Y%z)X=ri zxv}DB6)0@t*nt&+FMe=w8Sa@RrnF9HIs7G`{n;yza7>LwN;`ZCW^7&6G6M@YJQ}x$ zWnOdI<^NTqC{Z*F2$P_cDTnWk6`%Z}4j2NyO&gAt-cPX5Eo(5+;h)8bPjD2$G!$z5 z)D1Mg=K}bLu8o+vhrCs)bl`Fe(}w@zCK}5Bl7h(xO@0j$4FQ~5PZ1q4tjTEjEs=XR zTYtpDXyeCaR!bc;rV2_AQU}anFTg)oxDn;e#t?M_yMjJqS&aBj#lx5dyWFf@aBn2+ zA|Fi)>su!NdO}@!$w0lP#r(+ikuJDhCwu6v9Z5q*s6il~0Co6*xu zMX8#M&Zj3{ZQ{ss2BzQdZS+9ThNMb3y^FWujp+t{Rh}G%ypr936oQ#GLOM~qj@?`X zn;z3dJ&#;rX<6&~bJ2#kO6)9~*fbsK=i3bFA^$Leb;v<(eRYH=gng`IxwXyT=eq2o z$9ouk-)Qy4r^nqC4cOw8G!#+1flfa52jhTnHf^MPH)>;jjV~Bakv=x+6+0EZex0R- zPIN82gAcWTfb*X!#|`grklJ@y$Kx2|7-PA}9#A53&4C5XeY z3ka%?9DaYiNJe-RG$w)R+DIH$IB5n@G0|{Jpl46c=d4Zj5?HuVO3&Wt$d=<8qulF^ z+em=nfGHL9=Yq)R#gVKJHq@v=g8XsSf^;dCWmL~b;}&{0?4o>ZR6>%0%ulX(hf$;Y zi7&tQ%9tAajy=uL4IF%ytfX>4?Kx%dwj z0H6!yAEUFcicAEhfu&i1lvKxhuVSJs{^)&}<@DSuf0d7>js7`aoWq**VMSv&3f#rH z*2@eLJ(EQGBYU_8IO$r%gE`{ot_ByNnWjYy^b;$F*(DCi+NMQ}!@)lEeXx|ZP=JF< z>8_~)3Z`8ahj2s$)s?i7y@RN0_&TS?P-964aUH$VMKtfA_#8*N9pJ*UM4U^ol%ktL zK}y<)okE;ink9cVYaH_t%P{>i?E<1WLi8n8zd+P%f_9AEV$Spmcx@JFqS29 zWFfWr{4#IoZkFy4frIqFnZZw>qic~bxYHvqJSTyv8CwJJ$?{18pgph-QrFw)3=0pjBO^-PpC78#Z!Qs*vmxzwFaZeFGVkM(lRP$I=*YnSGI ztYn2l?b8+TOLT997aE$SL+^K^yzYT*4d4(RoVbaSl7mk*e$Y<_{v3eW|=XqwJFEolP7hdAJ4OIiXv)9a*xI$VVCz`|z{wiQYjiGj;3F{yD)s*F5gGGvY zjNq_@J~qLZ9zwhW{!mUg=7wD?RLf9-DAcvF{@p~LIsb@}9yAvHCC}VQRW#NX8t)*X z(0!@{p+@*kk&9H=_*Ef|>=tG%`zPOMpbEzN_*nbw`v><1MjF-8pPtitGYtp`8VgHB zGG-a4Pg(0e-t@d zXbje(5Ye5ELAu^Ko{qbEp7P3`lvgb%Geq@XEY6Q@>tqWroE|F*%p3rUXkxV}z#V1y z!}QZVUZTL9DxlD&MSboe3SLrM;HsdaleG)JUL`O*nl`ShCqnM#xM+;SA(BLRp((Ad zje8(b)GcW5(nr)EKGrsv2c-ZyYTG>cmPJ#dagIXl}U0eGXVabX$V8 z&6-fBaG-JdlE}v`BB-wXV;nYxZzHznu&V&K{$UMP*1%)a#v@7fZAAXTKa}OhFT5;H zmp<)mf*Odo{2Jm*4`S%!Kki`d@@GpbeO;SSd_&lA1G`iZYS^?1o8rYj3`4uiKo5A z;nrF3qQ9-CqW9}9U@KYc+T>t=>f8R`Tu>_q=}F7V#9{6}Mt7HLsqOSAaE zJp9GLCIhLjy7^p*KPkG-I(V*nohUAAV#&{3sDc-P#xHT=C?rjBia)uhuh@&@KhU^j zt7(%528j=_Ac6vAsL99#{taT#&_r>Wj6IxvIF?n90%%^>qOp+K^|jyFE@-A}Q^FFA z-0r2U(+jf38_+P<@I;kR@dH4ilwZluJfCa zp~s|uW4))iuM?@R-C3|K#(GcTLdiuLYU-dUF&e8q5<1hEKHg3nE$_L$v!{L-EH2=V zL->Z$`qZOt;sO>!WdWMlw5g{Siwg+O2T}Rhv^PVmt?olBWp(92u`y+q{YZF;HEnsn zkFHJI=Pv57LITm{v=#+OJ+K#HCz1v$DDHsC99$*=1EOthosa0=vRe;pKeV^RH zucQ=d%GhE3tj{1~25v1iuWQrq^%7^z2^z~X9Y@U0jqbqxw!0K=dW4%edpw*Al!D~- zS-3$+(qJyr7akR7v9t$*$_uAwxr+~Des>8seF<(J#TF+xDCk8pooAX7Ts9pL z`^@CwitVlp@nM8=DX`VFnIRKtHIg)ehRGJGDLzC3E=fb-I3mP3EJUEaGUbfB7K?K# zH@4K8uFZ&TC(i7COKMDFK|xWJ_z(#W09_T#zCWCn9yr1NQ7;uV`@uk3y2)Gi21R4$ zt#Au2`{9qG!;x3*U0uX{=_O=J%nSV32iEk_q1BJ3m41-j`B(&vpkDllI^@7a1IAk zS~O8QogZ0h9N1fDy$>}|D@>ao?QiVOye3gvaX(L(mY5Z8eDdRUT;b^^9~7)=79Sz- z034*mHZ2k7_&&nke28;TJn0|1sq_!E{DVSWTOdlsd**Q$6xg%{(~epnZddIx<$@WP ztUVDN=ca&Z+JbrR)`zjx+TA(DUpOby+IikvJO-gq*A^~}wswB~h($JG91E8;SReSa z&edPuT2Wzru&z-4suZ+%UO1Kf$CqsjAJMfWoLA4r_LIq0*B1416Q%JSX|Eh+(R+4r zAC_HAL!kze5x$F67PzJI!bLc?mua4ig-!-*$*=6ObjQX}Ti7;54k*bPe)a#tFb>fr z(c#)+iyQU+qTKqfatGAFr^a#f!wOh*Ep?zbt$8rgg7y{*7tC`=oytE{CtNbjZf)C1 zE$g$AmwXT@-bW-8Fetm&aRFO-uQ*rFwx{o=^599VAQnF*H{R+yHq<#i*EM^7H_AVg zCkIERvD8slt$wlUv@r{3Uz9|%-G=a<)+uQQI|WjC^a&n=7)R<`Rn|8W`C~*WD0N|w z^$mPBb$9k#`)58h6W{8nuDIC0tE0I~2lE^kwRCOiV*ylfpf?kBNDo>XkWPz#dfD|# zvJY^x7QK_~>gwrdgQx%@5pu50O2GAK1@hjXoOh?agmol_K%0rh;%1Bs59dMYjI6P8Q3d4$oaY*Vj)HUtne^1&~KAN_&LnKXFsO}_ZX;&81Q2K8=AGU_&bZzB#`IHlFV`ym#SzsJ|m$I zrR2;$fi(HbIM+Kn^SOAMwr)1Rl31siL;YylttTzANxf81=CDYL^_lJ*wrpDF@EV%x zH__$%naMXP8HZRQn91e%Vsf$a1angKa8Qbyz?z? z&LJoBb~{?u{4{d`7T{=ZqU4yJ44pNW`Q<@M!j~sds!=Ml(LrzA7|GJIp)&udqNHO7 z<*!a)FVWPzWNCS&)6Bp6&|A^j?1fC!(J6-#v7HYSs3s1td`c|c$4!{1W8z?1VZKcv zkFGfa1TFbAkb$X!zn`EbNYZo)G-~C?h~wFEO2qm)+N+N22#lsh$Oa6JNka|S9C`;z zJ$KU8Zyo!>DHqX$QXhCw(On&bOUl#K6v{Hk2r@k1pg>kr8Orgm2x*KScVLab2Tl=en2C9l7W z{R1il^xqO`5k4cYLpYWttIc9cUZF1h05Dz4y03;7VTA-5Qv+M(ZBJ`2FBOZ=II^ZE zQl{yaZ0xKy4w{O|glq!6f<~uyG^MVMltm%UYIB)J7V^#3O6ysc0GfX$j(tqg7#={2 z?jP^!9$ByWP@H=$D@XxL@S-Gqnxnb`mNm|u#$nyE);TlCivNbDJ6Y-ZF+n7NVf&JWV!Jb%e+o*>`(V#<+MkwrXVp&v;_D3lLgF(I1 zgR(vRDD^AVU;LS9FXYT{N301LrAnKzp9{pq5R`#&`AtOui*Bg>-x*EkOvcEITW!qgZ;OvArN{yV?s$h+v1+TP9K%QOp5(qiMg1z+PY*NOT@xv$caUo`k&PRrqcMJgx7v4Z;DPs>Z8(Xd+g7@Q31(BODZ!IZ5hN=_6-~ z8^wJyTVBRs7F7CEG7>9phL5pp=ODX!6lO{nP{97NDk6+l;VfW_PFG=LdC5h!w+7a_ zwyIqsEx|?SZcb@eJ&{fe#_Z$qG(}EGR&AL`x#kHfxD}dojG`&<(^~ktqVcq#xw9YS znIq_+#tW6S0Fx_+QrZO|mCoF*P`+2+u-+^$Wu zYVHs0K>&vuKYP%e{BmdauxYD)N~gG&3)xSV#?HpmM6V)NVT#mPZpQ|SM<|a(stRUb zqBIBxT_{;{$9YgR@}HXGqe*QyNDS-7uYNfank{&$F4PKbXuu#!c=lh`2;c%kvl* zSV1g)-jmG~g8+yY9C`Diu`;1aeW*T{pXN_%9uASvRYBgd97=g4*gsW%YS#a8$2B6PDFy-3$ z9b+l5b3iLD{VQnr(5~F04$BbD{1>Z8pYS>>2*%R2{J;Yg`rEUzK#iyQV{+)tOFLY$ z(pf~CCE?U%_!2e)8*253J`|)f zwVKdBcc-x2%T5IW*r$Os#0)!OqN`88L?duM8`@Itu=>;i8sXKJai9=fE_R?1-T5plu=o-GXX0#k(*0rL`jg<8ue~BxXDw-8aBUAF&>DZLTT@xr9 zFD+>P^53rE(#0~Ovkn6i3tVrPj1z_1{a zTFfhXCa|*0G`f1TG?aYMfSe|QQSV9!eD)AD1d|bMd_4<%z_Y1DX6%QCq+G0W5KVo7HT{dOd}8q2rjC>6wj!j z*#4K;P>i&AhlA2TZk5`m_(}}r^qwK->&leHAGV|P-{-p&RGe5&;SJkag~|*|5-AKi zmB3&*u>EHxb$^*Z0j1_qaJ!vx6gV!R27Ckshte&L2%E2UnxW+NgESV01n{d8t|UC1 z#y%e@+big6rY0I!j()-uMdniSuSVlWg;ufd$!_I<3W?5br;gS zGV1(vI_m>8Y+C7?VU%}~uNF|Ul+Ha*xyN{eTPdg%ADS_yZJF&I8Y^i^qBh9#K2Qyx zx>ovpFpcSQ(Piv4k491P1*=QzrMMu8Fd?>IDpW#!$(GnXOd#-~O@Y_d>Eib)=Qk!3@#(7K4!#+qI)S zDF-=aWdVe?K;Ql<5^sU9@twVvE7 zp?TCO2%{-iJXsbt!`cOL#uc24b^r%P2wf}dT}gSts=p{h{ZVCo4tL>wq88kt)8TGb z-ye=zd#P_ETIY$YIq^{hq^5#?#UX)$B+=Qqs<3WtBjvxwvG}S< z+0hDPS>`qVSXFF4FBK)2p_obrurn2+Bu$+q<{Im|xEnTP1HKMF1rF5sGf}*ZFkm!M z#$H!hP06iyn%85$vY`J*lBJ0>*TcAi0}mLG(2RPZZx7(ekv<36o*BR;C zSRczBu~_Ot5wf=USqn1ao6%W`-L2rNLMGuy(W*n#F z5d8M|1o3h}y(Fk~y20ll6r^*%1t(qG@Nq0N9TnBJeC9-T!VQtTDWx{T z0yz|y4cq$Bk_|rmzGdhe&NGWWIxD8QaCm?`ceMvJ!}TB*0qgsi>V`0 zFr|P79-Fq|+gR(;f#>9+oqTNC#+AX;@yI3?zh2s<;^{=nxOj|}>?R8~HdY$H;m~GV zST032sR)XrWm}h9@T*c`1@?jB5dU!s&JX*7M(iF+YojsI5ZN0l(qE!iv9|RPX5(mv z0-L*-jdNN+sf4Qt2@rEI$AvB=Yt-;-v<7;3h+fW7ssb}dJ6^yg&R$vZZLIMXVrpc8 zBKozjWwz_Qn5JzS6-h&%9l*13;25H9%4wjbFSXpxfP$%Y)DG!S?gLv!x@l$%bws8T z$e}!WQ)3VMt*rBvf0#9j>Zse!23N1#T(X!l{Be z!Qy3PCj=Lzpv}3vjrWn1S=L~-mG?B#Llghv8Jm(NCdTSzrWi&Io3^dVm7>e0e4k@LXt-mZPhu)-_Iak<} zK&xw&6O(8bf=WgREWrIf0+4po{6&E?rc<3nj~M{#H-(;xiOLmK#3SE-$uZ zCDydcW$nm*WPs$NI&$S(ku*DV8!L!aSoz`}W1%_QQW}eQP*5F$sZ>(&<}O2>^CsIItd z8{#m2nRh@si$c2X*=sbno%+y(_-OL?2J-h9!&?-9kAnUgmM3v2PXbeHpX?;|B5oXi zm<7*xh`mTs4W!D0wk`Lkl&&4+a)-$lOM7a_Ds4`GOuezs(GCmu~UZ{}b{;2Bt z7o{!Mj4Z4uv@xGqE##Q)3PtiSnG$SlB;Gniap4_59LTYHm$1R zGCgd4jk^c9prA3{*nq{*ddRX_YOLzVtJE%QJo_o*sQT$VwS(h=gJQe=H9;$T@DWu7 zYt=gE66@srQ=s@X9iN&WBs|tDQT$fNZMUA zCc))F+ryFE1ZnYFe0}?V*!GP#rN-og94jl5jJt7k7-le4cs80=PFHyq6c$vz; z_5B_DzM&NeDuFK4D79l-(;#7fN2N#)+VNXCt>_oPE~RL|aqog#I#hoN)2e%h8GEtH zBDKyns$cLH51sx-x`KQ(t$Luh7=sOepj6$X8aL86^!iJpRBWr$E9t3GhWxNnYxLE# z(#0oOM1U+xW7UbF;*vWk0d`WUCvttMgER z&;+PY`DyjHQIy$>_Xy1P`s9HRVd_7M@*z4v1a|6n9aM%W^xr#~U>W@&z#Y*tDJA9Ym=^ zT^ztTc3P6i_ovTX(~X^vWr$Lk(rlt>JAILfqH8~=_@MD}qu9FsYo~%xFe_ASMMz?b zvp4Kat2P2L=Wo&J&Qeb?!OWG#(06q#r{y>q+uixn+WBdfk%RY3i?Mfgjzf(%orco2 zo!7Ar^Y#q(G5CnC?P?!Q%PXd_!+^C-+tu|JJs!SJj>&Rzgc{EuG~{ir_*FUXuE=m> zG=NF1qYyQ=)KGuiCoXZ+GmT0?{kyA!O6+vI_D4~FEHIyPBHgvgPi#G}$pYnLyLJSL zt!6Iy5^39ApJ7RD>M_v?+fVM&F1x-uPxeob@Svn>d=Xopi^Gx^oKCmvrXMZ)^tx++ zt!aN#Y;he_(zTinXT_G*$z5yij}}{TDh((D;S#9PIY4anZ8=e9(`u|uv;vz;kW~3t z&9Fdm;@otX2L0&v-gTkg!`9cTtF zwdvwxWU-PofW@Z0*WP*4%j(`c zh{Z=O6wKIP-xZsF@DJq<@5~rn{}A5$5(ij==z>nh+1{&o2zIb6iQSB|eN-fq5-!2F z-cz0PG-bIOWsJhoFHowX@15)A#<|P;n2QDh?XLU!+1b$UD zGXHko>h8x0mQ=wX$kOcIi?4TOSzgIF8emz4>S)iy)+;L$I$2~K8@Q+%+Y&liJE)sj zrK0sgZBBn9mg0j(;((He40>|JWs#1x3#gB6JLpM#ai+gm#{ISyzy>{??k$#nX6JDX zX!Sw;4>!_MTnd3dSo}f3I4!sYq18~I;u3X#CE2Icv 0 -MAG_AUTO < 21 -FWHM_IMAGE > 0.3 / 0.187 -FWHM_IMAGE < 1.5 / 0.187 -FLAGS == 0 -IMAFLAGS_ISO == 0 -NO_SAVE - -[MASK:flag] -FLAGS == 0 -IMAFLAGS_ISO == 0 -NO_SAVE - - -[MASK:star_selection] -# Star selection using the FWHM mode -MAG_AUTO > 18. -MAG_AUTO < 22. -FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 -FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 -FLAGS == 0 -IMAFLAGS_ISO == 0 - -[MASK:fwhm_mag_cut] -FWHM_IMAGE > 0 -FWHM_IMAGE < 40 -MAG_AUTO < 35 -FLAGS == 0 -IMAFLAGS_ISO == 0 -NO_SAVE - -# Split the 'star_selection' sample into -# two random sub-samples with ratio 80/20 -[RAND_SPLIT:star_split] -RATIO = 20 -MASK = star_selection - -# The following selection is only used for plotting - -[PLOT:size_mag] -TYPE = plot -FORMAT = png -X_1 = FWHM_IMAGE{fwhm_mag_cut} -Y_1 = MAG_AUTO{fwhm_mag_cut} -X_2 = FWHM_IMAGE{star_selection} -Y_2 = MAG_AUTO{star_selection} -MARKER_1 = + -MARKER_2 = . -MARKERSIZE_1 = 3 -MARKERSIZE_2 = 3 -LABEL_1 = All -LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" -TITLE = "Stellar locus" -XLABEL = "FWHM (pix)" -YLABEL = Mag - -[PLOT:hist_mag_stars] -TYPE = hist -FORMAT = png -Y = MAG_AUTO{star_selection} -BIN = 20 -LABEL = "stars" -XLABEL = "Magnitude" -YLABEL = "Number" -TITLE = "Magnitude of stars" - -[PLOT:fwhm_field] -TYPE = scatter -FORMAT = png -X = X_IMAGE{star_selection} -Y = Y_IMAGE{star_selection} -SCATTER = FWHM_IMAGE{star_selection}*0.186 -MARKER = . -LABEL = "FWHM (arcsec)" -TITLE = "FWHM of stars" -XLABEL = "X (pix)" -YLABEL = "Y (pix)" - -[PLOT:mag_star_field] -TYPE = scatter -FORMAT = png -X = X_IMAGE{star_selection} -Y = Y_IMAGE{star_selection} -SCATTER = MAG_AUTO{star_selection} -MARKER = . -LABEL = "Magnitude" -TITLE = "Magnitude of stars" -XLABEL = "X (pix)" -YLABEL = "Y (pix)" - -[STAT:star_stat] -"Nb objects full cat" = len(FWHM_IMAGE) -"Nb objects not masked" = len(FWHM_IMAGE{flag}) -"Nb stars" = len(FWHM_IMAGE{star_selection}) -"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. -"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 -"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 -"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 -"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 -"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 diff --git a/workflow/config/cfis/star_selection.setools b/workflow/config/cfis/star_selection.setools new file mode 120000 index 000000000..a355664cf --- /dev/null +++ b/workflow/config/cfis/star_selection.setools @@ -0,0 +1 @@ +../../../example/cfis/star_selection.setools \ No newline at end of file From db957416dfa0f15a929b205e0d8cc29f26c8414e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:20:27 -0400 Subject: [PATCH 064/100] docs(smk): one home per concept in the Snakefile, rules and profile Each explanation now lives where the mechanism is introduced; the other sites carry a one-line pointer. Consolidated: the manifest-vs-log contract (completeness.py), the params-formatted-once hazard (unit_pre), the localrule argument (Snakefile localrules), the exposure/tile reclamation asymmetry (clean_tile.py, clean_tile_targets), group resource composition (prepare.smk), the ngmix_range resume hazard (tile.smk range_hash), and what can be carried through apptainer-args (profiles/nibi --bind /local). Also drops two stale claims: tile_vignets no longer declares a temp() directory output, and the profile's group label list said tile_finish. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- profiles/nibi/config.yaml | 42 ++---- workflow/Snakefile | 86 ++++-------- workflow/rules/exposure.smk | 68 ++++------ workflow/rules/prepare.smk | 6 +- workflow/rules/tile.smk | 263 +++++++++++++----------------------- 5 files changed, 159 insertions(+), 306 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 97df987ef..f6cf35d28 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -9,11 +9,8 @@ # Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). # software-deployment-method wraps every job's shell in `apptainer exec` — the # user never types apptainer. WHICH image is resolved in the Snakefile through -# workflow/scripts/container.py: this user's writable sandbox if they built one, -# else their cached SIF (`sp container pull`), else the `container:` path in -# workflow/config.yaml — the shared /project .sif, and the default whenever the -# cache is empty. `sp container status` prints which layer is live. `sp container -# pull` needs the network: login node or salloc, never a batch job. +# workflow/scripts/container.py, whose docstring documents the layers and the +# resolution order; `sp container status` prints which layer is live. executor: slurm @@ -118,43 +115,34 @@ show-failed-logs: true printshellcmds: true # No `keep-incomplete` here. The declared output IS the manifest, so letting -# snakemake delete a failed job's output is exactly the semantics this workflow -# wants: a `.json` on disk means the stage succeeded, and a resume cannot -# schedule downstream work on top of a failure. The post-mortem evidence is the -# rule's `log:` (`/logs/.json`), which completeness.py writes on -# every run and which snakemake preserves through the failure that produced it — -# which is also what `show-failed-logs` above surfaces on the console. +# snakemake delete a failed job's output is exactly the manifest-vs-log +# semantics this workflow wants (completeness.py's docstring argues it), and +# `show-failed-logs` above surfaces the surviving log on the console. # rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still # propagate — completeness.py writes the manifest only on change, so mtimes move # only when reality moves. # # `input` is dropped because reclamation needs the tile->exposure edge to be -# CONDITIONAL: a tile whose final_cat is on disk declares no exposure inputs, so -# a neighbour rebuilding a shared exposure cannot drag it along (see tile.smk). -# With the `input` trigger on, that same conditional reads as "set of input files -# has changed" and reruns every finished tile — against an exposure store that -# reclamation has already deleted. Measured on fixture t4: 70 jobs with the -# trigger, 28 without, for one damaged tile in a four-tile chain. +# CONDITIONAL; with the trigger on, that conditional itself reruns every +# finished tile against a reclaimed exposure store. tile.smk's tile_finished +# commentary carries the mechanism and the fixture-t4 job counts. # # Nothing this workflow relied on is lost. The two genuinely data-derived input # sets are covered another way: a changed exposure list arrives through the # (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides # in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute # + this profile), so a bare snakemake invocation cannot quietly recombine -# reclamation with the `input` trigger; a runtime assertion is impossible -# (the trigger set is unreadable at parse time) — the gate is the launcher. +# reclamation with the `input` trigger. rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the # attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned -# ngmix thread count. The RULES own threads and resources; this profile only -# sets defaults for rules that state nothing (default-resources above). +# ngmix thread count, and would flatten the group resource composition. The +# RULES own threads and resources; this profile only sets defaults for rules +# that state nothing (default-resources above). # -# `group:` fusion of the short rules (PRD D-profile) lives in the rule files, not -# here: the labels are `tile_prep` (prepare.smk), `exp_short` (exp_split + -# exp_mask) and `tile_finish` (tile_merge_cats + tile_make_cat), each documented -# in its file's docstring. Grouping composes resources per toposort level (max -# mem/threads, summed runtime for a linear chain) and leaves the attempt scaling -# above intact, which is the other reason set-resources must stay out of here. +# `group:` fusion of the short rules (PRD D-profile) lives in the rule files, +# each label documented in its file's docstring: `tile_prep` (prepare.smk), +# `exp_short` (exposure.smk) and `tile_shape` (tile.smk). diff --git a/workflow/Snakefile b/workflow/Snakefile index 3813c8b3d..30ba7c8ce 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -21,12 +21,9 @@ output is its MANIFEST (`/manifests/.json`), written by `completeness.py check`. Product files are not declared — a missing CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. -Failure evidence rides on the rule's `log:` (`/logs/.json`, -`unit_log` below): the check writes its full verdict there on every run, and the -manifest only when that verdict is a success. Snakemake deletes a failed job's -declared output and never touches its log, so the two directives say exactly the -two things this workflow needs — "this stage succeeded" and "here is what -happened". +Failure evidence rides on the rule's `log:` (`/logs/.json`): +the manifest says "this stage succeeded", the log says "here is what happened" +(the contract is argued in completeness.py's docstring). """ import functools @@ -324,18 +321,9 @@ STAR_CAT_HASH = script_hash("star_cats.py") # failure family as the script's own no-fallback rule, across time rather than # across processes. # -# ADDING it is itself a params change, and on a RESUME that change is -# destructive rather than merely expensive: it replans the fused group WITHOUT -# tile_vignets, so every chunk trips TILE_VIGNET_REQUIRED and the group fails. -# The failed group then removes each member's existing outputs, final_cat -# included. That was derived from snakemake's cleanup path for months; it is now -# OBSERVED — job 20818649 against smk-g5, 2026-08-30, 32 seconds, and the tile's -# directory on the persistent root was empty afterwards. tile.smk's range_hash -# comment carries the run. The TILE_LOCAL warning does not cover this case at all: -# tile_local() sits in three rules' params and drags tile_vignets back in, -# this sits in one — and clean_tiles: false, the shipped default, is the -# configuration it bites. Land this hash, and any later edit to the split, at -# a campaign boundary on a fresh root. +# Land this hash, and any later edit to the split, at a campaign boundary on a +# fresh root: on a RESUME the params change is destructive, not merely expensive +# (observed, with the transcript, at tile.smk's range_hash). NGMIX_RANGE_HASH = script_hash("ngmix_range.py") # --- exposure reclamation (D5, S5) ----------------------------------------- @@ -370,13 +358,11 @@ def flag(value, default=False): # read no index of their own and must schedule no deletions. CLEAN = flag(config.get("clean", False)) and PHASE == "compute" -# Reclamation REQUIRES the `input` rerun-trigger to be off (profiles/nibi sets -# the list; tile.smk explains why). The two are already tied together: CLEAN is -# gated on SP_PHASE=compute, which only workflow/bin/sp sets, and bin/sp always -# launches with that profile. A bare snakemake invocation without SP_PHASE -# schedules no clean job at all, so it cannot meet the incompatible combination. -# There is no runtime assertion because the trigger set is not readable at parse -# time (workflow.dag_settings is None until the DAG is built). +# Reclamation REQUIRES the `input` rerun-trigger to be off (see tile.smk's +# tile_finished cascade note). The gate is structural rather than asserted: CLEAN +# needs SP_PHASE=compute, only bin/sp sets it, and bin/sp always launches with +# profiles/nibi — so the incompatible combination is unreachable. A runtime +# assertion is impossible (workflow.dag_settings is None until the DAG is built). # Tiles that must not pin an exposure's store. See config.yaml: a permanently # failed tile otherwise holds every exposure it touches (~80) forever, because @@ -455,26 +441,20 @@ def tile_tombstone(tile): def clean_tile_targets(): """Which tiles this invocation may clean: the in-scope ones, and no test. - The exposure case needs clean_targets()' eligibility test because an - exposure's consumer set closes over the campaign. A TILE's store has no - consumer outside that tile — tiles read exposures, nothing reads another - tile's store — so there is nothing to wait for and nothing to defer on. Every - ready tile is eligible the moment its own final_cat exists, and the rule's - input is exactly that. + No eligibility test, because a tile's store has no consumer outside that + tile (clean_tile.py argues the asymmetry with clean_targets()). Every ready + tile is eligible the moment its own final_cat exists, and the rule's input + is exactly that. - The restriction to TILES_READY is therefore not eligibility, it is scope - containment, and it runs the opposite way from the exposure case: an + The restriction to TILES_READY is therefore scope containment, not + eligibility, and it runs the opposite way from the exposure case: an OUT-OF-SCOPE tile's tombstone would drag that tile's final_cat, its tile_make_cat job and the whole fused tile_shape group into this DAG, where the `params` rerun-trigger can find a finished tile out of date and rerun it against reclaimed exposure stores (see the TILE_LOCAL warning in tile.smk). In scope the edge is free: `rule all` already requests the same final_cat. - - Nothing is lost by skipping an out-of-scope tile, because the tile list only - ever grows — a later invocation has it in scope and cleans it then. That is - also what makes flipping `clean_tiles` on RETROACTIVE: the tiles already - reclaimed are exactly the ones with a tombstone, so the missing tombstones - schedule exactly the missing cleans. + Nothing is lost by skipping an out-of-scope tile — the tile list only ever + grows, so a later invocation has it in scope. HEAD PROCESS ONLY, for the same reason as clean_targets() above. """ @@ -505,11 +485,10 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT -e/--exclusive): * ``output/``, ``manifests/`` and ``logs/`` — the last two are a SIBLING - pair, not one dir with two naming conventions: clean_exposure deletes - BOTH wholesale, because a log attesting "complete" for a store that has - been reclaimed contradicts the unbuilt chain the DAG must now see. (The - benchmark tsv is deliberately outside both: it measures the job, it does - not claim anything about the store, so reclamation must not eat it.) + pair, not one dir with two naming conventions, because clean_exposure + deletes both wholesale (clean_exposure.py says why). The benchmark tsv + is deliberately outside both: it measures the job, not the store, so + reclamation must not eat it. * tile: ``tile_numbers.txt`` (dot format — what get_images reads); * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the ORIGINAL exposure name from the index (``2605805p``), so get_images @@ -575,21 +554,10 @@ def sp_shell(stage, config_name, *, check_args="", post=""): else (see unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number by construction (D4). - The check runs even when shapepipe_run failed, so a failed unit still leaves - a record of why: the verdict goes to the rule's `log:` on EVERY run, and to - the declared `.json` manifest only when it is a success. Snakemake - deletes a failed job's output and never touches its log, so the manifest - stays success-only currency and the log is the evidence `sp report` reads. - - The verdict is COMPOSED, which is why `--job-rc "$rc"` is passed: the count - floors and shapepipe_run's exit status are two independent statements about - the same job, and the record must reflect both. Without it, a job whose - counts cleared their floors but whose shapepipe_run died (a late runner - raising after the counted ones wrote their files) publishes a SUCCESS - manifest and still exits nonzero — snakemake deletes that manifest as a - failed job's output, and the log is left claiming "complete" for a stage - with nothing to show for it. With the rc composed in, both files agree that - the job failed. + The rc is CAPTURED rather than ``&&``-ed onto, so the check still runs when + shapepipe_run failed, and it is passed back as ``--job-rc "$rc"`` so the + verdict composes the count floors with shapepipe_run's own exit status. Both + halves are argued in completeness.py's docstring. """ return ( "{params.pre}\n" diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index fc7c3af7d..9fe3e7b29 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -34,11 +34,11 @@ The two rules NOT in it are structural, not taste: of it, so the group has one clean external edge. Different exposures share no DAG edge, so this is one group job per exposure. -NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); -never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering -would make the #746 startup validation turn tolerated per-CCD attrition into a -whole-exposure hard failure. That is now a property of the committed configs -(config_exp_Sp.ini has NUMBER_LIST = $SP_UNIT_NUM; Gie/Ma/psfex have none). +NUMBER_LIST ($SP_UNIT_NUM, see unit_num in the Snakefile) is set only for +exp_split, whose numbering scheme IS the exposure id; never for get_images / +exp_mask / exp_psf, whose per-CCD or download numbering would turn tolerated +per-CCD attrition into a whole-exposure hard failure. It is a property of the +committed configs (config_exp_Sp.ini alone carries the entry). """ rule exp_get_images: @@ -66,14 +66,8 @@ rule exp_get_images: # per HEALPix chunk, into a run-independent chunk store under config `star_cats`. # `exp_star_cat` then reads the chunks covering an exposure's focal plane and # cuts them to it — no network at all. workflow/scripts/star_cats.py holds both -# halves and the geometry they must agree on; its docstring is the reference for -# the chunking and the padding. -# -# The arithmetic: exposures overlap ~7-10 deep and a tile's exposures all look at -# the same square degree, so the old one-cone-per-exposure design re-fetched the -# same sky ~8 times over. Chunked by sky, a full-UNIONS footprint is ~1.5k -# queries where the exposure count would have been ~25k, and a campaign that -# grows within the fetched footprint issues none. +# halves, the geometry they must agree on, and the arithmetic that motivates the +# split; its docstring is the reference for chunking, padding and query counts. # The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE # pointing at a path that does not exist inside the image, so requests is pointed @@ -134,13 +128,9 @@ STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" rule star_catalogue: output: manifest = STAR_CAT_MANIFEST - # No `log:` — see write_manifest() in star_cats.py: this rule runs no - # shapepipe_run and computes no completeness verdict, so there is nothing to - # split between a manifest and a log. Under `set -euo pipefail` it either - # completes or aborts, and snakemake's captured stderr is the evidence. - # `cmd` is a params value, so it is substituted AFTER the shell string is - # formatted: a `{output.manifest}` placeholder in here would survive - # literally (see unit_pre in the Snakefile). Hence the explicit path. + # No `log:` — see write_manifest() in star_cats.py. + # `cmd` is a params value, so placeholders in it are not formatted (see + # unit_pre in the Snakefile). Hence the explicit manifest path. params: cmd = in_container( f"python {SCRIPTS}/star_cats.py fetch" @@ -164,9 +154,7 @@ rule star_catalogue: # numbering scheme needs. Local: one header read for the focal-plane footprint, # a load of the chunks covering it, a radial cut. # -# A LOCALRULE for the same reason as clean_exposure: seconds of work, and one -# sbatch per exposure would be ~20k submissions well under the 15-minute floor -# cluster policy asks us to bundle away. +# A LOCALRULE, for the reason the Snakefile's localrules line gives. # # The per-unit farm is a REAL directory holding exactly this exposure's 40 # numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR @@ -191,9 +179,8 @@ rule star_catalogue: # whose target the purge removed. A purged cut or a deleted farm makes the rule # out of date, it reruns, and it re-cuts or re-links as needed. def star_cat_cmd(exp): - """The whole rule body, as bash — carried as a params value, never inlined - in ``shell:``: it contains literal ``{}`` (the manifest JSON) and snakemake - formats a shell string once, which would consume those braces.""" + """The whole rule body, as bash — carried as a params value because it + contains literal ``{}`` (the manifest JSON); see unit_pre in the Snakefile.""" cut_dir = f"{STAR_CATS}/exp" cat = f"{cut_dir}/star_cat-{exp}.fits" work = exp_dir(exp) @@ -336,26 +323,19 @@ rule exp_psf: # tile's tile_vignets manifest — vignets is the last stage that reads exposure # products, everything after it reads tile-level files. # -# Three properties make the late append behave (see clean_exposure.py): -# * the job deletes the exposure's manifests too, so a tile appended after the -# clean sees an unbuilt chain and regenerates it instead of running against -# an empty store. The tombstone deliberately does NOT stand in for those -# manifests — it is not an input to anything but itself. -# * a finished tile is not disturbed: Snakemake demands a missing intermediate -# only when something downstream of it must run. -# * params.consumers carries the consumer set, so growing it makes the -# tombstone stale under the default `params` rerun-trigger; the clean job -# reruns after the new tile's vignets, against the enlarged set. +# What the job deletes, and why a late append still behaves, is argued in +# clean_exposure.py's docstring; params.consumers is what makes a grown consumer +# set stale (same file). # -# The tile side reads the exposure manifests through ancient() (see tile.smk), -# which is what keeps this deletion from rebuilding every neighbouring tile. -# This rule's OWN inputs are deliberately not ancient: a tile that really did -# rebuild its vignets must reschedule the cleans of the exposures it read. +# The tile side reads the exposure manifests through ancient() and cuts the +# reclaimed edges of finished tiles (see tile.smk), which is what keeps this +# deletion from rebuilding every neighbouring tile. This rule's OWN inputs are +# deliberately not ancient: a tile that really did rebuild its vignets must +# reschedule the cleans of the exposures it read. # -# A localrule (declared in the Snakefile): it is an rmtree, not science, and one -# sbatch per exposure would be ~20k scheduler submissions at DR6 scale. Local -# execution serialises them under local-cores, which costs nothing at rmtree -# speed and never blocks the compute chains (this rule is in none of them). +# A localrule (declared in the Snakefile). Local execution serialises the cleans +# under local-cores, which costs nothing at rmtree speed and never blocks the +# compute chains (this rule is in none of them). rule clean_exposure: input: # ONLY the consumers this invocation may actually build. A consumer that diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 6ca24f4e8..07405407f 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -36,10 +36,8 @@ exposure.smk, into a run-independent store. The tile side has no star-cat node because it has no mask rule yet — see tile.smk. """ -# NUMBER_LIST is never set for get_images (download stage; nothing on disk to -# validate against — #746 would hard-fail the unit). The committed -# config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the -# config, not of an injection step. +# No NUMBER_LIST for get_images — a download stage has nothing on disk to +# validate against (see exposure.smk's docstring for the convention). rule tile_get_images: group: "tile_prep" output: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index f0babc3bf..7873aa555 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -82,13 +82,9 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # config_tile_Ng_template.ini and config_tile_Mc.ini ($NGMIX_VIGNET_DIR), and # config_tile_Ng_template.ini alone for $SP_WCS_DIR. # -# TWO node-local stores, not one, and the second is small on purpose. The -# vignette store is ~5-8 GB of bulk; $SP_WCS_DIR holds a single 11.3 MB file, -# log_exp_headers-.sqlite, which ngmix reads once per object per epoch. -# Moving 5.6 GB bought 4.3x and left 44% of every chunk blocked in NFS RPC on -# that one file. What costs on a network filesystem is the operation, not the -# byte, and the two stores are the same fix applied at the two ends of that -# distribution. +# TWO node-local stores, not one: the ~5-8 GB vignette store and the 11.3 MB +# WCS sqlite, the same fix applied at the two ends of the size distribution +# (tile_local()'s docstring carries the thread-state measurement). # # WHY IT NEEDS THE GROUP. The store is node-local, so it exists only on the # machine that wrote it and only while that job lives. Unfused, tile_vignets and @@ -97,16 +93,9 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # run in one allocation on one node, so the store the first member writes is # simply there for the rest. (Verified live: group members share the directory.) # -# HOW THE PATH GETS IN HERE. Not through the environment. profiles/nibi passes -# --bind /local, and tile_local() below builds the path from the tile wildcard, -# which snakemake substitutes at DAG time. $SLURM_TMPDIR was the obvious choice -# and does not work: snakemake escapes the `$` when it splices apptainer-args -# into the job command, so the container gets the literal string and every fused -# job dies on the guard (campaign 20798193, cancelled after 8 minutes). The guard -# stays, because its job now is to catch a missing --bind rather than a missing -# variable -- and either way the failure must be loud, never a silent fall back -# onto NFS or into /tmp, which in this container is a tmpfs charged to the job's -# memory cgroup. +# HOW THE PATH GETS IN HERE. Not through the environment: profiles/nibi passes +# --bind /local and tile_local() below DERIVES the path from the tile wildcard. +# Why nothing can be communicated instead is on that profile line. # # THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, # not one chunk, because the store dies with the job. At ~1 h per fused tile @@ -147,14 +136,12 @@ def tile_local(tile): """The node-local prologue, as bash, for one tile. A FUNCTION of the tile rather than a constant, because the path has to be - literal by the time apptainer sees it. `$SLURM_TMPDIR` cannot be used: - snakemake escapes the `$` when it splices `apptainer-args` into the job - command, so the container receives the eight characters `$SLURM_TMPDIR` - and every fused job dies (observed live, campaign 20798193). Passing it - through `--env` fails the same way, and `APPTAINERENV_*` would have to be - exported by the job wrapper, which is snakemake's and not ours. - - So the path is derived instead of communicated. `/local/scratch` on nibi is + literal by the time apptainer sees it. Nothing can be COMMUNICATED into the + container -- `$SLURM_TMPDIR`, `--env`, `APPTAINERENV_*` and + `{resources.tmpdir}` all fail, one of them in production; the post-mortem is + on the `--bind /local` line of profiles/nibi/config.yaml. + + So the path is derived instead. `/local/scratch` on nibi is `drwxrwxrwt root:root` -- world-writable with the sticky bit, verified in the container (probe job 20798618) -- and the tile id is a wildcard snakemake substitutes at DAG time, so the shell string carries a concrete @@ -186,19 +173,18 @@ def tile_local(tile): A copy, never a symlink: a link resolves straight back to NFS. The prologue runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in `tile_merge_cats`, which has no pre_run), so the staging is attempted nine - times per tile and eight of those are concurrent siblings in one toposort - level. It is written as copy-to-a-temp-name plus `mv -f`, which is atomic - within a filesystem, and it is UNCONDITIONAL -- no `cp -u`, no - already-there test. `cp` is not atomic, so an interrupted copy leaves a - truncated destination whose mtime is NEWER than the source; `-u` would then - skip it forever, and nothing downstream would catch it, because - TILE_VIGNET_FRESH and TILE_VIGNET_REQUIRED both look only at the vignette - store. A silently truncated WCS store reaches ngmix as wrong astrometry - rather than as an error, which is the worst failure mode available here. - Nine unconditional copies of 11 MB per tile, once at job start, is not a - cost worth reasoning about; a staleness rule would be. Renaming over a file - a sibling chunk already has open is safe -- the open descriptor keeps the - old inode, whose contents are identical. + times per tile, eight of them concurrent siblings in one toposort level. It + is copy-to-a-temp-name plus `mv -f` -- the same all-or-nothing publish + shapepipe.utilities.file_io.write_atomic argues -- and it is UNCONDITIONAL, + no `cp -u` and no already-there test. An interrupted `cp` leaves a truncated + destination whose mtime is NEWER than the source, so `-u` would skip it + forever, and nothing downstream would catch it: TILE_VIGNET_FRESH and + TILE_VIGNET_REQUIRED both look only at the vignette store, and a truncated + WCS store reaches ngmix as wrong astrometry rather than as an error. Nine + unconditional copies of 11 MB per tile is not a cost worth reasoning about; + a staleness rule would be. Renaming over a file a sibling chunk already has + open is safe -- the open descriptor keeps the old inode, identical in + content. `tile_merge_headers` is upstream of the whole group, so the file exists by the time any member runs; failing loudly if it does not is correct, because @@ -258,14 +244,10 @@ find /local/scratch -maxdepth 1 -name 'sp-*.*' -user "$(id -u)" -mmin +1440 \ # rerun trigger, so adding it invalidates nothing. TILE_SLURM_EXTRA = "--tmp=16000" -# One label for the whole shape chain. Composition, verified empirically -# against snakemake 9.23.1 (resources.py::GroupResources.basic_layered, and a -# real sbatch): within a toposort level siblings SUM their mem_mb and cores and -# MAX their runtime; across levels the sums are MAXed and the runtimes SUMMED. -# So the eight chunks are one wave -- 8 x 1 core, 8 x mem, ONE chunk's runtime -# -- and the group's declared runtime is the sum along the chain. That sum is -# what gates partitions on nibi, so each member's runtime below is measured -# p99 plus margin, NOT the old defensive ceiling. See the tile_ngmix docstring. +# One label for the whole shape chain; the composition arithmetic is in this +# file's docstring. The group's declared runtime is a SUM along the chain and +# that sum gates partitions on nibi, so each member's runtime below is measured +# p99 plus margin, not the old defensive ceiling. TILE_GROUP = "tile_shape" # Cleanup, on the LAST member only. $SLURM_TMPDIR is Slurm's to reclaim, but @@ -466,15 +448,8 @@ rule tile_detect: sp_shell("tile_detect", "config_tile_Sx.ini") # PSFEx interpolation to galaxies + vignet postage stamps: the last stage that -# reads exposure products, and the bulk intra-tile intermediate. -# -# The vignette store is declared as a SECOND, temp(directory()) output alongside -# the manifest. This is the ONE scoped exception to "no directory() outputs" -# (D5): the store is ~tens of GB per tile and must be reclaimed when its last -# intra-tile reader finishes, but it must not become DAG currency — so the -# manifest stays the edge, and the directory rides along purely so native temp() -# fires at the right moment. Its readers (tile_ngmix, tile_make_cat) declare -# BOTH. `--notemp` keeps it for debugging. +# reads exposure products, and the bulk intra-tile intermediate. The store it +# writes is node-local (see TILE_LOCAL above). rule tile_vignets: group: TILE_GROUP input: @@ -486,14 +461,12 @@ rule tile_vignets: # tile_merge_headers above. fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: - # THE MANIFEST IS THE ONLY DECLARED OUTPUT NOW. The vignette store used - # to ride along as a second temp(directory()) output purely so native - # temp() would reclaim it; it lives on node-local storage under - # $SLURM_TMPDIR, whose path is not knowable at DAG time, so it cannot - # be declared at all — and does not need to be. It was never DAG - # currency (the manifest always was the edge), its readers are now all - # inside this group job, and Slurm reclaims it with the job. The one - # affordance lost: `--notemp` can no longer keep it for debugging. + # THE MANIFEST IS THE ONLY DECLARED OUTPUT. The vignette store used to + # ride along as a second temp(directory()) so native temp() would + # reclaim it; node-local, it cannot be declared at all — and need not + # be. It was never DAG currency, its readers are all inside this group + # job, and TILE_CLEAN's trap reclaims it. Lost affordance: `--notemp` + # can no longer keep it for debugging. manifest = f"{TILE_DIR}/manifests/tile_vignets.json", log: f"{TILE_DIR}/logs/tile_vignets.json" @@ -519,11 +492,8 @@ rule tile_vignets: runtime = 20, slurm_extra = TILE_SLURM_EXTRA shell: - # The completeness check is pointed at the NODE-LOCAL run root. - # completeness.py is unmodified: --run-dir is its own documented override - # for "$SP_RUN", and --unit keeps the manifest's unit field the tile ID - # instead of the basename of the local root (it would otherwise default - # to "sp-tile"). See sp_shell for the rest. + # The completeness check is pointed at the NODE-LOCAL run root; see + # sp_shell's check_args for what the two flags do. sp_shell("tile_vignets", "config_tile_PiViVi.ini", check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') @@ -567,80 +537,55 @@ rule tile_ngmix: 'eval "$ngmix_range_out"', tile_local(wc.tile), TILE_VIGNET_REQUIRED]), script_hash = SCRIPT_HASH, - # The one rule whose fingerprint has to cover ngmix_range.py. - # `pre_run` already puts the INVOCATION in params, but the invocation - # is invariant to the script's body — and what that body decides is a - # PARTITION. Resume a tile across an edit to the split without this and - # the already-done chunks keep the old ranges while the reruns take the - # new ones: some objects measured twice, others by nobody, the tile - # green, no error anywhere. - # - # ADDING THIS PARAM IS ITSELF A PARAMS CHANGE, AND ON A RESUME IT DOES - # THE VERY THING IT PREVENTS, BY ANOTHER ROUTE. Snakemake compares the - # SET of recorded param values, and smk-g4's metadata for a finished - # chunk (record_format_version 6) holds two of them -- the SCRIPT_HASH - # digest and the pre string -- so a third entry is new and the job - # replans with "Params have changed since last execution". On snakemake - # 9.23.1 under profiles/nibi's trigger set, on a fixture mirroring this - # group with everything up to date, that reschedules the eight chunks, - # tile_merge_cats and tile_make_cat -- and NOT tile_vignets or - # tile_detect, whose manifests exist, so missing_output never queues - # them and nothing propagates down to them either. + # The one rule whose fingerprint has to cover ngmix_range.py: `pre_run` + # already puts the INVOCATION in params, but the invocation is invariant + # to the script's body, and what that body decides is a PARTITION (the + # Snakefile's NGMIX_RANGE_HASH argues the corruption). # - # A group replanned without tile_vignets is precisely the state - # TILE_VIGNET_REQUIRED exists to catch. Every chunk trips the guard and - # the group fails; from there GroupJob.postprocess(error=True) fans out - # over every member and removes its EXISTING outputs -- tile_make_cat's - # final_cat on the persistent root among them. So the cost of getting - # this wrong is a deleted science product, not a wasted hour. + # LANDING THIS PARAM MID-CAMPAIGN DELETES SCIENCE PRODUCTS. A new param + # entry replans the fused group with "Params have changed since last + # execution", scheduling the eight chunks, tile_merge_cats and + # tile_make_cat but NOT tile_vignets or tile_detect (their manifests + # exist, so missing_output never queues them). That is exactly the state + # TILE_VIGNET_REQUIRED catches: every chunk trips the guard, the group + # fails, and GroupJob.postprocess(error=True) removes every member's + # EXISTING outputs -- tile_make_cat's final_cat on the persistent root + # among them. # - # ALL OF THIS IS OBSERVED, end to end, on a real campaign root -- SLURM - # job 20818649 against smk-g5, 2026-08-30. It had been derived twice from - # snakemake's cleanup path and never seen, because "Group jobs: inactive - # (local execution)" puts it out of reach of any login-node fixture. The - # run took 32 seconds: - # * the dry run scheduled tile_ngmix x8 + tile_merge_cats + - # tile_make_cat, reason "params have changed since last execution", - # with tile_vignets and tile_detect ABSENT -- the group's own SLURM - # label came back as tile_shape_tile_make_cat_tile_merge_cats_tile_ngmix, - # with no tile_vignets in it; - # * every chunk tripped the guard below; - # * the group FAILED, and final_cat-186.307.fits was GONE from - # /project afterwards. The tile's directory on the persistent root - # was empty. - # (Restored from a copy taken first; the design and the transcript are in - # sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md.) + # OBSERVED end to end, not derived: SLURM job 20818649 against smk-g5, + # 2026-08-30, 32 seconds. The dry run scheduled exactly that member set + # (the group's SLURM label came back + # tile_shape_tile_make_cat_tile_merge_cats_tile_ngmix, no tile_vignets); + # every chunk tripped the guard; the group failed; and + # final_cat-186.307.fits was GONE from /project afterwards, the tile's + # directory on the persistent root empty. It had never been seen before + # because "Group jobs: inactive (local execution)" puts it out of reach + # of any login-node fixture. (Restored from a copy taken first; design + # and transcript in sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md.) # - # WORTH SITTING WITH: the guard below is what makes this loud rather than - # silent, and the loud failure is precisely what triggers the deletion. A - # guard that turns a wrong catalogue into a failed job is still the right - # trade -- but on a fused group, "fails loudly" is not a cheap outcome, - # and that is an argument for never reaching this state rather than for - # relaxing the guard. + # WORTH SITTING WITH: the guard is what makes this loud rather than + # silent, and the loud failure is precisely what triggers the deletion. + # Still the right trade -- but it argues for never reaching this state, + # not for relaxing the guard. # - # WHY THE TILE_LOCAL WARNING ABOVE DOES NOT ALREADY COVER IT, which is - # the whole reason this needs its own note: tile_local() sits in the - # params.pre of tile_vignets, tile_ngmix AND tile_make_cat, so an edit - # there drags tile_vignets back in and the store is rebuilt. - # range_hash is on tile_ngmix ALONE. Same family, opposite shape -- it - # is the ONE-rule fingerprint change that is dangerous, not the - # three-rule kind that warning describes, and reading it as covering - # both is the mistake to avoid. job_head.sh's runbook sweep for the - # guard state does not help either: it skips any tile with a final_cat, - # which is exactly the set this breaks. + # WHY THE TILE_LOCAL WARNING ABOVE DOES NOT COVER IT: tile_local() sits + # in three rules' params.pre, so an edit there drags tile_vignets back in + # and the store is rebuilt. range_hash is on tile_ngmix ALONE. Same + # family, opposite shape -- the ONE-rule fingerprint change is the + # dangerous one. job_head.sh's runbook sweep does not help either: it + # skips any tile with a final_cat, exactly the set this breaks. # # INVERTED BY clean_tiles, so read the default carefully. With - # reclamation ON the tombstoned tile has lost tile_detect.json, and the + # reclamation ON the tombstoned tile has lost tile_detect.json and the # structural "Input files updated by another job" propagation puts - # tile_vignets back in the group (clean_tile below measures this) -- - # the store is rebuilt and nothing trips. The SHIPPED DEFAULT - # clean_tiles: false is therefore the dangerous configuration, which is - # the opposite of how reclamation reads everywhere else in this file. + # tile_vignets back in the group (measured at clean_tile below), so the + # store is rebuilt and nothing trips. The SHIPPED DEFAULT + # clean_tiles: false is the dangerous configuration -- the opposite of + # how reclamation reads everywhere else in this file. # # So: land this hash, and every later edit to ngmix_range.py, at a - # campaign boundary on a fresh root. Same rule as tile_local()'s, same - # escape hatch and same caveat if a resume is unavoidable -- - # `--rerun-triggers mtime code software-env`. + # campaign boundary on a fresh root. Same rule and same escape hatch as + # tile_local()'s -- `--rerun-triggers mtime code software-env`. range_hash = NGMIX_RANGE_HASH # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over @@ -816,17 +761,12 @@ rule tile_make_cat: # (measured on 186.307; the per-directory breakdown is in clean_tile.py). At # DR6's 23,114 tiles that is 26.9 TiB against a 1 TiB quota and 3.17M inodes # against a 1M one. BOTH BOUNDS BIND, and the byte one binds first: without this -# rule no batch may exceed ~859 tiles. The inode arithmetic is why the whole -# directory goes rather than just `output/` — the 62 non-output inodes per tile -# are still 1.43M at DR6, i.e. over quota on their own. -# -# ELIGIBILITY IS TRIVIAL, AND THAT IS THE POINT. An exposure's consumer set -# closes over the CAMPAIGN (~7-10 tiles read each one), which is why -# clean_targets() has to test every consumer and why clean_exposure carries the -# set in params so a late append makes the tombstone stale. A tile's store has -# NO consumer outside that tile: tiles read exposures, and nothing reads another -# tile's store. So there is no consumer set, no eligibility test, no staleness -# to detect — only an ordering edge on the tile's own final_cat. +# rule no batch may exceed ~859 tiles. +# +# ELIGIBILITY IS TRIVIAL, AND THAT IS THE POINT: a tile's store has no consumer +# outside that tile, so there is no consumer set, no eligibility test and no +# staleness to detect — only an ordering edge on the tile's own final_cat +# (clean_tile.py opens with the asymmetry; the survivor set is argued there too). # # THE INPUT IS final_cat, ON THE PERSISTENT ROOT, AND NOT ancient(). It is # already the campaign's designated tile-finished marker (see final_cat()'s @@ -837,28 +777,9 @@ rule tile_make_cat: # is the whole reason the rule is worth having; ancient() would keep the # dependency but drop the ordering, and the batch would peak at its full size. # -# SCOPE: TILES_READY only, never every tile in the index. clean_targets() has to -# reason about eligibility; this one has to reason about the converse, and for a -# different reason. Requesting an OUT-OF-SCOPE tile's tombstone would put that -# tile's final_cat — hence its tile_make_cat job, hence the whole fused -# tile_shape group — into this DAG, where the `params` rerun-trigger can find it -# out of date (see the TILE_LOCAL warning above) and reschedule a finished tile -# against exposure stores clean_exposure has already reclaimed. In scope, that -# costs nothing: `rule all` requests the same final_cat anyway, so the clean adds -# an edge and no jobs. Deferral, never loss: the tile list accumulates, so every -# later invocation has the tile in scope and picks it up — which is also how -# flipping `clean_tiles` on later reclaims retroactively (the tiles already -# cleaned are exactly the ones with a tombstone). -# -# FOUR SURVIVORS, not one, and clean_tile.py argues each. Two of them are the -# corrections this rule's first design needed: `manifests/tile_find_exposures.json` -# (rule prepare_all_tiles declares it for EVERY tile in the list, and the list -# accumulates — delete it and the prepare group reruns per cleaned tile per -# invocation, writing uncompress's 382 MB back into the store) and the Fe -# exposure list under `output/` (build_index.exp_list_path re-reads it at EVERY -# compute parse; delete it and the default SP_MISSING_THRESHOLD of 0.0 makes the -# first `sp run` after the first reclamation a fatal parse). Ten inodes per tile -# survive in total, ~231k at DR6. +# SCOPE: TILES_READY only, never every tile in the index — see +# clean_tile_targets() in the Snakefile for what an out-of-scope tombstone drags +# into the DAG. # # THE SURVIVING tile_vignets.json DOES NOT TRIP TILE_VIGNET_REQUIRED, AND THE # REASON IS WHAT GETS DELETED, NOT WHAT GETS KEPT. Read that guard above first. @@ -890,12 +811,10 @@ rule tile_make_cat: # safe because nothing upstream of it survives, and tile_detect.json in # particular must never join the list. # -# A LOCALRULE (declared in the Snakefile), same as clean_exposure: it is an -# rmtree, not science, and one sbatch per tile is ~23k scheduler submissions at -# DR6 scale for work shorter than the scheduling latency. Unlike exp_star_cat it -# is a DAG LEAF, so being local can never make it both a dependency and a -# dependent of a group and there is no cycle to avoid — no `group:` label here, -# and none possible. +# A LOCALRULE (declared in the Snakefile), same as clean_exposure. Unlike +# exp_star_cat it is a DAG LEAF, so being local can never make it both a +# dependency and a dependent of a group — no `group:` label here, and none +# possible. # # WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to # tile_local() mid-campaign reruns finished tiles unsatisfiably because their From 928297e875d90f20250b638ba77a8ed0d4c4ac8a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:22:18 -0400 Subject: [PATCH 065/100] docs(scripts): point the reclamation and report scripts at their one home clean_tile.py stops re-arguing clean_exposure.py's symlink safety, log deletion and tombstone-first ordering; run_report.py stops restating the manifest-vs-log contract and the ngmix products caveat; build_index.py's module docstring stops duplicating build(); ngmix_range.py points the RNG claim at position_seed and the resume hazard at NGMIX_RANGE_HASH. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- workflow/bin/sp | 11 +++--- workflow/scripts/build_index.py | 11 ++---- workflow/scripts/clean_exposure.py | 7 ++-- workflow/scripts/clean_tile.py | 57 ++++++++++++------------------ workflow/scripts/ngmix_range.py | 11 +++--- workflow/scripts/run_report.py | 26 +++++--------- 6 files changed, 44 insertions(+), 79 deletions(-) diff --git a/workflow/bin/sp b/workflow/bin/sp index 104efe1b0..0a7d56565 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -11,10 +11,7 @@ # sp report [ARGS...] emit run_report.json now (mid-run is fine). # sp container VERB manage the image every job runs inside: pull, status, # sandbox, exec, resolve. `sp container --help` documents -# the two layers and the resolution order (sandbox -> your -# cached SIF -> the `container:` path in config.yaml). -# `pull` needs the network: run it on a login node or in an -# salloc allocation, never from a batch job. +# the layers and the resolution order. # # Anything else is passed straight through to snakemake with the same profile and # state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). @@ -49,9 +46,9 @@ RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" # "durable, low-volume products go on /project". It is neither: one small, hot # metadata file per output, rewritten on every job, which at DR6 is >170k files # of churn against a backed-up filesystem with a 1M-inode group quota. It is also -# reconstructible — losing it costs a re-parse, not a re-run. (The lesson arrived -# the hard way: a hard 27/27 TiB group quota killed a metadata write mid-run, -# live. The quota has since eased; the placement is right on its own merits.) +# reconstructible — losing it costs a re-parse, not a re-run. (A hard 27/27 TiB +# group quota killed a metadata write mid-run, live. The quota has since eased; +# the placement is right on its own merits.) # --directory only moves state: all data paths are absolute, and the Snakefile # resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 8dc9cb13a..2dd191306 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -34,15 +34,8 @@ find_exposures output. No other parse builds anything: a prepare parse or a passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is already on disk. (There is no ``sp index`` verb; the CLI below stays for -hand-inspection.) It -iterates the *declared* tile list and checks each tile's Fe output at its -deterministic path (no globbing — O(tile-list) existence checks, respecting the -no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in -``missing.json`` and the index is built over the rest. The build fails only if -the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the check -entirely), so a keep-going download storm that lost a few tiles does not cost the -run. The threshold is checked BEFORE anything is written, so a failed build -leaves the previous index and ``missing.json`` intact. +hand-inspection.) ``build`` below documents the missing-tile policy and the +write ordering. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index e65c0dbcf..728f6c527 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -104,9 +104,7 @@ def main() -> None: args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") targets = [t for t in candidates if t.is_symlink() or t.exists()] - # Tombstone first, complete, fsync'd — then delete. See the module docstring: - # the crash window has to sit where the data still exists, not where the - # record does not. + # Tombstone first, complete — then delete (see the module docstring). args.tombstone.parent.mkdir(parents=True, exist_ok=True) tmp = args.tombstone.with_suffix(".json.tmp") tmp.write_text(json.dumps({ @@ -120,8 +118,7 @@ def main() -> None: removed = [] for target in targets: - # NEVER rmtree a symlink: star_cat_exp is a link into the shared pool in - # legacy unit dirs, and rmtree would follow it and empty that pool. + # NEVER rmtree a symlink (see the module docstring). if target.is_symlink(): target.unlink() else: diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py index 1beeb904d..85686df69 100644 --- a/workflow/scripts/clean_tile.py +++ b/workflow/scripts/clean_tile.py @@ -90,8 +90,9 @@ benchmark TSVs are gone. Until it does, per-chunk cost attribution stops at the first reclaimed tile even though the numbers are still on disk. -DELETION IS SYMLINK-SAFE, and here that is not a nicety. A finished tile holds -NINE symlinks in TWO classes, and the second is the one that matters: +DELETION IS SYMLINK-SAFE (``clean_exposure`` gives the general reason), and here +that is not a nicety. A finished tile holds NINE symlinks in TWO classes, and +the second is the one that matters: * ``exp_forest///output`` — 7 links into the EXPOSURE stores, each shared with 7-10 other tiles. Rebuildable, but only by re-running those @@ -112,29 +113,20 @@ next should know that the worst case is not a scratch store they could rebuild — it is a rmtree walking into half a terabyte of shared, backed-up survey data. -LOGS ARE DELETED, NOT ABSORBED, for the same reason ``clean_exposure`` deletes -them: on a finished tile every ``logs/.json`` is BYTE-IDENTICAL to the -``manifests/.json`` beside it (verified across all 16 stage records of -186.307), because completeness.py writes the same verdict to both and a tile -with a failed stage has no final_cat and so is never cleaned. Absorbing them -would duplicate the manifests the tombstone already carries. +LOGS ARE DELETED, NOT ABSORBED, as in ``clean_exposure``, and here the +duplication is exact: on a finished tile every ``logs/.json`` is +BYTE-IDENTICAL to the ``manifests/.json`` beside it (verified across all +16 stage records of 186.307), because a tile with a failed stage has no +final_cat and so is never cleaned. -ORDER: tombstone FIRST, complete and atomically renamed, and only then any -deletion — the reverse of the obvious order, and the same choice -``clean_exposure`` makes. A crash between the two leaves a tombstone beside a -store that still exists: the next invocation treats the tile as cleaned and only -disk is lost. Deleting first would put the crash window where the record that -replaces the store was never written. +ORDER: tombstone FIRST, then deletion — ``clean_exposure``'s docstring argues +the crash window. There is no ``consumers`` field and no consumer-set staleness to detect, because -a tile has no consumers. What reruns this job is the ordinary machinery: the -``script_hash`` param carries this file's content hash (the ``code`` trigger -hashes only the rule's shell string, not the scripts it calls), and the ``mtime`` -trigger fires if final_cat is ever rewritten. A rerun over an already-pruned tree -deletes nothing — and, because absorption is ADDITIVE over the existing -tombstone, does not blank the record either. See ``previous_record``: a naive -re-absorption reduced a 16-manifest tombstone to the two surviving manifests, -found by re-running the fixture clean twice. +a tile has no consumers. What reruns this job is the ordinary machinery +(``script_hash``, and ``mtime`` if final_cat is rewritten). A rerun over an +already-pruned tree deletes nothing and, because absorption is ADDITIVE, does +not blank the record either — see ``previous_record``. """ import argparse @@ -198,12 +190,11 @@ def previous_record(tombstone: Path) -> tuple: THE ABSORPTION IS ADDITIVE, and this is why. A second clean of an already-cleaned tile finds only the two surviving manifests on disk, so a fresh absorption would overwrite a complete record with a two-entry one and - the tile's history would be gone — silently, and for good. That is not a - hypothetical rerun: the ``script_hash`` param reruns this job on any edit to - this file, and the ``mtime`` trigger reruns it if final_cat is ever - rewritten. So the previous record is the BASE and what is on disk is laid - over it: a rebuilt stage's fresh manifest still wins, and a reclaimed one - keeps the only copy that exists. + the tile's history would be gone — silently, and for good. Found by running + the fixture clean twice; not hypothetical, since ``script_hash`` reruns this + job on any edit to this file. So the previous record is the BASE and what is + on disk is laid over it: a rebuilt stage's fresh manifest still wins, and a + reclaimed one keeps the only copy that exists. """ if not tombstone.exists(): return {}, {} @@ -262,10 +253,8 @@ def prune(root: Path, keep: set, removed: list) -> None: survivor, recursed into iff it is a real directory on the way to one, and deleted otherwise. - ``is_symlink()`` is tested BEFORE ``is_dir()`` and before the recursion, - both for the reason ``clean_exposure`` gives (a dangling link is invisible to - ``exists()``) and for a sharper one here: ``exp_forest/`` holds ~8 links into - the shared exposure stores. + ``is_symlink()`` is tested BEFORE ``is_dir()`` and before the recursion — + see the module docstring on the two symlink classes. """ ancestors = {p for k in keep for p in k.parents} for entry in sorted(root.iterdir()): @@ -298,9 +287,7 @@ def main() -> None: manifests.update(absorb_manifests(mdir)) benchmarks.update(absorb_benchmarks(mdir)) - # Tombstone first, complete — then delete. See the module docstring: the - # crash window has to sit where the data still exists, not where the record - # does not. + # Tombstone first, complete — then delete (see the module docstring). args.tombstone.parent.mkdir(parents=True, exist_ok=True) tmp = args.tombstone.with_suffix(".json.tmp") tmp.write_text(json.dumps({ diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index fc9fea7b9..ce9d46680 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -30,9 +30,9 @@ 200.302), the new ones at 1.0000x-1.0002x, and the sum over tiles of slowest-chunk cost falls 132,875 -> 113,710 predicted CPU-s, 14.4%. -Moving the boundaries is scientifically free: ngmix seeds its RNG from object -POSITION (SEED_FROM_POSITION), so which chunk an object falls in cannot change -its measurement. +Moving the boundaries is scientifically free: ngmix's RNG is seeded per object +from its sky position, so which chunk an object falls in cannot change its +measurement (see ``ngmix_package.ngmix.position_seed``). DETERMINISM HERE IS CORRECTNESS, NOT TIDINESS. The n_chunks chunks are separate processes that each run this script independently and must derive the IDENTICAL @@ -46,9 +46,8 @@ failing it. The cross-TIME case is not covered here and cannot be. NGMIX_RANGE_HASH -(Snakefile) makes a RESUME across an edit rerun the chunks rather than mix two -partitions, but a fused group holds its eight chunks open over the LIVE -checkout for hours — smk-g4 measured 6,236-7,762 s of elapsed per chunk — and +(Snakefile) handles a RESUME across an edit, but a fused group holds its eight +chunks open over the LIVE checkout for hours — smk-g4 measured 6,236-7,762 s of elapsed per chunk — and each chunk reads this file only when its own shell starts. An edit landed mid-flight is therefore read by some chunks and not others. Seen once, live: an invocation four seconds after a rewrite returned chunk 5 of 186.307 as diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 7c8b01585..f559e3276 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -13,11 +13,9 @@ tile->exposure edges that let an exposure failure be blamed on the tiles it blocks; * the **verdicts** written by ``completeness.py check`` — per-runner - found/expect/floor and log-scraped failure reasons — which land in two places - with two different lifetimes. Every run writes the unit's ``logs/.json`` - (the rule's snakemake ``log:``, which snakemake never deletes); a run whose - verdict is a success ADDITIONALLY writes ``manifests/.json``, the - rule's declared output, which snakemake deletes when the job fails. + found/expect/floor and log-scraped failure reasons — in the unit's + ``logs/`` (every run) and ``manifests/`` (successes only; completeness.py + argues the split). Both dirs are read here and the status comes from the file BODY, so a failed stage speaks through its log while a successful one is corroborated by two @@ -146,12 +144,10 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, survivors: frozenset = frozenset()) -> set: """Fill in reclaimed units from their ``cleaned.json``; return their ids. - A cleaned exposure has neither ``manifests/`` nor ``logs/`` — - ``clean_exposure`` deleted both, after copying every manifest verbatim into - the tombstone (the logs duplicate them). Read them back, or - the report inverts the truth exactly when reclamation works: the exposure - shows as "not run" and blocks the very tiles whose completion authorised the - deletion. + A cleaned exposure has neither ``manifests/`` nor ``logs/``; every manifest + was copied verbatim into the tombstone first. Read them back, or the report + inverts the truth exactly when reclamation works: the exposure shows as "not + run" and blocks the very tiles whose completion authorised the deletion. Manifests on disk win if both exist — that is a re-built chain, and the tombstone is then a stale record of the previous generation. @@ -233,12 +229,8 @@ def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: exactly, including a warn on any one of the eight ngmix chunks (keep_worst is what makes that true on the tombstone path as well as on disk). - The per-runner ``products`` aggregate is NOT a per-chunk total, before or - after reclamation: a stage contributes the runners of its one surviving - record, so tile_ngmix attrition is counted over one chunk of eight. Reading - it as a whole-tile figure over-states completeness by 8x. That is a property - of collapsing the chunks to one stage row, not of cleaning, and the two - paths now agree on it. + The per-runner ``products`` aggregate is NOT a per-chunk total: reading it + as a whole-tile figure over-states completeness by 8x (see ``keep_worst``). """ per_stage = {} for stage in stages: From c2763f8086e9359deaf18e3e9188a565fdf0d8bb Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:22:56 -0400 Subject: [PATCH 066/100] docs(ngmix): position_seed owns the position-seeded RNG story The physics lived in four places (module docstring, class Notes, the runner guard, the process() loop). It now lives in position_seed's docstring; the others carry a line and a pointer. Drops two stale claims there: the seed is no longer "off in production", and the noise-realisation rng is documented as per-object rather than per-tile. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- .../modules/ngmix_package/__init__.py | 8 ++-- src/shapepipe/modules/ngmix_package/ngmix.py | 38 ++++++++----------- src/shapepipe/modules/ngmix_runner.py | 7 +--- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index fb603566d..ef05bed76 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -58,11 +58,9 @@ ======================== Each object gets its own random number stream, seeded from its sky position -and CCD (``position_seed``, ngmix#796). The output is therefore identical -whether a tile is processed in one go or split into object chunks with -``ID_OBJ_MIN``/``ID_OBJ_MAX``. The older tile-seeded mode is retired; the -config option ``SEED_FROM_POSITION`` is obsolete. Setting it to ``False`` -raises an error, so a stale config cannot silently change the RNG. +and CCD (``ngmix.position_seed``, which argues what that buys). It is the only +mode: the config option ``SEED_FROM_POSITION`` is obsolete, and setting it to +``False`` raises rather than silently changing the RNG. """ diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index ffbab40dd..fca3aa1ea 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -137,7 +137,8 @@ def position_seed(ra, dec, ccd): difference, inflating ``sigma_m``. Seeding the per-object RNG from sky position instead makes the same object draw the same added noise (and the same fit guesses) in every branch, so both cancel and the m-bias error - shrinks. Off in production; a knob for the sim path only. + shrinks. It also makes the result independent of how the tile is split into + ``ID_OBJ_MIN``/``ID_OBJ_MAX`` chunks, which is why it is now the only mode. Box math (kept exactly as Fabian's issue #796):: @@ -299,8 +300,7 @@ def __init__( self.ra = [] self.dec = [] # CCD number of the first epoch, used only to build the per-object - # position seed (see :func:`position_seed`, ngmix#796). ``None`` when - # position seeding is off, so the field costs nothing on the hot path. + # position seed (see :func:`position_seed`). self.ccd = None self.bkg_sub = bkg_sub self.megacam_flip = megacam_flip @@ -442,10 +442,8 @@ class Ngmix(object): Notes ----- - The RNG is always per object and seeded from that object's sky position - (:func:`position_seed`). Results therefore do not depend on how the tile is - split into object chunks, and metacal's ``fixnoise`` counter-noise and the - fit guesses cancel across Pujol image-simulation branches (ngmix#796). + The RNG is always per object and seeded from that object's sky position; + :func:`position_seed` says what that buys. Raises ------ @@ -1014,16 +1012,12 @@ def process(self): n_no_epoch += 1 continue - # Position-seeded per-object RNG (ngmix#796). Each object draws from - # a stream fixed by its own (ra, dec, ccd), so the result is - # independent of which chunk the object lands in and of detection - # order, and the same object gets the same fixnoise counter-noise - # and fit guesses in every Pujol shear branch, so both cancel in the - # branch difference. The prior is rebuilt from the same per-object - # RNG because the guesser draws its initial guess via prior.sample() + # Per-object RNG, seeded from (ra, dec, ccd) — see + # :func:`position_seed`. The prior is rebuilt from that same RNG + # because the guesser draws its initial guess via prior.sample() # (ngmix guessers.py), which consumes the RNG the prior was - # CONSTRUCTED with — a per-object rng alone would leave the guess - # drawing from a shared stream and break both properties. + # CONSTRUCTED with: a per-object rng alone would leave the guess + # drawing from a shared stream and break the invariance. obj_rng = np.random.RandomState( position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) ) @@ -1247,8 +1241,8 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): stamp.ra.append(tile_cat.ra[i_tile]) stamp.dec.append(tile_cat.dec[i_tile]) # CCD of the first surviving epoch — Fabian's coord_list[0] convention - # for the position seed (ngmix#796). All epochs of one object share the - # ra/dec above, so first-epoch CCD pins one deterministic seed stream. + # for the position seed. All epochs of one object share the ra/dec + # above, so first-epoch CCD pins one deterministic seed stream. if stamp.ccd is None: stamp.ccd = int(ccd_n) @@ -1549,8 +1543,8 @@ def prepare_ngmix_weights( weight : numpy.ndarray flag : numpy.ndarray rng : numpy.random.RandomState - Random state for the noise realisations (seeded per tile for - reproducibility). + Random state for the noise realisations (seeded per object; see + :func:`position_seed`). bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. If supplied, unmasked pixels use ``1 / bkg_rms**2`` as the ngmix inverse variance. @@ -1678,8 +1672,8 @@ def make_ngmix_observation( wcs : galsim.BaseWCS Local WCS Jacobian at the object position. rng : numpy.random.RandomState - Random state for the noise realisations (seeded per tile for - reproducibility). + Random state for the noise realisations (seeded per object; see + :func:`position_seed`). bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. centroid_source : {"hsm", "wcs"}, optional diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index d6a004b2e..072288e00 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -142,11 +142,8 @@ def ngmix_runner( else: dilate_neighbour = 1 - # Position-seeded RNG is the only mode: every object's RNG comes from its - # own (ra, dec, ccd), so results do not depend on how the tile is split - # into chunks, and metacal's fixnoise counter-noise cancels across Pujol - # image-simulation shear branches (ngmix#796). The retired tile-seed mode - # had neither property. Old configs that disable it must fail loudly. + # Position-seeded RNG is the only mode (see ngmix.position_seed). Old + # configs that disable it must fail loudly, not silently change the RNG. if config.has_option(module_config_sec, "SEED_FROM_POSITION"): if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): raise ValueError( From 8726e9be80e4545e1f790c5aa864e7942aa8123b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 17:40:18 -0400 Subject: [PATCH 067/100] feat(ngmix): materialise the chunk partition once per tile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eight ngmix chunks of a tile each ran ngmix_range.py in their own shell and trusted the others to have landed on the same boundaries. That trust was misplaced once: a fused group holds its chunks open over the live checkout for hours, so an edit landed mid-flight was read by some chunks and not others — tile 186.307, chunk 5 split 17649..22060 where its siblings had 17129..21229, 520 objects orphaned and 831 measured twice, and merge_sep_cats would have concatenated it green. Single writer instead of eight agreeing readers. ngmix_range.py gains --write (compute the whole partition, publish it as JSON via temp-name + rename) and --read (look one chunk's row up). tile_vignets — the first member of the fused group, strictly preceding every chunk in its DAG — writes $SP_LOCAL/ngmix_ranges.json in its pre_run; each chunk reads its row. The partition math is untouched, so the boundaries are identical for identical inputs. Read mode does not fall back to recomputation when the file is absent; it exits non-zero pointing at tile_vignets. The fallback would be the race again. The file is group-internal plumbing, not DAG currency: it lives and dies with the group job exactly as the vignette store does, so it is no rule's output. NGMIX_RANGE_HASH now fingerprints the script that PRODUCES the file and rides on tile_vignets as well as tile_ngmix. It guards reproducibility across a resume, not sibling agreement — and putting it on both rules also removes the one-rule-fingerprint failure mode that deleted a final_cat on smk-g5, since an edit to the split now drags tile_vignets back into the group. Tests: the invariance property — write the partition, read every row back — reproduces the per-chunk computation exactly, plus the index-validation and no-fallback cases. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- tests/unit/test_ngmix_range.py | 73 ++++++++++++-- workflow/Snakefile | 28 ++++-- workflow/rules/tile.smk | 107 +++++++++++++++----- workflow/scripts/ngmix_range.py | 173 +++++++++++++++++++++++++------- 4 files changed, 304 insertions(+), 77 deletions(-) diff --git a/tests/unit/test_ngmix_range.py b/tests/unit/test_ngmix_range.py index a96ba5cf3..30027cd29 100644 --- a/tests/unit/test_ngmix_range.py +++ b/tests/unit/test_ngmix_range.py @@ -1,18 +1,23 @@ """The ngmix chunk split tiles ``[1, n_obj]`` exactly. -``workflow/scripts/ngmix_range.py`` is run INDEPENDENTLY by each of the -``n_chunks`` ngmix processes, which then trust each other to have computed the -same ranges. Nothing downstream cross-checks: merge_sep_cats concatenates the -chunk catalogues, so an overlap silently duplicates objects and a gap silently -drops them. The one invariant that catches both is that the ranges partition -``1..n_obj`` — that is what this module asserts, over the epoch-weight +Nothing downstream cross-checks the chunk ranges: merge_sep_cats concatenates +the chunk catalogues, so an overlap silently duplicates objects and a gap +silently drops them. The one invariant that catches both is that the ranges +partition ``1..n_obj`` — that is what this module asserts, over the epoch-weight distributions the splitter actually has to cope with. +``workflow/scripts/ngmix_range.py`` now runs ONCE per tile (tile_vignets writes +the whole partition to the group's node-local scratch) and is then only read +from, once per chunk. The second thing this module pins is that the trip through +JSON changes nothing: row ``k`` of the written file is exactly what asking the +splitter for chunk ``k`` used to return. + Deliberately container-free: the range function takes a plain sequence of epoch counts, so nothing here imports astropy, numpy, or shapepipe. """ import importlib.util +import json from pathlib import Path import pytest @@ -99,6 +104,62 @@ def test_ranges_are_deterministic(n_obj, n_chunks, data): assert all(isinstance(b, int) for lo, hi in first for b in (lo, hi)) +# --------------------------------------------------------------------------- # +# Write-once / read-per-chunk is the same partition +# --------------------------------------------------------------------------- # + + +def _roundtrip(epochs, n_chunks): + """What a chunk shell sees: the doc as it comes back off disk.""" + return json.loads(json.dumps(ngmix_range.partition(epochs, n_chunks))) + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=200), + n_chunks=st.integers(min_value=1, max_value=12), + data=st.data(), +) +def test_written_rows_reproduce_the_per_chunk_computation(n_obj, n_chunks, data): + """THE INVARIANCE TEST for materialising the split. + + Writing the whole partition and reading chunk ``k``'s row back must give + byte-for-byte what the old per-chunk ``id_ranges(...)[k - 1]`` returned. If + this ever fails, the two mechanisms have drifted and a tile's coverage is + what pays. + """ + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=40), + min_size=n_obj, + max_size=n_obj, + ) + ) + doc = _roundtrip(epochs, n_chunks) + assert doc["n_obj"] == n_obj and doc["n_chunks"] == n_chunks + expected = ngmix_range.id_ranges(epochs, n_chunks) + got = [ngmix_range.chunk_range(doc, k) for k in range(1, n_chunks + 1)] + assert got == expected + assert_tiles(got, n_obj, n_chunks) + + +def test_chunk_index_outside_the_written_partition_is_fatal(): + """A chunk index the file does not hold must not index from the end.""" + doc = _roundtrip([3] * 40, 4) + for bad in (0, -1, 5): + with pytest.raises(SystemExit, match="outside 1..4"): + ngmix_range.chunk_range(doc, bad) + + +def test_reading_an_absent_ranges_file_is_fatal(tmp_path): + """NO FALLBACK: a missing file fails, it does not recompute. + + Recomputing per chunk is precisely the mechanism this design removed. + """ + with pytest.raises(SystemExit, match="no ranges file"): + ngmix_range.read_ranges(tmp_path / "ngmix_ranges.json") + + # --------------------------------------------------------------------------- # # Degenerate shapes — each pins a decision, not just an absence of crash # --------------------------------------------------------------------------- # diff --git a/workflow/Snakefile b/workflow/Snakefile index 30ba7c8ce..3f3f1d340 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -312,18 +312,26 @@ CLEAN_TILE_HASH = script_hash("clean_tile.py") # config.yaml's star_cats block on clearing the store after a semantic change. STAR_CAT_HASH = script_hash("star_cats.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it -# emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks -# are a PARTITION of its object IDs: resume a tile across an edit to the split -# and the chunks that already succeeded keep the old ranges while the reruns -# take the new ones, so within one tile some objects are measured twice and -# others by nobody. merge_sep_cats concatenates whatever it is handed, so the -# tile completes green with a corrupt catalogue and no error anywhere. Same -# failure family as the script's own no-fallback rule, across time rather than -# across processes. +# emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are +# a PARTITION of its object IDs: resume a tile across an edit to the split and +# the chunks that already succeeded keep the old ranges while the reruns take the +# new ones, so within one tile some objects are measured twice and others by +# nobody. merge_sep_cats concatenates whatever it is handed, so the tile +# completes green with a corrupt catalogue and no error anywhere. +# +# ACROSS TIME IS ALL THIS GUARDS NOW. The within-group version of the same +# corruption — eight sibling chunks each computing the split and disagreeing — is +# structurally closed: the partition is materialised ONCE per tile, by +# tile_vignets, and the chunks only look their row up (tile.smk's +# TILE_NGMIX_RANGES, which also keeps the incident transcript). So this hash buys +# REPRODUCIBILITY of the file across attempts and resumes, not agreement between +# siblings. It therefore fingerprints the script that PRODUCES the file, and +# rides on tile_vignets as well as tile_ngmix — both rules, deliberately; the +# one-rule version of this param is the dangerous one, argued at tile.smk. # # Land this hash, and any later edit to the split, at a campaign boundary on a -# fresh root: on a RESUME the params change is destructive, not merely expensive -# (observed, with the transcript, at tile.smk's range_hash). +# fresh root: a resume across it re-measures the tile, and the failure modes a +# mid-campaign params change can reach are catalogued at tile.smk's range_hash. NGMIX_RANGE_HASH = script_hash("ngmix_range.py") # --- exposure reclamation (D5, S5) ----------------------------------------- diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 7873aa555..109566b1c 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -271,6 +271,25 @@ TILE_VIGNET_FRESH = r""" rm -rf "$SP_VIGNET_OUT/run_sp_tile_PiViVi" """ +# THE SINGLE WRITER OF THE ngmix CHUNK PARTITION, and tile_vignets ONLY. +# +# The ranges are materialised once per tile, here, and every chunk then looks +# its own row up (TILE_NGMIX_RANGE_READ below). tile_vignets is the first member +# of the fused group and strictly precedes all chunks in its DAG, so this is the +# one place in the group where a single process can write something all chunks +# read — no flock, no first-one-wins race. The sexcat it reads is tile_detect's, +# upstream of the whole group and on the SHARED root ($SP_RUN); only the OUTPUT +# is node-local. +# +# Group-internal plumbing, deliberately not a rule output: the file lives and +# dies with the group job exactly as the vignette store does, so a chunk can +# only ever read what its own group job wrote. ngmix_range.py owns the rest. +NGMIX_RANGES = "$SP_LOCAL/ngmix_ranges.json" +TILE_NGMIX_RANGES = ( + f'python {SCRIPTS}/ngmix_range.py --run-dir "$SP_RUN" ' + f'--n-chunks {NGMIX_CHUNKS} --write "{NGMIX_RANGES}" || exit 1' +) + # THE FUSE'S ONE SHARP EDGE, made loud rather than mysterious. # # The vignette store is not a declared output any more, so snakemake cannot see @@ -473,8 +492,14 @@ rule tile_vignets: params: pre = lambda wc: unit_pre("tile_vignets", wc.tile, forest=forest_dir(wc.tile), - pre_run=[tile_local(wc.tile), TILE_VIGNET_FRESH]), - script_hash = SCRIPT_HASH + pre_run=[tile_local(wc.tile), TILE_VIGNET_FRESH, + TILE_NGMIX_RANGES]), + script_hash = SCRIPT_HASH, + # tile_vignets now PRODUCES the chunk partition (TILE_NGMIX_RANGES), so + # it carries the splitter's fingerprint too. Same hash, same constant — + # the Snakefile's NGMIX_RANGE_HASH argues what it guards, and + # tile_ngmix's copy below carries the mid-campaign-edit warning. + range_hash = NGMIX_RANGE_HASH # 8, not 16, for the same reason tile_ngmix is 1: `-b {threads}` is SMP # batch size over input FILE SETS, and a tile is one set -- this run's own # log says "Batch size: 16 / Total number of processes: 1". 16 was the @@ -497,11 +522,12 @@ rule tile_vignets: sp_shell("tile_vignets", "config_tile_PiViVi.ini", check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') -# ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own -# CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params -# function cannot, because params evaluate before the sexcat exists. Closed, not -# open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour straggler's -# root cause (ngmix treats id_obj_max <= 0 as unbounded). +# ngmix shape measurement — N chunks per tile (D4). Each chunk LOOKS UP its own +# CLOSED object-ID range in the file tile_vignets materialised at the top of this +# group job (TILE_NGMIX_RANGES); the ranges are knowable only at EXECUTION time, +# from this tile's own sexcat, which is why a params function cannot supply them. +# Closed, not open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour +# straggler's root cause (ngmix treats id_obj_max <= 0 as unbounded). # # Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and # merge_sep_cats — DAG-serialised after all chunks — is the gather. @@ -531,19 +557,51 @@ rule tile_ngmix: # discards the script's exit status, so a missing sexcat would fall # through to shapepipe_run with an unset range and fail as something # else. Capture, check, then eval — the range script fails as itself. - pre_run=[f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py --run-dir ' - f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' - f'|| exit 1', - 'eval "$ngmix_range_out"', - tile_local(wc.tile), TILE_VIGNET_REQUIRED]), + # tile_local FIRST, because it is what exports $SP_LOCAL — and the + # ranges file the lookup reads lives there, written once by + # tile_vignets (TILE_NGMIX_RANGES). This chunk only looks its row + # up; it never recomputes, and the script refuses to. + # + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing ranges file would + # fall through to shapepipe_run with an unset range and fail as + # something else. Capture, check, then eval — the lookup fails as + # itself. + pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, + f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py ' + f'--read "{NGMIX_RANGES}" --chunk {wc.chunk}) || exit 1', + 'eval "$ngmix_range_out"']), script_hash = SCRIPT_HASH, - # The one rule whose fingerprint has to cover ngmix_range.py: `pre_run` - # already puts the INVOCATION in params, but the invocation is invariant - # to the script's body, and what that body decides is a PARTITION (the - # Snakefile's NGMIX_RANGE_HASH argues the corruption). + # `pre_run` already puts the INVOCATION in params, but the invocation is + # invariant to the script's body, and what that body decides is a + # PARTITION (the Snakefile's NGMIX_RANGE_HASH argues the corruption). + # Carried on tile_vignets too, since the split moved there. + # + # WHY THE PARTITION IS MATERIALISED ONCE, kept as history because the + # design only reads as over-careful until you have seen this. The eight + # chunks used to each run ngmix_range.py in their own shell and trust the + # others to have landed on the same boundaries. A fused group holds them + # open over the LIVE checkout for hours (smk-g4: 6,236-7,762 s elapsed + # per chunk), and each chunk read the script only when its own shell + # started, so an edit landed mid-flight was read by some and not others. + # Seen once, live: an invocation four seconds after a rewrite returned + # chunk 5 of 186.307 as 17649..22060 where the other seven had been split + # 17129..21229 — 520 objects orphaned, 831 measured twice, and + # merge_sep_cats concatenates whatever it is handed, so the tile would + # have completed green. (Twelve sequential and thirty-six concurrent runs + # against a stable checkout gave the identical correct partition; the + # race was purely the edit.) That is now STRUCTURALLY closed: tile_vignets + # writes the ranges once and the chunks only look their row up, so there + # is nothing left for them to disagree about. The hash below no longer + # buys sibling agreement — it buys REPRODUCIBILITY across a resume. # - # LANDING THIS PARAM MID-CAMPAIGN DELETES SCIENCE PRODUCTS. A new param - # entry replans the fused group with "Params have changed since last + # WHAT A ONE-RULE FINGERPRINT DOES MID-CAMPAIGN: IT DELETES SCIENCE + # PRODUCTS. Observed when range_hash lived on tile_ngmix ALONE, which is + # no longer the case -- tile_vignets carries it too now that it is the + # rule that runs the splitter, so an edit drags tile_vignets back into + # the group, the store and the ranges are rebuilt, and nothing trips. + # Kept because the mechanism is general and the next single-rule param is + # one edit away. A new param entry replans the fused group with "Params have changed since last # execution", scheduling the eight chunks, tile_merge_cats and # tile_make_cat but NOT tile_vignets or tile_detect (their manifests # exist, so missing_output never queues them). That is exactly the state @@ -568,12 +626,13 @@ rule tile_ngmix: # Still the right trade -- but it argues for never reaching this state, # not for relaxing the guard. # - # WHY THE TILE_LOCAL WARNING ABOVE DOES NOT COVER IT: tile_local() sits - # in three rules' params.pre, so an edit there drags tile_vignets back in - # and the store is rebuilt. range_hash is on tile_ngmix ALONE. Same - # family, opposite shape -- the ONE-rule fingerprint change is the - # dangerous one. job_head.sh's runbook sweep does not help either: it - # skips any tile with a final_cat, exactly the set this breaks. + # THE RULE THE INCIDENT LEFT BEHIND: a fingerprint that changes on the + # chunks but NOT on tile_vignets is the dangerous shape. tile_local() + # was always safe for exactly this reason -- it sits in three rules' + # params.pre, so an edit there pulls tile_vignets in and the store is + # rebuilt. range_hash is now the same shape by construction. job_head.sh's + # runbook sweep would not have helped either: it skips any tile with a + # final_cat, exactly the set this breaks. # # INVERTED BY clean_tiles, so read the default carefully. With # reclamation ON the tombstoned tile has lost tile_detect.json and the diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index ce9d46680..c3194dfe4 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -1,12 +1,29 @@ #!/usr/bin/env python3 -"""Print one ngmix chunk's closed object-ID range as bash exports. +"""Materialise the ngmix chunk partition once per tile, then serve it. -Run inside the chunk's shell, from the tile's own sexcat, because the range is -only knowable at execution time (PRD D4):: +TWO MODES, ONE WRITER. The partition is computed ONCE, by tile_vignets — the +first member of the fused tile_shape group — and written to the group's +node-local scratch; each of the n_chunks chunk shells then only LOOKS UP its +own row:: - eval "$(ngmix_range.py --run-dir $SP_RUN --chunk 3 --n-chunks 8)" + # tile_vignets, once (see tile.smk's TILE_NGMIX_RANGES) + ngmix_range.py --run-dir $SP_RUN --n-chunks 8 --write $SP_LOCAL/ngmix_ranges.json + + # each chunk, at its own start + eval "$(ngmix_range.py --read $SP_LOCAL/ngmix_ranges.json --chunk 3)" # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 +The file is group-internal plumbing, NOT DAG currency: it lives on $SP_LOCAL and +dies with the group job, exactly like the vignette store. It is deliberately not +a rule output — a chunk can only ever read the file its own group job wrote. + +Read mode does NOT fall back to recomputation when the file is absent; it exits +non-zero. The fallback would restore the very failure this design removes (see +DETERMINISM, below). + +The range is still only knowable at EXECUTION time (PRD D4) — a params function +cannot compute it, because params evaluate before the sexcat exists. + SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering [1, N] processes every object exactly once. The bounds are CLOSED, never ID_OBJ_MAX = -1 — ngmix treats ``id_obj_max <= 0`` as unbounded @@ -34,36 +51,42 @@ from its sky position, so which chunk an object falls in cannot change its measurement (see ``ngmix_package.ngmix.position_seed``). -DETERMINISM HERE IS CORRECTNESS, NOT TIDINESS. The n_chunks chunks are separate -processes that each run this script independently and must derive the IDENTICAL -range set; if two of them disagree, objects are silently measured twice or -silently dropped and nothing downstream notices — merge_sep_cats concatenates -whatever it is given. Hence integer arithmetic end to end (weights scale to -milli-epochs so no float ever decides a boundary), and NO FALLBACK: if the -EPOCH extensions cannot be read this script exits non-zero. Such a fallback -would be the worst available behaviour precisely because it would apply only to -the chunks that hit the failure, shredding the tile's coverage instead of -failing it. - -The cross-TIME case is not covered here and cannot be. NGMIX_RANGE_HASH -(Snakefile) handles a RESUME across an edit, but a fused group holds its eight -chunks open over the LIVE checkout for hours — smk-g4 measured 6,236-7,762 s of elapsed per chunk — and -each chunk reads this file only when its own shell starts. An edit landed -mid-flight is therefore read by some chunks and not others. Seen once, live: an -invocation four seconds after a rewrite returned chunk 5 of 186.307 as -17649..22060 where the other seven had been split 17129..21229, orphaning 520 -objects and double-measuring 831; twelve sequential and thirty-six concurrent -runs against a stable checkout gave the identical correct partition. Nothing in -this script can catch that. The rule is the one tile_local() already states — -land edits between campaigns — and it binds harder here, because tile_local() -fails loudly while this fails green. +DETERMINISM HERE IS CORRECTNESS, NOT TIDINESS. A tile's chunk ranges are a +PARTITION of its object IDs: if two chunks disagree about the boundaries, +objects are silently measured twice or silently dropped and nothing downstream +notices — merge_sep_cats concatenates whatever it is given. + +SINGLE-WRITER IS WHY THAT NO LONGER DEPENDS ON AGREEMENT. Every chunk reads the +same file, produced by one process from one read of the sexcat, so sibling +chunks cannot disagree even in principle — the boundaries are a fact of the +group job rather than a computation eight processes each have to land on. The +mechanism that closes is a real one: this script used to run independently in +each chunk's shell, and a group holds its chunks open over the LIVE checkout for +hours (smk-g4: 6,236-7,762 s of elapsed per chunk), so an edit landed mid-flight +was read by some chunks and not others. Observed once, live — the transcript is +at tile.smk's range_hash, which is the home for that history. + +The rest of the determinism discipline stays, because it is what makes the +written file trustworthy rather than merely shared: integer arithmetic end to +end (weights scale to milli-epochs so no float ever decides a boundary), and NO +FALLBACK — if the EPOCH extensions cannot be read, or the ranges file is absent +in read mode, this script exits non-zero. A fallback would be the worst +available behaviour precisely because it would apply only to the processes that +hit the failure, shredding the tile's coverage instead of failing it. + +What remains uncovered is the cross-TIME case, and NGMIX_RANGE_HASH (Snakefile) +is what covers it: a RESUME that reruns some chunks after an edit to this script +would otherwise mix old and new boundaries within one tile. """ import argparse +import json from pathlib import Path # Weights are integers in milli-epochs (one epoch = 1000) so every boundary is -# decided by exact integer comparison, identically on all n_chunks processes. +# decided by exact integer comparison, identically on every run of the splitter +# — which is now what makes the written file REPRODUCIBLE across attempts and +# resumes, rather than what makes eight siblings agree. MILLI_EPOCH = 1000 # The per-object setup cost, expressed in epochs so one integer weight carries @@ -229,20 +252,96 @@ def object_epochs(run_dir: Path): return counts +def partition(epochs, n_chunks: int) -> dict: + """The whole partition, as the dict that gets serialised to JSON. + + Minimal on purpose: the ranges themselves, plus the two numbers a reader + needs to check that the file is the one it expects (``n_obj`` so a mismatch + against the catalogue is visible, ``n_chunks`` so a chunk index is validated + against what was actually written, not against what the caller believes). + ``chunk`` is 1-based, matching SP_NGMIX_CHUNK and the run-directory suffix. + """ + ranges = id_ranges(epochs, n_chunks) + return { + "n_obj": len(epochs), + "n_chunks": n_chunks, + "chunks": [ + {"chunk": k, "id_min": lo, "id_max": hi} + for k, (lo, hi) in enumerate(ranges, start=1) + ], + } + + +def chunk_range(doc: dict, chunk: int) -> tuple[int, int]: + """Chunk ``chunk``'s closed range out of a ``partition()`` document. + + Validated rather than indexed: a negative ``chunk`` would index from the end + and hand this process some other chunk's range without any error. + """ + n_chunks = doc["n_chunks"] + if not 1 <= chunk <= n_chunks: + raise SystemExit( + f"[ngmix_range] FATAL: --chunk {chunk} outside 1..{n_chunks}" + ) + row = doc["chunks"][chunk - 1] + if row["chunk"] != chunk: + raise SystemExit( + f"[ngmix_range] FATAL: ranges file row {chunk - 1} is chunk " + f"{row['chunk']}, not {chunk}" + ) + return int(row["id_min"]), int(row["id_max"]) + + +def read_ranges(path: Path) -> dict: + """Load the ranges file, or die saying what should have written it.""" + try: + return json.loads(path.read_text()) + except FileNotFoundError: + raise SystemExit( + f"[ngmix_range] FATAL: no ranges file at {path}.\n" + " tile_vignets writes it once per tile, on the node-local scratch " + "this group job shares.\n" + " Its absence means tile_vignets did NOT run in this group job " + "(same cause as a missing vignette store).\n" + " FIX: rm the tile's tile_vignets.json and resume.\n" + " NOT recomputed here on purpose: a per-chunk fallback would let " + "chunks disagree about the partition." + ) + + def main() -> None: - """Emit this chunk's range as the two bash exports the config reads.""" + """Write the whole partition, or emit one chunk's range as bash exports.""" p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--run-dir", required=True, type=Path) - p.add_argument("--chunk", required=True, type=int) - p.add_argument("--n-chunks", required=True, type=int) + p.add_argument("--write", type=Path, metavar="JSON", + help="compute the whole partition and write it here") + p.add_argument("--read", type=Path, metavar="JSON", + help="look one chunk's range up in a file --write made") + p.add_argument("--run-dir", type=Path, help="--write: the tile's run root") + p.add_argument("--n-chunks", type=int, help="--write: chunks to split into") + p.add_argument("--chunk", type=int, help="--read: 1-based chunk index") a = p.parse_args() - # Explicit, because a negative --chunk would index from the end and hand - # this process some other chunk's range without any error. - if not 1 <= a.chunk <= a.n_chunks: + + if bool(a.write) == bool(a.read): raise SystemExit( - f"[ngmix_range] FATAL: --chunk {a.chunk} outside 1..{a.n_chunks}" + "[ngmix_range] FATAL: pass exactly one of --write / --read" ) - lo, hi = id_ranges(object_epochs(a.run_dir), a.n_chunks)[a.chunk - 1] + + if a.write: + if a.run_dir is None or a.n_chunks is None: + raise SystemExit( + "[ngmix_range] FATAL: --write needs --run-dir and --n-chunks" + ) + doc = partition(object_epochs(a.run_dir), a.n_chunks) + # Temp name plus rename, the same all-or-nothing publish tile_local() + # uses for the WCS store: a reader must never see a half-written file. + tmp = a.write.with_name(f".{a.write.name}.tmp") + tmp.write_text(json.dumps(doc)) + tmp.replace(a.write) + return + + if a.chunk is None: + raise SystemExit("[ngmix_range] FATAL: --read needs --chunk") + lo, hi = chunk_range(read_ranges(a.read), a.chunk) print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") From bedf996841c1cf7c8de8253ed6b3a8ea8ef6a4c1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 18:01:58 -0400 Subject: [PATCH 068/100] feat(sp): snapshot the code at launch so edits cannot reach a running campaign The SLURM executor re-invokes snakemake on every job's node, so jobs re-parsed the Snakefile and read workflow/scripts/* and the ini chain out of the LIVE checkout, hours after launch -- how the 186.307 partition split into two disagreeing halves. `sp run` now copies workflow/ (config symlinks dereferenced), src/ and the profile into /code, records HEAD and a dirty flag in snapshot.json, and runs the campaign entirely out of that tree. Refreshed wholesale on every `sp run`; the other verbs run out of the existing snapshot, `sp container` excepted. workflow.basedir IS the snapshot, so every internal path follows for free. The profile's PYTHONPATH pin is the one that cannot (YAML splices nothing, and snakemake escapes a literal $): the snapshot's copy of the profile has that one path rewritten and --profile points at the copy, so container.py keeps reading one apptainer-args line -- the snapshot's, from inside a job; the checkout's, for interactive `sp container exec`. Docs follow: workflow/README.md gains the "launch code snapshot" section as the one home, and the mid-campaign-edit warnings in the Snakefile header, tile.smk and ngmix_range.py shrink to pointers. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- profiles/nibi/config.yaml | 29 +++++--- workflow/README.md | 22 +++++- workflow/Snakefile | 5 ++ workflow/bin/sp | 120 ++++++++++++++++++++++++++++++-- workflow/rules/tile.smk | 13 ++-- workflow/scripts/ngmix_range.py | 9 +-- 6 files changed, 173 insertions(+), 25 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index f6cf35d28..23d385bd3 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -36,19 +36,23 @@ software-deployment-method: [apptainer] # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp # >>> EDIT THIS: THE ONE NON-PORTABLE LINE IN THIS FILE. <<< # The apptainer-args string below hardcodes one user's home -# (--home /home/cdaley) and one checkout's src/ (the -# PYTHONPATH pin). A different user must edit both by hand. +# (--home /home/cdaley). A different user must edit it by hand. # IT CANNOT BE FIXED HERE: YAML cannot splice an env var, and # snakemake escapes a literal `$` before the string reaches # any shell (see the $SLURM_TMPDIR post-mortem below, which -# cost a whole campaign). Nor can bin/sp inject it: snakemake's -# --apptainer-args REPLACES the profile value rather than -# appending to it, so the launcher would have to restate this -# whole string -- and workflow/scripts/container.py reads THIS -# LINE to give `sp container exec` the same environment jobs -# get, so a second definition is exactly the divergence that -# mechanism exists to prevent. One edited line beats two -# sources of truth. +# cost a whole campaign). Nor can bin/sp inject it through +# --apptainer-args: that REPLACES the profile value rather +# than appending to it, so the launcher would have to restate +# this whole string -- and workflow/scripts/container.py reads +# THIS LINE to give `sp container exec` the same environment +# jobs get, so a second definition is exactly the divergence +# that mechanism exists to prevent. +# THE PYTHONPATH PIN IS THE ONE EXCEPTION, and it is not +# edited by hand: `sp run` copies this whole file into the +# campaign's code snapshot and rewrites that ONE path to the +# snapshot's src/ (bin/sp, "the launch code snapshot"). Still +# one source of truth for the flags -- the copy is generated, +# never edited -- and jobs stop reading a live checkout. # --bind /local NODE-LOCAL NVME. $SLURM_TMPDIR on nibi is # /local/scratch/..0, and the mount is /local # (/dev/nvme0n1, ext4, 3.5 TB) -- binding /localscratch is @@ -106,6 +110,11 @@ software-deployment-method: [apptainer] # live p3-batch1 job reads it) and not the sif default # (frozen pre-#843, and pre-#873). Production later rebuilds # the sif at the validated commit and DROPS this --env line. +# The path written HERE is the checkout's, and is what +# `sp container exec` uses; the value JOBS see is the +# snapshot's src/, substituted per the paragraph above. So a +# running campaign holds its launch code even after this +# line (or the tree it names) changes. apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch --bind /local" latency-wait: 60 # NFS: wait for outputs to appear after a job diff --git a/workflow/README.md b/workflow/README.md index 52e8d58c3..08bfd2291 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -102,6 +102,26 @@ therefore two snakemake invocations over one Snakefile: `sp run` chains both so the UX is one command; both exit codes are checked and the run fails if either phase failed. +## The launch code snapshot + +`sp run` copies the code it is about to launch — `workflow/` (config symlinks +dereferenced), `src/` and the profile — into `/code`, records HEAD +plus a dirty flag in `/code/snapshot.json`, and runs the campaign +entirely out of that copy. It matters because a campaign is not one process: the +SLURM executor re-invokes snakemake on every job's node, so jobs re-parse the +Snakefile and read `workflow/scripts/*` and the ini chain hours after launch. +**Editing the checkout while a campaign runs is therefore harmless; a change +takes effect on the next `sp run`.** + +Everything workflow-internal hangs off `workflow.basedir`, which *is* the +snapshot, so it follows for free. The one exception is the profile's `PYTHONPATH` +pin, which YAML cannot interpolate: `sp run` rewrites that single path in the +snapshot's copy of the profile and launches `--profile` at the copy. The snapshot +is refreshed wholesale on every `sp run` — a new `sp run` *is* the relaunch — and +the other verbs run out of the existing snapshot, `sp container` excepted (it is +about the image you are working with now, not about a campaign). The mechanism +and its rationale live in one place: `bin/sp`. + ## Execution mode: one SLURM job per rule The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule @@ -131,7 +151,7 @@ profile-only pass. workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks config.yaml the run: tile list, paths, container, chunk count - bin/sp committed launcher (module load + /project venv + run/report/container/cancel) + bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3f3f1d340..b925adc06 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -16,6 +16,11 @@ rebuilding it changes which jobs exist without invalidating completed work; it ACCUMULATES across invocations, so a later clean_exposure (S5) sees every consuming tile of the whole campaign, not just this tile list. +Neither invocation reads the checkout: `sp run` snapshots the code into the +campaign's state dir and points snakemake at THAT tree, so `workflow.basedir` +below is the snapshot and mid-campaign edits are inert. The mechanism and its +rationale live in bin/sp. + The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared output is its MANIFEST (`/manifests/.json`), written by `completeness.py check`. Product files are not declared — a missing CCD is often diff --git a/workflow/bin/sp b/workflow/bin/sp index 0a7d56565..9b5550bea 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -4,6 +4,9 @@ # Three verbs, nothing else: # # sp run [ARGS...] bring the products on disk up to date with the tile list. +# Snapshots the code into the state dir first (see "the +# launch code snapshot" below) and runs out of the copy, so +# editing the checkout mid-campaign cannot reach the jobs. # Two snakemake invocations over one Snakefile: # 1. PREPARE snakemake prepare_all_tiles # 2. COMPUTE snakemake all <- its PARSE builds the index @@ -26,7 +29,6 @@ set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ REPO="$(dirname "$HERE")" VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" -PROFILE="$REPO/profiles/nibi" SCRIPTS="$HERE/scripts" CONFIG="$HERE/config.yaml" @@ -53,20 +55,115 @@ RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" # resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" +# --- the launch code snapshot ---------------------------------------------- +# THE ONE HOME for this concept; everything else points here. +# +# WHY. A campaign is not one process. The SLURM executor re-invokes snakemake on +# every job's node, so each job RE-PARSES the Snakefile, and each shell reads +# workflow/scripts/*.py and workflow/config/cfis/* when it STARTS -- hours after +# launch, out of whatever the checkout contains THEN. Editing the checkout +# mid-campaign therefore fed different code to different jobs of the same run +# (the 186.307 partition incident: an edit landed four seconds into a tile split +# it into two disagreeing halves). +# +# WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and +# runs the campaign entirely out of that copy: the Snakefile, the rules, the +# scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points +# into example/, and the copy must be self-contained), src/, and the profile. +# Every workflow-internal path hangs off `workflow.basedir`, which IS the +# snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the +# one that cannot (YAML splices nothing) and is rewritten below. +# +# CONSEQUENCE, and the reason the old "never edit workflow/ while a campaign +# runs" warnings are gone: mid-campaign edits to the checkout are inert. A change +# takes effect on the next `sp run`, and not before. +# +# LIFECYCLE: one snapshot per campaign state dir, REFRESHED WHOLESALE on every +# `sp run` -- a new `sp run` IS the relaunch, and snakemake's params/code rerun +# triggers decide what the new code invalidates. Nothing accumulates, and the +# path is stable, so a later `sp --unlock` reaches the same workflow that took +# the lock. +SNAPSHOT="$STATE_DIR/code" +snapshot_code() { + mkdir -p "$SNAPSHOT" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ + "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + else + rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" + cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + + fi + + # The profile's `apptainer-args:` hardcodes one checkout's src/ in its PYTHONPATH + # pin and CANNOT interpolate anything (snakemake escapes a literal `$` before the + # string reaches any shell -- the post-mortem is on that line). So the pin is + # rewritten HERE, in the snapshot's copy, and `sm` points --profile at that copy. + # The checked-in profile keeps its literal checkout path and stays the source of + # truth for the FLAGS; exactly one path is substituted, by one line of sed-work. + python - "$SNAPSHOT/profiles/nibi/config.yaml" "$SNAPSHOT/src" <<'PY' +import pathlib, re, sys +f, src = pathlib.Path(sys.argv[1]), sys.argv[2] +text, n = re.subn(r'(--env PYTHONPATH=)\S+', lambda m: m.group(1) + src, + f.read_text(), count=1) +if not n: + sys.exit("sp: no --env PYTHONPATH= found in the profile's apptainer-args; " + "the snapshot would run against the live checkout's src/") +f.write_text(text) +PY + + python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' +import json, pathlib, subprocess, sys, time +out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] +def git(*a): + try: + p = subprocess.run(["git", "-C", repo, *a], capture_output=True, + text=True, timeout=30) + except OSError: + return None + return p.stdout.strip() if p.returncode == 0 else None +status = git("status", "--porcelain") +out.write_text(json.dumps({ + "_comment": "Code snapshot taken by `sp run`; the campaign runs out of THIS " + "tree, never the checkout. Refreshed wholesale on every `sp run`.", + "source": repo, + "taken_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "head": git("rev-parse", "HEAD"), + "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(status), + "dirty_files": status.splitlines() if status else [], +}, indent=2) + "\n") +PY + echo "sp: code snapshot refreshed at $SNAPSHOT" >&2 +} + # SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of # declared tiles allowed to be missing their exposure list (default 0.0). export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" -# --snakefile pins the workflow to this checkout: sp must work from any cwd -# (an sbatch head job starts in the submission directory, not the repo). +# WHICH code a verb runs. `sp run` refreshes the snapshot and runs out of it; +# every other verb and the escape hatch run out of the EXISTING snapshot when the +# state dir has one -- `sp --unlock` and `sp report` must speak for the campaign +# that is actually on disk, not for a checkout that has moved since. Before the +# first `sp run` there is no snapshot and the live checkout is all there is. +# (`sp container` is the exception: it is interactive tooling about the image, +# not about a campaign, and stays on the checkout.) +code_root() { [ -f "$SNAPSHOT/workflow/Snakefile" ] && echo "$SNAPSHOT" || echo "$REPO"; } + +# --snakefile pins the workflow to one tree: sp must work from any cwd (an sbatch +# head job starts in the submission directory, not the repo), and the SLURM +# executor re-invokes snakemake with this same path inside every job -- which is +# exactly what makes the snapshot stick. # # SP_PHASE is REQUIRED by the Snakefile (a bare `snakemake` would build an empty # DAG and exit 0). `sp run` sets prepare/compute on its two invocations; every # other verb and the escape hatch fall through to `passthrough`, which parses the # index without building it and schedules no side effects. sm() { + local root; root="$(code_root)" SP_PHASE="${SP_PHASE:-passthrough}" \ - snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" \ + snakemake --snakefile "$root/workflow/Snakefile" \ + --profile "$root/profiles/nibi" \ --directory "$STATE_DIR" "$@" } @@ -74,6 +171,7 @@ cmd="${1:-}" case "$cmd" in run) shift + snapshot_code # PREPARE failing is NOT fatal to the run: keep-going means a failed tile # poisons only its own cone, and the tiles that lost their exposure list are # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which @@ -100,13 +198,25 @@ case "$cmd" in fi ;; report) + # Out of the snapshot when there is one, so a manual report reads the trees + # with the same script the campaign's own onsuccess/onerror hooks use (they + # call it through the Snakefile's SCRIPTS, which is the snapshot). Read-only + # either way -- this is consistency, not safety. shift - python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + python "$(code_root)/workflow/scripts/run_report.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" \ --status manual "$@" ;; container) # The image layer, outside snakemake entirely: the script is stdlib-only so # it runs on the bare host, and the apptainer module is already loaded above. + # Deliberately the CHECKOUT's copy, not the snapshot's: `sp container` is + # about the image you are working with now (status compares against HEAD, and + # the snapshot has no .git), and `exec` should hand you the environment your + # NEXT run will have. It reads the checked-in profile's apptainer-args, so a + # running campaign whose snapshot predates an edit to that line gets the old + # value while `exec` shows the new one -- the only place the two can differ, + # and it differs in the direction that matters (jobs stay pinned). shift python "$SCRIPTS/container.py" "$@" ;; diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 109566b1c..74d985e2a 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -127,8 +127,10 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # whole exposure edge set, rebuilding the reclaimed chains from VOS. That is # the rerun avalanche D5 exists to prevent, arriving through the params # trigger instead of through inputs. -# So: land changes to this function BETWEEN campaigns, never mid-campaign. If a -# resume is genuinely needed after an edit, run it once with +# So: land changes to this function BETWEEN campaigns. An edit cannot reach a +# RUNNING one (the launch code snapshot, bin/sp) -- the hazard is the `sp run` +# after it, which is a resume against finished tiles. If that resume is genuinely +# needed, run it once with # `--rerun-triggers mtime code software-env`, accepting that clean_exposure's # consumer-set staleness detection (which rides on params) is off for that # invocation. @@ -581,9 +583,10 @@ rule tile_ngmix: # design only reads as over-careful until you have seen this. The eight # chunks used to each run ngmix_range.py in their own shell and trust the # others to have landed on the same boundaries. A fused group holds them - # open over the LIVE checkout for hours (smk-g4: 6,236-7,762 s elapsed - # per chunk), and each chunk read the script only when its own shell - # started, so an edit landed mid-flight was read by some and not others. + # open for hours (smk-g4: 6,236-7,762 s elapsed per chunk), and each chunk + # read the script only when its own shell started -- against the LIVE + # checkout, as every job did before the launch code snapshot (bin/sp) -- + # so an edit landed mid-flight was read by some and not others. # Seen once, live: an invocation four seconds after a rewrite returned # chunk 5 of 186.307 as 17649..22060 where the other seven had been split # 17129..21229 — 520 objects orphaned, 831 measured twice, and diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index c3194dfe4..a6879c504 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -61,10 +61,11 @@ chunks cannot disagree even in principle — the boundaries are a fact of the group job rather than a computation eight processes each have to land on. The mechanism that closes is a real one: this script used to run independently in -each chunk's shell, and a group holds its chunks open over the LIVE checkout for -hours (smk-g4: 6,236-7,762 s of elapsed per chunk), so an edit landed mid-flight -was read by some chunks and not others. Observed once, live — the transcript is -at tile.smk's range_hash, which is the home for that history. +each chunk's shell, and a group holds its chunks open for hours (smk-g4: +6,236-7,762 s of elapsed per chunk), reading — as every job did before the launch +code snapshot (bin/sp) — the LIVE checkout, so an edit landed mid-flight was read +by some chunks and not others. Observed once, live — the transcript is at +tile.smk's range_hash, which is the home for that history. The rest of the determinism discipline stays, because it is what makes the written file trustworthy rather than merely shared: integer arithmetic end to From 84435c49570a1ac76748708f9fe0cac73d2b45c9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 18:04:12 -0400 Subject: [PATCH 069/100] ngmix: hoist per-object sqlitedict reads out of the epoch loop Each vignet.[str(obj_id)] access unpickles the object's whole all-epoch dict; six of them sat inside the per-epoch loop, so an object with n epochs paid n full unpickles per store where 1 suffices. Read each store's per-object dict once before the loop and index the locals inside it. The psf/gal dicts read for the empty-check in process() are now passed down instead of being re-read. Also cache f_wcs_file[exp_name] in a per-call dict: it was fetched twice per epoch (WCS, then header), each time unpickling all CCDs of the exposure. No numerical change: same values, fewer unpickles. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PsP9yhF9okPYrWKydmnpcS --- src/shapepipe/modules/ngmix_package/ngmix.py | 85 ++++++++++++++------ 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index fca3aa1ea..5ca65b9bc 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -1000,13 +1000,24 @@ def process(self): id_last = obj_id count += 1 - # Skip objects with no multi-epoch PSF or vignet data - if (vignet_cat.psf_vign_cat[str(obj_id)] == 'empty' - or vignet_cat.gal_vign_cat[str(obj_id)] == 'empty'): + # Skip objects with no multi-epoch PSF or vignet data. + # Read each store once here and pass the dicts down: every + # sqlitedict access unpickles the object's whole all-epoch dict. + psf_obj = vignet_cat.psf_vign_cat[str(obj_id)] + gal_obj = vignet_cat.gal_vign_cat[str(obj_id)] + if psf_obj == 'empty' or gal_obj == 'empty': n_empty_cat += 1 continue - stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat, self._bkg_sub) + stamp = prepare_postage_stamps( + vignet_cat, + obj_id, + i_tile, + tile_cat, + self._bkg_sub, + psf_obj, + gal_obj, + ) if len(stamp.gals) == 0: n_no_epoch += 1 @@ -1134,25 +1145,49 @@ def process(self): # Log mean ellipticity statistics self.log_mean_ellipticity() -def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): +def prepare_postage_stamps( + vignet, + obj_id, + i_tile, + tile_cat, + bkg_sub=True, + psf_obj=None, + gal_obj=None, +): # define per-object lists of individual exposures to go into ngmix stamp = Postage_stamp(bkg_sub=bkg_sub) + # Read each store's per-object dict ONCE: every sqlitedict access + # unpickles the object's whole all-epoch dict, so keeping these out of + # the epoch loop below saves O(n_epoch) full unpickles per store. + if psf_obj is None: + psf_obj = vignet.psf_vign_cat[str(obj_id)] + if gal_obj is None: + gal_obj = vignet.gal_vign_cat[str(obj_id)] + bkg_obj = ( + vignet.bkg_vign_cat[str(obj_id)] + if stamp.bkg_sub and vignet.bkg_vign_cat is not None + else None + ) + flag_obj = vignet.flag_vign_cat[str(obj_id)] + weight_obj = vignet.weight_vign_cat[str(obj_id)] + bkg_rms_obj = ( + vignet.bkg_rms_vign_cat[str(obj_id)] + if vignet.bkg_rms_vign_cat is not None + else None + ) + wcs_cache = {} #identify exposure and ccd number from psf catalog - psf_expccd_names = list(vignet.psf_vign_cat[str(obj_id)].keys()) + psf_expccd_names = list(psf_obj.keys()) for expccd_name in psf_expccd_names: exp_name, ccd_n = re.split('-', expccd_name) - gal_vign = ( - vignet.gal_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + gal_vign = gal_obj[expccd_name]['VIGNET'] if np.all(gal_vign == 0): continue if stamp.bkg_sub: - bkg_vign = ( - vignet.bkg_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + bkg_vign = bkg_obj[expccd_name]['VIGNET'] gal_vign_sub_bkg = background_subtract( gal_vign, bkg_vign @@ -1185,9 +1220,7 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): if stamp.megacam_flip and tile_seg is not None: tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) - flag_vign = ( - vignet.flag_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + flag_vign = flag_obj[expccd_name]['VIGNET'] if tile_vign is not None: flag_vign[np.where(tile_vign == -1e30)] = 2**10 v_flag_tmp = flag_vign.ravel() @@ -1195,23 +1228,27 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): if len(np.where(v_flag_tmp != 0)[0]) / v_flag_tmp.size > 1 / 3.0: continue - weight_vign = vignet.weight_vign_cat[str(obj_id)][expccd_name]['VIGNET'] + weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( - vignet.bkg_rms_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - if vignet.bkg_rms_vign_cat is not None + bkg_rms_obj[expccd_name]['VIGNET'] + if bkg_rms_obj is not None else None ) - epoch_wcs = vignet.f_wcs_file[exp_name][int(ccd_n)]['WCS'] + # One unpickle per exposure (all CCDs), reused across this object's + # epochs; the cache is per-call, so bounded by the object's exposures + if exp_name not in wcs_cache: + wcs_cache[exp_name] = vignet.f_wcs_file[exp_name] + ccd_wcs = wcs_cache[exp_name][int(ccd_n)] + + epoch_wcs = ccd_wcs['WCS'] jacob = get_galsim_jacobian( epoch_wcs, tile_cat.ra[i_tile], tile_cat.dec[i_tile] ) - header = fits.Header.fromstring( - vignet.f_wcs_file[exp_name][int(ccd_n)]['header'] - ) + header = fits.Header.fromstring(ccd_wcs['header']) # rescale by relative zero-points ( @@ -1227,9 +1264,7 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): # gather postage stamps in all of the epochs stamp.gals.append(gal_vign_scaled) - stamp.psfs.append( - vignet.psf_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + stamp.psfs.append(psf_obj[expccd_name]['VIGNET']) stamp.weights.append(weight_vign_scaled) stamp.flags.append(flag_vign) stamp.bkg_rms.append(bkg_rms_vign_scaled) From cad7eb588b787a48cb94bd4a08532e4589994b9f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 17:18:42 +0200 Subject: [PATCH 070/100] feat(mask): declare healsparse + hpgeom dependencies First step of the external-healsparse-mask path (#846): the reader rasterizes maskforce healsparse products onto image pixel grids in place of internal mask generation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CfRuCQa2UHo44yp2MZDsJX --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8b259deec..5388adc86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ dependencies = [ "cs_util>=0.2.1", "galsim>=2.8", "h5py", + "healsparse", + "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", From 3bb39e3884e3fec364ae5ad4dd2d46e71507181e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 17:46:04 +0200 Subject: [PATCH 071/100] feat(mask_ext): rasterize external healsparse masks to pipeline flag images New mask_ext module (PRD in #847): reads the image WCS, evaluates the pixel grid in memory-bounded row chunks, queries the healsparse map, applies a config-driven bit->flag mapping, optionally sums the external instrument flag (matching Mask._build_final_mask semantics), and writes the standard int16 _flag.fits. Same module serves tiles and exposure CCDs; mask files and bit meanings live only in config. Example tile/exposure configs; 7 unit tests on synthetic maps (bit mapping, chunk-seam independence, RA wrap, off-map, ext-flag sum, WCS round-trip). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CfRuCQa2UHo44yp2MZDsJX --- example/cfis/config_exp_MaExt.ini | 50 +++ example/cfis/config_tile_MaExt.ini | 51 +++ .../modules/mask_ext_package/__init__.py | 71 ++++ .../modules/mask_ext_package/mask_ext.py | 319 ++++++++++++++++++ src/shapepipe/modules/mask_ext_runner.py | 117 +++++++ tests/module/test_mask_ext.py | 230 +++++++++++++ 6 files changed, 838 insertions(+) create mode 100644 example/cfis/config_exp_MaExt.ini create mode 100644 example/cfis/config_tile_MaExt.ini create mode 100644 src/shapepipe/modules/mask_ext_package/__init__.py create mode 100644 src/shapepipe/modules/mask_ext_package/mask_ext.py create mode 100644 src/shapepipe/modules/mask_ext_runner.py create mode 100644 tests/module/test_mask_ext.py diff --git a/example/cfis/config_exp_MaExt.ini b/example/cfis/config_exp_MaExt.ini new file mode 100644 index 000000000..ecd416e13 --- /dev/null +++ b/example/cfis/config_exp_MaExt.ini @@ -0,0 +1,50 @@ +## ShapePipe configuration file for exposure-CCD external-mask (healsparse) +## rasterization. Reprojects the unified UNIONS healsparse mask through each +## single-exposure single-CCD WCS and sums in the instrument flag file, +## producing the pipeline_flag.fits artifact consumed downstream. + +[DEFAULT] +VERBOSE = True +RUN_NAME = run_sp_exp_MaExt +RUN_DATETIME = False + +[EXECUTION] +MODULE = mask_ext_runner +MODE = SMP + +[FILE] +LOG_NAME = log_sp_exp +RUN_LOG_NAME = log_run_sp +INPUT_DIR = $SP_RUN/output +OUTPUT_DIR = $SP_RUN/output + +[JOB] +SMP_BATCH_SIZE = 1 +TIMEOUT = 96:00:00 + +[MASK_EXT_RUNNER] + +# Parent module: single-exposure single-CCD images and instrument flags +INPUT_DIR = last:split_exp_runner + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Path of the external healsparse mask file (band-agnostic; r-band for shear) +MASK_PATH = $SP_CONFIG/mask_r.hsp + +# Healsparse bit value -> output flag value mapping +BIT_FLAG_MAP = 64:1 + +# Flag value for pixels outside the healsparse footprint (0 = unflagged) +OFF_MAP_FLAG = 0 + +# External instrument flag file: summed into the rasterized mask +USE_EXT_FLAG = True + +# HDU of the external instrument flag FITS file (optional, default 0) +HDU = 0 + +# File name prefix for the output flag files +PREFIX = pipeline diff --git a/example/cfis/config_tile_MaExt.ini b/example/cfis/config_tile_MaExt.ini new file mode 100644 index 000000000..410a6890d --- /dev/null +++ b/example/cfis/config_tile_MaExt.ini @@ -0,0 +1,51 @@ +## ShapePipe configuration file for tile external-mask (healsparse) rasterization +## Rasterizes the unified UNIONS healsparse mask onto each tile pixel grid, +## producing the pipeline_flag.fits artifact consumed downstream. + +[DEFAULT] +VERBOSE = True +RUN_NAME = run_sp_tile_MaExt +RUN_DATETIME = False + +[EXECUTION] +MODULE = mask_ext_runner +MODE = SMP + +[FILE] +LOG_NAME = log_sp_exp +RUN_LOG_NAME = log_run_sp +INPUT_DIR = $SP_RUN/output +OUTPUT_DIR = $SP_RUN/output + +[JOB] +SMP_BATCH_SIZE = 1 +TIMEOUT = 96:00:00 + +[MASK_EXT_RUNNER] + +# Input directory, tile image +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input file pattern(s): only the tile image is needed (WCS + pixel grid) +FILE_PATTERN = CFIS_image + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Path of the external healsparse mask file (band-agnostic; r-band for shear) +MASK_PATH = $SP_CONFIG/mask_r.hsp + +# Healsparse bit value -> output flag value mapping +BIT_FLAG_MAP = 64:1 + +# Flag value for pixels outside the healsparse footprint (0 = unflagged) +OFF_MAP_FLAG = 0 + +# No external instrument flag for tiles +USE_EXT_FLAG = False + +# File name prefix for the output flag files +PREFIX = pipeline diff --git a/src/shapepipe/modules/mask_ext_package/__init__.py b/src/shapepipe/modules/mask_ext_package/__init__.py new file mode 100644 index 000000000..4e2123986 --- /dev/null +++ b/src/shapepipe/modules/mask_ext_package/__init__.py @@ -0,0 +1,71 @@ +"""MASK EXT MODULE. + +This package contains the module for ``mask_ext``. + +:Author: Cail Daley + +:Parent module: ``split_exp_runner`` (exposures) or ``get_images_runner`` / + ``uncompress_fits_runner`` (tiles), or None + +:Input: Single image (tile or single-exposure single-CCD) and, optionally, an + external instrument flag file + +:Output: Per-image pixel flag file + +Description +=========== + +This module produces the ShapePipe per-image pixel flag file +``_flag.fits`` by rasterizing an **external healsparse mask** onto +the image pixel grid, rather than generating masks internally (the job of the +``mask`` module). It is the ShapePipe consumer of the unified UNIONS healsparse +mask products (PhotoPipe footprint + bright stars, MaxiMask, manual galaxy +masks), merged and stored as ``healsparse.HealSparseMap`` files. + +For each image the module: + +1. Reads the image header into an ``astropy.wcs.WCS``. +2. Evaluates the pixel grid to (RA, Dec) in row chunks (bounded memory). +3. Queries ``HealSparseMap.get_values_pos`` at every pixel centre. +4. Maps healsparse bit values to ShapePipe flag values through the + config-driven ``BIT_FLAG_MAP``, producing an ``int16`` flag image. +5. Optionally sums in an external instrument flag image (``USE_EXT_FLAG``), + matching the combination semantics of the ``mask`` module. +6. Writes ``_flag.fits`` carrying the image WCS in its header. + +Everything downstream (SExtractor ``IMAFLAGS_ISO``, setools star selection, +vignetmaker, ngmix) consumes this artifact unchanged; the module is a drop-in +replacement for ``mask`` at the flag-image contract. + +The same module serves tiles and exposure CCDs: only the input image (hence +WCS) differs. The healsparse file, its bit meanings, and all flag values live +**only in config** — the mask products evolve and are swapped at zero code cost. + +Module-specific config file entries +=================================== + +MASK_PATH : str + Path to the healsparse mask file (``.hsp``/``.fits``); band-agnostic +BIT_FLAG_MAP : str + Mapping from healsparse pixel bit value to output flag value, formatted as + ``":, :, ..."`` (e.g. ``"64:1, 2048:2"``). A pixel + carrying several bits receives the bitwise-OR of the mapped flag values. +OFF_MAP_FLAG : int, optional + Flag value assigned to pixels that fall outside the healsparse map + footprint (sentinel pixels); default is ``0``. Off-footprint pixels are + usually unobserved, so a non-zero value flags them. +USE_EXT_FLAG : bool, optional + If ``True``, sum an external instrument flag file (given as the second + input) into the rasterized mask; default is ``False``. Needed for + exposures, where saturation / bleeding / bad columns arrive with the data. +HDU : int, optional + HDU of the external instrument flag FITS file; default is ``0`` +PREFIX : str, optional + Prefix prepended to the output file name base ``flag``; default is ``""`` +CHUNK_SIZE : int, optional + Number of image rows evaluated per chunk; default is chosen from the image + size to bound memory (``1e8`` px for tiles, ``1e7`` px for exposure CCDs) + +""" + +__all__ = ["mask_ext"] diff --git a/src/shapepipe/modules/mask_ext_package/mask_ext.py b/src/shapepipe/modules/mask_ext_package/mask_ext.py new file mode 100644 index 000000000..4edc2a124 --- /dev/null +++ b/src/shapepipe/modules/mask_ext_package/mask_ext.py @@ -0,0 +1,319 @@ +"""MASK EXT. + +This module contains a class to rasterize an external healsparse mask onto an +image pixel grid, producing the ShapePipe per-image pixel flag file. + +:Author: Cail Daley + +""" + +import re + +import numpy as np +from astropy import wcs + +from shapepipe.pipeline import file_io + +# Per-chunk pixel budget: chunk the image in row bands so that no more than +# this many pixel coordinates are held / queried at once. A tile is ~1e8 px and +# an exposure CCD ~1e7 px; this bound keeps peak memory to a single band. +DEFAULT_PIXEL_BUDGET = 10_000_000 + + +class MaskExt(object): + """Mask Ext. + + Rasterize an external healsparse mask onto an image pixel grid and write + the resulting ShapePipe flag file. + + Parameters + ---------- + image_path : str + Path to the image whose pixel grid and WCS define the output + mask_path : str + Path to the external healsparse mask file + bit_flag_map : dict + Mapping from healsparse pixel bit value (int) to output flag value + (int); a pixel carrying several bits gets the bitwise-OR of the mapped + flags + image_num : str + File number string, inserted into the output file name + output_dir : str + Path to the output directory + w_log : logging.Logger + Log file + off_map_flag : int, optional + Flag value for pixels outside the healsparse footprint (sentinel + pixels); default is ``0`` + path_external_flag : str, optional + Path to an external instrument flag file to sum into the mask; default + is ``None`` (not used) + image_prefix : str, optional + Prefix prepended to the output file name base ``flag``; specify + ``'none'`` or ``''`` for no prefix, default is ``''`` + outname_base : str, optional + Output file name base, default is ``flag`` + chunk_size : int, optional + Number of image rows evaluated per chunk; default is derived from the + image width and :data:`DEFAULT_PIXEL_BUDGET` + hdu : int, optional + HDU of the external instrument flag FITS file; default is ``0`` + + """ + + def __init__( + self, + image_path, + mask_path, + bit_flag_map, + image_num, + output_dir, + w_log, + off_map_flag=0, + path_external_flag=None, + image_prefix="", + outname_base="flag", + chunk_size=None, + hdu=0, + ): + self._image_path = image_path + self._mask_path = mask_path + self._bit_flag_map = bit_flag_map + self._img_number = image_num + self._output_dir = output_dir + self._w_log = w_log + self._off_map_flag = int(off_map_flag) + self._path_external_flag = path_external_flag + + if (image_prefix.lower() != "none") and (image_prefix != ""): + self._img_prefix = f"{image_prefix}_" + else: + self._img_prefix = "" + + self._outname_base = outname_base + self._chunk_size = chunk_size + self._hdu = hdu + + self._set_image_coordinates() + + @staticmethod + def parse_bit_flag_map(map_string): + """Parse Bit Flag Map. + + Parse the ``BIT_FLAG_MAP`` config string into a dictionary. + + Parameters + ---------- + map_string : str + Mapping formatted as ``":, :, ..."``, e.g. + ``"64:1, 2048:2"`` + + Returns + ------- + dict + Mapping from healsparse bit value (int) to output flag value (int) + + Raises + ------ + ValueError + If an entry is not of the form ``:`` + + """ + bit_flag_map = {} + for entry in map_string.split(","): + entry = entry.strip() + if not entry: + continue + if not re.fullmatch(r"\d+\s*:\s*\d+", entry): + raise ValueError( + f"Invalid BIT_FLAG_MAP entry '{entry}'; expected " + + "':'" + ) + bit, flag = (int(part) for part in entry.split(":")) + bit_flag_map[bit] = flag + if not bit_flag_map: + raise ValueError("BIT_FLAG_MAP is empty") + return bit_flag_map + + def _set_image_coordinates(self): + """Set Image Coordinates. + + Read the image header into a WCS and record the image shape, mirroring + ``Mask._set_image_coordinates``. + + """ + img = file_io.FITSCatalogue(self._image_path, hdu_no=0) + img.open() + self._header = img.get_header() + # get_data().shape is (n_y, n_x) + self._img_shape = img.get_data().shape + img.close() + del img + + self._wcs = wcs.WCS(self._header) + + def _default_chunk_size(self): + """Default Chunk Size. + + Rows per chunk such that one band holds at most + :data:`DEFAULT_PIXEL_BUDGET` pixels. + + Returns + ------- + int + Number of rows per chunk (at least 1) + + """ + n_x = self._img_shape[1] + return max(1, DEFAULT_PIXEL_BUDGET // n_x) + + def _map_bits_to_flags(self, bit_values, off_map): + """Map Bits to Flags. + + Translate healsparse bit values to output flag values through the + bit→flag mapping, OR-combining every matching bit, and assign the + off-map flag to sentinel pixels. + + Parameters + ---------- + bit_values : numpy.ndarray + Healsparse values queried at the pixel centres + off_map : numpy.ndarray + Boolean mask, ``True`` where the pixel lies outside the footprint + + Returns + ------- + numpy.ndarray + Output flag values (``int16``), same shape as ``bit_values`` + + """ + flags = np.zeros(bit_values.shape, dtype=np.int16) + for bit, flag in self._bit_flag_map.items(): + flags[(bit_values & bit) != 0] |= np.int16(flag) + if self._off_map_flag != 0: + flags[off_map] = np.int16(self._off_map_flag) + return flags + + def rasterize(self): + """Rasterize. + + Evaluate the pixel grid to (RA, Dec) in row chunks, query the + healsparse mask, and build the ``int16`` flag image. + + Returns + ------- + numpy.ndarray + The rasterized flag image, shape ``(n_y, n_x)``, dtype ``int16`` + + """ + # Lazy import: healsparse is an optional heavy dependency, imported at + # use rather than module load. + import healsparse + + hmap = healsparse.HealSparseMap.read(self._mask_path) + sentinel = hmap.sentinel + + n_y, n_x = self._img_shape + chunk_size = self._chunk_size or self._default_chunk_size() + + flag_image = np.zeros((n_y, n_x), dtype=np.int16) + # Column indices are shared across every row band. + x = np.arange(n_x) + + for y0 in range(0, n_y, chunk_size): + y1 = min(y0 + chunk_size, n_y) + yy, xx = np.meshgrid(np.arange(y0, y1), x, indexing="ij") + # WCS uses 0-based pixel coordinates here (origin=0). + ra, dec = self._wcs.all_pix2world(xx.ravel(), yy.ravel(), 0) + # Normalize RA into [0, 360) so the wrap at 0/360 is handled; + # healsparse expects lon in that range with lonlat=True. + ra = np.mod(ra, 360.0) + + bit_values = hmap.get_values_pos(ra, dec, lonlat=True) + off_map = bit_values == sentinel + + band = self._map_bits_to_flags(bit_values, off_map) + flag_image[y0:y1, :] = band.reshape(y1 - y0, n_x) + + return flag_image + + def _combine_external_flag(self, flag_image): + """Combine External Flag. + + Sum an external instrument flag image into the rasterized mask, + matching ``Mask._build_final_mask`` semantics (element-wise sum of the + two integer flag images). + + Parameters + ---------- + flag_image : numpy.ndarray + The rasterized flag image + + Returns + ------- + numpy.ndarray + The combined flag image (``int16``) + + """ + external_flag = file_io.FITSCatalogue( + self._path_external_flag, + hdu_no=self._hdu, + ) + external_flag.open() + ext_flag = external_flag.get_data()[:, :] + external_flag.close() + + return (flag_image + ext_flag).astype(np.int16, copy=False) + + def _output_path(self): + """Output Path. + + Full path of the output flag file, matching the ``mask`` module naming + (``_flag.fits``). + + Returns + ------- + str + Output file path + + """ + name = ( + f"{self._img_prefix}{self._outname_base}" + + f"{self._img_number}.fits" + ) + return f"{self._output_dir}/{name}" + + def make_mask(self): + """Make Mask. + + Rasterize the healsparse mask, optionally combine the external + instrument flag, and write the flag file carrying the image WCS. + + Returns + ------- + str + Path to the written flag file + + """ + flag_image = self.rasterize() + + if self._path_external_flag is not None: + flag_image = self._combine_external_flag(flag_image) + + output_path = self._output_path() + out = file_io.FITSCatalogue( + output_path, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + hdu_no=0, + ) + out.save_as_fits( + data=flag_image, + image=True, + image_header=self._wcs.to_header(), + ) + + self._w_log.info( + f"Wrote healsparse-derived flag file {output_path}" + ) + + return output_path diff --git a/src/shapepipe/modules/mask_ext_runner.py b/src/shapepipe/modules/mask_ext_runner.py new file mode 100644 index 000000000..a43265d1b --- /dev/null +++ b/src/shapepipe/modules/mask_ext_runner.py @@ -0,0 +1,117 @@ +"""MASK EXT RUNNER. + +Module runner for ``mask_ext``. + +:Author: Cail Daley + +""" + +from shapepipe.modules.mask_ext_package.mask_ext import MaskExt +from shapepipe.modules.module_decorator import module_runner + + +@module_runner( + version="1.0", + file_pattern=["image", "flag"], + file_ext=[".fits", ".fits"], + depends=["numpy", "astropy", "healsparse"], + numbering_scheme="_0", +) +def mask_ext_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Mask Ext Runner. + + Rasterize an external healsparse mask onto the input image pixel grid and + write the ShapePipe flag file. + + Notes + ----- + Only the image is strictly required: it supplies the pixel grid and WCS + onto which the healsparse mask is rasterized. Unlike the ``mask`` module, + no weight file is used — the healsparse mask already encodes the footprint, + and missing-data handling is not this module's concern. A second input, an + external instrument flag file, is consumed only when ``USE_EXT_FLAG`` is + ``True`` (needed for exposures, where saturation / bleeding / bad columns + arrive with the data). + + """ + n_inputs = len(input_file_list) + use_ext_flag = config.getboolean(module_config_sec, "USE_EXT_FLAG") if ( + config.has_option(module_config_sec, "USE_EXT_FLAG") + ) else False + + if use_ext_flag: + if n_inputs != 2: + raise ValueError( + f"Found {n_inputs} inputs but USE_EXT_FLAG is True, which " + + 'expects "image" and "flag" in the MASK_EXT_RUNNER section ' + + "of the config file." + ) + image_path = input_file_list[0] + ext_flag_name = input_file_list[1] + else: + if n_inputs != 1: + raise ValueError( + f"Found {n_inputs} inputs but USE_EXT_FLAG is False, which " + + 'expects only "image" in the MASK_EXT_RUNNER section of the ' + + "config file." + ) + image_path = input_file_list[0] + ext_flag_name = None + + # Path to the healsparse mask file + mask_path = config.getexpanded(module_config_sec, "MASK_PATH") + + # Bit -> flag mapping + bit_flag_map = MaskExt.parse_bit_flag_map( + config.get(module_config_sec, "BIT_FLAG_MAP") + ) + + # Flag value for pixels outside the healsparse footprint + if config.has_option(module_config_sec, "OFF_MAP_FLAG"): + off_map_flag = config.getint(module_config_sec, "OFF_MAP_FLAG") + else: + off_map_flag = 0 + + # HDU of the external instrument flag file + if config.has_option(module_config_sec, "HDU"): + hdu = config.getint(module_config_sec, "HDU") + else: + hdu = 0 + + # Output file name prefix + if config.has_option(module_config_sec, "PREFIX"): + prefix = config.get(module_config_sec, "PREFIX") + else: + prefix = "" + + # Rows per chunk (optional; default derived from image size) + if config.has_option(module_config_sec, "CHUNK_SIZE"): + chunk_size = config.getint(module_config_sec, "CHUNK_SIZE") + else: + chunk_size = None + + mask_inst = MaskExt( + image_path, + mask_path, + bit_flag_map, + file_number_string, + run_dirs["output"], + w_log, + off_map_flag=off_map_flag, + path_external_flag=ext_flag_name, + image_prefix=prefix.replace(" ", ""), + outname_base="flag", + chunk_size=chunk_size, + hdu=hdu, + ) + + mask_inst.make_mask() + + return None, None diff --git a/tests/module/test_mask_ext.py b/tests/module/test_mask_ext.py new file mode 100644 index 000000000..f1843f2c3 --- /dev/null +++ b/tests/module/test_mask_ext.py @@ -0,0 +1,230 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MASK_EXT. + +Drives ``MaskExt`` against a synthetic healsparse map and a synthetic TAN WCS +to lock in the rasterization contract of ``mask_ext_runner``: the config-driven +bit->flag mapping (with bitwise-OR of multiple bits), chunk-seam independence, +RA wrap across 0/360, off-footprint (sentinel) handling, external-flag summing, +``int16`` output dtype, and WCS round-trip in the written FITS. + +The synthetic map is a high-resolution healsparse map covering only a tiny +patch on the sky; the synthetic WCS points an image at that patch so a subset +of pixels land on masked healpix cells and the rest fall off the footprint. +""" + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits +from astropy.wcs import WCS + +healsparse = pytest.importorskip("healsparse") +hpgeom = pytest.importorskip("hpgeom") + +from shapepipe.modules.mask_ext_package.mask_ext import MaskExt + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +# Sky patch the synthetic image and mask share. +CRVAL1 = 150.0 +CRVAL2 = 2.3 +NSIDE_SPARSE = 131072 # ~1.6 arcsec, matching the real UNIONS masks +NSIDE_COVERAGE = 32 +PIXSCALE_DEG = 0.187 / 3600.0 # UNIONS ~0.187"/px + + +def _make_wcs(naxis1, naxis2, crval1=CRVAL1, crval2=CRVAL2): + """Synthetic TAN WCS header centred on the shared patch.""" + w = WCS(naxis=2) + w.wcs.ctype = ["RA---TAN", "DEC--TAN"] + w.wcs.crval = [crval1, crval2] + w.wcs.crpix = [naxis1 / 2 + 0.5, naxis2 / 2 + 0.5] + w.wcs.cd = [[-PIXSCALE_DEG, 0.0], [0.0, PIXSCALE_DEG]] + return w + + +def _write_image(path, wcs_obj, naxis1, naxis2): + """Write a zero-valued image carrying the given WCS (defines pixel grid).""" + data = np.zeros((naxis2, naxis1), dtype=np.float32) + hdu = fits.PrimaryHDU(data) + # NAXIS* are set from the data shape; merge only the WCS keywords. + hdu.header.update(wcs_obj.to_header()) + hdu.writeto(path, overwrite=True) + + +def _make_map(masked_ra, masked_dec, bits, sentinel=0): + """Healsparse int32 map with the given (ra, dec) cells set to ``bits``.""" + hmap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.int32, sentinel=sentinel + ) + pix = hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec) + bits = np.asarray(bits, dtype=np.int32) + # Several fine image pixels can share one healpix cell; keep the first bit + # value per unique cell (replace requires unique pixels). + pix, first = np.unique(pix, return_index=True) + hmap[pix] = bits[first] + return hmap + + +def _run(tmp_path, image_wcs, naxis1, naxis2, hmap, bit_flag_map, + off_map_flag=0, chunk_size=None, path_external_flag=None): + """Instantiate MaskExt on written synthetic inputs and rasterize/write.""" + image_path = str(tmp_path / "image.fits") + mask_path = str(tmp_path / "mask.hsp") + _write_image(image_path, image_wcs, naxis1, naxis2) + hmap.write(mask_path, clobber=True) + + inst = MaskExt( + image_path, + mask_path, + bit_flag_map, + image_num="-000-000", + output_dir=str(tmp_path), + w_log=_NullLogger(), + off_map_flag=off_map_flag, + path_external_flag=path_external_flag, + image_prefix="pipeline", + chunk_size=chunk_size, + ) + return inst + + +def test_parse_bit_flag_map(): + """String config parses into an int->int dict; malformed entries raise.""" + assert MaskExt.parse_bit_flag_map("64:1, 2048:2") == {64: 1, 2048: 2} + assert MaskExt.parse_bit_flag_map(" 64 : 1 ") == {64: 1} + with pytest.raises(ValueError): + MaskExt.parse_bit_flag_map("64") + with pytest.raises(ValueError): + MaskExt.parse_bit_flag_map("") + + +def test_bit_flag_mapping_and_off_map(tmp_path): + """Masked cells map their bit; unmasked (off-footprint) get off_map_flag. + + The image centre pixel lands on a cell carrying bit 64 -> flag 1; the map + covers only that centre cell, so every other pixel is off-footprint. + """ + naxis1 = naxis2 = 16 + w = _make_wcs(naxis1, naxis2) + # RA/Dec at the central pixel (0-based grid, origin=0). + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, off_map_flag=8) + flags = inst.rasterize() + + assert flags.dtype == np.int16 + assert flags.shape == (naxis2, naxis1) + # The centre cell is flagged 1; the rest of the image is off the footprint. + n_flagged_1 = np.sum(flags == 1) + assert n_flagged_1 >= 1 + assert np.all(flags[flags != 1] == 8) + assert np.sum(flags == 8) == flags.size - n_flagged_1 + + +def test_multi_bit_or(tmp_path): + """A cell carrying two bits gets the bitwise-OR of the mapped flags.""" + naxis1 = naxis2 = 8 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + # bit values 64 and 2048 set together on the centre cell. + hmap = _make_map([float(ra_c)], [float(dec_c)], [64 | 2048]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1, 2048: 2}) + flags = inst.rasterize() + # 1 | 2 == 3 on the masked cell. + assert 3 in np.unique(flags) + assert np.all(np.isin(np.unique(flags), [0, 3])) + + +def test_chunk_seam_independence(tmp_path): + """Result is independent of chunk size (no seam artifacts).""" + naxis1 = naxis2 = 40 + w = _make_wcs(naxis1, naxis2) + # Mask a band of cells spanning several image rows. + ys = np.arange(0, naxis2) + xs = np.full_like(ys, naxis1 // 2) + ra, dec = w.all_pix2world(xs, ys, 0) + hmap = _make_map(ra.astype(float), dec.astype(float), [64] * len(ys)) + + inst_full = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + full = inst_full.rasterize() + + for cs in (1, 3, 7, 40): + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, chunk_size=cs) + npt.assert_array_equal(inst.rasterize(), full) + # Sanity: some pixels actually got flagged. + assert np.any(full == 1) + + +def test_ra_wrap(tmp_path): + """RA-wrap near 0/360: an image centred on RA~0 rasterizes correctly.""" + naxis1 = naxis2 = 16 + w = _make_wcs(naxis1, naxis2, crval1=0.0, crval2=2.3) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + # Straddling pixels produce RA both just below 360 and just above 0; the + # map cell is queried by its wrapped-into-[0,360) coordinate. + hmap = _make_map([float(np.mod(ra_c, 360.0))], [float(dec_c)], [64]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + flags = inst.rasterize() + # No crash, correct dtype, and the centre is flagged despite the wrap. + assert flags.dtype == np.int16 + assert np.any(flags == 1) + + +def test_external_flag_summing(tmp_path): + """External instrument flag image is summed into the rasterized mask.""" + naxis1 = naxis2 = 8 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) + + # External flag: a constant field of 16 (e.g. a saturation bit). + ext_path = str(tmp_path / "ext_flag.fits") + ext_data = np.full((naxis2, naxis1), 16, dtype=np.int16) + fits.PrimaryHDU(ext_data).writeto(ext_path, overwrite=True) + + inst = _run( + tmp_path, w, naxis1, naxis2, hmap, {64: 1}, + path_external_flag=ext_path, + ) + out_path = inst.make_mask() + + with fits.open(out_path) as hdul: + written = hdul[0].data + + # FITS stores big-endian; still a 2-byte signed int (int16). + assert written.dtype.kind == "i" and written.dtype.itemsize == 2 + # Every pixel gets +16 from the external flag; the centre cell also +1. + assert np.all(written >= 16) + assert 17 in np.unique(written) + + +def test_write_wcs_roundtrip(tmp_path): + """Written FITS carries the image WCS; pix2world round-trips.""" + naxis1 = naxis2 = 12 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + out_path = inst.make_mask() + + with fits.open(out_path) as hdul: + assert hdul[0].data.dtype.kind == "i" + assert hdul[0].data.dtype.itemsize == 2 + w_out = WCS(hdul[0].header) + + ra_in, dec_in = w.all_pix2world(cx, cy, 0) + ra_out, dec_out = w_out.all_pix2world(cx, cy, 0) + npt.assert_allclose([ra_in, dec_in], [ra_out, dec_out], rtol=0, atol=1e-9) From 6baa474494a92f74d295078b6016f2e54cd1b205 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 17:46:06 +0200 Subject: [PATCH 072/100] feat(make_cat): per-band MASK_ columns from external healsparse maps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optional MASK_EXT_PATHS (band:path pairs, env-expanded) in the make_cat section: each band's healsparse map is queried at object world positions (XWIN_WORLD/YWIN_WORLD) and added as a MASK_ column — the ShapePipe end of UNIONS-WL/spherex#38. Strict no-op when unset; off-map objects carry the map sentinel (-1 for integer maps). 3 unit tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CfRuCQa2UHo44yp2MZDsJX --- .../modules/make_cat_package/make_cat.py | 62 +++++++++ src/shapepipe/modules/make_cat_runner.py | 10 ++ tests/module/test_make_cat_mask_ext.py | 129 ++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 tests/module/test_make_cat_mask_ext.py diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 24e72693a..d6c38134d 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -209,6 +209,68 @@ def save_sm_data( return n_obj +def parse_mask_ext_paths(paths_str): + """Parse Mask Ext Paths. + + Parse the ``MASK_EXT_PATHS`` config value into a ``band -> path`` mapping. + + Parameters + ---------- + paths_str : str + Comma-separated ``band:path`` pairs, e.g. + ``u:/path/mask_u.hsp, g:/path/mask_g.hsp`` + + Returns + ------- + dict + Mapping from band name to healsparse map path + + """ + band_paths = {} + for pair in paths_str.split(","): + band, path = pair.split(":", 1) + band_paths[band.strip()] = path.strip() + + return band_paths + + +def save_mask_ext_data(final_cat_file, band_paths, w_log): + """Save External Mask Data. + + Query per-band external healsparse masks at each object's world position + and write one ``MASK_`` column per band into the final catalogue. + Object positions are read from the SExtractor windowed world coordinates + (``XWIN_WORLD`` = RA, ``YWIN_WORLD`` = Dec, both in degrees) carried in the + ``RESULTS`` extension. Objects falling outside a map's coverage receive + that map's sentinel value (``healsparse.HealSparseMap.get_values_pos`` + returns the map's sentinel — ``-1`` for integer maps — verbatim), which is + the documented off-map flag. + + Parameters + ---------- + final_cat_file : file_io.FITSCatalogue + Final catalogue + band_paths : dict + Mapping from band name to healsparse map path + w_log : logging.Logger + Logging instance + + """ + import healsparse + + final_cat_file.open() + ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"]) + dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"]) + + for band, path in band_paths.items(): + w_log.info(f"Query external mask for band {band}: {path}") + mask_map = healsparse.HealSparseMap.read(path) + values = mask_map.get_values_pos(ra, dec, lonlat=True) + final_cat_file.add_col(f"MASK_{band}", np.asarray(values)) + + final_cat_file.close() + + class SaveCatalogue: """Save Catalogue. diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 307dc2ffe..176341757 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -137,4 +137,14 @@ def make_cat_runner( if save_psf: err_msg = sc_inst.process("psf", galaxy_psf_path) + # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38): + # add one MASK_ column per band, queried at each object's world + # position. Absent config is a strict no-op. + if config.has_option(module_config_sec, "MASK_EXT_PATHS"): + band_paths = make_cat.parse_mask_ext_paths( + config.getexpanded(module_config_sec, "MASK_EXT_PATHS") + ) + w_log.info("Save external mask data") + make_cat.save_mask_ext_data(final_cat_file, band_paths, w_log) + return None, None diff --git a/tests/module/test_make_cat_mask_ext.py b/tests/module/test_make_cat_mask_ext.py new file mode 100644 index 000000000..91d0f5154 --- /dev/null +++ b/tests/module/test_make_cat_mask_ext.py @@ -0,0 +1,129 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MAKE_CAT external-mask columns. + +Exercises the optional per-band external healsparse mask lookup added to +``make_cat`` (PR #847 §4, the ShapePipe end of UNIONS-WL/spherex#38). A small +synthetic ``final_cat`` FITS carrying known ``XWIN_WORLD`` / ``YWIN_WORLD`` +object positions is queried against synthetic healsparse maps of known value, +locking in: (1) the ``MASK_`` column name and per-object values, (2) the +off-map sentinel (``-1`` for integer maps) written verbatim for objects outside +coverage, (3) multi-band handling, and (4) that absent config leaves the +catalogue untouched. +""" + +import numpy as np +import numpy.testing as npt +import pytest + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.make_cat_package import make_cat +from shapepipe.pipeline import file_io + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Object world positions (RA, Dec in degrees). The last object sits far from +# the map coverage so it exercises the off-map sentinel path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +def _make_map(value, dtype=np.int16, sentinel=-1): + """Build a healsparse map covering the first three RA/Dec positions. + + All covered pixels carry ``value``; everything else reads the sentinel. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=sentinel + ) + smap.update_values_pos( + RA[:3], DEC[:3], np.full(3, value, dtype=dtype), lonlat=True + ) + return smap + + +def _write_final_cat(path): + """Write a synthetic final_cat FITS with a RESULTS ext of known positions.""" + data = np.empty( + len(RA), + dtype=[ + ("NUMBER", "i4"), + ("XWIN_WORLD", "f8"), + ("YWIN_WORLD", "f8"), + ], + ) + data["NUMBER"] = np.arange(len(RA)) + data["XWIN_WORLD"] = RA + data["YWIN_WORLD"] = DEC + + cat = file_io.FITSCatalogue( + str(path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + cat.save_as_fits(data, ext_name="RESULTS") + return cat + + +def test_parse_mask_ext_paths(): + """band:path pairs parse into a stripped mapping, whitespace-tolerant.""" + parsed = make_cat.parse_mask_ext_paths( + "u:/a/mask_u.hsp, g:/b/mask_g.hsp,r:/c/mask_r.hsp" + ) + assert parsed == { + "u": "/a/mask_u.hsp", + "g": "/b/mask_g.hsp", + "r": "/c/mask_r.hsp", + } + + +def test_mask_ext_columns(tmp_path): + """Per-band columns carry the map value on-map and the sentinel off-map.""" + u_map = _make_map(16) + g_map = _make_map(32) + u_path = tmp_path / "mask_u.hsp" + g_path = tmp_path / "mask_g.hsp" + u_map.write(str(u_path)) + g_map.write(str(g_path)) + + cat_path = tmp_path / "final_cat-000.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue( + str(cat_path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + make_cat.save_mask_ext_data( + cat, + {"u": str(u_path), "g": str(g_path)}, + _NullLogger(), + ) + + cat.open() + data = cat.get_data() + # On-map objects (first three) carry the map value; the off-map object + # (last) carries the map's -1 sentinel. + npt.assert_array_equal(data["MASK_u"], [16, 16, 16, -1]) + npt.assert_array_equal(data["MASK_g"], [32, 32, 32, -1]) + # Integer dtype preserved from the map. + assert np.issubdtype(data["MASK_u"].dtype, np.integer) + cat.close() + + +def test_mask_ext_absent_is_noop(tmp_path): + """Not calling the lookup leaves the catalogue columns unchanged.""" + cat_path = tmp_path / "final_cat-001.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue(str(cat_path)) + cat.open() + cols = set(cat.get_data().dtype.names) + cat.close() + + assert cols == {"NUMBER", "XWIN_WORLD", "YWIN_WORLD"} + assert not any(name.startswith("MASK_") for name in cols) From ba20d53961dfb5ee179a8aae2bcf67f99ab06d79 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 18:32:24 +0200 Subject: [PATCH 073/100] feat(mask_ext): fail fast on boolean masks with non-1 bits The real 2025 r-band UNIONS mask (mask_r_nside131072.hsp) is a boolean healsparse map (True = masked), not an integer bit-flag map. With a boolean map, any BIT_FLAG_MAP bit other than 1 silently selects nothing (True & 64 == 0), producing an all-clean flag image. Raise instead, and document the two mask flavours in the example configs. Found by the real-data smoke: rasterizing the candide mask copy onto the CFIS.233.293 tile grid. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_013btLTHCgmiiggZmxJ4hM3n --- example/cfis/config_exp_MaExt.ini | 4 +- example/cfis/config_tile_MaExt.ini | 4 +- .../modules/mask_ext_package/mask_ext.py | 14 ++++++ tests/module/test_mask_ext.py | 44 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/example/cfis/config_exp_MaExt.ini b/example/cfis/config_exp_MaExt.ini index ecd416e13..ccb527dd1 100644 --- a/example/cfis/config_exp_MaExt.ini +++ b/example/cfis/config_exp_MaExt.ini @@ -34,7 +34,9 @@ NUMBERING_SCHEME = -0000000-0 # Path of the external healsparse mask file (band-agnostic; r-band for shear) MASK_PATH = $SP_CONFIG/mask_r.hsp -# Healsparse bit value -> output flag value mapping +# Healsparse bit value -> output flag value mapping. +# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks +# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. BIT_FLAG_MAP = 64:1 # Flag value for pixels outside the healsparse footprint (0 = unflagged) diff --git a/example/cfis/config_tile_MaExt.ini b/example/cfis/config_tile_MaExt.ini index 410a6890d..fce00da1d 100644 --- a/example/cfis/config_tile_MaExt.ini +++ b/example/cfis/config_tile_MaExt.ini @@ -38,7 +38,9 @@ FILE_EXT = .fits # Path of the external healsparse mask file (band-agnostic; r-band for shear) MASK_PATH = $SP_CONFIG/mask_r.hsp -# Healsparse bit value -> output flag value mapping +# Healsparse bit value -> output flag value mapping. +# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks +# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. BIT_FLAG_MAP = 64:1 # Flag value for pixels outside the healsparse footprint (0 = unflagged) diff --git a/src/shapepipe/modules/mask_ext_package/mask_ext.py b/src/shapepipe/modules/mask_ext_package/mask_ext.py index 4edc2a124..c24aaabcd 100644 --- a/src/shapepipe/modules/mask_ext_package/mask_ext.py +++ b/src/shapepipe/modules/mask_ext_package/mask_ext.py @@ -213,6 +213,20 @@ def rasterize(self): hmap = healsparse.HealSparseMap.read(self._mask_path) sentinel = hmap.sentinel + # Mask products come in two flavours: integer bit-flag maps (per-band + # bits, e.g. 64 = r) and boolean maps (True = masked, e.g. the + # candide copy of the 2025 r-band mask). For a boolean map the only + # meaningful bit is 1 (True); any other bit silently selects nothing + # (``True & 64 == 0`` -> an all-clean flag image), so fail loudly. + if hmap.dtype == np.bool_: + bad_bits = [bit for bit in self._bit_flag_map if bit != 1] + if bad_bits: + raise ValueError( + f"Mask {self._mask_path} is a boolean healsparse map; " + + f"BIT_FLAG_MAP bits {bad_bits} would never match " + + "(use '1:' for boolean masks)" + ) + n_y, n_x = self._img_shape chunk_size = self._chunk_size or self._default_chunk_size() diff --git a/tests/module/test_mask_ext.py b/tests/module/test_mask_ext.py index f1843f2c3..6958c9e02 100644 --- a/tests/module/test_mask_ext.py +++ b/tests/module/test_mask_ext.py @@ -228,3 +228,47 @@ def test_write_wcs_roundtrip(tmp_path): ra_in, dec_in = w.all_pix2world(cx, cy, 0) ra_out, dec_out = w_out.all_pix2world(cx, cy, 0) npt.assert_allclose([ra_in, dec_in], [ra_out, dec_out], rtol=0, atol=1e-9) + + +def _make_bool_map(masked_ra, masked_dec): + """Healsparse boolean map (True = masked) at the given (ra, dec) cells.""" + hmap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.bool_, sentinel=False + ) + pix = np.unique(hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec)) + hmap[pix] = np.ones(len(pix), dtype=np.bool_) + return hmap + + +def test_bool_map_flag_1(tmp_path): + """A boolean mask (True = masked) rasterizes through BIT_FLAG_MAP 1:1. + + This is the flavour of the real 2025 r-band UNIONS mask + (``mask_r_nside131072.hsp``): dtype bool, sentinel False, valid pixels + only where masked. + """ + naxis1 = naxis2 = 16 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {1: 1}) + flags = inst.rasterize() + + assert flags.dtype == np.int16 + assert np.sum(flags == 1) >= 1 + assert np.all(np.isin(np.unique(flags), [0, 1])) + + +def test_bool_map_wrong_bits_raise(tmp_path): + """Boolean mask + bits other than 1 would silently select nothing: raise.""" + naxis1 = naxis2 = 8 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + with pytest.raises(ValueError, match="boolean healsparse map"): + inst.rasterize() From f554e2b51e417596472ae50091b4121a39bcb730 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 18:29:02 -0400 Subject: [PATCH 074/100] Add MASK_EXT_PATHS example to make_cat config The catalogue-level mask query path (parse_mask_ext_paths, save_mask_ext_data) added in this PR had no shipped example. Document MASK_EXT_PATHS in config_make_cat_psfex.ini, the tile-level MAKE_CAT_RUNNER config, matching the design language used for the rasterizer's BIT_FLAG_MAP in config_tile_MaExt.ini. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KQnUyBXB85PdFF4xAKC6WC --- example/cfis/config_make_cat_psfex.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/example/cfis/config_make_cat_psfex.ini b/example/cfis/config_make_cat_psfex.ini index a7407d990..0690a69b9 100644 --- a/example/cfis/config_make_cat_psfex.ini +++ b/example/cfis/config_make_cat_psfex.ini @@ -75,3 +75,9 @@ SM_STAR_THRESH = 0.003 SM_GAL_THRESH = 0.01 SHAPE_MEASUREMENT_TYPE = ngmix + +# Optional per-band external healsparse mask lookup (band:path pairs, +# comma-separated). All bits of each map are queried at each object's +# (RA, Dec) and written verbatim to a MASK_ column; no filtering +# is applied here. Absent this key, the step is a no-op. +; MASK_EXT_PATHS = r:$SP_CONFIG/mask_r.hsp, u:$SP_CONFIG/mask_u.hsp From eb93838adad500adcfa59872b381b52b94f143de Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:45:26 -0400 Subject: [PATCH 075/100] refactor(mask): delete internal mask generation and its star catalogues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShapePipe stops making masks (#846, part of #845). Gone in one piece: the `mask` module (GSC 2.3 Vizier queries, Messier/NGC region files, the WeightWatcher .ww default and the halo/star .reg templates), the `mask_ext` rasterizer that briefly stood in for it, and every config that ran either — the *.mask / *.mask_simu mask configs, config_{exp,tile}_Ma_onthefly.ini, config_{exp,tile}_MaExt.ini, the defunct MaMa and Sx_exp_* variants, and workflow/config/cfis/config_exp_Ma.ini with its three dangling symlinks. The star-catalogue staging the mask module needed goes with it: the `star_catalogue` / `exp_star_cat` Snakemake halves (workflow/scripts/star_cats.py), scripts/python/create_star_cat.py and collate_star_cat.py, and shapepipe.utilities.vizier. Two utilities become dead in the same stroke and are deleted rather than left orphaned: `focal_plane` (only create_star_cat and star_cats.py called focal_plane_disc) and `utilities.file_io` (write_atomic had exactly those two callers; it is prose-cited from tile.smk, fixed there). Nothing replaces them with code. The sky-fixed masks are healsparse maps and are queried per object — MASK_ columns in make_cat, FLAG_EXT in the new mask_query module — so what used to be a pipeline stage is now a config entry. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_exp_MaExt.ini | 52 - example/cfis/config_exp_Ma_onthefly.ini | 79 - example/cfis/config_onthefly.mask | 86 -- example/cfis/config_save.mask | 86 -- example/cfis/config_tile_MaExt.ini | 53 - example/cfis/config_tile_Ma_onthefly.ini | 82 -- example/cfis/config_tile_onthefly.mask | 89 -- example/cfis/config_tile_save.mask | 89 -- example/cfis/defunct/config_MaMa_onthefly.ini | 105 -- example/cfis/defunct/config_MaMa_save.ini | 109 -- .../cfis/defunct/config_tile_Sx_exp_mccd.ini | 274 ---- .../cfis/defunct/config_tile_Sx_exp_psfex.ini | 248 ---- .../mask_default/MEGAPRIME_star_i_13.8.reg | 24 - example/cfis/mask_default/Messier_catalog.npy | Bin 4209 -> 0 bytes .../mask_default/Messier_catalog_updated.fits | Bin 8640 -> 0 bytes example/cfis/mask_default/default.ww | 40 - example/cfis/mask_default/halo_mask.reg | 50 - example/cfis/mask_default/ngc_cat.fits | Bin 201600 -> 0 bytes .../config_exp_Ma_onthefly.ini | 76 - .../cfis_image_sims/config_onthefly.mask_simu | 86 -- .../config_tile_Ma_onthefly.ini | 82 -- .../config_tile_onthefly.mask_simu | 89 -- scripts/python/collate_star_cat.py | 895 ------------ scripts/python/create_star_cat.py | 104 -- .../modules/mask_ext_package/__init__.py | 71 - .../modules/mask_ext_package/mask_ext.py | 333 ----- src/shapepipe/modules/mask_ext_runner.py | 117 -- .../modules/mask_package/__init__.py | 185 --- src/shapepipe/modules/mask_package/mask.py | 1271 ----------------- src/shapepipe/modules/mask_runner.py | 120 -- src/shapepipe/utilities/file_io.py | 45 - src/shapepipe/utilities/focal_plane.py | 92 -- src/shapepipe/utilities/vizier.py | 101 -- tests/module/test_collate_star_cat.py | 70 - tests/module/test_mask_ext.py | 274 ---- workflow/config/cfis/config_exp_Ma.ini | 86 -- workflow/config/cfis/config_onthefly.mask | 1 - .../config/cfis/config_tile_onthefly.mask | 1 - workflow/config/cfis/mask_default | 1 - workflow/scripts/star_cats.py | 301 ---- 40 files changed, 5867 deletions(-) delete mode 100644 example/cfis/config_exp_MaExt.ini delete mode 100644 example/cfis/config_exp_Ma_onthefly.ini delete mode 100644 example/cfis/config_onthefly.mask delete mode 100644 example/cfis/config_save.mask delete mode 100644 example/cfis/config_tile_MaExt.ini delete mode 100644 example/cfis/config_tile_Ma_onthefly.ini delete mode 100644 example/cfis/config_tile_onthefly.mask delete mode 100644 example/cfis/config_tile_save.mask delete mode 100644 example/cfis/defunct/config_MaMa_onthefly.ini delete mode 100644 example/cfis/defunct/config_MaMa_save.ini delete mode 100644 example/cfis/defunct/config_tile_Sx_exp_mccd.ini delete mode 100644 example/cfis/defunct/config_tile_Sx_exp_psfex.ini delete mode 100644 example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg delete mode 100644 example/cfis/mask_default/Messier_catalog.npy delete mode 100644 example/cfis/mask_default/Messier_catalog_updated.fits delete mode 100644 example/cfis/mask_default/default.ww delete mode 100644 example/cfis/mask_default/halo_mask.reg delete mode 100644 example/cfis/mask_default/ngc_cat.fits delete mode 100644 example/cfis_image_sims/config_exp_Ma_onthefly.ini delete mode 100644 example/cfis_image_sims/config_onthefly.mask_simu delete mode 100644 example/cfis_image_sims/config_tile_Ma_onthefly.ini delete mode 100644 example/cfis_image_sims/config_tile_onthefly.mask_simu delete mode 100755 scripts/python/collate_star_cat.py delete mode 100755 scripts/python/create_star_cat.py delete mode 100644 src/shapepipe/modules/mask_ext_package/__init__.py delete mode 100644 src/shapepipe/modules/mask_ext_package/mask_ext.py delete mode 100644 src/shapepipe/modules/mask_ext_runner.py delete mode 100644 src/shapepipe/modules/mask_package/__init__.py delete mode 100644 src/shapepipe/modules/mask_package/mask.py delete mode 100644 src/shapepipe/modules/mask_runner.py delete mode 100644 src/shapepipe/utilities/file_io.py delete mode 100644 src/shapepipe/utilities/focal_plane.py delete mode 100644 src/shapepipe/utilities/vizier.py delete mode 100644 tests/module/test_collate_star_cat.py delete mode 100644 tests/module/test_mask_ext.py delete mode 100644 workflow/config/cfis/config_exp_Ma.ini delete mode 120000 workflow/config/cfis/config_onthefly.mask delete mode 120000 workflow/config/cfis/config_tile_onthefly.mask delete mode 120000 workflow/config/cfis/mask_default delete mode 100644 workflow/scripts/star_cats.py diff --git a/example/cfis/config_exp_MaExt.ini b/example/cfis/config_exp_MaExt.ini deleted file mode 100644 index ccb527dd1..000000000 --- a/example/cfis/config_exp_MaExt.ini +++ /dev/null @@ -1,52 +0,0 @@ -## ShapePipe configuration file for exposure-CCD external-mask (healsparse) -## rasterization. Reprojects the unified UNIONS healsparse mask through each -## single-exposure single-CCD WCS and sums in the instrument flag file, -## producing the pipeline_flag.fits artifact consumed downstream. - -[DEFAULT] -VERBOSE = True -RUN_NAME = run_sp_exp_MaExt -RUN_DATETIME = False - -[EXECUTION] -MODULE = mask_ext_runner -MODE = SMP - -[FILE] -LOG_NAME = log_sp_exp -RUN_LOG_NAME = log_run_sp -INPUT_DIR = $SP_RUN/output -OUTPUT_DIR = $SP_RUN/output - -[JOB] -SMP_BATCH_SIZE = 1 -TIMEOUT = 96:00:00 - -[MASK_EXT_RUNNER] - -# Parent module: single-exposure single-CCD images and instrument flags -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of the external healsparse mask file (band-agnostic; r-band for shear) -MASK_PATH = $SP_CONFIG/mask_r.hsp - -# Healsparse bit value -> output flag value mapping. -# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks -# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. -BIT_FLAG_MAP = 64:1 - -# Flag value for pixels outside the healsparse footprint (0 = unflagged) -OFF_MAP_FLAG = 0 - -# External instrument flag file: summed into the rasterized mask -USE_EXT_FLAG = True - -# HDU of the external instrument flag FITS file (optional, default 0) -HDU = 0 - -# File name prefix for the output flag files -PREFIX = pipeline diff --git a/example/cfis/config_exp_Ma_onthefly.ini b/example/cfis/config_exp_Ma_onthefly.ini deleted file mode 100644 index 71cd45b54..000000000 --- a/example/cfis/config_exp_Ma_onthefly.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -# Path to check for existing output mask files -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/example/cfis/config_onthefly.mask b/example/cfis/config_onthefly.mask deleted file mode 100644 index 7c185c602..000000000 --- a/example/cfis/config_onthefly.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_save.mask b/example/cfis/config_save.mask deleted file mode 100644 index 497dedda9..000000000 --- a/example/cfis/config_save.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_tile_MaExt.ini b/example/cfis/config_tile_MaExt.ini deleted file mode 100644 index fce00da1d..000000000 --- a/example/cfis/config_tile_MaExt.ini +++ /dev/null @@ -1,53 +0,0 @@ -## ShapePipe configuration file for tile external-mask (healsparse) rasterization -## Rasterizes the unified UNIONS healsparse mask onto each tile pixel grid, -## producing the pipeline_flag.fits artifact consumed downstream. - -[DEFAULT] -VERBOSE = True -RUN_NAME = run_sp_tile_MaExt -RUN_DATETIME = False - -[EXECUTION] -MODULE = mask_ext_runner -MODE = SMP - -[FILE] -LOG_NAME = log_sp_exp -RUN_LOG_NAME = log_run_sp -INPUT_DIR = $SP_RUN/output -OUTPUT_DIR = $SP_RUN/output - -[JOB] -SMP_BATCH_SIZE = 1 -TIMEOUT = 96:00:00 - -[MASK_EXT_RUNNER] - -# Input directory, tile image -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s): only the tile image is needed (WCS + pixel grid) -FILE_PATTERN = CFIS_image - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits - -# Path of the external healsparse mask file (band-agnostic; r-band for shear) -MASK_PATH = $SP_CONFIG/mask_r.hsp - -# Healsparse bit value -> output flag value mapping. -# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks -# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. -BIT_FLAG_MAP = 64:1 - -# Flag value for pixels outside the healsparse footprint (0 = unflagged) -OFF_MAP_FLAG = 0 - -# No external instrument flag for tiles -USE_EXT_FLAG = False - -# File name prefix for the output flag files -PREFIX = pipeline diff --git a/example/cfis/config_tile_Ma_onthefly.ini b/example/cfis/config_tile_Ma_onthefly.ini deleted file mode 100644 index 8f7ef4206..000000000 --- a/example/cfis/config_tile_Ma_onthefly.ini +++ /dev/null @@ -1,82 +0,0 @@ -# ShapePipe configuration file for masking of tiles - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/config_tile_onthefly.mask b/example/cfis/config_tile_onthefly.mask deleted file mode 100644 index 69ad20769..000000000 --- a/example/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/config_tile_save.mask b/example/cfis/config_tile_save.mask deleted file mode 100644 index 82c4b66af..000000000 --- a/example/cfis/config_tile_save.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/defunct/config_MaMa_onthefly.ini b/example/cfis/defunct/config_MaMa_onthefly.ini deleted file mode 100644 index 84f117e65..000000000 --- a/example/cfis/defunct/config_MaMa_onthefly.ini +++ /dev/null @@ -1,105 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_MaMa_save.ini b/example/cfis/defunct/config_MaMa_save.ini deleted file mode 100644 index 4bd1b00ef..000000000 --- a/example/cfis/defunct/config_MaMa_save.ini +++ /dev/null @@ -1,109 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, star_cat_tiles - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight, star_cat - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner, star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini b/example/cfis/defunct/config_tile_Sx_exp_mccd.ini deleted file mode 100644 index fec79f177..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini +++ /dev/null @@ -1,274 +0,0 @@ -# ShapePipe configuration file for single-exposures, MCCD PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -## Detection on tile -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -## Detection on single exposures -[SEXTRACTOR_RUNNER_RUN_2] - -INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2 - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner_run_2 - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) if True -DETECTION_WEIGHT = False - -# Se to True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[MCCD_PREPROCESSING_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -INPUT_DIR = last:setools_runner - -# Input are individual CCDs, thus single-exposure single-HDU images -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - - -[MCCD_FIT_VAL_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -NUMBERING_SCHEME = -0000000 - - -[MERGE_STARCAT_RUNNER] - -INPUT_DIR = last:mccd_fit_val_runner - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -PSF_MODEL = mccd - -NUMBERING_SCHEME = -0000000 - - -[MCCD_PLOTS_RUNNER] - -# Now MCCD has created a focal-plane PSF model, including all CCDS per images, -# thus single-exposure files -NUMBERING_SCHEME = -0000000 - -PSF = mccd - -PLOT_MEANSHAPES = True - -# X_GRID, Y_GRID: correspond to the number of bins in each direction of each -# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. -X_GRID = 5 -Y_GRID = 10 - -PLOT_HISTOGRAMS = True - -# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape -# before drawing the plots. -REMOVE_OUTLIERS = False - diff --git a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini b/example/cfis/defunct/config_tile_Sx_exp_psfex.ini deleted file mode 100644 index ea86ea048..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini +++ /dev/null @@ -1,248 +0,0 @@ -# ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 40 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -[SEXTRACTOR_RUNNER_RUN_2] - -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -# True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[PSFEX_RUNNER] - -# Use 80% sample for PSF model -FILE_PATTERN = star_split_ratio_80 - -NUMBERING_SCHEME = -0000000-0 - -# Path to executable for the PSF model (optional) -EXEC_PATH = psfex - -# Default psfex configuration file -DOT_PSFEX_FILE = $SP_CONFIG/default.psfex - - -[PSFEX_INTERP_RUNNER] - -# Use 20% sample for PSF validation -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat - -FILE_EXT = .psf, .fits, .cat - -NUMBERING_SCHEME = -0000000-0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = VALIDATION - -# Column names of position parameters -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# If True, measure and store ellipticity of the PSF (using moments) -GET_SHAPES = True - -# Minimum number of stars per CCD for PSF model to be computed -STAR_THRESH = 22 - -# Maximum chi^2 for PSF model to be computed on CCD -CHI2_THRESH = 2 diff --git a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg deleted file mode 100644 index 4e4164aaf..000000000 --- a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg +++ /dev/null @@ -1,24 +0,0 @@ --11.5 68 --6 186.5 -7 188 -10 64.5 -31 55 -50 38.5 -56.5 11.5 -188 8 -192 -4 -59.5 -11.5 -45 -33 -13.5 -64 -5 -154 --6 -155 --11 -64.5 --40 -44.5 --51.5 -30.5 --62.5 -22.5 --68 -9.5 --177 -2 --176 3 --78 12.5 --67.5 14.5 --38.5 50 diff --git a/example/cfis/mask_default/Messier_catalog.npy b/example/cfis/mask_default/Messier_catalog.npy deleted file mode 100644 index ef07eb032b08de4fcf508418524692e27f92ad75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4209 zcmb7{dvKIj6~H$S0z_V+Av_zF=Wf!F0D%A@VJ{fCEf+Q}Ge}K(fi%99nrA zDunWq77DdgDz+6YRTR<6s3U8&j-3Igj&%@bimd}oA2U?zlbKdM_uP9w(m(ve`$y*c zzCFM5y62pGb0i#Dx@=XB?^WN{Ks?dak_puMR|f+dThsACg+H)u`P4vJg+CbRYTc6H z4~^zMo@mjpb;;T0Jrn)RYq*17Tj|%eB$n*RU?!Sg*O7@Qw1q{{wyp$y?@C2G6Z9UO zJ*|SwZ1?}atFSw_CB3P=b8}@&M!PQkRQogQ9WJxR1Qh>397tE&bDwclCSdu1(AH6 z4(J=nqclLjNNy$y{Uf>I4k$$U+;$icOckc`Be|`!t%iZDp)MQ_Fa8fL{-b;tQzLp( z1zK|XR4X~i+6ssD2bS^mr21)@q|-L|e`)L2lj@%utcLWY`f8xj0Yf~cz08Nmysu}I z@{^g-7~8uRih{w2Z!HWBN-@I6ONt@pAt6OgeuG|8G~c*H>$1v0xQBsaDf~%ZQ5pK+ zj9#xu(RJZAt&dbL!Y~#qk)qmYi%L-!8?ToXsYiGI;Jwj}Di2{ei;a+?!V|0AWsRl> zym+Os9#lTUNERCeCHYt zJUw6aL%5fLQZM+Z%ud_-6Iwr&qT_?AH)5Y0Q~eRfvY5YD%-O+3yrf8%k}`2kQ5lS5 zF$!?}vw9tm%8M`a(@7E=S+VZ$>vz-ugb6G*QHqIP%nRl)Kw>5JofNeih%kx8CQH%J ziDe7S6Z!TxZ4=cXgaC_C$oH}-GAe=*UV1+mP=gW5Sgf4nY({85j5bP5FwOf$skxcg z`ew}uH3Z>41}X@U&gz10ej=di1-q4`b^hq(-Jdoa+e?F#{-Xl{Y#R$_Fm@dV%C0vB0w`Q0Pu=9N?r10aa z1fiP6W{_UL)7z_%e`NK-A8(x7p+->k!%PNhq!_BzUb;+rXK_@tft|MwtqK197itW` zEEbzBMTsX?^IN<3T|d4!Y5&KoR4Kw77ON%ms|DMaqS5$4ebivn6}t8M9V&xgsj&!i z8K@&*oT+IkM%@MMhIoJxJ1e4yYTsCB3-YHfhMU}t3w%~Us)i{I@1A>5M z-PR@B7L@6soDj-%^Uc>N)_z{GGL!si)iAfUj; z5chgC? z3g;oLX0bJ-x6I~7j4@k7dKc!~37<-tWQGvdGVllib}C9S%nNnqSzCL_=S?yN!a4@l z6EN0g|6g7*%bn5qnc^)}C2BrGlz|ul+q+-jSP-A>&68b=j-t$X$xCiGfE6@H@}#1%&`}s!}C4|3xRHS%ncEW8jNY^fzTg z8JXpd^5&~XtPx=|i*1pj&>|L+d5Nq!9g?sS0L0-4w@4|Wsp3qZx=0U6~aLVzCyqRm{F~3&hHc>Jfya3>+iC-sHUVhk$^KK6XW4(DKwegl8D|DgmX=&0@1}8gR!+><@H9 z+WAAZ9^qLAjuT*e#Csq>?XndVKW$FmD8h3Le2oCVEeJOk2$!g6{6`KDwc_*QyqTuQPCx0DJ5Z>D8!r%S;}ikXzqWafEL$@B#sK&c5wX z9YclwMt!znzPT4B5ME^9B?1bpOTA8Mbw^hpRnUe7=8}*^_$C9V2=Fdu-YK1(=oR%* zt|gU1IL*MfJ`;oX=kB5NT zfba?f-uzXxoJl@$H4aq@J{Kw+tpw#gl=-AJ5(Csd;(*Y{{>v*ks<&9 diff --git a/example/cfis/mask_default/Messier_catalog_updated.fits b/example/cfis/mask_default/Messier_catalog_updated.fits deleted file mode 100644 index 6a9f00096170565aeb7fab3429c2256ce56f3533..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8640 zcmeHL4NO&K82bxk?Xcg61^5lKgn(d%puWZoviN(qOUV;yoAdgbQ9S23R5C zGJc%mUqCidTB!L;nwUE8!IV-Emzk_l)0xbfKNvc7R`!19<}PlRM&quncFy*U!+YNM z`JV52-?M$^Ch6l6Vxt8?D|on!!gOJNj?I!|&(0GZIf71aOcL@OCY#x0H+#s(5YrnI z^eKJtg(9EHZZ{PQW|PAtI0|zm&n)h!0aEly_^(HOFCm{TKRZLR3ptjD%+_qlmS@ee z<+(|E-9@JuqYY7g@ELn0_VZRD>u0W-raGWHpgN#BpgN#BpgN#B@ZWcUA19Oa@dm9B z$n*vyu-NFpUb~tuWLRw`d!gVkWn@W$RN!zOPJ5+1di$@QpJ)8@^Rsdu5B7Y14xWX4 zS&}W&v4Ee)%@0rF0_OL1DSR_L^MwZY!H4;MT?t}*vH{QW{rFzUwCCjK3b`h`DO++# zcF*4nulyY!*XREJQ20EvxYZhU`smoGB+vRn9^rG1jKwNtnfqonipd7O(vLv1l&P%K zd;Z)D_w!ESQ~KdWJ`L|jpvgWz+iDYpf$N*;MLuI{LbS_2yUFFz!1U?j6XRTb_*Jc3 z1LNcR9`<9t7x=h77oYd^x%!)DEt2Npr~knA%k{bVyr<7KFN(fYZ|HOJc~4)6GS3UG zHswc^f&1s;18>Zq`uoOTE~2Q~q|Mla|%g za-h=JE%N?j_uw*?bnuLd%4718+1H`mV&Nd-x*iGk9UB7p&}C4$PlbH^_CmVeV&Nd- z=89JNK<*p#3k)C#Ou1FXUhvhhA8^?Mtb-}voMsd9Z@}Xk4L!{R2OH5KblZ)dP$=zM!+^*|xtz<#td_j+GQ1-&6ZGat%$AEx3wsX?S2kT*O z(P(-PbvTh(obuFT5Z{>yK3MWT-;4Ig4*6`)85)KG1QE-xCN>W;=W>HMh*)Vk)ja6M z@syJyu$7@FVSU62dK5DdMBK3RxExb`isC~K0*QVOCn9^$ zbp(OdhJ(xyr{#o-Fcz?5&!N=z7J3m$omj$lnD%Km2LvbsqXagOsg}PA-bQ25LIe>V zY_+_lgL<|@I z07LL-iY?$qV<2@4R4q?dbnI3#Dr+G=U=KWr-9NxdBlb3dj*Wz&SaOD!L_8P^mIHNs zdT16GHv7tAi0Nvf^(agkZEeVSNc<=n#-hoUN=9c0WOjG+Cb?dqlKPu$YDEYqwBwk(z^jfXBY2yG-{#>NlWyeBp( z@5@r2p?zSM!|o1OUcz)9@V|8#^oBHrL)mTU;l)flYcOm;Vdr^9f&XuELZ^<7!rdZ> z7d$Tc4;qc|FXe?ZLlMI-okBI`F@A;7=F| BE!zM9 diff --git a/example/cfis/mask_default/default.ww b/example/cfis/mask_default/default.ww deleted file mode 100644 index c2797f904..000000000 --- a/example/cfis/mask_default/default.ww +++ /dev/null @@ -1,40 +0,0 @@ -#--------------------------------- Weights ------------------------------------ - -WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) - -WEIGHT_MIN 0. # Pixel below those thresholds will be flagged -WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged -WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels - -#---------------------------------- Flags ------------------------------------- - -FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) - -FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels -FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs -FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits - -#---------------------------------- Polygons ---------------------------------- - -POLY_NAMES "" # Filename(s) of input DS9 regions -POLY_OUTFLAGS # FLAG values for polygon masks -POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks -POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? - -#---------------------------------- Output ------------------------------------ - -OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename -OUTFLAG_NAME flag.fits # Output FLAG-map filename - -#----------------------------- Miscellaneous --------------------------------- - -GETAREA N # Compute area for flags and weights (Y/N)? -GETAREA_WEIGHT 0.0 # Weight threshold for area computation -GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area -MEMORY_BUFSIZE 256 # Buffer size in lines -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -WRITE_XML N # Write XML file (Y/N)? -XML_NAME ww.xml # Filename for XML output -XSL_URL file:///usr/local/share/weightwatcher/ww.xsl - # Filename for XSL style-sheet -NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/example/cfis/mask_default/halo_mask.reg b/example/cfis/mask_default/halo_mask.reg deleted file mode 100644 index c44f25167..000000000 --- a/example/cfis/mask_default/halo_mask.reg +++ /dev/null @@ -1,50 +0,0 @@ - 274.66813 -1.25966 - 272.54579 32.47406 - 266.21222 65.67579 - 255.76731 97.82190 - 241.37579 128.40544 - 223.26462 156.94408 - 201.71942 182.98775 - 177.07997 206.12573 - 149.73486 225.99312 - 120.11532 242.27660 - 88.68848 254.71937 - 55.94996 263.12519 - 22.41606 267.36151 - -11.38436 267.36151 - -44.91826 263.12519 - -77.65678 254.71937 --109.08362 242.27660 --138.70315 225.99312 --166.04827 206.12573 --190.68772 182.98775 --212.23292 156.94408 --230.34409 128.40544 --244.73561 97.82190 --255.18052 65.67579 --261.51409 32.47406 --263.63643 -1.25966 --261.51409 -34.99339 --255.18052 -68.19511 --244.73561 -100.34123 --230.34409 -130.92476 --212.23292 -159.46341 --190.68772 -185.50708 --166.04827 -208.64506 --138.70315 -228.51245 --109.08362 -244.79593 - -77.65678 -257.23870 - -44.91826 -265.64452 - -11.38436 -269.88084 - 22.41606 -269.88084 - 55.94996 -265.64452 - 88.68848 -257.23870 - 120.11532 -244.79593 - 149.73486 -228.51245 - 177.07997 -208.64506 - 201.71942 -185.50708 - 223.26462 -159.46341 - 241.37579 -130.92476 - 255.76731 -100.34123 - 266.21222 -68.19511 - 272.54579 -34.99339 diff --git a/example/cfis/mask_default/ngc_cat.fits b/example/cfis/mask_default/ngc_cat.fits deleted file mode 100644 index f51546da7fc88ae1ca919a3c13882fe1a7df6768..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 201600 zcmeGF2~?NW_CJoJC^(`d4mctzj+u&vn#6MsJc>gOJW`TaIFXu~17=1VYN#lVC{9`A zfF^3U*7~iL>ptto=A6A> zdpOfR16KEE`t~vd0zw1+=3oDSCjy4PGB*B|31cP(B)k&PwR?XnU}D1HvBL&W81^^i zL6Y73_w7F5Kk(}aeuF1W7(69l*x-c00SQya4gY&CZfb8apu2@1{}x{x@Ebd6%#h&| z0$z!CaUMBl_}GafUl}{m&EzltWe(_X_Kx@u{QCc;;Q#PHnc4ra>_5f+vA{nT_{ReO zSl}NE{9}QCEbxy7{=aVlnoe5X`}7VCXezt+?k{97v*};7dLm%R$gzVbObJLBJY@9n zfZ?wt=*j6{iu|Ad`Ty$tx!wQ&^JkuqxBdJ1bK4N`8$EpNO9>-rK0eGf6AwW7|MZXf zwfh@>ZQHi*(EdO0L;3&ouWxvt=-!ye{}0v+FHLx5(zt+egC`6gGdyAVguh=ebo>9s zukHU~Kj!!MT-<2;ckOQWim?73Ux)vLpAMP$k;6w1`_Ekahe!AB&i-gRZ1_to=dbbS z7W_Z_FXqSo_%HGcCVw;?JYndVkz)e_{`dIW{}=i7kLhb_`%M_EZS=px*R@a7GnyY( z)%@>&$B*#2{4w;uz>n~0e*Zmu+P@P=zCQd#EdBp?`Vv0P@4tsn#|z_&`ETIU{Qi6R zI&eI{JaQ~2RsQ$()BMDLBmVx$ZXxzixrB3$GW6Q(0ez6|7tpwsPev zE9_$Zz2elE+^{4Mk1&5CT)25il=*6ESZeMuO}OIgE6V;pk$!lZtK9R-la(jFLG{Ce zgM&?@R`xVmw#zfjG&lE{X_QTFpo;r{VpIYT4-XH^@R^e$4$l1-Ges54DBmR2@g6(K z08_?cRP44*R{!E(VYY_{OED^5k1^7gTU-U;EThuiC|UO7iAJ^HUxETctHr1ZQx=%Kcc z>e6aXiYz-~7y%<~Eu(H>f;j#D?66GD)H3QnxkmhJSO7Bx5rmtE)MtC1F!n(?CNwx6 z;tccqBy1<3L#$~8U8p0z+%$}G(2~t@Ljv7JmJ`7|UZzeZ5xO&1b%ycChkTEkj*#08u z1Gi0Q8in>3eKv%}$H!}6b9P-&&wXxi37FF|7NCiRz)1zJ){t5u)GmYsGGQu}SpIc} z+EXEvLmO30W5q_H_DtR9YJ`@t_>XjPZTfC%JWq3p#cMppwecNDq$VsG(^*_wn#fha z)HIeL=&r$0umx2Efq1;Ph{dwS(gFj)v=LVUeA7rfGL);JKIvl_OP3rL*Sg+(pRn$*MJnDbjw~neYf%DumDycd@N&)SDq-I(u$P>({Ll>$u(-1_>-aM z64G_!)`wXOI=OASZgW;xbaXUV2`1ledNEDxO4&kfMYost6MV#u?>}Z;n8*9hLb21= zmuwLVf@NeOaCbtijrhQ48d>r2VwZn8cYAr1P!uV4Rj$FVgtp*uJVfk5UkD6Zx%baz zJ96glRDy`zjm=k!9l5ZYyS-$Ej(6nyVQf>NwTx|Gy0iVWVK;Q|U>RGUIO51R_b`Et zydApiG)Hi&NH!l@A( zO!(x~Y4u{Sr`&$|!YCBI8<}p;efo8Vy&gSGyf%(1pUvGVWI1sCEHIs*7+BJ^!xPi^yWs?kwXT2H@Q-yKogI z{OOlyufM#Bbs^}Y%xBxA^d2rOj170uDT(r4dFs#ZHw}VLRfIR9@aB=544n)l%2&Lu zPLEq}OhMa+2`o_#eNdfl+l+~@M=n))giyX`E!AUMPeuywo1((zEY(W<$=y~a!a-E5 zc}{sfJH?2@phe1w3bD$hiy(&F$z`8CnqVixkC|^XXUAn9*lk1qQqlM z4y#=`dq{mEb3}!vUvc2F)&1vh6qk++A;*}glV3rKn0^Af7(b%z3;%6?Mi1~H=*+af=O{HQ zX$d`9*~>RgXeRVxl=9&FRPrFN^Tq2o!cwnyFb z@nM5Hdoxnjw%$|wjcCl2NGz(DqF%=+L+Xxzh^DANI8s%s(}yEjS9#({*hsV}*g_+% z7ZwyY3=N2+;Il-%S$Re<)CWWQEK$Gx1tapCpP9fV>h}#XI--LuK#3$$uD+EK7K`5U zJeb-{fVM-r&U43DeN1^`(a{5g_I`a#zAX)#nusMDPH82B4(to`9x^iE7cd8~e-|5o!87jd<{ax_ zY>LL4Zi$)d&xc)(XmaYbIQr))HZtqdNG^2ZzU$*|$W6YSp*F>SY{bVu}_=hRQ~-uHgymaniOq8qk1M2iPwFW{DQR z`N$^K284(Duad8l`Yo5Alcl5Da|HRXT-8^!7`Hy>5Y{h1Nxm^f%e8@GzgHeR3ihx> ztG*#(f3*SJIYO)@n&%!<2fS~SE;^E1qGQ{SUJ4?0IJ6#HEPmTjm3c7J)=h;Nj`%2% z0vt0(sr{JqXQB#+L(B9wYJcp<-1b?@<7iQ#eLrU&NfZu`f8YF|N8}0agIKx8cU^EI zKW<@o06JG`m*ioOTbIv%0nrle1I8GKR{!Y2VTq0vuZ!ZXGu`13;=f8w!Hf+kbrg0S zbUNpI48Q~o3R+fY(`cr6!Ek7sTK0J@-{5L3IjC!8p z`#;jTMKScJ+h0&$^g1RkAfyVYO6WJy>i4+D+!rz-tf^2T(|Q_RF+*uZVvRa|M46l* zcb8*eJ$EX+q09Ry@#Gr2Efx};blbOtv@*o0l-lG1?S}A1`QpK*Ta1)-3GeK!`eEFM z#dQM`!?K-P*B)b6t|VaHo;V^))opITsKnS3kr`v`Ow9Un!IXLQXm(3g7}NRT#98z> zS}W(b`8BvdWnFr9_O`#rkmLmu`V)^{_4nFE$3xsbOs`(Y>@zi#I~@9=qu)KggFD8a zX_BhSVS4 z*Qj|!VNG}wa|if@BES^=e@IhNYdo0;J1QEJyeLfa0GkYLis(PmRn($4*2IGJmEHMIesT(7)XvvquC6tF#(z=(k#z zSF8;C`kQ0PFFYF;uZ9KL*U=4sjY!TQH%vgke)IVR9->*f3H>hH-=RZ+Z*|a3#MJt+ zdwp3fBw6B7LaR9S*q^6dNW;ZM@c8z_imnD^ijm9DseNc-2sn*&WcD<*FYgm}WiMhn zY4T+Id(2=-IVOybj&>G3e?Y_`)-iE4#jDR%u=A%JWaV-hM}m*j|6UOrnIn1f;7Gdw z10P&Td`vO<)4O&7G9>t|Zu^r96LM~Ux11ftQeIo#$DUo#k2`uMOj(zob8l)h4hNPp zwMHAULoYS7J>Hmf$o^uW7rC;80O&uw)09mJv6h&c;A4MLn853(0wN?W3dsrIv6jWA zQUgjHTkOmXv8glB?e?wYyXq}0iu`piWr^vr&2yfa@hwLoGfjGDuTu%YAXa;S=6jJ! zR^G^s(1To=^ih%Xcc~>v>M{ZFO)$G=g*$9{%70VuAF6igpKhW#TH{6a}MW8N&tYNZiFeF zi)8(@0X*RBB&2gr@0Dc`QYB0+F()#`SoB9ai$yg{q?AjMWyTG3$5;xcaHTP8gC5$q zDR-h|={YA{Vl6QjQ(DhbyWMSW?zvRy^Ftc1_~J<0SxDQ?vzKrW1NAL2KV-5jlhmE6 z=%AZdr;k&m<5@OxDp_p)%|o)x+s!$WVfCS6am89T^02HNw6(+n_@zw3vxHRhNNrv0 zRK8Ww-5M6OE>?@%40J^ym|{8m>nA4-5A7&REb4bzdQ24SyHottWzz+9fgb6oIbb|u?_g)9qafvV zZ)e2)W<1`q=N6aR?TpC#gbbvarv2H|C^NqXG1LZHda$Q)VEsMj0W6kCYn<)WTn zz`!?_SiVt+I}m%r<)~#?Cfr0K(!Qd<3Sdg`&9 z<48l{x5U^KD116Kf~pqeF? zVryW~s+Qb5Y_iAFuQQxM>B_Ck@=M9a24tQH80^|tq+w#TWLdNrhi)97Dc(lzA~lH< zOu8RiwY9V3J*XF3j6L`EAWviV(@VLD0lq2LTw5<@Kx~pr{Z$i#otM`waN~IA**1pN z{U#F%D1zQy<0(CT*STZ;UH?^L%nh9`1yhv3)cpA;j__PtS3wGT^b*}$vKumPt&yd_ ziFU;ps^le#w_=;~1Rly+BBN6)lA!5vtp>F6ISVktb@OOJ|?bgn+ z;cw4!n`3P=dR|vMv73M&wD;F0-!&$O*xdDTiT4_3tNPX7VIJ&;wU|3pi~Ep)p@byFkE)v%m}`E+|(U-O)ig%lEsYu~Qj z66cgdNI@id4;E{Y<#@xDH$Y%r)=vyJwm$c^yUP0BGr}fxooB?LPfjIAZTL1vmf3#A zfMUAJhEu0y*)xG0Fw7%!$bIQmS0~)rV(X7y&w24mt~>pc`C^(Z3tfl@K1*Z{36y0| zDh`-TGU0|kvBnlmN5PPwn<6tONS57S)>Q>1uv)8%IUPW0T{5r7$g)`QLL|XQmEt3^ z%<<3N_So>V5H*qQp%P$fiH*_GvaDAgwJ7>bv2kD@S++qbW(vmOacrZgakZG+t*%n` zu4unexJzuru;!idfJ$hqZ^G!|-L9%z%BF^qqW6|xjnQzFLE66GBUG0CX@R>B+JyOf zEp&b47q-XylMczUCFx@(Y+7DHRL6V_QZ)2Uzg@`r3K4|g>f!bMW_64Sm`akP3P`#4 zv4TGJPE%U!>b&Lqv$u%`=n!BWEoJle^|GA*8SYR(L=~*FYoQ8$pbrFWmDItK*oLE+ zu<>89P9+dlLJI5wi=9h#Ds{~xN0IsU$68_y7+}Y{k|OMKw zu=L6V(o|x)JvusPBla3W&~DgKD@T?>XF$O40eZCz)fh(;;06u-0|c(m>Al=Zqnasp zbV(QOkb|WXFrg)~Zv^K|C|R1Lge7)j_F3Aaki{|~`^hyad9=cCXKtO}vM* zwk&o>?F7dM6L_H14QThg>mn<8JNI;4BKzWTr}watEVdUR-Tiz*&I6;64aD}?b!&~< z9UX0~MDs8R*q#M5q}Qaj959UIkt%zxt(Rrb-(lt0Q9ECYQcHARN~{0jwqpA`yE|=C zgC_!J&xkrXW51Z?9<=sQk_kgnfu4<9Vqe|nvNDFHO3;Oo?(3hZ3?xKSL5L>abRm_ku{(-^8h$Y9K`71mC@MkhL11DIlgz!ven~B?0%%O zF2(etoF#39S|aDbc~U*-WU;L&Ei{mV}aIOD9VRF775(bS=sjW_5UyvHmK2UH0c>b*$mQ|TD@mR!O{1U5hOGPf5d91NKZ9)}DMFty2*IXG4jQ0Xm3 zs!QOIO%vOL&!M2*0od?Vr|zGA+Pesrun*@s)gx2fCj8``y|VJ8*WGE#PgdlpJzz>m@mb<< z$YlE(PRsnTwN69MLN{Ec*R4s#g%R$J^u>tb2s zh|X(6A4~je_BLa7Nd}&!oWOQV5LTW5rJ3dh2g~v}ngCt2qfVT;ul68wMJ1R=o?R$? zOU_)eawlT;y5)=3zYAE--@S8hPBC1p))`fZGBcJ@;tNjfdp5rRtVz;{E zb8mB%0y5BPEM|6m-<;N_TK&Sv6ww!3&mfSIEm3fxjOB7C?Ec#-T+e3$g zDGI;N5Fgw4>h zLEw^KQmMpu*uZ=Wn{5zDpC!J3ElQkf*@ax7Ln?nvXUD4a6lI;rCP0ca69v9S$dt3F& zG%uuy9;A$@Am z;yZg8DT{r?4c{RfSkW55lL?I@!ABXG80G4zf+c>$QLm~9+=on02aHj5j6!Z=tnKM_ z`S$eB>bi9)Lb{{s_JPb4I80F_cFU^M|8yfQiinU^-yG{|gyEtHi;=1r#a8OsmL-1J zzTP;3yyXozicNSLBTzZaq%+}LxWW{tvB6!g?gZ)?=mU>=*Jah!KT?a*%Y>p2j>xKK z^_cBJHZ96MCaXTVhIyb0l)D@!Qjz?Fw%TH6a7wLOx8d%VeSUA0^3?~IbRTpEJ%80R zySS;NnkCMRJg%O>{tFDG-B9%Ve$f+E;58kmKOs+7^->oStGlf;7@lW|KWPR`?zF^D zJ!^?~KJs>jC75Ezd*F%ZjdTQ^Kh00i%^W4`%`w=ONu)e4I<*h0tpPwTUtDx>2lzWxK@+$m30wRI+og=kY; zxEv>|b$FejGq5wiT~M=eP6?jZJ9-qlbZ5jzLCO42}N;0f4 zqa`k$xGc*e5dlYQ92a52GF7gU6zzr!J7SDo^_H=-J28%98;xCFd9H9U#aZk$4>_z4 z95U0hgCm_G2lf##Duohaw==}%0gL<+E-qoaB^dJ{C`J*`U0i}2YT_6Ue$cvH_{iH{ zaN`Zm)4@nM+BC>h6=SM!1KI+dB`(!Kz7T8VT$m6JH^rse39`(;V%f+n_7XBiRo*|& zJwK{h;%xRbXDH4p_-j|frgow7MHE}QmqLl#lT|V{Fo9Czc*G6=e#A;#LdwMV#UAVz z00WMVLYAFAnh72hK^L$3*;_F1;kN|NdQ|L{RSDv=My7kAc?J7oWfycZP@g_iT)~cn z|L|}M7#;jqkm>OTflLUrmMCsJP25IK`XThI6NANaWF=9Bh^Dxj8822;i8at(j1~+iaEnlm{5G;vN+IGxWdvB*RW6c`?$xbBhW~% zZAcaOa5R#{0y==boNOP&egme~TK!*JCW|8Mo5E`pc;JCn*#nygwrh;K^lTgFrRK+} zMbgk_iR&Gs!~>sB>};R~0Sn&qyPl<;A|hJiMs&3J6Ip!lAkn6{4y3PReH)At;A4qj z-`T6G;-qWPF%M>X3jyQNM+Yt} z2BstG95{w!!1#d-4W`4g50*vOmjpsQ;kf-~kn}IVh35`nYKg1ueVo|`mbn}9ZTKZA zY!3%NOS$vdVQ~wg4NY~+zJu=bJkmcvNx;CgLa3^bBWFOWW9+tHqN@KxDIPs~k#ctn zR*9d_-{BqxOigk7OrZMc&9SVUKOwz?ZJ&zHLMAZMdpMTVA3MCzgKT1odoU0P)DeW{ zl7a4ecqrfOX|7gjiTnFn$#y+Lc>rdyH=x{{AN4|$n99-_W7HcsQv+Stx3cu6I%;aE zMlPh`(xXYL_@#0UCO}HKcz`4CDQlLwIwebbV0BWb&X1g7g~lMH1&fP6en<>ied#eO zR19D7>ci$CJz`dgCop|N3DUbBeK;_oce`GZNgI6 z@DIcg{jfx-Jrz_S5{TqEmVzqA@4mhqQb}YFPhI^|B}T?x(va&hZhLrcD^`9bOHn9@Dl-;3Rj_P25BUDX)T_!7 z@dH-#K_GySDa&F*Hv}sjvMHLxQ%4_oQgU zu5y(pMsZZ7oH0h6MXwED+Q^SoDGzJCi(~UJpGN9^cU8`ludQb2Nu+K0pgb{RP%nZ` zU$`v)cxN#J-35tdVBXj~SRh8bk(Nt2t~y}Cnu`)_k8gU|l3(&(zpVVWr`OpR8Dd1opm9R?MdtobK z(ArkKea>m}wmz_&N>VDX-fwS5!W|4L@GR;3>Kgf4wTmJE*$sTAtc>kiuQP$XjASEM zdA5yg3TK~pHRQ@?7OG4%4-m)=-DQ_DJAuWrwpF}^+OYkK zyG2z&BJ+Km$UcNAek;<}H?B}N!(ugFwW5^rrR_K@Q(1}=C<;oBHnIA?*ljAntt0Tz0rOZu zp?J{iDfS#B0T|Borrn;*+vrdbOuOcZ0_cJW(pCHpjglegKOjN7^05WQYHCS04l{y> zPq8|TbF8SZy=NY>P8HcdW zL)#?A@otciZ0E;E7ZSxZapG3IF%4*MaRsype6xXp{?bpj1&X7Eg*%sWWEmpl*ih3MVrxF5~V)8u#DQJMwe(}Rp zvKewpFfy+zWsPp5*8oQ(IE>Es;6KWxCT=`xR2MoYjvgY;><%fL_m{5mh!}3GzQ#iAE1JJod zt<9n0$Hl&^3tPR$TiN!kbKg+F(AQnotQF-{MPkMaz&t3%YQ0=bEo-1JqA;+UpWjmN zB1uik0hlRk;cin3Qa7-L_FV1S-m?7Vn(Qc>Tv;3ElRe?v0_|H%)^2-Qdj9+s$0;+d z-7!b}k~opqbKrv#(T8MdjJKn~lyTG^vr&4UxXe5N-ID%sg~A`_9a2$(dH6p_6B|!F z&p2RHl<=9MS_OSzpgs}7^z#(yIXIH7p2WZ&Asp*0Jzv<%eIYZgxuJp6t`=uD39TjT z^e&c-cDH35?3Y>(g47cj=fF@mu{v12HhR4;hm=1dt$$^stnkJU9NJu^_MXY&<380G z80%8|-85AR-39SLi74s6c$!nUbuph*VH~w~20Qgy58}4Zj;i~0uq@lB8f%+M%GF6o zRMl}WfL3d)t^X$x>aChO13WpHkfosHYr()K?-mzO;l}=yC57awjQ?8$|L0pJ!qBFSJuNWb6MQ= zq7opQvVPz7Mm6MucLIm5@(7O&sj3a;Qz)*(p~2I8#jj6Ru^q8LSHcw9 zww;!Y+g06nTg|OdB zvb1m(m>%#neDqOKjkJDloKYFGVB!JBC^0w7{ssX9hPp2dh|iOCk+-~z{CWVzN5Go# zvTo=;_ARP)mkrv*h{qSb$%BPWJnHtp;0(msFNzOl+MwTM;{ zErUF|Gj#B=WFss_f~v&2JA(iuM7F`V1zI^6X!u>_g_|V^I+~J$^X0hf#IXX;sfK| z&Tgn8pFV@>`3=_;F7V44;_Ql7dA`n$YE%(NG;dFKx9mo5rpfAqmQj4s2P5PBSg9Sq z@F)hmnz9j&oRvB~j-{~rfh|Jq9XKbAfYC7)ICPY%j&TkWwD$vRC#dmQi@|GT6Y$8p zAWI?E!H^6MLOw3D;ADln$~M3K{bJ7W5i6T;1U^VpWpLq=q_8U+4LWVSgYA}TAl#s^ zY}7DPRT^@ib@5>6$*E7T!Cf!Z*T5QOT@Tydc%<<*8W82_WFu^1%)sRh6xXex5$1){ zam|E~vT}_O4x@2{l1c!aC4)*YlmSQ)5>k``p;DX-K*|Iqw6hxr2CAB9>Oe^)pxlAe z>H@|}+Q02eQP`wqEjt|@>_Z++YF1EFP(0Tqwh6YcOBY2l4_FLUc1NqyS8^$6=?j-l zwr*6Du+fexI_R1_z!7gu6j7Xv1Yj>G8}XPsoCo7f65A9z&5d!H#fC0PWFV9{_+ZIl z>MqEPC{cC2n66LUY4w9zWU28;bU)N3XtD3Xsd&!-RrDmU$%{cMH6nsbu*I58J*`ro ztKhEEEFxQ$-{!}jW3f$NU2|{=QsDrlaWu^+G?rqKNCYSczkC(ryn*t)qubn_I!66gpVwL6)~b_+}IHfbEq>z!7!0gR-8F>s8I6a6uP2Pj>o#flse zjlkUr0^nP+B~rgp7;gzW+q7k8Z()^c6bexEh0B&96@-QUl*FQnDVyI*P!iMgO0WlO z+bl9gE&Q2p`Rd?rj{CPS&3d?JVagVnc{!&b1=i9fT4ATT8BXjH(o8bpBZQV{mB+Bo zPC}frE-kaJtK|0jun+j4N}nS+8!&OkZ?$r*PI-#{xDQmKizVA;`x-Wm69c+dt}X7G z+Iyqigw_d(GN|7=GBVAkP1$-xiVVUD_Y!TxW$VShGN_9_^9_5L5>j&31w5dRk)VTs zL%x+ThXE)ZFpqx_EBa#ho=PxMT7dlkOvlQMiNSbFd4J zXc92!1H%8cMC%1J)Uq0Fc-t6NV8X{@?3*w$A!V^`ho>HV2k%Vy>##Is+p?)@9QI8~ z3W%tJL2KNTYsk?$VA?K>weRScD0GQ-Et8Gskm`i$I)d6^Gqwo^VCbQxw8IgaCb;?d zkPy5xLmWBx4X5@BXmZuAZgVvr#~LA0M{?UX>s7qIO{%4||MaeGh1(0zLkC@lj!`lg zcZqFC;lS1d5Y+QxdZBJq#;WaWv6+Yc_9%* z-Vgb`k9-{a`!`^9%>%7r%#i!;0l3r5eAyD~>|WpjGg`7^%5m8oxg4qzOnp#CoQG*X z@eOuEE+Kt#eS})xI+NG?6RAXpf>ml9kivvyBvW?4?b>lTg9fI$MTJahrD}h)f_$q@ z7@;ENd+8Nf>;n<5^Bnw z^4ureOc-0NUdPq|iDl(NaJ}?(lz50VBrQX|j%PNYsgALb6Vr0$;~MmNS8E8Fm#mD8 zF@LFo8>s@j<~v~^e3tCoF-JB<+Jy)JX37wx)2HBwW;GBBsHY3LkfOrA|CoCR5KY;! z;cj&tcOwBzcNd*1gxdeY1)&mPmh6P<$MNn%i>B<Bq#ZYhDiiad5(3o! zcO!RhK2%lK52R?nbiu&`|JlPC7`w7_+smpJ&eYrhQec5DmEzQt&+2mjkxIIBR*~vh z$x#hm;)k@Hp1o&q&H&Xc8H%uMv}CH=4WZqS%i1e)*n~UTKv#=-{gUZ&!&A5`>W_dq z4Mx{b5B1gZg8tsJK=4x_`-IzGC*5^K!F4NUx2hjy3A z=L|3G&?LFqeiyuFFbCZQ8Hly*a{aI!iYXivBW+FDrT+!{Ev!9If~pObVVzdVCU{7r zga=9-Id05hj35VfFQE19@?FoFe&Y?`>*5mer7;Tct6-GnUqQY#;^}ni+4q zc4!#KBPGHnR}ii6oCERTDo_77MzzF&c0#(6;`Hfr4eUl3)$xNa5%zRX<%1^>Vn9GC zs2>%hd@xu%EYbO`2q$@dXNn+gJMn-`ENLQeo8WRW>T4sL z3CK*ms}I&Q^l&^0F%5GV6xSufH>IfYxbHz#{3$-n`=@0PF0w%}id9Q?ZImyYVrR94 zV9JQn_w72pXRy^lLH$KKIn=u#t~V& zNCv&79|XyzDv>yX*JQHI-e*Te%0SsiaVGllR$C-8W~+CfFDPLlO<;aLx5a9QL6SU`%3tR9~D# zpb{X$>gd;3z$euZFbpgbhe^E=$!;FaZrPs?E^s%oZdjbH?lH?ajKKo=K1)U}ub`HF zvS0|Bm)1AIK z`gj8~?TQzd-nzd`P?hs!kI0%8oaC2=leRj>dVCOTC&Pp`8W7N0vS*$BYBOGt$ptV? zcyG5;?XPD9dTvT_s;zyQ+^Bi|S%C1No)%*1II0>;|*u8^ftKh{Sy zA=;9?angP>W^BX*%9*k^)?%B{<1W%}Fl9fyQ}H{_^TRLHBBErJw~$TyC$gj19#K(; z(OuNyNQM;fz_mRe94R7QP|cKm=~1F5*BBpaf+hQ2sG}r?lbz5dlft2Io&8R!#v_?& zDz&J-SeoCNH<+&xbfwO*&z&5{Z)G`d9w=eSn7l-}xbb|R(IW!D@0Yk!^kGE?=0Tl-l^Um3VRPxB*#4C}t4vG-W8i(5z>@vnNLLf* zq;T)R@ew^N!hRR2PC(ZkY}D3`PWLY9JjabEi$zD?QgnR~#dRb{zdzJo`)zBsIx~$v zdR?u)(wSEl;A6>WdnVhTqanjuWJXg)<9rZ3J%Asyfu8+3Sk=REQ%KMTdiMAIsvdH% z&_l<1^xNr9Rh+q`7Im0$AZ2oQ-gOqsz@Ed47@tj^YyblVFT~_W%4 z>~{7GBYiG(yjttdDVg#S9s~Z|tEOS5N%VqpH^}M(GUL_8CoA&Uz$FG$4RI@%df9JGzqW5rYH%IA6$Zj&*syXt!EY@&pt7Ysu%aeo4clss%171M8-% zB{-x9kva;W$J-ryLu`8efqDptMZ2x+bR)5PG#&W)EqfV`6%Y@I1=CZf)mNAXo`){r z143dSr*iIF#`kd4apEy>#(lMHdLxQc@(l<$ny?#FIH1%4GjRHDwKU`jZiFCOav-MG zJ4-AE(O~+0y5sX=DPAKR#55MS!Cphc%ytFtz#)O^UfM+pN$t7Vb}?${`jJK&;uLvY zQx0ksmy?ML4@s!Po*UHKFDDc2mTU^K&^9N{X%nvx*zKhFh{Xi{lnI z(!PWTO56xmTd{`&9J)`A#kTRTnWMPZE+Ab7RlKft+lAZ+S2B*@eC+ha>$o!j!ICeS z4eaH30F;;lhb3dNPgnydW#bVr_)Pi2)bZ-MW7jx7H1y)oT!{*XT(9A54?sd_@d;C@AP!4jLM?!*WeN_mb>Nn*3<{NY^pN& z0tl~kWaupRkt&1HeeT6>6Y7&qEE$)Vs6N00T(AYTIa3bCl6Xl&ebFG7#+Si+CfhS` zIIh4IL64M(D?Xy?Vb==`sUd?0w$ZC%XiS0>5FjNv)oF*TNH<_H@PTrbj@fVNJgzP= zc=di&tz=3IEa0&!K~>u~iH0Nk!sU?qJ?$BAC4MN8g6#RSs&8^q#b%4)c+|Ag-M9fk z5_tT3iAQieqRWum6;v)>)gWzCN&O+6y`9JK#@$Xphd!1ZHZeoZ#72nWGGYACeB%df zq?V|UDl*V{rDSVyC29N8h&t+3%uonAl(XbWY%G_$vzn!_fnNHtP*uiuEkx>OI-*0g zeH4p#)Te5u9EamkDNVQWJctP+njcqHfg#g1+j>;Wk>jd7a^u0eAzO0jD@U9;$X3lakkSAi7HZ`p^rklEsy0ae$Z|h)gVZf!*&Y@ zx~Y#!?4#C|v|ETa<){sJ)q2n0*)QN@$q`8YgtV`>nn)NVyxsIhdE=aeUV zp9vH2%Go{~6}17KtBh}vMOUgM+N>PMQgF++U*8ZmGT@tX^a4NUEi9Tz7p^h^<4biU zBJhK@>4a59syed84@->0bCywPQ7d6}=8=F{uk=`Z?oX2_EGOZp*HM!nMAasoc&V0Z zhhyRoc}$#}(9?{3VFxJPHd^zb^)3U+3%=|%mz9{9#xNO5L&>c<3# zw&cX~@y_GJ!-Ws}pq=EG1oS}z@WU2FP$n^*_)WAkVB)9luJ5$~4^htVJWZG->XvQ{)@wbefwIUBWe?7pRFJ^ zmwfwbz!)bAPuh`zdQrRmzl)vJKGB(py)_iq7JDr|&k0L!!(%1fY01er0g$LC-#WBs z?i(u8)0L5mK|Fv;Bw{}1`I2z^<&CN#j_0(}q7(Yl7_!)uQ{$aD#Az_b+iGO7*FJ6F zBw~xAz=br?qtF?A)sK~9u~V@EzTd*1*FZrOFw>Axw6N*tuiKm@r@!SRPU32p2Rw%_ zTux6uBu=i~#S{3IbcXfKR>#y&$n*lKHu9S|Q1+Ai^0_6GlE#EZ<2?g-jd+0Yj-Rm- z^V5ffH};%!;;;Y010Ud9a``iz<(3J5Ng++cv^XwwABWTpDv`7*R4w>kzaRs&mV6V< z^abpR)3z~I^QN4BBGRbc{#jnlqsre9X5b~?ws@!JJb*zLFs-r1d9i*|ntG8LLHHro ziQH7j-F?nzIL&F_;Ct>rl3157dpe1MyWGbb=M0a=ji7F#F6?1biJ3D3)l_V@k%15a zrkrsuRlR{HL&(SseddJwDiLWHh$Oq3a_02QDiIqYAZUxt#5yVwIoNn01rJNkDCO%6 z%-JJ6;q2oiWyX?)jwo}IZ3?@Z@~xKFbGjfa35)9{_Ev|xIbpa?S%Slovq$8}={wr< z=`fg}yPWyheW!IvK1kzugfOs{6Zq0x?h8@Ek|}d;$tANs;K72$F6s!vwSnB@f&gIm zJ)NQ07=pE^4>4tO%l)e1vdUx}-H?-;p0i)VqM3j}-UtE z>yHx#I#?(eC1)X#xdA*NK^y3;7mqjtOYUeg(^;*Wt5gukruZy5tNJ08S~9-C4G?=c z-f4y_B2Y}1nB8yM!MF7jD!N0NjkEmC@!JA?I|=FRL7l@|lniUkV|H9&*dWhQ9H-D2 zNS7Tle#5y70;a{PY3dt1?MqVFm9tLfsqezw53|ZSl|93JaqB1-_5f4x7<1a*H+wk8 zDcf{bYdOvFxcyWj)|9il2?r^4?i^X{?5_^nv^t>@ zn($2ndo8Z~2f!~-5Xyaf%YF~XaDYK~O37Fh#g;r}n@Xel>|YY>bqFccr#M5EKaSa1 z2wbSHwVmT#!F~@-9lzC)JjZK|y#RwJO6ahBMB6!W9`-yOJAs4}9;tD5O41U8c(6U@ ztiEn1m%P&pK9-!b`>s6)&z<52t;-yoLn)8*4w+Dn?J>8Ek9c>reo_}`O*toKsM8w# z$9Y$Y*?AGp^CO1wDI_m4;and-yZel9Io44Hgg&u&C}0c^7A$ryrcUo7Z;4PKv8J5c z+t2P*c_X`lot^SYwi*Ck@LR3^-1*~eYr-$w1w$V&eT2vHLi=q@SaVUu<`Ty|?eq(E zsEKKd&CMuO%duL6gc7Fj^i<2k_R#D#6xA#_FL;#N5d0}OF;;)xAIDV9l6f+lFtyq! z;}U|OEXE~L{Vo{4eA}8N=sqa5YGF<#%;`wEOtMF6^dVIqz0^tsuCB7bx&0(swuL^Z z6^TtfwlU{t?3|s4O;O5{^XnI@b+7m3oIy2eb18_^nJGg>RdfbWu8*8QsXMxA7D{Ta3jnXcwr;Asi~fKmcaS1?>*2atmYKL(+nfbE@2; zH(8fnC@TJY;` zdp%ylq7q5umxWc*>@xbkSV@V}F;0`h16;y`OBf#ZN?b&tDq5GC+ngqNPa5?#j)jZH z*q-Qd$;hnSqQ(*SRvd026LOVB=3V=%zaE@k)VtXJ0tsfSqH$D8ciLUmZ!>_0Dd#77 zI5RT}36u_)MTJFn8*~P+mTpmtKCkoibS#=tT(_tN-}P}u;{lA5uG!YYe}}&KXtLcD zlRSXZ+Ag{kuO96Gfql!`F6ws92}7GB&oN=~FGcn~Jgi7!*|$qtc-vjk_DL*0Q!Z%| zZHsx=Ne_*5iHxzkVqdHhq!dsev?TJD-393%;(;obT#}HjCSflLztx@6;;o*Jx6-fj zXdLaZC_+g31IG;tx<@RQ*1T2Ni4r$Jv?**wxp{L>U z`BH2x;sTPtQvCRRLSAu_Xza_bOB0j zk7e5nRXgN^NEa=(V5ofuyIut#KoX>U+()&=-;p7v!9+-J7->I?GXm}shceVuyvbC} zB_;i*V6|8-WLyMk*hA}*j=MQ%a|9&QHkX@g2e$mu|v16z50 zp_6c^A&JyRUU|EMlYlGk{;s6`${&s>4@``5(M*}g%G1U6gOa%=N&(oDZPbIcy9_WS zKHcSty(vytQ|r=;EVi;R%c+5}60=vG#$H~rk25duK2M#v#H#1B)GtUM!^paKST%0F z`UOvfu5`7i)n@{YOn4vKY8)V_30Tfj3GnDHSK^QTj75SOKWN{sE4PUF+NT@JA3jmTJUeS-JOm_E8}?3h1wuGvzx=Bb@1%HuLO)OROE1 zsNTdfAs#qjMoX?Q<72;%#9c1LLP1Ne#g|BPS7$%$F*tI3&7fAwHPtyGO4(t0%p~JXmbzq*OJe1KqZXI55SB63$_J)52Jh0m3*Y;6TVNbZmqpm5^=> z&sVow=@+Qr7fWvZ5Pw*rWI=;crp$bAnj=5E?Y8MAY{s6%on{XJ!v`KyH#$9WM*^PL zZGYpOI!@11y;uq}eXmE6^X{&8B$7&)a`TC4cGdJYA^^bffl#n2=L(()y#@k4F#R;z zncebB!+;cgrrcCD#F>p%<3mDLMu@uh?)Uh)WOlJ>`8H=>YMfXE9$Y006V`H`zteJu zz7}$8Q}I*|j+-XwDp7K4M8wlxc)SL`CB`7cd4#URUjQP4O@#NK&vNDlXE3k~!m$<8 zrt~$Q^!G5xQBiUWvgaTCsnhywv8}@!s9E6+U9&AqzMp%{nb-Hthi`_;&6m@iq*Z;n z5pq~=35;=Gz>DOF9&N&{SH`PUtO{=EIEC1mqnz>{g}hdW2`#x5f7+;TEWgP_e7ei6 zb8b1kpX_gVK)HAdx(|jNR$EePa~=l3EV&)edSAr}L~<0wn(~9qp-zvI85=w5gRSG8 z9>~B$f^I-N_6-$bcvJ+%$!ezDQJkjmJSQ_{DIZ3r$oLrxT{AXQ?i`k_zUefHoeeDA z<*voP>cHRv_v~WxYjvFUU34}OAz{e_{dUW@KWxJWCte5{ zJ$*DYg^?_|8*9Qk7;jMm|AQN{&)j$BH3(v)3~U$jg*|SqaaY-Mag|tx=dm7E*@Nbh zi}@Jp(-$T8Cao76kgAB&1XJ$)Dn_is)6D^{u-rX7+IbVtNS;1@3O5k}-~SS7)Y6nK{W5Ucqm7{Gx+cjF)7 zMp^63I}9EEwd6+*au>MSM$l2plKafN&g)He?*OKz{P;nvI@zEHdoGhgYJY1S3BV5C z6ulz>N8y}OAIy_;!GnPvFvh5*80QdhdQx+sS97%#(;nM};~+e@9dleN!Br08l2HL( zi%v%e%eNN}Vm~{+79u!D{Aop`)5veGq!wR^>>4|l)Lt_&hz_V3ARJD#`jf`L%V zk_X?r>?B>PX_RI%h|OCv1ii6-$0Twx?4H<6Hs4A(QM;&?H)J!h;X3;1}j`WL%;8_)C3O5|%gRk>}2- z@3zil-{z9G$9yl#i9-S@<5a@br&x}|GjQOcL+WUFupA#dftj+Ak8Wrny)Z+;59CTf z&uk;ThVwI$IxK&{yTw^E7P2lks1N#LM3g#JcLF^xO!`>zC?+GDaCr+q&=)1YxE7)o zZ_8vH5N*k0WB1Ck1w5hAemOQFP?kk%4Tksh-lypSvs_{6}&4 z=^HF0k{~6H#eUaEokYM8p%+V8QAc{!`H_lKd_Wi6;r#}04|*Y_{3)bzahPLv2OV@E z0BQROXBtk+K{0LQGcEhbte0zXBlI9w9&ge>cEBY88=k{w%G^F_PSU*1%v2LbA9Iqj zlG$?*YsnLRJ!E?ve>v}p*aqJSi0Pbo4}i1^dnqrC3{ojQj{8uVAR`5%rhEOy2#CvL`NEBUp4Z{l;Ux z>`{UL@~Cb=Un9G5ZdqjxOV;INO(A>WXf@db@GW_=dn?&vRvveKY{I-Mv8n@@Cb=T{ z>t{TjW~R<-GxU>-3Z;BiKLiGSOnGvDoQ!;1|J5DD83>=(k&$>L5+aEYq&(hPR>EEZ zF%6~|%f}x+i^NH{c$g4`aQVaRZ8Bq^7hk%XLaw|T5F+39`hcxoK!p5yp0e%N7uhe+ zxVt>rvX4l|g@;zU3q^_aJQ=w|-%ZdsR<083acU2+wO>wVt`QsXzya#(M)+;0r|g28 zI)oHeELm7CMNS|66#EvwvE;WakBbPr2253e6eaS7_^99v1;6f-zcV~kf1K6_5AvQV ze+)Y!=MLHBT1a%0r=Ca?+c#%X&=DU?{)or-=8nx2rF0YPE(`mPai-ulSR8P`rk4CJ z_L!51ds=Xk2KIeetn(W3V}z7>oSqph4n~}0S53A5o|YOMRp_&Rjtc z?S>zErHj2dVMZnJS@QI2w^VIp1PVZ4ZGYHvPI`xpWxv2`mOS-ag!;bo3bMMvOuxIA z;JgtrpY6do9x3r7?l*Wp&u^z|i~Wesz#I3Yk^r5>{x~;OdXJw*47Dyr6%r(#qvKwR z#eV<(Drc_8heiNOtR$r0FDi8AKIO#?D4zWC{jEGFCHvuyW_MW>xKJGWc`gOMZc#

4h9YH;BjCK1aktyi9r? zzQt$C^H#pBd$rgdmOtYKs(xsJeBM`jmHUSF9~8T`SZDG*>9{Lg_Q0t9wl6z zA^Qy*Nt%Y{5|0aS$I1RBmwVU^KaDx<%*C-2s7@mgO4Lmk_e(aYSjr<+F8vlEmw10g z7zjFaX?NI}SLFydpbU1yzY{Kw_my>#=LZHdk|i%rik5Ye%qY--8zq0SGUW4kK?cUr zv3@ZtS>)p~Z6*u^aH#T0w)hrPIMk;x%959QB#0Tm`A{fm-(IPaCkOrBf-MH3DX$Ek zCSSl_BarGUSMbc&ko<=)E}8P`xb<>;$vqtQ%hd=UIpi;YFi2iqNcQ2=1j@o@p~ zurBy2hIQFVt}HH-EeF4H#ErDrD^Cu=+0i5zD40rI!Of)wd!r2C(7F^4xFv_+nky0T zp~M$ia>(_n?l>*RP=J)YJJzq>A1^H&(|U;GYTOvv0PjoygKiC1)7OZDnDdjxSnSm` zodxD~T!N((KXqN!#NrGlq$=p7Upi+LW3Mq6eqmj%-boQhO8zPWN~y68J$I`P#3(vInkocS5YUBK3>W^AE=#=OrEh1~*(e?ey#1m&d3~ z*0%QyQSw_|qPRHL*a;l4Cu!SPUiaxKpKbS-I}+Fa6(kZ%GHBrlwinEpV7zKtzf5*`fw+PX=I&f@ts<0}Jp-Br^LKt+FJEwcfC2POsVGV{hcA28zYX4)Au~#C{)4CwvgLTaF@eUZsl*2A(%P( z>y~ZFo4EKn0+)L6TW!-@7;wJAB@!|+9unAg@E*jW*k^#NV)h_-}eLE%2N2CsS z5439&Uf|KPEqOYMP3L&wDIvCUVi!=Zl&8Z)FQlrjm*bAM*UiWfNI?T3*;i!L<6?QWccHeFt* zIxKcrBek=&)h|p6cFI_$Oahp$^5?cjqJ$j@B&dd%F8|z-D7LTyUnlmponk$UD!|rl zUwll?zgM@0X{HK?y*=Q7_tn$_-PGZeWpA4m?0uCHiU(jY9V!3J-Qcw3>Tih8QGez5 z2`_mjz+mk0*MiaFAhRS0M*Y7gtQD`(Nh^zi1z@|1oN=U&4#FPVgnx@_v8MFVy7>5; z@hNYfXsfSR_W0|Gqn;x7))DHb%fFrr_H1<9i$W^@dgG*L1D&*zg2l4r-w&tlYoFCh zOVP?T7@r`YY0*wQN>`~}H9~w!_9&1lXQj(phE6No0X*7f$=c!t;?sbsb-iV6`DC$` z^wCH#5mdWppIE^#P#H)w{kO?Dv6W376w?ITkB2p1yML`VGLo`vKavn<9@FDO6WV`s z2z(ivk7d{+ALzEq)OVb7tZ|)f`+XZIJ~YRG zU0`!(IhhA0>meLc(gBv~yuky8hp-1W&9WQ5S}Irlx21(Wm4Ya@d9L@Rf9MXy_WNO>XVIr27sJ8!yAYf@DE?<(N1tUkxTRE_Wcf!qTk~jeQ@r?~;07%P zNtC#C0^3Hbl^)v2c7sQY#0RlD|Cmcyzgv95OdQy{$F&=*ND&(ls|be2x?-_`c_ju? zM3k!x_C~Y#SHuU0U%K5eEJExZd!t6GaWo=V)+!EhdeD{#NVgj|J}uV_D80C-VmFRR zkOgOkXwQ|=8tleBi)F#_)k+rwtTE}N3a&TfFNAM}Av&W6u-U$oW=-|iD>2{;qWV!028XcAzX~TXKJd0V}@dGK<;BiZ|wUK+# z{%!-Z8$TuF>gwNgl@eWITZPy_2AK>TMK#-Q_|hCPW#t6DWuuukYFaDPNZ*^SN@%j+NNdht7FHyU$PbYML>53#ulX`=_{iIMa{U{LTOaO8a!ckjWVoP|9` z9Q6kD4YhJ`C0HPky|Qp-vgHR4kBXqi%(T|^oT4L$YX6JCv}s-TSY1LRZG3~DY;)wK zR$t?2e7z9s$PWWo)kIJ(d$(BUUfqEyglj^bDXdC@!8F1qO~%RV?ro-dXrzr#UXa7N zm_u>(1II&?#X9Z*U#!x2QKc9(y+%YKh2Lnq@pob3+)qoi)wTLfZrLiYwpQy;W0q=_ zP2yV0CM3@bbw{vv6HYOv6-O!anby$cWU`$8>v(-90H#!VcaKb4Rjs_HklJ<=&vFQx z)gP)efbjk-nYQtm`Xi-Gd#*HI6WkK;_`ge(#mck?X8D*>=&S zlg78Q{GyRxKIkc8PKf!`j}rTYCzZ=^)HhvDld8j>QFM+T0>1Zs9c*iAZ_B zLS9b>{~!q39{vL=Wv55B`vN!1_HPmC>Bc}nF%4I~^=~@Ub3Iv#;;2IKsuEi~quCeB zfM8106mcD))Iy}eI9~`s{j%(U*2S_V zXV74%oHY`{IoYzZTy(n&%nrX#v|c(9Sr|O!qxLM8mX|utsri zJOiZsAc$(&Do7_pG*tpf(nMAaUnf1rZaQqOyp!{Vh&6txXVZY&i$s6=Acey)y=jqk z2Yb={VX6c?aa1hj)J-|S431^{L}R`U7fS@*xK%#O+7_ytF2}zfbf-`(W#L~b=cg5_5s8V-}@`TD|u&kDtA9;7S74M@qdn|?Q4&ib)NkApl) zO0x%M$$S6)R$C0Tl;9TDhfSB)8K`OOZc*_123-R9l!zT63Q9UFhAz|CYC#U7OQbwF zPqrXC5MoUfHqBZh3RZktS0Zp`g_&Cqc_Z6eW zEi1)pmO2q@da0%}S*UtLw@SsB5+CjpAMwprB`_g84g`yjJV$l^0dxqX&N}Nj74DXx z%0KZk=F4YwX8;jkOJsigVVH9l6HWVE zn9ty7c3*%T`EpKOx7GBuG-ux5Yt=x&0vJ--tQ1=xouvDZJVj_)b;io(Xp;)5e2rtk zO7TmzdHKs!$zRJ2?7HLocC%9_72kUs@@igg{uD z;BWTiEUS_M8hZumR8^K8*lM0k_-wQ4gVZO9Dlf0Ks_3;Ln7|F}5G{jP9o+>XPeBjt z94~|1vyy;=M?WE(bNp6Gp&C5C8RyKTb3{`UQh~&J6k8NC+|jrl$O6Eb-#*X*1ALg) zBTNosPkpuWtqI(~(4CGmX@pKD7No4s%W|^o%k6#YkF*1)R){Ov^HexAu;yPB$raNq z9k^AhmuhA&@jmj6PE>2G9>9az%jUFHlQWlal4AwG=%+XgA#mSz@$->qw3HIXBhXtZ zHmuQk6x|Zu0Fh5Vlnm2-X4%dA@0QI7?e%yLi$Q2`NaVlKPZJLSI zEB+vl5uP_DY~DUbJVtgP^ibX(V>iEHrtE1(C&rb{Ck4wMydhKLi}mKu&yty~LbRhW zLAu?%UyRtu`47d^;E0PAPqFBx2r#BZ-#y}~O|NN!sWK>8JjWWna<)bq)ONqj8@N%g z85ro+K2gNZF4RvEuuq-@^y~QZ(kxd^7s)_+!A-q6dPH_#wQAXmK~Il#zLodQ~^WmiU9G@lh11PVGPoGRR+{LJhP;$ zP>pby8|i&?Qa-1-ogky=Ps;(8q2puA1aDZg-!HCi*g=7FoQGNXk3w6IVj47T zr{`JL=oJp#DFsDkI*ZSlCk;`b5}(eKQ7=r<-v%^>fajtPn)EoD(4tY8Oyhl117r76u+5om~%NZu;GN_bx@XP8!oEL_w(1e)*=66T_V%BpBZY)#Gttgq)gpR9DeBTJX&tT>XOn&S@SkX73e= ztf3&mL}81&YsFx)N1?3|+v1u%p0C{drq~4Ncc)k_-OmZo$3R$VH6cHLhA-AzPB|^- zpR2O?uj+Er?H2cCJE4 zZHtWubUHdD7!$VGo8y$SJFBYTsC2s(+14HzW?stEgqEur2WLK@b1beKf#+r z#)Av^7UCGY7HWec6JlWkOC30f!Si~KcmD*1_Hp>Vx1Zy2p1@( z=;FUY1h`A`2FIyDXGw*5LxxgWb}*YM8*er515pL0vkFBtN1MQ)zPS*X@U~}-9t=h~ zMd-q?7=|X+TRgtVyPmU7${t!ut9QnUZJctALb-A!r4=*2C^kV$e7$9>U(Sfy;@fm5 zjU2_2260n@8W#Ut){sc;3o z_08MGn}ihgO$XbW<*+x4{;hLm>uC|sRs^NG=`LE|zSr47|50|ainCiRGdELQZT;h&E8MIayHsmv&5BcnJ0Q{V(VBIs zrSJf@Yt!0fv2@f+x?|J21as)<_FF&KQ}Zeng{^;_^X@E`!_Y%j0qIXm-o12~ngg|gKnf9R%etp0g`vyO=-|1~(;9LkidF2wn? zO}E>85hho$uTF6zHaIRv3} zl87x-wf|KjsnB|!9CtD95ed7^?~!s9-KTB`eoT69OZf9@e=)s&Q{8`X?@OlDQT|WO~BCjc1EoA>?Kbd*==5}6+1{th51aZ zx4E&>S?g!z(oo z!(&R0jK3|_*Ti7Ti;n$y7J%ZeO^Xl8{-Y+37&zNu#PFOC`OVFMwKw(z_ae72s%KyJeHmjSCfdtLQ0vkN60%I zCrq5D-C)GtcF6NQd*b}pEpcyzx|FA@7$eW$o!aTwHma?pP|CJLHbjaH`qMh3H=S{| zae~N$6iOlGtvP}>F|{kfGSUwHu~t4YZMnYl?lL{(tao!EQGzN-lN~>%k^EOF2;tEL z$0pkve<=0S?a=S`$`#SKiw`(-G(?pMZCWgT;O0^}>X)dH3cXM$AD#H7z!v)WtsU~` zO3zUytEz+%TYkv-nvW;NO(cgL%Jep3V>t>bxe`xwR@$@B!pX^Y( zNIbp#OS_)RD%Eo#7ioZOi2{6VUd(*^XGU~uk`GykjREWha z5K-K;hS1jzicmK z+Q`pta1IvztFP-DT|Te1mT_jHuF9rL=LGkos(eOnKRsF&t!}M*ZNoHok#i->4E*7$ zKXsOK<9tt`!MwYXJBOblI009$*E)YvgUeP?X*tS7Cx}66 zuWOI$SkK4w{E8{REW6!(rSb{#o|oenU8UWMt?~&LL-Ps#JY}G^YtP6AT-||(X>)B~ zDix;~OsEpMDhlfpUMLn-=bLHhNU`m<_b(7JW~8KC0nx0^6FLVmOny-f}hz4omc6k$1Q5Y)GRAJKGf;{jcgtotOL)*PAMJ`h>RU$)6nZ{vb zP9UR)f!^-REpqw2nHEM?aYhxU8b_FW#1W_PNw?cQkS}j~^IbK+HO_APSCJUwUJW%& zzn$m2&U3dxs9&Xd?3wADb#G25XkGq$+B1Y9e!j2EXM%fQ>jRc3pCu@P81lS}Ar7-k?mFQl`~s%zuK` zpv2}WSGJ2v^rUdlvmE8L)!VfSbT;smAR?5ZkuqSV$8xWihHI-Q1$%~bof5^AJ1H@~ zNZe1>Y9uJMF!c|0{$@)r*Efb~*LAeBfrrl!Y246m)gkddsRotB4AYV`vf<$Y>JNn$ zrbXvu!~Z;07clL}nNiEbRvA<&*HWfdcv5*|8IdM%+jlu6@1mnetck+KI3kr%A`3%*|dGb z0MDn~6Ibd34#FcRWl9T^vm64t{By=B5&tOlO_$T2yprEN{;|TLLV_xz13l+>lM4*0 zRi)eQcle3(2PW%Y3dRsPYnkk(2Zj#>HtZVnI%SmaOW2+v%M`MSV?On>?e;fhdYiIa zt(3z^NO?nusSJ!TpK*45=)6w3{{0JjFacw#Y@Y6%Azd;kl}*y^u(lV(Kc~0pO)ij9 zqA=Teh_M3_T;(2G=;R><*rwYGYY{0TNn@P{FkK}qW`kJ8DJ1?dCE6d%cTzcCsX~fs zS$242ptvI75B-VGP=v6z0%YPrV^bq`_H^%3XM~!Us-}*Szf17)!jy&TDjY`IVPy7M z+Hj@rP~ZXdD^EJ7Nt;+ol_1S@>~1G@%!eAf78|x?mNN_-9*~x(a0r{Y#aY6ciCj!* zaEyy|mL$#9eldilhn)a!qbuboMU{tRosn*)c=SP+6De}dT|X-+u02+)bgtm#VxXF~ zANJ84r-YvFA`f1vlB>&p)7NftbTj>Bl(&ctJ3p$_Xr|-O$_~$&3uH#hO=o0>1--Pz zv@YQc;UaE6rU*uC_@D}L73aqNP4fVlh!e}0dI8&5y~Ffux#rc@8V8an(ff!sg3VY& zy2|~W?9`HPD+(N@#M}xQ!6T6Tq2esv4(Gx34$tn<+QJyZeU;#7pzLXyY50SIKz~95 z!bTc26hMxaPDW_ zau-#!lnz{G={EO1-EOs!JA9WQr@5z0JjyQ-vyaF%H@>5nvZ$72cUVm#V@`)v(7+X&LD{;lMD!^N<9J(S-)c5O0y7~ zZr2OCuL0%BMHiy?0`F_;aNk%z(EkpT;b0 zkL*J4t*;zV8@l=;tlgpHu(R?%PwB%g+Ak4ZcFVZ7r*xcZrX9XJ;(Ym3R$a#y!TGTY zUPGd|3I2#d<798Sc-2Zd zjVxy{bgLYL2d95vsECL?K2to)6Ml;!%vUiM!D`@0-YE((M$WQ3^03u$4wGUF*B%dy zb6(}Rhq7s|YGNHzg*Z;y00j&JV(Srh70J0kCnQzyQsRzLvh!L;Yg?juY}+rVoHAD- z4I#ZkEFWR%E8SEZX?Kh|=WM_BJ^dxjQ|fo*&_Dz3Nbq0NUOKMbL@0w6gOYwj%Vh1yS&@5ZnV|$=SXi9pT=^e zPz9#7TfC8EOM>9Wn03?}b?t(>{-e{_LUD-HM-(?7i_T9SkOhx))XoNNmffjSq!`8O z`FfaYq@Bhki=Rlv@C4pBx^T;;-^K=dQsl4ganB-e-`ZK)4Tgt^^Y&*tTZ0KTuufOR zd;8^@i(4>83aLw4^5M&aX|8g1ry=LOEtyv)vHlUILaI|}j<>~Uo3*11kHS)KOQDag zxyqH3B9sjXP@0hH^cVSHIfPXP0j61Y=MHnc&HrhrqS#dF-0h%T#M!TMl+z_TU9rg9 z@)mtX)x{wedIvl=LLY2PP@YpIR7x2o#SpHE_r^c)rv5f)dvtD_EOW-4(*1`~?gJsJ zIMa&+RVCf-OkV4U9z3Q)%9NN*vfHbcY8;pVDS4UR7FXY-1{1FO_aF2IowRiX88;mF z6Q%#%s2g&Iil9!2~X{t2~_JZNq|lp3gv?4(#?ueqfGRg)2vO9vUXT4{ur*{GCo#cq2bA)y~!??7VH3 zNZ?bDWLhOv`tEVAA`uuEl=o3Tez_*_hjIgi{*#@ny4|L~K%m9he{A^{hh4bq4TP_K0 zGC(Bxt=3XJ%IcBr_Q>gswv?k_3b5RG=Ra(7#9=~RqI1Go=NO3=fNf%|^OhLr8~QQ+ ztBC)L$E$P1Hqx;8nb4*LM^>kEvtCItetF~o-v6|&A$R_KtFyxWSWpcUe!tW6K8F|b zFGlk5bDsBE{-M5!0ou{`-$NK-j)jP*$I-^+bP&Va<`WXwHt4Ldvfmn?r z^6Ex%#H9v`3|}ZljrmYRFW2hxCA@_y{Gk*?+o4OuD{Q1@!8A|nLXwpRcbw5-X)Km{ zQmo<>lB!aot4y0I+kSC_CWLFIK?@x9!nRVMU%DO19?n%fK)=g`6jIt`JI97<&$;$! zf57<65RoF@~6B2CB$8&atIZLtn3P zj8AYr;^R4=P6?YJ_Ue4+8!YC*x5hv`FaM2PsUw*x@VFM}Y=SE2s1M=R?andJ(Ym%D zIc>RftZb-)ZTj`dF@D|{26^BxA=SBZgg54#e(l#pa^&PC-tJ`PP-0;W4)p@+zBlw^ z*x4Fs)ft(x*4zm|EG2eVILVXB^#=z@|BLX$Zt)Q(W+-l2ROGd5z0*lq20`R#=wI%C zH-ljh^<6hqR$H5ynukN+2h-Xu!oy)frL8tlm)JtN{@FF!>KbWOvqI-x9@$YmbbLf{ zccl%7Yn3bcVW8g|d0Th;Q}r-L+o(x#vaNN=z$o(j9A_W>sgh!N1Xnv1BoFrkdYQ6E z)ZJUHjm3W|P$nFrmPLs1T+4bLXJ;#I zT#YnCBh5?nhP?ZY?t=^=X|gw%YXr(3`Km-0uI;VjhCj-wA7X#MV0}%VJpM327j-_- z`kLT}ptK!hcL|#*erH-8?(<6&ei_OPA`yOiva_kX`r zuYVX_f=4(Be4;6CtllLu&`G#p&iwRuriNb`~1YkRkiWHuAy?|d}4B?gS z&H?UuDyzdI-R|;UnpnXRycl?Rls%$@4~eTk4@gybE#Cn&XBgM_E)Zirb?FYw^ z?P8L7h}J|43>s%_ExIagBJENLd~AQSS0CCz=Bg6X|k5X*z@GiL=B*o{`GxCO)EfEEnHF zfT8&!M?ZbUxq3;YlA?a;cGvmUvVdca7V4L1d#tW?1lb$-uW5wQcf<>om;}XK9(M}y z5Gxa|6qNgAgUsNnTP{_Mwr2`tMhAUb&a~UEZEEG*ht0it_#ieYP2T31G^561Pv$$z z#y_QdDHx;vg*chv|EQLNUm#o#J$lbkab4j%dWR-RSNSSbHfsG~UGs=$zw%pph8%(} zMl1Jovg}Lx0;ODuCe$pE{W!D;Z1atZb`FUE9{5xQZT0B#h2l7`x#A?#ZAE`xD{f}X zm;Wlu1MJI4k;4myWxn}K^okT`IXCQ(;8uBujSwDRtb*XtRX%4)jqeL3Tq&PUu->X1 zp+CS`D)^&+8zCF5KcNR`#vb*;vU-6iC?BhR>y|hx8oz%*6O6^aohL%wkGL`N8vr~AwOps*X1q(>)It)wweE~I)Z1uKwTTBiD+)T! zVCEc@Q^4U*6~m)Tuyx9Wr0HS0lISUb`Aax{F(s@Bk?*dM?9#weCy1_OkIM6HFLWgx z%w+dfH&Y_JP%Mr6P0#qWJ-X&(ilv+l^w2y&I@0d?;$)dUQFm1~`2K^0A1SAG)jd=`0?wnu_i?xi`-3y=JvhhLOJ zY$a*V-HE2QN4M59MMpx4Ka_7M(JxSRWI+U57+~EdjgVarbkOZZGvzp@_FVR-7 z$KzP6=QoC5$}dK2@d9u1b@6pQe>YEr^BwCjiic*}z2QnRb8dh2MK=ro-8l|)inmJC zcuY&_-ha2q-=!D1jib6}$H`pI($?_^DH1m^K+7cxgS2uNszqC#MZ~x2mu`3OzF$a= zHc>+TAf!Z!o*c7K7Srl?AGTKXy!4z>_laA?H#Ad97<+UdbJUtfLNc;y7!$^PG1__h z{b|}jI4jHU{&A7aVV6fq(du`4@d|$imcDYTvXYwlbdcc~b#a><| zrbB>LNNOw?x>bJJAWl(bv9AGjFCXFTVHXg^RH-aGreUQxo}rIen0~$cuScCXxx|VD zGal>y?@4Fv@HKV7VtO19m8^|}(%2*BRYt*Gr`Lr;%&imTE#W70d&y9qi@CKx-a;Z{ zKU$?G)Y}G&IWWR`g7*na4D@t6W=_j@xYI1KxEj~NV{4cwVn<>2_IAm4ZTON zS~=yXPZS0dLG_t_=;saHIjC-3yT{N3ak#R)?vvDG*gF^%Tdlq z8N5M!$KkkqbTNdEX-+rytdrKZN5_LsH~Jue85})EP8KI8=`}WEakaHb5L(9+wjVZC9Nc48E06+j(J*0cr%5jg~q*u^TkPHLWR0@L~+;I^#}(m#7XuE6;o}op2J7URbF#18pcRDS}Bfjur2>0k9vgi z5#k8Si%m;5c-ygxuYyi9jX7KE6!lr`8-Kv`i?h~SoDflB0Y{ZDFIaEU<0>BE z+NQgTy!6TX<7t4AfUCtPAJ=tTJsT|%$2jDR3DplC^+M{4(cX|eeP!2_=*cYU8^6`s zFJ($>z5U7_Cn~*j$;D{-{L=Hz15&Wxpkf^pAm!fz@0|1Biz45CRL`-&)^sLCpft|z z`RHCL()DX!97;nm(|^R~L|Cid`^r)!v>RgiWG1d#HBz=6ZJ*=gQ& z5&At00~r}qVZss(Y|~2dHWSTU2%x3PS7+p{O_%$CL1;40yN>t8Kx$$vmZWBlhOZH6 zKrdIG>(!!>T)N=6Rt~FCsQ97N=mq{(q)9amGOr%vLQ4& zB36_5TS-AVn6{1(qe++>Os9nRAh6t#eOIv<3*ic-ZLcne#F1U^>s84z1-)0B04wFq zxtgigrI)Of*KRA*YZo(giF#5Z&RSDB_?VCP4G(VnwC1d}G9lp7kuuq7%^`3!b&+=1 z>U3kx0P`6W_Ue`5bYp?23Ie6T5xCN6Ko*G*pBs8{!LWejYG`VLuGf_@)@aV+=MmZ< zMasy@RtCo!m4Wy{SiM!w$T+BPE$ZW9y>2AIEJ-xh!-FbFnM^*&|2(Cm5VZeG{X0U% zb2rY>bJHMQ<(9415RO>6IFRz(WEuI;KUx>Vbmal@97uNool-D;e!qBrbGm8?rvK>m zOiSVA@O8ck2Y5VvR(Nl^OLtp_$BTuseX9w2C~1HdkwJzOT`*Lo((T^OxnIe0y3)l+ zc`I8y%q?FJ;ebsL`$L?(-hHu8SLyXvn9OodK^vz3P%!U2ZpZL-xltB4DVCiwJD>%G;R*~fn%4d(z`>cbqyB{@~NVM z^=7bnEf<7Ti5gvkbJ7tlZmPQ(+dJZ{;2A)DR>4$h6W~mASFYd#(|UwND3GVLb%sY&SKS?|0*Nfc5n6yI=<0Q z#h5Y@CDOaU(;4%Zq025+B`&&DP8a`J ze!g&s3k(pCkc*N3nsA5Iav{-^{$s8@cvv2a;RRkM_lF{*^;R_ z7+h`p9(D#kqPIAW3F8Ls7bRriR_Yrb^$5drMG2tzujx?YQV)pd$%g_`We=d&lX6?I zY{`?T3Ws5O(;ng3_l7o*vB%@1oEA(pOD+N%S?C0q`M%5a;nU6#H}k?S6%KJ`(tI&F zqaC+#o@mEDAq-Otjyo2KkdeQu`sVW*cV~<@_=yj-#i~?K7nh&r454MCt#M`C2NT5W zF*-AtiJ&+(x%yvvek^YMUZ;QG64fE8pKZroh!ZC&?y76Zan(CTCDTa$>++~qg5F^= z0|E4K!{Mj1*3BeoQZ@w(CgcHwvpnuwiaoGemffd&BYADwSNZ~~A+*d*9ZSnDM+#P> zMEk(ht1+Q6a)xq4pJka&dsgxk!B6v8x6kRoWCQ~lV11&NJCUq>s}c~??LM67?!YVV zKsQ$JbNh5Tk0p7f3ngHBUA!E|sc;Wp5B+KEb9<3<6In(SQw_b(n0;dZngtdjO*8E? zXMw!-p&5Ejx< zv@?lK(2|P=;@InB+?-ZCjQZ6Jw2j^RU71Z48(s;6!u{$e7t#Y%{5TXba`Xg zJ$h&j(vfzb(>&a``##0Pe1ZA~otAgPNaC-f6=P0|L9X2 zDYkK7NZErQMrQkI57%z6mI=$gcg_<#dJpr(X}aB)z}*q@gN9BO)Mv=D#`#CVc5%$f z5jzGq*Mu@{kMqmD?Q3W0vtQa`edh&=SIUMektVeJ&R-~AS@?}6fD|0jbL5kvTt^U$ zv+Ta}CyQ6gPiTS`+n0-dubilR_}%W?b-D~Lj?z8ME?s3nt<3I|tj`zvtN7?^hsoPs z3fIuF8U#+czFhjKPkjhUljZoUbjoE_3PMVSoKVtP$CxJ6+a&MH;@=x=9$z)3FCqD-e1**p~wqsuc3oL-FjR24%g2=iP=J`}3B0TUpc zk*rLp62^0D4|}iTqO@WPIK=YeMJ$WMHNKBU-y;*`B+><0v|EGYy;$*il{TNN%a;{0 zxrKSm1;%LW?Dn=}7f|siSBToU#{EwV;}>4d7%^T?jsuluNFT>=SY1QSapummNcWge7f}Gvz9~J z_aM~wD8KXzh>_E0>+2A%#lG4jZwt|l(0up%E{~J9H?29E?3+EWtjGB887BCxbnkM+JyartI2m^F6EUei}edRD;{T|`y%vy zZRUu_Z#$&br$oA4A3FCdH}NG@%2ACH{SS%9nKcHXT$Kja_l($eSDU)9?ALF**kux= z7z6bib57pLIt*?wU0=W9i$o$L1C=oaxBwelD2J0dUoq9L>^CA)lyba*KNMrcX6==2 z`W?{9!G|gn^F=A=r7(^w<+f^3${MVyVk~xdttjF;Lydt>m3uQqH@+_lhbi$uq~|?O z{QE&zuhuU5EsYo5IVGQA0vE!PND;$jCyZ<&sNeEJv59+p${r0@aZ6 zM1(VuaPYvC67X0P>rCVt0TN8}=*Pl;R_8t1w=ky4`pKdjpI3@2a1gi$`~};Spj4va zQL;g--QPtMjFfld#bT~8Dx|=t$~TcR^Xq%Ga^WhH>+|^Hh;vQQ!+IKNN>nZsyIHEJ z0eYE+UU67N^W>${#Q-}^k~h9UilBkTlc4Rc1-;@DcphW$Z)sMp@^utm?(F!f`PMabY%n#@n6j-NYbJ<*DtmF=Gd14}O#=-YFZ;T{`!@)7`c1XuCk$$x zsQpr-d5p=FIoFvNrDHUXhYyM`++3o zzm(y*63v4gRqcY!abzQw%#%MR_EW}D3W78}*@@sH8yJj{`%kZwf~y%-SWHXlpA)g~ zQLZ5=BO`?>^$6L)`&O8<3MPX3-y0$1U$cEJiYgBZ@e&u66fpQeSUO64FyZaG!0lg8 z+ZWEsiPLXUD@N*vkvElEpPOrprg`+wKIGX<=jic`a{9l%K~5-WbqRsPlcPCqT7)a} zRXFtjWRx7!z`X2~s{*F~%W0yB8{GV1BB=lM1>ytJ(dAKvA5}gOa>CdfbrhCp9Pk*y zw-V!+HrL6 zR1Tvjp-Z`HQ3J>lkh&s8TMVngbOujoxL-m)&2)OH=Rx-Of$dssc99&*qAWL`)c(9; zxS9-H^d+xRr1 zi-Hf~g$U1icg@jAd1stl@xo`i1f?j!*^Qgm-J{z}khTY@TW{mV&p6-G-he&l#0icB zP~VlZuGs0rM;r;J-6r&lk$!DY)va6&_^8(NCy6FOsj6U)iVY(5!8>$EU!##8*zG*U zF^gU3Vt{>f!Lhi3iTPaUReQvlKjvLx^dvxf@agOGuOesw>7?%Ne3!ytkPfJAWF6t8>};Pa6B?FUU-B-G(j`MxNeDQ=dAdX$L4bqM ze6+lqEv#~wdX!BQ8pL_}c6w5Wl!jivO29m4W1q$P&|;LX(upZ?o^DiByp1IF6tRM;kr@c0Pi&h!J|w=&psr+os~~k6CeI3MWJq4lEB3a&~>xYpV`^Ls(EGCm#Ax$5@Ciaeu7%@X@o{ zl>np4{RhPXCK7oU{qjhROu0VG=gKK|0^f!=nP-F2w5WuIXT+QAi+Ox)KY@fo-v*fX z)iJ>syFTHbJ)+`lwl9*g*ov*ZKcF4_*QI@I&ho<=D}Ta%b}ZLtFMv`Z02(z6tGw63$y*+B_1rjT0Y|2n>t6 zQAmF656O6dwcJMd>=~gh2`KfA33u!h|Jit~ZVZ#Kb+q`1%Yo&-a2Pl$K}_OyEf`!L zd)JB@j?mQjdYA++_SkBcvakutIljEmpg?~GgVJ#RyXR|V<4lghAdJm z90`i)z_4@s@_9XMDc!hMe_)r%;$McN{MQ8Tz^)vBA)Pj$RJD=z!0wHl$JwDlq#~r- z18**nE4bHK?o)1HWQ_A5mwWPkUCzJ>2j#pcpH(w)Rf;Ovlf`vpUVw*LnHYF)rnC0P zTlA6|B>*;mvh!5iXLYY_2ut>h$Dhj5210#!W*d|W96Yu)O09J#t47MU{L~94Zr5$UOtq-`C3atslA}EQ z*NplG?w%!{U{F&h<*jJ3mL)aUFYtJCp|j|pQF^FABtqb7#(=2TE%Z=db_i^vZ{T~o z{vy(Yk@8mDKyede13%dyZ++rJ6{E^Gwmqmpp?Hp7N>xD;y0D7lNwiuzH)e~);M zORQ+Bng=ER8*5GGNPwymuEY)sYAH5zM3euT5`!89i%qXxS0}cfltE-$`#1KHx;?Ey z{-aW>r)^VG4AMc(_lg4MrUgWDxt4M{jzR3IQPJQUCTs^nFL_^0&@aT9t^(;?-@`oCrAtiFV*tEQZ);3%N z>zQda<~p{*VF=Ns;yE+DH=V(tp376ecei{EumP1Kma97oDMm)hpkzn+X>)_dWQrGgGlk;nhbyl? zDMxZ75zo88ZWiJ>cc@^1P2J)g=HuxPuySqFDW~OCIBFN20YrdJ^Ap8f-av}_VULLw zVw0Ow#UPztsY}!W|={Dsam&RwtxVmjDC^ACw)_quJK z3c5jg=VZz$pSbdOFKWk>;oYq51lW#*BSdbEXskn-XPQOVKB7%Wz%J;!FsFWg54l(HDmcPw{) z;n^zwYnu9?dftCU$fa3DN~Gze2hHUm#zSUZ3iwDV%aKd(eOd=hzLxU(BH5dy0Lp~n z8pr-3vBph@YwW=dh&b*@6i@>A@HlYD8$}8xg#+Ix0tpQb4>w>AHgawupEL$`wLLY@ ziX$P}Y~S>L5INoA9-5&`7#`K-l=G&&N$6#4OL$76M43Ir3%sii^q;)v5vD}%3??%9A4WgJAHqfR^Q-wZ>Jpj z>v-!c{9*_^IrSVXSLmUf1ygicXT7egK!osJn9QxwI|;5E&ie@u`7RXFT?$8HvqIVW zx#gDKQk%B9*Tag?B|!u0xKsSV(MUuZBPVtV7C&&R z6CS1!CiW0c8}`;HZd{qz`>d10QQ~rdX#*wpA0hgDQKXqdL`q^oY6+VuOMRi8XeU^e zoDB8&qA+n_tjuC59I>iX0{Yl$+4rMQ^&+GJcKvqoJv(PH3|TM|OqU)Ox4Y@+s0QI? zo`_}US%Q>2<;wbmoA$_20UdooXWNMrcFL=cWNE9HX^TB_(izE}0Hqu_V0w49xXwN1 zX>_@Bf^4(QydpvsJU4fXRYjVw^{8)fupx8Dlqh}l%|IW2R?Hwph60wak=BEL?>Uj_ zZVZ_cE3)N{5&nAOZKM>8mUnPEj6d86+E^gIoBX&YAR44E##wKIG#eb@n!wvWInB@N zrfvwkj>tQUx+!g4&%F^N@7xsavj-M?MF`0aZVx3gG}4!MI!W9+$pyM53|St$Zy6t8 zM7ja<{y5qAqZGZCLkW2B4B+!bAyw202f*H{l@D_1s{A6b&kl((goDy0UrRY2A;R6G zURp}x4~N7}cTChnC*$lxN9E%4-&+bg)nHl`FDl8|r*N1m^$4dptxqZ!io1bZk159r z_L1H~0pkbsA12FSwx5xrE=}QEwp+@0hqns#jVmt{ip+@_IzbymaQuEj4&$Is2_-x_ zEPo4@7Ms%wn65IUQ?j?uRVjM?BVR!u@?e3tFG<~1m2hRCAwg?pDrcSMQ$_P2nWB97 zp80kAghP0P>RJv^T|pJVQDyuR>npyMQP^XbYPUl;NB;{qmlPpSm*{!aNoQUO2DeJb zmU0;DLHySxbS3ROpH~lrT&>IH>JNz>A^qJO5ZcH?Xr?J&w$REMT?VGf6vw2}H`vO+VTVGRoPY*$D@`)&|-)5jh8EazDc~oP@zhf=$k87xaZg~ z<4Ai51J=1Lh5J**XghSTe4yCmW%1M6jy@?KCKojSRT`(;L$a%#NS0hnr~(fN51zCR zyXnOZj~qWa_`U#(5^6F5uzFQ)J}KJ|?Ij*2PHVK0d2p=ySgZyGWWb{ zKyEk`owIgg50IKZdB`)d;umIGs)X^&YimXB_E5cYRiKf+eZ*SNtwpN1&fc~~F8=sz z-F@UCrDv=+*f><)Hzs^%i%dx#Bi8WEuF`HeQeeHzrCuCu8qkm<+pYcPz_RPguZyj> zCY7sCyh4kSj~9zOnorX0rA&1OLw;`L*la$cx~uKU3)b70u1eM~QGMFl$+09bxME35 zlfm_4i&mMSeS0>=deJ@djv2A~e-~ID-aAp+8hiYAr?s6g5Q@1mcA}DJX3d5Sr2uxe zRGcMSV>!SK;jg{cOaFYMUFp%`a4ud{lYNd7Dw>gUK0+L0XBB&xCYIEcgzQYb(bR-x zQllf*OC(lQB{YttfEe-P@P>L6JX=9e>afyDX2z#Vpi7pW)IQL9nZ6LHrejMAKV-ei zBGY=G#gfQ^_ak`&0i}NMXt703ySbole45m2j+1N-#~D{9#hn&Ea(h8lF*p*6tk+09 z0aG)3P3pN^CNWzJ0!NwZl#+N}KB>txUvG&Nxzf6WgOBrlgYTpPTjjKy_FPh>%^bOw zv85kU%2g9f8oSUcV0sTj(;AY-9M}V@-1gdO&7(%!^mg*mvV{dG zSGgH2t(VD@=K-lnG}4>H6i71ELkX=1P5jl(>Jb z`1L8B#oP2_No0;YnbpeoiMH*e$0DuoNoiMwfecbQgCAZB2Zr18Cq1`N+{^r>fKJJX z5uw47R-tjiQ zRMID!3vK-4xdQalRh5qF@@c>N>C#jI2#8;ut2=|2ttPbJxXfcS4evJ1`Q%@=hmFup% zR=Gh%^GG{s=OMXxN230Ec(ii#V-C3(GXREIke2VZ&d{+@#TDxfkhihdSOX5KAcZqm zPcsMdyC|ie)lVi{Ue>i$m2wU2WWHGPp{3)4YDhU>Y<4+D%5Nr zIwUCHQsQ!^NfnFalc%QY2r|kAwUoEbTB!VLJ}1eYL#-csp3^&cMas9y1EQ^;cxnMq zhH28}VVN9rm^c0YP!VJy8tBPetzVe(pqOz~G8w3bvSgA+c;h`vWLA1313%TH;20IS z$iezu*=yP`9OU4025E5X`8?~uf2El#^(^mZ}d zy)r`;c(5lP<)+A}(aoctE3+4hRZ9`SPh&A-c;k1U6DL&Q)n$@->dT1H@H?l+xCtnNA%i2fs5*o6uC5 zTOoR|4F+t}p(I}$D<7&bDX9#Ptg|w4@)6}E70K9xt)pY4u+LzWV`-)fMvGOiZ&1@F zMFiNKmeRl90g*+L$9P@h!5DEL>$nhHU6y5vv7d(M@j6nd^4J#7?>viG1`a=6WmTN# zckZjN$1kXlO&=)`53#zTN|-A7X<~G@+jO%1H+8IYhj%1Xz!0SD(h}0>k92kkCw7~ zzX;Dhp-pI%tJi<*jTLvHe%3{?^|WPAYaYX%IK+Y$=w$~zZ>=)d2MGt1JFwf@%Z^ou zZ$&Y=ELOB*gpz}D<+=fV5-&zMGt@BBbtT)^4e4_DuiKE1M>q{RDVsrOzz-?MT8jRA zGWB{P)gb)mto0#VM@VsTe3LBFIQ*n|*eD6^H1yMVVKed z$CIa~lDShL`FqXpOkP@yuH?)7Xb%;A> zU@(3eI_Sc_^Ne7X8~ja`Gvf7Cn>A82#vXrccYbwuc?^%?hpj1`yNCe?)nJ;EFS3qz z)2$NG>Gn{|Pu$FMSPtqVniBELWdi9OmZDtA|F+n5xz3v`{9i!{l%vXZ8$?5{GUXb3 zz?8>Z{n!*%(AR5Sa*B0k4vPB-nbyDWeWe<)X{JMS&WRibSC~&Rg~!ATB8X&BvmqE9 z(`H(KPR!69ERqmA>ySW#=t{TMSXY^SR$O8xXNgY*hSIeMm|Y1h@dQOvDI@#y7)(z@Mxw5OGGwzy@2ilduD<4$Tt;r zW2vF!gUus{NP_P$$IunEB97*v#O7%pY;d=Bd1Po44oP&9SyDrSiS?muw8xO^MnMN0 z5%f;JNaJ}|iYvw--4rdZH0MuMz(9Cyr??{Z4P^_pmIL9vwT>TYc@e4FKNfp^k7zULTANZ041K6lvPj0P9;lz*G{E+r7K!fdCqvk`T{Hl?(uFGU`0!|IC)OR1 zqQJr9dRNahBZm{M*b4&^DRh^%#*hsi`Aaf zC5HZUM&xo^5M7kRslsOc5|$bGuW5wC*aRKGhY#4sV#C_3^qloStb3Rst^Vb}hBZ$R z*KxGE3?A@->8U_(vgchI$aq40Vd2NU}AKMj0M(T>C=>N&G&X#J;JC2>rYNo!qBwrVL6E+kZehSGF|Yn z8?!}o9va1KrXL$NH`((O$#0Z$5Ye`ta?`U#8}2p2)O=rt%}x+^)6pwuS7~7P1?U4* zC6qAg-?K=Z9g?a0LWn50sMb2iGQ)9n!DxfdgIO@#9J{g86L@p5vEpr!-|E;armI zX4p@A#bJ(kD{V~`Mg~XHFVu}bhp}>XgtgBM6JtnWPWbJ%#iBPx&R6XvWe9h8t8{!B zBc~)y7T?tN)VknXs$3pHDLnR5$k-BFn4W*w_Xou%Y(OYiCMYAPw3sE1a8|*!2Lv{k z5=n}P9;zC;w5<@IR-G63(>(mO`j_(8n^f?=4SStlH!qhKO-e7Xf zn1-AZT`2~VSl|k*9;8cjTObk$%L24DghAs(=Ekc;7&gs!)wfRZ-pLFT$uJ#8RzTV< zQrsx~b-7%#@RBUeDR*q}_T|`X6uMw}s@#$5ZP~`8f-+3WI2|3M+lm=Ur%c{!J<0() z>}i^5%E4sI!=R=Nu^AdiPO%8%YPo_QVwi3aDdfz+TIz#=l-WzHtsit$)-s`;vSe23 zJA81!p!yJO`tVNKgu{aT;o4($tr*K>WEUm!RN$sO9x6_;8kpc4C#O6cZ><>6T8}P_ zl;^ity|^2xc=+imJ7!v6bCRjV*RoS?JiMGH;><`tR zt&Cyj^LuA{@D!)a z(xYHXQ6+h+x9h1DTDd3<{gm)V|9+n~a=0!LS>Ww<<#r|3bo9gfRCv3#Oj1luoDN?$ z(@R=A-8>AiekVOIGS$d4y~9X5Sr zdpcvFX@24m*P+UAwvn=`&>OufSeM9E6RY7@gvyZxMH+`;IypghV`hQZl#u|&u4s@o zQV&Oo5eOXf9L=Dn$`_}@7iEj&@&0 zzu=ALB3u4NV2_TrelPx0k1aLQ)GHQQHu)WLOy>v>QXq5>>#KFyr7iY!n5U9#jr`%N z|6qY^IO-)WHc$0I!wYLY-!eW`IE-?eD)+t2nJXnWLj!v*&Uzr*T*zIozH`HuMaYJ| zwu;%fLE{*{Y0f*vT$!;jG9|#Vr9ez2vmgvjBOJbUiI~D7LK0H2DJ3}lvytwz24H}L zu)C4fn-8BlGf|*g)bKYCiA3|-jSGxyYl%#Xf&fMpO1yDET-)hM#Z*ax2aB?4WGUjm zF7$c?r%x@4@uI*33p(d-eLIXrTe;bZLj8{#lD=CTFWHh3Y;CKrEHre zC#*5YfsELCggu#Z0wIN_<~x@fxY8T*uX*v)K!5*&xBFdf^i13sxiZe%eYQ^2;6~va z1#-Ome3mJ3Vyj5XYNfRe(H0}<)_mPp#~ARj$B%Qw5FYyhx*FJ|+o_>Dz0p4}*2Awj zrETg!6Jw}u43s+hq_-;<*r{*6*~7VKu{fr)9`)%e!@q1KZ$EaoZnp-V7OMSwKeeYH?WKqlMI0{`Z54zMz)%%*UDa?Oe6ZR)50Mi78)F_u3Aj%u;)+N?! zZSUUdIl*2udMGSFf1=nM^LeR`WK(5oBk?smc*5#mU9Xv%a zRaTUW-gl^w>R8X$eqmke={GzUB2^Viyu@)2PE;vExVBjNWZ7cc6x9+GffBzJS}Qp8 zoIwO>T~dGDCnoUC=CqvZ{y_Ry!oDxun;4Wzx2=|=Jma`{5l59MUE)vX`|N5cVEkZu zpin-3V{biTLo^0zbyDZlQl;SW{|GzxxT>o3 z|5KJ0sYzmLmM9vAk=a;z$;)1Aa})(bIijMdc%!`F6-6@+;-=Ag@7Jw_A(b`8O~%S0xOOm!}37@}x2tR28H=6D5vu79W&m*kk=O+aboT z@!Ww9j-QpQYO!y{Ib&I*Ry;5=JT}COG8U@T+~cBl_O1wLOoD!8*L*CjcWXIgIg?j* z)j%7Mi_hmw*M8}sL#hU0+iCGRx8iY<8Gc#Y(#7Ybg8{aO^rr;3H7Rx!Q+_^MgcZ82 zSI^c=sRl47B3Q;2#m4|!q>WsDIrt&V`dfjHKD}*gagOx!2Nw5qc|S(9W5iY$@AHEP zi&X1LCj~>#m2MsJ&BX1hxQAo!7TNmAz5;s)p}oI1OWs5-dCx$2?oh6%;#ua}SKG@s z#ZGOqYAC-LXV)-LX}awG${}3>1Ht2B2d9n6Ijp)av?XEP-3;Ir4{YMMMP#|#y8HD? zib6}3&nv{|q{2?ZK>Yp}p(00ozHFP;#gw2$2`^DT3R2~KmiRkE1wT@@Diex&Q=Ng_ zCsa%|Fj4P-Gmv2q7(7Rn*OEoPDD_G647BK+Txs$rs~!R#4_$WKK3}1mX+RtKM7T&V zh*Atq716l36vOCQl96)TLGciMGR4jJR&?Cy%*1O*QNXarLtUKK+?~ZZ28U>ORLm{@ zR!f0LZ(GEdI}N?t*Hi;X&q${shZYqN7(?i_(`obFC<|Rw1Au9tG^ZhVR?$`ggYd{^ z(c#sh;uh>-c%+|kmT)GI{~9BUUNKH&^S!FL3Vy027hB6&yFh|hBB9iY;oN6lD3Zmj zEVnrenK333sUk+ZQG?9W-|zw0E2rJ4apr_srgp=qG^Yv2PJ*GZ6ay1`JV&fg2bd`_ z{G1ie`LQ%(BzR;dI^pymTTOf*g=c5NIf+naX5tWfh0BI)#x^xWrxaUExfJ~PhqCIv zu%-!fGo8BhK^GB=w*SkfGb2T_!KFGZt2Fe2fO9iz7yO~%Q)NN26VB_DILY&VNt|;J zo>S0w>wwwXUgu>|RVuX`c#(}m1G+y&AE0l`abihQ4Gz!y>1SNY_%sXy5~e_{9^uGC>@pWuu2jST#p{_S8SLat3L^aZ$-xNreL*gw&hqPAj%J z`LAd7oks<0(%Q(GTH6B^PA`_^tCia^UVJmZmG1cs)5A@i-lZ`*3Mqv>+$-Wb?g2N% zruYWJduPOTw>_vGRigGh#oJfJ=WGk2m|~3jH3&zttj?p%t8Tax9@|r7mlkGc1Rjde zOb$qYNB2@j?9XQ$i|qx9dt$%3AZk^Fhk9Jwmi4+gXT6)`h=#`rA--TZs&HU6aGX3Q zzF=Y%Mra!X7laB~f<}VZ#K;;M%aElKDG)K?8yl@yF7;whRjs!zJ05b|wBJ%~LfNUA z+eWlfP0aXK#$*PM;Rde?I4Xo3M`8g#9X;^vys9LqAG$mk z>)ga(MBPmYDqSQGOH`b;s~a+DUYwyOiI@3`$tWQzxE)7*6(2CQE1Wnw6#g)7kVBK5 zp)6IPm{+B5EoaD|S6Eb0>QklNUh(qGD$#;r&r8a;vfV~enN?Q(DUkPV6DPT&sVv5i z5<{XL(rbmh507C7oQ=EyQkM>8xAMMxEZjMk`i-_JK?h-4ksDR_5gn(1gU1~QW&d{P zl(S7}%OxG%&g65<0~ka~^gZr&<~bmMc}5;H$4R{QsRc<&DsjJgvp!5cttcOgk^+?Q z@T$zIaQe|2$_y|Cy^i?&_fuNRMU8aS74ZcN|DJLn9TiP-=ofSq_&^vFFTSDy@rNn} zfzRsaKfSNjhp{3Y6`!AMt@JQ4Ci5-@A0xSIZ4*I=&AZg&5T~-#cxGuf&=dae{WM zbTL)d2yu!GcJT1LzlH*Pev0tyz#myEcxfl!v&kc-FJ|#7l($+A2FJ7GVMtu-q>YSCg zOnXjmf|hBqCyzO!NOu|ln5ptZs*}Nfb`=3*O8h-ioWN0@w(#H}*U6VMbpL^92w#+o zQA{-`?zN~Rh2jK@e;zPOFgs~P&d?wx+^s{qGD4gnTO%Spdwg_7Msisp^J<)Oko?3} zoyW*CGN-~%us){z!VjijH4!IBTcmgx-+sMXobU!D#$x2kI#K;v7A8DLXTTBx#g%eC z+rBf_$zCLNi^@~pw;N3j9^lZTGGRa&$Y$587c+~VbQvvld|WvoPVvG6#nlfUH=GeC zzHXsI3dR8Y*9CEsMf)(*UBKg}W6Fej{sNIIt?LIr2Zt&F4oduyhaG#?8BdN~Fc{_R zn2kY~eGf!>RcbJe%?%cN=Phv-Olwt$3fe8Ycy0f`r)6a6o2tGSr~fUMkq4J*0u}>U z1Q*!JmjkeF23W%m;uKdiszMFOCW+T6?jvb%oxavfNK`}H!5y7nY3eXk8bj!wx$g^}(1C{uOFLtWbBU&|bO~syx6KwO%t-{L zF~AIF4LU^Qu?D|!OZXQPL%*f!18Q$42ue9cNE6?2?*!whYKR?q zRvaghASgZMmK|^=^Fqu;ni!?1viMN&60hxfQXr7>@C%+bQ#cHdC0oRo#26TqJ1LQO zNqpI5Z74n<%$+JudfTLwKy0wkxsE$DibtlF@=A_4$$%s&gd_gOsqWn`*m}%wO7P(6 zN&G^6#TcgB!(?w~kG7aW`tLbT$6pRuVb}x4l&C@YJ=*C+?sU%{K0E%p1gGOM^E?ua zsq*1yagzLWC}x_6{pm)L3G@<72+?oH-xlrs{P4z5Xrs%!U1Z;$M>TZqm-t(Sb2o`e zI7??1cjB#%Zr2OXX%iYA^^OO3vF11%N5v_z@saVtpXn`?az>Z)VdA8>`G{zM{gNt9 zF=-B>T$si&w8HgO{!?tCQ!-4Szu@+KFJARfiVsXj#JKH-9oJ0Zfwrjuw_ch#GNPHr z-yZ4KC2c?@${Fa76a@dnf*BG#LeKWj^(?6cA!uOn4U@&0NA^|wHa_#H+b+!H8#g=> z!h-)`ac4RaWN?t%_RR06LTwHlPhD|uXU_r*#{2P0wz#*C_GzZNioI&JT>Xzu+NMpF z`tb!(&W|_W5eiFqhzMENJF*HmO7u+?Uofp!O!HJ|$7jbnVH};E4Pm!xFYy!h$s5Q? zon&GR0$01fJW;F-6wnr{L8t*eCtb$;^Yv?Z49S)wy~LXa>7gUeDK0DUhZUxQameP3 zmk-tuGLOoEj6e9p^o8*UGo5$6S3`BI$Gb&hh<9EaKIn3wKoRs{qD0$+H>ZPM+V?yy zh4b29bAgOMOq|v<;iztMAV&dllKR1v{2v3o1tP=rmu!3 z2-6Na?_hOWf}DkIPK7|QBKRp_(k%L#WDlFo8B>VpFw;a6lI zX7l`EOz0auzQ?C*3^lxJplLhAH(ZB;p^0Q48+NI`*HIWukL%lBDsL+stxKQ}V(&?j z?O3+0R*s#bAG~8pRHMXer^F8o;gax+F%Sp*&vN8l0fR`1cPeCL*BkX&$nUo%Nk3*)>-tG=z@f zZXDa3E$A(4D|_?|Pjl~kWO{YL^h`SA#xa5gl+m_l(rWiU(yx_Z4}(rA-j#D=6Svy~L zA`iR*X4KCpbvAICvPEc2*Rw7OxY)6JE2LacPA~t*oZL~N36JH8?!9EN^M{tgdk##r z9B;1xZ7DJ3nA6Y8*p{h*wK*-jdJ#kkpwHgpEa2jWQm#ZZWgRAuP8xz|sRO?MV@K$kx}rf#$|&t%jwBiNpA<^&5^L8FRC7r_L1@+t8& ziS8|vv^}2NA%5Y4q0%;}OEBdr_43rz)g>x(odpdCtMx+l1IPI=C!awt^_3(D7dJZ# zz3XyXO0S!aIbU(n0EVg*B{%@Fz*`_{s%?5U!|Fu(#y}`=do@aR{z(<3zA14!Rs8UH zGu;|6qu=h;y}eVwOIOchLd2rW&++n}mX)G6F;=BZa6tE0PVDlBX=1(Z3W(nr7;S}6 zqQv&tCeRM1mRA^y)v_a@cPGn1W6fmgPQu&7^SC>h~&%bL)Nb zjqZaqj$W(gxb@!uRXNExs@MAQZoP4fbqND380PkPe!2ccBMGJ)fBB7(qXK56Y)W)m zd0!x7!ndQH?S0Q%-=bWZ#!;h0`BW!?6e@Y9z4+{2+ls9#q^3ZEX+XVdl=!s0lf|{0 zG}JfH*B6L?d2@Ab^wJ1}5)b2?!0F$Qwwg3`+HbFreJ&$CS95z(bO^($NE3@j;g! zIpS7QVnXo5PK|IrpO&UisuZghmB2Iiw{d0vW`ZtGIVzz=n2h7}@os>j93=*4b$^8s z8=yQM*}MTk5Bc}E+ejckB@e8IhLcpJ?W#z^VZOlC5)u0%OgVqdJ2 zoYh53fe5e~1Tx0nasCGth3Z@PKX~-MZjXCiM(a@T0FPjn7{UEpr5tvp$~)slMdd%W z6hq*y#y_uJ|48_MS2`Y>A9(B-dw7_hp zglWR9r=7MeA<;}tC!KJZEHsSyRTB%S!0r7|z_~E9N;eOr`0Uc8BJb3FkYCf72Q$uBuUG_&GPO!8}o3vQBK)V#9Zc5ng&vs!`&uT23pNg2OZN zzstq1Rr-{y0oJ?uQ8AJ=0|ksyN^DjzpK(wg&NjgO`<%bfC-a8_M~O-2oF8{|)k_ss zy2`^x#ZZ#Hs1i+;=X!Sv6C?R(@rNg6O^jP-_6+?sg$E@bJ}M)4=v;|K5==#cjG$F| zVikcAN>F3)qVTNRt~z7Y#+8atrmOsAv=zyaRzc?nkHS6jR(fsz>)E5~sB<Ye6Ja3b7xam0?lE z*SO(A9kJXyWn!vC#97@hK3N?OeIDKD-oQr-re=|;&(O`HJxfS&K!*s^xAsW?4f+GD zfT4cbE;-DbiDxR$^@-gh9tIS8nE2?^AWVjFQ@CL$=xP#LCCf18`@lB7?bG&@NE}(I zW6UrO-yyraI#k<38>mlqeWz&e*P<4Z@|3oH28D}97>))IO9`LdXF$AIVqS4Kp6heR zDcOZ{^b}W&De*_;`V1)&UAdV84)vo1nJ2rN@0|I}_ZcDdH3H0M{L&}mn0SO6?byPE zRG;)+V$u3!?H2=V;w~|u`ZW`fzLepdBCqvDpl{U{Ta;g_G8`&`Q7#Z$den2aaTI3* z2Qs0l3WY--5^%}#^Gc*=vFYQ*`=Ush zM>@-hv!O$J5RRTH{X3Vx5NQIY&#@h)pLy$+M!7(&?6UKmg~e58fYo0P$XmQ6S0urs ze}Wv~CGpa_B!(51Rx%ac8p4rSKhden>5eLNp#(fK^W`{hP;EA&Pk1aymQU0z*T-Kd z<+l?j?RBm7YxDsbzY6Wd`bWiZZ~Hn=g;e5Q2dwY7bOk7769`O3zGEg1Y-L8Do%rA` z>-3Z@)r~OGx2<#`hiqztqfFT&@sSGa+m69%MRIHb_Aauf+S|?u}>75edWeZ)xt0JSe4bU^RHW6fbjr`&toRvkw#u^&g4hr{o;(H1<|ykEEAYyB#NVt&X6?f6sAm)~5qzer~ zR;R>}Z1FirJ6yHcic8i?HZhb5jgkLV;9QB0R*lfOA?Zw^)A39*-3WmX4}TLmX39{F z)Bt}O9)@t~UD?i&$dJ;$Ow42=mBzm1TqS;nS{(gz;@N)})JN$1BkCHk@{d@k`t zJwgF`Z@VuU${J47>G@3W(APQZS}%9eiwF%BkG^ekTrqq~^(I%}wmW1qrn6u$uI#&X zv$X`z2SUpA&FUh4BIzCGGf~)=w2AdMKBEU{EtJLju01Ax^pftP4^m3v#EGei_tv7bQOC3bI(QVR!J3jj4zVI$;7!8DS5iYq>XYo7p73%)ApSX;z!=p!9ePV zlw;ZAytj~P2poA|Jb1l^u6aCqFw>oQ=8`zi(Y4KCVafKR&Bj}G_~2aS@hC&MAI(<0 z3Cl-&C&-vuyLGHLRn6nk0U0vp*uDCiiD6oK*16c-WPs3OAN_2LbJ5&uG)N!)Zin+L zqrNJ1DO1Wly0@9g=Rm3=1eJ1+`p3&*u|1Uf%If&`FK6YZ?@jVws`%_ji!0>)tQjb) z8yssd$@@7{s2qin{C0B8UgzSZbJ}ydL~?^m)|1TUr}N=6RZ5P?PLm?lL_%pym4~97 zpPS6oUk{_p`HoUOCx#RcdQC!>YaVRHJ{Ewl-7Cp;c0kWm*JsK7O2x@2C5s?GNWlmg)K~*OMWIG?`AT> ze-&d&(Dl_`V(D-&z>-hPDeh;XC)!XhGgFpq*Dc!s>sPyd(BU9gLyTNoOL=RZY})kd z3(7F9LD);u|4X-M9L9uuj?3nK&6J^7)jTnc6sqKf1+RF=f$$9m>TyV}B=a@JO_$Rzx|?hqzf3C^rb~Rj zSx)8~1$0lBz?|R(a=KC9*dsYmE{o}LtFshX2D_sna7q-c!Wsh8AQ;w|OEm~cS zQM7+9gH13fSHP4JY}B_$EQ+gNirw$oaG92JPzRk>uKzt9gRhunN`)5Xn2Jt&qe{nV zK&$`V4%w0$A&ZE@G{v;vtC!^X7cW*jyMM3K)^i`+b_``U=|0A2l zx9s%z(bgDQ0mHmqjSs$>+mr3(5JbYbbn^6;Vz{DwC`Y+pN9pQFf0Nwbuf9<1k(yuxCx?JFY+N1xmqF`TUjU}OgNr{ejzCnV*p|$P*`~}(Q z$UNP$jkepfWFubQ0F(*-8U{*9?BLd6XJof&**-gEP>g$n^>p=uNPpgZyoJpr#2Tb2 zZFUA9K9!)ALp1jIF;h0UsGmpjbXmDhPDVK}c!W!dvI#NPGlVpyPqcN(40}xQr!<-> zoBgGSh7K0Lo$^qsypz=)Y@v({uun7HyBC;GG-m8=r|@1<<4UvCsV$b$C0e#JI6S8F z!o{uEwbIHZsi5o6OC-fzZ>(;&{bxtW`5)=0^OPBpvS6QR?!CxHm40^ruB+v;{387Y z(#}o^#EQnuZuO!vZ;@FnhBibw!{hy( zq7kcK3WrgSL<`r4FVcZqrmHZqcYQF$D-qlxZy}#ST}tF>MT9|GNDW^uux_s4x=wggq1IL~NGXB*qy0_F! z`<0xP8>akV@rSDBxBcx)MJByAe<*z*bl)qwaZp&{$kRy6yU4V$pNJ}EYZc03eyf>C z;T|9MFeUs;lSQMg=7CQbQ=$f8S*mDM{e+*-_P>|moF^#&UQ-sM$_uH%os5*QhZ$|9 zMx7SnESJL2n9%=R9V__bIiht=bGBO&&blnd3 zzNB(}49B=(cd|^6x=x1_jA2?9E7Px=sACaT;km@P}CLI;m;Y$861{zSS ziF+eQz0yLiObsh_JMmRlRq~Xh2F#5T0kSuML74zxH3$pqiva5ml<=4?P8R_dRh0L& z`UBqWVjbqioZX=wX281v>+rWdlnd05*0u+{5D*(U(p(m5Km$&s2meg0S0WZHemk{k zg7eEgW2y(NsZArC&&ZswkirL~XHrU6kvj~pdD{MYoorTYo_xSSlz4BKC}$}P#Y`g{ z@Y*SP>$ayC30BxyoqH_*cLkA&0)rDCpk zY%QQl)FhCcU@pD|l(AT9gG(aUEFgM>mV3ki`cs8eLr+aU6zto5m2T=_foW<~u(x+f zK>Iee`8n~|M&|ClDbcn_JnuabYY2}OJ4KuzQf>f#Z#&g@K!o4*j1p;P7O4rx#LWhn z5^MjD68Bb!OWb=6qlv)-1N9y)o@YhzB1{7s*!ViuFLf7crZ9%bxCHALIs+xv;J9at zyo+ONTd_J)(B=1Tt~KL{>S;pi!+T|Yq)#fs;_8Rk)rr=5uDT&Xm7+wG zj%7# zj>*SvcT~`s?jkk&jC?HePdXe7^o_M-#I=`<1~!TkH*hR^w?T^7uj6HdGh@_T%xF7s z!x{I!z$5xdGg$ogz;=G ztp_9*l}!hZzY_d`b*W0!->RjIj1#r`pVe&+&)N3CrU|mScgH(V8ED{RIo7FQZ*6sL zv4QOGog+^#5=?vye7vK${_{23x9~u@w~ojuw=@Yg51&1-4_A-!3QS;$# zs$*>6tWxVILK_LjrUMtoTR$`7Qv{=nERwfzDK8N6?Lc3Yye(;=0_IgI++(S2V-!=R z=#-{3lN%qbuS6h*Qvr^$178tdkt^CKODJ4cwjVl-yNIBh@$twT^eb{c%`m zq=eY@$ui>djOzZhrj+tjagam4M3B*C&<(q+d1M->j*r19assCqv6vA%h-YV7b74y1 z;ODmo)sM7JG2Gw}&mMn7I;c^qHHYgiA2Wn&r~~GsgR(9+>;f1FNg)07sI2QHN5ZZ! zjZU+2=?g2dm{xA!>20zB51%PuMr>TH^(E&|m4UDtRUX-FC3~5kVeGR9_25N&@5G&! zGRVGQ^=Bn-IymBViAN)?pkqU~<2Ewk6UJk?SK4t|L3pCI1h zK9J%8FvPZtl+6z=4NU?dOz0r|q_b2djB+KT_aA(vlRh2=4%ELu+GcLpy~S|DAYED@ zhVP8j(@3fzWk`{%*QlPhSm1vXY5;q_Kn!J!yA`BH>A_I_ zkWz!NFJ88M@uG!d>gTry-?uH;m-ib0WqR$w31LnO-DePOwJrm{m?MksyIqGAqG3u{ zwz=b8JwPi_AFV+b3*;>$t_w9&m~ub+J?|=H2c_-c8>8e692mN4MOt_RH~1*=bF<4oevO?X;Hb#OJ=tdcbN3dShSH>uDz0h6{hQ*`H4IGUOkmBvB840f9!81L641TR^oT*nThd1!m2*R8*Q6}h20 z(Oyu8=Ehyrv<%;#hVt35$@1dPrNh>!bHbPwSu6KlKdz{=Ndwrs>s(8BO?6h|B-AAOc>5#T*GUA~F z`ul<;aFEWy_wl!Sb)!nzW5}|p;xOkO6m%_i2#-NMw(B9aeqww(2d{oGacpgd59BEN;N+KO9$iDX+otYBMQ! z4$~cqanzfqrQE7}InOV@q{`dkwpVx3LmQ-sa~u`LTI%PshhD!!-o|bd^*sZ?SB*S0xmY%(8&^DF z43C$)h>LI6(Izy?i9J%b8>T=Rn+`o(VeKG^1{ZSGxQ;4s?+{z)J{41(1%au?#izPe zH;~hJAS=OuUuiE|0Oo#n0%D{A1#0md`gUE$2)bCcD4alL&}E< zVsl!5?H90M_0Ph@v&_VSZv2uqF+*;A`&|`ZreCkop$rY2kzJMzS54h$o9;XAEajbM zP@2whDA%#OFlnZQ0hWH>Hfxyog0dD{qq`XT{CJrtKhrIHx7K!Vq3rV7bD_XRIUeci z@?z6#l;cT|E*xkFscQQ^JH30Pbqn(bR5wOWf2Y7HX27}?&!HeCnpMa;WGZP0QcBTn zm8^4~x?!2_wtx>|ak|rh^QSRXfiYr#EtYNB2FLqK+unBi&1dDkcj^6XPyN_jdB?Kh zy4%uK(qBEce>dG0B29^5z4tg|xy=jcrN=D?_PlsyL9vZF}rfL z8-m(EKWE5P_SN~rtHRZcJHFeYP%23<9Z+93>%3ogD2B&?Gx84RQT(BtMTsHNvKgyi zO4~fG?N=9^EVAL?BoEk6qn)*+>8VW#6r_sF(G@P$U@g_uvDiOOi$O_`Xl)VO+aC5q zEwPFl{=txehLZr}d2hiTNq&1cPmuQ_rHm?J2;DcTkQy9uB=Je zbKH8K&HxOi#E3r8&JNt5fKd%8V^+&61oJwSQGWz4fIP#=t`cxi3Si-NWR|BVCy*+p_O* zkrIaR#}XO)V%>YZ)D&O=`jITzmS+H!E{KM}GjILwD^uq4aKuE4pBNeNUxOnfGf~{Y zbfua=x}bH@uj@W}t5PmwZL`v)%R8&xdlO?B$I={LywGldF;Xta$)4W`Ef&#!J0mh) z`qt_+fyVn8mFt{^SMJeFt2EQ5xw0o)ScH@b?Tp8>#JT+@?HO2l+oQ%FcUOG*j0T30 z{PxIZo1Nh^9?%vGQ&L7oop#i*69wH6TJLg(lZXRQiU4#{sPuekyLOcJZAQ`@*>k~e zWeWw2+qPjHdw3a^6&7^4xLMBQr9A4ZpKXuGKP8{Gmuo3VLM*v%MtOrg1MKBgXSugm zkfeBwWOd{p_%;t7rogxq%W&xo~F@ zO$>&{b0cqza`H;wRG>VYeqZ3sX9TO%*HSX}x0k)xenK%tM3+B8&v>y&4qDnodoG}m zW|SV1gD~OlP*{#)H*y%g)KwlMXGe&CykYWO8RgnvlD)|6;~5#G?Nem0A5DMyf0THs zT)fRCfl7b{wQ@5vWskgL)dm_>NA%c0-s0Nma#7hMV@jzkNeQ2iqq0sv6QA77xwaKTn2p-_@*`tPa z>G2_lIkrMTV@mJ@4_8zbQ)Bf}9gD^Gq91j@lqnpezTD1|X^(ddcU;B2V`e==MmPrH;!G#&mL11IKj91*L;z_9?cF$#J92-NK087$RJ_zf< zlj7=dof>Z=cJDFK_wBVxITH?}_SF}Cy(3me+gYWud1Jrs#^FPi3kR*%oG*+EeZ@wR z4rbnsuZEPfQ7%esAaJpTvDoMbGn@|IZBozz{d53sgAd z98vGQe!Hco)}?=(9Fd%^7#gdOsk2j#Xl<0csJe?$?C}l%p-c5wy#~|K*}J;)fPo%M zm1!gYG5ANnlk9Y)5eAWBO!#ZM9D4m{)m27w4khKzcQnE1(rTaV&)s_u7|M;zl|yPT z)gvWcV$76MXAF5ZGhm7dC^7Y{9254m4$G}dxiL8$*CnlBn|D9H%9(cfeH5KOnTvlC3%9O9|y)pFL*h(H_MtRe-@XpfSBp%e%R@ zhe#7+WBQfKyLrY-RnbVt&dhfL9BNf1@|2XZ^&_2*?42kaM(p%tCxb@}nGKVzk@PT0RN<%W(E%R6hYUoCM>+)WW;|9F^6l828)ciB zx2WQt8)^_-#c^^P+~J5PjOOs#7!Z^Sr8uG==A5x9se~_ zGDqzQp5>s9!l6ryJ$6ubEO=axKfpqj$AtB3onN(BL<2oXSeLxPW>e+;%qQzx&oQ6l zzsjx5i+Cjh zDS$F2d~8XfjCtM6_cbu}THggubt2OE<+0u4rQ}e&V#?2FKjy5Hl2d8~i?P__^`^>u z_ui~STSI?5VvoG{{joZNfbX{-&pRfPN!+^|i)~fVAK#S_e8^k43n*aYNvm1#=4oY9 zHLw9``(v_j#wLvvNmOCEl<$5B2ZZlPAN^T79Un%@H^Tb%%3IX|^Z2B5OwP>1BStK} z_P3mpM-LDBF`ao-879;?#<7Y&y<@OC_{WuZv7W1K)-9VV zi;df#DY`O|P*wOr;82OhqeIF-G0KGFUe6GxIgbk{)1izXbjVpqvZyc!l`5phH{5LP z;Y^K+Q^S<`{w0oA0^0=4c(#of^2iQ#l!7oFwoQ&Ay|_OlWqjsFtCTMg>lTVgiTr4( zav`CIVhmx{B{}NAZXL-mhVbi78AV3nDoO-Yz+^>4xN8Ql*WXHj`R$4O_quDI{!Swe zDw~e~ce(TP#Oao?oNkt~;%g<^gDTYJP3 zdWSsJ*T5z;*y7wwtIP~_!Q)9M^`C~{DqDE!UpX#zaF~=T9)Uz)S!+yoLKp*#FX1Fs zt`t*#etSZz&7zQ50iZm>hBJSfnGX9%lndSdhXI-(3#dgWZ`#)d(otxOYzG%%p zay76~MRM|udfH_ApouGUtvk3|tt^I3a6|ti;;-z!DY1rd`#ITqka6YmiRde^eB4<*pLwrplL7#f?l>!^}sEaL5RA zzUIpTL5Xf_XJzklzUO$hvKZh1cK(u3SvB~t`XOavx%k@bR~i#$aXWvlEYe~Pj2JoqDydAayuuxc85BpNi_%wrDFT~WTlwW z7^Z1^gmvxK^Q2X=J#Ha+XBbHN0sXlmxnkf(P0%Ksv^-yoWAU#x1`4VYlR9O|Xim#1 zu^RfME2-AIESM=^T1wXEP2_Ao^DmjtD^-3k5*3{6$_q)E+%L;2<;o28jnyZQI3(sm zP)v>3oA=21JIzx*F!kA!6Sj!mEQYEQ@PUv~D&{f4^Kd|z-bCcHrbKbEDe^}Lm}90d#_gu@raRu-zNc^uv>wlW|AgYn$tven`R(m8~M zz8;gykBhC1>Z$(N^vN}tP8ks{*0GYe)pRI+drIzJ(Yrc3p>0o@xmOfzzog?7K9nGz z(zX`@I@UF?sdpU+*5N{30E>Z^690@Ag{vJ6ohm+i3U3{4BWH$+WF5FuTa;QkT)Ux) z@&44PLsm9ouc|*~M-$nID@2MZKfgV-ZM0a%LKO^EDW5&{&N*T^#!=7>jyvPUGKP|> z1V5_8v=__BF6JpmmAg~LvVA8)7K2cOsaPkLKX^tl^nh8k%IX93$?Q=yekBHDxG>#fZp4xkzSjM<8V34YS=_r?xin1;QVijO%5n?%m zUaaLw8FWZ2zxj?3bd>9wAxc?hD5DB|sM2?;*g`%>RmIqYS0c7}mq^Rxlf}!WAm2u)} z{!l!UG_bWP4Av*-NHFoRRyGedwSQ^;w}~t(bSl5I#wUW1CC%& zk|?oqj}C36984kHJym?iCPokfd=%lb_@49Hl`w?>(wDo)v0o->9=gP|SHk631`~>@ zE-}6B6?sGJ7S*9WJuXAu&_<~Ls-JC7`+mIafAl>)71c9vjbrMT`f|mU zsTw*8V$;L>8Fp2 z^*pJpJjc&xXLmf*V-V-*lz{@;9^a&kQ|xJh(ge)(b2;L_<_#)ipzIb~WPLXLi7&;} zZ)f*DB5zr^OgmfKH2bw^>jy@CU{F#hk;*+!-bPn8MG9hvc93;h{o)VfsO$+4xQ(um z@}mmR|JF^@{g1L3ZeZ_q?kz7WktX=F^9p6%sYW@l!1Qpk7-d#!j6JeT$b&|&t#knk zOy4Fo7I*OKA~sV4`*^&J9JxZ5&{DE%5Gul@dWV<#2FHx)7lOC(uBbZ3cTuZ71(X@n9o0pLK^r%wCH%^cp2T&6JI9DzpIQ@yLpj>4Rr#<+N{Sa;~ox z>7X#A?PXI7pS8+el|K)}EU z9y7Ygmespn@TfuHsg{;3%mB;`NM>fRou58U4dE18cyJR`%u3NXG}2jno4M1TsH4Ag zc%Kr}3*<1?lfj|*AcduyyLexW;))1?6@j*U*H?FaGrzBpt@a$y4LPW)Fem#f_{i`~dgo`C`? zK5{so%~F=urA!f?n7G<~YX9EqFHjEmqoNrn1Ih&MtduLVo_8TV7 zg6TKm96b!iF8@jz_6Gx*2|>VL50H; zx2B8zQGG&4A$(RUbI$2xm5PWx?uruo$(iA~0Rm56jwf|1^-VAJ#DNR4A(K_)8ujPC z8*o>~WoR6R$FQmLvFO)zd(j3`Qlh-1s+6$pCmuT_A7g?CQxj6NPo0$uBL{}G_1jOr zpW{yW?Q8WZP>lWd+zuDq35QTANR*aW#@>}9=X&_b1%!%kuYRs^mho%8p|{Q=xq zNl0m{&wU`yoyZ}(43-&ml_%btBWL!TrwK9Yd;4U=eb%8XZ)qHP%5zUP4!7R-wimF; z|FGDT*IyAIGCoz3C;{|cXRTs}l1UJZ)j#PXm)RzxUr-@^a>W+0kDPA1u_;m@T&ZOR z5}Sk~*=Nsb&9y9EMd%FlGUb;!M=r@(YxEAUanzFq0onGKC-g@g(HN-vUaKjO9@Y+Z zpHD8`Cm&)p5DBJ<&Aqd}c$@rE5Ii0?#L8L37!r&do}5)Ehu>>f@r^Fa&&gqhzgS_R zg_$QKcgZM*$^(I*Yk5|_;4KgZl?i7t+IpeQQ`OF%%P9D^ z>-Oj|E+v2-wNp;>rW)E}PyVt^=6-%sn-DYl?c5ew;sp7VkziV7?#l<{n6Uw6aidG_ zeV4`YRdaOP$JstRw||^C&N&nqdL@RR6UUwZRJZKh!UXZA_g-V3!jU`oxHw)BZiQhn zYyyv0!o_i}%T@DugKXk&1~m^vQ)NGdX=(a%qSep+LjQ7xA*!tJh3tb4zmNgpUqtn^Fp0{=7#{2%71v z5u0{aF8QvT(pG`P9xSL9vH=M}>EpBKb&Zu1{u!--X%o(SWFI_wS6gh}fqgPN?kC+D zPz~ruQsh*o3Fu+`lDFoNHJgtG1XYa^^!&5#FvpTm@P82Uq7&upPf`^IgJb@)74Do~ zzv@F+nn&KssWNtGa`nJDuVt<*J`(Ha$Xak(|yY_=}+!%d6-hW7fT^?GeySE4Rtm7=5`%DMx+wa+GJWHw)yPugB>U z8v1;O3N0g^*RF&BkECPbImV}8Fi7Xe2IT!M{;LQckNa}v>LVYh|0)wwMa0Q=YiFwe zs-JD=&p6`d-|)NcVK5RU66(u#@0ux3m1<)1nRtKy$UfaCYaa8yUM)Au35ucOL5Y!5 zWxFjcs=c4zbe-E^+!0xqPvd6QJIr5FF7M}c%FK}W=TEFJ$G%`|rrPTBIZE7$G^3qy zcBMj^S5_)lY|tBJYQY>}+j+_8;afshx9#~8OXb*1z58a`-291`+?ASpHrPGj1r$5k<%t^)l(k$`0f1E1lgMB8Um)N`|SMX zMdEur7es7{F2S+JS${cF?c4l89po&gkyJ5A^S6}B##~|rwoxv>DpGVJt5-(I+4)(c z<>)GXdPp?@q{CMU#yTHPt-lPzgNpG69TVY+~9uZP(8P%R3F`|SlOJ48NN6OdqRn*VJrIi-H* z>Nb~uWvU!)X3>g|&t4EXW1YH?rW5F}Zvl_6a+!JiY+VJ$5SkUs6s`|~(lA|6w^U}f zGxL`u71|4)i4zMtn%7+}Y995k$SJH$poeK53&_$u;V}^kOP{^)e2Tl2%z_HKkrH1h zC$)G(Dds7cm@X%AtN?o&9{EQKqPYIo+_{geq!uU*jms5_0%d~L)6juL5I#2WUJ_(Ms8aNALt*x&>0Y|UfAPow2@ zuDB~6AjM)GcFHx6>!$=27R36C<*MK@Z6Hd)orE@#hSXp~#%t0l*OF)h?yC~^H2 zxg`0J`mfP;;Z0lQk~>c6;4e|1f`zx9mPyU)iy2saGm?&KYrAETJT7* z3x=kPt?V~c88`Usf{4q~V(qiUDCf5e;=^S9Uiyxe(gi~AC|RF`xk|ZA6_yKsJ0?eW z%+XfYVhiSV5L6ryQL@U!LE?;wBslJ{c_noXLv;N2ON(=owMSEnnQ|DOaaLF78%K zM^KUmR(mhX&DZS(eL#A1u3X$Mzq&;gT(?dxmal7qsWR%E9LxL`)jg)S!n9MVhHiMQ z%92UsRZt~N1T8I=gG)?qN{C3gd|B*$rk-w5Rmy~m&d0mC9DPH5&!)d!5(ij7R2Jh$ zm4C;If4ycB=I&O6MZD_vV_>0fa{zSg?1OT*1}Qtr&i>{y z9pYLRBF$%Q5sPlq`Fc>?#8^$jkc09GE^H&g#Ob07iQ@gHdfC?m|Kfp}&YOR|tS8=^ zmDoiu=gK~3E@>$m$0F|FJ@N7a%_B)``>POD=Wo+~0gG)fnteb%!H8Y~Gf2sl_C!ul zna_M)7thRe4up4;5AvBO2^UhnIW0RbGn;K%%HjnboC8Y=b;c!2l~~+qi`dA3Rk?u@ z7?}<3j33O4Y?&%7Yck!wOV0OBxCb;+I+Uj?@Ol<&4Sm6-ou)Z!cMV!JT(U zwvG>&BKCz;afriDS4%7|Ef%Y|xzw43JM=;9+i7Bz*(WqSY7%PglLe2|)?!VS-=js~ z)7iQKRjPJd_;Q?E(sq=#m`1v!`(7t_@(lq>m68q|+tTGD%z=nv73+xIoFzxrJFlAu zOnIs0WQOcdmer<|;ODcKj6N+!vORebiy5Xv62)J>y?U925_6KpWZn}G5-_I3@Ezhe z^3zp?Vru=$>C*pCh)vU8lO+TJla4v6`)C zB?ag{yRccYv#IlpP!`P;dx>*dY|rndy1s5@+P7V<$Qdl#VpGKwV9r_b8Xp|QdIWhz zW-s}$+EGg$i;>xH&A6t-U3=tHtxS?g#G>t`Q{sK@1SlLx`d@^cVmad9X>yoWWU1>U%u?=*q$y z8Cv&`2$ityr&^tMpJXd-w+RPuaJ*pvvql9yQ#GK%hmMO~EbOWh_y(~~K+eEGXlkSs zmP9!7$??CLb~{CdW#O=O;@$2h=^dr8>Benx$y?^_YwQY-XD`Slds3AIyBnVgeY9-5@Mx)+!>u&01i%g#i+8w>d=X#}{r1w^GTpgtFY1871U~zz zmfOS^y+`XMHQ@X0r>5@{Sxs-!Mn*I|SO6&K8mrozk_5rsC=0rq*E%)fsf8)>=7$bw zZ8h|#HWi6C+4rEnr~a%I`AqZ`ov^M-F@5UwEi#G?h%(dDL1=eME~#~OpAf5u=gXo8 z2kH_a_1l3}tKFv;y`pCYL1n_HZm2Jx9`=m-?kjL8_d~8MjvcLsP8!G37UlA$?q(BI z`(pHK_lPyRFd!ED*z-JTx{J2)k*yH(hcXs>R^oK7`*<YRv`BI(S@$B(3z{okQblIv6ba#|2FzFmT z_1kxpi+VO!@udh@{f|hOb_kb?I_ce66NO8E%@MCH{#7>*t^Ts+8{PcDGu38{>f8PH zvcPJ0{$F0wA%*DP_R{~7d$2k&tIrP9+9_M`(aHoE%GvhP9sA^@r{`7&ZlDoqz3(v- zJOh2mM!BqRhJ~h{E;R^MnQ}Pq;h>8$5(auWTHcb873wY!JMgG%dF@kR0d5Os(OrPS zd|m_FE{HOYiDL^@&1Wy0U&|evU}>uxrf#CVu6dj4uW2B2ubj5QWPi~}1IPEuhgmPY zNSmVyV!w-&4?kF6g{2x#`RwJ*I=D~1I7%}mw2|`tPIpkxqx!RkSWI}~iaU@YoWg;T zAf!Z#PZ{J@8ekAw%#q#yoS*~7lo(Vj>s0tduJqf>Um5Q%>@-s&ML`H(opuXZ4pTfd zVcBbY|UofKICE%JO$}-G)cs(+d%sm9|Aw&$@-!qY7sSwfc3B$^PX1DglR9e|f#j z5PEe(toqq@k#CPXixcju3Z?vZ(PKN^{O>R5_z0-pvS{*VH~;hLA@w25zUo4N8v7D@3)zNnRu!#OI73se;P?G;_a+>u>R=y@fj&~|txIc9c+LYoq! z^4&h0^&*02F%nGA+qYU-+<30227$fC#CN7^SC%NGD^^Z*TX2|zKa^d4_KKnkx5d{j zR3Bz|tSF0hr-dEURbWh&kN3G#|9zt-pgx38YPr+ypRbXIsS+z+Z12t^p&|b@A+_@P zCT{LKZPg#94_Z02i93Rk<3+U9JXYqOawoH5gjh@SSW%KHJ8`@Lg8JF^$|GCc1suJ{ ze9H1Zd*#RB?t-qd)r6uicFCtdN(;pYzzWvMsXGd^a(Noqv3z%E`Eb3or`@n(bvHTh zE0gAn620w}&5LB5_h#%?6);ahUN!`QA>JTMSd0_EZuw_7Y)B|n&cvd+1iH^~*DgMz+_C;YzR zs28AAXF!R#LOHwe4&ARC9`_14dr6_{7}d|Vi~oL3?A?8J&A?}`s(U7QjO{0kqeR2B zd3$GK)o)@Z>DNZ97FGPqI+?*VLHSl&toYpsndhC^M6_+M8pC&v?6+k|;Rlc0d^wp< z0mV&~XZq(ly?NIQPzs#iekL{7N#rbkC4``I1cPPU^Bp6q&>00iJf+&46MN3(Idy!n2{hp3> z&0~!(;0oT4ia|L|Slc|)T~>I9&SDkRNVgWbop|yR3C8&?XHMkxr{yWFnLPU^2mK=nauv$JFY?Y0uJ5GXP9khAK%d3(VSI7l8#))mWi zY$SP$Sa)wh@Oo0oCBamO)Vip2ahLa~9egP9aHR7PbJM)gKwy3RYPXodL|v+& zF(v-C!|g|&O~tfIL(i?_{Ef{e{!rBr`&pLTuXwN;y{b}F2^0pore=kOps$bY=%zh% z&0fL!m6zSZdDp4&x{^edvR&2*a0ICWbeR72vh@XrCo_>^pr4EnKFfkTpp-t87PvDH>KmJ`FQmI+2CR5&Robq9a*Igdc=kmI{2-J}weD_XUbP3U-`;Th zUbpd;xAbx;C4Bb!uPVBK!P*v@n$B@Oxo&=WXP*pU4@>}$(}C`9bB~V_%HdS`F0%U; zvdwp9*i%<)d-hECb(E;21UOWQbKQSsR-l+_vFk5HcmMd>bJ80k5}X^IiQ32kWx@^h z_gcSld4oR;jtvF1T>H*o^g}8B2op@5=B`5=XK5lTu{b*suYCHrOpG~t_7u0ZexQKXVqeJvmPEkdm~>D z$qoPACLFNG?~&pjE`U%){g9HI=??EPNlEYsH?I>{emo~xrBREBK;KvH9A@uTmB`ao zI7ff^%j4>JmHNTs^=8gVhNFm7Kg5aA%#CLYD+n+o!DDriSngc`(AsXgF2aq^KBJ9{6u-Tx-BCB?p$CBARoT?2 zmg6ImFTylWh190O@oopw@+x3Df;RqiR-B9bNzbAY4Uakd+^*ybucQP7s&qIl9wwQG zvU-^Uw(0RL&S7q}gF*c$G5(x$xJvgw#$ubY0?y${=QJ==qVECcz_r1@>AQ_?UfrMc z9}PNK zSF!7ho}e3{(dC;a&hxaFt8RF9Vn=sLv9`Ez<)+=qVj7#m)K|J7<@FRXjdXOFPjyNV zK8ttW;?w965L?;Bd5hiwo2o|Wv!5Lk=5{@`LZ3p?HhuQ-FtAE2jT7bP5R``}1AFDQFlfH&Ej3Y*%oE22dunpY4<((pX`EU^?k%B{@WH zd{tW=a8wy~+@sLKnvl+b>idV zZ}q1Dg3sREn)5-tA_7At$+kDghlyu6{SyQS`uOe5&vtNQEBv|yIH>Y{u{-6~hU!wE z3M)$ND|BPG>_Vb9cG&!5H#eU4^8Xln*SM;R?|&Rc#Vd+p;uS+fK~#)P#qjKz%~2G* zu|*?MQHs=}BrP>k5k*Chn%9IvQMsrnN@Q7H&fdq;C@eGzv9$1$W~HH+rk3`9ubF)~ zyuSS&{2%NGXPud~-s`ew&t=Vw3CybQeEhIO!v$<}Q6+@b&-pG*{}Bh10D~#zQiV?E zTS9_KIE{>M%qfvgC62S9jAKC;^`*sjy8GDAVQ&*)%fQ_^I@u9~TXWzO*^KX8=;i2x zVG~fwSl_7~a|FF7hmiuu&d~*qAnfFda=LP7-%G|IER&L05H)otZio67NKLV-x-;44 z=(Ipw(?q49oaLC(Y4z8F2TFmkvCt80#SsQhd+cQH>{}#XUj|sZ0$+H5fWV)il5!k!4C;_Hcb=T%W zj)334a=i_^n`St+4z>%X{sev3Yl+MgiA$s{9#!2nA9;=#0+u;_yDKHmD8Xz`3A`_v zE~=UXZ;GKk=jR^b?>K2BqOldvb(@mf`u z=r|(+P-5>%$7^%iiW2$4Q9sV+%ZFSRpXi_HMc%zS($V7UJmDx|)7^7JjA--N)l@R_ z?wNMh18aPQ1Cl^ElHq8vw1;rRRkHf-6*a6F3n67F0gX{2Cxb;`m_XXP5s%!ao{p&_ zUJ(0$mH#&Zoh#PpJ6b%T7SpQc2G%$%E9K$x?Z_{???)&!6a^^L9{>6%;pSzOr~)ZX zf)x!rmZ7N(huj%{j!`?S#PpA_SgvKh@f(f|Az-jPszg^9*Kn%nZD)g$`x0}sU@wc# zCX_>o^iap}#}?A80+5p~<1Z=C;KNr19zxsP?O_Hbm5_mCiM0hRw&4hpU=sc;cJv*j z3pW71rslQ7#oimd#HvP1a%Jvs6~@0vumcY1%3NE9r3l075D+BO8`q8HP#>DgMk&uL z+0hp%yd(ugtD4s)$@mPr^avPC04eGf#~7RsLrj4bC1US8hJ6v{e8oWM=b;}!!va?N;j=cKcD0j8|L_!qAjRgq#=%@i~!F*?D~IzVQa3FY##{2Zx1a-5A)C^7Pqk&7ejfk6gc-XMSb z0}LcmL69Yert2-Sm;wy)CCrN)YrKQQJmxx0m=_aaLNH8#Sm~F%@5efphR+hg-%+T4 z0sWwzRl@35gyf4HQ`k=}W(Z)R}=8|$T#<-VMCYTE4^1Ti_Uii#cwEU9CEicCl-;Z^M19%)R zXDhInl3M4vBhhRH&i+AtIpEE26=YnSWf3M6ObV6hs(>MVbl{gSbBqS~G`e_H z2eBR)YGfklB#z#gr$bvf9Lr$BFZn$^jZ2sgYYANt>zbP1Hqqi2e6H;jY0#%0l8OYfb!Ed7|%+IwumSO{06~v+x2y;@5k1#0^f8OQoF>Bop2C5>a1)gDS z*=P5uz64hA^bunrj`Rm88Tc)WAP_zVZ=z#dczgjbLlTYo97V2&I6T~3C|cfa79 z5aYm4AG#b>@N>Aa04b~aPMiL>#5eQxWf(=1SipyrOAn39a>|vA1j6Nq#xJCC+_Ib4sB!M$eQ6%9%mP3sq&4&h$71)!y~g(>)|khiJS zoc&~2{ona|8n#lye3I#&7V{lV;0Dx}NcRjZFz#dgVgo@S-Sc#saX+h%lVDZ%-~zU2 z%xxjQOknq8cmu4P5>u(%qe9*Qd*4Ew-`hQHF0kcT^Q00{V&sm;?d2U%H=AU(Gy z@ra`(a%nT2;jm}S5w;XBVH4nh6o`EWCz7Lc)g8K|3c`kDwgwCSBo;P7iA_GnDMMa> zA_xJ&H*456(}d6%RkFN|;_bbtkvE~QtknnO)W~)S2U%s`5l1W60u`#P z!$||U><1D|9P5kN3god}1=1=Cm_2I`v)X6-3Fs&_Qr+7q!1%KNV8sp4AqgdL9Bzj( z^|spYx$w|Pz;Vg>;9(QG+{$uv2(c^e;*D1Vv z$9oxLZnhPEfuT`l)I;M5B$Pp<)ON3yW0cgNQQWI;Qr^czCRCRZv^P3i`2+(A9aW-8 zxxL8|Mme@35DxGGVL^&fo_J4WGf2Xm3%mu^=ZFU$lvs41?ZORYHV}XVgp?xgar!Uk z8wFsC!?*{MIALJ&3kXYmK#&v6sD>&_FL4i?xKskMlCU9&d-NYHctnX3MGv_L66;|c z84i0_S8xwpLQRCSIv!v8h|ikjv96l!zb{u4B;n|6hxhec;sruAh?8Q!uS|R$$ z0@z+`L}`YNY;b`YL1-KgPpG$Ve(&k#ywxL_rj;*%RRr*+BNOT|z0LG=+*G62g+pa6 z_Y7D`z)Y*xRVYWPBwL; zklNQTR$qs%w;NO+IeB1GErhSPVFT|S`rwJ3TuWu#}a3J>?5R8fgXj047W5TUlRxUh%JNKlc5Hfc(?G|63=JMhR`tda*tG<0}W9)(fGj&@1Q zL`cCVDDh+s8!S(omq-i4eAy68W0Nj60#?Yp*ifXdTyFn6OSH=|K{h?g+7LL?fLF9AFH>u2{=Pe1KghEWPgNii|rQCBPKA zobcAqt>D7yvI=gdbz3e^D3kVh6y{ibTHZ4PEMWR^7TbyGV-z=W{1olzhHF*xVdOHQ z+`08^r#V{;9*}Zxt0NG{Yt2Q&CTzOD$1z7AOsT~KhlF08#k7q5POGEJ^}9^_d#*Fa zAolxBY!_yWVLsWQ6#lT4?ZS~%s4v@(!n?uxK`iM(G3mMep}~$c9IytY64?HB{toZ1 zTSzQ*(KU5{`}>Y|9W;UCs__0TFZKr-CE_9MvA?UYqdoSmK?=1HsM2J;l7ea5RB1vG z0?Hjd5k)?jUvEGL+TT0c=r6;`3OGurq2K>RC~FE`U<(szV4{6Rq>n&~ z65!FllC}I$-cLY+4gsE zC?fPA1EEBiJ3EW9DSnt@!|t=QPwpb+=%Et-AzY^tf``OmO<>QX?*MDj17Ow&_AAE7 zsjvqmfiOOXw|-jQDr*zaAM@6)18F|0NFJ{Qvjym};a~YS>|dU1L}4-ms>^WLzr5HO zjm6k1AeBsCs^wi?$tT#P9FV>`n|F~3hcc4)FTKu|m{<8gLExC@$-8WcafQ_W%w%H> zCJj|Sex;85aNFf>~v1wZw4s&7>fbje`p=IpsF@#7}899P5= z4oCshYvHUF=08ZeR5Ia#Ub7t$*myw%lmg+Yt%{06L`{pK%ALgNIMgV}Q`0Up57B$(ykBaRX>k`MSNI)a|BxA*=)`@_nJji=~e8T<#j`6)f` zoOJ2}9Nog073bvur93hAK(7#`2e$0yqdi46g8qLA-LENr81EBwS>pfda$v||Wd^c; z;5F%}12eK6UP$-23IkPH=h+~2qsY4P7dBlMqMtw&=pw`6zz%=If#cSrfCKRX9^12-0;>}afP-=$yF2kO(P2`cpi64jmHwFg%RweUl?P#LEEembzVu5`i#Rq5Ggl~1k5w)5^k9>bEJy^1 zCguFueJoGvz=3Kg(QB-6`C^o7y0oZO7H@$sA=v{8qDsf>Y!tTcKy_)tqRu&NCytdM zrr?1Rz2bDz6rLj-VCr9?e1UEt@sK5OtQ8i@oE;KW=@M-mMFvqmsznLqo~*E6#N0S| zm`I;YW7DvPgZk1QMT5_X5-4tJi-QH?%`KUz2C)N^S^SKLF3%M`9mY<>gd_#Jpv0&` z)&qBQlt~^SumaW^%HdxV$M7(H0FoDo;3kxdEqC;LL8T>5G6A4RR_gsQS|jD)3L+fS z8zKV$KTKd_{PZAsRwofhszt*qSgd)of-aa2f5;{xZ|im)kLY>I`ny*}z(}!Ah4VLB z)QT)Xk^q6R1CDZZ%w#6)gQV^R}!8Tz=0I*G_>pYBN?A6EzQ=0(xA1Q_Y%3-wpZguU) z{?7V4*yT%Aq?EnB%I_asbD9uR@?+VS_v+n%b49V=Eajw$7$ucP;$*IcvK zMW5#?=g^+s1`b)}%VYXlbb6_ZG|-7yHW0fwQ{kwNLdxmctS8?6k~;5~cV^wN#0ix2 zAlCB}^^4{ViG=)nqi~;?JV!am2aoxc*RWPmcW$mz_ z>jG8@zo6R#P$sZT-oxHBx7wo$cpNQKZaCIErGW4st6yGk;J$g+;jTs30$5`l2%^`; z`b{_HgU~i(3`p+=Gr!w!31F}qnBJbx8kl#bg$Tl5>8u&TvJARJ2{$}+XH9VQBJ63t zK>wG=!A5r0u>MHA=zb{ggGC*pSYx~% zz#zSU@Tm;P0!$85H~@=PJ=nUIHNoeEbP>Ki*wT|-z)^#+mW+>sZ920I3`J2~y5Zns zN0|G9cPU^@vE!21FpNE_oL{knU1QkS!6I9Zm_kY*w*ARti=_Godl$2(;{ph!RR3VV zFy@ZlIPnk`JE){kMuZp?0WEZyUf@_cCf^nO2l1WrM8M%+6S|KZpLgC)8v~ge#mB+% z1;%d{fBIL3!@;k89J|jg5+5AlsDtCP*g%}c*bzMNKne~J9*EZi^-Tk*;^-3A8}_(L zAB%(k(Kar{as>TT;(=-?@mw{tAi1T)`6B<9urR{-9sB2WFa>;6Syp6(;RSLbU7XEU z;3G~vQiU#WU1Oavkx6a<0aaEXVY6`vPd=2BB?__Y2n`!mfn8BzZ8@8U#HA1r+yoD- znonCOPXvt;B|h+Uyl4|cNYmgbvUBa?Fvo zM&3>-4fJgg`#g7+Xr&~uU#=LBVbY$I!vk$?(Z?@gj|4%Nz^t|_u@CDlP9J;}##$nA8ud*h|2vkoT)|z<#UXbe))EW4q+A)jK!>_tbG%~C zNyB>}xHEQ48R>G=A_a|hlz6xjKV;w@YI}+|!0T}vy5IqiHfMN) zLpR8L^r-3~T%5QXdE6b%(9?A^NZNWHFTM*bF-yd8b;6vlR!t(Lmhqa_%4voj0 zB}T6#MVRo=6n7<|e(KvH9Nb$NxsQrVM;+Q@cRU-PB>FI@2IxB@9Ahw9-UK)R41~45 zY>{~s1FC_r?V4VKTe{t(*pcd?Jexfatpg#IRrYMv`{GUv6esmjqM%mqhs>RhuqjG` z$0w(hckva=cfPlWzQ5u~`{HY=VwTX)u+zxvONC7()2|ZsA&}tWQQlb(!|oR1fl^>v zenx-#NVMPqJ|NV@IpSLm6}m{V=kK!AcgKoe4&bB2PbtcKuI~%{f~&JKv1S^h<5#IfanD@GW4KrXKj@tS!6RLvd)~>nlF(w4(i~I2 zL9iIK9jO+#DIT_M{B_p^K=B}EB%=$5;)De}cs4s@JaV-`#Vo=22y?KBkP;%Kuo&Z` zbBib>!@v|(#^$i|xCakE$k0|0jhAp@rPpS!F)s}F?(tp;f7`%U9n5WZq$ zQ#>HFV65A%fnv}sw0+-qlcNvrp-1`G! z*af_V042q~KP1MW{%IL_$PyC*l)2`*X_Tlk&0qQ6>=&kzw(n01VZYnZTrW6@}Vn5Ja=k)RW+rpLfw;GF{ zfz&oT!8kNvvCu_I$@VptYdKCgK+5(=5Lyc8O&xBaR+842VjA={U6{2-9+*dbum|SZ=|Z`Z#szE;raK@-27gH_cXqX(x3Gu6 zQ4(>+h(&f7j3fP0;$O%vBgV);vP58_@;73gc;KS7oon?w&WSR@-Oc6Ybo0gl&~fY`!rnc2LgN$h9?d)FkeU(0}n_U5jPB5 zBAhJ@2-bpOrR9SNrHS-K#)>efgyQt5YRPPG)wjUW0sUQe9lwf~^ zC-x2J<>N;Qq@WrIw~w)x5mN0yC_^JXY^j%*avB3(8|*>q58Jn| zkd4R9fy6@)j@&uP{>I*WXe*cw)1S^}e`CKXRUxEdm;c#nY&|yGX)(=$>tc*P*xE%% zNvx%LlM3Y-EQ7h3pJ6sl$|k_A(xg+=texYC51W`;M+jz3m&_1 zg4IQI;Q(Fo*o}L@^UWnaQKbkQrV0=rb=rP4&(oL)SE4>KjkbK9?rTgmzxyE}A<>rK zB7JznS4#<))b{HqYL$*Fya{{I zT}gU?JWpDxzYD7~cd;X+#Ppo<ZBDMliol*XEVn60DHmHw|QEeQgg> zq>s_?UNe_2{jrndy15JnJ{lW5J;LyTa;8nKY)H1dF$v=ah!hPQW1CZyG|a*Qy3}^m zhuJIuX%>(|kClx%dtLbpqXO`dV#m0-8KV(n#1!B(HWt2(vd$FRW)fj?s0RzfLaHfN zV{yIj8qZ-Hl4(;bdjTh=%-htOb%$?qNJ>1c9s|EXq{Q(&+OeCM{sBX(rn1?8)Uaoe zssW2ju}Rn}H#hfz;GqzYq?{yn9wR;*VjZHbZ0^BQKI{_@N}`aCdad_ecDLR@g1x@+ z6uXBrnTRQ*Xlx!|9~Ql+H)$%G3$_$_#oJ^?H-XdGvWxCY+O^J3ZLMs@3_sQvy{gS``gbW0RkeWa;pa{Hd1kLFZ7U1Z4Z+z=aKybh7!lR zsw~Snxo#?T*;pHEEJgAYl@Lrfuc)wun{R{Ac1sH%%P7~ht;V+bZc;K{?x5gbdJ#32 z*>Ju7uX!SdfS!%J=6k^VC?QRVtR3!-WaBy<+uNQpRxRpX_l;uPk>xtF?=oRiVW6E| z0@(FGrZ@veV>xXGv7}OY9*QjSO^z`VTQb3v9xKbecGnn-WDWv`M`L+k*6N#)%waQI zZMFwV2i^X+P6U5PiqkzmC9!TYZCx6>p`HyfaohYQK-24Z?k|_*e4SUzNo& zGdj7{->>RZ{tiWI#e z)UV@lC@Iu_2MNn{zz%&Jq;xlTlnIL+dbQL@LW_$Z5_<86*-97l)*TTa@4pvdOu+V1 zQ;Nn)-f3=sh_5=~zyreO6#d88%c7Bnqcry6m$k-7tO8OA7)WCuH4igJ;go<9curLL z5L=lRqW@9H^b;>HdoWI&H9e=Xk3SAFlF(1i2aiO2#b}`#YH(uqH_gFBU#ync}5!k#Sa+{XMRstR@BR< zW97}m*!bWZ!ckEezp$!K^VrC9i^Mwz_!>L68Y^dL40NXn&)th=z0vZ6AUapk!^GHm%(#n2V5X~<3c}8Ua2zy+@>v1qjjF7~X3H#x< zLZc~;2S$BqkBejDj8SklAr;~9$+pexvdgnq8D zn5gp0FU*pQgduVRJg2d1841c9=#dWscz}R|O!uvi6IIHH>93z>=nV$GNj*ntvELf| z9{(`g-Np1qMG9|QB0gy{q;Age<38}JFD?3nol5;$(Q%VEc@_nlT(Sq>;4v{Ib@53d^;q zFRR?FykbAK;9X%LA@+7-oTX--ydc!SjWhngLAE}ANSofq0Xh}Pw(J@IylX>#eQqC9J%IAeH-TPiOWL=GaJV2|4OsK)LcHN~AhV{Da{4Hv9ZQtFeD@ z`$s!$5ziBU>rM zFo6t&N8@e}JoIXev&kNU&|rbx=!9)I?EpOl+H!Y4Kjn=TToA5OmBzasvgKIgO9T&5 zqKVgawhS%IZB!xUcoUy!B@gQ3U!fdt(yK_XLZbwq2v>5C-)nU@j0!T%_k}m@8)^*0 z@+6Q7tMg`PpmXJfyRZjup69FofS@C*!zNbV;>lA+6nX~0A(?s>d9%x&L!7a$anG`N z7K)o#qnu1V8-*Krf6W(tDS#Wv+vpdJk(kDIb6TwJf@sTT*TCD#+d2GLHHJK7AfcSM zXBw+Uj~><{##Y|8c(E}P-+fZ94DT)XE=@4{j!B~*1kB3Y-yCGwTCabK>!!RcyU_b2 z9>8Jc9S#&IA+7`}jd#eoaDrwPps7^f2a8jA=7q^%I*$AN;eDbpR-32<2paE*F3Sdl z19YLs$~(v7=5uUw#lO-GUFJ9Eo#ER}m=0S(uIzfPv*L|qbU>G`?6&Hz{sS6msuD%I z_|8gD`pHC4sa!V<-k+E6z7*SiVV2Sj{SOqEp6mXEuQ3#zJmQf`;qW-d9_x@u4o68G zk2g!uFJcUI8}U&>G1mPTX2f94lS;rIs1jOjyo&B(-C7fL?Lj1a4f(`K=-Q(Hq|LU`aeuI6wIO2`6akhwK2oI{zfnz z(6>MtfZ#`Q>D#An6z~Cw2gzE}Vo&{%#s|!Ji6qdY@~0knpu{~F;mM*JBSpEOz{rwE zNacKG06IowAc)rZz@>+I2OMD!r1V(%z^6Rf(*5yHduV*{_9HxC?puOTM!JL!w)0-* zggeAq`Jl@AC&q(moq<9F!+5}3Tg84;paqXk^9%*uVn~qo7<@O|U}$AfU)pr&npz%! z5l*TM2aSh)n9X}!mHi-;()iG1oP&lrr4o2w7--m_=DbH6nQJWxHF3Nrc)$h3)XFWb zaMKlVG=a7-qxS!YFvi=m2YH_mNm8P@MGLX)!6tb8kkBp73ytx1f2TdHoU5~Wz@EEK zzi6C)iJRbmcHI%el?&C%Rt&4JI&mmP-o{XK&Hx6oa<%b&-eINdz+A4jF6AAP2MQd} z$I6-81!E}o6T+1wN#jhvaN-R#235}HB3#A$^6?4r?u!!Po#yc#gPVyM6V49*Xg+@; zYPr+d8V^5>gGd{Rja#yH7~Z3j_d;U;H;^PNkNB%O@3kpM@PJLMe0W8Q{T~>abg_|N zM)WFDmZAwJ0|~JsahiCq53f@FFt|bEg7C>Wm*<9Gk2fMQTmdNQ%7{;@jae9sfFOB{ z{%0%ifrA#nP)dnfbjf%X*~cB7jnbHge#%RjAR-DOlWKywjTKoC@xD(c>M6N z4=^bUU=rzs@m|A_Brn9Gl*T8Z?I^~S34X|TZhFLBo;f^Gq}7Z?Hnl8x|0c zxNecgCisPTM3J^r7G2V}V4|uH`V{{PVgqy013gDFvVcA#CWhIOLjupy z6V2l>(s~B7fmA{;P1v!?XyMAFw(?mGBG^|r7{z%YJD>HLjqUi-$H_zE^FOokH9J0{ zz2>y_M&omjjumGj5L2P;T+H_DKmr7QNF%?sFo<2mUgCV15Ra8FL}o}iS~y4%=LhlB z=jItI2F+FAVWY5Iij6(I^&hULZ|OR3CCc{)@tOiUN^Fl*CgQUODbj>X&u1tdJijD2 zQZ*}2n>62e8U51|*}_=)vdKQiNHb1_`peLaKZ7nDj6;u=FL$pr9>aVgRRIo-uWWi( z&p|`qQ8EQ#vXA~QGVQ5^F!CzveU^?R?Z8lKyK+T^F%~JB_*Vj3nH_2j$5@PblnJK0 zLyQq}6N%)p@@Tpdj&xJAgq5$nzE%GPtANl&I%?(l&c+x(NAyS@>z>)l-x;x5OhUq@ z8ee@k*zm#XN;?1pIE}AKImWt;C=wQfSSw$XU!gz18&xM|jpL#IKXV2i>Z1y-m+yf2 z(>j~lE=B5>kq?9Vghk`Fno|AOqsN4iK~TBvcBTHSYYqj3fBp4e%~f-tweq!1JL^~R zd9^tWwE3Vve{XsZB1j{zAJkll$3TZ|G7a-V&_x2v zc^IlpHK&lkN8>rz!We_WI~dAlE9bx^V}VS5lIrig7j3M@K~E(RE2`wS+{&B1--eEE zWZg+@ep^5NA@X4~^k9dOP4j^x!(4qs6_hBdv3HFcsz3@oR=$V(o!Dmgcb=&d&G-6O z=+#j}SumL5iRSydUa}v>(oYVmpbiMH$Jq4?^C%IFvi?Efu|_DeTp@uTE8myurgwFp zElw!wPEra%@NTr0ic1~`zushwlufWy?og_mu^!jnp*~q&0@Gm_o zubk@5Bff4&|H_c6M4U!w^}eQ+ORs~+I8h~vtX`Rs#UpT9G=7*&(f^2m)nQ=rt;)ZP zeyFd;t|M5B9xJbY;y&MT;t1`2kPXU>M`?WD_<`aDLJ5uka;AU}-`|1Mr^m{F`Q3*P zM^6dBh=<01ZM}(AVB!uxB+_3u<*?i47DP8u0>kP`9Gi$Avcz@IBkbsi^2e|8?*87M3vR^vB{BiOenQ6&we@tXAe%#g=@$o8Y==WFZ>%sZe8sju;y z4gJ`+NGm{n$@FGm82c9M9+`E7zV567-3`+~V7fle-UA}*2)FiCv#;>cBIttYtvfNS z0!wK`fT1;BiyOz!PHHZ^pGw-^{-m0fVk#VBWk}r)skAId@U;9xMN6dAPj;PQz*Gd>bD1-^4yg6gpk0y15@? zI@&_wLC{sV7CCxza{&`bHMP+@jBmUy@1lS{n(FaqI4^X&AiBkZ;AV|6thj^;^@%Zf zRF>-Qt|=rCsw3>Tk(uEP2i5Ivu)U_ZFPsA}Vv zuknq4%VTJTSoeeeiXVa>stez$jq`l@^Oz!l=Y@gP#*wwGE#{M{gaFoLZGn-w@-I;W z27)fny4h(G5>4F+SD77QiRwhaqe zf6T*>a%JS(wn{Yn9lLBuEFP=cw%-xf5%W?MFhXqG1$R$m%y?MWT&QgqS12RRo}i?r^l+cuW8Q4V&b24K?xA<*mUPMHGtF9$3C;^ z9g*lqB_v_xd_~23Tlac&=$N1%M#q`r17_6J4)d;A{4kY8CWL6K+Trz+mdDK@AQVK2 zi^(h;;XoxsoOXDWt5N}NbLjt|)O{z0r%zsu)?STOPnp$9!ywaZ6LpMrNEKPVic)h?BBYyehQY)*@HdvZNngzcrN z5Q|4syFD4p7GWNjs^C%8ZvBoJqp;{_df%#c3ywA-T}c!mydJ~GU~QgwpbB^-iqLP510}n~?#ks|eaWZL;GaS0_Niyun zy=dq3t#6pS{u9QPU`W-hs&9<1J`p1vh^0qUeG{{l5~QucYclA3`8?x?6MKgS}iGT+sjwc%vKlIUgi=^gxcH?dM{GO;4cObD@|19)b=SB=nveBA7c~4-=_X?YZ1j z{~Yr$C6EFmf$(0ur5H#dlB$64VTI+J`Wte4ev1=|u$z=hfR9yuqE{*NM`HkyG8~== zPGY^$fzL+?ltKyLE5;P$(h{&T()Nk5rK}fv(&Pq!v#S32H7q$P)M+tQ_21{g=3sjm zB*-fMXS0-2EIyJhAZn`rrEKLpgf?IkWALaxY>B|7b|e<1G_{xeY-0q{tVjwr@5rw@PsldI!d`Hpne*0v_O_slDHdXW{r*=#bKpnD$r2_r!e@ll1=GOkO#3bZRc^z^L5kdni`0dfG*}oE0drH zUb|pSkuRY{+Nb>`qcO5O2$(3*r>KU%jNSz>kfd>HpM_q`SH?MMtEqiTD)k&7ty3#LurXAgG9s-W6u6#wxz;N+HK=E?y?n_lLKgu&NUdaEkdo?r_k)Jl zu-33TKuKUhxQaB_++inh1nIeKH+FLXsRS1ENtm83_nONpeV_5<5onYkfh4JF-_glN zj2u`=V0~ZR#0qhvG#F9|P3^m+IS)sCl*o6^s`gz!pN9v@;|m}Oglu2-b;A)Z=zWW` zxcW_N;z18ed=SsWv4UrlR-&kn2Vh9PV0Y!hWIij3$_r1qOny zs{Kx;pFm=ptF`R+VXzT^<^0?7H3g4*gIFspgP}Ny9jW$jcZ{_`8-$>hy7Ygil(obT z2oekEn%aN+5u-8uLM2cQgraQL8oNoU1h9Z`JjQ4&7ayq#m|`T}+B-#jUl1cG@nf9* zTVzt#Y1{v125-FaDPkyX8r&_L1>q$mU7|?a;DMLyWf<^6ti%zlrSry78oc<3(Qt93u&ETg+|y`; zBUDnQZ>?(Z@&Ka|Hqw{LaM08NtqPcihK-ypstjl!!y++!t5a?O_IXAkM*y}UqE#Ib z8mn)S8AQZqq&lEFft{$A7Of80bC*w>X@{vj=%T41SmX-)xw}|h5mG`vy21l7dyXGcY)H8q z4-8-H;|0pQc*U;5`y$2$ zHk`+!<;L`7tT>>2)>6jVr1Gl;mGceHuw=qu^Uap;^N5kTF;cOr>w1t}SF7 z-9QwVNC)}FGKSk2yW`CU7EK+*=jj^`O?F0*RULHZK8wa^jkFaw29;l6cC1AwLR(?v z!MI`fQ#7zq&R1;khuQjhnFKDy4#t6XW6ZBQs%h%rlRm5!E`TMAp&AHPLF_W1lP*9D z9GFyzZ&52+ONd5^Un5uz#vUYA@^}=j6EaE<s=)dIgU>-|0TPXqknI7rvjRNL!_A{@C>c5)@q@apbeFkIG*Wql!0i>Gx%&Vn* ze53D#2_aTf!+vSbmcx}%vMn5^4jXqt&#AW#qz;?x!zN>S0re%YVXN*MtFb2$K9R}| zTN`Su3Ed_LAX?R784>ImbL*c_ZdgVF8;LfVa6my#wLIx(^gvRp4Fre=;ps?yEm|r3 zpb}Qq619~b!3l@Z)GYCQywO08U*zjy+18m2L(V3OOE*|BZQE5IZy}@5ax8^)#mRB7 zg~{|_9J_%@CNPz{SXQ56cd(1I#HovAO)a}~x=73@N?_ktvOlnfLrC!e>CFswbH^yB z)iqT~^3^wD{sWpy<&>9>=^JIxnS{;pdLe)ThlH+t8_Mos2R8x310D|wS(;iRv=s(o zZE}r<7y$!=tirnb8Vm65*I8XTyO>Y>JVTTyBVepw1*4V@C8Ryr;u`i1zUp8o)#ttK zdL_nlnbMV3mE((i4t?!96Y@b<*f&?|u4M+(`-k~r94Cc1l@{aE7qbd;eIMX7m9K+X zbXiO(AiV9ysvC+EbtH7Y{|pO2nkR}&ZFy}hD?$rHZU7cdRWVt9&A*!nOJNT+p_2XD zV!E)pBz!a5_%pDOB1pDl;bAe1u}4A|!6STP8asj}*aj)7wC5~*Mi`%n0UlWlk5vsf zvREnxymdIje~qvw<8uP0q@bpT-?7=JA}y&dKCC@)=d^r9O|ifD^Eu|hMkr-ftwHfz z!$egb59{lp?9=TcA4Y0xEr`?C<0>Bl23=5Pe>}^L+(#&-O^3gp%vNE|0t^YMrVjro zj%8pbuLRIh4TR(FECZ>pR6>Xye)1Um6icm9U<%8l#OH3j8PcQb2qW4gv7G4<&N#KI zBSyO$tDEf?Mg|W}9T9Pt<>4H7)R#C$U^3$qd~k>$Y&rthL+4>W85m4Lj62KgC_Y#O zA3UZevRurtqKd>ZBEB=r{Y%7HSQ$Yd5pQR?vbm5Z9I-e}v0=nlh2RHTl-N|Md?8cL zOeq^8ly%j0&{cJWo?%?Xs}4}6*a$bCh&eVgkmQkjO<9jv#}8@35x+e&zJH}gl#sgo z4kxJmPsu2%EbsP4J zfgwFsb>!0(Y%&f)M{()fk$c=&*zf7i5~@0?vyYLAy@0T|RBqIg(qZpms9{>ns*bvO zLGKI2beK@sbX4tny_b2YyMP|q*qyCHnwHH8eKamS^~41z5KG^FO^w9N_q#|&M{$WX z5|h;HaDhQbXR95#+K)}Z=^jKVBdbT+*iVkQg^!Lx~`k9#tLPypVk& zZ*iAF7j=FRk9ul{tA!aoAzWXB-WJ6=s(=SJ_MQCvEQyq2M@K}nPjJ2iOlT6GgOq<7 z*DEo#hJAc@g76DUfycBW_HoQFE?q{yG?smg3xiD0p~OXBV>L$eC{B-69la-)oy6CJ zO5g$XW8r-2ta7Imt2*YnSYssS*j!aU&*f9I`jBGufXCNysA6%30|;jm_*9&aiz-t6 zF*64lBVc4`D&u4Hx4wMp^WEv?p$Aod^WanE)Si5!#>_7@M%3HLrjEWjh)-R0ww{!m z7x>gI+w0-D=f+yJeF`v5hk+2x5Y3i68mz3wQQbfL$Odp<9;?UB;z*7COPuyoZK72%g z`l#}IC7t=O`LYTKUghiq**}#ysKkr-ie)-OdtB$S?8S;t zUG^AT8_OfH7ELAaK_rCMfdd&(73<6m3PZ1TG=Z90BkjdB3P zCgLH1ZA)S=VxA5Jscp1>jUI{&04gDvM)ywAhhP~DKcusxlY{j1`lp;l=f<;{NRESI zW{G=)48Lbai{U-YsHqyd*00NjAu3^2C-e)pZ^n0D$9pSqObEKBD{`GswjUG1F6qPS zB`m8GX3S$rm|%t!v&uWA`fu{=Faigr*_ZTmbkD(1;+TNG)LQiUsR|xVov?f>TZ<&{ zg*ax^pMXtR)tPPlwOq7?-~qz_Li8~h^5BQbS#2 z5H?k^kT)g}1{n?$x4E$pykf8?NkWOoBlwG0N&`dbxrvh)Pn5T*nN=>F($`}Nyu=wF z6Orn*4jZ?KAgn&g%}rm2jgBrJSWH2pjw>7{{#MP_!M8vu#ZC$gQx0Gi5Ckc9QeR)? zAchHbR-ZJjRM~~S)pbZG4RmK~k!L`<;89gUK+u^MQ`Lz-r1159#f>>6T2m*Ts9`52 z%R>@`k)Q2_(+RKMb0P(g8_Ddlx%vjC<3Mm{mp_|Hs?(#Y&u%PaRxB?NFes?0&whK- zvJ4S)8|q&rtH(AfwxpUzatmP3e&NUd!e<^oB#visy4Y7Z@t{t**mjjHZc%SqDp7P@LVm${fL z{`W+DbR=z`+k2Q>ew2BifUm00?cd5RIJ^@`Nn@)zc}yCc4ZjdURGGZGnzc7i;skt6 zoxE)jd;90^0(z7nWIbdXWCDaV;pB^4rETH`W2G#9(L= zdM7E*)}LUG3y4sr;#Ic+2OjXiK^&9npPn5zwSql{>>nxt=$blpp9c@Tn@t4Cbn*f_ zU+wWiJ+WJSX%{CIm&(N*KdHQj8x2}Ic})Ia2w&j-xUiUDIt5+Yx3R>DACl>mmf>uh znPmbKTGc77{aNmBw}n^`G0QJez281h8puWGmB<9R^#-64Ta(D%;G&dRb!a zeI*N0Odddr6B09VJv1ax73lKNosY(be4EqR(>_RHZ=aW!luKPOh4gk*z2$}YNzIK+ z9DYLvLN!f|?^eUwV{n%0Z0O_t(^z|SH^>brVO8Uo6zP>1H&B&C8k@#{QfOr2lSBl7 z1CNicC_iEG0RNiy_}?+U>g+%fFhHuQ)5n(THRkdp;9J${MZs)!6Yi=q{m15P6-GGt zm;9osGm4HV$>wUc;4$O-9OZxbs#6I;Nbrm_o`hI}F0`F-HIcPP71LtiaW$CD#Bwc_ zkWA48&o&Qc1rJr7fsfz9cc8K1=R8BOKAZ@>azTki9e>m8hd3#Laam5;FM1yB?c zRyEjSKh+CWkkI2+Irha$WG)NoV^v@DiDvV$gawh(l`mT2*d#Nf3d(8fi?0^xv(bekivb4_n(MP%NByYk zi)(V&*8BAiG6L}vcFr{C0qM=h4Rh{{4hH@K8tt1vDtm@3vyX-icEr60z zW)1hYm!pec1tUWrO`V1Lk7JE}MXZCUsk0YE+b@|DM6fGLxLx3@u)1P{8w6qYD>W6{&zmdp|B4csB=%KxELZmL5d?wsm2QdLhHo|)%AlL$@6Fz< zp9?n!YxCL2J$IEDTFVx|`v65&)6_Z7r&unSr_NRh0?T4=Ak~XXl#xB=@KbC)j-zu` z!OYd0^{*$GGd_W>#i?&lo&16-@wse)dHNiPR&@@vO)(d^QR4qa$OvL7?YEOiauiBr zEN15<2I;vu8CmQcPR<7odLZRq5_=WNf+Q9X2!D@duV8t>B!I_%3AOAMbKfhZXlk-& zB%6%B0~ts#O@90^TZ{f_9gbwnCbkaK9d%-pN88w%>GDQ;p>1+pF`Hwi&q=1!gV=h6 zHljz`BYDCZwlVz>{g5_IP6%N(WHOOh5o5`*UTht{gj7Z1m{iKvHR~jBWRkYY&j&D@ z{Ug_locy9cTZhj)@sK!>9kvcbb6_x~V1awBJRZj^@v1*tjTuFhkc3yiIK|g?O?GMv zDJx6anxg!Ag!P-)2IR8ArY6(*SJ(!O3G^nV)NIa{|FyRBlv~)NIRxGpJYMrZ z!mOCIrz*lgujLlA?&z-rm?@=OHeVAXQ{Pf4=#pQ?TNx~po@JG}&unE~u^;(roz=^u z`I?LJa96=}?%Nlbj}$4H&OLL9J%tI?I$(2aAM&}+APUr3OV+%0gr4mrzs?N=E z<8StS;F?33Hz2r1$6kWap`KMMsgOsOsyhtL+AMl%c-lF|TC+UsfXz1A@ky zI`>KpU-hM&#TUUp_rXKH>V1(HOiVR(UawNV>cx5!%x~b7)pwDtPrx8WQ|HgLGY>e5 zaNx133-*WeAsf7e2~kQ@7nEl4!55CZ1|th@gz|86L%$IF=FwV~WlmK9n5r%uRLWO! zKiAjv&EYkCu>0R4q=Zf1Y?{PpFKFg`O*Qq+6%~9Ly0m2grW3G~VcC52(7CQEDQqzx z4JlAeDwkq!&SxFEA^PipuBnUM$MUz{xatb+#XE{vj+P^ogBdk-ahtn5uJb!$Tq&w7 z3C6AElbg7tESYkjFGgqsB~{bZ)UDOZQlv68fWiL`kzq{pf*9Xp@DQ230%Axfm4s^)K> z6}N|(Oy4ZzFU;5Jhm5hM4}BRf8Wl)|8&Yqb=m?T;`%yUpjW*E9`tpR;~B=On0V+Dq7n zgAKWWP3}aMq)U7qcKMUV1Yy}uFW$e|9f4G6yKMI{-XDvPghPm3hMb9h*xHo|dkA35 zz3*}ykfH!K@zK=f&!+ICMajbI&_`3#E3UB&Oqrm*RBna4Kiee%d*L!rHD%zAduT;>o=SIr#6+jta-K{J4%O2$cc2J;^PMvtnlnitJmW62jdWOI?x z`JE*K`#y~G4JH97dg?&lbn*@SK5ub{Uuq94lt8ZK~f^dH=FtF~}%A~H| z@5Z}*-$cBGLjBc75cmE5C(=bqS@Ttmq2ts8HyKia>YAao+=|;wuEM5Ztf^};zHOxb zEqFwcD>qC`WO+CM$E-3|-LSNPPh8?Jj4ZU>aMqniq{KNz7P@!)}{^aGP*!yB2N$~s%(u?awlLbECre~1xyI(EavccpH8peeVI>qv@u^T>_$4#_ zlCcrX3pNCd0G9b_ih;w#2$+PP)g;YmgVY2aRfK`EIvru%a9b836+E(h<5*Wr+)<1P zDOqoL9Nmvbnv@ev-yN1{WVcyDKP1z4PhTVcm%y^ScpA%5f(U|XcDGZ;GVIiW zktL4ouqNDJ2_;_4j5xbZ(c^)a+FkjA_^(1qm{wve#xAFQN-a%0VBL$cd*D z%3Q5O_MS7wMBH(hiYjH~{p`Y0VNWlkUkB1r){l)1GVUouQ4`V`H0M$23{D+6# z$B4$A2PjUDRn7h(+<41O)c{kgx_wxh(FePI^5H0$QB${%tu&VA{41)U1PEJe*a%m` zG6)wclwC;GAgc?DZLdjJcA~Fc=j`qOcq;ADVKsqS)g72&9e^W;8WN>A+N zvB{PWCBg?;{%0<739&n@zKSLHfs==(?ihYVX(7);kp|lFVvf=hDH3HeJ_f2eO{4Yq zuuG3{0Gw6LY38e!#9kJ@MJY|qX;*4l^K0F@7zmFiSiZ;YlyxS|@vT&rU=f3mf`_W+ z6h$hl&DDEhpd5XxGJNpQq6(@(%9V1Z@1HxI1XQ_o-M-D7b%OUGCESf|!5(iNOaTsr zJ6`s+t{y!I|AgBQnrGUh%1Cu*XJ%h@eUH;~nz}Qj*4T@DZQ=o|S=F6S#@hG)*g%vJ zICc&#wLgxvHE1fsVdwI23(N$Ur35DVKX@u}-(i69GsyHTSDXeK;~zfYKzCW{Eiz)R(TzYq80Q zMrh|d4U}i8Rcu&os#7j+TAb0%+}k9W=8ZeUwxd3@C41mD`5E4ZFJ3X|K_g?_Q5bO{EbV=_3fJ zf)aPUjAyah1Psy*`P^H1xwkz21Vr4go}Ohyp>d=*g~sF5`~%mGL!ZrbwjU^QG|uRO znQt?aaTWSswMru}wSg(n;->G`^NqsSVrkAmt^^dcPc{nCJ%`1}NGRcT*!cfAd;2h( zsy6T+0ANJ%<_NtvQX16h$hfQXVShnUqRRDay-~CrMOF2{o0> z>@7-0ROBtg5UG?JPl{5%&suArIp_QMUf1`!et+yg&N}z|z1G@muXV5cE@=0(d5anK zm;aEenqASzSe*tSu0P?f;5^EglsevJs{_ky-axGnWtmeG-Px5kY7fFRES*xaAQGfU zdFYa(r4$~Cy!g(q!c5lthHO)!sU}LT){T{Jd#-lTV4eN?d4*RGkpGnXmA?8^$t<4T z|1DmPC}v0Y%J0{?Irh8K_MkV;tH#*A(2Lz$6hLe}h$zsPoE&&A`UU((Zid`tGp1O< z-Z)zBUg)2i8%f_dHo?{9#I)$fVsHF+Tj}Mz#j139v*nc1q3p6?LJRTcMMq1AFn<)< zS)1x|!@9d8qsPme?F-eN94!{0J`Z)@7FKfykyk;}Fl^s^d6qoh^n25j@h>zu^!w@J zvu05P3T1c_M+e>~y+sw5RavX$zY}jT7Q<>lEdSrKMCVm9+Oyj2z#2x=FmkN=DKD^| zE1Afl=}=arWq$v1w$zZpc8Bwe%H(B9Pr5c@TQ{6vW#<1~-DT};`4<8H_|~WKMUtE4 zw2Kg3b97zl!;FT#jBF#=+g)PanZ$lo3;X5mQ1?Dwg*{fzrAZ* zX*Q`#P~Qr+x_+@*kTXl$)IzK#bHk$O1?%e9rvwU+LvukmN-eBeUpk)sG1S*INXMVB zF|dG`D}LDS@XqB$CPA$3$p+<}jBye;J3PP-zv1S6?i50J5!)*F&KG%sn(RqK!;Y7C zxIzt%u&9@?Vo{Uz6M3hLBq01GBEN+rs1PlcPw&L2}W}H(r<}_MHh_*M8%H&-;lVbAne;U zN2Ue#`|A%&^nbg0vJ`7&r89&Sv>cJ`=PFVIAtf^s6PM zM?Ps~wp-kT7kUSDo#2t-j_GXp=G7D`4eV<`4D2=tOpD zpSNe?=o1Qz3G7n=4*pFkK!)`K580<=zmDPxS^|=>5TUS zOENR%ZPwZe{wg@|%B&<=3tOK?D6ry|9kPn~R5$R10E4Q0hwCROXgRJIdwuX#RUr3kg{=%21b_1+K(kLgsv=^&S?R_n& z@LlSD_V)pOCsF$*BuzP8De;3AH5X|~l4_4ba(-wu8c3Dy@v_h+EY%IIs;_e!`>$6+ zqZKzNOX*_;{&yQ8z3 zCl6c|*poP{YTAlMdK zAL5W42rF3EjLD@B`L9CJI8RGz`iGpCUi|XnT=!F3)6ZJHXZu2TC%Y_I z+$xvW;;6fw3%T0+unkG?HpQ;j*OZ&HD1Av*Sxf?*YW^UUj6Un7oQ$uCZThM7CD^!_ zKv3{Clq3B=&Xjllnjh|CjwrGNt;fh~!jk6fDRI}^%|FX!`0Xd%b(|ETnAIik^O=ES zMgt9|C(G3_6TN=y!qH(Y#MQ~k-W3}TXq#HmN9`$*H!ic)kB#R>T(sM}c~Y64U@ezX z#GM!On%6|D>HaaQYn=NbM_7gaOG`$R)qFiu4!T$_*9}i|UEavrKxiD2F%(ir;$`9Y zkD!b@TjBLs9x47T6K3Ri4ps+TOB<3~)tvs#nfjoipO8#i9OKoy^5p)RPbj_GHV<^a@qe+hdGO9|=l9dL3^tp~SGX zzJ0K~m;l{tBaSL~YNj{1U6vtg3`AyhRUOg6xAu6Xq=Tx-Z6B?iG0-Diw;e+OH!@f^ zJY2fZOFw3`wf1Ki>8qKt}Y( zMtAahjlI~+9|mIfb9wH!go+jd!4s6ni>3rFj_lGp^4OfwUZ0wo#{1A9A^QW9z1|3> z*%}&8-o<$XvG{|nZ-es0-XbsU{SA7rPfsX8c`Bhuo*Zl(t#zTHO|d-5p)dsX6O=jq za#XEbZLl$PLh=-;uupN@crO;i&>?yH{S;Npo{U%n%_-bg(uppm4%vpQ9q#dye zz8sD`ef+fC?_VD@Y@g1WZLU%Sp-~|EzYv_g$aFq=bcZp1nLB;GN__kg&C+)5c{eAj zng{Kr2}BRcoYgbDG4_PlLOe}bvus{Th;1c(?r^gF!*a9`0~wV00g`8N!%oWq;gHN9 zzgm8sW7EGHhy|o?WY1A=xa%h*&(i5?P^KvXd+>y0!3vQ-?2TpHKNenbTq;?gg0F27 zl!dqKP&MTu&B5w&9*q~tbX1){Pz1A`fAP(U?qd`FFuf2OAz8fXjH<~kEQngsm*~+z z5IFtRP6mrfzV`@Y|5SL&SxgCxM{Y9}*sWma<+Oi^TE0Ko8OE}G>4|t%v+MFOm!K@= z2)r&402s)!eTnPomE@uWdMaf7ma*=kjbgAgq`W^`ovFUwTTt3Ec8(h!sti;7guUb&_x z8(mf`pP*_kvG?YIARRyRN3A83!YCnmEqAl3`O8n?hM>IKbV-T6%%Zs(r7jNGowHY{FGn#>#CFEesFBH)rIOj%AUj zYulFLtObyrlw(L%3Ld9gWb#R5o=-#a>AQ>F@0c!1p@BXj`D{m;)TJ;Dwy;D$Z&BZ?TRPW37%rcaT)t7S zZ}nL@UER;L$dq6 z3a|Ffwkne$x_fthcX#qyBPn(b%Ko3@yfL?gvhj*Ka zzDx^t;PyEBX}7Q|gF@hfSA%#5{not-XK;^Q|ZDj*v z?M(O0**acbwxFxzICx~Y`^aE3AKN~5_`W)_j&H&$h>AbgOAgZ>XJxqUiXrg@k{SMO$5zdhxI0FCH&nI1`2hjKUq=5XET67B$ zS|T?y2$91Y=!cZC8%{Dw>joe~^1}_u8!EYcuKO4b!{vMOT0e}oenUup)K*K=%_mQ2FL_BUa8hAqT+obfs@3?_~_2bANO*YHvSIu=OYq8&dpCqmcsCG0h2(EOthRE4YA-VkoPw%G0)FuZ2DC1NflzgOE><-b zmY7ZlYZ~&4RE^h9nJxv5psL$qT4325zZ#+z<$sgh*ZDftJ@fn_Gi?=WqV@JwsOpqN zZRn3ERqv&BlEpQo0)Nb@dT*!6Kz|>G#*k`IbG52RL?-y!1R-_7>~X3_mvrMwq@|%? zlH6~ryI6?Wsf*P;O~046Oe{P>)o9BOul@A(hQoZJ>2`3CYN;Sh#@3);~XbfgVsbu+1fi_OT9Pg~l{KAa%R|C`=AZ}tThGRXiHG(DTtw$z>xNd`ZXxu(>NT%gO`aj6 zDa)+=f*C`q&Emd2R$r} zwq)Tg;(A9Zg3&l!wdH*~-k+r5;;8dzw4=7*LaUq>jtu^!`l%pwnxqRxq#9nU#|9# zNkMZ|$2}z_ncQjCk^%(;l}K*kaJowUPHg|29sa5DKCFRgoaUY(;!d-*a?IJ{o_Y95 zv)bCLCGFC>T(%NQTm^9{pXRkjchc`lYoM-sa;0IUQjMd!yqf6MrRcIo!LHOA;Qev| zM--*F0pUWbYsEVEK3>3DQVUUDEOC4UXb8|kDmA`Dnm4&3V#02hp9p;5pC%YdFaI{P zv^jAo+H+-^Xd0Vt3GC(gL}_4sny_(=Hn4sim45AH)tG?x0=R$%#IG~F)~x8)*l`i$tpv&~pLU0pQyfr#hMt45H% zveJv0a@>e+Q2Ji5q7>>8<^F$^C&)OHB8yhquoGjTe(dzVuy0CuREAVvc4~_8+$k&u zw~*>LV1m4G@Mww~2ZLGY&YaXGj`Wb9hd`5gwY1sRHD%N=;3Dkju?LqxE?sij% zECHNzH^yoX;GXXXQ{aV+-bNNwwd);s<^v1MI4oR#c>t) z?p4Mw20|{*^al06T@MW{7!?hAZ1e^V``Mu2EQ%n+>~??SrG-!I0Lo|-`N_XP}6XlZS<}guru;HaX`5y`wb%)XdxBM%kye`d(3EnH8l^4OT0l`wMRK? zpn>zcd({aC!lqWafw#tc1IXQz&j7M)55#zbK7KMX_OpUXUS?;TWW^92bj5V^Gz1)2t83<59YVdp0yg{cML{J7FPWA>x z#gM2WZL4`h37*uxH7G-1TLyY=mn#+~JGhXh4CW^p1`^sy}yB3LpNRLedB13j(LmXVoyWcY3?QVmqUSPR`u}Ltju<&YP4d0$4?F8#=+pRRnRnHL z7a1C$g;XB3NduB47tZ~r`7g&tn*4O?8Ur*Sc#VmQ{)xw^R9aPC!?H^Bd_aB?*>0I)Mz`R zQJgmskH|Tk5WJ6M5CNfv~sh%_MwSp`rNB^x#sIsukTQM=4ptIo}cRVxNU6YWik5lSZ{RS z`Nqg)x))Nc)EnKaURZQTjlQChH@f$~q8hq)_eQgK0YUvZ>e}OpUJk3RLfdhJYV4*n z-q84|T1t%;Qsd&Ld$mc$(3bD)hPt0#|6{y2lz0aX0gaHFF!ZF9y;A7ZVI9SjtFCXi z&btv;@~?#$dtkD6TcPR^n=6Jz3(~Vyew4{?>ovoU7Fhew>%%I#;;$2vNr+M+w5LCC0 zo#a-KJ)0h6iJsSR%e*C?bNZNZ6zXTIoBqi4?x550!wPoukCom;76P!eqv6)k^Sv90 z5zvAeXWyQ>%xm|Q{Rl8}O#Ww{yyQoA4c3C)_DYgBiIt;9G2LMjZU`mW{KR;5%;7?b^(=42YrmOS0s4g0jI&d`{E<^5Hk~&4g!jUOdn0lbTyfmH^Q)2Jz8+FDf1eh3 zkU%cFgSBbFe>1%?l#1*f?zl74j=JaA{aPM2P(fv~cLTQuO6{tTS~htb-TUaLh;D8A zQ1NN^zSGCF>iVJl&4-oV7+!0O!bBhXc(pf*oIB`Y+0Gg|)9X0q8{;_|L+WAr$4#8g zYpWaeXSJE>UG;ozLlin8HEW=FSNRD+j0qo|R9^Z%!7az|9MXo=V;98qk0llk87!Aa zWQI55)TW3WkDlyMTI{cbfC{Nc9t2{_h_D<%^~lr9ysMc|AgHCK9?o@f-qk0@M!f$h z?}q%>6Ld-l>+DBAp6-oUP(Px6AsaH=ercjZ8ugQ_9=o;DTM%asQmt}Nb9}v$gF`r2 z6FzI(XKpVpec^GlFtTzyv$v>J z7gE?8w)!)_Z7ltd{eH1*pE$bFTf{9rEM{pu+cDR@f8!c+v{aUVz-Yf@wY5mF29?p}JF5l#HMuB?)) z7Tq@AUHWmRF(Dlv3y9atedMIclQRK#!&;bw##4WUa=e!r9&ZE?ulp%hOz3pmYwr{PN<|Sl(VFt31<86wzKQ-7yWmreO^XoeI zeLl``rREe;YwKrtb>6gb(x41iYoCdAKe~ReIa)lWZTjx=RJVM8ZG(cdLTdF#>E5E< zmxW^{Lh6IUZ8C?5e+1DFb!hsmcURWfV^)ra%X@ch@#cSEZ)(y7f?(A{Ky+r<&C5#)M zpxU(~P1U%-UNbQU+WYZw_29lsO%E#6qjL9aTy*^1Y<5{U?0N2tEa2c!bE&3#;ocV0 zWWk)#`iE^ev@S_KEqgV%mE*wDd{zCd$%kU)IB-p+ypd>vjqUt#VDwm7>E~uB(`?U! zICN^aYWBHJjAqzYv`-R`_idH7hhNW-zoODMtHbZ)%b%RzmLeEXA$6D{kY|ayK`}cj zkHoE(KiOqrGSH1db);jioPB4hmeNXkq*IZcU3**)K>axCNQ+fc;^*xF3Y4jf?PMoyD%UjxArT>YlJ!G>tCbPsx<8iC_lB()i)TSXGDU(yE3NTtD7VTuwol z=+A9()MVbbh4{tL__-oW)#zJGJIZQ%^4=Y)#@!~tfpz7{C(C6&Cq+P5N1d$PtZH@Y zAMx!k6EozSYIF6!mc}oXuGu){MbiKp6>~nHDH|X9Q=2cNOgH>G1fm>sXnPoo{XB{F zX+gHW8)fcCsmf{X&R3s`A51b>S9!QQDaB z6pqUBHxO;ZoDQlnuSS08=l@n9=bqWJZTj8Xp4QpF%{&)ait5^gJVAAO`Y;t6v&Kxp zKv5v)T)FD`+kLu3thLzbpX1a8XHpCeAcE?5F7VHoA4PIz3|D`U7;R-%!01v<3--tLv9h_J-OR8wrZf_l zb7VBN{o_nbciD8rY{~l#7nIB9>+Xzn?KA9-7V}zzpmo%l&UNJDUb#jNh&t-bOIfmc zq^Y~2dq7B?d2610L{<&l=o5Wuh7`@TIjN2MXFi@TE7^jk!)aMU>g+YMo7seS)OaQ6ZAZGUc+C_^4$4X95?T| zV+P{p*|L=EFMwN=zb_jtr#QqwC@bl?PCMMI$cuj}Y@ol>R>>*8J4W4aWqDQwmDw9K{J9hOE6d08s1EjNzBNI@s&lMYhBjnqtLm-Og%mUwGa8(lE%RmWvRkSq6sKnE z7+KhHe3W2q(_~@(m8Jp0(D+xL%pciR%b=g2Q*+-+ne)`vs_3P40&l|J0pkTQoCD)YW=7U(ksv?wUAi*=%Kq64eMg$OWxUlYlxszcib>{ z5rKXT9?dD})a!U!)!;Y|JuK0B51x>BiT?4~0x|QPyi3>#_|~TNW=)oNxc~x$et;ld z;jzWfnVyVIsi)O5Reohc@r_Kw5TXc(+CvVFLc{grI`vO+%e^RggI(5ZhePd@xKU<9)eXOm?G$6#=;#ua0T3f|ZO`4g8wZ&T^r?Up4A>~ch{M5Sg(u`(&?}LKH;p|o5Jk(}8D90WhXeC*c7e9~ z1)LIpz}SzV42dZvoX2!JOXKDBveFN*Gm>7g=d_$6$`yio4uQtjN+}MWiP+ zE_5p8Br%GmbSYzx3%7Sr^|@SJ2wQ`4;jk@U3)mv1CEB>jO4W?mK!oxe&MThw=utW3 zr$8_!Z2Z^fM-mdB-L=)EfL?wGSnM>H8PAsX|#c(Wgk~EQB0eF1~R7XBg2rkhY_p^C%k`i zR}_0_ZL^&wTpX`mH*&h-H2Hag7sGbP@-Rx%{2gA*HN(uNJ<^7p^BW45c`>tFnlC;J z@xKjm+?#Jk^s&~(XdAa?mTF#VTwtvp*E7zm&G7>G*86dPPf&3rcP+E(J5Jo+#j0+5 zN83WvpsZQ56|(n>6Q(DFHr#18yhPPGK3?PdG`7s}Y8Lh}AKs_*@QC}Yj#rafdRjRn zN3(J9s`m8G#tm?yidoSd6?b!0v2?gN@m}>ht;`3nF=6xCCsq90KN&fY1(eM*-Mt)O z9)qZ*ad4XZE|EJ8=~6UO*6jLN701mstsD<1e@=8y{MsY%4G^ZGd996J-S2HuGt(`b zw<~hbung2hkv8ZwAG5`)d$);9wEr`!IVo z+vb0b^%|rvjLfGkVq@L2$4pWIYq1uqcgUYCC3O#iFxAxV?(RO}FY>T!$Z7FgxvHo3 zM21*=Lab_Wt-bROO2~<)GHr{oleCkpqvDh1saiMO8tJ$#HsyJ>8k$XeD|-CH+fOtUj6TPL?Bw$ zPF3-@pA7fHkdyG%Xs=#=-N?cy;q6?nE+3kRZO2^8rn|k`B+wb)N8LCrx5VsVbDno#mm7vmzf4*k3_Ch9*o>22IAS( zs)?U0%5X`1V54m6S0geRB$6Jd$%Lo`^@-HP_;|nF_+O-7Yl-9Md$qYl0teea5^tn5 zeC#L_I-8;;Z8dO}9Af`;0{zr-Ih5vIz~*o%v!&6cb&I3wv0Zr*BHH#1gOXq5)nIwf zXtF4+X6*E07Mox}Yq8eVXQ&viGbDs(?N)ozygCm}G(-_B^EkKE9xd9gofKOnyIBVnVnQCu zymd-Z4Y^AUxYf4Jr^~#qo0S>rgAGLE({4^t;2zCi2bUxaVrcty$ zKpf8%VHzK7C+VQm@siOZgz;eutsrI12INRO$|0qGfEct{di&v~7UH^8xrERrjBLkC z$MKnx=I6_?5L1pz0@H=&!UK(IBAsF;8cB`SJ5KK|Eu*i=cbru&mqlOe?D*mqN%=pB zF4-sjxFnDeUZHe+HBqwub$Pfuz~${_auNO17ad(z&ya3ZVTPqI={s|z8wIv}2*>I8 z-ch;MPYO`Rjx!Cb#M$*-m?$*f+ad$m5bqFw=-+V@MxO)F#o zr(O}8Zpo5sT32}=8s#}cx!W)e8p!vV&+MQ(E_5LJ<1Fc83tL%NcKo?mMssbCe{GlQ z`145-{}!ip)Zq;G252m&(^(q-Oq0Hk-WcW*ayr!PvsCz7?z!k#2!V`K+MRDY3h>B5_`oww-#P5SgE9jBMoSB(ZX(y*{X+ zGM!EoxERI<9zU#dok)u}jCf1mw~o_k+%QR@vbA;;G-$XbRz}o(GD@(Cb!0@$ME$Q1 zarbVy#!nz@?9plJOzFo_i59F(`?k})>m+dK;xJJ}zkjt{e)!gK0|w%Hdh_QGg^2>e zcjc4xa|F{5qCeb822v3Pf*J~xC8wnY+WHNkEZZhIbbi?Chlb)bX-NVTc`@=M_$I&w)1f=?d5TZZtlrdcI(Arv} zr{X1m_aj_REs;wJzm3RoE?q`o4=p+m#RfT=&?u1xl6Q`HqB0^_=L^zhfS=~vaOs>d zP6pawQY&fa%topntDJ)HgwlEBY*mjh94#pi)dz==3pz4hU`w*>hP8JZNf8c++?CRV zi{t$8iH3H|B^TckCC8-6s$SG)xkKSZ89^|I9zX~NjR`xY&z5mvqEx@RqrNnZBoBw} zooQ0f`qoF88Ye9un;pgF(W9!~G?M{S+ZCcSljZtzX2Z-nsxt-Y+C(SJ?L2di+(2#@ z?FOsMgA`7Tc4g<88B*Jau#yt`a+b6Cgz&`H`KdVZX1@{Th9?VU5H+X(_leF;mK&I* zd;_7&ym-msqd8)+c}wI=F!FHI3WLUeJtWDR>Is)3}4S=|~(H>{LAVw`#D zX^k;5d%*$Xf;9~q@EirD@~mGp8W3xSNitbiv^{uWyJn&!-|?Wa7!bpq&g;%dka8Xo zi+va`S2HIUGSk5c8muiF5iS;SR9U86$*loc`doJ8$_NfYH3W^Ie7RCakZVu(F}TpM zbF7SFbwPvQwZEAzIb7c0U*BTiFd1ChH%g9gGbR527<0#~$xxe<$wxVnDLwsqz?Mtp z6lsxGY6L?L>`}Q{TBMAPNP20_Oi>Xol-{hnPnuydz%^^Cb=EyC&B&I-56k7!<`blk z|9Z6$T!bA!miH>rFQpX6wF^QK(JyVaO&mhat1NbDo0F1ELNeU|4T^-e&D9WEef=Ou z`xVj}Cn1N=mP|(>@;Yf}1A$v_Q?|)O|D4d$xMP(x zVUHegU)$+Rq}Tnmq9o;V&lL%EjV?%vU?iDs8F@$1@Jx{m-oGHqrt@Y=XL3|Uh`umi zhL&`Qvd7}daxHJ6|ZU2dK0~>eBUu{{7 zomegHqplx2m;RC_V=vnhg+c+gE_TV{`-QTIBfc1Al*66lolDvKY{>^6GWZbZsDk)w z4`U!2Go4FmNDJ%>_Ydm5R+}VQ_uC3WMs(fKjX)i(oUum=^$D+IQG=A0XiB4*GTe4t zO?bSM!pT`Iu}<1zMwgTer%BJ~Oui|NPD(3ZQXgWAG$ELyAiNArX*ov{t6vn>cDR!g zJI)=-Sv26*4JqwnC6+8S$HM(Ig>#@$tjCZ+qd86r8B@km9$KUD1f7(AJ5}P%<|IH1 zM?=?CN#@<)qo98Oc@p&BK%meNm?W|OM>9~MkkUVcjc(of3-xwbO5Pc%$DI+tt@l$1 zTbs_xkC2l9y$FVM9nl;LeqarM~;}Wdsu)VpO40(0D5e$e- z8cHPTzKVWps%%Pn;Kn3Vr#opVQ0`;aw%4GwUts%miFCyt2ir zuEe*#EB~|8y^@VgtsGDgWf~u}yM9;>8rEh?CtKxC)1jfbL`HCk;S<#jr)5OOMIrjwfFWPo)5Y#7UKR*twe7;v>1yI}6T-7p7XlzI zOHS@bgGK2=ZvAl)&xM>W?KjG}rX$1ZQ@Qwx>2fjgxK)1X(mP9X`8e|pMAhM;ajIVQ zw?~&TtYMrr{R_h{zAn@jRFQgG*-jT0M(zBSqcLHZCks`*K(=<2HUV;Qvn`L!j0oHH zt7Z5_!7y6{J60^CDzdb4wx4$SPGr>9*CQtEa&)7Niqs^hwlOuz+H$r=|F8z?+Hjj> z`067^rqi`cjJoEAWJANsQ8~rCKdzh38)`ymoUWXV4D~NefkKVtG4oV)igi>;+La94 zXW_!|u!41EukQh_%Iml-=mee1Zq8LLh?Uf$gGO!ReRX6phr)~ojYb*bCK;;MO^+Kn z7U|KDnn>kh!jbu*#it>dCokBPcDliFQgb)U*JQ!d5C{guP1EEoSGxIOZJIhMN!46j zZX`9zrLv<~ll>sg*2t0iT#RZ+dW$MuQvbV6&Ted{t!_Ka<@q~GU*=9?X?TUw?ZG1V zg(sU=1++VtKe^F+aKj}gP6;|xF8-iGPP3t^t&Va*r(4_E@=A0Cp>FN6N~ayN^^%O} z-I!s1dv?3#WvbdGrow|YQ1{+DRJEnIM*2thvDLhJ*A*L^7HD0%{gkJw z`D5SsrTbKl^QqheOD!$ck3U(ZMUPC5-M66&C-2}}BX`H9r#*A>Zgsh0UxrGIerff% zIJ5sT0zs=#dQej*)sAY-CFJzrB(bxkGr^mG$ zC8z&;5u5h7E7?6rk>CJ=SsHh*cTaA9LL121qsIeza+dqqSWEW+*gmsE)o%Hb=~7mX zkEg3zoB^Z0Rj$WjHq6ikaI0JozC1&0|7cxI$L+CQ++TX!7daN}ai|0?zl8-0Iz4_D z$>-GwxG%@KB)5_U$tn8jG2Qul)=!oW7ycZWO&T5v(%@@|Y^NvT zgh87b-KyV{ucV78Orza^ZxHO2Gt$>zC|X_Km@j#J;30<(K^V?e9O;2#$8FC~b7j0= zC&04(Y^CJ;S0RCdD2uCcgzzGR6+NxtdsBgmi|sRrGwT(}(@F;E&c zF?!e)z3mRVA=7%_cHFd!x5?#x-VLK|T9-l@9BtFI%d?~l*G!hsVfGp>-SVZat^4K+ z)_sYLq2v_K_5~w6YAB}(n9%1kY@Q7BPeqM^(#B1coBZ{yeY9w0llVE+M=P(Y8=qJ?-vUk`sAXQiFTiG#TfA0b8PTrpZV@l^hK~P?1gi4=Zai zb_1^`n!$JFa3}49&2lySQh;mmvz;`LWeA;Yg79MD7yjp)ol6L2Lze2aaNC0E-S#<%H?|0y4ywx>jTu?q)5 z-=@2gB$D_$iH|X!KbkqiPI{k~ICC zlyIS66SXL5TjXZ$crU6Fj7`YqtdlhajX|f^p!~qG#9@I)P#+V(CHsU7WVSrTxCMfS z;0nnk|5&ND2PkY)_9tDkrhz^*Tr(^1HB(<(AlB${+lw>47pJBMZo{{G_59I`#rFU_ z2T_ei17&Br@YM*o^?t8OlVt=)Hr>P1MX%dWN-lZfB5t6bc(X-^`!@`Cx?XHl)n-ux zgfA(hVMqTCS`Xh151a`cB{;t+T&yntJ{s6b0475Dl0Fb4pOSM}(?AzQe`2@9CWuR> zk_Zj=gp47=%r_z3fyrPDH>b567KJFreK*`1?l7?ZEMGp0)QhHJTe@s}uWECLhA%TD zV0WE-1ASE~0Y2U%f)U;RXCyHRD0_EEChM`mbUIkmP(C4W=IMVM^=YKx@HuHt8Ww)| zf*s3ozvyUVt{{x)-$~$&k`7Uh`nHpl5mta6+8#jsk`^c>|0&>FABdjpAWhim<%cgR zJI(|AOaq_+@kfcIlMYuK2(*yXn;Odpk@6I-EYuLa;~GgkuYxr+7}0xk0CDgddz;ks zkKWCTrHl^=@U0xZ*_Uem*&{|U!=*PnS7k&pA*H4fbb7Z;mhvZU8cEP-XqzZKqW3L( zCry)!cq4;i?a}+9S<;&CeJv@C2-dcmlyNI9($FzYzQ}#R$YF`59G4$>HsstG_3kmGP;m8KM=m79$F$5_nXV3nk~vbmL?lcMO85DO@+%H z1aKb>pL1<0*)CBn|Vh?CMc}Bi!X$~KK-#&XhaDeO{S`HqRd%BuzXU0Vr zA7ViV=|s3`L6!Q?O_tAWHKZ_&ZE}nlG^QNi4a+Bpifrr-cY4286xc;x7A>iD6wA@l zx3)x?@bx<4v4s>dVR3F?7oR+^^$o=3jRStzJVrv7Rf}aaGcLx_bZGcsj%3?&Q(v(6 z=E)(-yJ8Fdpxj3rWh4VxqtE~f;e_pY%|~c_R4zv_GE(|PH}902*e&)A1Paw$`;gZO z88i)i`w3f}k@*8I1cC4F-yy6V4O`>o3vMiH8lcfo)=0{uD_i&8mLwC>O*(Dg*M7ipLl+R@*93DmkPqw@vV>Y`)DbfQdONH=%iOm3!JfKsx?Hm zlU}PlP(eV*BDnB`obiqS@-fB1A?kDKbV5E^a5T&oh>Uc1N52|T+GdaQl3Gp)K+rBo(!0-+Kx7LV6dIn1 z9zf~6W911ZM^x8@heNbbHgb!?R}K_nHy-1KQ)nTO)WO--TE1YxB-zgQTtSsz*kroF zF98Z+Xbd_h+w0sI)`f=QX_7-i5Ky$mXc#4HWY&f^jqCcB6mUlX2_)AI`D2inAhz}#4HG8F7sG8b4=d>{ngN<_ct_%E1rp2a63Q+``R2Vo$;A4K3`crpGiJz&&LfpfoS2MA4JWwueiEzha`auqkpT6PJl}6zi=%nA@NKO;VQ;1-e=&TizflnalTb($A z&xqXdxj^IbWI2M70oTuXC;iz>`J!Ic$*SX|KNllk`>Rjrpv(MX`7HXimcC$~Y;T^b zO{fV2v80+@P?;alL{FLVvUIaNQo7GL+v>8gL=OB>FYE?TC`OqdxkH0sR3q@BBN5Wy z;{1gdFkG~xK+w=N`g|_^)!9<^ca9+n7a+L3Fwn6jl@XnWx2ws)oJiUtC;iqIY zXf4LH^=_*4;`^l((HZW#VW+gRXD9ZRlupoMI&Tj&)pmvGyT$V1xsF;WU;Ph~r8DnJ z{`Dn&|Ag$a*BNvJ+HUMFSzM`FZf9I*Y>JVx2iHaQAWn&A2u*0>B|=|KeW|B;W#dz2pDzd}`NpJ_H+8Te*{{MThlxrw{A(5U*&!eL46DIQV-cOS995X?#dzy92ffDN@c( zFSgJR4Q*JVunhEx!j_VT^KxxAEg)#<8Yg%BHQx;N0`1#ADV!DH7mTCjpyATp^7Z(( znudO8;7IS&fvY0U?vt7*pWe7DiY-h0ZTlkGJe@xMisdW+EXi7|zsRv;W=Fwx%RaJB zHu)iERYN^zCI%pC5gmF8U>=+j&A75u2TWpVtGt>-Oj)s)`g{ZM8WpPa!KJ!T>I7< z=*A>T@mH``(n)#JlV!0+feu9Pm?w7<>jXjHmCVUsu-5jwB}DJbkXxC?t63w%_Reau zku@#^eFNRkp&ux))M%hk_dbPl!o74lxPU^<*!6y)9xK?~4AChN_}7<|i+#OW)A6rw z!o@4(2Ck}T!7R#usIW~DYTtwqT^u9x2|Um!gH3mMeU9A4Jf{1HZFuphEaqJqfq470 z^dPLjHxM*RR!MdC#QTz=Es|rz2jM8)2+HBjl8)C-F>QfH1F@r%IDFJrrvbL#XUTqU zqiePn;^aK}JSz~z<@8P&#AX)|zH%I0SJ+55Uzb0Zh+<2=5bs-}Bxl@D?FDUQ&_Yh% z8t0^6-yx<;CFqXZmz2v>Y>0%V(f5KR2}G`A15xXYWN>2$aH~sSb_Dt&16-`6eJ`vp z7n1=(3q~U-4R*`tx3$p5(U0Twjo&9%aC{97{Qwc4CI#&awUju%sMZ+Z$=uGHsL+d#jE*&O=$-A)^2)IBrh5clbO=5Mue9CtEUe-az`2~Y8v_@Gk zTRADxY`PY|}t4~3P3%b5DI>-d|72CN~B}s&lQ`0tf80X!(GHTs>lGTBC~O~IEuV2e3KZYS zkB*jIq;S)tk_VK+ObPHQsvAH9<>_j2Io-i;0OA>}PM#|*m=&Gf(fuS9L3F=?P|+rQ zHWh|vy1p-slLR{MBIH1Hy#MNHxtBRv(js2>L`noF(@*+8S0JwvY#0XI~GI!cDK8*s612Hm~nIEyWJ~aGTDe14TG$=;9pBO7&aOr!n6&)H}KkiDNZwP8SIQ#DjvYUMexLCFs zb(hI3PToP$PteI={>UJX4RGs*jF@=ISh`mK(2wI}5OlQrn9ZC|L&(Xf*+}-V?7@US zQT8o&Bb4T1AsS4SHf;WB8t9VkWH1MP-SA=kLqD*MPn3Lr3I+|L2v)?Ajds*>-K8>` zrOS>;nR{roIxc-Lv7al@0HSrF42WdVg=q2|xrP}gA>2PQI_3txI((~M<=6oKjLyk& zIh~Ha;fv1c%G~HROh(6LvXP9S8luc-%N2kx4!s*?kFLc!+iUo+4Z%83k*`@e7h+S; z5G;L;WUGfws~2L?vLPobs3e1R&;~xuag^;r8KW{}!|^wbv#oLz z#I0aqvbuGbTcqpV=Ju9v;U%awyj0Ku+F=!l5lgs{d zNtg=|73*XhRa7+uXh0mUBm4c$9oz8h2^mT9EcEcT{qJ-cNyJQ~;irv~zh+aIC^SyR z%P=Z{fbW~|pQEyiYbI4(`ZcJOHX}=nO=%1|{Tg5PE7-8n@(Rga z0oRY?^cz|%Ygrip?h8hoziE=)&+>! zS<;t`u?1FrAnvKA+Q)Y_7F%O{%hA9J_Je%ofOvAUtf;>|+yD)-HZ4CB$$Refdm&T$ zu;wqc;{^z|^*eE{4j12ZPji8cTsLiDTdKdb`f-qY==A4qQ>=~4v2yh1GeNA8wWQfj ze>SPRFY2}PA{hJrd5U6~urz=}H6*Q* zYZ2ehns5t^PaG0@kDt~od_N`0@6N*)(9>Z2eucAXK$0hLxC8Z;bOp^}|Hb%Srh-_FSM zO}{rN2!?XMuaY7Qt{Z9iPE@PQs^Ts}PN2>fng8o2nyr3x!!&uxKbyBGhm%!4A7_Q( z2_-YSpl^xfM}hhnhZAG$%-CGEm0R-^|2Xq zgAI*7@p8K#bz|k|SFDn6NH*G98kwtA>~B?BHG)py-f^X0^L?uYvnVsCxZAnOwVbYP z`e}d!OG|7&x9SIOiIpNYb^+I}M6l~}R2*qFY0!`31a8lfcOy5shC2b0XRV5fxAf7v;T9oyPLOHzBEhlbVqhJCczFP0zk}}pTHm&+=GgVSl z!YL|6-?fM~_t*^_G?y~sPNFp#N zaGsaG88^s~$?BpT zok(|(Vbjz_biy7r)`fkb-)DBAu%5&*%ETf?yJoC%c zRF%-qZvKG+#62adG3!&^PZNwT_s&oW^g@2vZYfVD$$}T6l2^!_4l8c)xb zcl?4<)*ftkymQFB=6xE^Zj%M|AhAo_lhEVNrlnlMD)r$Vk3 z$`}3f32t>8JgJ3)#z%#+*pE@90UB?TU5w8aUl$-sb7f(RP*_qLR*zF{vQ3bUZlqz| zWEJ<336SwS-G)@vxQC6Yu@IXls8-+J66M=ZSt$Pdc4Ht5vE_tnwd3DrWSa)nWrdeL z?kTgBWMm@Pp?zK!iCLR6vT1&2wY|7yY3v=RnnyZ6C_fI9SK_BeabYiKUyVIc zCj4c#ie+|SGSOWk*>P!vpVL}Ugu^aDhjV%Zn6Yq|v8KU930ik)mW|9pscS*qofs#-Qc za1>8vS~D6TN<;Tb)pC@H+|ekGqk812q_6F{DGfmMovvE?aj>=_BTps#kPw-J6v628 z1N*dM`a!TK;@$U%P%UJFrPI+6nx)!(oEfDY`wlO6jWad{1uj<~Rf!iijZ7Dc5X~2U zX%dApccsk7o`CBI6fOpTMI8w}Nw3kut0~^emi)|nQXwrrnQ4nj#N;wfCINO(l!bRmaeWQ=^5m87F zydNb8JBsB5=;~?965TvOo?{)PxmY)Rwpr%4xg<*c_vWd@`xYAYt)yGlNihe@;OhsQ zZYz``--JHVnR3uSzP1hPJE>+1tCH`b{-@QXoJ&__(BOeaStlvwL;`T1=%@RnJo9TU zhkk(gbhea#eQy{A1AR6{%84#TD2+mcdqOq1YpDq(hA0iBT`gxLQya)CN9x7-8=lqR ze*f5ZMqN13Rz$JmW!nnX_~EY&7fW>iFgf^1)M=zTTp^38Xu!YvL6?dh@-Fv*B61um zk%Q5h(lYZ3y$kZ|n*7ODjvdQX;$wj*&mF{bc7>yfqV0D_<%KgFqI&XIyHzVctA-Wq zr!(>bp&CA0hZpWXulmM6O)o4pV}IXD)zS~YEi**FKOx`QC5u)N1N|^XzO#wqG#%I; znxv9R6sL*WhSPDf?t_J4Z9(~NnrebQe72xe){%o$wMQuJC?F2)b8ld8ypRSOHA;Qe zCjW4l1{OP>tGiUx*#B{&YU*!=g960&CsZQK9z3TXX#BK7B@#%kxfr4jE7%wPh+V&- zgS^N~iyzu*pq!bZTHM(r;!3AZeK}Oyq)oJ4%3&|~5Jgj~ba5K(lr@A%Ktn$$_s={P zx7l7nwD$PtsER9z8kG(yhSyI1B#Q01bk#08g^CliM76y08ZEXJ%(;-faG?vsJvrz& ztukey|B5zTOv5LwB5rV6Y>`8p6KY9$LL|y7R&h_K=^sAP*di6nl{;;*1Ostl2bEaQ zTp{uyT9?bgNM;UbG@h+maigqCTPJar5Qbv}vs~IRkfZnMoMuTXu4~ozKBC9&Q0={m*-#VB0=j#bwQod}%p+?;j+KrY{E&S2U7C#Z`4h5j|tF zEG{rRJ(?2`9ZOUavBf@H8u+}ALiR}&zVXdBhLY^ls zC&G64Bq_SAmua98l(eHNnWTu?QD(fbjj!79n)gw#N5&Z%s&-ju=(kuU-O`L-Mmb-iImJmj!EUJ2|LcdZy{PRNnRuqCo=fD+&)2$ z6DD3-)e9foAuq6CMRi~OCs)Xe(FszV852~CuGfa;(4wDIEjU~P#W(VED`cU~DeKdi zTSrPG>r-v@v8u(7v%+o|?l?q;mG(UtW(&luB-M^PnrrA%g~ko@x~s;|F9>r1;@Pn( zzVW9~o|`vOwTxWH))1Xk0{f@hggi977AuF?5YhI)zli?Eba{(oQr&TR#yieSMJ&H= zi0We-Qe_d@#e91}gKd{X(KjHRjZ5TZ?lt-l`1Xx$s_m_PHA~+fe9p%C3z9OU%X{@z zLdvVU!)Ykw*pVoUzUdTJjs|WDzchDql<1rJ79)>yWV$$-F7r8T(=;H8U|+6O@t5_A zBFakoC41Y)=fdUlmsltwsTK@L$?UVhKa8aj8o#cSqD%YgY0Dq``}y)@10Vqtf`X&C zuYT~D*Gob5Q@ZBcFiel}v{%(%?b|8k-UzW%yExqmvyI{$kP8jh1$ zBUxVYZ$KbN$jM~OwTQSJ{W~f-gw6y1}V5z5Nb!5kR zxRXh-!S^mc8y+4A)}D#&_-Tz|mCJOx%QAc4!{?GUNtTnS1+jGlGzh<4HrAx1@=^L8 zS4kxQ_4_(RLzVL9?3Tt+5T;>liM$nAYomS+n=0LY2zwuhA$ck>zN;2nKWKYnp1j7! zT~tH5yw*Hbv(yg|S2a?NZ$2Hx<>iU;ZY1ak&z((nd)b&8tsrdgnj=g7s7~V-=dbB* z+HV^J{M*0KxbiX#Y z~TBsu1B(XlsXQaP&6SVo=mab8x}Y(xZgWtqH&8gYQ(xPj8HkOu++coGU%Z z?b-nxgQ0QA8PKqiYV_8FW{becF>q3$SM8=RP53?vhMa+%S8nZl(403IIR?rJS0CA& zf{Se!8t3MsEgjc-ZlJTIv@sKk7Twr%;EP+_b7bQyM0BJ6!29aB`COL+Ut1n7OD4Iu z5Mc(2PxOa#?*7|P8eJ@n4=Ms@NO=gjPjpjEX*;4!CV*mSWRckUA1cJa!9rxU%~jR> z%yEWBRM?WVi%o(~R%tca^zZ-Ls39y$c4EBzZhdRz$Znk{zq9RvP(H-vMJA#O zmUi4`6ZZNCcf4t^2Fjkg*gX&l)CoG-TrvB~KbwbB$jP2@(%p4yq%g0OU6AJPvIjCg z+qXu$l}8=pIip~->$pQgw7CHt`w`O0RV%s_nD z?p^2p$hkWB*0)Tmgw&D!z*b- zxsAi*_wBDl8R(CF@-5}nAZQIV=(;%fSMIG9gh%C|TjSlsWbxKL+2}IpuFdWtE<9)m z&_d3jh0EN-9Jio{wdtUXmZ(1KF4x1uYCCA|O7|dH@b-o~%%Fvn+|TKBS~+8}K?CAc zb56W94a;Tu1ounEi`Ldc42en#Oo1hnTA)eBL92Za~|^byS}&Wl{EE6S7aXxg%la$e9r5 z9%$6tIJ-=DsT?YiY~}C~#Vn1S&Ka_mlK`~UkN$kgb@!359}ZUioOjN-dl?=D;mIIp z=kfoqy|0a{s@VRgNKGx1La&O2Qdtz4g@tw_&z$Wk3>0OHg5o2FiK3z?8lrFz6*+)B z$djU^C@P8)S>D8CFS`~gX@;qkrIrtsY3aR|b+zAj*6hRKbN#>lzq@|iFPJm4)^|P3 z?AbGG));5}esvAO!M`3gPOI@p87k!NNaF}j#n+<=gtKWOz5R>}(3s@HI6@w)F)rc0 z0uVI|e50w|X?Ho|%EzV!HzkQLFv6g{N%V1dqX}c7B#Iik7V>-*-K)j&5Uf<#jQfFh zwCDq}U}vO_8Tbg+5dT83wat&$rAsLVh3K)yd7Q+M!nw&oDfrd_Ynwgo*^?P*Fz)wg zW8PU0tw9#gOV&2;s3a>)Gz3v5r+$5bJCFjlJKXqrD94_g(CZqF&&=!%++^sz85Dw9 zpJdBThW2WvkRfW8qi7&z;>#}gJ1wVaA*EH~ICc_&K|VHZsC%e!2tEpw%9KMNsxvP7 zyz3-7L>oH0S)4p|PGV3>9(ppsxQL4s9B2mCx;FG{U*ig{@~U@=Kg|7_aSAc4QtPbK zVX)kV!Bj=5C=us_>8#*RZG~kv@5R`1uB=NjxKnes&1COrhBO7T;=4Y%z)X=ri zxv}DB6)0@t*nt&+FMe=w8Sa@RrnF9HIs7G`{n;yza7>LwN;`ZCW^7&6G6M@YJQ}x$ zWnOdI<^NTqC{Z*F2$P_cDTnWk6`%Z}4j2NyO&gAt-cPX5Eo(5+;h)8bPjD2$G!$z5 z)D1Mg=K}bLu8o+vhrCs)bl`Fe(}w@zCK}5Bl7h(xO@0j$4FQ~5PZ1q4tjTEjEs=XR zTYtpDXyeCaR!bc;rV2_AQU}anFTg)oxDn;e#t?M_yMjJqS&aBj#lx5dyWFf@aBn2+ zA|Fi)>su!NdO}@!$w0lP#r(+ikuJDhCwu6v9Z5q*s6il~0Co6*xu zMX8#M&Zj3{ZQ{ss2BzQdZS+9ThNMb3y^FWujp+t{Rh}G%ypr936oQ#GLOM~qj@?`X zn;z3dJ&#;rX<6&~bJ2#kO6)9~*fbsK=i3bFA^$Leb;v<(eRYH=gng`IxwXyT=eq2o z$9ouk-)Qy4r^nqC4cOw8G!#+1flfa52jhTnHf^MPH)>;jjV~Bakv=x+6+0EZex0R- zPIN82gAcWTfb*X!#|`grklJ@y$Kx2|7-PA}9#A53&4C5XeY z3ka%?9DaYiNJe-RG$w)R+DIH$IB5n@G0|{Jpl46c=d4Zj5?HuVO3&Wt$d=<8qulF^ z+em=nfGHL9=Yq)R#gVKJHq@v=g8XsSf^;dCWmL~b;}&{0?4o>ZR6>%0%ulX(hf$;Y zi7&tQ%9tAajy=uL4IF%ytfX>4?Kx%dwj z0H6!yAEUFcicAEhfu&i1lvKxhuVSJs{^)&}<@DSuf0d7>js7`aoWq**VMSv&3f#rH z*2@eLJ(EQGBYU_8IO$r%gE`{ot_ByNnWjYy^b;$F*(DCi+NMQ}!@)lEeXx|ZP=JF< z>8_~)3Z`8ahj2s$)s?i7y@RN0_&TS?P-964aUH$VMKtfA_#8*N9pJ*UM4U^ol%ktL zK}y<)okE;ink9cVYaH_t%P{>i?E<1WLi8n8zd+P%f_9AEV$Spmcx@JFqS29 zWFfWr{4#IoZkFy4frIqFnZZw>qic~bxYHvqJSTyv8CwJJ$?{18pgph-QrFw)3=0pjBO^-PpC78#Z!Qs*vmxzwFaZeFGVkM(lRP$I=*YnSGI ztYn2l?b8+TOLT997aE$SL+^K^yzYT*4d4(RoVbaSl7mk*e$Y<_{v3eW|=XqwJFEolP7hdAJ4OIiXv)9a*xI$VVCz`|z{wiQYjiGj;3F{yD)s*F5gGGvY zjNq_@J~qLZ9zwhW{!mUg=7wD?RLf9-DAcvF{@p~LIsb@}9yAvHCC}VQRW#NX8t)*X z(0!@{p+@*kk&9H=_*Ef|>=tG%`zPOMpbEzN_*nbw`v><1MjF-8pPtitGYtp`8VgHB zGG-a4Pg(0e-t@d zXbje(5Ye5ELAu^Ko{qbEp7P3`lvgb%Geq@XEY6Q@>tqWroE|F*%p3rUXkxV}z#V1y z!}QZVUZTL9DxlD&MSboe3SLrM;HsdaleG)JUL`O*nl`ShCqnM#xM+;SA(BLRp((Ad zje8(b)GcW5(nr)EKGrsv2c-ZyYTG>cmPJ#dagIXl}U0eGXVabX$V8 z&6-fBaG-JdlE}v`BB-wXV;nYxZzHznu&V&K{$UMP*1%)a#v@7fZAAXTKa}OhFT5;H zmp<)mf*Odo{2Jm*4`S%!Kki`d@@GpbeO;SSd_&lA1G`iZYS^?1o8rYj3`4uiKo5A z;nrF3qQ9-CqW9}9U@KYc+T>t=>f8R`Tu>_q=}F7V#9{6}Mt7HLsqOSAaE zJp9GLCIhLjy7^p*KPkG-I(V*nohUAAV#&{3sDc-P#xHT=C?rjBia)uhuh@&@KhU^j zt7(%528j=_Ac6vAsL99#{taT#&_r>Wj6IxvIF?n90%%^>qOp+K^|jyFE@-A}Q^FFA z-0r2U(+jf38_+P<@I;kR@dH4ilwZluJfCa zp~s|uW4))iuM?@R-C3|K#(GcTLdiuLYU-dUF&e8q5<1hEKHg3nE$_L$v!{L-EH2=V zL->Z$`qZOt;sO>!WdWMlw5g{Siwg+O2T}Rhv^PVmt?olBWp(92u`y+q{YZF;HEnsn zkFHJI=Pv57LITm{v=#+OJ+K#HCz1v$DDHsC99$*=1EOthosa0=vRe;pKeV^RH zucQ=d%GhE3tj{1~25v1iuWQrq^%7^z2^z~X9Y@U0jqbqxw!0K=dW4%edpw*Al!D~- zS-3$+(qJyr7akR7v9t$*$_uAwxr+~Des>8seF<(J#TF+xDCk8pooAX7Ts9pL z`^@CwitVlp@nM8=DX`VFnIRKtHIg)ehRGJGDLzC3E=fb-I3mP3EJUEaGUbfB7K?K# zH@4K8uFZ&TC(i7COKMDFK|xWJ_z(#W09_T#zCWCn9yr1NQ7;uV`@uk3y2)Gi21R4$ zt#Au2`{9qG!;x3*U0uX{=_O=J%nSV32iEk_q1BJ3m41-j`B(&vpkDllI^@7a1IAk zS~O8QogZ0h9N1fDy$>}|D@>ao?QiVOye3gvaX(L(mY5Z8eDdRUT;b^^9~7)=79Sz- z034*mHZ2k7_&&nke28;TJn0|1sq_!E{DVSWTOdlsd**Q$6xg%{(~epnZddIx<$@WP ztUVDN=ca&Z+JbrR)`zjx+TA(DUpOby+IikvJO-gq*A^~}wswB~h($JG91E8;SReSa z&edPuT2Wzru&z-4suZ+%UO1Kf$CqsjAJMfWoLA4r_LIq0*B1416Q%JSX|Eh+(R+4r zAC_HAL!kze5x$F67PzJI!bLc?mua4ig-!-*$*=6ObjQX}Ti7;54k*bPe)a#tFb>fr z(c#)+iyQU+qTKqfatGAFr^a#f!wOh*Ep?zbt$8rgg7y{*7tC`=oytE{CtNbjZf)C1 zE$g$AmwXT@-bW-8Fetm&aRFO-uQ*rFwx{o=^599VAQnF*H{R+yHq<#i*EM^7H_AVg zCkIERvD8slt$wlUv@r{3Uz9|%-G=a<)+uQQI|WjC^a&n=7)R<`Rn|8W`C~*WD0N|w z^$mPBb$9k#`)58h6W{8nuDIC0tE0I~2lE^kwRCOiV*ylfpf?kBNDo>XkWPz#dfD|# zvJY^x7QK_~>gwrdgQx%@5pu50O2GAK1@hjXoOh?agmol_K%0rh;%1Bs59dMYjI6P8Q3d4$oaY*Vj)HUtne^1&~KAN_&LnKXFsO}_ZX;&81Q2K8=AGU_&bZzB#`IHlFV`ym#SzsJ|m$I zrR2;$fi(HbIM+Kn^SOAMwr)1Rl31siL;YylttTzANxf81=CDYL^_lJ*wrpDF@EV%x zH__$%naMXP8HZRQn91e%Vsf$a1angKa8Qbyz?z? z&LJoBb~{?u{4{d`7T{=ZqU4yJ44pNW`Q<@M!j~sds!=Ml(LrzA7|GJIp)&udqNHO7 z<*!a)FVWPzWNCS&)6Bp6&|A^j?1fC!(J6-#v7HYSs3s1td`c|c$4!{1W8z?1VZKcv zkFGfa1TFbAkb$X!zn`EbNYZo)G-~C?h~wFEO2qm)+N+N22#lsh$Oa6JNka|S9C`;z zJ$KU8Zyo!>DHqX$QXhCw(On&bOUl#K6v{Hk2r@k1pg>kr8Orgm2x*KScVLab2Tl=en2C9l7W z{R1il^xqO`5k4cYLpYWttIc9cUZF1h05Dz4y03;7VTA-5Qv+M(ZBJ`2FBOZ=II^ZE zQl{yaZ0xKy4w{O|glq!6f<~uyG^MVMltm%UYIB)J7V^#3O6ysc0GfX$j(tqg7#={2 z?jP^!9$ByWP@H=$D@XxL@S-Gqnxnb`mNm|u#$nyE);TlCivNbDJ6Y-ZF+n7NVf&JWV!Jb%e+o*>`(V#<+MkwrXVp&v;_D3lLgF(I1 zgR(vRDD^AVU;LS9FXYT{N301LrAnKzp9{pq5R`#&`AtOui*Bg>-x*EkOvcEITW!qgZ;OvArN{yV?s$h+v1+TP9K%QOp5(qiMg1z+PY*NOT@xv$caUo`k&PRrqcMJgx7v4Z;DPs>Z8(Xd+g7@Q31(BODZ!IZ5hN=_6-~ z8^wJyTVBRs7F7CEG7>9phL5pp=ODX!6lO{nP{97NDk6+l;VfW_PFG=LdC5h!w+7a_ zwyIqsEx|?SZcb@eJ&{fe#_Z$qG(}EGR&AL`x#kHfxD}dojG`&<(^~ktqVcq#xw9YS znIq_+#tW6S0Fx_+QrZO|mCoF*P`+2+u-+^$Wu zYVHs0K>&vuKYP%e{BmdauxYD)N~gG&3)xSV#?HpmM6V)NVT#mPZpQ|SM<|a(stRUb zqBIBxT_{;{$9YgR@}HXGqe*QyNDS-7uYNfank{&$F4PKbXuu#!c=lh`2;c%kvl* zSV1g)-jmG~g8+yY9C`Diu`;1aeW*T{pXN_%9uASvRYBgd97=g4*gsW%YS#a8$2B6PDFy-3$ z9b+l5b3iLD{VQnr(5~F04$BbD{1>Z8pYS>>2*%R2{J;Yg`rEUzK#iyQV{+)tOFLY$ z(pf~CCE?U%_!2e)8*253J`|)f zwVKdBcc-x2%T5IW*r$Os#0)!OqN`88L?duM8`@Itu=>;i8sXKJai9=fE_R?1-T5plu=o-GXX0#k(*0rL`jg<8ue~BxXDw-8aBUAF&>DZLTT@xr9 zFD+>P^53rE(#0~Ovkn6i3tVrPj1z_1{a zTFfhXCa|*0G`f1TG?aYMfSe|QQSV9!eD)AD1d|bMd_4<%z_Y1DX6%QCq+G0W5KVo7HT{dOd}8q2rjC>6wj!j z*#4K;P>i&AhlA2TZk5`m_(}}r^qwK->&leHAGV|P-{-p&RGe5&;SJkag~|*|5-AKi zmB3&*u>EHxb$^*Z0j1_qaJ!vx6gV!R27Ckshte&L2%E2UnxW+NgESV01n{d8t|UC1 z#y%e@+big6rY0I!j()-uMdniSuSVlWg;ufd$!_I<3W?5br;gS zGV1(vI_m>8Y+C7?VU%}~uNF|Ul+Ha*xyN{eTPdg%ADS_yZJF&I8Y^i^qBh9#K2Qyx zx>ovpFpcSQ(Piv4k491P1*=QzrMMu8Fd?>IDpW#!$(GnXOd#-~O@Y_d>Eib)=Qk!3@#(7K4!#+qI)S zDF-=aWdVe?K;Ql<5^sU9@twVvE7 zp?TCO2%{-iJXsbt!`cOL#uc24b^r%P2wf}dT}gSts=p{h{ZVCo4tL>wq88kt)8TGb z-ye=zd#P_ETIY$YIq^{hq^5#?#UX)$B+=Qqs<3WtBjvxwvG}S< z+0hDPS>`qVSXFF4FBK)2p_obrurn2+Bu$+q<{Im|xEnTP1HKMF1rF5sGf}*ZFkm!M z#$H!hP06iyn%85$vY`J*lBJ0>*TcAi0}mLG(2RPZZx7(ekv<36o*BR;C zSRczBu~_Ot5wf=USqn1ao6%W`-L2rNLMGuy(W*n#F z5d8M|1o3h}y(Fk~y20ll6r^*%1t(qG@Nq0N9TnBJeC9-T!VQtTDWx{T z0yz|y4cq$Bk_|rmzGdhe&NGWWIxD8QaCm?`ceMvJ!}TB*0qgsi>V`0 zFr|P79-Fq|+gR(;f#>9+oqTNC#+AX;@yI3?zh2s<;^{=nxOj|}>?R8~HdY$H;m~GV zST032sR)XrWm}h9@T*c`1@?jB5dU!s&JX*7M(iF+YojsI5ZN0l(qE!iv9|RPX5(mv z0-L*-jdNN+sf4Qt2@rEI$AvB=Yt-;-v<7;3h+fW7ssb}dJ6^yg&R$vZZLIMXVrpc8 zBKozjWwz_Qn5JzS6-h&%9l*13;25H9%4wjbFSXpxfP$%Y)DG!S?gLv!x@l$%bws8T z$e}!WQ)3VMt*rBvf0#9j>Zse!23N1#T(X!l{Be z!Qy3PCj=Lzpv}3vjrWn1S=L~-mG?B#Llghv8Jm(NCdTSzrWi&Io3^dVm7>e0e4k@LXt-mZPhu)-_Iak<} zK&xw&6O(8bf=WgREWrIf0+4po{6&E?rc<3nj~M{#H-(;xiOLmK#3SE-$uZ zCDydcW$nm*WPs$NI&$S(ku*DV8!L!aSoz`}W1%_QQW}eQP*5F$sZ>(&<}O2>^CsIItd z8{#m2nRh@si$c2X*=sbno%+y(_-OL?2J-h9!&?-9kAnUgmM3v2PXbeHpX?;|B5oXi zm<7*xh`mTs4W!D0wk`Lkl&&4+a)-$lOM7a_Ds4`GOuezs(GCmu~UZ{}b{;2Bt z7o{!Mj4Z4uv@xGqE##Q)3PtiSnG$SlB;Gniap4_59LTYHm$1R zGCgd4jk^c9prA3{*nq{*ddRX_YOLzVtJE%QJo_o*sQT$VwS(h=gJQe=H9;$T@DWu7 zYt=gE66@srQ=s@X9iN&WBs|tDQT$fNZMUA zCc))F+ryFE1ZnYFe0}?V*!GP#rN-og94jl5jJt7k7-le4cs80=PFHyq6c$vz; z_5B_DzM&NeDuFK4D79l-(;#7fN2N#)+VNXCt>_oPE~RL|aqog#I#hoN)2e%h8GEtH zBDKyns$cLH51sx-x`KQ(t$Luh7=sOepj6$X8aL86^!iJpRBWr$E9t3GhWxNnYxLE# z(#0oOM1U+xW7UbF;*vWk0d`WUCvttMgER z&;+PY`DyjHQIy$>_Xy1P`s9HRVd_7M@*z4v1a|6n9aM%W^xr#~U>W@&z#Y*tDJA9Ym=^ zT^ztTc3P6i_ovTX(~X^vWr$Lk(rlt>JAILfqH8~=_@MD}qu9FsYo~%xFe_ASMMz?b zvp4Kat2P2L=Wo&J&Qeb?!OWG#(06q#r{y>q+uixn+WBdfk%RY3i?Mfgjzf(%orco2 zo!7Ar^Y#q(G5CnC?P?!Q%PXd_!+^C-+tu|JJs!SJj>&Rzgc{EuG~{ir_*FUXuE=m> zG=NF1qYyQ=)KGuiCoXZ+GmT0?{kyA!O6+vI_D4~FEHIyPBHgvgPi#G}$pYnLyLJSL zt!6Iy5^39ApJ7RD>M_v?+fVM&F1x-uPxeob@Svn>d=Xopi^Gx^oKCmvrXMZ)^tx++ zt!aN#Y;he_(zTinXT_G*$z5yij}}{TDh((D;S#9PIY4anZ8=e9(`u|uv;vz;kW~3t z&9Fdm;@otX2L0&v-gTkg!`9cTtF zwdvwxWU-PofW@Z0*WP*4%j(`c zh{Z=O6wKIP-xZsF@DJq<@5~rn{}A5$5(ij==z>nh+1{&o2zIb6iQSB|eN-fq5-!2F z-cz0PG-bIOWsJhoFHowX@15)A#<|P;n2QDh?XLU!+1b$UD zGXHko>h8x0mQ=wX$kOcIi?4TOSzgIF8emz4>S)iy)+;L$I$2~K8@Q+%+Y&liJE)sj zrK0sgZBBn9mg0j(;((He40>|JWs#1x3#gB6JLpM#ai+gm#{ISyzy>{??k$#nX6JDX zX!Sw;4>!_MTnd3dSo}f3I4!sYq18~I;u3X#CE2Icv``): input runs live under -``/P/output`` and outputs are named -``validation_psf_conv--.fits``. ``v2.0`` removes the patch concept: -input runs live under a single ``/output`` root and outputs drop the patch -token (``validation_psf_conv-.fits``). - -HSM ellipticities and sizes are no longer rotated here. PSFEx and the in-repo -MCCD interpolation now measure adaptive moments directly in world coordinates -(galsim ``FindAdaptiveMom(use_sky_coords=True)``), so the WCS-Jacobian shape -rotation this script used to perform is redundant and has been removed. - -Caveat: the MCCD ``PSF_MOM_LIST``/``STAR_MOM_LIST`` columns are produced by the -external ``mccd`` fit-validation code (``mccd.auxiliary_fun.mccd_validation``), -which still measures HSM moments in the pixel frame. Those shapes are therefore -still rotated into world coordinates here, via the WCS-Jacobian rotation, until -``mccd`` itself adopts ``use_sky_coords``. This is the one branch that keeps the -rotation; the in-repo PSFEx and MCCD-interpolation paths measure adaptive -moments directly in world coordinates upstream and pass them straight through. -""" - -import sys -import os -import re -import glob -from tqdm import tqdm -from joblib import Parallel, delayed -import gc - -import numpy as np -from astropy.io import fits -import galsim - -from cs_util import args as cs_args -from cs_util import logging - - -def collate_paths(input_base_dir, output_base_dir, patch): - """Collate Paths. - - Return the ``(input run dir, output dir)`` for a patch. ``patch`` is None - for the patch-less v2.0 layout, which drops the ``P`` token; v1.x - passes the patch number. - - Parameters - ---------- - input_base_dir : str - input base directory - output_base_dir : str - output base directory - patch : str or None - patch number, or None for the patch-less v2.0 layout - - Returns - ------- - tuple - input run directory and output directory - - """ - if patch is None: - return f"{input_base_dir}/output/", output_base_dir - return f"{input_base_dir}/P{patch}/output/", f"{output_base_dir}/P{patch}" - - -def output_filename(file_pattern, patch, idx): - """Output Filename. - - Build the collated catalogue filename. ``patch`` is None for the patch-less - v2.0 layout, which drops the patch token. - - Parameters - ---------- - file_pattern : str - input file pattern (e.g. ``validation_psf``) - patch : str or None - patch number, or None for the patch-less v2.0 layout - idx : int - exposure run index - - Returns - ------- - str - output catalogue file name - - """ - patch_token = "" if patch is None else f"{patch}-" - return f"{file_pattern}_conv-{patch_token}{idx}.fits" - - -def transform_shape(mom_list, jac): - """Transform Shape. - - Transform shape (ellipticity and size) using a Jacobian. - - Parameters - ---------- - mom_list : list - input moment measurements; each list element contains - first and second ellipticity component and size - jac : galsim.JacobianWCS - Jacobian transformation matrix information - - Returns - ------- - list - transformed shape parameters, which are - first and second ellipticity component and size - - """ - scale, shear, theta, flip = jac.getDecomposition() - - sig_tmp = mom_list[2] * scale - shape = galsim.Shear(g1=mom_list[0], g2=mom_list[1]) - if flip: - # The following output is not observed - print("FLIP!") - shape = galsim.Shear(g1=-shape.g1, g2=shape.g2) - shape = galsim.Shear(g=shape.g, beta=shape.beta + theta) - shape = shear + shape - - return shape.g1, shape.g2, sig_tmp - - -class Loc2Glob(object): - r"""Change from local to global coordinates. - - Class to pass from local coordinates to global coordinates under - CFIS (CFHT) MegaCam instrument. The geometrical informcation of the - instrument is encoded in this function. - - Parameters - ---------- - x_gap : int - Gap between the CCDs along the horizontal direction; - default is ``70`` (MegaCam value) - y_gap : int - Gap between the CCDs along the vertical direction; - Default is ``425`` (MegaCam value) - x_npix : int - Number of pixels per CCD along the horizontal direction; - default is ``2048`` (MegaCam value) - y_npix : int - Number of pixels per CCD along the vertical direction; - default to ``4612`` (MegaCam value) - ccd_tot : int - Total number of CCDs; - default to ``40`` (MegaCam value) - - Notes - ----- - This is the geometry of MegaCam. Watch out with the conventions ba,ab that means where - is the local coordinate system origin for each CCD. - For more info check out MegaCam's instrument webpage. - - Examples - -------- - 'COMMENT (North on top, East to the left)', - 'COMMENT --------------------------', - 'COMMENT ba ba ba ba ba ba ba ba ba', - 'COMMENT 00 01 02 03 04 05 06 07 08', - 'COMMENT --------------------------------', - 'COMMENT ba ba ba ba ba ba ba ba ba ba ba', - 'COMMENT 36 09 10 11 12 13 14 15 16 17 37', - 'COMMENT --------------*-----------------', - 'COMMENT 38 18 19 20 21 22 23 24 25 26 39', - 'COMMENT ab ab ab ab ab ab ab ab ab ab ab', - 'COMMENT --------------------------------', - 'COMMENT 27 28 29 30 31 32 33 34 35', - 'COMMENT ab ab ab ab ab ab ab ab ab', - 'COMMENT __________________________' - """ - - def __init__( - self, x_gap=70, y_gap=425, x_npix=2048, y_npix=4612, ccd_tot=40 - ): - r"""Initialize with instrument geometry.""" - self.x_gap = x_gap - self.y_gap = y_gap - self.x_npix = x_npix - self.y_npix = y_npix - self.ccd_tot = ccd_tot - - def loc2glob_img_coord(self, ccd_n, x_coor, y_coor): - """loc2glob Img Coord. - - Go from the local to the global img (pixel) coordinate system. - - Global system with (0,0) in the intersection of ccds [12,13,21,22]. - - Parameters - ---------- - ccd_n: int - CCD number of the considered positions - x_coor: float - Local coordinate system hotizontal value - y_coor: float - Local coordinate system vertical value - - Returns - ------- - glob_x_coor: float - Horizontal position in global coordinate system - glob_y_coor: float - Vertical position in global coordinate system - - """ - # Flip axes - x_coor, y_coor = self.flip_coord(ccd_n, x_coor, y_coor) - - # Calculate the shift - x_shift, y_shift = self.shift_coord(ccd_n) - - # Return new coordinates - return x_coor + x_shift, y_coor + y_shift - - def flip_coord(self, ccd_n, x_coor, y_coor): - r"""Change of coordinate convention. - - So that all of them are coherent on the global coordinate system. - So that the origin is on the south-west corner. - Positive: South to North ; West to East. - """ - if ccd_n < 18 or ccd_n in [36, 37]: - x_coor = self.x_npix - x_coor + 1 - y_coor = self.y_npix - y_coor + 1 - else: - pass - - return x_coor, y_coor - - def x_coord_range(self): - r"""Return range of the x coordinate.""" - max_x = self.x_npix * 6 + self.x_gap * 5 - min_x = self.x_npix * (-5) + self.x_gap * (-5) - return min_x, max_x - - def y_coord_range(self): - r"""Return range of the y coordinate.""" - max_y = self.y_npix * 2 + self.y_gap * 1 - min_y = self.y_npix * (-2) + self.y_gap * (-2) - return min_y, max_y - - def shift_coord(self, ccd_n): - r"""Provide the shifting. - - It is needed to go from the local coordinate - system origin to the global coordinate system origin. - """ - if ccd_n < 9: - # first row - x_shift = (ccd_n - 4) * (self.x_gap + self.x_npix) - y_shift = self.y_gap + self.y_npix - return x_shift, y_shift - - elif ccd_n < 18: - # second row, non-ears - x_shift = (ccd_n - 13) * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 27: - # third row non-ears - x_shift = (ccd_n - 22) * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 36: - # fourth row - x_shift = (ccd_n - 31) * (self.x_gap + self.x_npix) - y_shift = -2.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 37: - # ccd= 36 ears, second row - x_shift = (-5.0) * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 38: - # ccd= 37 ears, second row - x_shift = 5.0 * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 39: - # ccd= 38 ears, third row - x_shift = (-5.0) * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 40: - # ccd= 39 ears, third row - x_shift = 5.0 * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - -class Glob2CCD(object): - r"""Get the CCD ID number from the global coordinate position. - - The Loc2Glob() object as input is the one that defines the instrument's - geometry. - - Parameters - ---------- - loc2glob: Loc2Glob object - Object with the desired focal plane geometry. - with_gaps: bool - If add the gaps to the CCD area. - """ - - def __init__(self, loc2glob, with_gaps=True): - # Save loc2glob object - self.loc2glob = loc2glob - self.with_gaps = with_gaps - self.ccd_list = np.arange(self.loc2glob.ccd_tot) - # Init edges defininf the CCDs - self.edge_x_list, self.edge_y_list = self.build_all_edges() - - def build_all_edges(self): - """Build the edges for all the CCDs in the focal plane.""" - edge_xy_list = [] - for idx in (0, 1): - edge_list = np.array( - [self.build_edge(ccd_n)[idx] for ccd_n in self.ccd_list] - ) - edge_xy_list.append(edge_list) - - return edge_xy_list - - def build_edge(self, ccd_n): - """Build the edges of the `ccd_n` in global coordinates.""" - if self.with_gaps: - corners = np.array( - [ - [-self.loc2glob.x_gap / 2, -self.loc2glob.y_gap / 2], - [ - self.loc2glob.x_npix + self.loc2glob.x_gap / 2, - -self.loc2glob.y_gap / 2, - ], - [ - -self.loc2glob.x_gap / 2, - self.loc2glob.y_npix + self.loc2glob.y_gap / 2, - ], - [ - self.loc2glob.x_npix + self.loc2glob.x_gap / 2, - self.loc2glob.y_npix + self.loc2glob.y_gap / 2, - ], - ] - ) - else: - corners = np.array( - [ - [0, 0], - [self.loc2glob.x_npix, 0], - [0, self.loc2glob.y_npix], - [self.loc2glob.x_npix, self.loc2glob.y_npix], - ] - ) - - glob_corners = np.array( - [ - self.loc2glob.loc2glob_img_coord(ccd_n, pos[0], pos[1]) - for pos in corners - ] - ) - - edge_xy = [] - for idx in (0, 1): - edge = np.array( - [np.min(glob_corners[:, idx]), np.max(glob_corners[:, idx])] - ) - edge_xy.append(edge) - - return edge_xy - - def is_inside(self, x, y, edge_x, edge_y): - """Is the position inside the edges. - - Return True if the position is within the rectangle - defined by the edges. - - Parameters - ---------- - x: float - Horizontal position in global coordinate system. - y: float - Vertical position in global coordinate system. - edge_x: np.ndarray - Edge defined as `np.array([min_x, max_x])`. - edge_y: np.ndarray - Edge defined as `np.array([min_y, max_y])`. - """ - if ( - (x > edge_x[0]) - and (x < edge_x[1]) - and (y > edge_y[0]) - and (y < edge_y[1]) - ): - return True - else: - return False - - def get_ccd_n(self, x, y): - """Returns the CCD number from the position `(x, y)`. - - Returns `None` if the position is not found. - """ - bool_list = np.array( - [ - self.is_inside(x, y, edge_x, edge_y) - for edge_x, edge_y in zip(self.edge_x_list, self.edge_y_list) - ] - ) - - try: - return self.ccd_list[bool_list][0] - except Exception: - return None - - -class Convert(object): - - def __init__(self): - - self.params_default() - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - ) - self._params = options - - # Save calling command - logging.log_command(args) - - def params_default(self): - - self._params = { - "input_base_dir": ".", - "output_base_dir": ".", - "version_cat": "v2.0", - "mode": "merge", - "patches": "", - "psf": "psfex", - "file_pattern_psfint": "validation_psf", - } - - self._short_options = { - "input_base_dir": "-i", - "version_cat": "-V", - "mode": "-m", - "psf": "-p", - "patches": "-P", - } - - self._types = {} - - self._help_strings = { - "input_base_dir": ( - "input base dir; for v1.x runs are expected in" - + " /P/output, for v2.0 (patch-less) in" - + " /output; default is {}" - ), - "version_cat": ( - "catalogue major version, allowed are v1.3, v1.4, v1.5, v1.6," - + " v2.0; v2.0 is patch-less; default is {}" - ), - "mode": ( - "run mode, allowed are 'merge', 'test'; default is" + " '{}'" - ), - "psf": "PSF model, allowed are 'psfex' and 'mccd'; default is {}", - "patches": "(list of) input patches; ignored for v2.0", - } - - # Output column names with types - self._dt = [ - ("X", float), - ("Y", float), - ("RA", float), - ("DEC", float), - ("E1_PSF_HSM", float), - ("E2_PSF_HSM", float), - ("SIGMA_PSF_HSM", float), - ("FLAG_PSF_HSM", float), - ("E1_STAR_HSM", float), - ("E2_STAR_HSM", float), - ("SIGMA_STAR_HSM", float), - ("FLAG_STAR_HSM", float), - ("CCD_NB", int), - ] - - # Extra columns for MCCD:737 - self._dt_mccd = self._dt.copy() - self._dt_mccd.append(("GLOB_X", float)) - self._dt_mccd.append(("GLOB_Y", float)) - - def update_params(self): - """Update Params. - - Update parameters. - - """ - if self._params["psf"] == "psfex": - #self._params["sub_dir_pattern"] = "run_sp_exp_202" - self._params["sub_dir_pattern"] = "run_sp_combined_psf" - self._params["sub_dir_psfint"] = "psfex_interp_runner" - elif self._params["psf"] == "mccd": - self._params["sub_dir_pattern"] = "run_sp_exp_SxSePsf_202" - self._params["sub_dir_psfint"] = "mccd_fit_val_runner" - self._params["sub_dir_setools"] = "setools_runner/output/mask" - else: - raise ValueError(f"Invalid PSF model {self._params['psf']}") - self._params["sub_dir_psfint"] = ( - f"{self._params['sub_dir_psfint']}/output" - ) - - def run(self): - """Run. - - Main processing function. - - """ - # Guard against a mistyped version silently falling through to the - # v1.x patch loop (e.g. ``-V v2`` or ``-V 2.0``). - allowed_versions = ("v1.3", "v1.4", "v1.5", "v1.6", "v2.0") - if self._params["version_cat"] not in allowed_versions: - raise ValueError( - f"Invalid version {self._params['version_cat']}; allowed are" - + f" {', '.join(allowed_versions)}" - ) - - # v2.0 removes the patch concept: a single patch-less run root. For - # v1.x, iterate over the requested sky patches as before. ``patch`` is - # None in the patch-less case, which drops the patch token from the - # input path and the output filename. - if self._params["version_cat"] == "v2.0": - patch_nums = [None] - elif self._params["mode"] == "test": - patch_nums = ["3", "4"] - else: - patch_nums = cs_args.my_string_split(self._params["patches"]) - - do_parallel = True - - # Loop over patches - for patch in patch_nums: - - patch_dir, output_dir = collate_paths( - self._params["input_base_dir"], - self._params["output_base_dir"], - patch, - ) - print("Running patch-less (v2.0)" if patch is None else f"Running patch: {patch}") - - if not os.path.isdir(output_dir): - os.makedirs(output_dir, exist_ok=True) - - subdirs = f"{patch_dir}/{self._params['sub_dir_pattern']}*" - exp_run_dirs = glob.glob(subdirs) - n_exp_runs = len(exp_run_dirs) - print( - f"Found {n_exp_runs} input single-exposure run(s) for patch" - + f" {patch_dir} ({subdirs})" - ) - - if self._params["mode"] == "test": - exp_run_dirs = exp_run_dirs[:2] - n_exp_runs = len(exp_run_dirs) - print( - f"test mode: only using {n_exp_runs} input single-exposure" - + f" runs" - ) - - # Loop over exposure runs - if not do_parallel: - for idx_exp, exp_run_dir in tqdm( - enumerate(exp_run_dirs), - total=n_exp_runs, - disable=self._params["verbose"], - ): - self.transform_exposures( - output_dir, patch, idx_exp, exp_run_dir - ) - else: - res = Parallel(n_jobs=-1, backend="loky")( - delayed(self.transform_exposures)( - output_dir, patch, idx_exp, exp_run_dir - ) - for idx_exp, exp_run_dir in tqdm( - enumerate(exp_run_dirs), - total=n_exp_runs, - disable=self._params["verbose"], - ) - ) - - def transform_exposures(self, output_dir, patch, idx, exp_run_dir): - """Transform exposures. - - Transform shapes for exposure for a given run (input exp run dir). - - """ - output_path = ( - f"{output_dir}/" - + output_filename( - self._params["file_pattern_psfint"], patch, idx - ) - ) - if os.path.exists(output_path): - print(f"Skipping transform_exposures, file {output_path} exists") - return - - psf_dir = f"{exp_run_dir}/{self._params['sub_dir_psfint']}" - try: - all_files = os.listdir(psf_dir) - if self._params["verbose"]: - print(f"Found {len(all_files)} file(s) in {psf_dir}") - except Exception: - if self._params["verbose"]: - print(f"Found zero PSFEx files in {psf_dir}, skipping") - return - - cat_list = [] - for file_name in all_files: - if self._params["file_pattern_psfint"] not in file_name: - continue - - tmp = re.findall(r"\d+", file_name) - - if self._params["psf"] == "psfex": - exp_name, ccd_id = int(tmp[0]), int(tmp[1]) - elif self._params["psf"] == "mccd": - exp_name = int(tmp[0]) - ccd_id = -1 - - if self._params["verbose"]: - print("Match found ", exp_name, ccd_id) - - psf_file_path = f"{psf_dir}/{file_name}" - - try: - if self._params["psf"] == "psfex": - psf_file_hdus = fits.open(psf_file_path, memmap=False) - psf_file = psf_file_hdus[2].data - psf_file_hdus.close() - mod = "RA" - else: - psf_file = fits.getdata(psf_file_path, 1, memmap=True) - mod = "RA_LIST" - except Exception: - continue - - if self._params["psf"] == "psfex": - # HSM ellipticities and sizes are measured directly in world - # coordinates upstream (FindAdaptiveMom use_sky_coords=True), so - # they are passed straight through; only positions are collated. - exp_cat = np.array( - list( - map( - tuple, - np.array( - [ - psf_file["X"], - psf_file["Y"], - psf_file["RA"], - psf_file["DEC"], - psf_file["E1_PSF_HSM"], - psf_file["E2_PSF_HSM"], - psf_file["SIGMA_PSF_HSM"], - psf_file["FLAG_PSF_HSM"], - psf_file["E1_STAR_HSM"], - psf_file["E2_STAR_HSM"], - psf_file["SIGMA_STAR_HSM"], - psf_file["FLAG_STAR_HSM"], - np.ones_like(psf_file["RA"], dtype=int) - * ccd_id, - ] - ).T.tolist(), - ) - ), - dtype=self._dt, - ) - cat_list.append(exp_cat) - - else: - l2g = Loc2Glob() - g2c = Glob2CCD(l2g) - new_ccd_id = np.array( - [ - int( - g2c.get_ccd_n( - psf_file["GLOB_POSITION_IMG_LIST"][ii, 0], - psf_file["GLOB_POSITION_IMG_LIST"][ii, 1], - ) - ) - for ii in range(len(psf_file)) - ] - ) - - # Local-to-CCD position: subtract each CCD's focal-plane shift. - new_x = np.zeros_like(psf_file[mod]) - new_y = np.zeros_like(psf_file[mod]) - - # The MCCD PSF_MOM_LIST/STAR_MOM_LIST columns come from the - # external mccd fit-validation code, which still measures HSM - # moments in the pixel frame; rotate them into world coordinates - # via the per-CCD WCS Jacobian. This rotation stays until mccd - # itself adopts use_sky_coords (see the module docstring). The - # in-repo PSFEx / MCCD-interpolation paths are already in world - # coordinates and are passed through unrotated. - new_e1_psf = np.zeros_like(psf_file[mod]) - new_e2_psf = np.zeros_like(psf_file[mod]) - new_sig_psf = np.zeros_like(psf_file[mod]) - new_e1_star = np.zeros_like(psf_file[mod]) - new_e2_star = np.zeros_like(psf_file[mod]) - new_sig_star = np.zeros_like(psf_file[mod]) - new_flag_psf = np.zeros_like(psf_file[mod]) - new_flag_star = np.zeros_like(psf_file[mod]) - for ccd_id in range(40): - m_ccd_id = new_ccd_id == ccd_id - if sum(m_ccd_id) == 0: - continue - - x_shift, y_shift = l2g.shift_coord(ccd_id) - - new_x[m_ccd_id] = ( - psf_file["GLOB_POSITION_IMG_LIST"][:, 0][m_ccd_id] - - x_shift - ) - new_y[m_ccd_id] = ( - psf_file["GLOB_POSITION_IMG_LIST"][:, 1][m_ccd_id] - - y_shift - ) - - header_file_path = ( - self._params["sub_dir_setools"] - + self._params["file_pattern_psfint"] - + f"{exp_name}-{ccd_id}.fits" - ) - try: - header_file = fits.getdata(header_file_path, 1) - except Exception: - continue - header = fits.Header.fromstring( - "\n".join(header_file[0][0]), sep="\n" - ) - wcs = galsim.AstropyWCS(header=header) - - g1_psf_tmp_l = [] - g2_psf_tmp_l = [] - sig_psf_tmp_l = [] - g1_star_tmp_l = [] - g2_star_tmp_l = [] - sig_star_tmp_l = [] - flag_psf_tmp_l = [] - flag_star_tmp_l = [] - - for obj in psf_file[m_ccd_id]: - try: - jac = wcs.jacobian( - world_pos=galsim.CelestialCoord( - ra=obj["RA_LIST"] * galsim.degrees, - dec=obj["DEC_LIST"] * galsim.degrees, - ) - ) - except Exception: - flag_star_tmp_l.append(16) - flag_psf_tmp_l.append(16) - g1_psf_tmp_l.append(0) - g2_psf_tmp_l.append(0) - sig_psf_tmp_l.append(0) - g1_star_tmp_l.append(0) - g2_star_tmp_l.append(0) - sig_star_tmp_l.append(0) - continue - g1_psf_tmp, g2_psf_tmp, sig_psf_tmp = transform_shape( - obj["PSF_MOM_LIST"], jac - ) - - g1_psf_tmp_l.append(g1_psf_tmp) - g2_psf_tmp_l.append(g2_psf_tmp) - sig_psf_tmp_l.append(sig_psf_tmp) - flag_psf_tmp_l.append(obj["PSF_MOM_LIST"][3]) - - g1_star_tmp, g2_star_tmp, sig_star_tmp = ( - transform_shape(obj["STAR_MOM_LIST"], jac) - ) - g1_star_tmp_l.append(g1_star_tmp) - g2_star_tmp_l.append(g2_star_tmp) - sig_star_tmp_l.append(sig_star_tmp) - flag_star_tmp_l.append(obj["STAR_MOM_LIST"][3]) - - new_e1_psf[m_ccd_id] = g1_psf_tmp_l - new_e2_psf[m_ccd_id] = g2_psf_tmp_l - new_sig_psf[m_ccd_id] = sig_psf_tmp_l - new_flag_psf[m_ccd_id] = flag_psf_tmp_l - new_e1_star[m_ccd_id] = g1_star_tmp_l - new_e2_star[m_ccd_id] = g2_star_tmp_l - new_sig_star[m_ccd_id] = sig_star_tmp_l - new_flag_star[m_ccd_id] = flag_star_tmp_l - - exp_cat = np.array( - list( - map( - tuple, - np.array( - [ - new_x, - new_y, - psf_file["RA_LIST"], - psf_file["DEC_LIST"], - new_e1_psf, - new_e2_psf, - new_sig_psf, - psf_file["PSF_MOM_LIST"][:, 3], - new_e1_star, - new_e2_star, - new_sig_star, - psf_file["STAR_MOM_LIST"][:, 3], - new_ccd_id, - psf_file["GLOB_POSITION_IMG_LIST"][:, 0], - psf_file["GLOB_POSITION_IMG_LIST"][:, 1], - ] - ).T.tolist(), - ) - ), - dtype=self._dt_mccd, - ) - cat_list.append(exp_cat) - - del psf_file - - if len(cat_list) == 0: - return - - # Finalize catalogue - patch_cat = np.concatenate(cat_list) - hdul = fits.HDUList() - hdul.append(fits.PrimaryHDU()) - hdul.append(fits.BinTableHDU(patch_cat)) - - # Write catalogue - hdul.writeto( - output_path, - overwrite=True, - ) - - del cat_list - del hdul - gc.collect() - - -def run_convert(*args): - - # Create instance - obj = Convert() - - obj.set_params_from_command_line(args) - obj.update_params() - - obj.run() - - -def main(argv=None): - """Main - - Main program - """ - if argv is None: - argv = sys.argv[1:] - run_convert(*argv) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py deleted file mode 100755 index 05e3d3bb9..000000000 --- a/scripts/python/create_star_cat.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script create_star_cat.py - -:Description: Create reference star catalogue for masking of -bright star halos and diffraction spikes - -:Authors: Axel Guinot, Martin Kilbinger - -""" - - -import os -import re -import sys - -from cs_util import args as cs_args -from cs_util import logging as cs_logging - -from astropy.io import fits - -from shapepipe.utilities.file_io import write_atomic -from shapepipe.utilities.focal_plane import ccd_center_and_radius, focal_plane_disc -from shapepipe.utilities.vizier import query_vizier as _query_vizier - - -# GSC 2.3 catalog ID -CDS_CAT_ID = "I/305/out" - - -def query_vizier(ra, dec, radius_arcmin): - return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) - - -def main(input_dir, output_dir, kind): - - file_list = os.listdir(input_dir) - - for f in file_list: - if "image" not in f: - continue - - img_number = re.split("image", os.path.splitext(f)[0])[1] - fpath = os.path.join(input_dir, f) - - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - if kind == "exp": - # One query covering the full MegaCam focal plane - ra, dec, radius_deg = focal_plane_disc(fpath) - radius = radius_deg * 60.0 - print( - f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" - ) - else: - # A single image: its own centre and half-diagonal. - ra, dec, radius_deg = ccd_center_and_radius(fits.getheader(fpath, 0)) - radius = radius_deg * 60.0 - - table = query_vizier(ra, dec, radius) - write_atomic(table, output_name) - - return 0 - - -def params_default(): - """Return default parameters, short options, types, and help strings.""" - _params = { - "input_dir": ".", - "output_dir": ".", - "kind": "exp", - } - _short_options = { - "input_dir": "-i", - "output_dir": "-o", - "kind": "-k", - } - _types = {} - _help_strings = { - "input_dir": "input directory containing image files; default is {}", - "output_dir": "output directory for star catalogues; default is {}", - "kind": "processing kind, 'exp' for full MegaCam focal plane, 'tile' for single image; default is {}", - } - return _params, _short_options, _types, _help_strings - - -if __name__ == "__main__": - - _params, _short_options, _types, _help_strings = params_default() - - options = cs_args.parse_options( - _params, - _short_options, - _types, - _help_strings, - ) - - cs_logging.log_command(sys.argv) - - main(options["input_dir"], options["output_dir"], options["kind"]) diff --git a/src/shapepipe/modules/mask_ext_package/__init__.py b/src/shapepipe/modules/mask_ext_package/__init__.py deleted file mode 100644 index 4e2123986..000000000 --- a/src/shapepipe/modules/mask_ext_package/__init__.py +++ /dev/null @@ -1,71 +0,0 @@ -"""MASK EXT MODULE. - -This package contains the module for ``mask_ext``. - -:Author: Cail Daley - -:Parent module: ``split_exp_runner`` (exposures) or ``get_images_runner`` / - ``uncompress_fits_runner`` (tiles), or None - -:Input: Single image (tile or single-exposure single-CCD) and, optionally, an - external instrument flag file - -:Output: Per-image pixel flag file - -Description -=========== - -This module produces the ShapePipe per-image pixel flag file -``_flag.fits`` by rasterizing an **external healsparse mask** onto -the image pixel grid, rather than generating masks internally (the job of the -``mask`` module). It is the ShapePipe consumer of the unified UNIONS healsparse -mask products (PhotoPipe footprint + bright stars, MaxiMask, manual galaxy -masks), merged and stored as ``healsparse.HealSparseMap`` files. - -For each image the module: - -1. Reads the image header into an ``astropy.wcs.WCS``. -2. Evaluates the pixel grid to (RA, Dec) in row chunks (bounded memory). -3. Queries ``HealSparseMap.get_values_pos`` at every pixel centre. -4. Maps healsparse bit values to ShapePipe flag values through the - config-driven ``BIT_FLAG_MAP``, producing an ``int16`` flag image. -5. Optionally sums in an external instrument flag image (``USE_EXT_FLAG``), - matching the combination semantics of the ``mask`` module. -6. Writes ``_flag.fits`` carrying the image WCS in its header. - -Everything downstream (SExtractor ``IMAFLAGS_ISO``, setools star selection, -vignetmaker, ngmix) consumes this artifact unchanged; the module is a drop-in -replacement for ``mask`` at the flag-image contract. - -The same module serves tiles and exposure CCDs: only the input image (hence -WCS) differs. The healsparse file, its bit meanings, and all flag values live -**only in config** — the mask products evolve and are swapped at zero code cost. - -Module-specific config file entries -=================================== - -MASK_PATH : str - Path to the healsparse mask file (``.hsp``/``.fits``); band-agnostic -BIT_FLAG_MAP : str - Mapping from healsparse pixel bit value to output flag value, formatted as - ``":, :, ..."`` (e.g. ``"64:1, 2048:2"``). A pixel - carrying several bits receives the bitwise-OR of the mapped flag values. -OFF_MAP_FLAG : int, optional - Flag value assigned to pixels that fall outside the healsparse map - footprint (sentinel pixels); default is ``0``. Off-footprint pixels are - usually unobserved, so a non-zero value flags them. -USE_EXT_FLAG : bool, optional - If ``True``, sum an external instrument flag file (given as the second - input) into the rasterized mask; default is ``False``. Needed for - exposures, where saturation / bleeding / bad columns arrive with the data. -HDU : int, optional - HDU of the external instrument flag FITS file; default is ``0`` -PREFIX : str, optional - Prefix prepended to the output file name base ``flag``; default is ``""`` -CHUNK_SIZE : int, optional - Number of image rows evaluated per chunk; default is chosen from the image - size to bound memory (``1e8`` px for tiles, ``1e7`` px for exposure CCDs) - -""" - -__all__ = ["mask_ext"] diff --git a/src/shapepipe/modules/mask_ext_package/mask_ext.py b/src/shapepipe/modules/mask_ext_package/mask_ext.py deleted file mode 100644 index c24aaabcd..000000000 --- a/src/shapepipe/modules/mask_ext_package/mask_ext.py +++ /dev/null @@ -1,333 +0,0 @@ -"""MASK EXT. - -This module contains a class to rasterize an external healsparse mask onto an -image pixel grid, producing the ShapePipe per-image pixel flag file. - -:Author: Cail Daley - -""" - -import re - -import numpy as np -from astropy import wcs - -from shapepipe.pipeline import file_io - -# Per-chunk pixel budget: chunk the image in row bands so that no more than -# this many pixel coordinates are held / queried at once. A tile is ~1e8 px and -# an exposure CCD ~1e7 px; this bound keeps peak memory to a single band. -DEFAULT_PIXEL_BUDGET = 10_000_000 - - -class MaskExt(object): - """Mask Ext. - - Rasterize an external healsparse mask onto an image pixel grid and write - the resulting ShapePipe flag file. - - Parameters - ---------- - image_path : str - Path to the image whose pixel grid and WCS define the output - mask_path : str - Path to the external healsparse mask file - bit_flag_map : dict - Mapping from healsparse pixel bit value (int) to output flag value - (int); a pixel carrying several bits gets the bitwise-OR of the mapped - flags - image_num : str - File number string, inserted into the output file name - output_dir : str - Path to the output directory - w_log : logging.Logger - Log file - off_map_flag : int, optional - Flag value for pixels outside the healsparse footprint (sentinel - pixels); default is ``0`` - path_external_flag : str, optional - Path to an external instrument flag file to sum into the mask; default - is ``None`` (not used) - image_prefix : str, optional - Prefix prepended to the output file name base ``flag``; specify - ``'none'`` or ``''`` for no prefix, default is ``''`` - outname_base : str, optional - Output file name base, default is ``flag`` - chunk_size : int, optional - Number of image rows evaluated per chunk; default is derived from the - image width and :data:`DEFAULT_PIXEL_BUDGET` - hdu : int, optional - HDU of the external instrument flag FITS file; default is ``0`` - - """ - - def __init__( - self, - image_path, - mask_path, - bit_flag_map, - image_num, - output_dir, - w_log, - off_map_flag=0, - path_external_flag=None, - image_prefix="", - outname_base="flag", - chunk_size=None, - hdu=0, - ): - self._image_path = image_path - self._mask_path = mask_path - self._bit_flag_map = bit_flag_map - self._img_number = image_num - self._output_dir = output_dir - self._w_log = w_log - self._off_map_flag = int(off_map_flag) - self._path_external_flag = path_external_flag - - if (image_prefix.lower() != "none") and (image_prefix != ""): - self._img_prefix = f"{image_prefix}_" - else: - self._img_prefix = "" - - self._outname_base = outname_base - self._chunk_size = chunk_size - self._hdu = hdu - - self._set_image_coordinates() - - @staticmethod - def parse_bit_flag_map(map_string): - """Parse Bit Flag Map. - - Parse the ``BIT_FLAG_MAP`` config string into a dictionary. - - Parameters - ---------- - map_string : str - Mapping formatted as ``":, :, ..."``, e.g. - ``"64:1, 2048:2"`` - - Returns - ------- - dict - Mapping from healsparse bit value (int) to output flag value (int) - - Raises - ------ - ValueError - If an entry is not of the form ``:`` - - """ - bit_flag_map = {} - for entry in map_string.split(","): - entry = entry.strip() - if not entry: - continue - if not re.fullmatch(r"\d+\s*:\s*\d+", entry): - raise ValueError( - f"Invalid BIT_FLAG_MAP entry '{entry}'; expected " - + "':'" - ) - bit, flag = (int(part) for part in entry.split(":")) - bit_flag_map[bit] = flag - if not bit_flag_map: - raise ValueError("BIT_FLAG_MAP is empty") - return bit_flag_map - - def _set_image_coordinates(self): - """Set Image Coordinates. - - Read the image header into a WCS and record the image shape, mirroring - ``Mask._set_image_coordinates``. - - """ - img = file_io.FITSCatalogue(self._image_path, hdu_no=0) - img.open() - self._header = img.get_header() - # get_data().shape is (n_y, n_x) - self._img_shape = img.get_data().shape - img.close() - del img - - self._wcs = wcs.WCS(self._header) - - def _default_chunk_size(self): - """Default Chunk Size. - - Rows per chunk such that one band holds at most - :data:`DEFAULT_PIXEL_BUDGET` pixels. - - Returns - ------- - int - Number of rows per chunk (at least 1) - - """ - n_x = self._img_shape[1] - return max(1, DEFAULT_PIXEL_BUDGET // n_x) - - def _map_bits_to_flags(self, bit_values, off_map): - """Map Bits to Flags. - - Translate healsparse bit values to output flag values through the - bit→flag mapping, OR-combining every matching bit, and assign the - off-map flag to sentinel pixels. - - Parameters - ---------- - bit_values : numpy.ndarray - Healsparse values queried at the pixel centres - off_map : numpy.ndarray - Boolean mask, ``True`` where the pixel lies outside the footprint - - Returns - ------- - numpy.ndarray - Output flag values (``int16``), same shape as ``bit_values`` - - """ - flags = np.zeros(bit_values.shape, dtype=np.int16) - for bit, flag in self._bit_flag_map.items(): - flags[(bit_values & bit) != 0] |= np.int16(flag) - if self._off_map_flag != 0: - flags[off_map] = np.int16(self._off_map_flag) - return flags - - def rasterize(self): - """Rasterize. - - Evaluate the pixel grid to (RA, Dec) in row chunks, query the - healsparse mask, and build the ``int16`` flag image. - - Returns - ------- - numpy.ndarray - The rasterized flag image, shape ``(n_y, n_x)``, dtype ``int16`` - - """ - # Lazy import: healsparse is an optional heavy dependency, imported at - # use rather than module load. - import healsparse - - hmap = healsparse.HealSparseMap.read(self._mask_path) - sentinel = hmap.sentinel - - # Mask products come in two flavours: integer bit-flag maps (per-band - # bits, e.g. 64 = r) and boolean maps (True = masked, e.g. the - # candide copy of the 2025 r-band mask). For a boolean map the only - # meaningful bit is 1 (True); any other bit silently selects nothing - # (``True & 64 == 0`` -> an all-clean flag image), so fail loudly. - if hmap.dtype == np.bool_: - bad_bits = [bit for bit in self._bit_flag_map if bit != 1] - if bad_bits: - raise ValueError( - f"Mask {self._mask_path} is a boolean healsparse map; " - + f"BIT_FLAG_MAP bits {bad_bits} would never match " - + "(use '1:' for boolean masks)" - ) - - n_y, n_x = self._img_shape - chunk_size = self._chunk_size or self._default_chunk_size() - - flag_image = np.zeros((n_y, n_x), dtype=np.int16) - # Column indices are shared across every row band. - x = np.arange(n_x) - - for y0 in range(0, n_y, chunk_size): - y1 = min(y0 + chunk_size, n_y) - yy, xx = np.meshgrid(np.arange(y0, y1), x, indexing="ij") - # WCS uses 0-based pixel coordinates here (origin=0). - ra, dec = self._wcs.all_pix2world(xx.ravel(), yy.ravel(), 0) - # Normalize RA into [0, 360) so the wrap at 0/360 is handled; - # healsparse expects lon in that range with lonlat=True. - ra = np.mod(ra, 360.0) - - bit_values = hmap.get_values_pos(ra, dec, lonlat=True) - off_map = bit_values == sentinel - - band = self._map_bits_to_flags(bit_values, off_map) - flag_image[y0:y1, :] = band.reshape(y1 - y0, n_x) - - return flag_image - - def _combine_external_flag(self, flag_image): - """Combine External Flag. - - Sum an external instrument flag image into the rasterized mask, - matching ``Mask._build_final_mask`` semantics (element-wise sum of the - two integer flag images). - - Parameters - ---------- - flag_image : numpy.ndarray - The rasterized flag image - - Returns - ------- - numpy.ndarray - The combined flag image (``int16``) - - """ - external_flag = file_io.FITSCatalogue( - self._path_external_flag, - hdu_no=self._hdu, - ) - external_flag.open() - ext_flag = external_flag.get_data()[:, :] - external_flag.close() - - return (flag_image + ext_flag).astype(np.int16, copy=False) - - def _output_path(self): - """Output Path. - - Full path of the output flag file, matching the ``mask`` module naming - (``_flag.fits``). - - Returns - ------- - str - Output file path - - """ - name = ( - f"{self._img_prefix}{self._outname_base}" - + f"{self._img_number}.fits" - ) - return f"{self._output_dir}/{name}" - - def make_mask(self): - """Make Mask. - - Rasterize the healsparse mask, optionally combine the external - instrument flag, and write the flag file carrying the image WCS. - - Returns - ------- - str - Path to the written flag file - - """ - flag_image = self.rasterize() - - if self._path_external_flag is not None: - flag_image = self._combine_external_flag(flag_image) - - output_path = self._output_path() - out = file_io.FITSCatalogue( - output_path, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - hdu_no=0, - ) - out.save_as_fits( - data=flag_image, - image=True, - image_header=self._wcs.to_header(), - ) - - self._w_log.info( - f"Wrote healsparse-derived flag file {output_path}" - ) - - return output_path diff --git a/src/shapepipe/modules/mask_ext_runner.py b/src/shapepipe/modules/mask_ext_runner.py deleted file mode 100644 index a43265d1b..000000000 --- a/src/shapepipe/modules/mask_ext_runner.py +++ /dev/null @@ -1,117 +0,0 @@ -"""MASK EXT RUNNER. - -Module runner for ``mask_ext``. - -:Author: Cail Daley - -""" - -from shapepipe.modules.mask_ext_package.mask_ext import MaskExt -from shapepipe.modules.module_decorator import module_runner - - -@module_runner( - version="1.0", - file_pattern=["image", "flag"], - file_ext=[".fits", ".fits"], - depends=["numpy", "astropy", "healsparse"], - numbering_scheme="_0", -) -def mask_ext_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Mask Ext Runner. - - Rasterize an external healsparse mask onto the input image pixel grid and - write the ShapePipe flag file. - - Notes - ----- - Only the image is strictly required: it supplies the pixel grid and WCS - onto which the healsparse mask is rasterized. Unlike the ``mask`` module, - no weight file is used — the healsparse mask already encodes the footprint, - and missing-data handling is not this module's concern. A second input, an - external instrument flag file, is consumed only when ``USE_EXT_FLAG`` is - ``True`` (needed for exposures, where saturation / bleeding / bad columns - arrive with the data). - - """ - n_inputs = len(input_file_list) - use_ext_flag = config.getboolean(module_config_sec, "USE_EXT_FLAG") if ( - config.has_option(module_config_sec, "USE_EXT_FLAG") - ) else False - - if use_ext_flag: - if n_inputs != 2: - raise ValueError( - f"Found {n_inputs} inputs but USE_EXT_FLAG is True, which " - + 'expects "image" and "flag" in the MASK_EXT_RUNNER section ' - + "of the config file." - ) - image_path = input_file_list[0] - ext_flag_name = input_file_list[1] - else: - if n_inputs != 1: - raise ValueError( - f"Found {n_inputs} inputs but USE_EXT_FLAG is False, which " - + 'expects only "image" in the MASK_EXT_RUNNER section of the ' - + "config file." - ) - image_path = input_file_list[0] - ext_flag_name = None - - # Path to the healsparse mask file - mask_path = config.getexpanded(module_config_sec, "MASK_PATH") - - # Bit -> flag mapping - bit_flag_map = MaskExt.parse_bit_flag_map( - config.get(module_config_sec, "BIT_FLAG_MAP") - ) - - # Flag value for pixels outside the healsparse footprint - if config.has_option(module_config_sec, "OFF_MAP_FLAG"): - off_map_flag = config.getint(module_config_sec, "OFF_MAP_FLAG") - else: - off_map_flag = 0 - - # HDU of the external instrument flag file - if config.has_option(module_config_sec, "HDU"): - hdu = config.getint(module_config_sec, "HDU") - else: - hdu = 0 - - # Output file name prefix - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - else: - prefix = "" - - # Rows per chunk (optional; default derived from image size) - if config.has_option(module_config_sec, "CHUNK_SIZE"): - chunk_size = config.getint(module_config_sec, "CHUNK_SIZE") - else: - chunk_size = None - - mask_inst = MaskExt( - image_path, - mask_path, - bit_flag_map, - file_number_string, - run_dirs["output"], - w_log, - off_map_flag=off_map_flag, - path_external_flag=ext_flag_name, - image_prefix=prefix.replace(" ", ""), - outname_base="flag", - chunk_size=chunk_size, - hdu=hdu, - ) - - mask_inst.make_mask() - - return None, None diff --git a/src/shapepipe/modules/mask_package/__init__.py b/src/shapepipe/modules/mask_package/__init__.py deleted file mode 100644 index bdfece65b..000000000 --- a/src/shapepipe/modules/mask_package/__init__.py +++ /dev/null @@ -1,185 +0,0 @@ -"""MASK MODULE. - -This package contains the module for ``mask``. - -:Author: Axel Guinot - -:Parent module: ``split_exp_runner`` or None - -:Input: Single-exposure single-CCD image, weight file, flag file (optional), - and star catalogue (optional) - -:Output: Single-exposure single-CCD flag files - -Description -=========== - -This module creates masks for bright stars, diffraction spikes, deep sky -objects (from the Messier and NGC catalogues), borders, and other artifacts. If -a flag file is given as input, for example from pre-processing, the mask that -is created by this module is joined with the mask from this external flag file. -In this case the config flag ``USE_EXT_FLAG`` needs to be set to ``True``. To -distinguish the newly created output flag file from the input ones, a prefix -can added as specificed by the config entry ``PREFIX``. - -An NGC catalogue with positions, sizes, and types is provided with -``shapepipe``, -`source `_. - -Masked pixels of different mask types are indicated by integers, which -conveniently are powers of two such that they can be combined bit-wise. - -To mask bright stars, this module either creates a star catalogue from the -online -`guide star catalogue `_ -database relevant to the the footprint. This is done by calling a CDs -(Centre de Données astronomique de Strasbourg) -`client program `_. -Note that this requires online access, -which in some cases is not granted on compute nodes of a cluster. In this case, -set the config flag ``USE_EXT_STAR = False``. Alternatively, a star -catalogue can be created before running this module via the script -``create_star_cat``. During the processing of this module, this star catalogue -is read from disk, with ``USE_SET_STAR = True``. - -The masking is done with the software ``WeightWatcher`` :cite:`marmo:08`, -which is installed by ``ShapePipe`` by default. - -Module-specific config file entries -=================================== - -USE_EXT_FLAG : bool - Use external flag file to join with the mask created here; - if ``True`` flag file needs to be given on input -USE_EXT_STAR : bool - Read external star catalogue instead of creating one during the - call of this module; - if ``True`` star catalogue file needs to be given on input -MASK_CONFIG_PATH : str - Path to mask config file -HDU : int, optional - HDU of external flag FITS file; the default value is ``0`` -PREFIX : str, optional - Prefix to be appended to output file name ``flag``; - helps to distinguish the file patterns of newly created and external - mask files -CHECK_EXISTING_DIR : str, optional - If given, search this directory for existing mask files; the - corresponding images will then not be processed - -Mask config file -================ - -An additional configuration file is used by the mask module, its path is -``MASK_CONFIG_PATH`` in the module config section, see above. The following -describes the config file sections and their entries. - -[PROGRAM_PATH] --------------- - -WW_PATH : str, optional - Full path to the WeightWatcher executable (``ww``) on the system ; if not - set the version controlled WeightWatcher installation in the ShapePipe - environment will be used -WW_CONFIG_FILE : str - Path to the WeightWatcher configuration file -CDSCLIENT_PATH : str, optional - Path to CDS client executable; required if ``USE_EXT_STAR = False`` - -[BORDER_PARAMETERS] -------------------- - -BORDER_MAKE : bool - Create mask around borders if ``True`` -BORDER_WIDTH : int - Width of border mask in pixels -BORDER_FLAG_VALUE : int - Border mask pixel value, power of 2 - -[HALO_PARAMETERS] ------------------ - -HALO_MAKE : bool - Create mask for halos of bright stars if ``True`` -HALO_MASKMODEL_PATH : str - Path to halo mask geometry (``.reg`` file) -HALO_MAG_LIM : float - Faint stellar magnitude limit for halo mask -HALO_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and halo mask size -HALO_MAG_PIVOT : float - Pivot stellar magnitude -HALO_FLAG_VALUE : int - Halo mask pixel value, power of 2 -HALO_REG_FILE : str - Output halo mask ``.reg`` file - -[SPIKE_PARAMETERS] ------------------- - -SPIKE_MAKE : bool - Create mask for diffraction spikes of bright stars if ``True`` -SPIKE_MASKMODEL_PATH : str - Path to diffraction spike geometry (``.reg`` file) -SPIKE_MAG_LIM : - Faint stellar magnitude limit for spike mask -SPIKE_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and spike mask size -SPIKE_MAG_PIVOT : float - Pivot stellar magnitude -SPIKE_FLAG_VALUE : int - Diffraction spike pixel value, power of two -SPIKE_REG_FILE : str - Output spike mask ``.reg`` file - -[MESSIER_PARAMETERS] --------------------- - -MESSIER_MAKE : bool - Create mask around Messier objects if ``True`` -MESSIER_CAT_PATH : str - Path to Messier catalogue -MESSIER_SIZE_PLUS : float - Fraction to increase Messier mask -MESSIER_FLAG_VALUE : int - Messier mask pixel value, power of 2 - -[NGC_PARAMETERS] --------------------- - -NGC_MAKE : bool - Create mask around NGC objects if ``True`` -NGC_CAT_PATH : str - Path to NGC catalogue -NGC_SIZE_PLUS : float - Fraction to increase NGC mask -NGC_FLAG_VALUE : int - NGC mask pixel value, power of 2 - -[MD_PARAMETERS] ---------------- - -MD_MAKE : bool - Account for missing data (zero-valued pixels) if ``True`` -MD_THRESH_FLAG : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked as flagged -MD_THRESH_REMOVE : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked for removal -MD_REMOVE : bool - Image is removed if marked for removal - -[OTHER] -------- - -TEMP_DIRECTORY : str - Path to temporary dictionary -KEEP_INDIVIDUAL_MASK : bool - Keep individual masks in addition to merged mask file -KEEP_REG_FILE : bool - Keep ``.reg`` mask file - -""" - -__all__ = ["mask"] diff --git a/src/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py deleted file mode 100644 index ba6d586ad..000000000 --- a/src/shapepipe/modules/mask_package/mask.py +++ /dev/null @@ -1,1271 +0,0 @@ -"""MASK. - -This module contains a class to create star mask for an image. - -:Authors: Axel Guinot, Martin Kilbinger - -""" - -import os -import re - -import numpy as np -from astropy import units, wcs -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.table import Table - -from shapepipe.pipeline import file_io -from shapepipe.pipeline.config import CustomParser -from shapepipe.pipeline.execute import execute -from shapepipe.utilities.file_system import mkdir -from shapepipe.utilities.vizier import query_vizier - - -class Mask(object): - """Mask. - - Class to create mask based on a star catalogue. - - Parameters - ---------- - image_path : str - Path to image (FITS format) - weight_path : str - Path to the weight image (FITS format) - image_prefix : str - Prefix to input image name, specify as ``'none'`` for no prefix - image_num : str - File number identified - config_filepath : str - Path to the ``.mask`` config file - output_dir : str - Path to the output directory - w_log : logging.Logger - Log file - path_external_flag : str, optional - Path to external flag file, default is ``None`` (not used) - outname_base : str, optional - Output file name base, default is ``flag`` - check_existing_dir : str, optional - If not ``None`` (default), search path for existing mask files - star_cat_path : str, optional - Path to external star catalogue, default is ``None`` (not used; - instead the star catalogue is produced on the fly at run time) - hdu : int, optional - HDU number, default is ``0`` - - """ - - def __init__( - self, - image_path, - weight_path, - image_prefix, - image_num, - config_filepath, - output_dir, - w_log, - path_external_flag=None, - outname_base="flag", - check_existing_dir=None, - star_cat_path=None, - hdu=0, - ): - - # Path to the image to mask - self._image_fullpath = image_path - - # Path to the weight associated to the image - self._weight_fullpath = weight_path - - # Input image prefix - if (image_prefix.lower() != "none") and (image_prefix != ""): - self._img_prefix = f"{image_prefix}_" - else: - self._img_prefix = "" - - # File number identified - self._img_number = image_num - - # Path to mask config file - self._config_filepath = config_filepath - - # Path to the output directory - self._output_dir = output_dir - - # Log file - self._w_log = w_log - - # Path to an external flag file - self._path_external_flag = path_external_flag - - # Output file base name - self._outname_base = outname_base - - # Search path for existing mask files - self._check_existing_dir = check_existing_dir - - # Set external star catalogue path if given - if star_cat_path is not None: - self._star_cat_path = star_cat_path - - self._hdu = hdu - - # Read mask config file - self._get_config() - - # Set parameters needed for the star detection - self._set_image_coordinates() - - # Set error flag - self._err = False - - # Guide Star Catalogue parameters - #self._CDS_cat_ID = "I/271/out" # GSC 2.2, does not have Fmag - self._CDS_cat_ID = "I/305/out" # GSC 2.3 - - # Keys in CDS astroquery result - self._cds_keys = ["GSC2.3", "RAJ2000", "DEJ2000", "Fmag", "jmag", "Vmag", "Nmag", "Class"] - - # Minimal scaling for halo and spike polygon templates - self._scaling_min = 0.1 - - def _get_config(self): - """Get Config. - - Read the config file and set parameters. - - Raises - ------ - ValueError - If config file name is ``None`` - IOError - If config file not found - - """ - if self._config_filepath is None: - raise ValueError("No path to config file given") - - if not os.path.exists(self._config_filepath): - raise IOError(f'Config file "{self._config_filepath}" not found') - - conf = CustomParser() - conf.read(self._config_filepath) - - self._config = { - "PATH": {}, - "BORDER": {}, - "HALO": {}, - "SPIKE": {}, - "MESSIER": {}, - "NGC": {}, - "MD": {}, - } - - if conf.has_option("PROGRAM_PATH", "WW_PATH"): - self._config["PATH"]["WW"] = conf.getexpanded( - "PROGRAM_PATH", "WW_PATH" - ) - else: - self._config["PATH"]["WW"] = "weightwatcher" - self._config["PATH"]["WW_configfile"] = conf.getexpanded( - "PROGRAM_PATH", "WW_CONFIG_FILE" - ) - if conf.has_option("PROGRAM_PATH", "CDSCLIENT_PATH"): - self._config["PATH"]["CDSclient"] = conf.getexpanded( - "PROGRAM_PATH", "CDSCLIENT_PATH" - ) - elif self._star_cat_path is not None: - self._config["PATH"]["star_cat"] = self._star_cat_path - else: - raise ValueError( - "Either [PROGRAM_PATH]:CDSCLIENT_PATH in the mask config file " - + " or a star catalogue as module input needs to be present" - ) - - self._config["PATH"]["temp_dir"] = self._get_temp_dir_path( - conf.getexpanded("OTHER", "TEMP_DIRECTORY") - ) - self._config["BORDER"]["make"] = conf.getboolean( - "BORDER_PARAMETERS", "BORDER_MAKE" - ) - if self._config["BORDER"]["make"]: - self._config["BORDER"]["width"] = conf.getint( - "BORDER_PARAMETERS", "BORDER_WIDTH" - ) - self._config["BORDER"]["flag"] = conf.get( - "BORDER_PARAMETERS", "BORDER_FLAG_VALUE" - ) - - for mask_shape in ["HALO", "SPIKE"]: - - self._config[mask_shape]["make"] = conf.getboolean( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAKE", - ) - self._config[mask_shape]["individual"] = conf.getboolean( - "OTHER", "KEEP_INDIVIDUAL_MASK" - ) - - if self._config[mask_shape]["make"]: - - self._config[mask_shape]["maskmodel_path"] = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MASKMODEL_PATH", - ) - self._config[mask_shape]["mag_lim"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_LIM", - ) - self._config[mask_shape]["scale_factor"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_SCALE_FACTOR", - ) - self._config[mask_shape]["mag_pivot"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_PIVOT", - ) - self._config[mask_shape]["flag"] = conf.getint( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_FLAG_VALUE", - ) - - if conf.getboolean("OTHER", "KEEP_REG_FILE"): - reg_file = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_REG_FILE", - ) - self._config[mask_shape]["reg_file"] = ( - f'{self._config["PATH"]["temp_dir"]}/' - + f'{re.split(".reg", reg_file)[0]}' - + f"{self._img_number}.reg" - ) - else: - self._config[mask_shape]["reg_file"] = None - - for mask_type in ["MESSIER", "NGC"]: - - self._config[mask_type]["make"] = conf.getboolean( - f"{mask_type}_PARAMETERS", f"{mask_type}_MAKE" - ) - - if self._config[mask_type]["make"]: - self._config[mask_type]["cat_path"] = conf.getexpanded( - f"{mask_type}_PARAMETERS", - f"{mask_type}_CAT_PATH", - ) - self._config[mask_type]["size_plus"] = conf.getfloat( - f"{mask_type}_PARAMETERS", - f"{mask_type}_SIZE_PLUS", - ) - self._config[mask_type]["flag"] = conf.getint( - f"{mask_type}_PARAMETERS", - f"{mask_type}_FLAG_VALUE", - ) - - self._config["MD"]["make"] = conf.getboolean("MD_PARAMETERS", "MD_MAKE") - - if self._config["MD"]["make"]: - self._config["MD"]["thresh_flag"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_FLAG" - ) - self._config["MD"]["thresh_remove"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean( - "MD_PARAMETERS", "MD_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean("MD_PARAMETERS", "MD_REMOVE") - - def _set_image_coordinates(self): - """Set Image Coordinates. - - Compute the image coordinates for matching with the star catalogue - and star mask. - - """ - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - self._header = img.get_header() - img_shape = img.get_data().shape - img.close() - del img - - self._wcs = wcs.WCS(self._header) - - # Compute field center - - # Note: get_data().shape corresponds to (n_y, n_x) - pix_center = [img_shape[1] / 2.0, img_shape[0] / 2.0] - wcs_center = self._wcs.all_pix2world([pix_center], 1)[0] - self._fieldcenter = {} - self._fieldcenter["pix"] = np.array(pix_center) - self._fieldcenter["wcs"] = SkyCoord( - ra=wcs_center[0], dec=wcs_center[1], unit="deg" - ) - - # Get the four corners of the image - corners = self._wcs.calc_footprint() - self._corners_sc = SkyCoord( - ra=corners[:, 0] * units.degree, - dec=corners[:, 1] * units.degree, - ) - - # Compute image radius = image diagonal - self._img_radius = self._get_image_radius() - - def make_mask(self): - """Make Mask. - - Main function to create the mask. - - """ - output_file_name = ( - f"{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - if os.path.exists(f"{self._check_existing_dir}//{output_file_name}"): - return None, None - - if self._config["MD"]["make"]: - self.missing_data() - - if self._config["HALO"]["make"] or self._config["SPIKE"]["make"]: - stars = self.find_stars( - np.array( - [ - self._fieldcenter["wcs"].ra.value, - self._fieldcenter["wcs"].dec.value, - ] - ), - radius=self._img_radius, - ) - - if not self._err: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._create_mask( - stars=stars, - types=_type, - mag_limit=self._config[_type]["mag_lim"], - scale_factor=self._config[_type]["scale_factor"], - mag_pivot=self._config[_type]["mag_pivot"], - ) - - if not self._err: - mask_name = [] - if self._config["HALO"]["make"] and self._config["SPIKE"]["make"]: - self._exec_WW(types="ALL") - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}halo_spike_flag' - + f"{self._img_number}.fits" - ) - mask_name.append(None) - else: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._exec_WW(types=_type) - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}' - + f"{_type.lower()}_flag{self._img_number}.fits" - ) - else: - mask_name.append(None) - - masks_internal = {} - if not self._err: - if self._config["BORDER"]["make"]: - masks_internal["BORDER"] = self.mask_border( - width=self._config["BORDER"]["width"] - ) - - if not self._err: - for _type in ("MESSIER", "NGC"): - if self._config[_type]["make"]: - masks_internal[_type] = self.mask_dso( - self._config[_type]["cat_path"], - size_plus=self._config[_type]["size_plus"], - flag_value=self._config[_type]["flag"], - obj_type=_type, - ) - - if not self._err: - try: - im_pass = self._config["MD"]["im_remove"] - except Exception: - im_pass = True - - if not self._err: - path_external_flag = self._path_external_flag - - if not self._err: - if im_pass: - final_mask = self._build_final_mask( - path_mask1=mask_name[0], - path_mask2=mask_name[1], - masks_internal=masks_internal, - path_external_flag=path_external_flag, - ) - - if not self._config["HALO"]["individual"]: - if mask_name[0] is not None: - self._rm_fits1_stdout, self._rm_fits1_stderr = execute( - f"rm {mask_name[0]}" - ) - if mask_name[1] is not None: - self._rm_fits2_stdout, self._rm_fits2_stderr = execute( - f"rm {mask_name[1]}" - ) - - output_file_name = ( - f"{self._output_dir}/{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - - self._mask_to_file( - input_mask=final_mask, - output_fullpath=output_file_name, - ) - - # Handle stdout / stderr - # _CDS_stdout/_CDS_stderr are only set when find_stars ran, i.e. - # when HALO_MAKE or SPIKE_MAKE is True (False for image sims) - general_stdout = "" - general_stderr = "" - if hasattr(self, "_CDS_stdout"): - general_stdout += f"\nCDSClient\n{self._CDS_stdout}" - if self._CDS_stderr != "": - general_stderr += f"\nCDSClient\n{self._CDS_stderr}" - if hasattr(self, "_WW_stdout") or hasattr(self, "_WW_stdout"): - general_stdout += f"\n\nWeightWatcher\n{self._WW_stdout}" - if self._WW_stderr != "": - general_stderr += f"\n\nWeightWatcher\n{self._WW_stderr}" - if hasattr(self, "_rm_reg_stderr") or hasattr(self, "_rm_reg_stdout"): - general_stdout += f"\n\nrm reg file\n{self._rm_reg_stdout}" - if self._rm_reg_stderr != "": - general_stderr += f"\n\nrm reg file\n{self._rm_reg_stderr}" - if hasattr(self, "_rm_fits1_stderr") or hasattr( - self, "_rm_fits1_stdout" - ): - general_stdout += f"\n\nrm fits1 file\n{self._rm_fits1_stdout}" - if self._rm_fits1_stderr != "": - general_stderr += f"\n\nrm fits1 file\n{self._rm_fits1_stderr}" - if hasattr(self, "_rm_fits2_stderr") or hasattr( - self, "_rm_fits2_stdout" - ): - general_stdout += f"\n\nrm fits2 file\n{self._rm_fits2_stdout}" - if self._rm_fits2_stderr != "": - general_stderr += f"\n\nrm fits2 file\n{self._rm_fits2_stderr}" - - return general_stdout, general_stderr - - def find_stars(self, position, radius): - """Find Stars. - - Return GSC (Guide Star Catalog) objects for a field with center - (RA, Dec) and radius :math:`r`. - - Parameters - ---------- - position : numpy.ndarray - Position of the center of the field - radius : float - Radius in which the query is done (in arcmin) - - Returns - ------- - dict - Star dictionnary for GSC objects in the field - - Raises - ------ - ValueError - For invalid configuration options - - """ - if "star_cat" in self._config["PATH"]: - self._CDS_stdout = Table.read(self._config["PATH"]["star_cat"]) - else: - # For some exposures, Vizier returned empty star list if input position - # is not single (? or double) precision - p = np.array(position, dtype='double') - - coord = SkyCoord(ra=p[0] * units.deg, dec=p[1] * units.deg, frame="icrs") - - self._CDS_stdout = query_vizier(p[0], p[1], radius, self._CDS_cat_ID) - - self._CDS_stderr = "" - - return self._make_star_cat(self._CDS_stdout) - - def mask_border(self, width=100, flag_value=4): - """Create Mask Border. - - Mask ``width`` pixels around the image. - - Parameters - ---------- - width : int - Width of the mask mask border - flag_value : int - Value of the flag for the border (power of 2) - - Returns - ------- - numpy.ndarray - Array containing the mask - - Raises - ------ - ValueError - If ``width`` is ``None`` - - """ - if width is None: - raise ValueError("Width for border mask not provided") - - # Note that python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - flag[0:width, :] = flag_value - flag[-width:, :] = flag_value - flag[:, 0:width] = flag_value - flag[:, -width:] = flag_value - - return flag - - def mask_dso( - self, - cat_path, - size_plus=0.1, - flag_value=8, - obj_type="Messier", - ): - """Mask DSO. - - Create a circular patch for deep-sky objects (DSOs), e.g. - Messier or NGC objects. - - Parameters - ---------- - cat_path : str - Path to the deep-sky catalogue - size_plus : float - Increase the size of the mask by this factor - (e.g. ``0.1`` means 10%) - flag_value : int - Value of the flag, some power of 2 - obj_type : {'Messier', 'NGO'}, optional - Object type - - Returns - ------- - numpy.ndarray or ``None`` - If no deep-sky objects are found in the field return ``None`` and - the flag map - - Raises - ------ - ValueError - If ``size_plus`` is negative - ValueError - If ``cat_path`` is ``None`` - - """ - if size_plus < 0: - raise ValueError( - "deep-sky mask size increase variable cannot be negative" - ) - - if cat_path is None: - raise ValueError("Path to deep-sky object catalogue not provided") - - m_cat, header = fits.getdata(cat_path, header=True) - - unit_ra = file_io.get_unit_from_fits_header(header, "ra") - unit_dec = file_io.get_unit_from_fits_header(header, "dec") - m_sc = SkyCoord( - ra=m_cat["ra"] * unit_ra, - dec=m_cat["dec"] * unit_dec, - ) - - unit_size_X = file_io.get_unit_from_fits_header(header, "size_X") - unit_size_Y = file_io.get_unit_from_fits_header(header, "size_Y") - - # Loop through all deep-sky objects and check whether the object's - # disc overlaps the image footprint - indices = [] - size_max_deg = [] - for idx, m_obj in enumerate(m_cat): - - # DSO size - # r = max(m_obj['size']) * units.arcmin - r = max( - m_obj["size_X"] * unit_size_X, - m_obj["size_Y"] * unit_size_Y, - ) - r_deg = r.to(units.degree) - size_max_deg.append(r_deg) - - # Add index to list if the DSO disc overlaps the image: - # distance between DSO centre and image centre smaller than - # DSO radius plus image half-diagonal. (Testing only the image - # corners against the DSO radius, as done previously, misses - # objects that are smaller than the image and lie away from - # the corners.) - dist = self._fieldcenter["wcs"].separation(m_sc[idx]) - if dist < r_deg + self._img_radius * units.arcmin: - indices.append(idx) - - self._w_log.info( - f"Found {len(indices)} {obj_type} objects overlapping with" " image" - ) - - if len(indices) == 0: - # No closeby deep-sky object found - return None - - # Compute number of DSO center coordinates in footprint, for logging - # purpose only - n_dso_center_in_footprint = 0 - for idx in indices: - in_img = self._wcs.footprint_contains(m_sc[idx]) - self._w_log.info( - "(obj_type, ra, dec, in_img) = " - + f"({obj_type}, " - + f'{m_cat["ra"][idx]}, ' - + f'{m_cat["dec"][idx]}, ' - + f"{in_img})" - ) - - # Note: python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - nx = self._fieldcenter["pix"][0] * 2 - ny = self._fieldcenter["pix"][1] * 2 - for idx in indices: - m_center = np.hstack( - self._wcs.all_world2pix( - m_cat["ra"][idx], - m_cat["dec"][idx], - 0, - ) - ) - r_pix = ( - size_max_deg[idx].to(units.deg).value - * (1 + size_plus) - / np.abs(self._wcs.pixel_scale_matrix[0][0]) - ) - - # The following accounts for deep-sky centers outside of image, - # without creating masks for coordinates out of range - y_c, x_c = np.ogrid[0:ny, 0:nx] - mask_tmp = (x_c - m_center[0]) ** 2 + ( - y_c - m_center[1] - ) ** 2 <= r_pix**2 - - flag[mask_tmp] = flag_value - - return flag - - def missing_data(self): - """Find Missing Data. - - Look for zero-valued pixels in image. Flag if their relative number - is larger than a threshold. - """ - # Open image - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - - # Get total number of pixels - im_shape = img.get_data().shape - tot = float(im_shape[0] * im_shape[1]) - - # Compute number and ratio of missing data (zero-valued pixels) - missing = float(len(np.where(img.get_data() == 0.0)[0])) - self._ratio = missing / tot - - # Mark image as to be flagged if ratio larger than 'flag' threshold - if self._ratio >= self._config["MD"]["thresh_flag"]: - self._config["MD"]["im_flagged"] = True - else: - self._config["MD"]["im_flagged"] = False - - # Mark image as to be removed if flag is True and - # ratio large than 'remove' threshold. - # Reset all other mask 'make' flags to False (no other mask needs - # to be created) - if self._config["MD"]["remove"]: - if self._ratio >= self._config["MD"]["thresh_remove"]: - self._config["MD"]["im_remove"] = True - for idx in ["HALO", "SPIKE", "MESSIER", "BORDER"]: - self._config[idx]["make"] = False - else: - self._config["MD"]["im_remove"] = False - - img.close() - - def sphere_dist(self, position1, position2): - """Compute Spherical Distance. - - Compute spherical distance between 2 points. - - Parameters - ---------- - position1 : numpy.ndarray - [x,y] first point (in pixels) - position2 : numpy.ndarray - [x,y] second point (in pixels) - - Returns - ------- - float - The distance in degrees. - - Raises - ------ - ValueError - If input positions are not Numpy arrays - - """ - if ( - type(position1) is not np.ndarray - or type(position2) is not np.ndarray - ): - raise ValueError("Object coordinates need to be a numpy.ndarray") - - p1 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position1[0], position1[1], 1) - ) - p2 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position2[0], position2[1], 1) - ) - - dTheta = p1 - p2 - dLong = dTheta[0] - dLat = dTheta[1] - - dist = 2 * np.arcsin( - np.sqrt( - np.sin(dLat / 2.0) ** 2.0 - + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(dLong / 2.0) ** 2.0 - ) - ) - - return dist * (180.0 / np.pi) * 3600.0 - - def _get_image_radius(self, center=None): - """Get Image Radius. - - Compute the diagonal distance of the image in arcmin. - - Parameters - ---------- - center : numpy.ndarray, optional - Coordinates of the center of the image (in pixels) - - Returns - ------- - float - The diagonal distance of the image in arcmin - - Raises - ------ - TypeError - If centre is not a Numpy array - - """ - if center is None: - return ( - self.sphere_dist(self._fieldcenter["pix"], np.zeros(2)) / 60.0 - ) - - else: - if isinstance(center, np.ndarray): - return self.sphere_dist(center, np.zeros(2)) / 60.0 - else: - raise TypeError( - "Image center coordinates has to be a numpy.ndarray" - ) - - def _make_star_cat(self, CDSclient_output): - """Make Star Catalogue. - - Create a dictionary from an astroquery request. - - Parameters - ---------- - CDSclient_output : str - Output astroquery - - Returns - ------- - dict - Star dictionary containing all information - - """ - header = [] - stars = {} - - for key in self._cds_keys: - stars[key] = CDSclient_output[key] - - return stars - - def _create_mask( - self, - stars, - types="HALO", - mag_limit=18.0, - mag_pivot=13.8, - scale_factor=0.3, - ): - """Create Mask. - - Apply mask from model to stars and save into DS9 region file. - - Parameters - ---------- - stars : dict - Stars dictionary (output of ``find_stars``) - types : {'HALO', 'SPIKE'}, optional - Type of mask, options are ``HALO`` or ``SPIKE`` - mag_limit : float, optional - Faint magnitude limit for mask, default is ``18.0`` - mag_pivot : float, optional - Pivot magnitude for the model, default is ``13.8`` - scale_factor : float, optional - Scaling for the model, default is ``0.3`` - - Raises - ------ - ValueError - If no star catalogue is provided - ValueError - If an invalid option is provided for type - - """ - if stars is None: - raise ValueError("Star catalogue dictionary not provided") - - if types not in ("HALO", "SPIKE"): - raise ValueError('Mask types need to be in ["HALO", "SPIKE"]') - - if self._config[types]["reg_file"] is None: - reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - else: - reg = self._config[types]["reg_file"] - - mask_model = np.loadtxt( - self._config[types]["maskmodel_path"] - ).transpose() - mask_reg = open(reg, "w") - - stars_used = [[], [], []] - - """ - star_zip = zip( - stars["RA(J2000)"], - stars["Dec(J2000)"], - stars["Fmag"], - stars["Jmag"], - stars["Vmag"], - stars["Nmag"], - stars["Clas"], - ) - """ - - # Get keys without object name - keys_to_use = self._cds_keys[1:] - star_zip = zip(*(stars[k] for k in keys_to_use)) - - for ra, dec, Fmag, Jmag, Vmag, Nmag, clas in star_zip: - # Compute mean magnitude over the available (finite) bands. - # Missing GSC bands are NaN and must be excluded: a single NaN - # would make the mean NaN and silently fail the - # ``mag < mag_limit`` test below, leaving bright stars with - # incomplete photometry (the ones that most need masking) - # unmasked. - mags = [ - band - for band in (Fmag, Jmag, Vmag, Nmag) - if band is not None and np.isfinite(band) - ] - if len(mags) > 0: - mag = sum(mags) / len(mags) - else: - mag = None - self._w_log.info( - f"No finite {types} magnitude for star at ra={ra} " - + f"dec={dec}; object not masked" - ) - - if ( - ra is not None - and dec is not None - and mag is not None - and clas is not None - ): - if (mag < mag_limit) and (clas == 0): - scaling = 1.0 - scale_factor * (mag - mag_pivot) - if scaling < self._scaling_min: - scaling = self._scaling_min - pos = self._wcs.all_world2pix(ra, dec, 0) - stars_used[0].append(pos[0]) - stars_used[1].append(pos[1]) - stars_used[2].append(scaling) - - for idx in range(len(stars_used[0])): - poly = "polygon(" - for x, y in zip(mask_model[0], mask_model[1]): - angle = np.arctan2(y, x) - ll = stars_used[2][idx] * np.sqrt(x**2 + y**2) - xnew = ll * np.cos(angle) - ynew = ll * np.sin(angle) - poly = ( - f"{poly}{str(stars_used[0][idx] + xnew + 0.5)} " - + f"{str(stars_used[1][idx] + ynew + 0.5)} " - ) - poly = f"{poly})\n" - mask_reg.write(poly) - - mask_reg.close() - - def _exec_WW(self, types="HALO"): - """Execute WeightWatcher. - - Execute WeightWatcher to transform ``.reg`` to ``.fits`` flag map. - - Parameters - ---------- - types : {'HALO', 'SPIKE', 'ALL'}, optional - Type of WeightWatcher execution, options are ``HALO``, - ``SPIKE`` or ``ALL`` - - Raises - ------ - BaseCatalogue.CatalogFileNotFound - If catalogue file not found - - """ - if types in ("HALO", "SPIKE"): - - default_reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - default_out = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}_flag' - + f"{self._img_number}.fits" - ) - - if self._config[types]["reg_file"] is None: - reg = default_reg - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute(f"rm {reg}") - - - else: - reg = self._config[types]["reg_file"] - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - - elif types == "ALL": - - default_reg = [ - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo{self._img_number}.reg" - ), - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"spike{self._img_number}.reg" - ), - ] - default_out = ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo_spike_flag{self._img_number}.fits" - ) - - if self._config["HALO"]["reg_file"] is None: - reg = default_reg - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute( - f"rm {reg[0]} {reg[1]}" - ) - else: - reg = [ - self._config["HALO"]["reg_file"], - self._config["SPIKE"]["reg_file"], - ] - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - else: - raise ValueError("Types must be in ['HALO','SPIKE','ALL']") - - if (self._WW_stderr != "") or (self._rm_reg_stderr != ""): - self._err = True - - def _build_final_mask( - self, - path_mask1, - path_mask2=None, - masks_internal=None, - path_external_flag=None, - ): - """Create Final Mask. - - Create the final mask by combining the individual masks. - - Parameters - ---------- - path_mask1 : str - Path to a mask (FITS format) - path_mask2 : str, optional - Path to a mask (FITS format) - masks_internal : dict, optional - Internally created masks - path_external_flag : str, optional - Path to an external flag file - - Returns - ------- - numpy.ndarray - Array containing the final mask - - Raises - ------ - ValueError - If all masks are of type ``None`` - TypeError - If border is not a Numpy array - TypeError - If Messier mask is not a Numpy array - - """ - final_mask = None - - if path_mask1 is None and path_mask2 is None and not masks_internal: - raise ValueError( - "No paths to mask files containing halos and/or spikes," - + " borders, or deep-sky objects provided" - ) - - if path_mask1 is not None: - mask1 = file_io.FITSCatalogue(path_mask1, hdu_no=self._hdu) - mask1.open() - dat = mask1.get_data() - final_mask = dat[:, :] - - if path_mask2 is not None: - mask2 = file_io.FITSCatalogue(path_mask2, hdu_no=self._hdu) - mask2.open() - if final_mask is not None: - final_mask += mask2.get_data()[:, :] - else: - final_mask = mask2.get_data()[:, :] - - for typ in masks_internal: - if masks_internal[typ] is not None: - if type(masks_internal[typ]) is np.ndarray: - if final_mask is not None: - final_mask += masks_internal[typ] - else: - final_mask = masks_internal[typ] - else: - raise TypeError( - f"internally created mask of type {typ} " - + "has to be numpy.ndarray" - ) - - if path_external_flag is not None: - external_flag = file_io.FITSCatalogue( - path_external_flag, - hdu_no=self._hdu, - ) - external_flag.open() - if final_mask is not None: - final_mask = final_mask.astype(np.int16, copy=False) - try: - ext_flag = external_flag.get_data() - except: - self._w_log.info( - "Problem while getting external flag data. Check" - + f" whether file {path_external_flag} is not corrupt" - ) - raise - final_mask += ext_flag[:, :] - else: - final_mask = external_flag.get_data()[:, :] - external_flag.close() - - return final_mask.astype(np.int16, copy=False) - - def _mask_to_file(self, input_mask, output_fullpath): - """Mask to File. - - Save the mask to a fits file. - - Parameters - ---------- - input_mask : numpy.ndarray - Mask to save - output_fullpath : str - Path of the output file - - Raises - ------ - ValueError - If input_mask is type ``None`` - ValueError - If output_fullpath is type ``None`` - - """ - if input_mask is None: - raise ValueError("input mask file path not provided") - if output_fullpath is None: - raise ValueError("output mask file path not provided") - - out = file_io.FITSCatalogue( - output_fullpath, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - hdu_no=0, - ) - out.save_as_fits(data=input_mask, image=True) - - if self._config["MD"]["make"]: - out.open() - out.add_header_card( - "MRATIO", - self._ratio, - "ratio missing_pixels/all_pixels", - ) - out.add_header_card( - "MFLAG", - self._config["MD"]["im_flagged"], - f'threshold value {self._config["MD"]["thresh_flag"]:.3}', - ) - - # Write WCS information to header - if self._wcs: - header_wcs = self._wcs.to_header() - for card in header_wcs: - out.add_header_card( - card, - header_wcs[card], - header_wcs.comments[card], - ) - out.close() - - def _get_temp_dir_path(self, temp_dir_path): - """Get Temporary Directory Path. - - Create the path and the directory for temporary files. - - Parameters - ---------- - temp_dir_path : str - Path to the temporary directory, a value of ``OUTPUT`` will include - the temporary files in the run directory - - Returns - ------- - str - Path to the temporary directory - - Raises - ------ - ValueError - If ``temp_dir_path`` is of type ``None`` - - """ - if temp_dir_path is None: - raise ValueError("Temporary directory path not provided") - - path = temp_dir_path.replace(" ", "") - - if path == "OUTPUT": - path = f"{self._output_dir}/temp" - - path += "/" - if not os.path.isdir(path): - mkdir(path) - - return path diff --git a/src/shapepipe/modules/mask_runner.py b/src/shapepipe/modules/mask_runner.py deleted file mode 100644 index 13fbf9b6c..000000000 --- a/src/shapepipe/modules/mask_runner.py +++ /dev/null @@ -1,120 +0,0 @@ -"""MASK RUNNER. - -Module runner for ``mask``. - -:Author: Axel Guinot, Martin Kilbinger - -""" - -from shapepipe.modules.mask_package.mask import Mask -from shapepipe.modules.module_decorator import module_runner - - -@module_runner( - version="1.0", - file_pattern=["image", "weight", "flag"], - file_ext=[".fits", ".fits", ".fits"], - depends=["numpy", "astropy"], - executes=["weightwatcher"], - numbering_scheme="_0", -) -def mask_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Mask Runner.""" - # Get number of input files - n_inputs = len(input_file_list) - - # Set options for 2 inputs - if n_inputs == 2: - ext_flag_name = None - ext_star_cat = None - - # Set options for 3 inputs - elif n_inputs == 3: - if config.getboolean(module_config_sec, "USE_EXT_FLAG"): - ext_flag_name = input_file_list[2] - ext_star_cat = None - elif config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = None - ext_star_cat = input_file_list[2] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag or " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Set options for 4 inputs - elif n_inputs == 4: - if config.getboolean( - module_config_sec, "USE_EXT_FLAG" - ) and config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = input_file_list[2] - ext_star_cat = input_file_list[3] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag and " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Raise error for invalid settings - else: - raise ValueError( - f'Found {n_inputs} inputs and these must be "image", "weight" and ' - + '"ext_flags", "ext_star_cat" (optional). Check the MASK_RUNNER ' - + "section of the config file to make sure you have the " - + "appropriate settings." - ) - - # Get path to mask configuration options - config_file = config.getexpanded(module_config_sec, "MASK_CONFIG_PATH") - - # Get mask HDU number - if config.has_option(module_config_sec, "HDU"): - hdu = config.getint(module_config_sec, "HDU") - else: - hdu = 0 - - # Get mask mask file name prefix - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - else: - prefix = "" - - outname_base = "flag" - - # Path to check for already created mask files - if config.has_option(module_config_sec, "CHECK_EXISTING_DIR"): - check_existing_dir = config.getexpanded( - module_config_sec, "CHECK_EXISTING_DIR" - ) - else: - check_existing_dir = None - - # Create instance of Mask - mask_inst = Mask( - *input_file_list[:2], - image_prefix=prefix.replace(" ", ""), - image_num=file_number_string, - config_filepath=config_file, - output_dir=run_dirs["output"], - path_external_flag=ext_flag_name, - outname_base=outname_base, - star_cat_path=ext_star_cat, - check_existing_dir=check_existing_dir, - hdu=hdu, - w_log=w_log, - ) - - # Process module - stdout, stderr = mask_inst.make_mask() - - # Return stdout and stderr - return stdout, stderr diff --git a/src/shapepipe/utilities/file_io.py b/src/shapepipe/utilities/file_io.py deleted file mode 100644 index 8a0964aa0..000000000 --- a/src/shapepipe/utilities/file_io.py +++ /dev/null @@ -1,45 +0,0 @@ -"""FILE I/O UTILITIES. - -Small, dependency-light helpers for publishing files the rest of the pipeline -treats as a cache. - -:Author: consolidated from workflow/scripts/star_cats.py and - scripts/python/create_star_cat.py - -""" - -import os - - -def write_atomic(table, path): - """Publish ``table`` at ``path`` all-or-nothing. - - Every caller's only cache test is existence (``Path.exists`` / - ``os.path.isfile``), so a write killed part-way -- job timeout, OOM, node - failure -- would otherwise leave a truncated FITS that every later run - trusts forever, and ``test -s`` passes on partial bytes. Writing to a temp - and renaming makes the visible file all-or-nothing: ``os.replace`` is atomic - within a directory. - - The temp keeps the target's suffix, because astropy picks its writer from - the extension. It is dot-prefixed and PID-tagged so it stays out of the - ``star_chunk-*`` / ``star_cat*`` globs the rules use, and two concurrent - writers cannot collide. - - Parameters - ---------- - table : astropy.table.Table - Table to write - path : str or pathlib.Path - Destination path - - """ - path = os.fspath(path) - directory, name = os.path.split(path) - tmp = os.path.join(directory or ".", f".tmp-{os.getpid()}-{name}") - try: - table.write(tmp, overwrite=True) - os.replace(tmp, path) - finally: - if os.path.exists(tmp): - os.remove(tmp) diff --git a/src/shapepipe/utilities/focal_plane.py b/src/shapepipe/utilities/focal_plane.py deleted file mode 100644 index 32770bd06..000000000 --- a/src/shapepipe/utilities/focal_plane.py +++ /dev/null @@ -1,92 +0,0 @@ -"""FOCAL PLANE GEOMETRY. - -The sky footprint of a MegaCam exposure, read from its image headers. Both -star-catalogue producers need exactly this and used to carry their own copy of -it -- ``workflow/scripts/star_cats.py`` (the HEALPix chunk store's ``cut``) and -``scripts/python/create_star_cat.py`` (the one-cone-per-exposure path the store -replaced). They must agree on which sky an exposure covers, so there is one -definition of it. - -:Author: consolidated from the two star-catalogue scripts - -""" - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.wcs import WCS - - -def get_wcs(header): - """Build the WCS by hand, from the linear terms only. - - Deliberately NOT ``WCS(header)``: it sidesteps distortion-convention - incompatibilities between these headers and astropy, and a footprint needs - nothing finer than the linear terms. - - Parameters - ---------- - header : astropy.io.fits.Header - Image header - - Returns - ------- - astropy.wcs.WCS - WCS object - - """ - final_wcs = WCS(naxis=2) - final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except KeyError: - final_wcs.wcs.cunit = ["deg", "deg"] - final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - final_wcs.wcs.cd = [ - [header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]], - ] - - return final_wcs - - -def ccd_center_and_radius(header): - """Return ``(ra_deg, dec_deg, radius_deg)`` for a single CCD. - - The radius is the half-diagonal: centre to the ``(0, 0)`` corner. - - """ - w = get_wcs(header) - (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( - [[header["NAXIS1"] / 2.0, header["NAXIS2"] / 2.0], [0, 0]], 1) - center = SkyCoord(ra_c * u.deg, dec_c * u.deg) - radius = center.separation(SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg - return float(ra_c), float(dec_c), float(radius) - - -def focal_plane_disc(image, n_ccd=40): - """Return ``(ra_deg, dec_deg, radius_deg)`` covering all CCDs of one exposure. - - The centre is the mean of the CCD centres; the radius is the largest - centre-to-CCD-centre distance plus that CCD's own half-diagonal. - - ONE ``fits.open`` for all the extensions: ``fits.getheader(image, ext)`` - opens the file, walks the HDU list to ``ext`` and closes again, so a loop - over it costs ``n_ccd`` opens and O(n^2) header seeks. - - """ - centers, radii = [], [] - with fits.open(image) as hdul: - for ext in range(1, n_ccd + 1): - ra_c, dec_c, radius = ccd_center_and_radius(hdul[ext].header) - centers.append((ra_c, dec_c)) - radii.append(radius) - - ras = np.array([c[0] for c in centers]) - decs = np.array([c[1] for c in centers]) - center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) - seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg - return (float(center.ra.deg), float(center.dec.deg), - float(np.max(seps + np.array(radii)))) diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py deleted file mode 100644 index 0bf077a78..000000000 --- a/src/shapepipe/utilities/vizier.py +++ /dev/null @@ -1,101 +0,0 @@ -"""VIZIER QUERY UTILITY. - -Consolidated Vizier query helper used by the mask module (to fetch the -reference star catalogue) and by ``scripts/python/create_star_cat.py``. -Retries over a list of mirror servers at progressively longer timeouts. - -:Author: Martin Kilbinger - -""" - -import random -import time - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astroquery.vizier import Vizier - - -VIZIER_SERVERS = [ - "vizier.cds.unistra.fr", - "vizier.cfa.harvard.edu", - "vizier.iucaa.in", -] - -VIZIER_TIMEOUTS = [10, 20, 40] - - -def query_vizier(ra, dec, radius_arcmin, cat_id): - """Query a Vizier catalogue with retries over timeouts and mirror servers. - - Parameters - ---------- - ra : float - Right ascension in degrees. - dec : float - Declination in degrees. - radius_arcmin : float - Cone-search radius in arcminutes. - cat_id : str - Vizier catalogue identifier (e.g. ``"I/305/out"`` for GSC 2.3). - - Returns - ------- - astropy.table.Table - First result table returned by Vizier. - - Raises - ------ - IndexError - If all server/timeout combinations return an empty result. - - """ - # Empirically, single-precision input positions can cause Vizier to return - # empty lists for some exposures; force double precision. - p = np.array([ra, dec], dtype="double") - coord = SkyCoord(ra=p[0] * u.deg, dec=p[1] * u.deg, frame="icrs") - - # Stagger concurrent queries to avoid hammering a single mirror. - time.sleep(random.uniform(0, 5)) - - for attempt, timeout in enumerate(VIZIER_TIMEOUTS): - for server in VIZIER_SERVERS: - v = Vizier( - row_limit=-1, timeout=timeout, vizier_server=server - ) - # cache=False: astroquery otherwise pickles every HTTP response into - # $HOME/.astropy/cache/astroquery/Vizier, ~2 MB per query. The - # workflow already caches the RESULT as a FITS catalogue on scratch - # and skips the query when it hits, so the pickle is pure duplicate — - # and at campaign scale (~25k exposures) it is ~50 GB against a - # 50 GB home quota. Home is for source and config, not for a second - # copy of the survey. - result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id, - cache=False, - ) - if len(result) > 0: - print( - f"Vizier query successful " - f"(server={server}, timeout={timeout}s)" - ) - return result[0] - print( - f"Vizier returned empty list at {coord}, " - f"{radius_arcmin:.2f} arcmin, " - f"server={server}, timeout={timeout}s" - ) - if attempt < len(VIZIER_TIMEOUTS) - 1: - wait = 10 * 2**attempt - print( - f"All servers failed, retrying in {wait}s " - f"(attempt {attempt + 1}/{len(VIZIER_TIMEOUTS)}, " - f"next timeout={VIZIER_TIMEOUTS[attempt + 1]}s)" - ) - time.sleep(wait) - - raise IndexError( - f"Vizier astroquery returned empty list at {coord}, " - f"radius={radius_arcmin} arcmin, catalog={cat_id}" - ) diff --git a/tests/module/test_collate_star_cat.py b/tests/module/test_collate_star_cat.py deleted file mode 100644 index 20d52ce23..000000000 --- a/tests/module/test_collate_star_cat.py +++ /dev/null @@ -1,70 +0,0 @@ -"""UNIT TESTS FOR STAR-CATALOGUE COLLATION PATHS. - -Pin the patch vs patch-less (v2.0) path and filename convention of -``scripts/python/collate_star_cat.py``. Runs up to v1.6 carry a ``P`` -token in both the input run directory and the output filename; v2.0 is -patch-less (``patch is None``) and drops that token, reading from a single -``/output`` root and writing ``validation_psf_conv-.fits`` — the -name still matched by the downstream ``validation_psf_conv-*`` glob. -""" - -import importlib.util -from pathlib import Path - -import pytest - -# The collation script lives under scripts/python (not an importable package), -# so load it by path. -_SCRIPT = ( - Path(__file__).resolve().parents[2] - / "scripts" - / "python" - / "collate_star_cat.py" -) -_spec = importlib.util.spec_from_file_location("collate_star_cat", _SCRIPT) -collate_star_cat = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(collate_star_cat) - - -@pytest.mark.parametrize( - "patch, exp_input, exp_output", - [ - ("3", "in/P3/output/", "out/P3"), - (None, "in/output/", "out"), - ], -) -def test_collate_paths(patch, exp_input, exp_output): - """v1.x carries the P token; v2.0 (patch None) drops it.""" - assert collate_star_cat.collate_paths("in", "out", patch) == ( - exp_input, - exp_output, - ) - - -@pytest.mark.parametrize( - "patch, expected", - [ - ("3", "validation_psf_conv-3-0.fits"), - (None, "validation_psf_conv-0.fits"), - ], -) -def test_output_filename(patch, expected): - """The patch token is present for v1.x and absent for v2.0.""" - assert collate_star_cat.output_filename("validation_psf", patch, 0) == expected - - -def test_output_filename_matches_downstream_glob(): - """Both layouts stay under the downstream ``validation_psf_conv-*`` glob.""" - for patch in ("1", None): - assert collate_star_cat.output_filename( - "validation_psf", patch, 5 - ).startswith("validation_psf_conv-") - - -@pytest.mark.parametrize("bad", ["v2", "2.0", "v1.7", ""]) -def test_invalid_version_raises(bad): - """A mistyped -V is rejected rather than falling through to v1.x.""" - obj = collate_star_cat.Convert() - obj._params["version_cat"] = bad - with pytest.raises(ValueError): - obj.run() diff --git a/tests/module/test_mask_ext.py b/tests/module/test_mask_ext.py deleted file mode 100644 index 6958c9e02..000000000 --- a/tests/module/test_mask_ext.py +++ /dev/null @@ -1,274 +0,0 @@ -"""UNIT TESTS FOR MODULE PACKAGE: MASK_EXT. - -Drives ``MaskExt`` against a synthetic healsparse map and a synthetic TAN WCS -to lock in the rasterization contract of ``mask_ext_runner``: the config-driven -bit->flag mapping (with bitwise-OR of multiple bits), chunk-seam independence, -RA wrap across 0/360, off-footprint (sentinel) handling, external-flag summing, -``int16`` output dtype, and WCS round-trip in the written FITS. - -The synthetic map is a high-resolution healsparse map covering only a tiny -patch on the sky; the synthetic WCS points an image at that patch so a subset -of pixels land on masked healpix cells and the rest fall off the footprint. -""" - -import numpy as np -import numpy.testing as npt -import pytest -from astropy.io import fits -from astropy.wcs import WCS - -healsparse = pytest.importorskip("healsparse") -hpgeom = pytest.importorskip("hpgeom") - -from shapepipe.modules.mask_ext_package.mask_ext import MaskExt - - -class _NullLogger: - def info(self, *_args, **_kwargs): - pass - - -# Sky patch the synthetic image and mask share. -CRVAL1 = 150.0 -CRVAL2 = 2.3 -NSIDE_SPARSE = 131072 # ~1.6 arcsec, matching the real UNIONS masks -NSIDE_COVERAGE = 32 -PIXSCALE_DEG = 0.187 / 3600.0 # UNIONS ~0.187"/px - - -def _make_wcs(naxis1, naxis2, crval1=CRVAL1, crval2=CRVAL2): - """Synthetic TAN WCS header centred on the shared patch.""" - w = WCS(naxis=2) - w.wcs.ctype = ["RA---TAN", "DEC--TAN"] - w.wcs.crval = [crval1, crval2] - w.wcs.crpix = [naxis1 / 2 + 0.5, naxis2 / 2 + 0.5] - w.wcs.cd = [[-PIXSCALE_DEG, 0.0], [0.0, PIXSCALE_DEG]] - return w - - -def _write_image(path, wcs_obj, naxis1, naxis2): - """Write a zero-valued image carrying the given WCS (defines pixel grid).""" - data = np.zeros((naxis2, naxis1), dtype=np.float32) - hdu = fits.PrimaryHDU(data) - # NAXIS* are set from the data shape; merge only the WCS keywords. - hdu.header.update(wcs_obj.to_header()) - hdu.writeto(path, overwrite=True) - - -def _make_map(masked_ra, masked_dec, bits, sentinel=0): - """Healsparse int32 map with the given (ra, dec) cells set to ``bits``.""" - hmap = healsparse.HealSparseMap.make_empty( - NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.int32, sentinel=sentinel - ) - pix = hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec) - bits = np.asarray(bits, dtype=np.int32) - # Several fine image pixels can share one healpix cell; keep the first bit - # value per unique cell (replace requires unique pixels). - pix, first = np.unique(pix, return_index=True) - hmap[pix] = bits[first] - return hmap - - -def _run(tmp_path, image_wcs, naxis1, naxis2, hmap, bit_flag_map, - off_map_flag=0, chunk_size=None, path_external_flag=None): - """Instantiate MaskExt on written synthetic inputs and rasterize/write.""" - image_path = str(tmp_path / "image.fits") - mask_path = str(tmp_path / "mask.hsp") - _write_image(image_path, image_wcs, naxis1, naxis2) - hmap.write(mask_path, clobber=True) - - inst = MaskExt( - image_path, - mask_path, - bit_flag_map, - image_num="-000-000", - output_dir=str(tmp_path), - w_log=_NullLogger(), - off_map_flag=off_map_flag, - path_external_flag=path_external_flag, - image_prefix="pipeline", - chunk_size=chunk_size, - ) - return inst - - -def test_parse_bit_flag_map(): - """String config parses into an int->int dict; malformed entries raise.""" - assert MaskExt.parse_bit_flag_map("64:1, 2048:2") == {64: 1, 2048: 2} - assert MaskExt.parse_bit_flag_map(" 64 : 1 ") == {64: 1} - with pytest.raises(ValueError): - MaskExt.parse_bit_flag_map("64") - with pytest.raises(ValueError): - MaskExt.parse_bit_flag_map("") - - -def test_bit_flag_mapping_and_off_map(tmp_path): - """Masked cells map their bit; unmasked (off-footprint) get off_map_flag. - - The image centre pixel lands on a cell carrying bit 64 -> flag 1; the map - covers only that centre cell, so every other pixel is off-footprint. - """ - naxis1 = naxis2 = 16 - w = _make_wcs(naxis1, naxis2) - # RA/Dec at the central pixel (0-based grid, origin=0). - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, off_map_flag=8) - flags = inst.rasterize() - - assert flags.dtype == np.int16 - assert flags.shape == (naxis2, naxis1) - # The centre cell is flagged 1; the rest of the image is off the footprint. - n_flagged_1 = np.sum(flags == 1) - assert n_flagged_1 >= 1 - assert np.all(flags[flags != 1] == 8) - assert np.sum(flags == 8) == flags.size - n_flagged_1 - - -def test_multi_bit_or(tmp_path): - """A cell carrying two bits gets the bitwise-OR of the mapped flags.""" - naxis1 = naxis2 = 8 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - # bit values 64 and 2048 set together on the centre cell. - hmap = _make_map([float(ra_c)], [float(dec_c)], [64 | 2048]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1, 2048: 2}) - flags = inst.rasterize() - # 1 | 2 == 3 on the masked cell. - assert 3 in np.unique(flags) - assert np.all(np.isin(np.unique(flags), [0, 3])) - - -def test_chunk_seam_independence(tmp_path): - """Result is independent of chunk size (no seam artifacts).""" - naxis1 = naxis2 = 40 - w = _make_wcs(naxis1, naxis2) - # Mask a band of cells spanning several image rows. - ys = np.arange(0, naxis2) - xs = np.full_like(ys, naxis1 // 2) - ra, dec = w.all_pix2world(xs, ys, 0) - hmap = _make_map(ra.astype(float), dec.astype(float), [64] * len(ys)) - - inst_full = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - full = inst_full.rasterize() - - for cs in (1, 3, 7, 40): - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, chunk_size=cs) - npt.assert_array_equal(inst.rasterize(), full) - # Sanity: some pixels actually got flagged. - assert np.any(full == 1) - - -def test_ra_wrap(tmp_path): - """RA-wrap near 0/360: an image centred on RA~0 rasterizes correctly.""" - naxis1 = naxis2 = 16 - w = _make_wcs(naxis1, naxis2, crval1=0.0, crval2=2.3) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - # Straddling pixels produce RA both just below 360 and just above 0; the - # map cell is queried by its wrapped-into-[0,360) coordinate. - hmap = _make_map([float(np.mod(ra_c, 360.0))], [float(dec_c)], [64]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - flags = inst.rasterize() - # No crash, correct dtype, and the centre is flagged despite the wrap. - assert flags.dtype == np.int16 - assert np.any(flags == 1) - - -def test_external_flag_summing(tmp_path): - """External instrument flag image is summed into the rasterized mask.""" - naxis1 = naxis2 = 8 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) - - # External flag: a constant field of 16 (e.g. a saturation bit). - ext_path = str(tmp_path / "ext_flag.fits") - ext_data = np.full((naxis2, naxis1), 16, dtype=np.int16) - fits.PrimaryHDU(ext_data).writeto(ext_path, overwrite=True) - - inst = _run( - tmp_path, w, naxis1, naxis2, hmap, {64: 1}, - path_external_flag=ext_path, - ) - out_path = inst.make_mask() - - with fits.open(out_path) as hdul: - written = hdul[0].data - - # FITS stores big-endian; still a 2-byte signed int (int16). - assert written.dtype.kind == "i" and written.dtype.itemsize == 2 - # Every pixel gets +16 from the external flag; the centre cell also +1. - assert np.all(written >= 16) - assert 17 in np.unique(written) - - -def test_write_wcs_roundtrip(tmp_path): - """Written FITS carries the image WCS; pix2world round-trips.""" - naxis1 = naxis2 = 12 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - out_path = inst.make_mask() - - with fits.open(out_path) as hdul: - assert hdul[0].data.dtype.kind == "i" - assert hdul[0].data.dtype.itemsize == 2 - w_out = WCS(hdul[0].header) - - ra_in, dec_in = w.all_pix2world(cx, cy, 0) - ra_out, dec_out = w_out.all_pix2world(cx, cy, 0) - npt.assert_allclose([ra_in, dec_in], [ra_out, dec_out], rtol=0, atol=1e-9) - - -def _make_bool_map(masked_ra, masked_dec): - """Healsparse boolean map (True = masked) at the given (ra, dec) cells.""" - hmap = healsparse.HealSparseMap.make_empty( - NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.bool_, sentinel=False - ) - pix = np.unique(hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec)) - hmap[pix] = np.ones(len(pix), dtype=np.bool_) - return hmap - - -def test_bool_map_flag_1(tmp_path): - """A boolean mask (True = masked) rasterizes through BIT_FLAG_MAP 1:1. - - This is the flavour of the real 2025 r-band UNIONS mask - (``mask_r_nside131072.hsp``): dtype bool, sentinel False, valid pixels - only where masked. - """ - naxis1 = naxis2 = 16 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {1: 1}) - flags = inst.rasterize() - - assert flags.dtype == np.int16 - assert np.sum(flags == 1) >= 1 - assert np.all(np.isin(np.unique(flags), [0, 1])) - - -def test_bool_map_wrong_bits_raise(tmp_path): - """Boolean mask + bits other than 1 would silently select nothing: raise.""" - naxis1 = naxis2 = 8 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - with pytest.raises(ValueError, match="boolean healsparse map"): - inst.rasterize() diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini deleted file mode 100644 index d5b521080..000000000 --- a/workflow/config/cfis/config_exp_Ma.ini +++ /dev/null @@ -1,86 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Input file patterns: image, weight, external flag, external star catalogue -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -# True: the cat comes from $SP_RUN/star_cat_exp, the per-unit farm the -# exp_star_cat rule builds (40 per-CCD links to this exposure's one cat). -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -# Path to check for existing output mask files -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask deleted file mode 120000 index 9cd6a04b1..000000000 --- a/workflow/config/cfis/config_onthefly.mask +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/config_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask deleted file mode 120000 index 4743a87f0..000000000 --- a/workflow/config/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/config_tile_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/mask_default b/workflow/config/cfis/mask_default deleted file mode 120000 index 0970152ab..000000000 --- a/workflow/config/cfis/mask_default +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/mask_default \ No newline at end of file diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py deleted file mode 100644 index 2407af71b..000000000 --- a/workflow/scripts/star_cats.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -"""The campaign's GSC 2.3 star catalogue, as a HEALPix-chunked sky store. - -Masking needs, for every exposure, the bright stars over its focal plane. The -sky does not change between exposures, so the network cost of that is a property -of the campaign's SKY AREA, not of its exposure count: exposures overlap each -other ~7-10 deep, and a tile's exposures all look at the same square degree. - -So the store is chunked by sky, not by exposure. One GSC 2.3 cone query per -HEALPix pixel of NSIDE=32, written run-independently under the ``star_cats`` -config root and never fetched twice. A campaign that grows past the fetched -footprint queries only the chunks its new tiles add; one that grows within it -queries nothing. - -Two numbers set the scale. A full-UNIONS footprint is ~1.5k chunks against ~25k -exposures, so the QUERY COUNT drops ~16x. The queried AREA drops ~4x: the old -design covered the footprint ~8 times over (that is just the exposure overlap -depth), the new one ~2 times, the 2x being the price of bounding a HEALPix -quadrilateral by the cone Vizier speaks (see ``pixel_cone``) — 5.6-9 deg^2 for a -3.36 deg^2 pixel, ~40-60k rows and ~3-4 MB per chunk. - -Two subcommands, one module, deliberately: ``fetch`` and ``cut`` must agree -EXACTLY on which pixel holds which star, and a shared NSIDE constant in one file -is the only version of that agreement which cannot drift. - - fetch --tile-list ... --store ... --manifest ... - The campaign side. Turns the tile list into the set of pixels its - exposures can possibly need, fetches the missing ones, writes a manifest. - - cut --images ... --store ... --out ... - The per-exposure side, purely local: read the focal-plane footprint from - the exposure's image headers, load the chunks covering it, deduplicate, - and cut to the focal-plane disc. Reproduces byte-for-byte the same sky - selection the old one-query-per-exposure cone did. - -Geometry, and why the fetch pad is what it is. Chunk-need is computed from the -TILE list rather than from exposure pointings, because tile IDs are the one thing -known before any download: a pointing center means reading a FITS header of an -image get_images has not fetched yet, and the DAG needs the chunk set at parse -time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so -each tile is a disc of half-diagonal 0.354 deg; find_exposures gives a tile every -exposure whose footprint covers it, and the MegaCam focal plane is a disc of -radius 0.73 deg (measured on the cached catalogues). An exposure center is -therefore at most 0.354 + 0.73 deg from the tile center, and its stars 0.73 deg -beyond that: 1.81 deg, padded to ``PAD_DEG`` = 2.0. The pad is a perimeter cost -— negligible for a contiguous campaign, and paid once. - -The pad is a bound, not a promise: ``cut`` verifies that every chunk covering the -exposure it was handed is on disk, and fails loudly if one is not. A missing -chunk means the geometry above is wrong, and that must not degrade quietly into -an under-masked exposure. -""" - -import argparse -import json -import sys -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -import numpy as np -import healpy as hp -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.table import Table, vstack - -# The PYTHONPATH pin in profiles/nibi puts this checkout's src/ on the path (see -# exposure.smk's in_container), the same way the vizier helper is reached below. -from shapepipe.utilities.file_io import write_atomic -from shapepipe.utilities.focal_plane import focal_plane_disc - -# GSC 2.3. The same catalogue the mask module's own CDS path uses -# (mask.py: _CDS_cat_ID), so the store is a drop-in for it. -CAT_ID = "I/305/out" - -# NSIDE=32 -> 3.36 deg^2 per pixel, 12288 pixels over the sky. Chosen so one -# chunk is a couple of MegaCam focal planes: small enough that a Vizier query -# stays within a small multiple of the per-exposure queries this replaces, large -# enough that a full-UNIONS footprint is ~1.5k chunks rather than ~25k. The -# cone-vs-quadrilateral overhead is scale-free, so NSIDE trades query count -# against query size and nothing else. NESTED, so a chunk id is a hierarchical -# sky address and a future NSIDE change is a subdivision. -NSIDE = 32 -NEST = True - -# Angular padding on the disc used to select chunks (see the module docstring). -PAD_DEG = 2.0 - -# The MegaCam focal-plane disc, and the margin added to a pixel's own bounding -# cone. Both in degrees. -MARGIN_DEG = 0.02 - -# GSC 2.3's object id: the deduplication key where chunk cones overlap. -ID_COL = "GSC2.3" - - -# --- the chunk store -------------------------------------------------------- - - -def store_dir(store: Path) -> Path: - """Chunks live under the catalogue and resolution that produced them, so a - later NSIDE or catalogue change is a new directory beside the old one rather - than a silent reinterpretation of files already on disk.""" - return Path(store) / CAT_ID.replace("/", "_") / f"nside{NSIDE}" - - -def chunk_path(store: Path, ipix: int) -> Path: - return store_dir(store) / f"star_chunk-{ipix:06d}.fits" - - -def chunks_for_disc(ra_deg: float, dec_deg: float, radius_deg: float) -> list[int]: - """Every pixel that touches the disc, as sorted ids. - - ``inclusive=True`` makes this a conservative superset — the guarantee ``cut`` - relies on is that no star inside the disc lives in a pixel this omits. - """ - vec = hp.ang2vec(ra_deg, dec_deg, lonlat=True) - return sorted(int(i) for i in hp.query_disc( - NSIDE, vec, np.radians(radius_deg), inclusive=True, fact=4, nest=NEST)) - - -def pixel_cone(ipix: int) -> tuple[float, float, float]: - """(ra, dec, radius_arcmin) of a cone that CONTAINS pixel ``ipix``. - - Vizier speaks cones, HEALPix speaks quadrilaterals, so the query is the - pixel's bounding cone: its center, and the largest center-to-boundary - distance plus a margin. The cone spills over the pixel edges, which costs a - little duplication between neighbours and buys the containment ``cut`` - depends on. The duplicates are removed on read, by ``ID_COL``. - """ - ra_c, dec_c = hp.pix2ang(NSIDE, ipix, nest=NEST, lonlat=True) - ra_b, dec_b = hp.vec2ang(hp.boundaries(NSIDE, ipix, step=8, nest=NEST).T, - lonlat=True) - center = SkyCoord(ra_c * u.deg, dec_c * u.deg) - radius = center.separation(SkyCoord(ra_b * u.deg, dec_b * u.deg)).deg.max() - return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) - - -def read_chunks(store: Path, ipixels: list[int]) -> Table: - """Load and deduplicate the given chunks. - - A missing chunk is fatal (see the module docstring): it means the fetch - footprint did not cover this exposure, and an under-masked exposure is worse - than a failed job. - """ - missing = [i for i in ipixels if not chunk_path(store, i).exists()] - if missing: - raise SystemExit( - f"star chunk(s) {missing} not in {store_dir(store)}. The campaign's " - f"star_catalogue fetch did not cover this exposure — re-run it " - f"(and check that its tile list contains this exposure's tiles).") - - table = vstack([Table.read(chunk_path(store, i)) for i in ipixels], - metadata_conflicts="silent") - _, keep = np.unique(np.asarray(table[ID_COL]), return_index=True) - return table[np.sort(keep)] - - -# --- exposure footprint ----------------------------------------------------- -# The WCS construction and the focal-plane disc live in -# shapepipe.utilities.focal_plane, beside the vizier helper and imported the same -# way: create_star_cat.py needs exactly the same geometry, and the two must not -# be able to disagree about which sky an exposure covers. - - -def exposure_image(images_dir: Path) -> Path: - """The one multi-extension exposure image in a get_images output dir. - - That dir is a symlink farm holding ``image-.fitsfz`` plus its weight and - flag; only the image carries the 40 CCD WCSs. - """ - found = sorted(p for p in Path(images_dir).iterdir() if "image" in p.name) - if not found: - raise SystemExit(f"no image file in {images_dir}") - return found[0] - - -# --- fetch ------------------------------------------------------------------ - - -def campaign_chunks(tile_ids: list[str]) -> list[int]: - """Every chunk the campaign's exposures can need, from the tile list alone.""" - from shapepipe.utilities.cfis import get_tile_coord_from_nixy - - needed: set[int] = set() - for tile_id in tile_ids: - nix, niy = tile_id.split(".") - ra, dec = get_tile_coord_from_nixy(nix, niy) - needed.update(chunks_for_disc(ra.degree, dec.degree, PAD_DEG)) - return sorted(needed) - - -def fetch(args: argparse.Namespace) -> None: - from shapepipe.utilities.vizier import query_vizier - - tile_ids = [ln.strip() for ln in Path(args.tile_list).read_text().splitlines() - if ln.strip()] - needed = campaign_chunks(tile_ids) - out_dir = store_dir(args.store) - out_dir.mkdir(parents=True, exist_ok=True) - todo = [i for i in needed if not chunk_path(args.store, i).exists()] - print(f"star chunks: {len(needed)} needed for {len(tile_ids)} tiles, " - f"{len(todo)} to fetch -> {out_dir}", file=sys.stderr) - - def one(ipix: int) -> int: - ra, dec, radius_arcmin = pixel_cone(ipix) - table = query_vizier(ra, dec, radius_arcmin, CAT_ID) - write_atomic(table, chunk_path(args.store, ipix)) - print(f"chunk {ipix}: {len(table)} rows " - f"(ra={ra:.4f} dec={dec:.4f} r={radius_arcmin:.1f}')", - file=sys.stderr) - return len(table) - - # A handful of concurrent queries, never one per exposure: the same modest - # concurrency the per-exposure rule reached through --local-cores, now an - # explicit number instead of an accident of the head node's CPU count. - if todo: - with ThreadPoolExecutor(max_workers=args.workers) as pool: - list(pool.map(one, todo)) - - write_manifest(args, tile_ids, needed, len(todo)) - - -def write_manifest(args, tile_ids, needed, n_fetched) -> None: - """The rule's declared output. - - Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and - has no per-runner count floors, so there is nothing to compose and no - separate ``log:`` — under ``set -euo pipefail`` the job either completes or - aborts at the failing query, and snakemake's captured stderr is the evidence. - The manifest keeps the workflow's "one rule, one manifest" currency: written - last, and only when the content changed, so an unchanged campaign leaves the - mtime where it was rather than churning the `mtime` rerun-trigger. - """ - body = json.dumps({ - "stage": "star_catalogue", - "level": "campaign", - "status": "complete", - "catalogue": CAT_ID, - "nside": NSIDE, - "nest": NEST, - "pad_deg": PAD_DEG, - "store": str(store_dir(args.store)), - "n_tiles": len(tile_ids), - "n_chunks": len(needed), - "n_fetched": n_fetched, - "chunks": needed, - }, indent=2, sort_keys=True) - path = Path(args.manifest) - path.parent.mkdir(parents=True, exist_ok=True) - if not path.exists() or path.read_text() != body: - path.write_text(body) - - -# --- cut -------------------------------------------------------------------- - - -def cut(args: argparse.Namespace) -> None: - image = exposure_image(args.images) - ra, dec, radius = focal_plane_disc(image) - ipixels = chunks_for_disc(ra, dec, radius) - table = read_chunks(args.store, ipixels) - - center = SkyCoord(ra * u.deg, dec * u.deg) - stars = SkyCoord(np.asarray(table["RAJ2000"]) * u.deg, - np.asarray(table["DEJ2000"]) * u.deg) - inside = table[center.separation(stars).deg <= radius] - - print(f"{image.name}: ra={ra:.4f} dec={dec:.4f} r={radius:.4f} deg, " - f"{len(ipixels)} chunks -> {len(inside)} stars", file=sys.stderr) - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - write_atomic(inside, out) - - -# --- CLI -------------------------------------------------------------------- - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - sub = p.add_subparsers(dest="cmd", required=True) - - f = sub.add_parser("fetch", help="fetch the campaign footprint's chunks") - f.add_argument("--tile-list", required=True, type=Path) - f.add_argument("--store", required=True, type=Path) - f.add_argument("--manifest", required=True, type=Path) - f.add_argument("--workers", type=int, default=4) - f.set_defaults(func=fetch) - - c = sub.add_parser("cut", help="cut one exposure's catalogue from the store") - c.add_argument("--images", required=True, type=Path, - help="a get_images output dir holding image-.fitsfz") - c.add_argument("--store", required=True, type=Path) - c.add_argument("--out", required=True, type=Path) - c.set_defaults(func=cut) - - args = p.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() From 659168ad791c72b96867c6b54a5d6ceb02b3c0e4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:45:44 -0400 Subject: [PATCH 076/100] feat(mask_query): flag exposure detections against external healsparse masks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One healsparse lookup, two consumers. shapepipe.utilities.mask_query holds the primitive: `query_map` reads a map and returns its value at each (RA, Dec), and `flag_positions` ORs several maps into one integer per object. `make_cat` now calls query_map for its per-band MASK_ columns instead of opening healsparse itself, and the new `mask_query` module runs between sextractor and setools on exposures, writing an integer FLAG_EXT (0 = clean) into a NEW sexcat_ext.fits beside the input — the spread_model `add` pattern, so the LDAC_IMHEAD HDU survives and nothing mutates in place. star_selection.setools gains `FLAG_EXT == 0` beside every `IMAFLAGS_ISO == 0`: setools expressions have no bitwise operators, so the bit selection happens in the module (MASK_PATHS, optional MASK_BITS) and the config only tests for zero. The one interpretive choice is off-coverage. get_values_pos returns a map's sentinel outside its coverage — False for boolean maps, -1 for integer ones. make_cat passes that through verbatim (its documented off-map flag), but flag_positions treats it as NOT flagged for both kinds: OR-ing -1 in would flag every object a map does not reach, i.e. a map whose footprint stops short of an exposure would silently reject all of its stars. Off-coverage counts are logged. Tests build tiny nside_sparse=4096 maps and a three-HDU LDAC fixture and lock in the boolean/integer/MASK_BITS/OR/off-coverage cases, that the LDAC structure and the input file survive, and that make_cat's verbatim behaviour is unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- .../modules/make_cat_package/make_cat.py | 11 +- .../modules/mask_query_package/__init__.py | 51 +++++ .../modules/mask_query_package/mask_query.py | 94 ++++++++ src/shapepipe/modules/mask_query_runner.py | 73 +++++++ src/shapepipe/utilities/__init__.py | 2 +- src/shapepipe/utilities/mask_query.py | 152 +++++++++++++ tests/module/test_mask_query.py | 206 ++++++++++++++++++ 7 files changed, 583 insertions(+), 6 deletions(-) create mode 100644 src/shapepipe/modules/mask_query_package/__init__.py create mode 100644 src/shapepipe/modules/mask_query_package/mask_query.py create mode 100644 src/shapepipe/modules/mask_query_runner.py create mode 100644 src/shapepipe/utilities/mask_query.py create mode 100644 tests/module/test_mask_query.py diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index d6c38134d..946df5a8e 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -16,6 +16,7 @@ from sqlitedict import SqliteDict from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query def get_output_name(output_dir, file_number_string): @@ -246,6 +247,9 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log): returns the map's sentinel — ``-1`` for integer maps — verbatim), which is the documented off-map flag. + The lookup itself is ``shapepipe.utilities.mask_query.query_map``, + shared with the ``mask_query`` module: one primitive, two consumers. + Parameters ---------- final_cat_file : file_io.FITSCatalogue @@ -256,17 +260,14 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log): Logging instance """ - import healsparse - final_cat_file.open() ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"]) dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"]) for band, path in band_paths.items(): w_log.info(f"Query external mask for band {band}: {path}") - mask_map = healsparse.HealSparseMap.read(path) - values = mask_map.get_values_pos(ra, dec, lonlat=True) - final_cat_file.add_col(f"MASK_{band}", np.asarray(values)) + values = mask_query.query_map(path, ra, dec) + final_cat_file.add_col(f"MASK_{band}", values) final_cat_file.close() diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py new file mode 100644 index 000000000..b4b1d92c1 --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -0,0 +1,51 @@ +"""MASK QUERY PACKAGE. + +This package contains the module for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +:Parent module: ``sextractor_runner`` + +:Input: Single-exposure single-CCD SExtractor catalogue + +:Output: The same catalogue with an added ``FLAG_EXT`` column + +Description +=========== + +ShapePipe consumes sky-fixed masks by querying them, not by rasterizing them. +This module sits between SExtractor and ``setools`` on the exposure chain: it +reads each detection's windowed world position (``XWIN_WORLD``, ``YWIN_WORLD`` +in the ``LDAC_OBJECTS`` extension) and looks it up in the configured healsparse +maps, writing one integer column: + +``FLAG_EXT`` + ``0`` for an object no configured map flags, nonzero otherwise. The nonzero + value is the bitwise OR of the contributing map values, so it says *which* + bits fired, but nothing downstream is required to read it that way. + +The single column exists because ``setools`` expressions support only +``< > <= >= == !=`` — no bitwise operators — so the bit selection has to +happen here. ``star_selection.setools`` cuts on ``FLAG_EXT == 0`` beside its +existing ``IMAFLAGS_ISO == 0``: instrument flags reject pixels, the queried +masks reject objects. + +The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with +``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is +written once. That module's docstring documents the off-coverage convention. + +Module-specific config file entries +=================================== + +MASK_PATHS : str + Comma-separated healsparse map paths to query +MASK_BITS : int, optional + Bit mask applied to integer maps (``value & MASK_BITS`` flags); default is + to flag on any nonzero value. Ignored for boolean maps, which flag on + ``True`` +PREFIX : str, optional + Output file prefix + +""" + +__all__ = ["mask_query"] diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py new file mode 100644 index 000000000..16f3c5ff5 --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -0,0 +1,94 @@ +"""MASK QUERY. + +Class to flag SExtractor detections against external healsparse masks. + +:Author: Claude Fable 5, for PR #847 + +""" + +import numpy as np + +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + + +class MaskQuery(object): + """Mask Query. + + Query external healsparse masks at every detection of a SExtractor + catalogue and write the result as a single ``FLAG_EXT`` column into a copy + of that catalogue. + + Parameters + ---------- + sexcat_path : str + Path to the input SExtractor catalogue + output_path : str + Path to the output catalogue + mask_paths : list + Paths to the healsparse maps to query + bits : int, optional + Bit mask applied to integer maps; default ``None`` flags any nonzero + value + w_log : logging.Logger, optional + Logging instance + + """ + + def __init__( + self, + sexcat_path, + output_path, + mask_paths, + bits=None, + w_log=None, + ): + + self._sexcat_path = sexcat_path + self._output_path = output_path + self._mask_paths = mask_paths + self._bits = bits + self._w_log = w_log + + def process(self): + """Process. + + Query the masks and write the flagged catalogue. + + Returns + ------- + int + Number of flagged objects + + """ + ori_cat = file_io.FITSCatalogue( + self._sexcat_path, + SEx_catalogue=True, + ) + ori_cat.open() + data = ori_cat.get_data() + ra = np.copy(data["XWIN_WORLD"]) + dec = np.copy(data["YWIN_WORLD"]) + + flag = mask_query_util.flag_positions( + self._mask_paths, + ra, + dec, + bits=self._bits, + w_log=self._w_log, + ) + # int32 is what the catalogue carries; the UNIONS bit table needs 12 + # bits, and no OR of it can overflow. + flag = flag.astype(np.int32) + + new_cat = file_io.FITSCatalogue( + self._output_path, + SEx_catalogue=True, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + ori_cat.add_col( + "FLAG_EXT", flag, new_cat=True, new_cat_inst=new_cat + ) + ori_cat.close() + + return int(np.count_nonzero(flag)) diff --git a/src/shapepipe/modules/mask_query_runner.py b/src/shapepipe/modules/mask_query_runner.py new file mode 100644 index 000000000..fc2a6679a --- /dev/null +++ b/src/shapepipe/modules/mask_query_runner.py @@ -0,0 +1,73 @@ +"""MASK_QUERY RUNNER. + +Module runner for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +""" + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.modules.module_decorator import module_runner +from shapepipe.utilities import mask_query as mask_query_util + + +@module_runner( + version="1.0", + input_module="sextractor_runner", + file_pattern=["sexcat"], + file_ext=[".fits"], + depends=["numpy", "healsparse"], +) +def mask_query_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Mask Query Runner.""" + sexcat_path = input_file_list[0] + + # Get file prefix (optional) + if config.has_option(module_config_sec, "PREFIX"): + prefix = config.get(module_config_sec, "PREFIX") + if (prefix.lower() != "none") & (prefix != ""): + prefix = prefix + "_" + else: + prefix = "" + else: + prefix = "" + + mask_paths = mask_query_util.parse_map_paths( + config.getexpanded(module_config_sec, "MASK_PATHS") + ) + if not mask_paths: + raise ValueError( + f"[{module_config_sec}] MASK_PATHS is empty; the module has" + + " nothing to query." + ) + + # Any nonzero map value flags unless a bit selection is given + if config.has_option(module_config_sec, "MASK_BITS"): + bits = config.getint(module_config_sec, "MASK_BITS") + else: + bits = None + + output_path = ( + f'{run_dirs["output"]}/{prefix}sexcat_ext{file_number_string}.fits' + ) + + mq_inst = MaskQuery( + sexcat_path, + output_path, + mask_paths, + bits=bits, + w_log=w_log, + ) + n_flagged = mq_inst.process() + + w_log.info(f"FLAG_EXT nonzero for {n_flagged} objects") + + # No return objects + return None, None diff --git a/src/shapepipe/utilities/__init__.py b/src/shapepipe/utilities/__init__.py index 5f0eab2d6..42e234003 100644 --- a/src/shapepipe/utilities/__init__.py +++ b/src/shapepipe/utilities/__init__.py @@ -7,4 +7,4 @@ """ -__all__ = ["file_system", "cfis", "galaxy", "summary"] +__all__ = ["file_system", "cfis", "galaxy", "mask_query", "summary"] diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py new file mode 100644 index 000000000..0cc64255d --- /dev/null +++ b/src/shapepipe/utilities/mask_query.py @@ -0,0 +1,152 @@ +"""MASK QUERY. + +The one healsparse lookup in ShapePipe. + +ShapePipe does not generate or rasterize masks. Sky-fixed masks are supplied +as healsparse maps and are consumed by *querying them at object positions*: +every object gets its mask value(s) as catalogue columns, and rejection happens +at the catalogue level, never at the pixel level. The only mask that still +reaches pixels is the per-exposure instrument flag image delivered with it. + +Two callers share the primitive defined here: + +* ``make_cat`` writes one ``MASK_`` column per configured map, carrying + the map value verbatim (no interpretation, no filtering); +* ``mask_query`` writes a single integer ``FLAG_EXT`` column onto the exposure + SExtractor catalogue, combining the configured maps into "clean (0) or + flagged (nonzero)" so that ``setools`` — whose expression language has no + bitwise operators — can cut on ``FLAG_EXT == 0``. + +Coverage +-------- +``healsparse.HealSparseMap.get_values_pos`` returns a map's *sentinel* for +positions outside its coverage: ``False`` for boolean maps, and typically +``-1`` for integer maps. ``make_cat`` passes that sentinel through verbatim, +which is the documented off-map flag for the final catalogue. + +``flag_positions`` instead treats off-coverage as **not flagged**, for both map +kinds. This makes the integer case agree with the boolean case (whose sentinel +is literally ``False``) rather than diverge from it, and it keeps a map whose +coverage does not reach an exposure from silently rejecting every star on it. +Off-coverage counts are logged so the situation is visible rather than silent. + +:Author: Claude Fable 5, for PR #847 + +""" + +import numpy as np + + +def parse_map_paths(paths_str): + """Parse Map Paths. + + Parse a comma-separated list of healsparse map paths. + + Parameters + ---------- + paths_str : str + Comma-separated map paths, e.g. ``/a/star.hsp, /b/maximask.hsp`` + + Returns + ------- + list + Map paths, stripped of surrounding whitespace, empty entries dropped + + """ + return [path.strip() for path in paths_str.split(",") if path.strip()] + + +def query_map(path, ra, dec): + """Query Map. + + Read a healsparse map and return its value at each world position. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Map value at each position; positions outside the map's coverage carry + the map's sentinel value + + """ + import healsparse + + mask_map = healsparse.HealSparseMap.read(path) + + return np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + + +def flag_positions(paths, ra, dec, bits=None, w_log=None): + """Flag Positions. + + Combine one or more healsparse masks into a single per-object integer flag. + + Each map contributes at each position: + + * boolean map: ``1`` where the map is ``True``, ``0`` elsewhere; + * integer map: the map value, optionally restricted to ``bits`` + (``value & bits``); ``0`` where the value is zero or the position is + outside coverage. + + Contributions are combined with a bitwise OR, so the returned flag is zero + for a clean object and carries the union of the bits that fired otherwise. + + Parameters + ---------- + paths : list + Paths to the healsparse maps to query + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + bits : int, optional + Bit mask applied to integer maps; default ``None`` means any nonzero + value flags + w_log : logging.Logger, optional + Logging instance + + Returns + ------- + numpy.ndarray + Integer flag per object, ``0`` for a clean object + + """ + ra = np.asarray(ra) + dec = np.asarray(dec) + flag = np.zeros(ra.size, dtype=np.int64) + + for path in paths: + values = query_map(path, ra, dec) + + if values.dtype == bool: + contribution = values.astype(np.int64) + n_off = 0 + else: + integer = values.astype(np.int64) + # Off-coverage: the sentinel, negative by healsparse convention. + # Zeroed rather than OR-ed in, see this module's docstring. + off_coverage = integer < 0 + n_off = int(np.count_nonzero(off_coverage)) + integer = np.where(off_coverage, 0, integer) + if bits is not None: + integer &= bits + contribution = integer + + flag |= contribution + + if w_log is not None: + w_log.info( + f"Mask query {path}: " + f"{int(np.count_nonzero(contribution))}/{ra.size} objects " + f"flagged, {n_off} outside coverage" + ) + + return flag diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py new file mode 100644 index 000000000..053dffd1e --- /dev/null +++ b/tests/module/test_mask_query.py @@ -0,0 +1,206 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MASK_QUERY. + +Exercises the exposure-side half of the query-everything mask design (PR #847): +``mask_query`` reads each SExtractor detection's windowed world position out of +the ``LDAC_OBJECTS`` extension, looks it up in the configured healsparse maps, +and writes a single integer ``FLAG_EXT`` column into a NEW catalogue beside the +input. + +What is locked in here: (1) boolean maps flag with ``1``, (2) integer maps +contribute their value and ``MASK_BITS`` restricts which bits do, (3) the +off-coverage sentinel (``-1``) never flags — the one place this differs from +``make_cat``'s verbatim pass-through, argued in +:mod:`shapepipe.utilities.mask_query` — (4) several maps OR together, and +(5) the input catalogue is left untouched while the LDAC structure survives. +""" + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Detection world positions (RA, Dec in degrees). The last one sits far from +# every map's coverage, so it exercises the off-coverage path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +def _write_map(path, value, dtype=np.int16, n_covered=2): + """Build an integer map carrying ``value`` at the first ``n_covered``. + + Everything else reads the ``-1`` sentinel, i.e. off coverage. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=-1 + ) + if n_covered: + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.full(n_covered, value, dtype=dtype), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_bool_map(path, n_covered=2): + """Build a boolean map, ``True`` at the first ``n_covered``.""" + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, np.bool_ + ) + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.ones(n_covered, dtype=np.bool_), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_sexcat(path): + """Write a synthetic LDAC SExtractor catalogue of known positions. + + Written with astropy rather than ``FITSCatalogue.save_as_fits``, because + creating an LDAC catalogue through that API requires an existing LDAC file + to copy the ``LDAC_IMHEAD`` HDU from. The three-HDU layout below is what + SExtractor writes and what ``SEx_catalogue=True`` (``hdu_no=2``) indexes. + """ + imhead = fits.BinTableHDU.from_columns( + [ + fits.Column( + name="Field Header Card", format="1A", array=np.array(["x"]) + ) + ], + name="LDAC_IMHEAD", + ) + objects = fits.BinTableHDU.from_columns( + [ + fits.Column(name="NUMBER", format="J", array=np.arange(len(RA))), + fits.Column(name="XWIN_WORLD", format="D", array=RA), + fits.Column(name="YWIN_WORLD", format="D", array=DEC), + fits.Column( + name="IMAFLAGS_ISO", + format="J", + array=np.zeros(len(RA), dtype="i4"), + ), + ], + name="LDAC_OBJECTS", + ) + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto( + str(path), overwrite=True + ) + return str(path) + + +def _read(path): + cat = file_io.FITSCatalogue(str(path), SEx_catalogue=True) + cat.open() + data = cat.get_data() + flag = np.copy(data["FLAG_EXT"]) + names = set(data.dtype.names) + cat.close() + return flag, names + + +def test_parse_map_paths(): + """Comma-separated paths parse, whitespace-tolerant, empties dropped.""" + assert mask_query_util.parse_map_paths( + " /a/star.hsp, /b/maximask.hsp ,, " + ) == ["/a/star.hsp", "/b/maximask.hsp"] + + +def test_flag_positions_integer_map(tmp_path): + """An integer map contributes its value; off-coverage stays clean.""" + path = _write_map(tmp_path / "m.hsp", 4, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [4, 4, 4, 0] + ) + + +def test_flag_positions_bits_restrict(tmp_path): + """MASK_BITS selects which bits of an integer map flag.""" + path = _write_map(tmp_path / "m.hsp", 1028, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=4), [4, 4, 4, 0] + ) + # A bit the map does not carry leaves everything clean. + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=2), [0, 0, 0, 0] + ) + + +def test_flag_positions_ors_maps(tmp_path): + """Several maps combine with a bitwise OR.""" + a = _write_map(tmp_path / "a.hsp", 4, n_covered=1) + b = _write_map(tmp_path / "b.hsp", 1024, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([a, b], RA, DEC), [1028, 1024, 1024, 0] + ) + + +def test_flag_positions_boolean_map(tmp_path): + """A boolean map flags with 1; its False sentinel stays clean.""" + path = _write_bool_map(tmp_path / "bool.hsp", n_covered=2) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [1, 1, 0, 0] + ) + + +def test_mask_query_writes_flag_ext(tmp_path): + """The module writes FLAG_EXT into a new catalogue and counts the hits.""" + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-0.fits") + out_path = tmp_path / "sexcat_ext-000-0.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=_NullLogger() + ).process() + + assert n_flagged == 2 + flag, names = _read(out_path) + npt.assert_array_equal(flag, [4, 4, 0, 0]) + assert np.issubdtype(flag.dtype, np.integer) + # The columns SExtractor wrote survive alongside the new one. + assert {"NUMBER", "XWIN_WORLD", "YWIN_WORLD", "IMAFLAGS_ISO"} <= names + # The LDAC structure survives: setools and psfex read HDU 2 by index. + with fits.open(str(out_path)) as hdus: + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + + # The input is not mutated: this module publishes a new file. + with fits.open(in_path) as hdus: + assert "FLAG_EXT" not in hdus[2].data.dtype.names + + +def test_mask_query_bits_and_all_clean(tmp_path): + """MASK_BITS reaches the module, and a miss leaves every object clean.""" + map_path = _write_map(tmp_path / "m.hsp", 1024, n_covered=3) + in_path = _write_sexcat(tmp_path / "sexcat-000-1.fits") + out_path = tmp_path / "sexcat_ext-000-1.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], bits=4, w_log=_NullLogger() + ).process() + + assert n_flagged == 0 + flag, _ = _read(out_path) + npt.assert_array_equal(flag, [0, 0, 0, 0]) From 51f25a5ee1d586da690ef64294b9c1dc44c01ab0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:46:11 -0400 Subject: [PATCH 077/100] refactor(pipeline): rewire the chains around the query design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposures. SExtractor now reads the instrument flag straight from split_exp (`FILE_PATTERN = image, weight, flag` — split_exp already writes an unprefixed flag.fits per CCD), with the run_sp_exp_Ma INPUT_DIR entry gone; mask_query_runner sits between it and setools in both the example and the workflow config, and star_selection.setools cuts on FLAG_EXT == 0 beside every IMAFLAGS_ISO == 0. Same rewiring in config_exp_mccd.ini, which shares that setools file. sextractor_runner's declared input_module follows the real parent. Tiles. config_tile_Sx_nomask.ini becomes THE tile config: tiles have no instrument flag image and no mask module now, so FLAG_IMAGE = False with default_noimaflags.param is the only variant, in example/cfis and in the sims dir. random_cat still needs a pixel mask image, but nothing in the pipeline produces one — its second input is now declared external (the healsparse-native replacement is #797), which is a real capability gap and is flagged as one in the config, the runner and docs/source/random_cat.md rather than papered over. Snakemake. `exp_psf` now depends on `exp_split` directly; `star_catalogue`, `exp_star_cat`, `exp_mask`, `star_cat_cmd`, `in_container` and the STAR_CAT_* helpers are gone, along with `config["star_cats"]` and the two mid-chain localrules. The `exp_short` group goes too, and the docstring says why: it existed to fuse exp_split with exp_mask, and a group of one rule submits exactly the job the ungrouped rule submits. completeness.py trades its exp_mask stage (mask_runner 40/1) for a mask_query_runner row inside exp_psf, floor 2 rather than 40 because setools tolerates the same sparse-CCD attrition either side of it; run_report drops the two stages; clean_exposure stops reclaiming link farms that no longer exist. Verified in the container: all 30 module runners import, every example and workflow config's MODULE list resolves through get_module_runners, and the workflow parses and builds its DAG (`snakemake --lint`, `-n`). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_Rc.ini | 9 +- example/cfis/config_exp_mccd.ini | 29 +- example/cfis/config_exp_psfex.ini | 38 ++- example/cfis/config_tile_Sx.ini | 17 +- example/cfis/config_tile_Sx_nomask.ini | 114 -------- example/cfis/star_selection.setools | 13 + ..._tile_Sx_nomask.ini => config_tile_Sx.ini} | 5 + src/shapepipe/modules/random_cat_runner.py | 4 +- .../modules/sextractor_package/__init__.py | 4 +- src/shapepipe/modules/sextractor_runner.py | 2 +- workflow/Snakefile | 34 +-- workflow/config.yaml | 26 +- workflow/config/cfis/config_exp_psfex.ini | 38 ++- workflow/rules/exposure.smk | 261 ++---------------- workflow/rules/prepare.smk | 11 +- workflow/rules/tile.smk | 34 +-- workflow/scripts/clean_exposure.py | 20 +- workflow/scripts/completeness.py | 23 +- workflow/scripts/run_report.py | 2 +- 19 files changed, 200 insertions(+), 484 deletions(-) delete mode 100644 example/cfis/config_tile_Sx_nomask.ini rename example/cfis_image_sims/{config_tile_Sx_nomask.ini => config_tile_Sx.ini} (91%) diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini index 2fec0a0fc..5d04e8e37 100644 --- a/example/cfis/config_Rc.ini +++ b/example/cfis/config_Rc.ini @@ -53,10 +53,13 @@ TIMEOUT = 96:00:00 ## Module options [RANDOM_CAT_RUNNER] -#INPUT_DIR = last:get_images_runner, last:mask_runner -INPUT_DIR = last:get_images_runner, $SP_RUN/output/run_sp_combined_flag:mask_runner +# The mask image is now an EXTERNAL product: ShapePipe generates no tile +# masks. Point the second entry at a directory of tile mask images matching +# NUMBERING_SCHEME below (the healsparse-native replacement for this module is +# the survey-window work, issue #797). +INPUT_DIR = last:get_images_runner, $SP_CONFIG/tile_masks -FILE_PATTERN = CFIS_image, pipeline_flag +FILE_PATTERN = CFIS_image, mask NUMBERING_SCHEME = 000-000 diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 092df0417..8e9ffbd8f 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -19,7 +19,7 @@ RUN_NAME = run_sp_exp_SxSePsf [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, +MODULE = sextractor_runner, mask_query_runner, setools_runner, mccd_preprocessing_runner, mccd_fit_val_runner, merge_starcat_runner, mccd_plots_runner @@ -62,11 +62,11 @@ TIMEOUT = 96:00:00 # - $SP_RUN/output #INPUT_DIR = . -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_MODULE = split_exp_runner -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag NUMBERING_SCHEME = -0000000-0 @@ -127,16 +127,31 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_MODULE = last:sextractor_runner -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat_sexcat NUMBERING_SCHEME = -0000000-0 +# External healsparse masks queried at each detection's (RA, Dec); see the +# mask_query module docstring. star_selection.setools cuts on FLAG_EXT == 0. +MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp + +; MASK_BITS = 1028 + + +[SETOOLS_RUNNER] + +INPUT_MODULE = last:mask_query_runner + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 199050927..10566d5ed 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -1,5 +1,8 @@ # ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -20,7 +23,7 @@ RUN_NAME = run_sp_exp_SxSePsfPi [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner +MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI @@ -57,12 +60,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Input from two modules -#INPUT_DIR = last:split_exp_runner, run_sp_exp_Ma:mask_runner -INPUT_DIR = last:split_exp_runner, last:mask_runner +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = last:split_exp_runner -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag # Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on # the decorator's 4-entry FILE_EXT default (length check fails at startup) @@ -127,16 +129,34 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_DIR = last:sextractor_runner -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 +# External healsparse masks queried at each detection's (RA, Dec). Any map +# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which +# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. +MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps ignore it. +; MASK_BITS = 1028 + + +[SETOOLS_RUNNER] + +INPUT_DIR = last:mask_query_runner + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini index 12f158508..9859e4237 100644 --- a/example/cfis/config_tile_Sx.ini +++ b/example/cfis/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options @@ -55,11 +60,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner, run_sp_tile_Mh_exp:merge_headers_runner +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag, log_exp_headers +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers -FILE_EXT = .fits, .fits, .fits, .sqlite +FILE_EXT = .fits, .fits, .sqlite # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 @@ -69,14 +74,14 @@ EXEC_PATH = source-extractor # SExtractor configuration files DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param DOT_CONV_FILE = $SP_CONFIG/default.conv # Use input weight image if True WEIGHT_IMAGE = True # Use input flag image if True -FLAG_IMAGE = True +FLAG_IMAGE = False # Use input PSF file if True PSF_FILE = False @@ -97,7 +102,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND, SEGMENTATION +CHECKIMAGE = BACKGROUND # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/example/cfis/config_tile_Sx_nomask.ini b/example/cfis/config_tile_Sx_nomask.ini deleted file mode 100644 index 731a8d338..000000000 --- a/example/cfis/config_tile_Sx_nomask.ini +++ /dev/null @@ -1,114 +0,0 @@ -# ShapePipe configuration file for tile detection - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers - -FILE_EXT = .fits, .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = False - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index 8330a1eff..f35043d99 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -1,4 +1,13 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties +## +## Two independent mask cuts, and they come from different places: +## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), +## delivered with the exposure and read by SExtractor; +## FLAG_EXT == 0 the external healsparse masks, queried per detection by +## the mask_query module (which map bits reach FLAG_EXT is +## that module's MASK_PATHS / MASK_BITS config). +## SETools expressions have no bitwise operators, so mask_query does the bit +## selection and this file only tests for zero. [MASK:preselect] MAG_AUTO > 0 @@ -7,11 +16,13 @@ FWHM_IMAGE > 0.3 / 0.187 FWHM_IMAGE < 1.5 / 0.187 FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 NO_SAVE [MASK:flag] FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 NO_SAVE @@ -23,6 +34,7 @@ FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 [MASK:fwhm_mag_cut] FWHM_IMAGE > 0 @@ -30,6 +42,7 @@ FWHM_IMAGE < 40 MAG_AUTO < 35 FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 NO_SAVE # Split the 'star_selection' sample into diff --git a/example/cfis_image_sims/config_tile_Sx_nomask.ini b/example/cfis_image_sims/config_tile_Sx.ini similarity index 91% rename from example/cfis_image_sims/config_tile_Sx_nomask.ini rename to example/cfis_image_sims/config_tile_Sx.ini index a5c12771c..78bcfb646 100644 --- a/example/cfis_image_sims/config_tile_Sx_nomask.ini +++ b/example/cfis_image_sims/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py index c85854cb5..f180c9f7e 100644 --- a/src/shapepipe/modules/random_cat_runner.py +++ b/src/shapepipe/modules/random_cat_runner.py @@ -12,7 +12,9 @@ @module_runner( version="1.1", - file_pattern=["image", "pipeline_flag"], + # The mask image is an external input: ShapePipe generates no tile masks + # (the healsparse-native replacement for this module is issue #797). + file_pattern=["image", "mask"], file_ext=[".fits", "fits"], depends=["astropy"], numbering_scheme="_0", diff --git a/src/shapepipe/modules/sextractor_package/__init__.py b/src/shapepipe/modules/sextractor_package/__init__.py index abb0b4572..40f40ca91 100644 --- a/src/shapepipe/modules/sextractor_package/__init__.py +++ b/src/shapepipe/modules/sextractor_package/__init__.py @@ -4,10 +4,10 @@ :Author: Axel Guinot -:Parent modules: ``mask_runner``, ``merge_headers_runner`` (the latter only +:Parent modules: ``split_exp_runner``, ``merge_headers_runner`` (the latter only when ``MAKE_POST_PROCESS`` is ``True``) -:Input: Single-exposure single-CCD image, weight and flag files +:Input: Single-exposure single-CCD image, weight and instrument flag files :Output: SExtractor output catalogue diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index c36a8596d..e45885b06 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -19,7 +19,7 @@ # first three entries only. @module_runner( version="1.0.1", - input_module=["mask_runner", "merge_headers_runner"], + input_module=["split_exp_runner", "merge_headers_runner"], file_pattern=["image", "weight", "flag", "log_exp_headers"], file_ext=[".fits", ".fits", ".fits", ".sqlite"], executes=["source-extractor"], diff --git a/workflow/Snakefile b/workflow/Snakefile index b925adc06..8a8a2a9bf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -82,10 +82,6 @@ RUN_DIR = Path(config["run_dir"]) # Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no # second path: one root, exactly the pre-D5 layout. PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) -# Run-independent root for the mask star catalogues: the HEALPix chunk store the -# star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it -# (config.yaml explains the placement). -STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the @@ -309,13 +305,6 @@ SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") -# Same argument for star_cats.py, which both star-cat rules call: their params -# otherwise fingerprint nothing but paths, so an edit to the chunking or the cut -# would never rerun them. ONE hash for both rules because it is one script — and -# that is also why fetch and cut live in one module (they must agree on which -# pixel holds which star). The hash does NOT key the store path — see -# config.yaml's star_cats block on clearing the store after a semantic change. -STAR_CAT_HASH = script_hash("star_cats.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -509,8 +498,7 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale pre-fix file with the bare id. There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at - the committed config dir, and ``star_cat_exp`` is a real per-unit directory - built by the ``exp_star_cat`` rule, not a symlink into a shared pool. + the committed config dir. Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -589,18 +577,16 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# The aggregation targets, clean_exposure, clean_tile, star_catalogue and -# exp_star_cat run in the head process. The two clean rules are seconds of rmtree -# and hang off `all`; exp_star_cat is seconds of local FITS work; all three would -# otherwise be ~20k (clean_exposure, exp_star_cat) or ~23k (clean_tile) sbatch -# submissions at DR6 scale for work shorter than the scheduling latency. -# star_catalogue is one job either way, and local keeps its CDS concurrency the -# explicit number its thread pool sets (see exposure.smk). +# The aggregation targets, clean_exposure and clean_tile run in the head +# process. Both clean rules are seconds of rmtree and hang off `all`; submitted +# they would be ~20k (clean_exposure) or ~23k (clean_tile) sbatch submissions at +# DR6 scale for work shorter than the scheduling latency. # -# star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out -# of any future `group:` label: a local job cannot be fused into a submitted group. -# The two clean rules are DAG leaves and have no such constraint. -localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat +# Both are DAG LEAVES, so neither constrains a `group:` label. (A mid-chain +# localrule would: a local job cannot be fused into a submitted group. The old +# star-catalogue rules were exactly that, and they are gone with the internal +# mask generation.) +localrules: all, prepare_all_tiles, clean_exposure, clean_tile rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index 8e4de8506..e3f0355aa 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -53,30 +53,6 @@ products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# The mask star-catalogue root — run-independent, shared by every campaign, and -# holding two things: -# /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 -# query per HEALPix chunk (~3.4 deg^2, ~25k rows), written by -# `star_catalogue` over the tile list's footprint and never fetched twice; -# /exp/star_cat-.fits the per-exposure cuts -# `exp_star_cat` makes from those chunks, with no network at all. -# Network therefore scales with SKY AREA, not exposure count: exposures overlap -# ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. -# -# On the PERSISTENT root: the sky store is a durable science product bought with -# ~1.5k catalogue-server queries at DR6 scale, and re-buying it after a scratch -# purge is the one cost in this workflow that cannot be paid with local compute. -# (It sat on scratch through smk-g3 only because def-mjhudson /project was then -# hard-full at 27/27 TiB.) -# -# THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to -# workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun -# both rules — the script's hash is a param on each — but a chunk already on disk -# is skipped and only re-cut. Clear the store by hand when the change must reach -# the data. Changing NSIDE or the catalogue ID is the exception: those name the -# directory, so a change there fetches into a new one beside the old. -star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache - # The run index, and — sharing its directory — missing.json and run_report.json. # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction @@ -142,7 +118,7 @@ clean_tiles: true # # READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its # exposures' stores. If you later retry that tile, those exposure chains are -# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# gone and will be REBUILT from scratch — get_images, split, psf, per # exposure. That is correct, and expensive. Ignore a tile when you have decided # it is dead, not while you are still debugging it. clean_ignore_tiles: [] diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 0af871d8e..08f45f99b 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -1,5 +1,8 @@ # ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -20,8 +23,7 @@ RUN_DATETIME = False [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - +MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI MODE = SMP @@ -57,11 +59,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Input from two modules -INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag # Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on # the decorator's 4-entry FILE_EXT default (length check fails at startup) @@ -126,16 +128,34 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 +# External healsparse masks queried at each detection's (RA, Dec). Any map +# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which +# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. +MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps ignore it. +; MASK_BITS = 1028 + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/mask_query_runner/output + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 9fe3e7b29..af60dadbe 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,17 +1,22 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -----> exp_mask -> exp_psf - -> exp_star_cat --/ - star_catalogue ---------------/ - -``star_catalogue`` is campaign-level, not per-exposure: one fetch of the whole -footprint's stars, which every exposure's ``exp_star_cat`` then cuts locally. + exp_get_images -> exp_split -> exp_psf Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a run log. There is no `prepare_exposures` aggregation target: these chains hang off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). +NO MASK RULE, and that is the design (PR #847). ShapePipe generates no masks. +The only mask that reaches pixels is the instrument flag image delivered with +the exposure, which ``exp_split`` splits per CCD alongside image and weight and +SExtractor reads directly. Sky-fixed masks are healsparse maps, queried once per +object: ``mask_query`` (inside exp_psf's config chain) writes ``FLAG_EXT`` onto +each CCD's SExtractor catalogue for setools' star cut, and ``make_cat`` writes +the per-band ``MASK_`` columns on the tile side. Neither needs a rule, a +star catalogue, or a network fetch — hence no ``star_catalogue`` / ``exp_star_cat`` +here, and no ``exp_mask``. + NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, not over one invocation — reclamation here is clean_exposure's job (S5), driven @@ -19,26 +24,21 @@ by the accumulating index. A temp() here would delete an exposure the moment this invocation's readers finished and cascade destructive reruns across spatial neighbours the next time a tile is appended. -GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — -one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under -the 15-minute floor Alliance policy asks us to bundle away. The composition -rules are in prepare.smk's docstring; this chain is linear too, so the group -asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. - -The two rules NOT in it are structural, not taste: - * exp_psf is heavy (16 GB, 4 h) and never fuses with a short rule; - * exp_get_images cannot join, because ``exp_star_cat`` — a LOCALRULE, and so - ungroupable — sits between it and exp_mask. Pulling get_images in would make - the group both a dependency and a dependent of exp_star_cat, i.e. a cycle. - Starting the group at exp_split leaves star_cat's inputs entirely upstream - of it, so the group has one clean external edge. -Different exposures share no DAG edge, so this is one group job per exposure. +NO GROUPING. The ``exp_short`` group existed to fuse exp_split and exp_mask — +two rules whose medians were 1:28 and 1:54, both well under the 15-minute floor +Alliance policy asks us to bundle away — into one sbatch per exposure. With +exp_mask gone there is nothing to fuse: a group of one rule submits exactly the +job the ungrouped rule submits, and the label would only obscure that. The +composition rules, should a second short rule ever appear here, are in +prepare.smk's docstring. exp_get_images stays separate for the same reason it +always did (a download, retried on its own), and exp_psf is heavy (16 GB, 4 h) +and never fuses with a short rule. NUMBER_LIST ($SP_UNIT_NUM, see unit_num in the Snakefile) is set only for exp_split, whose numbering scheme IS the exposure id; never for get_images / -exp_mask / exp_psf, whose per-CCD or download numbering would turn tolerated -per-CCD attrition into a whole-exposure hard failure. It is a property of the -committed configs (config_exp_Sp.ini alone carries the entry). +exp_psf, whose per-CCD or download numbering would turn tolerated per-CCD +attrition into a whole-exposure hard failure. It is a property of the committed +configs (config_exp_Sp.ini alone carries the entry). """ rule exp_get_images: @@ -58,198 +58,9 @@ rule exp_get_images: shell: sp_shell("exp_get_images", "config_exp_Gie.ini") -# --- mask star catalogues --------------------------------------------------- -# Two rules, and the split between them is the design: the NETWORK is a function -# of the campaign's sky area, the per-exposure catalogue is a local cut. -# -# `star_catalogue` fetches the footprint's GSC 2.3 stars once, one Vizier query -# per HEALPix chunk, into a run-independent chunk store under config `star_cats`. -# `exp_star_cat` then reads the chunks covering an exposure's focal plane and -# cuts them to it — no network at all. workflow/scripts/star_cats.py holds both -# halves, the geometry they must agree on, and the arithmetic that motivates the -# split; its docstring is the reference for chunking, padding and query counts. - -# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE -# pointing at a path that does not exist inside the image, so requests is pointed -# at the bundle explicitly (proven in the p3-batch1 bash precedent). -STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" - -# The rules run star_cats.py inside the container (healpy, astroquery, astropy) -# but call apptainer THEMSELVES rather than letting the SDM wrap them -# (`container: None` on both): the CA bundle above and the exposure rule's -# host-side farm loop both need the explicit exec. bin/sp has loaded the -# apptainer module. -# -# WHICH image and WHICH arguments are not this file's to decide, and hand-rolling -# them here was a real divergence: these were the only two rules that ignored a -# user's dev sandbox, because they read `config['container']` — the shared /project -# fallback — instead of the image the Snakefile resolved for everything else. -# `_image` is that resolution (sandbox -> cached SIF -> config), and the profile's -# own apptainer-args are the same string the SDM splices onto every other rule, -# PYTHONPATH pin for shapepipe.utilities.{vizier,cfis} included. -# container.profile_apptainer_args() exists precisely so this file can read them -# rather than restate them. Only the CA bundle is added on top, and only when the -# command touches the network. -def in_container(cmd, *, network=False): - args = list(_container.profile_apptainer_args()) - if not args: - # Silently falling back would run these two rules with no --cleanenv and - # no PYTHONPATH pin, i.e. against a different src/ than every other rule. - raise WorkflowError( - f"Could not read apptainer-args from {_container.PROFILE_FILE}; " - f"star_catalogue and exp_star_cat build their apptainer line from it.") - if network: - # The container's certifi bundle, one --env per variable (the profile's - # own PYTHONPATH entry uses the same one-assignment-per-flag form). - args += [a for k in ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", - "CURL_CA_BUNDLE") - for a in ("--env", f"{k}={STAR_CAT_CA}")] - return f"apptainer exec {' '.join(args)} '{_image}' {cmd}" - - -# The campaign's star catalogue: a first-class durable science product, keyed by -# sky rather than by run. Chunk-need is recomputed from the tile list on every -# run and only the missing chunks are fetched, so appending tiles costs exactly -# the chunks they add. -# -# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the -# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the -# per-exposure rule reached by accident through --local-cores, now an explicit -# number that does not scale with the head node's CPU count. -# -# `tile_list_hash` is what makes the incremental behaviour visible to the DAG. -# The tile list is parse-time config, not a rule input (and the profile drops the -# `input` rerun-trigger anyway), so appending tiles would otherwise leave this -# rule up to date against a footprint that has grown. Hashing the list into a -# param reruns it, and the rerun fetches only what is new. -STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" - - -rule star_catalogue: - output: - manifest = STAR_CAT_MANIFEST - # No `log:` — see write_manifest() in star_cats.py. - # `cmd` is a params value, so placeholders in it are not formatted (see - # unit_pre in the Snakefile). Hence the explicit manifest path. - params: - cmd = in_container( - f"python {SCRIPTS}/star_cats.py fetch" - f" --tile-list '{config['tile_list']}' --store '{STAR_CATS}'" - f" --manifest '{STAR_CAT_MANIFEST}'", network=True), - tile_list_hash = hashlib.md5( - Path(config["tile_list"]).read_bytes()).hexdigest()[:12], - script_hash = STAR_CAT_HASH - container: - None - threads: 4 - retries: 2 - resources: - mem_mb = 4000, - runtime = 720 - shell: - "set -euo pipefail\n{params.cmd}" - - -# The per-exposure catalogue and the 40 per-CCD symlinks the mask module's -# numbering scheme needs. Local: one header read for the focal-plane footprint, -# a load of the chunks covering it, a radial cut. -# -# A LOCALRULE, for the reason the Snakefile's localrules line gives. -# -# The per-unit farm is a REAL directory holding exactly this exposure's 40 -# numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR -# and the file handler INTERSECTS the numbers found across INPUT_DIRs, so a -# symlink to a shared whole-store pool contributes every other exposure's numbers -# and the intersection is empty ("numbers ... do not intersect", live). -# -# TWO declared outputs, and the second one is the point. -# -# The manifest keeps the "one rule, one manifest" currency of every other rule: -# written last, unique to this rule, a record of what the farm points at, and -# deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the -# chunk store at no network cost. -# -# But a manifest attests FOREVER, and the two things it attests to both live -# outside the unit's manifests/ dir: the cut catalogue on /scratch (60-day purge) -# and the farm itself. Either can vanish under a manifest that still says -# "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is -# declared too — one link stands for all 40, they are created by the same loop -# in the same instant, and declaring 40 buys nothing. Snakemake's existence test -# is os.path.exists, which FOLLOWS symlinks and is therefore False for a link -# whose target the purge removed. A purged cut or a deleted farm makes the rule -# out of date, it reruns, and it re-cuts or re-links as needed. -def star_cat_cmd(exp): - """The whole rule body, as bash — carried as a params value because it - contains literal ``{}`` (the manifest JSON); see unit_pre in the Snakefile.""" - cut_dir = f"{STAR_CATS}/exp" - cat = f"{cut_dir}/star_cat-{exp}.fits" - work = exp_dir(exp) - farm = f"{work}/star_cat_exp" - images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" - manifest = exp_manifest(exp, "exp_star_cat") - body = json.dumps({ - "stage": "exp_star_cat", "level": "exp", "unit": exp, - "status": "complete", "cat": cat, "link_dir": farm, "n_links": 40, - }, indent=2, sort_keys=True) - return "\n".join([ - "set -euo pipefail", - # LEGACY-SYMLINK HAZARD. Unit dirs built before this rule existed carry - # star_cat_exp as a SYMLINK into the old shared star-cat pool. `mkdir -p` - # is a no-op on an existing symlink-to-directory, so the 40-link loop - # below followed it and wrote this exposure's links INTO THE SHARED POOL - # (520 stray links found live). Replace the link — never `rm -rf` it, - # which would recurse into the pool, and never touch a real directory: - # a real farm is this rule's own output and `ln -sfn` refreshes it. - f"[ -L '{farm}' ] && rm -f '{farm}' || true", - f"mkdir -p '{cut_dir}' '{farm}' '{work}/manifests'", - in_container(f"python {SCRIPTS}/star_cats.py cut" - f" --images '{images}' --store '{STAR_CATS}'" - f" --out '{cat}'"), - f"test -s '{cat}'", - # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the - # one focal-plane catalogue (pattern from the p3-batch1 precedent). - f"for ccd in $(seq 0 39); do ln -sfn '{cat}' " - f"'{farm}/star_cat-{exp}-'\"$ccd\"'.fits'; done", - # Byte-stable, and written only after the links exist: an unconditional - # write would move the mtime, which is a rerun-trigger. - f"tmp='{manifest}.tmp'", - "cat > \"$tmp\" <<'SP_STAR_CAT_JSON'", - body, - "SP_STAR_CAT_JSON", - f"cmp -s \"$tmp\" '{manifest}' && rm -f \"$tmp\" || mv -f \"$tmp\" '{manifest}'", - ]) - - -rule exp_star_cat: - input: - rules.exp_get_images.output.manifest, - # The chunks this cut reads. star_cats.py fails loudly on a chunk that is - # missing anyway, but the edge is what makes the fetch happen first. - rules.star_catalogue.output.manifest - output: - manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", - # The sentinel: ccd-0 of the 40-link farm (see above). - link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" - # No `log:`, for the same reason as star_catalogue above. - params: - cmd = lambda wc: star_cat_cmd(wc.exp), - # star_cats.py is external to the shell string, so the `code` - # rerun-trigger does not see it — same reason SCRIPT_HASH exists. - script_hash = STAR_CAT_HASH - container: - None - threads: 1 - retries: 2 - resources: - mem_mb = 4000, - runtime = 10 - shell: - "{params.cmd}" - # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: - group: "exp_short" input: rules.exp_get_images.output.manifest output: @@ -266,33 +77,13 @@ rule exp_split: shell: sp_shell("exp_split", "config_exp_Sp.ini") -rule exp_mask: - group: "exp_short" - input: - # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs - # and this exposure's own star_cat_exp farm. - rules.exp_split.output.manifest, - rules.exp_star_cat.output.manifest - output: - manifest = f"{EXP_DIR}/manifests/exp_mask.json" - log: - f"{EXP_DIR}/logs/exp_mask.json" - params: - pre = lambda wc: unit_pre("exp_mask", wc.exp), - script_hash = SCRIPT_HASH - threads: 4 - resources: - mem_mb = lambda wc, attempt: 8000 * attempt, - runtime = 120 - shell: - sp_shell("exp_mask", "config_exp_Ma.ini") - -# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# SExtractor -> mask_query (FLAG_EXT) -> setools star selection -> PSFEx model +# -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's # :warn on psfex_interp_runner. rule exp_psf: input: - rules.exp_mask.output.manifest + rules.exp_split.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_psf.json" log: diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 07405407f..fdf5d726b 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -29,11 +29,12 @@ cached group resources and re-sets ``attempt`` on every member (jobs.py), and ``retries: 2`` still governs. A retry re-runs the whole group, which is safe because every rule ``rm -rf``s its own run dir at start. -Star catalogues for masking are NOT a prepare-phase concern and not pre-run -input: the compute DAG fetches the campaign footprint's stars once -(``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in -exposure.smk, into a run-independent store. The tile side has no star-cat node -because it has no mask rule yet — see tile.smk. +There is no masking node in this phase, or in any other: ShapePipe generates no +masks (PR #847). The instrument flag image ships with the exposure and is split +per CCD by ``exp_split``; the sky-fixed healsparse masks are queried per object +inside the ShapePipe configs (``mask_query`` on exposures, ``make_cat`` on +tiles). Nothing is fetched, staged or rasterized, so there is nothing to +prepare. """ # No NUMBER_LIST for get_images — a download stage has nothing on disk to diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 74d985e2a..981ac8d65 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -41,18 +41,14 @@ The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor run that the shape chain does not need co-scheduled, and folding it in would add its runtime to a sum that has no room. -Note there is no `tile_mask` rule: the committed config chain is the -"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask -run), and no tile-mask config was committed in the S2 sweep. Adding the masked -variant is a config + one rule, at the config selector the PRD describes. - -That rule also needs a tile-side analogue of ``exp_star_cat``: tile star cats key -on TILE id, so they are a separate cache namespace and a separate node, and the -earliest point it can run is after ``tile_uncompress`` (create_star_cat.py's -``-k tile`` mode reads the uncompressed tile image's primary header). The mask -config would then read a real per-unit ``$SP_RUN/star_cat_tiles`` directory, -built the same way and for the same reason (the file handler intersects numbers -across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). +There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe +generates no masks: tiles have no instrument flag image of their own, so +tile_detect runs SExtractor with FLAG_IMAGE = False against +default_noimaflags.param (config_tile_Sx.ini — what used to be the "sx_nomask" +variant, now the only one). Sky-fixed masks reach the tile as CATALOGUE columns +instead: ``tile_make_cat``'s make_cat queries the configured healsparse maps at +every object's (RA, Dec) and writes one ``MASK_`` column per band, which +is what downstream selections cut on. """ # --- the node-local tile root (the I/O + scratch fix) ---------------------- @@ -176,8 +172,8 @@ def tile_local(tile): runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in `tile_merge_cats`, which has no pre_run), so the staging is attempted nine times per tile, eight of them concurrent siblings in one toposort level. It - is copy-to-a-temp-name plus `mv -f` -- the same all-or-nothing publish - shapepipe.utilities.file_io.write_atomic argues -- and it is UNCONDITIONAL, + is copy-to-a-temp-name plus `mv -f` -- an all-or-nothing publish, so a + reader never sees a partial file -- and it is UNCONDITIONAL, no `cp -u` and no already-there test. An interrupted `cp` leaves a truncated destination whose mtime is NEWER than the source, so `-u` would skip it forever, and nothing downstream would catch it: TILE_VIGNET_FRESH and @@ -390,9 +386,8 @@ def exp_manifests(wc, stage): return [ancient(p) for p in paths] def tile_exp_split(wc): return exp_manifests(wc, "exp_split") -def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") -def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_psf(wc) # Build the per-tile symlink forest. Declaring the exposure manifests as input @@ -873,10 +868,9 @@ rule tile_make_cat: # safe because nothing upstream of it survives, and tile_detect.json in # particular must never join the list. # -# A LOCALRULE (declared in the Snakefile), same as clean_exposure. Unlike -# exp_star_cat it is a DAG LEAF, so being local can never make it both a -# dependency and a dependent of a group — no `group:` label here, and none -# possible. +# A LOCALRULE (declared in the Snakefile), same as clean_exposure. It is a DAG +# LEAF, so being local can never make it both a dependency and a dependent of a +# group — no `group:` label here, and none possible. # # WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to # tile_local() mid-campaign reruns finished tiles unsatisfiably because their diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 728f6c527..b763a5573 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,17 +7,12 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its -star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are -reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted -here like every other, so the exposure's chain must read as unbuilt, and 40 -symlinks left behind are a farm no rule now owns. The catalogue itself lives in -the run-independent cache, so rebuilding the farm costs a relink and no query. +run_sp_exp_Gie/Sp/SxSePsfPi), its ``manifests/`` and its ``logs/``. That is the +entire exposure store: since PR #847 removed ShapePipe's mask generation there +is no run_sp_exp_Ma tree and no star-catalogue link farm to reclaim beside it. Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not -``rmtree``d. Legacy unit dirs carry ``star_cat_exp`` as a link into the old -shared pool, and an rmtree would recurse through it and delete the shared cache -for every other exposure in the campaign. +``rmtree``d, so a link into a shared store can never be recursed through. Deleting the manifests is deliberate and load-bearing, not tidiness: @@ -98,10 +93,9 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - # is_symlink() first, and OR'd with exists(): exists() follows the link, so a - # dangling legacy star_cat_exp would otherwise be skipped and survive. - candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs", - args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") + # is_symlink() first, and OR'd with exists(): exists() follows the link, so + # a dangling link would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs") targets = [t for t in candidates if t.is_symlink() or t.exists()] # Tombstone first, complete — then delete (see the module docstring). diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index cf67363db..0dfac0ebb 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -14,8 +14,8 @@ ``shapepipe_run`` failed:: rc=0 - shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} --log {log} --job-rc "$rc" || rc=1 + shapepipe_run -c $SP_CONFIG/config_exp_Sp.ini -b {threads} || rc=$? + completeness.py check exp_split {output} --log {log} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory @@ -82,12 +82,18 @@ # --- exposure chain --- "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, - "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree # AND the bash baseline (both 120/exposure). + # + # mask_query is one sexcat_ext per CCD — the count the deleted exp_mask + # stage used to carry, now inside this chain because querying a healsparse + # map at ~2k detections needs no rule of its own. Its floor tracks the + # runners either side of it: setools tolerates sparse-CCD attrition, so a + # hard 40 here would fail exposures the chain is designed to survive. "exp_psf": { "sextractor_runner": dict(expect=120, floor=2), + "mask_query_runner": dict(expect=40, floor=2), "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), "psfex_runner": dict(expect=80, floor=2), "psfex_interp_runner": dict(expect=40, floor=0, warn=True), @@ -163,18 +169,17 @@ def check_floor(stage, run_dir): # var its config does, so chunk K's check looks at chunk K's dir. # # EVERY ENTRY HERE (and in COMPLETENESS above) HAS A RULE. The table used to -# carry two stages that did not: `tile_mask` (run_sp_tile_Ma, mask_runner 1/1) -# and `tile_detect_uc` (run_sp_tile_Uc). The committed config chain is the -# "sx_nomask" tile_detect variant and no tile-mask config was committed, so both -# were unreachable — tile.smk's docstring is where the masked variant is argued, -# and it is a config plus a rule plus these two rows, added back together. +# carry stages that did not — `tile_mask` (run_sp_tile_Ma) and `tile_detect_uc` +# (run_sp_tile_Uc) — and, until PR #847, an `exp_mask` stage that did. ShapePipe +# now generates no masks at all: the sky-fixed healsparse maps are queried per +# object inside the exp_psf and tile_make_cat chains, so masking has no stage +# of its own on either side and is not coming back. STAGE_DIR = { "tile_get_images": ("tile", "run_sp_tile_Git"), "tile_uncompress": ("tile", "run_sp_tile_Uz"), "tile_find_exposures": ("tile", "run_sp_tile_Fe"), "exp_get_images": ("exp", "run_sp_exp_Gie"), "exp_split": ("exp", "run_sp_exp_Sp"), - "exp_mask": ("exp", "run_sp_exp_Ma"), "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), "tile_detect": ("tile", "run_sp_tile_Sx"), diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index f559e3276..bd117c09c 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -59,7 +59,7 @@ TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", "tile_merge_headers", "tile_detect", "tile_vignets", "tile_ngmix", "tile_merge_cats", "tile_make_cat"] -EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] # The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names # the mechanism that owns each). Their presence is therefore NOT evidence that a From 1bceb2da112ea4fa54cc480742659eb7e026b835 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:46:28 -0400 Subject: [PATCH 078/100] docs(mask): describe the query design, drop the deps only masking used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipeline_tutorial.md's "Mask images" section becomes "Masks" and explains the design instead of the old two-run mask procedure: healsparse maps queried per object into FLAG_EXT and MASK_, the instrument flag as the one mask that reaches pixels, no internet access and no star-catalogue download anywhere. pipeline_canfar.md loses its mask-tiles and mask-exposures steps and the combine_runs flag_* staging; random_cat.md gains a warning that its input mask images now come from outside ShapePipe; workflow/README.md and the sims README describe the chains as they now are. Dependencies. weightwatcher leaves the Dockerfile and the three docs pages that listed it — the deleted mask module was its only caller (`ww` appears nowhere else). astroquery and hpgeom leave pyproject.toml: astroquery had exactly two importers, utilities/vizier.py and star_cats.py, and hpgeom was the mask_ext rasterizer's. `uv lock` regenerated the manifest (astroquery, html5lib, pyvo removed; hpgeom stays, healsparse pulls it in). canfar_avail_results loses its -m / pipeline_flag check mode, and scripts/README.rst its create_star_cat entry. Left deliberately: shapepipe.utilities.summary{,_params_pre_v2}'s mask_runner entries. Those describe the pre-v2 CANFAR job map and parse the logs of runs that already exist on disk — removing them would break summary_run against historical trees without making anything current cleaner. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- Dockerfile | 4 +- docs/source/container.md | 6 +- docs/source/dependencies.md | 1 - docs/source/installation.md | 4 +- docs/source/pipeline_canfar.md | 48 +++---------- docs/source/pipeline_tutorial.md | 59 ++++++++-------- docs/source/random_cat.md | 67 ++++++------------- example/cfis_image_sims/README.md | 8 ++- .../config_tile_PiViVi_canfar_sx.ini | 2 +- pyproject.toml | 2 - scripts/README.rst | 9 --- scripts/python/canfar_avail_results.py | 14 ---- uv.lock | 48 +------------ workflow/README.md | 30 +++++---- 14 files changed, 92 insertions(+), 210 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0a549ae8f..f6d323706 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV SHELL=/bin/bash \ COVERAGE_FILE=/tmp/.coverage # System dependencies — three categories: -# - astromatic binaries (psfex, source-extractor, weightwatcher) ship as +# - astromatic binaries (psfex, source-extractor) ship as # Debian packages on bookworm; preferred over building from source. # - compilers and dev libs needed to build the heavier wheels (galsim, # mpi4py, python-pysap, fitsio). @@ -55,7 +55,7 @@ RUN apt-get update -y --quiet && \ libcfitsio-dev \ libproj-dev proj-bin \ libgl1-mesa-glx \ - psfex source-extractor weightwatcher && \ + psfex source-extractor && \ apt-get clean && rm -rf /var/lib/apt/lists/* # OpenMPI from source — required for hybrid Apptainer MPI on HPC clusters. diff --git a/docs/source/container.md b/docs/source/container.md index 174cde9dc..1a83b0f55 100644 --- a/docs/source/container.md +++ b/docs/source/container.md @@ -165,7 +165,7 @@ The Dockerfile does **not** duplicate Python deps — those come from The asymmetry is deliberate: Python deps go through pyproject + lockfile (reproducible, auditable), system deps go through Dockerfile (Debian's versioning). Don't `apt install` something that has a Python wheel; don't -`pip install` something Debian packages directly (e.g. `weightwatcher`). +`pip install` something Debian packages directly (e.g. `source-extractor`). ## Why this shape @@ -175,8 +175,8 @@ versioning). Don't `apt install` something that has a Python wheel; don't - **`uv sync --frozen`** at build time means the image is bit-exactly reproducible from a tagged commit, and impossible to ship with a stale lockfile. -- **Astromatic binaries from Debian** (`psfex`, `source-extractor`, - `weightwatcher`) instead of source builds — Debian carries the +- **Astromatic binaries from Debian** (`psfex`, `source-extractor`) + instead of source builds — Debian carries the GCC-compatibility patches that the previous Dockerfile had to apply inline with `sed`. - **Two targets** so canfar batch deployments stay slim while interactive diff --git a/docs/source/dependencies.md b/docs/source/dependencies.md index 9378bfe00..b907a452d 100644 --- a/docs/source/dependencies.md +++ b/docs/source/dependencies.md @@ -57,7 +57,6 @@ packages (no source builds), plus the MPI stack: |---------|------------| | [Source Extractor](https://www.astromatic.net/software/sextractor/) | {cite:p}`bertin:96` | | [PSFEx](https://www.astromatic.net/software/psfex/) | {cite:p}`bertin:11` | -| [WeightWatcher](https://www.astromatic.net/software/weightwatcher/) | {cite:p}`marmo:08` | | OpenMPI (5.0.x) | | Python dependencies themselves are managed with [uv](https://docs.astral.sh/uv/); diff --git a/docs/source/installation.md b/docs/source/installation.md index 83df0b6bb..ee3f8e182 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -38,8 +38,8 @@ docker pull ghcr.io/cosmostat/shapepipe:develop-runtime We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance. ``` -The image bundles the astromatic binaries (`source-extractor`, `psfex`, -`weightwatcher`), MPI (`mpi4py` + OpenMPI), and every Python dependency, so +The image bundles the astromatic binaries (`source-extractor`, `psfex`), +MPI (`mpi4py` + OpenMPI), and every Python dependency, so there is nothing else to install or build. To process data on a cluster with MPI, run the pipeline through Apptainer the same way you would any MPI job. diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 96add5758..926d36578 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -170,22 +170,16 @@ The downloaded tile weights are compressed. The following call uncompresses all. shapepipe_run -c cfis/config_tile_Uz.ini ``` -### Mask tiles +### Masks -This step is done globally for all tiles. There might be job failures or interruptions. The following -command to the `ShapePipe` job script can be run repeatedly; already created masks will be skipped. - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 4 -``` - -If masks were created in more than one run, i.e. situated in more than one output directory, these have to be -combined for subsequent pipeline module runs. This is done by creating a new output directory with symbolic -links, using the script - -```bash -combine_runs.bash -c flag_tile -``` +There is no masking step. `ShapePipe` generates no masks: the sky-fixed +healsparse maps are queried once per object, by `mask_query` on the exposure +catalogues (`FLAG_EXT`, cut by `setools`) and by `make_cat` on the final tile +catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` +config entries at the maps and nothing else is needed — no star-catalogue +download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that +touches pixels is the instrument flag image shipped with each exposure, which +`split_exp` splits per CCD. ## Tile detection @@ -205,7 +199,7 @@ canfar_submit_job -j 16 -f tile_numbers.txt -P N_PAR -v -J JMAX ### Exposure Processing -#### Option 0: Global split and exp masks (deprecated; used for earlier v1.x patch runs) +#### Option 0: Global split (deprecated; used for earlier v1.x patch runs) For this option, set `sp_local=0`. @@ -217,21 +211,7 @@ For `sp_local=-` both `mh_local` (0, 1) are ok: export mh_local=0 ``` -#### Option 0: Mask exposures (deprecated) - -Run repeatedly if necessary: - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 8 -``` - -Combine all runs: - -```bash -combine_runs.bash -c flag_exp -``` - -### Option 1: Local split and mask exposures (recommended) +### Option 1: Local split exposures (recommended) Optional: Enable flags for local split processing and merge header runs as @@ -258,12 +238,6 @@ First, determine the number of maximum jobs with the option `-s` (see above). Th canfar_submit_job -j 2 -v -f exp_shdu.txt -v -P N_PAR -J JMAX ``` -### Mask exposures - -```bash -canfar_submit_job -j 8 -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - ### Exposure detection ```bash diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 95a0aa94a..36dde0ccd 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -44,11 +44,11 @@ Naming and numbering of the input files can closely follow the original image na A stacked image is also called *tile*. These files are used on input by `ShapePipe`. The pixel data can contain the observed image, a weight map, or a flag map. Tile images and weights are created in the case of CFIS by Stephen Gwyn using a combination of `swarp` and his own software. Examples of file names are - `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tile flag files - are created the mask module of `ShapePipe` (see [Mask images](#mask-images)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for + `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tiles have no flag file + (see [Masks](#masks)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for clarity, we include the string `image` for a tile image type. Default convention: **-.fits** - Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz`, `pipeline_flag-239-293.fits` + Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz` - Database catalogue files For very large files that combine information from multiple tiles or single exposures, `ShapePipe` creates `sqlite` @@ -128,8 +128,6 @@ for all options. This script creates the subdirectory `$SP_RUN/output` to store all pipeline outputs (log files, diagnostics, statistics, output images, catalogues, single-exposure headers with WCS information). -Optionally, the subdir `output_star_cat` is created by the used to store the external star catalogues for masking. This is only necessary if the pipeline is run on a cluster without internet connection to access star catalogues. In that case, the star catalogues need to be retrieved outside the pipeline, for example on a login node, and copied to `output_star_cat`. - The job script automaticall performs a number of subsequent calls to the `ShapePipe` executable `shapepipe_run`, as ```bash shapepipe_run -c $SP_CONFIG/.ini @@ -189,32 +187,31 @@ Finally, the headers of all single-exposure single-CCD files are merged into a s Two output directories are created, `run_sp_Uz` for `uncompress_fits`, and `run_sp_exp_SpMh` for the output of the modules `split_exp` (`Sp`) and `merge_headers` (`Mh`). -## Mask images - -Run -```bash -job_sp TILE_ID -j 4 -``` -to mask tile and single-exposure single-CCD images. Both tasks are performed by two calls to the `mask` runner. - -Note that internet access is required for this step, since a reference star catalogue is downloaded. - -The output of both masking runs are stored in the output directory `run_sp_MaMa`, with run 1 (2) of -`mask` corresponding to tiles (exposures). - -**Diagnostics:** Open a single-exposure single-CCD image and the corresponding pipeline flag -in `ds9`, and display both frames next to each other. Example -```bash -ds9 image-2113737-10.fits pipeline_flag-2113737-10.fits -``` -Choose `zoom fit` for both frames, click `scale zscale` for the image, and `color aips0` for the flag, to display something like this: - - - -By eye the correspondence between the different flag types and the image can be -seen. Note that the two frames might not match perfectly, since (a) WCS -information is not available in the flag file FITS headers; (b) the image can -have a zero-padded pixel border, which is not accounted for by `ds9`. +## Masks + +`ShapePipe` does not generate masks. Sky-fixed masks — star halos, stars, +manual masks for large galaxies, per-band coverage, MaxiMask defects — are +supplied as [healsparse](https://healsparse.readthedocs.io) maps and are +consumed by *querying them at object positions*, never by rasterizing them onto +pixels. Two modules do the querying, from the same shared lookup +(`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and +`setools` on the single-exposure single-CCD catalogues and writes one integer +`FLAG_EXT` column (0 = clean), which `star_selection.setools` cuts on so that +masked objects never enter the PSF star sample; `make_cat` writes one +`MASK_` column per band onto the final tile catalogue, carrying the map +value verbatim so downstream selections choose their own cuts. Map paths and +bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and +`MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. + +No internet access is needed at any point, and there is no reference star +catalogue to download. + +The one mask that still reaches pixels is the **instrument flag image** +(`p.flag.fits.fz`) delivered with each exposure, which records bad columns +and saturation. `split_exp` splits it per CCD beside the image and weight, +`sextractor` reads it as `IMAFLAGS_ISO`, and `ngmix` zero-weights flagged +pixels in its postage stamps. Tiles have no such image, so tile detection runs +with `FLAG_IMAGE = False`. ## Detect objects on tiles and process stars on single exposures diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md index a930d40ed..b5ae9931d 100644 --- a/docs/source/random_cat.md +++ b/docs/source/random_cat.md @@ -3,16 +3,25 @@ This section describes how to create tile-based random catalogues and healpix masks, and combined randoms and masks for a selection of tiles. -The masked regions are obtained on input from ShapePipe pixel mask ("pipeline flag") -files. +The masked regions are obtained on input from per-tile pixel mask images. + +```{warning} +**ShapePipe no longer produces those images.** The pipeline generates no masks +at all: sky-fixed masks are healsparse maps, queried once per object into +catalogue columns (see [Masks](pipeline_tutorial.md#masks)), and tiles have no +flag image. `random_cat_runner` therefore needs its mask images supplied from +outside the pipeline — point its second `INPUT_DIR` entry at a directory of tile +mask images matching its `NUMBERING_SCHEME`. The healsparse-native replacement +for this whole procedure (an n_epoch / n_pointings survey-window map built from +the maps directly) is issue #797. +``` ```{note} Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see [VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The `random_cat` module itself is current; -the input-staging and joint-mask steps now overlap with -[`sp_validation`](https://github.com/CosmoStat/sp_validation). The steps are -retained for reference. +helper, which is no longer shipped. The input-staging and joint-mask steps now +overlap with [`sp_validation`](https://github.com/CosmoStat/sp_validation). The +steps are retained for reference. ``` ## Set up @@ -41,46 +50,14 @@ If not, we can just download the headers to gain significant download time. shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini ``` -### Check pixel mask files - -Make sure that all pixel mask files are present. If they have been downloaded from ``vos`` as ``.tgz`` files, -type -```bash -canfar_avail_results -i tile_numbers.txt --input_path . -v -m -o missing_mask.txt -``` -In case of missing mask files, check whether they are present in the ``vos`` remote directory, -```bash -canfar_avail_results -i tile_numbers.txt --input_path vos:cfis/vos-path/to/results -v -m -``` -If missing on ``vos``, process those tiles. If processing only up the the mask is necessary, -the following steps can be carried out, -```bash -job_sp -j 7 TILE_ID -job_sp -j 128 TILE_ID -``` -The first command processes the tile up to the mask; the second line uploads the mask files -to ``vos``. +### Stage the pixel mask files -Now, download the missing masks with -```bash -canfar_download_results -i missing_mask.txt --input_vos vos-path/to/results -m -v -``` -Untar .tgz files if required, -```bash -while read p; do tar xvf pipeline_flag_$p.tgz; done -.fits` +with the committed `config_Rc.ini`). How you obtain them is outside ShapePipe; +older runs of the pipeline's own (now removed) mask module wrote them as +`pipeline_flag--.fits`, and those files still work. ## Create random catalogue and helapix mask per tile diff --git a/example/cfis_image_sims/README.md b/example/cfis_image_sims/README.md index b308cd6a8..fa0e00ac4 100644 --- a/example/cfis_image_sims/README.md +++ b/example/cfis_image_sims/README.md @@ -18,7 +18,10 @@ those cases are documented in the last column below and, at more length, under Every row is derived from the two bash scripts. "Module(s)" is the ShapePipe runner(s) the selected `.ini` names; "`.ini` selected" is what `job_sp_canfar_v2.0.bash` picks for that bit under sim settings -(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`, `star_cat_for_mask=onthefly`). +(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`). Bit 32 (mask exposures) is +gone: ShapePipe generates no masks (PR #847), so the bash scripts' +`star_cat_for_mask` setting and the `config_*_Ma_*.ini` configs it selected no +longer exist. | Bit | Stage | Module(s) | `.ini` selected (sim settings) | Sim special-casing | |----:|-------|-----------|--------------------------------|--------------------| @@ -27,10 +30,9 @@ runner(s) the selected `.ini` names; "`.ini` selected" is what | 4 | find exposures | `find_exposures_runner` | `config_tile_Fe.ini` | — | | 8 | retrieve exposure images | `get_images_runner` | `config_exp_Gie_symlink.ini` | symlink retrieval; completeness check expects 3 files vs. 6 for data | | 16 | split exposures, merge WCS headers | `split_exp_runner` | `config_exp_Sp.ini` | — | -| 32 | mask exposures | `mask_runner` | `config_exp_Ma_onthefly.ini` | — | | 64 | exposure PSF model | *(none — placeholder)* | *(none)* | **Placeholder.** For data this runs full exposure PSF modelling. For sims run_job writes a placeholder log and does nothing here; the sim PSF (`fake_psf_runner`) actually runs inside bit 512 | | 128 | merge exposure WCS headers → tile sqlite log | `merge_headers_runner` | `config_tile_Mh_exp.ini` | — | -| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx_nomask.ini` | `tile_det` is forced to `sx`, so the SExtractor-no-mask branch is always taken (the `uc` external-catalogue branch is never reached for sims) | +| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx.ini` | `tile_det` is forced to `sx`, so the SExtractor branch is always taken (the `uc` external-catalogue branch is never reached for sims). Tiles carry no flag image — ShapePipe generates no masks — so this runs with `FLAG_IMAGE = False` | | 512 | fake PSF + postage stamps | `fake_psf_runner`, then `vignetmaker_runner` ×2 | `config_exp_psfex.ini` (fake PSF), then `config_tile_PiViVi_canfar_sx.ini` (vignets) | **Two sub-runs.** run_job first calls the job script with `-j 64` → `config_exp_psfex.ini`, which despite its name runs `fake_psf_runner` (needs the sexcat from bit 256; run dir `run_sp_tile_fpsf`), then `-j 512` → `config_tile_PiViVi_canfar_sx.ini` for the two `vignetmaker_runner` runs. Data instead runs `psfex_interp_runner` + vignets here | | 1024 | multi-epoch shape measurement | `ngmix_runner` | `config_tile_Ng_batch_psfex_sx.ini` | — | | 2048 | create final catalogue | `make_cat_runner` | `config_tile_Mc_psfex.ini` | — | diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini index 28ed565af..26564e4fa 100644 --- a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -130,7 +130,7 @@ ME_IMAGE_PATTERN = flag, image, weight [VIGNETMAKER_RUNNER_RUN_3] # Cut per-object coadd-frame segmentation stamps from the tile SExtractor -# SEGMENTATION check image (config_tile_Sx_nomask.ini: CHECKIMAGE = BACKGROUND, +# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, # SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode # guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN # centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg diff --git a/pyproject.toml b/pyproject.toml index 5388adc86..7ec4c0528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,11 @@ requires-python = ">=3.12" # the code actually requires the newer API. dependencies = [ "astropy>=7.0", # major 6 → 7 - "astroquery", "canfar", "cs_util>=0.2.1", "galsim>=2.8", "h5py", "healsparse", - "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", diff --git a/scripts/README.rst b/scripts/README.rst index 1d3e265a6..2b8067cb0 100644 --- a/scripts/README.rst +++ b/scripts/README.rst @@ -9,7 +9,6 @@ Python scripts ============== 1. `create_log_exp_headers`_ -2. `create_star_cat`_ create_log_exp_headers ====================== @@ -18,11 +17,3 @@ This as to run after the module `split_exp_runner` it will create a master log file containing all the WCS information for each CCDs of each single exposures. To run the script : `python create_log_exp_headers.py path/to/split_exp_runner/output path/to/srcipt/output_dir` - -create_star_cat -=============== - -This script create all the star catalogs required to run the mask module for a -computational node without internet access. -To run the script : -`python create_star_cat.py path/to/image_dir path/to/script/output_dir` diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py index 62fbf0564..cb9afcb0b 100755 --- a/scripts/python/canfar_avail_results.py +++ b/scripts/python/canfar_avail_results.py @@ -113,13 +113,6 @@ def parse_options(p_def): action="store_true", help="only check final catalogues", ) - parser.add_option( - "-m", - "--mask_only", - dest="mask_only", - action="store_true", - help="only check mask files (pipeline_flag)", - ) parser.add_option( "-x", "--extension", @@ -159,10 +152,6 @@ def check_options(options): print("Invalid PSF model '{}'".format(options.psf)) return False - if options.final_only and options.mask_only: - print("One one of the options '-f' or '-m' can be given") - return False - return True @@ -379,13 +368,10 @@ def main(argv=None): if param.final_only: result_base_names = ["final_cat"] - elif param.mask_only: - result_base_names = ["pipeline_flag"] else: result_base_names = [] types = [ "final_cat", - "pipeline_flag", "logs", "setools_mask", "setools_stat", diff --git a/uv.lock b/uv.lock index a4d20ff52..27fe9cd5b 100644 --- a/uv.lock +++ b/uv.lock @@ -168,24 +168,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/27/0775134a2a939ddf7a073e77e66110b671e99cefbc36fb57b57851fd109a/astropy_iers_data-0.2026.8.10.0.32.39-py3-none-any.whl", hash = "sha256:04239afd6e9615165b84da1b84d73f03ca473f0ddf53b516c260e27349a95d03", size = 1998788, upload-time = "2026-08-10T00:33:34.729Z" }, ] -[[package]] -name = "astroquery" -version = "0.4.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "beautifulsoup4" }, - { name = "html5lib" }, - { name = "keyring" }, - { name = "numpy" }, - { name = "pyvo" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/48/273dbde090e071f9d264d084bc49193d126498d2906172b78febd9d62e28/astroquery-0.4.11.tar.gz", hash = "sha256:5537529bddc7fa07e773d5cd9baca593e3f5d93474edd1914f68e89506042b33", size = 12561055, upload-time = "2025-09-20T04:26:36.744Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/ca/944b328f2c60b83896a9223312e21e46cdc1fef57565e853da4544ff6a8e/astroquery-0.4.11-py3-none-any.whl", hash = "sha256:e34f114b285dd07a10ddb2065ebce829b01b0e740fd89dbc81a3077808e24b2d", size = 11139417, upload-time = "2025-09-20T04:26:31.881Z" }, -] - [[package]] name = "asttokens" version = "3.0.2" @@ -1151,19 +1133,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, ] -[[package]] -name = "html5lib" -version = "1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173, upload-time = "2020-06-22T23:32:36.781Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" @@ -2951,19 +2920,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, ] -[[package]] -name = "pyvo" -version = "1.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/35/12c0f4fa0879316837ac56f275942f006f0735e5aedba529b4449dddc36f/pyvo-1.9.1.tar.gz", hash = "sha256:2f26c99af7c32f3c34b919e2d14eaf1a95914176d693fb7769773f3ab0b7999d", size = 2167800, upload-time = "2026-06-11T14:44:01.349Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/09/04ff6e8beaa6cd60a960f925d837c2afa3fffba7860042ab1b52a8358297/pyvo-1.9.1-py3-none-any.whl", hash = "sha256:098648d00943440f56c1d00ac76330433578a0725ea6187afaa6bae08545bb53", size = 1152763, upload-time = "2026-06-11T14:43:59.016Z" }, -] - [[package]] name = "pywavelets" version = "1.9.0" @@ -3442,11 +3398,11 @@ version = "1.1.0" source = { editable = "." } dependencies = [ { name = "astropy" }, - { name = "astroquery" }, { name = "canfar" }, { name = "cs-util" }, { name = "galsim" }, { name = "h5py" }, + { name = "healsparse" }, { name = "joblib" }, { name = "matplotlib" }, { name = "mccd" }, @@ -3518,13 +3474,13 @@ test = [ [package.metadata] requires-dist = [ { name = "astropy", specifier = ">=7.0" }, - { name = "astroquery" }, { name = "build", marker = "extra == 'release'" }, { name = "canfar" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util?branch=develop" }, { name = "fitsio", marker = "extra == 'fitsio'" }, { name = "galsim", specifier = ">=2.8" }, { name = "h5py" }, + { name = "healsparse" }, { name = "hypothesis", marker = "extra == 'test'", specifier = ">=6.155.2" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=9.14.1" }, { name = "joblib", specifier = ">=1.4" }, diff --git a/workflow/README.md b/workflow/README.md index 08bfd2291..ac997afef 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,8 +22,7 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the -# star-catalogue cache root). +# Edit workflow/config.yaml: tile_list, run_dir, container. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -154,8 +153,8 @@ workflow/ bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures - exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) - tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + exposure.smk per-exposure: get_images, split, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) build_index.py prepare-phase run_index.sqlite builder (plain script) @@ -196,16 +195,19 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee committed under `workflow/config/cfis/` and version with the rules that set the env vars they interpolate — there is no `config_src` knob, and no per-unit config symlink; `$SP_CONFIG` points straight at the committed directory. -- **Mask star catalogues are built in the DAG.** `exp_star_cat` runs one Vizier - cone query per exposure into the run-independent cache at `star_cats:`, then - fans it out into a real per-unit `star_cat_exp/` directory of 40 per-CCD - symlinks, which `exp_mask` consumes. The directory must be per-unit and real: - the file handler intersects the image numbers it finds across a config's - `INPUT_DIR`s, so a symlink to the whole cache contributes every other - exposure's numbers and the intersection comes out empty. It is a `localrule`, - so the queries run serially in the head process — CDS is never hammered, and - the scheduler never sees a six-second job. The cache makes reruns and later - campaigns free. +- **There is no masking stage, on either side.** ShapePipe generates no masks + (PR #847). The one mask that reaches pixels is the instrument flag image + delivered with each exposure, which `exp_split` splits per CCD beside image + and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star + halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed + healsparse maps and QUERIED once per object: the `mask_query` module writes a + `FLAG_EXT` column onto each CCD's detection catalogue for setools' star cut + (inside `exp_psf`), and `make_cat` writes one `MASK_` column per band + onto the final catalogue (inside `tile_make_cat`). Map paths are config, not + code, so regenerated products cost a config edit. Nothing is fetched from a + catalogue server, staged, or rasterized, which is why the old + `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are + gone. - **The index is parse-time data, never a rule input.** Appending tiles changes which jobs exist without invalidating completed work. - **Exposure products are not `temp()`.** Exposures overlap tiles, so From 897febc1cffd55b0ab8fe17a686f12d3cff4c5c7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:48:53 -0400 Subject: [PATCH 079/100] feat(mask_query): narrow the shipped PSF-star diet to the star-body map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diet is settled: instrument flags (IMAFLAGS_ISO, already read by SExtractor) plus the UNIONS star-body map (bit 2, mask_ugriz_nside131072_n4.hsp), and nothing else. The committed [MASK_QUERY_RUNNER] examples in config_exp_psfex.ini (both copies) and config_exp_mccd.ini now name exactly that one map instead of the two-map placeholder. Halo bits 0 and 1 stay out on purpose — halos flag objects for the final catalogue, they do not reject PSF stars (mask-force telecon, 2026-07-21) — and MaxiMask is out too. The module docstring says so under its own heading, so that a reader who finds one path in the config knows it is a decision rather than an unfinished list, and knows widening it is a config edit and no code. That asymmetry also explains the contract: MASK_PATHS is a path list rather than a bit mask because the UNIONS products are one boolean map per bit, so choosing bits is choosing files. MASK_BITS survives for integer maps that pack several bits into one file, and its commented example drops from 1028 to 4 to match the diet. The setools header and the tutorial's Masks section carry the same note. Configs re-validated in the container (0 bad, all MODULE lists resolve) and the 10 mask tests still pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_tutorial.md | 4 +++- example/cfis/config_exp_mccd.ini | 20 ++++++++++++++----- example/cfis/config_exp_psfex.ini | 19 ++++++++++++------ example/cfis/star_selection.setools | 7 +++++-- .../modules/mask_query_package/__init__.py | 15 ++++++++++++++ workflow/config/cfis/config_exp_psfex.ini | 19 ++++++++++++------ 6 files changed, 64 insertions(+), 20 deletions(-) diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 36dde0ccd..a4db96579 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -197,7 +197,9 @@ pixels. Two modules do the querying, from the same shared lookup (`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and `setools` on the single-exposure single-CCD catalogues and writes one integer `FLAG_EXT` column (0 = clean), which `star_selection.setools` cuts on so that -masked objects never enter the PSF star sample; `make_cat` writes one +masked objects never enter the PSF star sample — a deliberately narrow diet, +the star-body map (bit 2) only, since halos flag objects without disqualifying +them as PSF stars; `make_cat` writes one `MASK_` column per band onto the final tile catalogue, carrying the map value verbatim so downstream selections choose their own cuts. Map paths and bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 8e9ffbd8f..68dcfbeef 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -137,11 +137,21 @@ FILE_PATTERN = sexcat_sexcat NUMBERING_SCHEME = -0000000-0 -# External healsparse masks queried at each detection's (RA, Dec); see the -# mask_query module docstring. star_selection.setools cuts on FLAG_EXT == 0. -MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp - -; MASK_BITS = 1028 +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets FLAG_EXT, which star_selection.setools cuts on as +# FLAG_EXT == 0. Comma-separated. +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 [SETOOLS_RUNNER] diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 10566d5ed..2f1c8e353 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -139,14 +139,21 @@ FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 -# External healsparse masks queried at each detection's (RA, Dec). Any map -# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which -# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. -MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets FLAG_EXT, which star_selection.setools cuts on as +# FLAG_EXT == 0. Comma-separated. +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, -# any nonzero value flags. Boolean maps ignore it. -; MASK_BITS = 1028 +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 [SETOOLS_RUNNER] diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index f35043d99..230017053 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -4,8 +4,11 @@ ## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), ## delivered with the exposure and read by SExtractor; ## FLAG_EXT == 0 the external healsparse masks, queried per detection by -## the mask_query module (which map bits reach FLAG_EXT is -## that module's MASK_PATHS / MASK_BITS config). +## the mask_query module. Which maps reach FLAG_EXT is that +## module's MASK_PATHS config, and the shipped diet is +## deliberately narrow: the star-body map (bit 2) only, no +## halos (they flag, they do not reject stars) and no +## MaxiMask. ## SETools expressions have no bitwise operators, so mask_query does the bit ## selection and this file only tests for zero. diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index b4b1d92c1..087e50722 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -34,6 +34,21 @@ ``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is written once. That module's docstring documents the off-coverage convention. +The diet is deliberately narrow +=============================== + +``MASK_PATHS`` is a *list of maps to reject PSF stars on*, not a list of every +mask that exists. The committed configs name exactly one map — the UNIONS +star-body product (bit 2) — beside the instrument flags SExtractor already +reads. Halo bits 0 and 1 are excluded on purpose: halos flag objects for the +final catalogue, they do not reject PSF stars (mask-force telecon, 2026-07-21). +MaxiMask is not in the diet either. + +Widening it costs a config edit and no code — add a path. That is why the +contract is a path list rather than a bit mask: the UNIONS products are one +boolean map per bit, so choosing bits *is* choosing files, and ``MASK_BITS`` +exists only for integer maps that pack several bits into one file. + Module-specific config file entries =================================== diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 08f45f99b..ffcac86b6 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -138,14 +138,21 @@ FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 -# External healsparse masks queried at each detection's (RA, Dec). Any map -# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which -# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. -MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets FLAG_EXT, which star_selection.setools cuts on as +# FLAG_EXT == 0. Comma-separated. +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, -# any nonzero value flags. Boolean maps ignore it. -; MASK_BITS = 1028 +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 [SETOOLS_RUNNER] From ba2271a55cd57f425962679b8830865c27b91636 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:53:36 -0400 Subject: [PATCH 080/100] =?UTF-8?q?fix(mask):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20CI=20smoke,=20SEGMENTATION=20checkimage,=20canfar?= =?UTF-8?q?=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups from review. CI. deploy-image.yml's runtime binary smoke still ran `weightwatcher --version` after the Dockerfile dropped the package, so the very next image build would have failed on a tool nothing calls. Step line and the comment's mention both go. SEGMENTATION, and this one was a real regression I introduced. Folding config_tile_Sx_nomask.ini into config_tile_Sx.ini carried the nomask variant's `CHECKIMAGE = BACKGROUND` over the masked config's `BACKGROUND, SEGMENTATION`. The segmentation check image is not masking furniture: vignetmaker cuts per-object segmentation stamps from it (config_tile_PiViVi_canfar_sx.ini FILE_PATTERN = sexcat, segmentation), and ngmix's uberseg blend handling raises at construction without them (ngmix.py ~496, shapepipe#776). Restored in both copies. The sims config already had it, which is why nothing else caught this. Docs. pipeline_canfar.md's "Collate star catalogues" section drove collate_star_cat.py, deleted with the star-catalogue tooling. The recipe is replaced with a warning that names what is missing rather than a silent cut: the merge step below it consumes validation_psf_conv files that now have no producer in this repo, and the star_cat/ + `combine_runs.bash -c psf_conv` staging around it does not apply either. The PSF measurement itself is unchanged — shapes are measured in sky coordinates during interpolation — so the collation is a pass-through that is straightforward to rebuild. No other docs page referenced a deleted script. Re-verified in the container: 30/30 module runners import, 65 configs' MODULE lists resolve (0 bad), the 10 mask tests pass, the workflow still parses and builds its DAG, and deploy-image.yml is valid YAML. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- .github/workflows/deploy-image.yml | 3 +- docs/source/pipeline_canfar.md | 72 ++++++------------------- example/cfis/config_tile_Sx.ini | 2 +- workflow/config/cfis/config_tile_Sx.ini | 2 +- 4 files changed, 20 insertions(+), 59 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 53e67dcca..6de369a66 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -83,12 +83,11 @@ jobs: # Smoke-test the binaries baked into the runtime image. Catches the # class of regression where the image builds but a runtime tool - # (sextractor, weightwatcher) is missing or unrunnable. + # (sextractor, psfex) is missing or unrunnable. - name: Test runtime — binaries run: | IMAGE=$(echo "${{ steps.meta-runtime.outputs.tags }}" | head -n1) docker run --rm "$IMAGE" source-extractor --version - docker run --rm "$IMAGE" weightwatcher --version docker run --rm "$IMAGE" psfex --version - name: Test runtime — shapepipe entry point (read-only fs) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 926d36578..108f948f4 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -309,61 +309,23 @@ shapepipe_run -c $SP_CONFIG/config_Pl_$psf.ini #### Collate star catalogues -Collate all input validation PSF files into star catalogues, gathering positions -(X/Y/RA/DEC) and the MCCD CCD id. - -Note: HSM shapes are no longer rotated into world coordinates at this step. The -PSF/star ellipticities and sizes are now measured directly in sky coordinates -during PSF interpolation (galsim `FindAdaptiveMom(use_sky_coords=True)`), so -`collate_star_cat.py` only collates and passes the shapes through. - -> **v2.0 is patch-less.** Runs up to `v1.6` are organised in sky patches -> `P1`..`P`, each patch a run directory. `v2.0` removes the patch concept: a -> single run root, outputs directly under it. `v2.0` is the default; select an -> older layout with `-V` (e.g. `-V v1.6`). For `v2.0` the patch loop and the -> `-P` option no longer apply, and the patch token drops from the output -> filename (`validation_psf_conv-.fits` instead of -> `validation_psf_conv--.fits`). - -```bash -cd /path/to/version -mkdir star_cat -cd star_cat -``` - -For `v2.0` (the default), run once against the patch-less run root, producing -files `validation_psf_conv-.fits`: - -```bash -collate_star_cat.py -i .. -v -``` - -For `v1.x`, pass the version explicitly and run once per patch, creating a -directory per patch `P?` and producing files -`validation_psf_conv--.fits` (for the v1.4 setup only one file): - -```bash -collate_star_cat.py -i .. -V v1.6 -P $patchnum -v -``` - -Combine previously created files as links within one ShapePipe run directory (for the v1.4 setup only one link). -First (and optiohnal), create a subdir for a run and link to the input patches: - -```bash -cd /path/to/version/star_cat -mkdir v1.6 -ln -s ../P1 -ln -s ../P2 -... -``` - -Next, create links to all `validation_conv` runs: - -```bash -combine_runs.bash -p psfex -c psf_conv -``` - -Merge all converted star catalogues and create `final-starcat.fits`: +```{warning} +**This step no longer ships.** `collate_star_cat.py` gathered each validation +PSF run's star positions (X/Y/RA/DEC) and MCCD CCD id into the +`validation_psf_conv-*.fits` files the merge below consumes, and it was deleted +with the star-catalogue tooling the removed mask module needed (PR #847). The +merge step that follows therefore has no producer for its input in this repo +until the collation is reimplemented, and the `star_cat/` + `combine_runs.bash +-c psf_conv` staging around it does not apply either. + +Nothing about the PSF measurement itself changed: HSM shapes are measured +directly in sky coordinates during PSF interpolation (galsim +`FindAdaptiveMom(use_sky_coords=True)`), which is why the collation was a +pass-through for shapes and is straightforward to rebuild. +``` + +Merge all converted star catalogues and create `final-starcat.fits` (this +reads the `validation_psf_conv` files the collation above used to produce): ```bash export SP_RUN=`pwd` diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini index 9859e4237..80350eead 100644 --- a/example/cfis/config_tile_Sx.ini +++ b/example/cfis/config_tile_Sx.ini @@ -102,7 +102,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 0ce9ea226..5487f547c 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -101,7 +101,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat From f7d1fd663d19c96fab76b6a0e7091c5f11b28392 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:00:37 -0400 Subject: [PATCH 081/100] perf(mask_query): read only the coverage pixels a catalogue touches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five review items, all in the query path. PARTIAL READS. query_map did HealSparseMap.read(path) on a 583 MB map, per CCD — ~22 GB of I/O per 40-CCD exposure to answer questions about 0.06 deg². It now reads HealSparseCoverage first, computes the coverage pixels the positions touch with hpgeom.angle_to_pixel at the map's nside_coverage, and loads only those. make_cat gets this for free, sharing the primitive. Measured on the DR6 star map for 2000 positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, IDENTICAL values. 125x faster, 21x less memory. The full-read footprint is 3.3 GiB — under the 4 GB mark — and the partial read adds ~0.15 GB to a rule already asking for 16 GB, so exp_psf's mem_mb is left alone. Per exposure this is ~4 s of map reading instead of ~8.5 min. test_partial_read_matches_full pins partial == full for both map dtypes. Two things the reviewer's sketch did not anticipate, both found by probing the real map rather than reasoning. (a) healsparse RAISES when no requested pixel is in the coverage map, so the all-off-coverage case is answered without asking it: one small probe read for dtype and sentinel, then the sentinel everywhere. (b) The neighbours padding is insurance, not necessity — every position falls in exactly one coverage pixel and all are requested — but at nside_coverage=128 it costs under a megabyte, so it stays. BOOLEAN OFF-COVERAGE. n_off was hardcoded to 0 for boolean maps, so a map that misses an exposure logged "0 flagged, 0 outside coverage" — indistinguishable from clean. valid_mask=True cannot fix this: the UNIONS products are BOOLEAN, and healsparse stores only the True pixels, so valid_mask returns the value itself (verified against mask_r_nside131072_n4.hsp). Coverage now comes from the coverage mask, which works for both dtypes, and all-off-coverage logs a warning saying the zero column means "the map does not reach here", not "clean". Also: FLAG_EXT's docstring no longer claims the value says which bits fired — true only for integer maps, since boolean maps can only contribute 1. config_Rc.ini's mask INPUT_DIR placeholder becomes rather than a $SP_CONFIG path that does not exist. A zero-detection CCD no longer raises on data["XWIN_WORLD"] — it writes an empty FLAG_EXT column, matching the sparse-CCD tolerance setools and the completeness floor already carry. hpgeom returns to pyproject.toml, now imported directly rather than via healsparse. pipeline_canfar.md's HSM sky-coordinates paragraph is lifted out of the collate_star_cat deletion warning into standing prose, pointing at the test that pins the convention. Verified: 15/15 mask tests pass, 30/30 runners import, 65 configs resolve, the workflow still builds its DAG, uv.lock regenerated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 10 +- example/cfis/config_Rc.ini | 2 +- pyproject.toml | 1 + .../modules/mask_query_package/__init__.py | 10 +- .../modules/mask_query_package/mask_query.py | 20 ++- src/shapepipe/utilities/mask_query.py | 163 ++++++++++++++++-- tests/module/test_mask_query.py | 113 +++++++++++- uv.lock | 2 + 8 files changed, 288 insertions(+), 33 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 108f948f4..350c4a403 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -317,12 +317,14 @@ with the star-catalogue tooling the removed mask module needed (PR #847). The merge step that follows therefore has no producer for its input in this repo until the collation is reimplemented, and the `star_cat/` + `combine_runs.bash -c psf_conv` staging around it does not apply either. +``` -Nothing about the PSF measurement itself changed: HSM shapes are measured +HSM shapes are **not** rotated into world coordinates at this step, and were not +before it was removed. The PSF and star ellipticities and sizes are measured directly in sky coordinates during PSF interpolation (galsim -`FindAdaptiveMom(use_sky_coords=True)`), which is why the collation was a -pass-through for shapes and is straightforward to rebuild. -``` +`FindAdaptiveMom(use_sky_coords=True)`) — see `tests/module/test_hsm_sky_coords.py`, +which pins that convention. The collation was a pass-through for shapes, which is +both why it carried no science and why it is straightforward to rebuild. Merge all converted star catalogues and create `final-starcat.fits` (this reads the `validation_psf_conv` files the collation above used to produce): diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini index 5d04e8e37..9f55f1812 100644 --- a/example/cfis/config_Rc.ini +++ b/example/cfis/config_Rc.ini @@ -57,7 +57,7 @@ TIMEOUT = 96:00:00 # masks. Point the second entry at a directory of tile mask images matching # NUMBERING_SCHEME below (the healsparse-native replacement for this module is # the survey-window work, issue #797). -INPUT_DIR = last:get_images_runner, $SP_CONFIG/tile_masks +INPUT_DIR = last:get_images_runner, FILE_PATTERN = CFIS_image, mask diff --git a/pyproject.toml b/pyproject.toml index 7ec4c0528..4c5fd7e02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "galsim>=2.8", "h5py", "healsparse", + "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index 087e50722..0391e5393 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -20,9 +20,13 @@ maps, writing one integer column: ``FLAG_EXT`` - ``0`` for an object no configured map flags, nonzero otherwise. The nonzero - value is the bitwise OR of the contributing map values, so it says *which* - bits fired, but nothing downstream is required to read it that way. + ``0`` for an object no configured map flags, nonzero otherwise. What the + nonzero value *is* depends on the maps: a boolean map — which is what the + UNIONS per-bit products are, and what the shipped config names — can only + contribute ``1``, so with those the column is 0/1 and says nothing about + which map fired. An integer map contributes its own value (optionally + ``& MASK_BITS``), and contributions are OR-ed, so a bit-packed map does + carry its bits through. Nothing downstream reads more than ``== 0``. The single column exists because ``setools`` expressions support only ``< > <= >= == !=`` — no bitwise operators — so the bit selection has to diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py index 16f3c5ff5..08e153a00 100644 --- a/src/shapepipe/modules/mask_query_package/mask_query.py +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -67,8 +67,24 @@ def process(self): ) ori_cat.open() data = ori_cat.get_data() - ra = np.copy(data["XWIN_WORLD"]) - dec = np.copy(data["YWIN_WORLD"]) + + # A CCD SExtractor found nothing on is tolerated all along this chain + # (setools' ~0.2% attrition, psfex_interp's floor=0 warn), so it must + # not be an error here either. An empty LDAC table has no columns to + # index, so read the positions only when there are rows, and still + # publish an output file — a missing sexcat_ext would look to the file + # handler like a crash rather than like an empty CCD. + if len(data) == 0: + ra = np.zeros(0) + dec = np.zeros(0) + if self._w_log is not None: + self._w_log.info( + "No detections in " + + f"{self._sexcat_path}; writing an empty FLAG_EXT column" + ) + else: + ra = np.copy(data["XWIN_WORLD"]) + dec = np.copy(data["YWIN_WORLD"]) flag = mask_query_util.flag_positions( self._mask_paths, diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index 0cc64255d..c3b150006 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -17,6 +17,28 @@ flagged (nonzero)" so that ``setools`` — whose expression language has no bitwise operators — can cut on ``FLAG_EXT == 0``. +Partial reads +------------- +Never read a whole map. The UNIONS products are ~550 MB each and ``mask_query`` +runs PER CCD, so a full read would cost ~22 GB of I/O per 40-CCD exposure to +answer questions about a 0.06 deg² footprint. Instead the coverage index is +read first (``HealSparseCoverage``, a few hundred kB), the coverage pixels the +catalogue actually touches are computed with ``hpgeom.angle_to_pixel`` at the +map's ``nside_coverage``, and only those get loaded. + +Every queried position falls inside exactly one coverage pixel and all of them +are requested, so the padding by ``hpgeom.neighbors`` is insurance rather +than necessity — at ``nside_coverage=128`` a coverage pixel of the star map +is ~23 kB, so the padding costs under a megabyte and buys immunity to any +edge convention we did not think of. ``test_partial_read_matches_full`` is +what actually holds the two paths equal. + +Measured on the DR6 star map (``mask_r_nside131072_n4.hsp``, 583 MB, +``nside_coverage=128``) for 2000 positions in one CCD-sized box, one process +each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, identical +values. Per 40-CCD exposure that is ~4 s against ~8.5 min of map reading, and +the query adds ~0.15 GB to a rule already asking for 16 GB. + Coverage -------- ``healsparse.HealSparseMap.get_values_pos`` returns a map's *sentinel* for @@ -24,11 +46,20 @@ ``-1`` for integer maps. ``make_cat`` passes that sentinel through verbatim, which is the documented off-map flag for the final catalogue. -``flag_positions`` instead treats off-coverage as **not flagged**, for both map -kinds. This makes the integer case agree with the boolean case (whose sentinel -is literally ``False``) rather than diverge from it, and it keeps a map whose +Coverage is reported from the COVERAGE MASK, not ``valid_mask=True``. For a +boolean map — which is what the UNIONS per-bit products are — healsparse +stores only the ``True`` pixels, so ``valid_mask`` returns the value itself +and cannot tell "inside the footprint and clean" from "outside it entirely". +The coverage mask can, at ``nside_coverage`` resolution, which is the scale the +question is asked at anyway: does this map reach this exposure at all? + +``flag_positions`` treats off-coverage as **not flagged**, for both map kinds. +This makes the integer case agree with the boolean case (whose sentinel is +literally ``False``) rather than diverge from it, and it keeps a map whose coverage does not reach an exposure from silently rejecting every star on it. -Off-coverage counts are logged so the situation is visible rather than silent. +That is also why the all-off-coverage case is logged as a WARNING: it is +indistinguishable from "nothing is masked here" in the output column, so it has +to be distinguishable in the log. :Author: Claude Fable 5, for PR #847 @@ -56,10 +87,106 @@ def parse_map_paths(paths_str): return [path.strip() for path in paths_str.split(",") if path.strip()] +def _covering_pixels(coverage, ra, dec): + """Covering Pixels. + + The map's coverage pixels touched by these positions, padded by their + neighbours and intersected with what the map actually holds. + + Parameters + ---------- + coverage : healsparse.HealSparseCoverage + Coverage index of the map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Coverage pixel indices to load, possibly empty + + """ + import hpgeom + + nside_coverage = coverage.nside_coverage + touched = np.unique( + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ) + padded = np.unique( + np.concatenate( + [touched, hpgeom.neighbors(nside_coverage, touched).ravel()] + ) + ) + # neighbors() returns -1 for a non-existent neighbour. + padded = padded[padded >= 0] + + return padded[coverage.coverage_mask[padded]] + + +def query_map_coverage(path, ra, dec): + """Query Map With Coverage. + + Read only the part of a healsparse map these positions need, and return + both its value at each position and whether each position is inside the + map's coverage. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + tuple + ``(values, in_coverage)`` — the map value at each position (the map's + sentinel outside coverage) and a boolean array, both of length + ``len(ra)`` + + """ + import healsparse + import hpgeom + + ra = np.asarray(ra) + dec = np.asarray(dec) + + coverage = healsparse.HealSparseCoverage.read(path) + nside_coverage = coverage.nside_coverage + + in_coverage = coverage.coverage_mask[ + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ] + + pixels = _covering_pixels(coverage, ra, dec) + + if pixels.size == 0: + # healsparse raises when no requested pixel is in the coverage map, so + # the empty case is answered without asking it: load one arbitrary + # coverage pixel purely to learn the dtype and sentinel, and return + # that sentinel everywhere. Same answer, one small read. + probe = int(np.flatnonzero(coverage.coverage_mask)[0]) + mask_map = healsparse.HealSparseMap.read(path, pixels=[probe]) + values = np.full(ra.size, mask_map._sentinel, dtype=mask_map.dtype) + return values, in_coverage + + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(pixel) for pixel in pixels] + ) + values = np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + + return values, in_coverage + + def query_map(path, ra, dec): """Query Map. - Read a healsparse map and return its value at each world position. + Return a healsparse map's value at each world position, reading only the + coverage pixels those positions touch. Parameters ---------- @@ -77,11 +204,9 @@ def query_map(path, ra, dec): the map's sentinel value """ - import healsparse - - mask_map = healsparse.HealSparseMap.read(path) + values, _ = query_map_coverage(path, ra, dec) - return np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + return values def flag_positions(paths, ra, dec, bits=None, w_log=None): @@ -123,30 +248,40 @@ def flag_positions(paths, ra, dec, bits=None, w_log=None): dec = np.asarray(dec) flag = np.zeros(ra.size, dtype=np.int64) + if ra.size == 0: + return flag + for path in paths: - values = query_map(path, ra, dec) + values, in_coverage = query_map_coverage(path, ra, dec) if values.dtype == bool: contribution = values.astype(np.int64) - n_off = 0 else: integer = values.astype(np.int64) # Off-coverage: the sentinel, negative by healsparse convention. # Zeroed rather than OR-ed in, see this module's docstring. - off_coverage = integer < 0 - n_off = int(np.count_nonzero(off_coverage)) - integer = np.where(off_coverage, 0, integer) + integer = np.where(integer < 0, 0, integer) if bits is not None: integer &= bits contribution = integer flag |= contribution + n_off = int(np.count_nonzero(~in_coverage)) if w_log is not None: w_log.info( f"Mask query {path}: " f"{int(np.count_nonzero(contribution))}/{ra.size} objects " f"flagged, {n_off} outside coverage" ) + if n_off == ra.size: + # Every object reads the sentinel, so the column is all-zero + # and looks exactly like "nothing is masked here". Say it. + w_log.warning( + f"Mask query {path}: NO object is inside this map's" + + " coverage — the resulting FLAG_EXT contribution is" + + " zero everywhere because the map does not reach these" + + " positions, not because they are clean." + ) return flag diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py index 053dffd1e..779a6bd69 100644 --- a/tests/module/test_mask_query.py +++ b/tests/module/test_mask_query.py @@ -35,8 +35,17 @@ class _NullLogger: - def info(self, *_args, **_kwargs): - pass + """Captures what the module logs, so coverage warnings can be asserted.""" + + def __init__(self): + self.info_msgs = [] + self.warning_msgs = [] + + def info(self, msg, *_a, **_kw): + self.info_msgs.append(str(msg)) + + def warning(self, msg, *_a, **_kw): + self.warning_msgs.append(str(msg)) def _write_map(path, value, dtype=np.int16, n_covered=2): @@ -73,7 +82,7 @@ def _write_bool_map(path, n_covered=2): return str(path) -def _write_sexcat(path): +def _write_sexcat(path, n=None): """Write a synthetic LDAC SExtractor catalogue of known positions. Written with astropy rather than ``FITSCatalogue.save_as_fits``, because @@ -81,6 +90,7 @@ def _write_sexcat(path): to copy the ``LDAC_IMHEAD`` HDU from. The three-HDU layout below is what SExtractor writes and what ``SEx_catalogue=True`` (``hdu_no=2``) indexes. """ + n = len(RA) if n is None else n imhead = fits.BinTableHDU.from_columns( [ fits.Column( @@ -91,13 +101,11 @@ def _write_sexcat(path): ) objects = fits.BinTableHDU.from_columns( [ - fits.Column(name="NUMBER", format="J", array=np.arange(len(RA))), - fits.Column(name="XWIN_WORLD", format="D", array=RA), - fits.Column(name="YWIN_WORLD", format="D", array=DEC), + fits.Column(name="NUMBER", format="J", array=np.arange(n)), + fits.Column(name="XWIN_WORLD", format="D", array=RA[:n]), + fits.Column(name="YWIN_WORLD", format="D", array=DEC[:n]), fits.Column( - name="IMAFLAGS_ISO", - format="J", - array=np.zeros(len(RA), dtype="i4"), + name="IMAFLAGS_ISO", format="J", array=np.zeros(n, dtype="i4") ), ], name="LDAC_OBJECTS", @@ -204,3 +212,90 @@ def test_mask_query_bits_and_all_clean(tmp_path): assert n_flagged == 0 flag, _ = _read(out_path) npt.assert_array_equal(flag, [0, 0, 0, 0]) + + +def test_partial_read_matches_full(tmp_path): + """The partial read returns exactly what a full read of the map returns. + + This is the guarantee that makes the optimisation safe: query_map loads + only the coverage pixels the positions touch, and must be indistinguishable + from HealSparseMap.read(path) at every queried position. + """ + for dtype, writer in ((np.int16, _write_map), (np.bool_, None)): + path = ( + _write_map(tmp_path / f"full_{dtype.__name__}.hsp", 7, n_covered=3) + if writer is not None + else _write_bool_map(tmp_path / "full_bool.hsp", n_covered=3) + ) + full = healsparse.HealSparseMap.read(path) + expected = np.asarray(full.get_values_pos(RA, DEC, lonlat=True)) + npt.assert_array_equal( + mask_query_util.query_map(path, RA, DEC), expected + ) + + +def test_coverage_reported_for_both_map_kinds(tmp_path): + """in_coverage is the coverage mask, not valid_mask. + + For a boolean map healsparse stores only the True pixels, so valid_mask + would equal the value and could not distinguish an unmasked object from one + the map does not reach. The distant object must read as off-coverage while + the near, unflagged ones read as covered. + """ + for path in ( + _write_map(tmp_path / "int.hsp", 4, n_covered=2), + _write_bool_map(tmp_path / "bool.hsp", n_covered=2), + ): + _, in_coverage = mask_query_util.query_map_coverage(path, RA, DEC) + # The 4th position is 190 deg away; the first two are on the map. + assert in_coverage[0] and in_coverage[1] + assert not in_coverage[3] + + +def test_all_off_coverage_warns(tmp_path): + """A map that reaches nothing warns, instead of logging a silent zero.""" + path = _write_map(tmp_path / "elsewhere.hsp", 4, n_covered=2) + far_ra = np.array([200.0, 201.0]) + far_dec = np.array([-40.0, -41.0]) + log = _NullLogger() + + flag = mask_query_util.flag_positions([path], far_ra, far_dec, w_log=log) + + npt.assert_array_equal(flag, [0, 0]) + assert any("NO object is inside" in m for m in log.warning_msgs) + assert any("2 outside coverage" in m for m in log.info_msgs) + # A map that DOES reach the objects must not warn. + log2 = _NullLogger() + mask_query_util.flag_positions([path], RA, DEC, w_log=log2) + assert not log2.warning_msgs + + +def test_flag_positions_empty_input(tmp_path): + """Zero positions is not an error and reads no map.""" + flag = mask_query_util.flag_positions( + [str(tmp_path / "does-not-exist.hsp")], np.zeros(0), np.zeros(0) + ) + assert flag.shape == (0,) + + +def test_mask_query_empty_ccd(tmp_path): + """A CCD with no detections still publishes a catalogue, not an error. + + setools tolerates sparse-CCD attrition and psfex_interp's completeness + floor is 0/warn, so an empty sexcat must flow through rather than raise. + """ + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-2.fits", n=0) + out_path = tmp_path / "sexcat_ext-000-2.fits" + log = _NullLogger() + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=log + ).process() + + assert n_flagged == 0 + assert out_path.exists() + flag, names = _read(out_path) + assert flag.shape == (0,) + assert "FLAG_EXT" in names + assert any("No detections" in m for m in log.info_msgs) diff --git a/uv.lock b/uv.lock index 27fe9cd5b..6a119eea6 100644 --- a/uv.lock +++ b/uv.lock @@ -3403,6 +3403,7 @@ dependencies = [ { name = "galsim" }, { name = "h5py" }, { name = "healsparse" }, + { name = "hpgeom" }, { name = "joblib" }, { name = "matplotlib" }, { name = "mccd" }, @@ -3481,6 +3482,7 @@ requires-dist = [ { name = "galsim", specifier = ">=2.8" }, { name = "h5py" }, { name = "healsparse" }, + { name = "hpgeom" }, { name = "hypothesis", marker = "extra == 'test'", specifier = ">=6.155.2" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=9.14.1" }, { name = "joblib", specifier = ">=1.4" }, From 2cc82eef82485ae27e5ce78c4abaf56d98359cf8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:03:28 -0400 Subject: [PATCH 082/100] =?UTF-8?q?revert(scripts):=20restore=20collate=5F?= =?UTF-8?q?star=5Fcat=20=E2=80=94=20it=20is=20PSF=20validation,=20not=20ma?= =?UTF-8?q?sking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scope error in eb93838a, mine to own: collate_star_cat.py was swept up with the GSC masking star catalogues on a name match. It is the PSF VALIDATION collation — per-exposure validation_psf files gathered into the validation_psf_conv catalogues config_Ms_psfex_conv.ini merges — and has nothing to do with mask generation. Restored byte-identical from origin/feat/snakemake-orchestration, with tests/module/test_collate_star_cat.py, and pipeline_canfar.md's "Collate star catalogues" section put back verbatim in place of the deletion warning ba2271a5 wrote (the HSM sky-coordinates paragraph returns with it, as the section's own prose, which is where it started). Two things asked for in the revert turn out not to be needed, checked rather than assumed: * focal_plane.py and utilities/file_io.py stay deleted. collate_star_cat.py imports nothing from shapepipe at all — only stdlib plus tqdm, joblib, numpy, astropy, galsim and cs_util. focal_plane_disc and write_atomic had exactly two callers between them, create_star_cat.py and star_cats.py, and both are masking and both stay deleted. * There is no scripts/README.rst entry to restore. That file listed create_star_cat (the masking one) and never listed collate_star_cat, so the edit in 1bceb2da was already correct. create_star_cat.py, star_cats.py and utilities/vizier.py remain deleted, as intended. Verified in the container: the 9 restored tests pass alongside the 15 mask ones (24 total), 30/30 runners import, 65 configs resolve, the workflow builds its DAG, and both restored files diff clean against origin. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 74 ++- scripts/python/collate_star_cat.py | 895 ++++++++++++++++++++++++++ tests/module/test_collate_star_cat.py | 70 ++ 3 files changed, 1020 insertions(+), 19 deletions(-) create mode 100755 scripts/python/collate_star_cat.py create mode 100644 tests/module/test_collate_star_cat.py diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 350c4a403..926d36578 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -309,25 +309,61 @@ shapepipe_run -c $SP_CONFIG/config_Pl_$psf.ini #### Collate star catalogues -```{warning} -**This step no longer ships.** `collate_star_cat.py` gathered each validation -PSF run's star positions (X/Y/RA/DEC) and MCCD CCD id into the -`validation_psf_conv-*.fits` files the merge below consumes, and it was deleted -with the star-catalogue tooling the removed mask module needed (PR #847). The -merge step that follows therefore has no producer for its input in this repo -until the collation is reimplemented, and the `star_cat/` + `combine_runs.bash --c psf_conv` staging around it does not apply either. -``` - -HSM shapes are **not** rotated into world coordinates at this step, and were not -before it was removed. The PSF and star ellipticities and sizes are measured -directly in sky coordinates during PSF interpolation (galsim -`FindAdaptiveMom(use_sky_coords=True)`) — see `tests/module/test_hsm_sky_coords.py`, -which pins that convention. The collation was a pass-through for shapes, which is -both why it carried no science and why it is straightforward to rebuild. - -Merge all converted star catalogues and create `final-starcat.fits` (this -reads the `validation_psf_conv` files the collation above used to produce): +Collate all input validation PSF files into star catalogues, gathering positions +(X/Y/RA/DEC) and the MCCD CCD id. + +Note: HSM shapes are no longer rotated into world coordinates at this step. The +PSF/star ellipticities and sizes are now measured directly in sky coordinates +during PSF interpolation (galsim `FindAdaptiveMom(use_sky_coords=True)`), so +`collate_star_cat.py` only collates and passes the shapes through. + +> **v2.0 is patch-less.** Runs up to `v1.6` are organised in sky patches +> `P1`..`P`, each patch a run directory. `v2.0` removes the patch concept: a +> single run root, outputs directly under it. `v2.0` is the default; select an +> older layout with `-V` (e.g. `-V v1.6`). For `v2.0` the patch loop and the +> `-P` option no longer apply, and the patch token drops from the output +> filename (`validation_psf_conv-.fits` instead of +> `validation_psf_conv--.fits`). + +```bash +cd /path/to/version +mkdir star_cat +cd star_cat +``` + +For `v2.0` (the default), run once against the patch-less run root, producing +files `validation_psf_conv-.fits`: + +```bash +collate_star_cat.py -i .. -v +``` + +For `v1.x`, pass the version explicitly and run once per patch, creating a +directory per patch `P?` and producing files +`validation_psf_conv--.fits` (for the v1.4 setup only one file): + +```bash +collate_star_cat.py -i .. -V v1.6 -P $patchnum -v +``` + +Combine previously created files as links within one ShapePipe run directory (for the v1.4 setup only one link). +First (and optiohnal), create a subdir for a run and link to the input patches: + +```bash +cd /path/to/version/star_cat +mkdir v1.6 +ln -s ../P1 +ln -s ../P2 +... +``` + +Next, create links to all `validation_conv` runs: + +```bash +combine_runs.bash -p psfex -c psf_conv +``` + +Merge all converted star catalogues and create `final-starcat.fits`: ```bash export SP_RUN=`pwd` diff --git a/scripts/python/collate_star_cat.py b/scripts/python/collate_star_cat.py new file mode 100755 index 000000000..f7ab25b41 --- /dev/null +++ b/scripts/python/collate_star_cat.py @@ -0,0 +1,895 @@ +#! /usr/bin/env python3 + +"""COLLATE STAR CATALOGUES. + +Collate the per-exposure PSF validation catalogues into star catalogues: gather +positions (X/Y/RA/DEC), assign the MCCD focal-plane CCD id, and merge into +``validation_psf_conv`` FITS files. + +Catalogue layout depends on the major version (``-V``). Runs up to ``v1.6`` are +organised in sky patches (``P1``..``P``): input runs live under +``/P/output`` and outputs are named +``validation_psf_conv--.fits``. ``v2.0`` removes the patch concept: +input runs live under a single ``/output`` root and outputs drop the patch +token (``validation_psf_conv-.fits``). + +HSM ellipticities and sizes are no longer rotated here. PSFEx and the in-repo +MCCD interpolation now measure adaptive moments directly in world coordinates +(galsim ``FindAdaptiveMom(use_sky_coords=True)``), so the WCS-Jacobian shape +rotation this script used to perform is redundant and has been removed. + +Caveat: the MCCD ``PSF_MOM_LIST``/``STAR_MOM_LIST`` columns are produced by the +external ``mccd`` fit-validation code (``mccd.auxiliary_fun.mccd_validation``), +which still measures HSM moments in the pixel frame. Those shapes are therefore +still rotated into world coordinates here, via the WCS-Jacobian rotation, until +``mccd`` itself adopts ``use_sky_coords``. This is the one branch that keeps the +rotation; the in-repo PSFEx and MCCD-interpolation paths measure adaptive +moments directly in world coordinates upstream and pass them straight through. +""" + +import sys +import os +import re +import glob +from tqdm import tqdm +from joblib import Parallel, delayed +import gc + +import numpy as np +from astropy.io import fits +import galsim + +from cs_util import args as cs_args +from cs_util import logging + + +def collate_paths(input_base_dir, output_base_dir, patch): + """Collate Paths. + + Return the ``(input run dir, output dir)`` for a patch. ``patch`` is None + for the patch-less v2.0 layout, which drops the ``P`` token; v1.x + passes the patch number. + + Parameters + ---------- + input_base_dir : str + input base directory + output_base_dir : str + output base directory + patch : str or None + patch number, or None for the patch-less v2.0 layout + + Returns + ------- + tuple + input run directory and output directory + + """ + if patch is None: + return f"{input_base_dir}/output/", output_base_dir + return f"{input_base_dir}/P{patch}/output/", f"{output_base_dir}/P{patch}" + + +def output_filename(file_pattern, patch, idx): + """Output Filename. + + Build the collated catalogue filename. ``patch`` is None for the patch-less + v2.0 layout, which drops the patch token. + + Parameters + ---------- + file_pattern : str + input file pattern (e.g. ``validation_psf``) + patch : str or None + patch number, or None for the patch-less v2.0 layout + idx : int + exposure run index + + Returns + ------- + str + output catalogue file name + + """ + patch_token = "" if patch is None else f"{patch}-" + return f"{file_pattern}_conv-{patch_token}{idx}.fits" + + +def transform_shape(mom_list, jac): + """Transform Shape. + + Transform shape (ellipticity and size) using a Jacobian. + + Parameters + ---------- + mom_list : list + input moment measurements; each list element contains + first and second ellipticity component and size + jac : galsim.JacobianWCS + Jacobian transformation matrix information + + Returns + ------- + list + transformed shape parameters, which are + first and second ellipticity component and size + + """ + scale, shear, theta, flip = jac.getDecomposition() + + sig_tmp = mom_list[2] * scale + shape = galsim.Shear(g1=mom_list[0], g2=mom_list[1]) + if flip: + # The following output is not observed + print("FLIP!") + shape = galsim.Shear(g1=-shape.g1, g2=shape.g2) + shape = galsim.Shear(g=shape.g, beta=shape.beta + theta) + shape = shear + shape + + return shape.g1, shape.g2, sig_tmp + + +class Loc2Glob(object): + r"""Change from local to global coordinates. + + Class to pass from local coordinates to global coordinates under + CFIS (CFHT) MegaCam instrument. The geometrical informcation of the + instrument is encoded in this function. + + Parameters + ---------- + x_gap : int + Gap between the CCDs along the horizontal direction; + default is ``70`` (MegaCam value) + y_gap : int + Gap between the CCDs along the vertical direction; + Default is ``425`` (MegaCam value) + x_npix : int + Number of pixels per CCD along the horizontal direction; + default is ``2048`` (MegaCam value) + y_npix : int + Number of pixels per CCD along the vertical direction; + default to ``4612`` (MegaCam value) + ccd_tot : int + Total number of CCDs; + default to ``40`` (MegaCam value) + + Notes + ----- + This is the geometry of MegaCam. Watch out with the conventions ba,ab that means where + is the local coordinate system origin for each CCD. + For more info check out MegaCam's instrument webpage. + + Examples + -------- + 'COMMENT (North on top, East to the left)', + 'COMMENT --------------------------', + 'COMMENT ba ba ba ba ba ba ba ba ba', + 'COMMENT 00 01 02 03 04 05 06 07 08', + 'COMMENT --------------------------------', + 'COMMENT ba ba ba ba ba ba ba ba ba ba ba', + 'COMMENT 36 09 10 11 12 13 14 15 16 17 37', + 'COMMENT --------------*-----------------', + 'COMMENT 38 18 19 20 21 22 23 24 25 26 39', + 'COMMENT ab ab ab ab ab ab ab ab ab ab ab', + 'COMMENT --------------------------------', + 'COMMENT 27 28 29 30 31 32 33 34 35', + 'COMMENT ab ab ab ab ab ab ab ab ab', + 'COMMENT __________________________' + """ + + def __init__( + self, x_gap=70, y_gap=425, x_npix=2048, y_npix=4612, ccd_tot=40 + ): + r"""Initialize with instrument geometry.""" + self.x_gap = x_gap + self.y_gap = y_gap + self.x_npix = x_npix + self.y_npix = y_npix + self.ccd_tot = ccd_tot + + def loc2glob_img_coord(self, ccd_n, x_coor, y_coor): + """loc2glob Img Coord. + + Go from the local to the global img (pixel) coordinate system. + + Global system with (0,0) in the intersection of ccds [12,13,21,22]. + + Parameters + ---------- + ccd_n: int + CCD number of the considered positions + x_coor: float + Local coordinate system hotizontal value + y_coor: float + Local coordinate system vertical value + + Returns + ------- + glob_x_coor: float + Horizontal position in global coordinate system + glob_y_coor: float + Vertical position in global coordinate system + + """ + # Flip axes + x_coor, y_coor = self.flip_coord(ccd_n, x_coor, y_coor) + + # Calculate the shift + x_shift, y_shift = self.shift_coord(ccd_n) + + # Return new coordinates + return x_coor + x_shift, y_coor + y_shift + + def flip_coord(self, ccd_n, x_coor, y_coor): + r"""Change of coordinate convention. + + So that all of them are coherent on the global coordinate system. + So that the origin is on the south-west corner. + Positive: South to North ; West to East. + """ + if ccd_n < 18 or ccd_n in [36, 37]: + x_coor = self.x_npix - x_coor + 1 + y_coor = self.y_npix - y_coor + 1 + else: + pass + + return x_coor, y_coor + + def x_coord_range(self): + r"""Return range of the x coordinate.""" + max_x = self.x_npix * 6 + self.x_gap * 5 + min_x = self.x_npix * (-5) + self.x_gap * (-5) + return min_x, max_x + + def y_coord_range(self): + r"""Return range of the y coordinate.""" + max_y = self.y_npix * 2 + self.y_gap * 1 + min_y = self.y_npix * (-2) + self.y_gap * (-2) + return min_y, max_y + + def shift_coord(self, ccd_n): + r"""Provide the shifting. + + It is needed to go from the local coordinate + system origin to the global coordinate system origin. + """ + if ccd_n < 9: + # first row + x_shift = (ccd_n - 4) * (self.x_gap + self.x_npix) + y_shift = self.y_gap + self.y_npix + return x_shift, y_shift + + elif ccd_n < 18: + # second row, non-ears + x_shift = (ccd_n - 13) * (self.x_gap + self.x_npix) + y_shift = 0.0 + return x_shift, y_shift + + elif ccd_n < 27: + # third row non-ears + x_shift = (ccd_n - 22) * (self.x_gap + self.x_npix) + y_shift = -1.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + elif ccd_n < 36: + # fourth row + x_shift = (ccd_n - 31) * (self.x_gap + self.x_npix) + y_shift = -2.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + elif ccd_n < 37: + # ccd= 36 ears, second row + x_shift = (-5.0) * (self.x_gap + self.x_npix) + y_shift = 0.0 + return x_shift, y_shift + + elif ccd_n < 38: + # ccd= 37 ears, second row + x_shift = 5.0 * (self.x_gap + self.x_npix) + y_shift = 0.0 + return x_shift, y_shift + + elif ccd_n < 39: + # ccd= 38 ears, third row + x_shift = (-5.0) * (self.x_gap + self.x_npix) + y_shift = -1.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + elif ccd_n < 40: + # ccd= 39 ears, third row + x_shift = 5.0 * (self.x_gap + self.x_npix) + y_shift = -1.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + +class Glob2CCD(object): + r"""Get the CCD ID number from the global coordinate position. + + The Loc2Glob() object as input is the one that defines the instrument's + geometry. + + Parameters + ---------- + loc2glob: Loc2Glob object + Object with the desired focal plane geometry. + with_gaps: bool + If add the gaps to the CCD area. + """ + + def __init__(self, loc2glob, with_gaps=True): + # Save loc2glob object + self.loc2glob = loc2glob + self.with_gaps = with_gaps + self.ccd_list = np.arange(self.loc2glob.ccd_tot) + # Init edges defininf the CCDs + self.edge_x_list, self.edge_y_list = self.build_all_edges() + + def build_all_edges(self): + """Build the edges for all the CCDs in the focal plane.""" + edge_xy_list = [] + for idx in (0, 1): + edge_list = np.array( + [self.build_edge(ccd_n)[idx] for ccd_n in self.ccd_list] + ) + edge_xy_list.append(edge_list) + + return edge_xy_list + + def build_edge(self, ccd_n): + """Build the edges of the `ccd_n` in global coordinates.""" + if self.with_gaps: + corners = np.array( + [ + [-self.loc2glob.x_gap / 2, -self.loc2glob.y_gap / 2], + [ + self.loc2glob.x_npix + self.loc2glob.x_gap / 2, + -self.loc2glob.y_gap / 2, + ], + [ + -self.loc2glob.x_gap / 2, + self.loc2glob.y_npix + self.loc2glob.y_gap / 2, + ], + [ + self.loc2glob.x_npix + self.loc2glob.x_gap / 2, + self.loc2glob.y_npix + self.loc2glob.y_gap / 2, + ], + ] + ) + else: + corners = np.array( + [ + [0, 0], + [self.loc2glob.x_npix, 0], + [0, self.loc2glob.y_npix], + [self.loc2glob.x_npix, self.loc2glob.y_npix], + ] + ) + + glob_corners = np.array( + [ + self.loc2glob.loc2glob_img_coord(ccd_n, pos[0], pos[1]) + for pos in corners + ] + ) + + edge_xy = [] + for idx in (0, 1): + edge = np.array( + [np.min(glob_corners[:, idx]), np.max(glob_corners[:, idx])] + ) + edge_xy.append(edge) + + return edge_xy + + def is_inside(self, x, y, edge_x, edge_y): + """Is the position inside the edges. + + Return True if the position is within the rectangle + defined by the edges. + + Parameters + ---------- + x: float + Horizontal position in global coordinate system. + y: float + Vertical position in global coordinate system. + edge_x: np.ndarray + Edge defined as `np.array([min_x, max_x])`. + edge_y: np.ndarray + Edge defined as `np.array([min_y, max_y])`. + """ + if ( + (x > edge_x[0]) + and (x < edge_x[1]) + and (y > edge_y[0]) + and (y < edge_y[1]) + ): + return True + else: + return False + + def get_ccd_n(self, x, y): + """Returns the CCD number from the position `(x, y)`. + + Returns `None` if the position is not found. + """ + bool_list = np.array( + [ + self.is_inside(x, y, edge_x, edge_y) + for edge_x, edge_y in zip(self.edge_x_list, self.edge_y_list) + ] + ) + + try: + return self.ccd_list[bool_list][0] + except Exception: + return None + + +class Convert(object): + + def __init__(self): + + self.params_default() + + def set_params_from_command_line(self, args): + """Set Params From Command line. + + Only use when calling using python from command line. + Does not work from ipython or jupyter. + + """ + # Read command line options + options = cs_args.parse_options( + self._params, + self._short_options, + self._types, + self._help_strings, + ) + self._params = options + + # Save calling command + logging.log_command(args) + + def params_default(self): + + self._params = { + "input_base_dir": ".", + "output_base_dir": ".", + "version_cat": "v2.0", + "mode": "merge", + "patches": "", + "psf": "psfex", + "file_pattern_psfint": "validation_psf", + } + + self._short_options = { + "input_base_dir": "-i", + "version_cat": "-V", + "mode": "-m", + "psf": "-p", + "patches": "-P", + } + + self._types = {} + + self._help_strings = { + "input_base_dir": ( + "input base dir; for v1.x runs are expected in" + + " /P/output, for v2.0 (patch-less) in" + + " /output; default is {}" + ), + "version_cat": ( + "catalogue major version, allowed are v1.3, v1.4, v1.5, v1.6," + + " v2.0; v2.0 is patch-less; default is {}" + ), + "mode": ( + "run mode, allowed are 'merge', 'test'; default is" + " '{}'" + ), + "psf": "PSF model, allowed are 'psfex' and 'mccd'; default is {}", + "patches": "(list of) input patches; ignored for v2.0", + } + + # Output column names with types + self._dt = [ + ("X", float), + ("Y", float), + ("RA", float), + ("DEC", float), + ("E1_PSF_HSM", float), + ("E2_PSF_HSM", float), + ("SIGMA_PSF_HSM", float), + ("FLAG_PSF_HSM", float), + ("E1_STAR_HSM", float), + ("E2_STAR_HSM", float), + ("SIGMA_STAR_HSM", float), + ("FLAG_STAR_HSM", float), + ("CCD_NB", int), + ] + + # Extra columns for MCCD:737 + self._dt_mccd = self._dt.copy() + self._dt_mccd.append(("GLOB_X", float)) + self._dt_mccd.append(("GLOB_Y", float)) + + def update_params(self): + """Update Params. + + Update parameters. + + """ + if self._params["psf"] == "psfex": + #self._params["sub_dir_pattern"] = "run_sp_exp_202" + self._params["sub_dir_pattern"] = "run_sp_combined_psf" + self._params["sub_dir_psfint"] = "psfex_interp_runner" + elif self._params["psf"] == "mccd": + self._params["sub_dir_pattern"] = "run_sp_exp_SxSePsf_202" + self._params["sub_dir_psfint"] = "mccd_fit_val_runner" + self._params["sub_dir_setools"] = "setools_runner/output/mask" + else: + raise ValueError(f"Invalid PSF model {self._params['psf']}") + self._params["sub_dir_psfint"] = ( + f"{self._params['sub_dir_psfint']}/output" + ) + + def run(self): + """Run. + + Main processing function. + + """ + # Guard against a mistyped version silently falling through to the + # v1.x patch loop (e.g. ``-V v2`` or ``-V 2.0``). + allowed_versions = ("v1.3", "v1.4", "v1.5", "v1.6", "v2.0") + if self._params["version_cat"] not in allowed_versions: + raise ValueError( + f"Invalid version {self._params['version_cat']}; allowed are" + + f" {', '.join(allowed_versions)}" + ) + + # v2.0 removes the patch concept: a single patch-less run root. For + # v1.x, iterate over the requested sky patches as before. ``patch`` is + # None in the patch-less case, which drops the patch token from the + # input path and the output filename. + if self._params["version_cat"] == "v2.0": + patch_nums = [None] + elif self._params["mode"] == "test": + patch_nums = ["3", "4"] + else: + patch_nums = cs_args.my_string_split(self._params["patches"]) + + do_parallel = True + + # Loop over patches + for patch in patch_nums: + + patch_dir, output_dir = collate_paths( + self._params["input_base_dir"], + self._params["output_base_dir"], + patch, + ) + print("Running patch-less (v2.0)" if patch is None else f"Running patch: {patch}") + + if not os.path.isdir(output_dir): + os.makedirs(output_dir, exist_ok=True) + + subdirs = f"{patch_dir}/{self._params['sub_dir_pattern']}*" + exp_run_dirs = glob.glob(subdirs) + n_exp_runs = len(exp_run_dirs) + print( + f"Found {n_exp_runs} input single-exposure run(s) for patch" + + f" {patch_dir} ({subdirs})" + ) + + if self._params["mode"] == "test": + exp_run_dirs = exp_run_dirs[:2] + n_exp_runs = len(exp_run_dirs) + print( + f"test mode: only using {n_exp_runs} input single-exposure" + + f" runs" + ) + + # Loop over exposure runs + if not do_parallel: + for idx_exp, exp_run_dir in tqdm( + enumerate(exp_run_dirs), + total=n_exp_runs, + disable=self._params["verbose"], + ): + self.transform_exposures( + output_dir, patch, idx_exp, exp_run_dir + ) + else: + res = Parallel(n_jobs=-1, backend="loky")( + delayed(self.transform_exposures)( + output_dir, patch, idx_exp, exp_run_dir + ) + for idx_exp, exp_run_dir in tqdm( + enumerate(exp_run_dirs), + total=n_exp_runs, + disable=self._params["verbose"], + ) + ) + + def transform_exposures(self, output_dir, patch, idx, exp_run_dir): + """Transform exposures. + + Transform shapes for exposure for a given run (input exp run dir). + + """ + output_path = ( + f"{output_dir}/" + + output_filename( + self._params["file_pattern_psfint"], patch, idx + ) + ) + if os.path.exists(output_path): + print(f"Skipping transform_exposures, file {output_path} exists") + return + + psf_dir = f"{exp_run_dir}/{self._params['sub_dir_psfint']}" + try: + all_files = os.listdir(psf_dir) + if self._params["verbose"]: + print(f"Found {len(all_files)} file(s) in {psf_dir}") + except Exception: + if self._params["verbose"]: + print(f"Found zero PSFEx files in {psf_dir}, skipping") + return + + cat_list = [] + for file_name in all_files: + if self._params["file_pattern_psfint"] not in file_name: + continue + + tmp = re.findall(r"\d+", file_name) + + if self._params["psf"] == "psfex": + exp_name, ccd_id = int(tmp[0]), int(tmp[1]) + elif self._params["psf"] == "mccd": + exp_name = int(tmp[0]) + ccd_id = -1 + + if self._params["verbose"]: + print("Match found ", exp_name, ccd_id) + + psf_file_path = f"{psf_dir}/{file_name}" + + try: + if self._params["psf"] == "psfex": + psf_file_hdus = fits.open(psf_file_path, memmap=False) + psf_file = psf_file_hdus[2].data + psf_file_hdus.close() + mod = "RA" + else: + psf_file = fits.getdata(psf_file_path, 1, memmap=True) + mod = "RA_LIST" + except Exception: + continue + + if self._params["psf"] == "psfex": + # HSM ellipticities and sizes are measured directly in world + # coordinates upstream (FindAdaptiveMom use_sky_coords=True), so + # they are passed straight through; only positions are collated. + exp_cat = np.array( + list( + map( + tuple, + np.array( + [ + psf_file["X"], + psf_file["Y"], + psf_file["RA"], + psf_file["DEC"], + psf_file["E1_PSF_HSM"], + psf_file["E2_PSF_HSM"], + psf_file["SIGMA_PSF_HSM"], + psf_file["FLAG_PSF_HSM"], + psf_file["E1_STAR_HSM"], + psf_file["E2_STAR_HSM"], + psf_file["SIGMA_STAR_HSM"], + psf_file["FLAG_STAR_HSM"], + np.ones_like(psf_file["RA"], dtype=int) + * ccd_id, + ] + ).T.tolist(), + ) + ), + dtype=self._dt, + ) + cat_list.append(exp_cat) + + else: + l2g = Loc2Glob() + g2c = Glob2CCD(l2g) + new_ccd_id = np.array( + [ + int( + g2c.get_ccd_n( + psf_file["GLOB_POSITION_IMG_LIST"][ii, 0], + psf_file["GLOB_POSITION_IMG_LIST"][ii, 1], + ) + ) + for ii in range(len(psf_file)) + ] + ) + + # Local-to-CCD position: subtract each CCD's focal-plane shift. + new_x = np.zeros_like(psf_file[mod]) + new_y = np.zeros_like(psf_file[mod]) + + # The MCCD PSF_MOM_LIST/STAR_MOM_LIST columns come from the + # external mccd fit-validation code, which still measures HSM + # moments in the pixel frame; rotate them into world coordinates + # via the per-CCD WCS Jacobian. This rotation stays until mccd + # itself adopts use_sky_coords (see the module docstring). The + # in-repo PSFEx / MCCD-interpolation paths are already in world + # coordinates and are passed through unrotated. + new_e1_psf = np.zeros_like(psf_file[mod]) + new_e2_psf = np.zeros_like(psf_file[mod]) + new_sig_psf = np.zeros_like(psf_file[mod]) + new_e1_star = np.zeros_like(psf_file[mod]) + new_e2_star = np.zeros_like(psf_file[mod]) + new_sig_star = np.zeros_like(psf_file[mod]) + new_flag_psf = np.zeros_like(psf_file[mod]) + new_flag_star = np.zeros_like(psf_file[mod]) + for ccd_id in range(40): + m_ccd_id = new_ccd_id == ccd_id + if sum(m_ccd_id) == 0: + continue + + x_shift, y_shift = l2g.shift_coord(ccd_id) + + new_x[m_ccd_id] = ( + psf_file["GLOB_POSITION_IMG_LIST"][:, 0][m_ccd_id] + - x_shift + ) + new_y[m_ccd_id] = ( + psf_file["GLOB_POSITION_IMG_LIST"][:, 1][m_ccd_id] + - y_shift + ) + + header_file_path = ( + self._params["sub_dir_setools"] + + self._params["file_pattern_psfint"] + + f"{exp_name}-{ccd_id}.fits" + ) + try: + header_file = fits.getdata(header_file_path, 1) + except Exception: + continue + header = fits.Header.fromstring( + "\n".join(header_file[0][0]), sep="\n" + ) + wcs = galsim.AstropyWCS(header=header) + + g1_psf_tmp_l = [] + g2_psf_tmp_l = [] + sig_psf_tmp_l = [] + g1_star_tmp_l = [] + g2_star_tmp_l = [] + sig_star_tmp_l = [] + flag_psf_tmp_l = [] + flag_star_tmp_l = [] + + for obj in psf_file[m_ccd_id]: + try: + jac = wcs.jacobian( + world_pos=galsim.CelestialCoord( + ra=obj["RA_LIST"] * galsim.degrees, + dec=obj["DEC_LIST"] * galsim.degrees, + ) + ) + except Exception: + flag_star_tmp_l.append(16) + flag_psf_tmp_l.append(16) + g1_psf_tmp_l.append(0) + g2_psf_tmp_l.append(0) + sig_psf_tmp_l.append(0) + g1_star_tmp_l.append(0) + g2_star_tmp_l.append(0) + sig_star_tmp_l.append(0) + continue + g1_psf_tmp, g2_psf_tmp, sig_psf_tmp = transform_shape( + obj["PSF_MOM_LIST"], jac + ) + + g1_psf_tmp_l.append(g1_psf_tmp) + g2_psf_tmp_l.append(g2_psf_tmp) + sig_psf_tmp_l.append(sig_psf_tmp) + flag_psf_tmp_l.append(obj["PSF_MOM_LIST"][3]) + + g1_star_tmp, g2_star_tmp, sig_star_tmp = ( + transform_shape(obj["STAR_MOM_LIST"], jac) + ) + g1_star_tmp_l.append(g1_star_tmp) + g2_star_tmp_l.append(g2_star_tmp) + sig_star_tmp_l.append(sig_star_tmp) + flag_star_tmp_l.append(obj["STAR_MOM_LIST"][3]) + + new_e1_psf[m_ccd_id] = g1_psf_tmp_l + new_e2_psf[m_ccd_id] = g2_psf_tmp_l + new_sig_psf[m_ccd_id] = sig_psf_tmp_l + new_flag_psf[m_ccd_id] = flag_psf_tmp_l + new_e1_star[m_ccd_id] = g1_star_tmp_l + new_e2_star[m_ccd_id] = g2_star_tmp_l + new_sig_star[m_ccd_id] = sig_star_tmp_l + new_flag_star[m_ccd_id] = flag_star_tmp_l + + exp_cat = np.array( + list( + map( + tuple, + np.array( + [ + new_x, + new_y, + psf_file["RA_LIST"], + psf_file["DEC_LIST"], + new_e1_psf, + new_e2_psf, + new_sig_psf, + psf_file["PSF_MOM_LIST"][:, 3], + new_e1_star, + new_e2_star, + new_sig_star, + psf_file["STAR_MOM_LIST"][:, 3], + new_ccd_id, + psf_file["GLOB_POSITION_IMG_LIST"][:, 0], + psf_file["GLOB_POSITION_IMG_LIST"][:, 1], + ] + ).T.tolist(), + ) + ), + dtype=self._dt_mccd, + ) + cat_list.append(exp_cat) + + del psf_file + + if len(cat_list) == 0: + return + + # Finalize catalogue + patch_cat = np.concatenate(cat_list) + hdul = fits.HDUList() + hdul.append(fits.PrimaryHDU()) + hdul.append(fits.BinTableHDU(patch_cat)) + + # Write catalogue + hdul.writeto( + output_path, + overwrite=True, + ) + + del cat_list + del hdul + gc.collect() + + +def run_convert(*args): + + # Create instance + obj = Convert() + + obj.set_params_from_command_line(args) + obj.update_params() + + obj.run() + + +def main(argv=None): + """Main + + Main program + """ + if argv is None: + argv = sys.argv[1:] + run_convert(*argv) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/module/test_collate_star_cat.py b/tests/module/test_collate_star_cat.py new file mode 100644 index 000000000..20d52ce23 --- /dev/null +++ b/tests/module/test_collate_star_cat.py @@ -0,0 +1,70 @@ +"""UNIT TESTS FOR STAR-CATALOGUE COLLATION PATHS. + +Pin the patch vs patch-less (v2.0) path and filename convention of +``scripts/python/collate_star_cat.py``. Runs up to v1.6 carry a ``P`` +token in both the input run directory and the output filename; v2.0 is +patch-less (``patch is None``) and drops that token, reading from a single +``/output`` root and writing ``validation_psf_conv-.fits`` — the +name still matched by the downstream ``validation_psf_conv-*`` glob. +""" + +import importlib.util +from pathlib import Path + +import pytest + +# The collation script lives under scripts/python (not an importable package), +# so load it by path. +_SCRIPT = ( + Path(__file__).resolve().parents[2] + / "scripts" + / "python" + / "collate_star_cat.py" +) +_spec = importlib.util.spec_from_file_location("collate_star_cat", _SCRIPT) +collate_star_cat = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(collate_star_cat) + + +@pytest.mark.parametrize( + "patch, exp_input, exp_output", + [ + ("3", "in/P3/output/", "out/P3"), + (None, "in/output/", "out"), + ], +) +def test_collate_paths(patch, exp_input, exp_output): + """v1.x carries the P token; v2.0 (patch None) drops it.""" + assert collate_star_cat.collate_paths("in", "out", patch) == ( + exp_input, + exp_output, + ) + + +@pytest.mark.parametrize( + "patch, expected", + [ + ("3", "validation_psf_conv-3-0.fits"), + (None, "validation_psf_conv-0.fits"), + ], +) +def test_output_filename(patch, expected): + """The patch token is present for v1.x and absent for v2.0.""" + assert collate_star_cat.output_filename("validation_psf", patch, 0) == expected + + +def test_output_filename_matches_downstream_glob(): + """Both layouts stay under the downstream ``validation_psf_conv-*`` glob.""" + for patch in ("1", None): + assert collate_star_cat.output_filename( + "validation_psf", patch, 5 + ).startswith("validation_psf_conv-") + + +@pytest.mark.parametrize("bad", ["v2", "2.0", "v1.7", ""]) +def test_invalid_version_raises(bad): + """A mistyped -V is rejected rather than falling through to v1.x.""" + obj = collate_star_cat.Convert() + obj._params["version_cat"] = bad + with pytest.raises(ValueError): + obj.run() From ed376f734cc9dd46c7c2f93a67b2180f397d110a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:07:24 -0400 Subject: [PATCH 083/100] fix(mask_query): public sentinel, empty-coverage guard, real product name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three from final review. `mask_map.sentinel` replaces `mask_map._sentinel` in the probe path — same value (verified for both int and bool maps on healsparse 1.12.2), public API. The probe also indexed `np.flatnonzero(coverage.coverage_mask)[0]` unguarded, so a map with no coverage at all died with a bare IndexError from inside the utility. It now raises `ValueError("healsparse map has empty coverage")`, naming the file, with a test. The shipped MASK_PATHS placeholder becomes the real DR6 product name, `mask_r_nside131072_n4.hsp`, in all three configs. The products are staged at /project/6001537/cdaley/masks/dr6/ (same tree as /project/def-mjhudson/cdaley/masks/dr6/), which is where the partial-read measurement in f7d1fd66 was taken — that docstring already named the file correctly, so nothing else mentioned the old placeholder. 25 tests pass in the container (mask_query, make_cat_mask_ext, collate_star_cat); 65 configs resolve. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_exp_mccd.ini | 2 +- example/cfis/config_exp_psfex.ini | 2 +- src/shapepipe/utilities/mask_query.py | 10 +++++++--- tests/module/test_mask_query.py | 11 +++++++++++ workflow/config/cfis/config_exp_psfex.ini | 2 +- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 68dcfbeef..5d2460312 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -145,7 +145,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 2f1c8e353..b4e2d1fa1 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -147,7 +147,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index c3b150006..d789d4100 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -169,9 +169,13 @@ def query_map_coverage(path, ra, dec): # the empty case is answered without asking it: load one arbitrary # coverage pixel purely to learn the dtype and sentinel, and return # that sentinel everywhere. Same answer, one small read. - probe = int(np.flatnonzero(coverage.coverage_mask)[0]) - mask_map = healsparse.HealSparseMap.read(path, pixels=[probe]) - values = np.full(ra.size, mask_map._sentinel, dtype=mask_map.dtype) + covered = np.flatnonzero(coverage.coverage_mask) + if covered.size == 0: + raise ValueError(f"healsparse map {path} has empty coverage") + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(covered[0])] + ) + values = np.full(ra.size, mask_map.sentinel, dtype=mask_map.dtype) return values, in_coverage mask_map = healsparse.HealSparseMap.read( diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py index 779a6bd69..85ce028cd 100644 --- a/tests/module/test_mask_query.py +++ b/tests/module/test_mask_query.py @@ -299,3 +299,14 @@ def test_mask_query_empty_ccd(tmp_path): assert flag.shape == (0,) assert "FLAG_EXT" in names assert any("No detections" in m for m in log.info_msgs) + + +def test_empty_coverage_raises_clearly(tmp_path): + """A map with no coverage at all fails with a message naming the file. + + The probe read in query_map_coverage indexes the first covered pixel; with + nothing covered that would be an IndexError from deep inside the utility. + """ + path = _write_map(tmp_path / "empty.hsp", 4, n_covered=0) + with pytest.raises(ValueError): + mask_query_util.query_map_coverage(path, RA, DEC) diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index ffcac86b6..2fa5dc525 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -146,7 +146,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map From c7c9e82f3d69d4a93393518d847f3e48243c2692 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:33:52 -0400 Subject: [PATCH 084/100] =?UTF-8?q?refactor(random=5Fcat):=20delete=20?= =?UTF-8?q?=E2=80=94=20randoms=20come=20from=20healsparse=20map=20algebra?= =?UTF-8?q?=20downstream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cail's call. random_cat computed a tile's effective unmasked area and drew randoms inside it by counting zero pixels in a tile mask IMAGE. That input was the deleted mask module's output, and with the query design there is no such image and no plausible producer for one: the survey window is map algebra on the healsparse coverage map (#797), done downstream, not a per-tile pipeline step that rasterizes to pixels first. Rather than keep a module wired to a placeholder path nobody can fill — which is what f7d1fd66 left, an honest but unpaid IOU — the module goes: the package, the runner, config_Rc.ini, docs/source/random_cat.md and its toc.rst entry. There were no tests to remove. The module was already unsound independently of masking, which is part of why keeping it had no value: `save_as_healpix` is called as `_save_as_healpix`, and `process` references `file_name` and `output_dir` which are never bound. Any run past the first few lines would have raised. `reproject` leaves pyproject.toml with it — random_cat.py held its only import (the two ngmix hits for "reprojection" are prose in comments). uv.lock regenerated: reproject, plus pims, pyavm, slicerator, toolz and zarr that came in behind it. Verified in the container: 29 runners import (was 30), 64 configs resolve (was 65, 0 bad), the 25 mask/collate tests pass, the workflow still builds its DAG, and random_cat / N_RANDOM / RandomCat / config_Rc / run_sp_Rc appear nowhere outside scripts/sh/, which stays untouched as agreed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/random_cat.md | 105 -------- docs/source/toc.rst | 1 - example/cfis/config_Rc.ini | 79 ------ pyproject.toml | 1 - .../modules/random_cat_package/__init__.py | 37 --- .../modules/random_cat_package/random_cat.py | 235 ------------------ src/shapepipe/modules/random_cat_runner.py | 81 ------ uv.lock | 222 ----------------- 8 files changed, 761 deletions(-) delete mode 100644 docs/source/random_cat.md delete mode 100644 example/cfis/config_Rc.ini delete mode 100644 src/shapepipe/modules/random_cat_package/__init__.py delete mode 100644 src/shapepipe/modules/random_cat_package/random_cat.py delete mode 100644 src/shapepipe/modules/random_cat_runner.py diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md deleted file mode 100644 index b5ae9931d..000000000 --- a/docs/source/random_cat.md +++ /dev/null @@ -1,105 +0,0 @@ -# Create random catalogues and masks - -This section describes how to create tile-based random catalogues and healpix -masks, and combined randoms and masks for a selection of tiles. - -The masked regions are obtained on input from per-tile pixel mask images. - -```{warning} -**ShapePipe no longer produces those images.** The pipeline generates no masks -at all: sky-fixed masks are healsparse maps, queried once per object into -catalogue columns (see [Masks](pipeline_tutorial.md#masks)), and tiles have no -flag image. `random_cat_runner` therefore needs its mask images supplied from -outside the pipeline — point its second `INPUT_DIR` entry at a directory of tile -mask images matching its `NUMBERING_SCHEME`. The healsparse-native replacement -for this whole procedure (an n_epoch / n_pointings survey-window map built from -the maps directly) is issue #797. -``` - -```{note} -Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see -[VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The input-staging and joint-mask steps now -overlap with [`sp_validation`](https://github.com/CosmoStat/sp_validation). The -steps are retained for reference. -``` - -## Set up - -### ID file and shell variables - -First, if if does not exist already, create the file ``tile_numbers.txt`` containing a list of tile IDs, -one per line. This is the same format as the input file to ``get_images_runner``. -For example, link to a patch ID list, -```bash -ln -s tiles_PX.txt tile_numbers.txt -``` -Next, set the run and config paths, -```bash -export SP_RUN=. -export SP_CONFIG=/path/to/config-files -``` - -### Get images or image headers - -We need to footprint of the image tiles. If they have been downloaded for a ``ShapePipe`` run, -check that they are accessible as last run of the ``get_images_runner`` module. - -If not, we can just download the headers to gain significant download time. -```bash -shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini -``` - -### Stage the pixel mask files - -Collect the tile mask images into one directory that `random_cat_runner`'s -`INPUT_DIR` points at, one file per tile ID in `tile_numbers.txt`, named to -match the module's `FILE_PATTERN` and `NUMBERING_SCHEME` (`mask--.fits` -with the committed `config_Rc.ini`). How you obtain them is outside ShapePipe; -older runs of the pipeline's own (now removed) mask module wrote them as -`pipeline_flag--.fits`, and those files still work. - -## Create random catalogue and helapix mask per tile - -Run -```bash -shapepipe_run -c $SP_CONFIG/config_Rc.ini -``` -The random catalogue and, with the config entry ``SAVE_MASK_AS_HEALPIX = True`` -a healpix mask FITS file, will be written to disk. - -## Create joint random catalogue - -The individual tile-based random catalogues can be merged into a numpy -binary (``.npy``) file with -```bash -merge_final_cat -i output/run_sp_Rc/random_cat_runner/output -n random_cat -v -``` - -### Results - -We can plot the random objects, -```bash -python ~/astro/repositories/github/sp_validation/scripts/plot_rand.py -``` -and also compute the effective survey area, -```bash -~/astro/repositories/github/sp_validation/scripts/compute_area.py -``` - -## Create joint healpix mask - -First, for convenience all image headers with WCS information are -linked from within one directory, with -```bash -prepare_tiles_for_final -i -``` - -Next, read all tile mask and WCS information, and create a joint full-sky -healpix mask with -```bash -/path/to/sp_validation/scripts/scripts/combine_hp_masks.py -p -v -``` -With the option ``-p`` the mask is plotted in Mollweid projection. - - diff --git a/docs/source/toc.rst b/docs/source/toc.rst index c08c4773c..6724b2b42 100644 --- a/docs/source/toc.rst +++ b/docs/source/toc.rst @@ -41,7 +41,6 @@ :caption: Miscellaneous post_processing - random_cat .. toctree:: :hidden: diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini deleted file mode 100644 index 9f55f1812..000000000 --- a/example/cfis/config_Rc.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for: create random catalogue - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Rc - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = random_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 24 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[RANDOM_CAT_RUNNER] - -# The mask image is now an EXTERNAL product: ShapePipe generates no tile -# masks. Point the second entry at a directory of tile mask images matching -# NUMBERING_SCHEME below (the healsparse-native replacement for this module is -# the survey-window work, issue #797). -INPUT_DIR = last:get_images_runner, - -FILE_PATTERN = CFIS_image, mask - -NUMBERING_SCHEME = 000-000 - -# Number of random objects -N_RANDOM = 50000 - -# N_RANDOM is per square degrees if True -DENSITY = True - -# Output healpix mask if True -SAVE_MASK_AS_HEALPIX = True - -# Healpix mask file base name (used if SAVE_MASK_AS_HEALPIX is True) -HEALPIX_OUT_FILE_BASE = mask_hp - -# Healpix mask nside (used if SAVE_MASK_AS_HEALPIX is True) -HEALPIX_OUT_NSIDE = 1024 diff --git a/pyproject.toml b/pyproject.toml index 4c5fd7e02..47ce2b2d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ dependencies = [ "python-pysap>=0.3", "PyQt5", "pyqtgraph", - "reproject>=0.19", "sf_tools>=2.0.4", "skaha>=1.7", "sqlitedict>=2.0", diff --git a/src/shapepipe/modules/random_cat_package/__init__.py b/src/shapepipe/modules/random_cat_package/__init__.py deleted file mode 100644 index 32c6f64b2..000000000 --- a/src/shapepipe/modules/random_cat_package/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""RANDOM CATALOGUE PACKAGE. - -This package contains the module for ``random_cat``. - -:Author: Martin Kilbinger - -:Parent module: None - -:Input: Images and masks - -:Output: Random catalogue FITS file - -Description -=========== - -This module creates a random catalogue, and computes the tile area accounting -for overlapping and masked regions. - -Module-specific config file entries -=================================== - -N_RANDOM : float - The number of random objects requested on output -DENSITY : bool, optional - Option to interpret the number of random objects per square degree; the - default is ``False`` -SAVE_MASK_AS_HEALPIX : bool - Output healpix mask if ``True`` -HEALPIX_OUT_FILE_BASE : str, optional - Output halpix mask file base name; used only if SAVE_MASK_AS_HEALPIX is - ``True`` -HEALPIX_OUT_NSIDE : int, optional - Output healpix mask nside; used only if SAVE_MASK_AS_HEALPIX is ``True`` - -""" - -__all__ = ["random_cat.py"] diff --git a/src/shapepipe/modules/random_cat_package/random_cat.py b/src/shapepipe/modules/random_cat_package/random_cat.py deleted file mode 100644 index cabbd18e0..000000000 --- a/src/shapepipe/modules/random_cat_package/random_cat.py +++ /dev/null @@ -1,235 +0,0 @@ -"""RANDOM CATALOGUE. - -This module contains a class to create a random catalogue, and to compute -the tile area accounting for overlapping and masked regions. - -:Author: Martin Kilbinger - -""" - -import os -import re - -import numpy as np - -import astropy.io.fits as fits -from astropy import wcs -from astropy.table import Table - -from reproject import reproject_to_healpix - -from shapepipe.pipeline import file_io -from shapepipe.utilities import cfis - - -class RandomCat: - """Random Catalogue. - - This class creates a random catalogue given a mask FITS file. - - Parameters - ---------- - input_image_path : str - Path to input image file - input_mask_path : str - Path to input mask file - output_dir : str - Output directory - file_number_pattern : str - ShapePipe image ID string - output_file_pattern : str - Output file pattern (base name) for random catalogue - n_rand : float - Number of random objects on output - density : bool - ``n_rand`` is interpreted per square degrees if ``True`` - w_log : logging.Logger - Logging instance - healpix_options : dict - Parameters for HEALPix output mask file - """ - - def __init__( - self, - input_image_path, - input_mask_path, - output_dir, - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ): - - self._input_image_path = input_image_path - self._input_mask_path = input_mask_path - self._output_dir = output_dir - self._file_number_string = file_number_string - self._output_file_pattern = output_file_pattern - self._n_rand = n_rand - self._density = density - self._w_log = w_log - self._healpix_options = healpix_options - - def save_as_healpix(self, hdu_mask, header): - """Save As Healpix. - - Save mask as healpix FITS file. - - Parameters - ---------- - hdu_mask : class HDUList - HDU with 2D pixel mask image - header : class Header - Image header with WCS information - - """ - if not self._healpix_options: - return - - mask_1d, footprint = reproject_to_healpix( - (hdu_mask, header), - 'galactic', - nside=self._healpix_options['OUT_NSIDE'] - ) - - t = Table() - t['flux'] = mask_1d - t.meta['ORDERING'] = 'RING' - t.meta['COORDSYS'] = 'G' - t.meta['NSIDE'] = self._healpix_options['OUT_NSIDE'] - t.meta['INDXSCHM'] = 'IMPLICIT' - - output_path = ( - f'{output_dir}/{self._healpix_options["FILE_BASE"]}-' - + f'{file_number_string}.fits' - ) - t.write(output_path) - - def process(self): - """Process. - - Main function to identify exposures. - - """ - # Read image FITS file header - try: - img = fits.open(self._input_image_path) - header = img[0].header - except (OSError, IOError) as error: - # FITS file might contain only header. - # Try as ascii file - try: - fin = open(self._input_image_path) - header = fits.Header.fromtextfile(fin) - fin.close() - except Exception: - raise - - # Get WCS - WCS = wcs.WCS(header) - - # Read mask FITS file - hdu_mask = fits.open(self._input_mask_path) - mask = hdu_mask[0].data - - # Save mask in healpix format (if option is set) - self._save_as_healpix(hdu_mask, header) - - # Number of pixels - n_pix_x = mask.data.shape[0] - n_pix_y = mask.data.shape[1] - n_pix = n_pix_x * n_pix_y - - # Number of non-masked pixels - n_unmasked = len(np.where(mask == 0)[0]) - - # Compute various areas - - # Pixel area in deg^2 - area_pix = wcs.utils.proj_plane_pixel_area(WCS) - - # Tile area - area_deg2 = area_pix * n_pix - - # Area of unmasked region - area_deg2_eff = area_pix * n_unmasked - - # Compute number of requested objects - if n_unmasked > 0: - if not self._density: - # Use value from config file - n_obj = self._n_rand - else: - # Compute number of objects from density - n_obj = int( - self._n_rand / area_deg2 * area_deg2_eff / area_deg2 - ) - - # Check that a reasonably large number of pixels is not masked - if n_unmasked < n_obj: - raise ValueError( - f"Number of un-masked pixels {n_unmasked} is smaller " - + f"than number of random objects requested {n_obj}" - ) - - else: - n_obj = 0 - - self._w_log.info(f"Creating {n_obj} random objects") - - # Draw points until n are in mask - n_found = 0 - xy_rand = [] - while n_found < n_obj: - idx_x = np.random.randint(n_pix_x) - idx_y = np.random.randint(n_pix_y) - - # Add points with additional random sub-pixel value - if mask[idx_x, idx_y] == 0: - d = np.random.random(2) - # MKDEBUG: the following seems to work, x and y interchanged - xy_rand.append([idx_y + d[1], idx_x + d[0]]) - n_found = n_found + 1 - xy_rand = np.array(xy_rand) - - # Transform to WCS - res = WCS.all_pix2world(xy_rand, 1) - if n_unmasked > 0: - ra_rand = res[:, 0] - dec_rand = res[:, 1] - x_rand = xy_rand[:, 0] - y_rand = xy_rand[:, 1] - else: - ra_rand = [] - dec_rand = [] - x_rand = [] - y_rand = [] - - # Tile ID - output_path = ( - f"{self._output_dir}/{self._output_file_pattern}-" - + f"{self._file_number_string}.fits" - ) - file_base = os.path.splitext(file_name)[0] - tile_ID_str = re.split("-", file_base)[1:] - tile_id = float(".".join(tile_ID_str)) - tile_id_array = np.ones(n_obj) * tile_id - - # Write to output - cat_out = [ra_rand, dec_rand, x_rand, y_rand, tile_id_array] - column_names = ["RA", "DEC", "x", "y", "TILE_ID"] - - # TODO: Add units to header - output = file_io.FITSCatalogue( - output_path, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite - ) - output.save_as_fits(cat_out, names=column_names) - - # Write area information to log file - self._w_log.info(f"Total area = {area_deg2:.4f} deg^2") - self._w_log.info(f"Unmasked area = {area_deg2_eff:.4f} deg^2") - self._w_log.info( - f"Ratio masked to total pixels = {n_unmasked / n_pix:.3f}" - ) diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py deleted file mode 100644 index f180c9f7e..000000000 --- a/src/shapepipe/modules/random_cat_runner.py +++ /dev/null @@ -1,81 +0,0 @@ -"""RANDOM CAT RUNNER. - -Module runner for ``random_cat``. - -:Author: Martin Kilbinger - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.random_cat_package.random_cat import RandomCat - - -@module_runner( - version="1.1", - # The mask image is an external input: ShapePipe generates no tile masks - # (the healsparse-native replacement for this module is issue #797). - file_pattern=["image", "mask"], - file_ext=[".fits", "fits"], - depends=["astropy"], - numbering_scheme="_0", -) -def random_cat_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Random Catalogue Runner.""" - # Get input file names of image and mask - input_image_name = input_file_list[0] - input_mask_name = input_file_list[1] - - # Set output file name - if config.has_option(module_config_sec, "OUTPUT_FILE_PATTERN"): - output_file_pattern = config.get( - module_config_sec, "OUTPUT_FILE_PATTERN" - ) - else: - output_file_pattern = "random_cat" - - # Get number of random objects requested on output - n_rand = config.getfloat(module_config_sec, "N_RANDOM") - - # Flag whether n_rand is total (DENSITY=False, default) - # or per square degree (DENSITY=True) - if config.has_option(module_config_sec, "DENSITY"): - density = config.getboolean(module_config_sec, "DENSITY") - else: - density = False - - # Get healpix output options - save_mask_as_healpix = config.getboolean( - module_config_sec, "SAVE_MASK_AS_HEALPIX" - ) - if save_mask_as_healpix: - healpix_options = {} - for option_trunc in ['FILE_BASE', 'OUT_NSIDE']: - option = f'HEALPIX_OUT_{option_trunc}' - healpix_options[option_trunc] = config.get( - module_config_sec, option - ) - # Create rand cat class instance - rand_cat_inst = RandomCat( - input_image_name, - input_mask_name, - run_dirs["output"], - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ) - - # Run processing - rand_cat_inst.process() - - # No return objects - return None, None diff --git a/uv.lock b/uv.lock index 6a119eea6..e1d016ab6 100644 --- a/uv.lock +++ b/uv.lock @@ -141,24 +141,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/ef/a5cf36a402c4511a776405f12d0b35196f55f4312ce407012a2fbcf1e4e0/astropy-8.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b482bc6c57c966e6c6234410a1d9afbcf92bcac858cf287812f8c99ddc3fafc", size = 10407732, upload-time = "2026-07-05T07:24:40.774Z" }, ] -[[package]] -name = "astropy-healpix" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/c1/aeb3fe3be2ee863708d625014267c71abfe20ddaa293b3d4ddb72ee1d6e9/astropy_healpix-2.0.1.tar.gz", hash = "sha256:0e3f1c94064c45da779900cb90c938df7aef99a924abb23eeb893b16540e77e6", size = 112256, upload-time = "2026-07-20T21:07:30.004Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/3d/0cde0db89ac8dd4e5347322470530298915739f7e9b356b01c1939de8c4f/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:179119d5a69e7b9245919cbe04c3e6bf0a485516b36c29f3402951aad5452251", size = 191178, upload-time = "2026-07-20T21:07:16.512Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c4/71cf2bd4374cc17e015413462be8051fe08bc49e077b7d48072fff4e465d/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c092c54124c48f8d98e04fb22f4b2aa4c8675e65d81c351523f41377f9a6df22", size = 193429, upload-time = "2026-07-20T21:07:18.015Z" }, - { url = "https://files.pythonhosted.org/packages/fa/5c/3a50f68225b836b395da4fb8dfd3d702ded1916042490b16a613caa0e4a6/astropy_healpix-2.0.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce875a29c598c1a99f8f68351daeb4173463044dce4f1c7ebfe8c233ec5e9a49", size = 188694, upload-time = "2026-07-20T21:07:19.244Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a7/c369703ca3fc6f5bee31b3d1f6d0f0b38c15ec84e7fbb6f552c0c7cedfb2/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78f76785852bcc748f5efab8bb4f2ab1fe959d7a998b48e7ed1e59a46cbf0e51", size = 198717, upload-time = "2026-07-20T21:07:24.657Z" }, - { url = "https://files.pythonhosted.org/packages/32/d5/0c4183611b8f36877112882e25cc8a91655a2d11e120ea1f5c153cb7d3a0/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02cefde735da1fe74654e786e02286261b04cc43f5c908a86a8457b2989ac2aa", size = 201205, upload-time = "2026-07-20T21:07:26.128Z" }, - { url = "https://files.pythonhosted.org/packages/e2/54/42bcd02b7c604d3a1132dfa93195bdc2677bb7844172cd098e149cbcb4e9/astropy_healpix-2.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65bee7b70f35ddb81d6b6c849c5bf46768afbce6869395e4f9e0a5c27cb0ce17", size = 196134, upload-time = "2026-07-20T21:07:27.55Z" }, -] - [[package]] name = "astropy-iers-data" version = "0.2026.8.10.0.32.39" @@ -476,15 +458,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -712,45 +685,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] -[[package]] -name = "dask" -version = "2026.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "cloudpickle" }, - { name = "fsspec" }, - { name = "packaging" }, - { name = "partd" }, - { name = "pyyaml" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/39/cbd21c9133d02b4e60899ed466ad5e876553ea68ebffee6209e7dcafc8d4/dask-2026.7.1.tar.gz", hash = "sha256:5727484427665f051e86bf87d021a64d6411141cdc8a20bfe3c1ad2968cc06b7", size = 11548794, upload-time = "2026-07-14T01:06:22.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/5f/7c22733da92b3a6cc4dddcaa8731089d213c2790bbc997e51c429a4e8f8b/dask-2026.7.1-py3-none-any.whl", hash = "sha256:985ffd6c5e9d7979ede515e84ae8d39b647d6aa64f77600f15714ff65f578fe6", size = 1496882, upload-time = "2026-07-14T01:06:20.341Z" }, -] - -[package.optional-dependencies] -array = [ - { name = "numpy" }, -] - -[[package]] -name = "dask-image" -version = "2026.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dask", extra = ["array"] }, - { name = "numpy" }, - { name = "pims" }, - { name = "scipy" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/5b/15d6d6ff8697b188787609be059fe4f07f99fc00f43f68e9e1540fa8733e/dask_image-2026.5.0-py3-none-any.whl", hash = "sha256:acf86cd7f0f1e97804198d30b7cc931efd29f9dec86c65ec004b50405c3f5227", size = 43814, upload-time = "2026-05-27T14:05:56.237Z" }, -] - [[package]] name = "datetime" version = "6.0" @@ -803,18 +737,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - [[package]] name = "dpath" version = "2.2.0" @@ -895,15 +817,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, ] -[[package]] -name = "fsspec" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, -] - [[package]] name = "future" version = "1.0.0" @@ -960,20 +873,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/ed/ae57eb7d344f43f87b74b3a281ead6ec7d6394eef72a7b1dcb28dd089550/gitpython-3.1.59-py3-none-any.whl", hash = "sha256:67a82f537384578643624c8b2c531938a9b82be431663e575dcf638526631d4c", size = 220996, upload-time = "2026-08-10T12:03:18.804Z" }, ] -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, -] - [[package]] name = "greenlet" version = "3.5.5" @@ -1809,15 +1708,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/f9/7b7b50f80b4585bcd78675ff3110c256877b11df32a8cde284f851762f57/llvmlite-0.49.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32adb84fdaae28aeb86fdb6253084ee707ee157289a2e98fe3caf48a62bee82", size = 58344482, upload-time = "2026-08-11T16:25:51.527Z" }, ] -[[package]] -name = "locket" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, -] - [[package]] name = "lsstdesc-coord" version = "1.3.1" @@ -2262,24 +2152,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/f9/3a7b6dbf81e01a48958b45ad2239edbc64707522ab17f11f9f18c44bf6d1/numba-0.67.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83ab968b0e0fa744eba03351282dd8000796e6ec8e4518f47bd3ed86c0a20c7b", size = 3614644, upload-time = "2026-08-11T23:03:55.794Z" }, ] -[[package]] -name = "numcodecs" -version = "0.16.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, - { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, - { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, - { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, - { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, -] - [[package]] name = "numpy" version = "2.5.2" @@ -2380,19 +2252,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] -[[package]] -name = "partd" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "locket" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, -] - [[package]] name = "pexpect" version = "4.9.0" @@ -2446,19 +2305,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, ] -[[package]] -name = "pims" -version = "0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imageio" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "slicerator" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } - [[package]] name = "platformdirs" version = "4.11.3" @@ -2553,15 +2399,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "pyavm" -version = "0.9.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/ac/a925d36dd37fc37f89afccc1f62ffa03b7344c8e4ff7850be7879b4497e6/pyavm-0.9.9.tar.gz", hash = "sha256:bc0f605d957c1fd6d7765523fcba8b9a72377ac6c51461c2a838fe44600bcd9a", size = 220572, upload-time = "2026-03-12T09:54:50.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/ab/ba8d2b40aee05cd986807d58ee324369eb16248b717a08b51eee977f8d33/pyavm-0.9.9-py3-none-any.whl", hash = "sha256:8bba0ee9645a8a9f215af9ceea67b494a6ee1fe380cebdc00ba99d781539ad76", size = 379786, upload-time = "2026-03-12T09:54:49.455Z" }, -] - [[package]] name = "pybind11" version = "3.1.0" @@ -3048,28 +2885,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] -[[package]] -name = "reproject" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "astropy-healpix" }, - { name = "dask", extra = ["array"] }, - { name = "dask-image" }, - { name = "fsspec" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pyavm" }, - { name = "scipy" }, - { name = "zarr" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/5b/8d9b51c754ab014194d374cb4873d0729b397f67997e67721538b643e682/reproject-0.21.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2132fc5d2fa3fbbd57337099f9d47582136e653ac82402783c7ceffa26804816", size = 1776590, upload-time = "2026-06-25T15:11:30.358Z" }, - { url = "https://files.pythonhosted.org/packages/da/2d/f9d76e8e308813978227e1e43a1f25f64cd0f4b030cecaeff8baeec9eeac/reproject-0.21.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8525baaca84e949a69532c02ff491c23993e3bc22a59f694af576e473f2a9d2f", size = 1791889, upload-time = "2026-06-25T15:11:31.741Z" }, -] - [[package]] name = "requests" version = "2.34.2" @@ -3417,7 +3232,6 @@ dependencies = [ { name = "pyqtgraph" }, { name = "python-dateutil" }, { name = "python-pysap" }, - { name = "reproject" }, { name = "sf-tools" }, { name = "skaha" }, { name = "sqlitedict" }, @@ -3503,7 +3317,6 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0" }, { name = "python-dateutil" }, { name = "python-pysap", specifier = ">=0.3" }, - { name = "reproject", specifier = ">=0.19" }, { name = "ruff", marker = "extra == 'lint'" }, { name = "sf-tools", specifier = ">=2.0.4" }, { name = "shapepipe", extras = ["doc", "jupyter", "lint", "release", "test", "fitsio"], marker = "extra == 'dev'" }, @@ -3556,15 +3369,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/d3/a3c1d569ae714ed9985adccefdfe2d6dd9f8ad7a297bd8fa4c7ed30c587b/skaha-1.7.0-py3-none-any.whl", hash = "sha256:68b0d3c925b98bf145c5f695237474fe1cea07cf9b69b4a9bae9910375dfa01a", size = 40795, upload-time = "2025-05-28T21:17:02.296Z" }, ] -[[package]] -name = "slicerator" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/52/f38586b82b2935f8b59a09b0a79c545a22ed062e728c9418bafeb51f61e0/slicerator-1.1.0.tar.gz", hash = "sha256:44010a7f5cd87680c07213b5cabe81d1fb71252962943e5373ee7d14605d6046", size = 38283, upload-time = "2022-04-07T18:54:08.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, -] - [[package]] name = "smart-open" version = "7.7.1" @@ -4027,15 +3831,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, ] -[[package]] -name = "toolz" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, -] - [[package]] name = "tornado" version = "6.5.8" @@ -4251,23 +4046,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, ] -[[package]] -name = "zarr" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "donfig" }, - { name = "google-crc32c" }, - { name = "numcodecs" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/15/436cb1d3bbe86173bd44ce7a34ecb210d0c0416946e337858149a905ef5a/zarr-3.3.0.tar.gz", hash = "sha256:cd0c8cf738b4bb4807815bc1255acad5bdf1a7b7264b606c5a1bc0d0392a306b", size = 943626, upload-time = "2026-07-30T16:35:10.491Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/c6/6b726ddf4c3ac5a123f285c3650fa8268902c28ea541a0282867f1336e65/zarr-3.3.0-py3-none-any.whl", hash = "sha256:323bf5366d4f909052ef6e2e03e7481a7434c3ee75d3a981eb3a71fc1ae22cef", size = 363685, upload-time = "2026-07-30T16:35:08.794Z" }, -] - [[package]] name = "zipp" version = "4.1.0" From edec5460b449a57906df566a6ac293c35c0562a5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:36:38 -0400 Subject: [PATCH 085/100] =?UTF-8?q?fix(config):=20the=202026=20DR6=20produ?= =?UTF-8?q?cts=20are=20named=20mask=5Fugriz=5F*=20=E2=80=94=20point=20the?= =?UTF-8?q?=20placeholder=20at=20the=20real=20ladder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Aug-2026 post-GSC2 band-combined healsparse products (arc:home/mhudson/masks/) use the mask_ugriz_nside131072_n.hsp naming; the mask_r_* files on canfar's ShapePipe/mask are the 2025 vintage. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_exp_mccd.ini | 2 +- example/cfis/config_exp_psfex.ini | 2 +- src/shapepipe/utilities/mask_query.py | 2 +- workflow/config/cfis/config_exp_psfex.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 5d2460312..68dcfbeef 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -145,7 +145,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index b4e2d1fa1..2f1c8e353 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -147,7 +147,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index d789d4100..e5c6c02eb 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -33,7 +33,7 @@ edge convention we did not think of. ``test_partial_read_matches_full`` is what actually holds the two paths equal. -Measured on the DR6 star map (``mask_r_nside131072_n4.hsp``, 583 MB, +Measured on the DR6 star map (``mask_ugriz_nside131072_n4.hsp``, 583 MB, ``nside_coverage=128``) for 2000 positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, identical values. Per 40-CCD exposure that is ~4 s against ~8.5 min of map reading, and diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 2fa5dc525..ffcac86b6 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -146,7 +146,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map From 97c44b759f4ce00fb6339e7ea9ebedb8bfc30d9a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 12:21:36 -0400 Subject: [PATCH 086/100] =?UTF-8?q?feat(mask=5Fquery):=20ship=20permissive?= =?UTF-8?q?=20=E2=80=94=20flag=20PSF-star=20candidates,=20cut=20nothing=20?= =?UTF-8?q?by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PSF star selection no longer cuts on the external masks. mask_query stays in the chain and keeps writing FLAG_EXT from the shipped MASK_PATHS (the star-body map, bit 2); star_selection.setools drops `FLAG_EXT == 0` from all four mask blocks and rejects on the instrument flags alone, leaning on outlier rejection for the rest. This makes the exposure side agree with the tile side, which was already permissive: make_cat writes MASK_ unfiltered. Flag transparently, cut downstream — so the effect of a mask on the star sample can be MEASURED before it is imposed, rather than baked in on the way past. The escape hatch is documented where someone would look for it, because a permissive default is only safe if reversing it is obvious. star_selection.setools' header now spells out the change (add `FLAG_EXT == 0` beside each `IMAFLAGS_ISO == 0`, one line per block) and says why it exists: if outlier rejection turns out not to be robust enough. The module docstring gains a "Nothing cuts on it by default" section, and the config comments beside MASK_PATHS say NOTHING CUTS ON IT rather than naming a cut that is no longer there. The tutorial's Masks section, pipeline_canfar.md and workflow/README.md lose the same stale claim; the shared utility now says setools *could* cut on the column, not that it does. The diet section is retitled: MASK_PATHS is a list of maps to RECORD against each candidate, not to reject on. Halos stay out for the same reason as before — they say nothing about whether a star is a good PSF sample. Verified: 25 tests pass, 29 runners import, 64 configs resolve, ruff clean. Only the four cut lines changed in the setools body — the mask blocks and their IMAFLAGS_ISO tests are otherwise untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 2 +- docs/source/pipeline_tutorial.md | 16 ++++++-- example/cfis/config_exp_mccd.ini | 6 ++- example/cfis/config_exp_psfex.ini | 6 ++- example/cfis/star_selection.setools | 32 ++++++++------- .../modules/mask_query_package/__init__.py | 39 +++++++++++++------ src/shapepipe/utilities/mask_query.py | 3 +- workflow/README.md | 8 ++-- workflow/config/cfis/config_exp_psfex.ini | 6 ++- 9 files changed, 77 insertions(+), 41 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 926d36578..1555f75f2 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -174,7 +174,7 @@ shapepipe_run -c cfis/config_tile_Uz.ini There is no masking step. `ShapePipe` generates no masks: the sky-fixed healsparse maps are queried once per object, by `mask_query` on the exposure -catalogues (`FLAG_EXT`, cut by `setools`) and by `make_cat` on the final tile +catalogues (`FLAG_EXT`, recorded but not cut on) and by `make_cat` on the tile catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` config entries at the maps and nothing else is needed — no star-catalogue download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index a4db96579..b2cf0640f 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -196,15 +196,23 @@ consumed by *querying them at object positions*, never by rasterizing them onto pixels. Two modules do the querying, from the same shared lookup (`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and `setools` on the single-exposure single-CCD catalogues and writes one integer -`FLAG_EXT` column (0 = clean), which `star_selection.setools` cuts on so that -masked objects never enter the PSF star sample — a deliberately narrow diet, -the star-body map (bit 2) only, since halos flag objects without disqualifying -them as PSF stars; `make_cat` writes one +`FLAG_EXT` column (0 = clean), recording the star-body map (bit 2) against every +detection; `make_cat` writes one `MASK_` column per band onto the final tile catalogue, carrying the map value verbatim so downstream selections choose their own cuts. Map paths and bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and `MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. +**Nothing in the pipeline cuts on these columns.** The PSF star selection ships +permissive: `star_selection.setools` rejects on the instrument flags +(`IMAFLAGS_ISO == 0`) and nothing else, leaning on outlier rejection for the +rest, and the final catalogue's `MASK_` columns are written unfiltered. +The principle is to flag transparently and cut downstream, so the effect of a +mask can be measured before it is imposed. If outlier rejection turns out not to +be robust enough, feeding the external masks to the star selection is one line +per mask block — add `FLAG_EXT == 0` beside each `IMAFLAGS_ISO == 0` — and that +file's header documents the change. + No internet access is needed at any point, and there is no reference star catalogue to download. diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 68dcfbeef..ec43bbf46 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -143,8 +143,10 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT, which star_selection.setools cuts on as -# FLAG_EXT == 0. Comma-separated. +# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships +# permissive (instrument flags only) and the column is carried for +# transparency and measurement — see star_selection.setools' header for the +# one-line change that would impose it. Comma-separated. MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 2f1c8e353..afa41cc7c 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -145,8 +145,10 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT, which star_selection.setools cuts on as -# FLAG_EXT == 0. Comma-separated. +# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships +# permissive (instrument flags only) and the column is carried for +# transparency and measurement — see star_selection.setools' header for the +# one-line change that would impose it. Comma-separated. MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index 230017053..bbda2e520 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -1,16 +1,24 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties ## -## Two independent mask cuts, and they come from different places: +## ONE mask cut, and it is the instrument flags: ## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), -## delivered with the exposure and read by SExtractor; -## FLAG_EXT == 0 the external healsparse masks, queried per detection by -## the mask_query module. Which maps reach FLAG_EXT is that -## module's MASK_PATHS config, and the shipped diet is -## deliberately narrow: the star-body map (bit 2) only, no -## halos (they flag, they do not reject stars) and no -## MaxiMask. -## SETools expressions have no bitwise operators, so mask_query does the bit -## selection and this file only tests for zero. +## delivered with the exposure and read by SExtractor. +## +## The external healsparse masks are NOT cut on here. mask_query queries them +## per detection and writes FLAG_EXT into this catalogue, but the selection +## ships permissive: the column is carried for transparency and measurement, +## and the star sample is defended by outlier rejection instead. Flag +## transparently, cut downstream. +## +## To feed the external masks to the star selection after all, add +## +## FLAG_EXT == 0 +## +## beside each IMAFLAGS_ISO line below. That is the whole change — one line per +## mask block — and it is the intended escape hatch if outlier rejection turns +## out not to be robust enough. Which maps reach FLAG_EXT is mask_query's +## MASK_PATHS config; SETools has no bitwise operators, so the bit selection +## happens there and this file would only ever test for zero. [MASK:preselect] MAG_AUTO > 0 @@ -19,13 +27,11 @@ FWHM_IMAGE > 0.3 / 0.187 FWHM_IMAGE < 1.5 / 0.187 FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 NO_SAVE [MASK:flag] FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 NO_SAVE @@ -37,7 +43,6 @@ FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 [MASK:fwhm_mag_cut] FWHM_IMAGE > 0 @@ -45,7 +50,6 @@ FWHM_IMAGE < 40 MAG_AUTO < 35 FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 NO_SAVE # Split the 'star_selection' sample into diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index 0391e5393..3acb8820e 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -29,24 +29,39 @@ carry its bits through. Nothing downstream reads more than ``== 0``. The single column exists because ``setools`` expressions support only -``< > <= >= == !=`` — no bitwise operators — so the bit selection has to -happen here. ``star_selection.setools`` cuts on ``FLAG_EXT == 0`` beside its -existing ``IMAFLAGS_ISO == 0``: instrument flags reject pixels, the queried -masks reject objects. +``< > <= >= == !=`` — no bitwise operators — so any bit selection has to happen +here, leaving the config a plain test for zero. + +Nothing cuts on it by default +============================= + +The shipped ``star_selection.setools`` does NOT cut on ``FLAG_EXT``. The PSF +star selection is permissive: it rejects on the instrument flags +(``IMAFLAGS_ISO == 0``) and leans on outlier rejection for the rest. The +column is written for transparency and measurement — so the effect of the +external masks on the star sample can be *measured* before it is imposed — +which is the same principle as ``make_cat``'s unfiltered ``MASK_`` +columns: flag here, cut downstream. + +Feeding the external masks to the star selection is a one-line change if +outlier rejection turns out not to be robust enough: add ``FLAG_EXT == 0`` by +each +``IMAFLAGS_ISO == 0`` in ``star_selection.setools``. The escape hatch is +deliberate, and the file's header says so. The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with ``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is written once. That module's docstring documents the off-coverage convention. -The diet is deliberately narrow -=============================== +What gets queried is deliberately narrow +======================================== -``MASK_PATHS`` is a *list of maps to reject PSF stars on*, not a list of every -mask that exists. The committed configs name exactly one map — the UNIONS -star-body product (bit 2) — beside the instrument flags SExtractor already -reads. Halo bits 0 and 1 are excluded on purpose: halos flag objects for the -final catalogue, they do not reject PSF stars (mask-force telecon, 2026-07-21). -MaxiMask is not in the diet either. +``MASK_PATHS`` is a *list of maps to record against each PSF-star candidate*, +not a list of every mask that exists. The committed configs name exactly one +map — the UNIONS star-body product (bit 2). Halo bits 0 and 1 are excluded on +purpose: halos flag objects for the final catalogue, they say nothing about +whether a star is a good PSF sample (mask-force telecon, 2026-07-21). MaxiMask +is not queried here either. Widening it costs a config edit and no code — add a path. That is why the contract is a path list rather than a bit mask: the UNIONS products are one diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index e5c6c02eb..cf79b21e0 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -15,7 +15,8 @@ * ``mask_query`` writes a single integer ``FLAG_EXT`` column onto the exposure SExtractor catalogue, combining the configured maps into "clean (0) or flagged (nonzero)" so that ``setools`` — whose expression language has no - bitwise operators — can cut on ``FLAG_EXT == 0``. + bitwise operators — *could* cut on ``FLAG_EXT == 0``. The shipped selection + does not; the column is carried for measurement (see that module). Partial reads ------------- diff --git a/workflow/README.md b/workflow/README.md index ac997afef..4aa24d29b 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -201,9 +201,11 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed healsparse maps and QUERIED once per object: the `mask_query` module writes a - `FLAG_EXT` column onto each CCD's detection catalogue for setools' star cut - (inside `exp_psf`), and `make_cat` writes one `MASK_` column per band - onto the final catalogue (inside `tile_make_cat`). Map paths are config, not + `FLAG_EXT` column onto each CCD's detection catalogue (inside `exp_psf`), and + `make_cat` writes one `MASK_` column per band onto the final catalogue + (inside `tile_make_cat`). Neither is cut on in the pipeline: the PSF star + selection rejects on instrument flags alone and everything else is a + downstream decision. Map paths are config, not code, so regenerated products cost a config edit. Nothing is fetched from a catalogue server, staged, or rasterized, which is why the old `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index ffcac86b6..858e3828f 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -144,8 +144,10 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT, which star_selection.setools cuts on as -# FLAG_EXT == 0. Comma-separated. +# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships +# permissive (instrument flags only) and the column is carried for +# transparency and measurement — see star_selection.setools' header for the +# one-line change that would impose it. Comma-separated. MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, From 43c32cdeb7f01fdda57924f48a260cce0d8fd22f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 12:44:37 -0400 Subject: [PATCH 087/100] feat(mask_query): ship the query off by default, rename FLAG_EXT to MASK_EXT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes, both about making the naming and the default say what the design means. MASK_EXT, not FLAG_EXT. The column is one family with make_cat's MASK_: an external mask, queried and stored per object. "FLAG" belonged to the other kind of mask entirely, which is the distinction the docstrings now lead with — instrument flags mark a CORRUPTED MEASUREMENT (bad columns, saturation) and are the only masks that reject anything in-pipeline, via IMAFLAGS_ISO in setools and zero-weighted pixels and dropped epochs in ngmix; healsparse masks are sky-fixed LOCATION flags that say where an object sits, not that its pixels are broken, so they are queried to columns and every rejection is downstream. Renamed in code, tests, docstrings, the setools header, the tutorial and both READMEs. MASK_PATHS ships commented out, and absence is now a strict no-op rather than a ValueError: the runner treats a missing key the way make_cat treats a missing MASK_EXT_PATHS, and MaskQuery with no maps copies the catalogue through byte-for-byte with NO MASK_EXT column. The copy is the load-bearing part — setools reads this module's output, so a no-op that wrote no file would break the chain instead of disabling the query. mask_query stays in the MODULE chain either way, so turning the query on is uncommenting one line and never editing the chain. Two tests cover it, one asserting the output is byte-identical to the input. The framing that ties both: the PSF star selection deliberately starts from outlier rejection alone, and MASK_EXT is the configurable pickup if that proves insufficient — write the column, measure the effect, impose it only if the measurement says to. Also corrected in passing: edec5460 renamed the measured file in the partial-read note along with the config placeholders, which credited the measurement to a file I did not run against. The note now says what was actually measured — the staged single-band copy mask_r_nside131072_n4.hsp — and names the mask_ugriz ladder it belongs to. Verified: 27 tests pass, 29 runners import, 64 configs resolve (MASK_PATHS absent, section and chain entry present in all three), ruff clean on the touched files. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 2 +- docs/source/pipeline_tutorial.md | 28 +++++++---- example/cfis/config_exp_mccd.ini | 17 +++++-- example/cfis/config_exp_psfex.ini | 17 +++++-- example/cfis/star_selection.setools | 34 ++++++++----- .../modules/mask_query_package/__init__.py | 50 +++++++++++++------ .../modules/mask_query_package/mask_query.py | 31 ++++++++++-- src/shapepipe/modules/mask_query_runner.py | 19 ++++--- src/shapepipe/utilities/mask_query.py | 18 ++++--- tests/module/test_mask_query.py | 46 +++++++++++++++-- workflow/README.md | 11 ++-- workflow/config/cfis/config_exp_psfex.ini | 17 +++++-- 12 files changed, 206 insertions(+), 84 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 1555f75f2..ecc621e9f 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -174,7 +174,7 @@ shapepipe_run -c cfis/config_tile_Uz.ini There is no masking step. `ShapePipe` generates no masks: the sky-fixed healsparse maps are queried once per object, by `mask_query` on the exposure -catalogues (`FLAG_EXT`, recorded but not cut on) and by `make_cat` on the tile +catalogues (`MASK_EXT`, recorded but not cut on) and by `make_cat` on the tile catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` config entries at the maps and nothing else is needed — no star-catalogue download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index b2cf0640f..51f09da7a 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -196,22 +196,30 @@ consumed by *querying them at object positions*, never by rasterizing them onto pixels. Two modules do the querying, from the same shared lookup (`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and `setools` on the single-exposure single-CCD catalogues and writes one integer -`FLAG_EXT` column (0 = clean), recording the star-body map (bit 2) against every +`MASK_EXT` column (0 = clean), recording the star-body map (bit 2) against every detection; `make_cat` writes one `MASK_` column per band onto the final tile catalogue, carrying the map value verbatim so downstream selections choose their own cuts. Map paths and bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and `MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. -**Nothing in the pipeline cuts on these columns.** The PSF star selection ships -permissive: `star_selection.setools` rejects on the instrument flags -(`IMAFLAGS_ISO == 0`) and nothing else, leaning on outlier rejection for the -rest, and the final catalogue's `MASK_` columns are written unfiltered. -The principle is to flag transparently and cut downstream, so the effect of a -mask can be measured before it is imposed. If outlier rejection turns out not to -be robust enough, feeding the external masks to the star selection is one line -per mask block — add `FLAG_EXT == 0` beside each `IMAFLAGS_ISO == 0` — and that -file's header documents the change. +The distinction that drives all of this is what a mask *means*. An **instrument +flag** marks a corrupted measurement — the pixels carry no usable signal — so +these are the only masks that reject anything inside the pipeline. The +**healsparse masks** are sky-fixed location flags: they say where an object +sits, not that its pixels are broken, so what to do about one is an analysis +decision and is made downstream. + +**Nothing in the pipeline cuts on the queried columns**, and on exposures the +query ships off entirely: `MASK_PATHS` is commented out, which makes +`mask_query` a strict no-op that passes the catalogue through with no +`MASK_EXT` column (the module stays in the chain, so enabling it is +uncommenting one line). `star_selection.setools` rejects on `IMAFLAGS_ISO == 0` +and nothing else, deliberately starting from outlier rejection alone, and the +final catalogue's `MASK_` columns are written unfiltered. `MASK_EXT` is +the configurable pickup if outlier rejection proves insufficient: add +`MASK_EXT == 0` beside each `IMAFLAGS_ISO == 0`, one line per mask block, as +that file's header documents. No internet access is needed at any point, and there is no reference star catalogue to download. diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index ec43bbf46..be3f2690c 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -143,11 +143,18 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships -# permissive (instrument flags only) and the column is carried for -# transparency and measurement — see star_selection.setools' header for the -# one-line change that would impose it. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +# detection sets MASK_EXT. Comma-separated. +# +# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue +# is passed through with no MASK_EXT column, the same gating make_cat gives +# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning +# the query on is uncommenting one line and never editing the chain. +# +# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive +# (instrument flags only) and the column is carried for transparency and +# measurement — see star_selection.setools' header for the one-line change +# that would impose it. +; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index afa41cc7c..fe3ecad43 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -145,11 +145,18 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships -# permissive (instrument flags only) and the column is carried for -# transparency and measurement — see star_selection.setools' header for the -# one-line change that would impose it. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +# detection sets MASK_EXT. Comma-separated. +# +# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue +# is passed through with no MASK_EXT column, the same gating make_cat gives +# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning +# the query on is uncommenting one line and never editing the chain. +# +# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive +# (instrument flags only) and the column is carried for transparency and +# measurement — see star_selection.setools' header for the one-line change +# that would impose it. +; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index bbda2e520..32197b466 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -1,24 +1,32 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties ## ## ONE mask cut, and it is the instrument flags: -## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), +## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation) ## delivered with the exposure and read by SExtractor. ## -## The external healsparse masks are NOT cut on here. mask_query queries them -## per detection and writes FLAG_EXT into this catalogue, but the selection -## ships permissive: the column is carried for transparency and measurement, -## and the star sample is defended by outlier rejection instead. Flag -## transparently, cut downstream. +## That is deliberate, and it follows from what the two kinds of mask MEAN. +## An instrument flag marks a CORRUPTED MEASUREMENT — the pixels carry no +## usable signal — so a flagged star is not a star we could model badly, it is +## one we cannot model at all. The healsparse masks are sky-fixed LOCATION +## flags (a star halo, a manual region, a band with no data): they say where an +## object sits, not that its pixels are broken, so whether one disqualifies a +## PSF star is a judgement, not a fact about the data. ## -## To feed the external masks to the star selection after all, add +## So the star selection deliberately starts from OUTLIER REJECTION ALONE for +## that judgement. mask_query still queries the external masks per detection +## and writes MASK_EXT into this catalogue — carried for transparency and +## measurement, so their effect on the star sample can be measured before it is +## imposed. Flag transparently, cut downstream. ## -## FLAG_EXT == 0 +## MASK_EXT is the configurable pickup if outlier rejection proves +## insufficient. To impose it, add ## -## beside each IMAFLAGS_ISO line below. That is the whole change — one line per -## mask block — and it is the intended escape hatch if outlier rejection turns -## out not to be robust enough. Which maps reach FLAG_EXT is mask_query's -## MASK_PATHS config; SETools has no bitwise operators, so the bit selection -## happens there and this file would only ever test for zero. +## MASK_EXT == 0 +## +## beside each IMAFLAGS_ISO line below — one line per mask block, and that is +## the whole change. Which maps reach MASK_EXT is mask_query's MASK_PATHS +## config, which ships commented out; SETools has no bitwise operators, so the +## bit selection happens there and this file only ever tests for zero. [MASK:preselect] MAG_AUTO > 0 diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index 3acb8820e..fed9dcd92 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -8,18 +8,34 @@ :Input: Single-exposure single-CCD SExtractor catalogue -:Output: The same catalogue with an added ``FLAG_EXT`` column +:Output: The same catalogue with an added ``MASK_EXT`` column + +Two classes of mask +=================== + +ShapePipe distinguishes them, and they are not interchangeable: + +**Instrument flags** mark a CORRUPTED MEASUREMENT — bad columns, saturation — +in the flag image delivered with each exposure. A flagged pixel carries no +usable signal, so these are the only masks that reject anything inside the +pipeline: ``setools`` drops flagged stars via ``IMAFLAGS_ISO``, and ``ngmix`` +zero-weights flagged pixels and drops epochs that lose too many. + +**Healsparse masks** are sky-fixed LOCATION flags — a star halo, a manual +region, a band with no data. They say where an object is, not that its pixels +are broken, so what to do about one is an analysis decision. They are queried +into columns and every rejection happens downstream: ``MASK_EXT`` here, +``MASK_`` in ``make_cat``. Nothing in the pipeline cuts on either. Description =========== -ShapePipe consumes sky-fixed masks by querying them, not by rasterizing them. This module sits between SExtractor and ``setools`` on the exposure chain: it reads each detection's windowed world position (``XWIN_WORLD``, ``YWIN_WORLD`` in the ``LDAC_OBJECTS`` extension) and looks it up in the configured healsparse maps, writing one integer column: -``FLAG_EXT`` +``MASK_EXT`` ``0`` for an object no configured map flags, nonzero otherwise. What the nonzero value *is* depends on the maps: a boolean map — which is what the UNIONS per-bit products are, and what the shipped config names — can only @@ -32,22 +48,24 @@ ``< > <= >= == !=`` — no bitwise operators — so any bit selection has to happen here, leaving the config a plain test for zero. -Nothing cuts on it by default -============================= +Off by default, and a no-op when off +==================================== + +``MASK_PATHS`` ships COMMENTED OUT. With no maps configured the module is a +strict no-op: the catalogue is copied through unchanged, with no ``MASK_EXT`` +column at all — the same gating ``make_cat`` gives ``MASK_EXT_PATHS``. It stays +in the ``MODULE`` chain either way, so enabling the query is uncommenting one +line and never editing the chain. -The shipped ``star_selection.setools`` does NOT cut on ``FLAG_EXT``. The PSF -star selection is permissive: it rejects on the instrument flags -(``IMAFLAGS_ISO == 0``) and leans on outlier rejection for the rest. The -column is written for transparency and measurement — so the effect of the -external masks on the star sample can be *measured* before it is imposed — -which is the same principle as ``make_cat``'s unfiltered ``MASK_`` -columns: flag here, cut downstream. +Even with maps configured, nothing cuts on the result. The PSF star selection +deliberately starts from outlier rejection alone, rejecting only on the +instrument flags, and ``MASK_EXT`` is the configurable pickup if that proves +insufficient. Writing the column without cutting on it is what lets the effect +of a mask on the star sample be *measured* before it is imposed. -Feeding the external masks to the star selection is a one-line change if -outlier rejection turns out not to be robust enough: add ``FLAG_EXT == 0`` by -each +That pickup is one line: add ``MASK_EXT == 0`` beside each ``IMAFLAGS_ISO == 0`` in ``star_selection.setools``. The escape hatch is -deliberate, and the file's header says so. +deliberate, and that file's header says so. The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with ``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py index 08e153a00..aff212b5a 100644 --- a/src/shapepipe/modules/mask_query_package/mask_query.py +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -6,6 +6,8 @@ """ +import shutil + import numpy as np from shapepipe.pipeline import file_io @@ -16,9 +18,16 @@ class MaskQuery(object): """Mask Query. Query external healsparse masks at every detection of a SExtractor - catalogue and write the result as a single ``FLAG_EXT`` column into a copy + catalogue and write the result as a single ``MASK_EXT`` column into a copy of that catalogue. + With no maps configured this is a strict no-op: the input catalogue is + copied through unchanged, with no ``MASK_EXT`` column, matching + ``make_cat``'s ``MASK_EXT_PATHS`` contract (absent key, nothing happens). + The copy is what keeps the module in the chain — ``setools`` reads this + module's output, so producing no file would break the chain rather than + disable the query. + Parameters ---------- sexcat_path : str @@ -26,7 +35,7 @@ class MaskQuery(object): output_path : str Path to the output catalogue mask_paths : list - Paths to the healsparse maps to query + Paths to the healsparse maps to query; empty means no-op bits : int, optional Bit mask applied to integer maps; default ``None`` flags any nonzero value @@ -61,6 +70,20 @@ def process(self): Number of flagged objects """ + if not self._mask_paths: + # No maps configured: copy the catalogue through byte-for-byte. + # A rewrite through FITSCatalogue would round-trip the LDAC HDUs + # for no reason; this way an unconfigured run is provably + # identical to its input. + shutil.copyfile(self._sexcat_path, self._output_path) + if self._w_log is not None: + self._w_log.info( + "No MASK_PATHS configured; passing " + + f"{self._sexcat_path} through unchanged, no MASK_EXT" + + " column written" + ) + return 0 + ori_cat = file_io.FITSCatalogue( self._sexcat_path, SEx_catalogue=True, @@ -80,7 +103,7 @@ def process(self): if self._w_log is not None: self._w_log.info( "No detections in " - + f"{self._sexcat_path}; writing an empty FLAG_EXT column" + + f"{self._sexcat_path}; writing an empty MASK_EXT column" ) else: ra = np.copy(data["XWIN_WORLD"]) @@ -103,7 +126,7 @@ def process(self): open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, ) ori_cat.add_col( - "FLAG_EXT", flag, new_cat=True, new_cat_inst=new_cat + "MASK_EXT", flag, new_cat=True, new_cat_inst=new_cat ) ori_cat.close() diff --git a/src/shapepipe/modules/mask_query_runner.py b/src/shapepipe/modules/mask_query_runner.py index fc2a6679a..a636560b2 100644 --- a/src/shapepipe/modules/mask_query_runner.py +++ b/src/shapepipe/modules/mask_query_runner.py @@ -39,14 +39,16 @@ def mask_query_runner( else: prefix = "" - mask_paths = mask_query_util.parse_map_paths( - config.getexpanded(module_config_sec, "MASK_PATHS") - ) - if not mask_paths: - raise ValueError( - f"[{module_config_sec}] MASK_PATHS is empty; the module has" - + " nothing to query." + # Absent (or empty) MASK_PATHS is a no-op, not an error — the same gating + # make_cat gives MASK_EXT_PATHS. The module stays in the MODULE chain and + # passes the catalogue through, so enabling the query is a config edit and + # disabling it never means editing the chain. + if config.has_option(module_config_sec, "MASK_PATHS"): + mask_paths = mask_query_util.parse_map_paths( + config.getexpanded(module_config_sec, "MASK_PATHS") ) + else: + mask_paths = [] # Any nonzero map value flags unless a bit selection is given if config.has_option(module_config_sec, "MASK_BITS"): @@ -67,7 +69,8 @@ def mask_query_runner( ) n_flagged = mq_inst.process() - w_log.info(f"FLAG_EXT nonzero for {n_flagged} objects") + if mask_paths: + w_log.info(f"MASK_EXT nonzero for {n_flagged} objects") # No return objects return None, None diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index cf79b21e0..982bfd3d3 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -12,10 +12,10 @@ * ``make_cat`` writes one ``MASK_`` column per configured map, carrying the map value verbatim (no interpretation, no filtering); -* ``mask_query`` writes a single integer ``FLAG_EXT`` column onto the exposure +* ``mask_query`` writes a single integer ``MASK_EXT`` column onto the exposure SExtractor catalogue, combining the configured maps into "clean (0) or flagged (nonzero)" so that ``setools`` — whose expression language has no - bitwise operators — *could* cut on ``FLAG_EXT == 0``. The shipped selection + bitwise operators — *could* cut on ``MASK_EXT == 0``. The shipped selection does not; the column is carried for measurement (see that module). Partial reads @@ -34,11 +34,13 @@ edge convention we did not think of. ``test_partial_read_matches_full`` is what actually holds the two paths equal. -Measured on the DR6 star map (``mask_ugriz_nside131072_n4.hsp``, 583 MB, -``nside_coverage=128``) for 2000 positions in one CCD-sized box, one process -each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, identical -values. Per 40-CCD exposure that is ~4 s against ~8.5 min of map reading, and -the query adds ~0.15 GB to a rule already asking for 16 GB. +Measured on the DR6 star-body map — the bit-2 rung of the ladder the configs +name ``mask_ugriz_nside131072_n4.hsp``, run against the staged single-band copy +``mask_r_nside131072_n4.hsp``, 583 MB, ``nside_coverage=128`` — for 2000 +positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB +peak RSS, full 12.8 s / 3364 MiB, identical values. Per 40-CCD exposure that +is ~4 s against ~8.5 min of map reading, and the query adds ~0.15 GB to a rule +already asking for 16 GB. Coverage -------- @@ -284,7 +286,7 @@ def flag_positions(paths, ra, dec, bits=None, w_log=None): # and looks exactly like "nothing is masked here". Say it. w_log.warning( f"Mask query {path}: NO object is inside this map's" - + " coverage — the resulting FLAG_EXT contribution is" + + " coverage — the resulting MASK_EXT contribution is" + " zero everywhere because the map does not reach these" + " positions, not because they are clean." ) diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py index 85ce028cd..f52d2e669 100644 --- a/tests/module/test_mask_query.py +++ b/tests/module/test_mask_query.py @@ -3,7 +3,7 @@ Exercises the exposure-side half of the query-everything mask design (PR #847): ``mask_query`` reads each SExtractor detection's windowed world position out of the ``LDAC_OBJECTS`` extension, looks it up in the configured healsparse maps, -and writes a single integer ``FLAG_EXT`` column into a NEW catalogue beside the +and writes a single integer ``MASK_EXT`` column into a NEW catalogue beside the input. What is locked in here: (1) boolean maps flag with ``1``, (2) integer maps @@ -14,6 +14,8 @@ (5) the input catalogue is left untouched while the LDAC structure survives. """ +import pathlib + import numpy as np import numpy.testing as npt import pytest @@ -120,7 +122,7 @@ def _read(path): cat = file_io.FITSCatalogue(str(path), SEx_catalogue=True) cat.open() data = cat.get_data() - flag = np.copy(data["FLAG_EXT"]) + flag = np.copy(data["MASK_EXT"]) names = set(data.dtype.names) cat.close() return flag, names @@ -171,7 +173,7 @@ def test_flag_positions_boolean_map(tmp_path): def test_mask_query_writes_flag_ext(tmp_path): - """The module writes FLAG_EXT into a new catalogue and counts the hits.""" + """The module writes MASK_EXT into a new catalogue and counts the hits.""" map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) in_path = _write_sexcat(tmp_path / "sexcat-000-0.fits") out_path = tmp_path / "sexcat_ext-000-0.fits" @@ -196,7 +198,7 @@ def test_mask_query_writes_flag_ext(tmp_path): # The input is not mutated: this module publishes a new file. with fits.open(in_path) as hdus: - assert "FLAG_EXT" not in hdus[2].data.dtype.names + assert "MASK_EXT" not in hdus[2].data.dtype.names def test_mask_query_bits_and_all_clean(tmp_path): @@ -297,7 +299,7 @@ def test_mask_query_empty_ccd(tmp_path): assert out_path.exists() flag, names = _read(out_path) assert flag.shape == (0,) - assert "FLAG_EXT" in names + assert "MASK_EXT" in names assert any("No detections" in m for m in log.info_msgs) @@ -310,3 +312,37 @@ def test_empty_coverage_raises_clearly(tmp_path): path = _write_map(tmp_path / "empty.hsp", 4, n_covered=0) with pytest.raises(ValueError): mask_query_util.query_map_coverage(path, RA, DEC) + + +def test_mask_query_no_maps_is_noop(tmp_path): + """No MASK_PATHS means a strict pass-through, not a zero column. + + The shipped configs comment MASK_PATHS out, so this is the DEFAULT path. + mask_query stays in the MODULE chain either way — setools reads its output + — so the no-op has to publish a file, and that file must be identical to + its input with no MASK_EXT column at all (the gating make_cat gives + MASK_EXT_PATHS). + """ + in_path = _write_sexcat(tmp_path / "sexcat-000-3.fits") + out_path = tmp_path / "sexcat_ext-000-3.fits" + log = _NullLogger() + + n_flagged = MaskQuery(in_path, str(out_path), [], w_log=log).process() + + assert n_flagged == 0 + assert out_path.exists() + assert out_path.read_bytes() == pathlib.Path(in_path).read_bytes() + with fits.open(str(out_path)) as hdus: + assert "MASK_EXT" not in hdus[2].data.dtype.names + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + assert any("No MASK_PATHS configured" in m for m in log.info_msgs) + + +def test_flag_positions_no_maps(tmp_path): + """The utility with no paths returns an all-zero flag and reads nothing.""" + flag = mask_query_util.flag_positions([], RA, DEC) + npt.assert_array_equal(flag, [0, 0, 0, 0]) diff --git a/workflow/README.md b/workflow/README.md index 4aa24d29b..5ace34019 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -201,11 +201,14 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed healsparse maps and QUERIED once per object: the `mask_query` module writes a - `FLAG_EXT` column onto each CCD's detection catalogue (inside `exp_psf`), and + `MASK_EXT` column onto each CCD's detection catalogue (inside `exp_psf`), and `make_cat` writes one `MASK_` column per band onto the final catalogue - (inside `tile_make_cat`). Neither is cut on in the pipeline: the PSF star - selection rejects on instrument flags alone and everything else is a - downstream decision. Map paths are config, not + (inside `tile_make_cat`). Neither is cut on in the pipeline: instrument flags + mark corrupted measurements and are the only masks that reject anything here, + while the healsparse masks are location flags and every decision about them is + downstream. On exposures the query ships off — `MASK_PATHS` is commented out, + making `mask_query` a no-op pass-through — so turning it on is a config edit, + not a chain edit. Map paths are config, not code, so regenerated products cost a config edit. Nothing is fetched from a catalogue server, staged, or rasterized, which is why the old `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 858e3828f..92b230739 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -144,11 +144,18 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships -# permissive (instrument flags only) and the column is carried for -# transparency and measurement — see star_selection.setools' header for the -# one-line change that would impose it. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +# detection sets MASK_EXT. Comma-separated. +# +# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue +# is passed through with no MASK_EXT column, the same gating make_cat gives +# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning +# the query on is uncommenting one line and never editing the chain. +# +# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive +# (instrument flags only) and the column is carried for transparency and +# measurement — see star_selection.setools' header for the one-line change +# that would impose it. +; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map From d5c73a163a8581baf39fc9b1ff5ccf42b9bb3a6c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 10:55:11 -0400 Subject: [PATCH 088/100] ngmix: clarify position seed rationale --- src/shapepipe/modules/ngmix_package/ngmix.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 5ca65b9bc..1167f3199 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -128,16 +128,9 @@ def get_prior(pixel_scale, rng, T_range=None, F_range=None): def position_seed(ra, dec, ccd): """Deterministic RNG seed from an object's sky position (ngmix#796). - For image-simulation m-bias with the Pujol estimator, the same scene is - simulated under different applied shears ("image branches") and the shear - response is read from the branch difference of the SAME objects. Metacal's - ``fixnoise`` adds a counter-noise realisation drawn from an RNG; if that RNG - is seeded per tile, an object gets *different* added noise in different - branches (detection order differs), and the noise fails to cancel in the - difference, inflating ``sigma_m``. Seeding the per-object RNG from sky - position instead makes the same object draw the same added noise (and the - same fit guesses) in every branch, so both cancel and the m-bias error - shrinks. It also makes the result independent of how the tile is split into + Position seeding gives the same object the same RNG stream in each image + branch, provided its sky position falls in the same seed box. It also makes + the result independent of how the tile is split into ``ID_OBJ_MIN``/``ID_OBJ_MAX`` chunks, which is why it is now the only mode. Box math (kept exactly as Fabian's issue #796):: From 899763aff7f2581c61a5415b27c491fdb5809b68 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 10:56:00 -0400 Subject: [PATCH 089/100] ngmix: remove obsolete seed configuration --- example/cfis/config_tile_Ng_template.ini | 9 --------- .../config_tile_Ng_batch_psfex_sx.ini | 6 ------ src/shapepipe/modules/ngmix_package/__init__.py | 4 +--- src/shapepipe/modules/ngmix_runner.py | 12 ------------ workflow/config/cfis/config_tile_Ng_template.ini | 5 ----- 5 files changed, 1 insertion(+), 35 deletions(-) diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini index bd2af3ea1..d928cc1be 100644 --- a/example/cfis/config_tile_Ng_template.ini +++ b/example/cfis/config_tile_Ng_template.ini @@ -100,15 +100,6 @@ CENTROID_SOURCE = wcs # the pure Sheldon UberSeg mask. # DILATE_NEIGHBOUR = 1 -# SEED_FROM_POSITION (optional): FALSE (default) seeds one RNG per tile. -# TRUE seeds a per-object RNG from the object's sky position instead, so that -# metacal's fixnoise counter-noise and the fit guesses are identical for the -# same object across Pujol image-simulation shear branches and cancel in the -# branch difference — shrinking the m-bias error. This is FOR IMAGE -# SIMULATIONS ONLY (Pujol noise cancellation, ngmix#796); leave it off (or -# omit) for real data, where it has no benefit. -# SEED_FROM_POSITION = FALSE - # METACAL_PSF (optional): the metacal reconvolution-kernel scheme # (metacal_pars['psf']). "fitgauss" (default) fits a Gaussian to the PSF and # rounds it; "gauss" reconvolves with a fixed round Gaussian sized from the diff --git a/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini index 8ca298565..3479a972a 100644 --- a/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini +++ b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini @@ -72,11 +72,5 @@ BKG_SUB = False SAVE_BATCH = 1000 -# Position-seeded per-object fixnoise RNG (#796/#803): the SKiLLS shear -# branches share their sky-noise realization, so seeding the metacal fixnoise -# from sky position makes the added noise identical across branches too and -# the Pujol estimator (Pujol, Kilbinger, Sureau & Bobin 2018, 621, A2) cancels it. -SEED_FROM_POSITION = True - ID_OBJ_MIN = -1 ID_OBJ_MAX = -1 diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index ef05bed76..66323822c 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -58,9 +58,7 @@ ======================== Each object gets its own random number stream, seeded from its sky position -and CCD (``ngmix.position_seed``, which argues what that buys). It is the only -mode: the config option ``SEED_FROM_POSITION`` is obsolete, and setting it to -``False`` raises rather than silently changing the RNG. +and CCD (``ngmix.position_seed``). """ diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 072288e00..345a5a43a 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -142,18 +142,6 @@ def ngmix_runner( else: dilate_neighbour = 1 - # Position-seeded RNG is the only mode (see ngmix.position_seed). Old - # configs that disable it must fail loudly, not silently change the RNG. - if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): - raise ValueError( - "SEED_FROM_POSITION = False is no longer supported: the" - " tile-seeded RNG mode has been retired because it makes" - " results depend on the object chunking. Remove the" - " SEED_FROM_POSITION entry from the ngmix config section" - " (position-seeded RNG is now the only mode)." - ) - # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index f0e374b08..dff44a3ca 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -117,11 +117,6 @@ MAG_ZP = 30.0 # Pixel scale in arcsec PIXEL_SCALE = 0.186 -# SEED_FROM_POSITION: per-object RNG seeded from sky position (ra, dec, ccd) -# instead of one ordered per-tile stream, so results are bit-identical under -# any chunking (D4/#796). Required for the chunked ngmix scatter/gather. -SEED_FROM_POSITION = True - # ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, # computed at execution time from the tile's own object count and expanded # via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not From 6890d8a3fecc30f9ee22318b15875965c09c034b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 10:56:38 -0400 Subject: [PATCH 090/100] workflow: rename tile merge catalogue config --- scripts/sh/job_sp_canfar.bash | 4 ++-- .../cfis/{config_merge_sep_cats.ini => config_tile_Ms.ini} | 0 workflow/rules/tile.smk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename workflow/config/cfis/{config_merge_sep_cats.ini => config_tile_Ms.ini} (100%) diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash index c99435823..e64ed1e5f 100755 --- a/scripts/sh/job_sp_canfar.bash +++ b/scripts/sh/job_sp_canfar.bash @@ -564,11 +564,11 @@ if [[ $do_job != 0 ]]; then cat $SP_CONFIG/config_merge_sep_cats_template.ini | \ perl -ane \ 's/(N_SPLIT_MAX =) X/$1 '$nsh_jobs'/; print' \ - > $SP_CONFIG_MOD/config_merge_sep_cats.ini + > $SP_CONFIG_MOD/config_tile_Ms.ini ### Merge separated shapes catalogues command_sp \ - "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" \ + "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ms.ini" \ "Run shapepipe (tile: merge sep cats)" \ "$VERBOSE" \ "$ID" diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_tile_Ms.ini similarity index 100% rename from workflow/config/cfis/config_merge_sep_cats.ini rename to workflow/config/cfis/config_tile_Ms.ini diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 74d985e2a..6a7974a0c 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -771,7 +771,7 @@ rule tile_merge_cats: runtime = 10, slurm_extra = TILE_SLURM_EXTRA shell: - sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") + sp_shell("tile_merge_cats", "config_tile_Ms.ini") # The run's science product. make_cat also reads the vignette store's # psfex_interp output, so it — not ngmix — is the store's last reader. From d82f214e741ff204e0917581c5fd38e9c60df3d8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 10:57:21 -0400 Subject: [PATCH 091/100] workflow: standardize tile catalogue run names workflow: update remaining tile run references --- docs/source/pipeline_canfar.md | 2 +- docs/source/pipeline_tutorial.md | 4 ++-- example/cfis/config_Ms_psfex.ini | 2 +- example/cfis/config_Ms_psfex_conv.ini | 2 +- example/cfis/config_make_cat_mccd.ini | 2 +- example/cfis/config_make_cat_psfex.ini | 2 +- example/cfis/config_make_cat_psfex_nosm.ini | 2 +- example/cfis/config_merge_sep_cats_template.ini | 2 +- scripts/python/create_final_cat.py | 2 +- scripts/sh/combine_runs.bash | 4 ++-- scripts/sh/init_run_exclusive_canfar.sh | 4 ++-- src/shapepipe/utilities/summary_params_pre_v2.py | 4 ++-- workflow/config/cfis/config_tile_Mc.ini | 4 ++-- workflow/config/cfis/config_tile_Ms.ini | 2 +- workflow/rules/tile.smk | 2 +- workflow/scripts/clean_tile.py | 4 ++-- workflow/scripts/completeness.py | 4 ++-- workflow/scripts/ngmix_range.py | 2 +- 18 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 96add5758..f8d535984 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -400,7 +400,7 @@ Rename to general PSF and star catalogue used for all ("a") sub-versions: ```bash -cp output/run_sp_Ms/merge_starcat_runner/output/full_starcat-0000000.fits \ +cp output/run_sp_tile_Ms/merge_starcat_runner/output/full_starcat-0000000.fits \ unions_shapepipe_psf_2024_v1.6.a.fits ``` diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 95a0aa94a..991f9bf04 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -328,8 +328,8 @@ Included are galaxy detection and basic measurement parameters, the PSF model at galaxy positions, the spread-model classification, and the shape measurement. Two output directories are created. -The first one is `run_sp_Ms` for the `merge_sep` run. -The second is `run_sp_Mc` for the `make_cat` task; the name is the same for both the `MCCD` and `PSFEx` PSF model. +The first one is `run_sp_tile_Ms` for the `merge_sep` run. +The second is `run_sp_tile_Mc` for the `make_cat` task; the name is the same for both the `MCCD` and `PSFEx` PSF model. ## Upload results diff --git a/example/cfis/config_Ms_psfex.ini b/example/cfis/config_Ms_psfex.ini index 5698ee6dd..a27b43920 100644 --- a/example/cfis/config_Ms_psfex.ini +++ b/example/cfis/config_Ms_psfex.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: False RUN_DATETIME = False diff --git a/example/cfis/config_Ms_psfex_conv.ini b/example/cfis/config_Ms_psfex_conv.ini index 3a68acc41..9d0b97207 100644 --- a/example/cfis/config_Ms_psfex_conv.ini +++ b/example/cfis/config_Ms_psfex_conv.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: False RUN_DATETIME = False diff --git a/example/cfis/config_make_cat_mccd.ini b/example/cfis/config_make_cat_mccd.ini index 858341e1f..860f2020a 100644 --- a/example/cfis/config_make_cat_mccd.ini +++ b/example/cfis/config_make_cat_mccd.ini @@ -8,7 +8,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc +RUN_NAME = run_sp_tile_Mc # Add date and time to RUN_NAME, optional, default: True ; RUN_DATETIME = False diff --git a/example/cfis/config_make_cat_psfex.ini b/example/cfis/config_make_cat_psfex.ini index a7407d990..823cb8938 100644 --- a/example/cfis/config_make_cat_psfex.ini +++ b/example/cfis/config_make_cat_psfex.ini @@ -8,7 +8,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc +RUN_NAME = run_sp_tile_Mc # Add date and time to RUN_NAME, optional, default: True ; RUN_DATETIME = False diff --git a/example/cfis/config_make_cat_psfex_nosm.ini b/example/cfis/config_make_cat_psfex_nosm.ini index 1983f91c7..50f5c0612 100644 --- a/example/cfis/config_make_cat_psfex_nosm.ini +++ b/example/cfis/config_make_cat_psfex_nosm.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc +RUN_NAME = run_sp_tile_Mc # Add date and time to RUN_NAME, optional, default: True ; RUN_DATETIME = False diff --git a/example/cfis/config_merge_sep_cats_template.ini b/example/cfis/config_merge_sep_cats_template.ini index 36b99e8da..96c4557f5 100644 --- a/example/cfis/config_merge_sep_cats_template.ini +++ b/example/cfis/config_merge_sep_cats_template.ini @@ -8,7 +8,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: True ; RUN_DATETIME = False diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 75e3a67c0..24ebd6903 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -394,7 +394,7 @@ def process(params): run_prefix = "run_sp_tile_Mc_*" else: patch_name = rf"P{params['patch']}" - run_prefix = "run_sp_Mc_*" + run_prefix = "run_sp_tile_Mc_*" patch_pattern = re.compile(patch_name) diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash index 8cbf9bbdf..91b2ca42a 100755 --- a/scripts/sh/combine_runs.bash +++ b/scripts/sh/combine_runs.bash @@ -104,9 +104,9 @@ run_out="run_sp_combined_$cat" if [ "$cat" == "final" ]; then # v1 - #run_in="$pwd/$out_base/run_sp_Mc_*" + #run_in="$pwd/$out_base/run_sp_tile_Mc_*" # v2 - run_in="$pwd/tile_runs/*/$out_base/run_sp_Mc_*" + run_in="$pwd/tile_runs/*/$out_base/run_sp_tile_Mc_*" module="make_catalog_runner" pattern="final_cat-*" diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh index 27efadc11..556153b0d 100755 --- a/scripts/sh/init_run_exclusive_canfar.sh +++ b/scripts/sh/init_run_exclusive_canfar.sh @@ -574,7 +574,7 @@ fi if [[ $do_job != 0 ]]; then # Remove previous runs of this job - rm -rf run_sp_Ms_20??-* + rm -rf run_sp_tile_Ms_20??-* fi @@ -582,7 +582,7 @@ fi if [[ $do_job != 0 ]]; then # Remove previous runs of this job - rm -rf run_sp_Mc_20??-* + rm -rf run_sp_tile_Mc_20??-* fi diff --git a/src/shapepipe/utilities/summary_params_pre_v2.py b/src/shapepipe/utilities/summary_params_pre_v2.py index 2def4d33b..ea67442e3 100644 --- a/src/shapepipe/utilities/summary_params_pre_v2.py +++ b/src/shapepipe/utilities/summary_params_pre_v2.py @@ -228,7 +228,7 @@ def set_jobs_v2_pre_v2(patch, verbose): jobs["256"] = summary.job_data( "256", - "run_sp_Ms", + "run_sp_tile_Ms", ["merge_sep_cats_runner"] * 2, "tile_IDs", path_main=path_main, @@ -241,7 +241,7 @@ def set_jobs_v2_pre_v2(patch, verbose): jobs["512"] = summary.job_data( "512", - ["run_sp_Mc"], + ["run_sp_tile_Mc"], ["make_cat_runner"], "tile_IDs", path_main=path_main, diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 55daef12e..4bcc6d6bf 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc +RUN_NAME = run_sp_tile_Mc # Add date and time to RUN_NAME, optional, default: True RUN_DATETIME = False @@ -60,7 +60,7 @@ TIMEOUT = 96:00:00 # tile's fused group job wrote to node-local storage (see config_tile_PiViVi.ini # and workflow/rules/tile.smk). $NGMIX_VIGNET_DIR names it; the other two # inputs and this run's own output stay on $SP_RUN. -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_Ms/merge_sep_cats_runner/output # Input file pattern(s), list of strings with length matching number of expected input file types # Cannot contain wild cards diff --git a/workflow/config/cfis/config_tile_Ms.ini b/workflow/config/cfis/config_tile_Ms.ini index 9dda5cf0f..08397399a 100644 --- a/workflow/config/cfis/config_tile_Ms.ini +++ b/workflow/config/cfis/config_tile_Ms.ini @@ -8,7 +8,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: True RUN_DATETIME = False diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 6a7974a0c..d67126242 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -808,7 +808,7 @@ rule tile_make_cat: # publishes a catalogue for a manifest snakemake is about to delete. sp_shell("tile_make_cat", "config_tile_Mc.ini", post="if [ $rc -eq 0 ]; then\n" - ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner' + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_tile_Mc/make_cat_runner' '/output/final_cat*.fits | head -1)" {output.final_cat}\n' "fi\n") diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py index 85686df69..0f385a91d 100644 --- a/workflow/scripts/clean_tile.py +++ b/workflow/scripts/clean_tile.py @@ -12,8 +12,8 @@ WHAT IT DELETES: the tile's whole ``/tiles///`` directory. Measured on 186.307 (smk-g4, a finished 34-tile-campaign tile): 1,279,231,196 bytes across 137 inodes (71 regular files, 9 symlinks, 57 directories) — -``output/run_sp_tile_Sx`` 745 MB, ``run_sp_tile_Uz`` 382 MB, ``run_sp_Mc`` -46 MB, ``run_sp_Ms`` 39 MB, ``run_sp_tile_Mh_exp`` 11 MB, and a non-``output/`` +``output/run_sp_tile_Sx`` 745 MB, ``run_sp_tile_Uz`` 382 MB, ``run_sp_tile_Mc`` +46 MB, ``run_sp_tile_Ms`` 39 MB, ``run_sp_tile_Mh_exp`` 11 MB, and a non-``output/`` remainder of 62 inodes totalling 15,726 bytes. Deleting only ``output/`` is NOT enough: 62 x 23,114 DR6 tiles is 1.43M inodes against a 1M quota, so the inode bound binds on its own and the directory has to go as a whole. diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index cf67363db..334f6043e 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -180,8 +180,8 @@ def check_floor(stage, run_dir): "tile_detect": ("tile", "run_sp_tile_Sx"), "tile_vignets": ("tile", "run_sp_tile_PiViVi"), "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), - "tile_merge_cats": ("tile", "run_sp_Ms"), - "tile_make_cat": ("tile", "run_sp_Mc"), + "tile_merge_cats": ("tile", "run_sp_tile_Ms"), + "tile_make_cat": ("tile", "run_sp_tile_Mc"), } diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index a6879c504..4aafb266d 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -193,7 +193,7 @@ def object_epochs(run_dir: Path): order and ``CCD_N < 0`` where the object misses that exposure. Summing ``CCD_N >= 0`` across them reproduces the final catalogue's ``N_EPOCH`` column exactly — checked row by row against 186.307's - ``run_sp_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, + ``run_sp_tile_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, 116,727 pairs, mean 3.31. The post-process is upstream of the whole tile_shape group, so the extensions always exist by the time ngmix runs; their absence is a broken tile, not a case to accommodate. From cbf7e11ab12b382efc08bb346e2303c9b27ef003 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 10:59:14 -0400 Subject: [PATCH 092/100] workflow: simplify explanatory prose workflow: simplify explanatory prose --- workflow/README.md | 8 ++++---- workflow/Snakefile | 33 ++++++++++++++++---------------- workflow/rules/exposure.smk | 14 ++++++-------- workflow/rules/tile.smk | 2 +- workflow/scripts/clean_tile.py | 24 +++++++++++------------ workflow/scripts/completeness.py | 6 +++--- workflow/scripts/container.py | 8 ++++---- workflow/scripts/ngmix_range.py | 17 ++++++++-------- workflow/scripts/run_report.py | 2 +- workflow/scripts/star_cats.py | 2 +- 10 files changed, 56 insertions(+), 60 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 08bfd2291..d5908f03c 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -6,7 +6,7 @@ bash layers and the per-site sbatch reimplementations. **Module code is untouched**: rules call `shapepipe_run -c ` on the existing config chains. Design and rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) -(the living PRD). +(the design document). Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is unsupported: `sp` sets the state directory, the SLURM profile, and @@ -40,7 +40,7 @@ v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became plugins, and full `rerun-triggers` became the default. Anything other than `run`, `report`, `container`, `cancel` passes straight through to -snakemake with the workflow's profile and state dir — the escape hatch for +snakemake with the workflow's profile and state dir — the direct command path for `sp --unlock`, `sp --dag`, `sp exp_psf ...`. ## The container image @@ -49,7 +49,7 @@ snakemake with the workflow's profile and state dir — the escape hatch for only exists if you ask for one: * your **cached SIF** (`~/.cache/shapepipe/shapepipe.sif`, `SP_CACHE_DIR` or - `SP_CONTAINER` to move it) — a pristine pull of the published image, private + `SP_CONTAINER` to move it) — a read-only pull of the published image, private to you, so nobody else's refresh moves the ground under your running jobs; * an optional **sandbox** (`~/.cache/shapepipe/sandbox/`, `SP_SANDBOX`) — the same image unpacked writable, so a `pip install` into it sticks. The escape @@ -114,7 +114,7 @@ Snakefile and read `workflow/scripts/*` and the ini chain hours after launch. takes effect on the next `sp run`.** Everything workflow-internal hangs off `workflow.basedir`, which *is* the -snapshot, so it follows for free. The one exception is the profile's `PYTHONPATH` +snapshot, including the profile's `PYTHONPATH` pin, which YAML cannot interpolate: `sp run` rewrites that single path in the snapshot's copy of the profile and launches `--profile` at the copy. The snapshot is refreshed wholesale on every `sp run` — a new `sp run` *is* the relaunch — and diff --git a/workflow/Snakefile b/workflow/Snakefile index b925adc06..11810c421 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,6 +1,6 @@ """ShapePipe real-data orchestration — Snakemake workflow. -Design / rationale: CosmoStat/shapepipe#848 (the living PRD), D1-D5. +Design / rationale: CosmoStat/shapepipe#848, D1-D5. `sp run` is TWO snakemake invocations over this one Snakefile: @@ -72,8 +72,8 @@ if _kind == "none": container: _image # --- paths ----------------------------------------------------------------- -# TWO ROOTS (D5). RUN_DIR is the scratch root: bulk intermediates, sized so a -# batch finishes inside the purge window. PRODUCTS_DIR is the persistent root +# RUN_DIR is the scratch root: bulk intermediates, sized so a batch finishes +# inside the purge window. PRODUCTS_DIR is the persistent root # for the durable, low-volume products — the final catalogues, the index, the # report. Snakemake's own state is the one durable-looking thing that stays on # scratch (bin/sp explains why: tens of thousands of small, hot metadata writes @@ -88,9 +88,9 @@ PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" -# The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the -# configs and the rules that set the env vars they interpolate are one artefact -# and must version together. Hence no `config_src` knob. +# The config chain is the repo's committed directory (D2). The configs and +# rules that set their environment variables must be versioned together. There +# is no `config_src` knob. CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" sys.path.insert(0, str(SCRIPTS)) @@ -98,7 +98,7 @@ import build_index # noqa: E402 from completeness import STAGE_DIR # noqa: E402 # SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two -# invocations of `sp run`, `passthrough` on the escape hatch (`sp --unlock`, `sp +# invocations of `sp run`, `passthrough` on direct commands (`sp --unlock`, `sp # --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index # build and the report hooks — so that a passthrough parse never mutates durable # state or dies on a threshold it was not asked about. @@ -139,7 +139,7 @@ NGMIX_CHUNKS = workflow._scatter["ngmix"] # there — no "some Fe output exists" guard. That guard used to make a # totally-failed prepare produce an empty index, an empty DAG and a green exit 0; # with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the -# SP_MISSING_THRESHOLD gate and fails loudly, as it should. +# SP_MISSING_THRESHOLD gate. # # In every other phase (prepare, or unset for a passthrough invocation) the parse # builds NOTHING and only loads whatever index is already on disk. @@ -148,8 +148,8 @@ NGMIX_CHUNKS = workflow._scatter["ngmix"] # snakemake inside every job, that re-invocation parses this file again, and it # inherits SP_PHASE=compute from the submitting environment. Without the guard, # every one of the run's jobs re-runs the build — hundreds of concurrent sqlite -# writers on Lustre, which is exactly the "database is locked" storm that killed -# the first real run (2026-07-31). Job parses only LOAD the index below. +# writers on Lustre, which previously caused "database is locked" errors +# (2026-07-31). Job parses only LOAD the index below. if PHASE == "compute" and workflow.is_main_process: build_index.build( TILES, RUN_DIR, INDEX_DB, @@ -331,8 +331,8 @@ STAR_CAT_HASH = script_hash("star_cats.py") # TILE_NGMIX_RANGES, which also keeps the incident transcript). So this hash buys # REPRODUCIBILITY of the file across attempts and resumes, not agreement between # siblings. It therefore fingerprints the script that PRODUCES the file, and -# rides on tile_vignets as well as tile_ngmix — both rules, deliberately; the -# one-rule version of this param is the dangerous one, argued at tile.smk. +# is used by both tile_vignets and tile_ngmix because either rule can produce +# the file. # # Land this hash, and any later edit to the split, at a campaign boundary on a # fresh root: a resume across it re-measures the tile, and the failure modes a @@ -404,13 +404,13 @@ def clean_targets(): run's scope or has already produced its vignets on disk. Without that test, requesting a tombstone for an exposure shared with a LATER batch would drag that batch's whole tile chain into this DAG through the clean rule's input — - scope expansion by cleanup, which is not a trade anyone asked for. Ineligible + cleanup would otherwise expand the scope. Ineligible exposures are simply skipped; the invocation that finishes their last consumer picks them up. Deferral, never loss. Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in `clean_ignore_tiles` neither gates eligibility nor appears in the job's - input — which is the whole point of that list. + input — which is the purpose of that list. HEAD PROCESS ONLY. This feeds `rule all` at module level, so it runs on every parse — including the ~800 per-job re-parses under the slurm executor, none of @@ -426,8 +426,7 @@ def clean_targets(): continue # An empty set AFTER filtering means every consumer is ignored: nothing # is left that could ever read this exposure, so it is eligible now. - # That is the whole point of clean_ignore_tiles — all() of an empty set - # is True, and it is true here in the intended sense. + # all() of an empty set is True, so this case is eligible. tiles = clean_consumers(exp) if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() for t in tiles): @@ -435,7 +434,7 @@ def clean_targets(): return sorted(out) # --- tile reclamation (D5) -------------------------------------------------- -# A SEPARATE FLAG from `clean:`, deliberately (config.yaml carries the full +# A separate flag from `clean:` (config.yaml carries the full # argument): exposure reclamation costs nothing but a rebuild if a tile is # appended later, while tile reclamation destroys the per-tile AUDIT TRAIL that # the campaign's cost model was derived from. They are not the same decision and diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 9fe3e7b29..8ddbbe193 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -107,15 +107,13 @@ def in_container(cmd, *, network=False): return f"apptainer exec {' '.join(args)} '{_image}' {cmd}" -# The campaign's star catalogue: a first-class durable science product, keyed by -# sky rather than by run. Chunk-need is recomputed from the tile list on every -# run and only the missing chunks are fetched, so appending tiles costs exactly -# the chunks they add. +# The campaign's star catalogue is keyed by sky position rather than by run. +# Chunk needs are recomputed from the tile list on every run, and only missing +# chunks are fetched. Appending tiles therefore fetches only their chunks. # -# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the -# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the -# per-exposure rule reached by accident through --local-cores, now an explicit -# number that does not scale with the head node's CPU count. +# A LOCALRULE (declared in the Snakefile) runs the network I/O in one job. The +# fetch loop uses a four-wide thread pool, independent of the head node's CPU +# count. # # `tile_list_hash` is what makes the incremental behaviour visible to the DAG. # The tile list is parse-time config, not a rule input (and the profile drops the diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d67126242..debe148ff 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -646,7 +646,7 @@ rule tile_ngmix: # how reclamation reads everywhere else in this file. # # So: land this hash, and every later edit to ngmix_range.py, at a - # campaign boundary on a fresh root. Same rule and same escape hatch as + # campaign boundary on a fresh root. The same rule and direct command as # tile_local()'s -- `--rerun-triggers mtime code software-env`. range_hash = NGMIX_RANGE_HASH # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py index 0f385a91d..9e7b1e7b0 100644 --- a/workflow/scripts/clean_tile.py +++ b/workflow/scripts/clean_tile.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 -"""Reclaim ONE finished tile's scratch store and leave a tombstone (PRD #848 D5). +"""Reclaim one finished tile's scratch store and leave a tombstone (PRD #848 D5). Run as the shell of the in-DAG ``clean_tile`` rule, never by hand: the rule's ``input:`` is the tile's ``final_cat`` on the PERSISTENT root, so by the time -this executes the tile has published the only thing the campaign wanted from it. +this executes the tile has published its final catalogue. A tile's scratch store has no reader outside that tile — tiles read exposures, nothing reads another tile's store — so unlike the exposure case there is no consumer set to close over and no eligibility test to make. Writer, then cleaner, and the DAG edge is the whole ordering argument. -WHAT IT DELETES: the tile's whole ``/tiles///`` directory. +What it deletes: the tile's whole ``/tiles///`` directory. Measured on 186.307 (smk-g4, a finished 34-tile-campaign tile): 1,279,231,196 bytes across 137 inodes (71 regular files, 9 symlinks, 57 directories) — ``output/run_sp_tile_Sx`` 745 MB, ``run_sp_tile_Uz`` 382 MB, ``run_sp_tile_Mc`` @@ -18,11 +18,11 @@ enough: 62 x 23,114 DR6 tiles is 1.43M inodes against a 1M quota, so the inode bound binds on its own and the directory has to go as a whole. -THE FOUR SURVIVORS, AND WHY EACH ONE IS NOT RESIDUE ---------------------------------------------------- -Everything kept here is currency some OTHER mechanism owns and re-reads long +The four retained paths +------------------------ +The retained paths are used by other mechanisms after the tile is complete. after this tile is done. Nothing is kept for tidiness, and the three that -already exist are a CONTRACT: ``require_survivors`` checks all of them BEFORE +already exist are required: ``require_survivors`` checks all of them before anything is deleted, because a silent drift in one of these paths would not show up as a broken clean — it would show up much later as a campaign that cannot resume. The fourth, ``cleaned.json``, is this job's own output and is the one @@ -72,7 +72,7 @@ file), i.e. ~231k inodes at DR6 against the 1M scratch quota — versus 3.2M if nothing were reclaimed and 1.4M if only ``output/`` were. -WHAT IS LOST, STATED PLAINLY. The per-tile audit trail, for BOTH tools that +What is lost: the per-tile audit trail for both tools that read it — ``sp_tilecost.py`` and ``sp_costmodel.py``. They attribute the fused ``tile_shape`` group job's cost per tile by reading the tile's SExtractor catalogue (NAXIS2 of the sexcat = the object count, the cost model's independent @@ -90,8 +90,8 @@ benchmark TSVs are gone. Until it does, per-chunk cost attribution stops at the first reclaimed tile even though the numbers are still on disk. -DELETION IS SYMLINK-SAFE (``clean_exposure`` gives the general reason), and here -that is not a nicety. A finished tile holds NINE symlinks in TWO classes, and +Deletion is symlink-safe (``clean_exposure`` gives the general reason). A +finished tile holds nine symlinks in two classes, and the second is the one that matters: * ``exp_forest///output`` — 7 links into the EXPOSURE stores, @@ -113,13 +113,13 @@ next should know that the worst case is not a scratch store they could rebuild — it is a rmtree walking into half a terabyte of shared, backed-up survey data. -LOGS ARE DELETED, NOT ABSORBED, as in ``clean_exposure``, and here the +Logs are deleted, not absorbed, as in ``clean_exposure``. Here the duplication is exact: on a finished tile every ``logs/.json`` is BYTE-IDENTICAL to the ``manifests/.json`` beside it (verified across all 16 stage records of 186.307), because a tile with a failed stage has no final_cat and so is never cleaned. -ORDER: tombstone FIRST, then deletion — ``clean_exposure``'s docstring argues +Order: write the tombstone first, then delete — ``clean_exposure``'s docstring the crash window. There is no ``consumers`` field and no consumer-set staleness to detect, because diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 334f6043e..2053881eb 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -8,7 +8,7 @@ its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) sits between ``floor`` and ``expect`` and is tolerated. -This file is also the ``check`` CLI — the second half of every rule's shell line +This file is also the ``check`` CLI, the second half of every rule's shell line (PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing onto it, so the check runs — and the verdict is recorded — even when ``shapepipe_run`` failed:: @@ -23,7 +23,7 @@ the counts and shapepipe_run's own exit status, because a runner can raise after the counted ones have written their files. -WHERE it writes the verdict is the whole point, and it is two files with two +The verdict is written to two files with different different jobs: * the LOG (``--log``, the rule's snakemake ``log:``) gets the full verdict on @@ -128,7 +128,7 @@ def count_products(run_dir, runner, spec): try: with os.scandir(out) as entries: for e in entries: - # A dead symlink is the one thing that must not count (the bash + # A dead symlink must not count (the bash # `ls | wc -l` semantics this ports counted live files only), and # a symlink is the only entry that can be dead — so it is the # only one worth a follow-stat. diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py index bbd7e0bb9..afc7019ed 100644 --- a/workflow/scripts/container.py +++ b/workflow/scripts/container.py @@ -3,13 +3,13 @@ Two layers, and the second only exists when you ask for one: -* the **SIF** (``~/.cache/shapepipe/shapepipe.sif``) -- a pristine, read-only +* the **SIF** (``~/.cache/shapepipe/shapepipe.sif``) -- a read-only copy of the published image, pulled into your own cache. Per-user by construction: one file, one owner, nobody else's refresh moves the ground under a running job. * an optional **sandbox** (``~/.cache/shapepipe/sandbox/``) -- the same image unpacked into a writable directory, so a ``pip install`` inside it sticks. - The escape hatch for work that needs a package the image does not carry yet. + The direct path for work that needs a package the image does not carry yet. Resolution order, shared by this CLI and by the workflow: **sandbox if it exists, else the cached SIF if it exists, else the ``container:`` path in @@ -75,7 +75,7 @@ class ContainerError(Exception): or Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "shapepipe" ).expanduser() -# This user's pristine image. Override with ``SP_CONTAINER`` (absolute path). +# This user's read-only image. Override with ``SP_CONTAINER`` (absolute path). DEFAULT_SIF = CACHE_DIR / "shapepipe.sif" # The optional writable unpacking of it. Override with ``SP_SANDBOX``. @@ -257,7 +257,7 @@ def cmd_pull(args): def cmd_sandbox(args): - """Unpack the image into a writable directory -- the opt-in escape hatch.""" + """Unpack the image into a writable directory -- the opt-in direct path.""" _require_apptainer() sandbox = local_sandbox() if sandbox.exists() and not args.force: diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index 4aafb266d..aa745c2ce 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -36,18 +36,17 @@ (object, epoch) pairs, not objects: the campaign measured 0.2714 CPU-s per pair with an intercept consistent with zero, plus ~0.05 CPU-s per-object of setup. The old equal-count split therefore produced chunks whose cost varied -1.6x WITHIN a single tile, tracking their epoch counts at R^2 0.91-1.000. Those +1.6x within a single tile, tracking their epoch counts at R^2 0.91-1.000. Those chunks are siblings in the fused tile_shape group job, which ends when its -SLOWEST member does, so the spread was pure wall clock: 7.7 hours over the -34-tile smk-g4 campaign, and on the worst tile (196.307) the slowest chunk ran +slowest member does, so the spread was pure wall clock: 7.7 hours over the +34-tile smk-g4 campaign, and on tile 196.307 the slowest chunk ran 26.3 min past the median and took the group to 97.4% of its wall limit. Re-splitting all 34 of that campaign's sexcats: the old boundaries leave the -slowest chunk at 1.131x-1.627x its tile's median predicted cost (worst -200.302), the new ones at 1.0000x-1.0002x, and the sum over tiles of +slowest chunk at 1.131x-1.627x its tile's median predicted cost on tile 200.302; the new ones at 1.0000x-1.0002x, and the sum over tiles of slowest-chunk cost falls 132,875 -> 113,710 predicted CPU-s, 14.4%. -Moving the boundaries is scientifically free: ngmix's RNG is seeded per object +Moving the boundaries does not change measurements: ngmix's RNG is seeded per object from its sky position, so which chunk an object falls in cannot change its measurement (see ``ngmix_package.ngmix.position_seed``). @@ -94,7 +93,7 @@ # both terms of the measured cost law: 0.05 CPU-s of setup / 0.2714 CPU-s per # (object, epoch) = 0.184 epoch-equivalents. It is what keeps the zero-epoch # objects (13 of 35,298 on tile 186.307) from weighing nothing — they still -# cost ~6% of a typical object, and a chunk handed thousands of them for free +# cost ~6% of a typical object, and a chunk handed thousands of them at no extra cost # would be a straggler of a new kind. Only the RATIO of the two costs moves a # boundary, which makes this robust: refitting on 186.307's eight measured # chunk CPU times with the setup term held fixed gives 0.2619 per GEOMETRIC @@ -111,7 +110,7 @@ def id_ranges(epochs, n_chunks: int) -> list[tuple[int, int]]: CONTIGUOUS — ``NGMIX_ID_MIN``/``NGMIX_ID_MAX`` is an interval, not a set — and tile ``[1, n_obj]`` exactly, so every object is measured once. - The objective is the SLOWEST chunk, not the average one, because the + The objective is the slowest chunk, not the average one, because the group job waits for it. So this minimises the maximum chunk weight exactly: binary-search the smallest feasible capacity, then fill left to right under it. Ties in that maximum break toward the earlier chunks, @@ -206,7 +205,7 @@ def object_epochs(run_dir: Path): law is 0.2681 CPU-s per pair at R^2 0.997, against 0.2619 at R^2 0.973 on the geometric count. The better number is unavailable before ngmix runs and is not worth wanting anyway: splitting on it moves boundaries by up to 177 - objects and improves the true slowest chunk by 1.07%. + objects and improves the slowest chunk by 1.07%. Only the EPOCH extensions are read. ``LDAC_OBJECTS`` carries a 10 kB VIGNET per row (376 MB on 186.307) and pulling it in would cost more than diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index f559e3276..0149f6bc9 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -63,7 +63,7 @@ # The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names # the mechanism that owns each). Their presence is therefore NOT evidence that a -# tile's chain was rebuilt, which is the one thing absorb_tombstones has to know +# tile's chain was rebuilt, which absorb_tombstones needs to know # to read a reclaimed tile's record out of its tombstone. SURVIVING_TILE_STAGES = frozenset({"tile_vignets", "tile_find_exposures"}) diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py index 2407af71b..0c8ea5ce8 100644 --- a/workflow/scripts/star_cats.py +++ b/workflow/scripts/star_cats.py @@ -34,7 +34,7 @@ selection the old one-query-per-exposure cone did. Geometry, and why the fetch pad is what it is. Chunk-need is computed from the -TILE list rather than from exposure pointings, because tile IDs are the one thing +TILE list rather than from exposure pointings, because tile IDs are the relevant identifier known before any download: a pointing center means reading a FITS header of an image get_images has not fetched yet, and the DAG needs the chunk set at parse time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so From 4ceb91afce91e02ef138b5308837e71b644f7b04 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 11:02:05 -0400 Subject: [PATCH 093/100] tests: mark UNIONS-specific tile pruning test --- pyproject.toml | 1 + tests/README.md | 1 + tests/unit/test_clean_tile_prune.py | 3 +++ 3 files changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8b259deec..67a0a3c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,4 +138,5 @@ testpaths = ["tests"] markers = [ "slow: heavy compute (minutes); excluded from the fast inner loop.", "candide: needs the candide cluster and/or its real data; auto-skipped elsewhere.", + "unions: uses UNIONS-specific survey layout or data.", ] diff --git a/tests/README.md b/tests/README.md index 82bd066d7..0b0df1524 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,6 +30,7 @@ candide needs the candide cluster and/or its real data; auto-skipped elsewhere ``` `--strict-markers` is on, so a typo'd marker is an error, not a silent no-op. +Use `pytest -m "not unions"` to run the survey-generic tests. A `candide`-marked test is **collected everywhere** (so `--collect-only` shows it exists) but **skipped off-cluster** with a clear reason. Candide is detected diff --git a/tests/unit/test_clean_tile_prune.py b/tests/unit/test_clean_tile_prune.py index 9830ff2d9..d7f3e092d 100644 --- a/tests/unit/test_clean_tile_prune.py +++ b/tests/unit/test_clean_tile_prune.py @@ -26,6 +26,9 @@ import pytest +pytestmark = pytest.mark.unions + + REPO_ROOT = Path(__file__).resolve().parents[2] SCRIPT = REPO_ROOT / "workflow" / "scripts" / "clean_tile.py" From 455583ba47702d9eb3b8b667527437bf44942546 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 11:10:51 -0400 Subject: [PATCH 094/100] workflow: centralize configured inputs and outputs --- workflow/README.md | 6 +++--- workflow/Snakefile | 12 ++++++++---- workflow/bin/sp | 7 +++++-- workflow/config.yaml | 18 ++++++++++++------ workflow/config/cfis/config_exp_Gie.ini | 2 +- workflow/config/cfis/config_tile_Git.ini | 2 +- workflow/scripts/clean_tile.py | 8 ++++---- 7 files changed, 34 insertions(+), 21 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index d5908f03c..6f48d3352 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,8 +22,8 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the -# star-catalogue cache root). +# Edit workflow/config.yaml: tile_list, inputs.tiles/exposures, outputs.run_dir, +# outputs.products_dir/star_cats/index_db, and container. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -150,7 +150,7 @@ profile-only pass. ``` workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks - config.yaml the run: tile list, paths, container, chunk count + config.yaml the run: tile list, input/output paths, container, chunk count bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures diff --git a/workflow/Snakefile b/workflow/Snakefile index 11810c421..aa20e0bcc 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -78,15 +78,17 @@ container: _image # report. Snakemake's own state is the one durable-looking thing that stays on # scratch (bin/sp explains why: tens of thousands of small, hot metadata writes # against a backed-up, file-count-limited filesystem). -RUN_DIR = Path(config["run_dir"]) +INPUTS = config["inputs"] +OUTPUTS = config["outputs"] +RUN_DIR = Path(OUTPUTS["run_dir"]) # Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no # second path: one root, exactly the pre-D5 layout. -PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) +PRODUCTS_DIR = Path(OUTPUTS.get("products_dir") or RUN_DIR) # Run-independent root for the mask star catalogues: the HEALPix chunk store the # star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it # (config.yaml explains the placement). -STAR_CATS = Path(config["star_cats"]) -INDEX_DB = Path(config["index_db"]) +STAR_CATS = Path(OUTPUTS["star_cats"]) +INDEX_DB = Path(OUTPUTS["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There @@ -526,6 +528,8 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, f"export SP_RUN='{work}'", f"export SP_UNIT_NUM='{unit_num(unit)}'", f"export SP_CONFIG='{CONFIG_DIR}'", + f"export SP_INPUT_TILES='{INPUTS['tiles']}'", + f"export SP_INPUT_EXPOSURES='{INPUTS['exposures']}'", # Also set via apptainer-args in the profile; kept here so a hand-run of # this same line outside snakemake behaves identically. f"export {_THREAD_CAPS}", diff --git a/workflow/bin/sp b/workflow/bin/sp index 9b5550bea..fdc3d4f59 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -40,8 +40,11 @@ source "$VENV/bin/activate" # snakemake depends on PyYAML, so this parses the file rather than pattern-matching # it -- quotes, inline comments and nesting are the parser's problem, not ours. cfg() { python -c 'import sys, yaml -print(yaml.safe_load(open(sys.argv[1])).get(sys.argv[2]) or "")' "$CONFIG" "$1"; } -RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +value = yaml.safe_load(open(sys.argv[1])) +for key in sys.argv[2].split("."): + value = value[key] +print(value or "")' "$CONFIG" "$1"; } +RUN_DIR="$(cfg outputs.run_dir)"; INDEX_DB="$(cfg outputs.index_db)" # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO # THE RUN on /scratch, never on the persistent root — the one exception to D5's diff --git a/workflow/config.yaml b/workflow/config.yaml index 8e4de8506..0fc3fc9c7 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -18,6 +18,13 @@ # broken.) tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt +# Inputs are the only site-specific data paths; all committed configs consume these +# through SP_INPUT_TILES and SP_INPUT_EXPOSURES. +inputs: + # Pre-staged P3 data (get_images RETRIEVE=symlink). + tiles: /project/def-mjhudson/unions-wl/tiles + exposures: /project/def-mjhudson/unions-wl/exposures + # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif @@ -27,7 +34,8 @@ container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # intermediates, sized so a batch finishes inside the 60-day purge window. The # sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-g6 +outputs: + run_dir: /scratch/cdaley/shapepipe-output/smk-g6 # products_dir is the PERSISTENT root: the durable, low-volume products — the # final catalogues (/tiles///final_cat-.fits, @@ -44,15 +52,13 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g6 # # Snakemake's own state is the one durable-looking thing that stays on scratch # (-state; bin/sp explains why). -products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 + products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / # $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules # are one artefact and must version together, so the dir is fixed by construction. -# Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). - # The mask star-catalogue root — run-independent, shared by every campaign, and # holding two things: # /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 @@ -75,13 +81,13 @@ products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # is skipped and only re-cut. Clear the store by hand when the change must reach # the data. Changing NSIDE or the catalogue ID is the exception: those name the # directory, so a change there fetches into a new one beside the old. -star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache + star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache # The run index, and — sharing its directory — missing.json and run_report.json. # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction # would otherwise have to rebuild from tile headers. -index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite + index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini index 7c6297cf0..b0cce78e8 100644 --- a/workflow/config/cfis/config_exp_Gie.ini +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -72,7 +72,7 @@ NUMBERING_SCHEME = -000-000 # Input path where original images are stored. Can be local path or vos url. # Single string or list of strings -INPUT_PATH = /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures +INPUT_PATH = $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES # Input file pattern including tile number as dummy template INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini index 72a0f0be7..cccf0a8b9 100644 --- a/workflow/config/cfis/config_tile_Git.ini +++ b/workflow/config/cfis/config_tile_Git.ini @@ -66,7 +66,7 @@ NUMBERING_SCHEME = # Input path where original images are stored. Can be local path or vos url. # Single string or list of strings -INPUT_PATH = /project/def-mjhudson/unions-wl/tiles, /project/def-mjhudson/unions-wl/tiles +INPUT_PATH = $SP_INPUT_TILES, $SP_INPUT_TILES # Input file pattern including tile number as dummy template INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py index 9e7b1e7b0..bdeefb794 100644 --- a/workflow/scripts/clean_tile.py +++ b/workflow/scripts/clean_tile.py @@ -98,10 +98,10 @@ each shared with 7-10 other tiles. Rebuildable, but only by re-running those chains from VOS. * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` - — 2 links into ``/project/def-mjhudson/unions-wl/tiles``, the staged survey - imaging: 621 GB across 2,536 files, on the BACKED-UP, GROUP-SHARED - ``/project``, and not this campaign's to lose. get_images RETRIEVE=symlink - is what puts them there, so every tile in every campaign carries a pair. + — 2 links into ``$SP_INPUT_TILES``, the staged survey imaging: 621 GB across + 2,536 files, on the backed-up, group-shared input filesystem, and not this + campaign's to lose. get_images RETRIEVE=symlink is what puts them there, so + every tile in every campaign carries a pair. Both classes are handed WHOLESALE to ``shutil.rmtree`` — ``exp_forest/`` as a top-level entry, ``run_sp_tile_Git/`` as one inside ``output/``, which ``prune`` From 98bc085748faf21bc142b8b827f918d2274c7da0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 11:21:46 -0400 Subject: [PATCH 095/100] workflow: select PSF model from configuration --- src/shapepipe/modules/mccd_interp_runner.py | 32 ++- .../mccd_package/mccd_interpolation_script.py | 25 +- workflow/README.md | 2 + workflow/Snakefile | 12 + workflow/config.yaml | 3 + workflow/config/cfis/config_MCCD.ini | 120 +++++++++ workflow/config/cfis/config_exp_mccd.ini | 237 ++++++++++++++++++ workflow/config/cfis/config_exp_psfex.ini | 5 +- workflow/config/cfis/config_tile_Mc.ini | 4 +- .../config/cfis/config_tile_Ng_template.ini | 2 +- .../config/cfis/config_tile_PiViVi_mccd.ini | 178 +++++++++++++ ...iViVi.ini => config_tile_PiViVi_psfex.ini} | 2 +- workflow/rules/exposure.smk | 8 +- workflow/rules/tile.smk | 16 +- workflow/scripts/completeness.py | 58 ++++- 15 files changed, 662 insertions(+), 42 deletions(-) create mode 100644 workflow/config/cfis/config_MCCD.ini create mode 100644 workflow/config/cfis/config_exp_mccd.ini create mode 100644 workflow/config/cfis/config_tile_PiViVi_mccd.ini rename workflow/config/cfis/{config_tile_PiViVi.ini => config_tile_PiViVi_psfex.ini} (98%) diff --git a/src/shapepipe/modules/mccd_interp_runner.py b/src/shapepipe/modules/mccd_interp_runner.py index 82ec7b65f..e61001fbf 100644 --- a/src/shapepipe/modules/mccd_interp_runner.py +++ b/src/shapepipe/modules/mccd_interp_runner.py @@ -8,6 +8,7 @@ import os +from shapepipe.pipeline.exp_utils import get_exp_output_dirs from shapepipe.pipeline.run_log import get_last_dir from shapepipe.modules.mccd_package import ( @@ -87,9 +88,34 @@ def mccd_interp_runner( ) elif mode == "MULTI-EPOCH": - module = config.getexpanded(module_config_sec, "PSF_MODEL_DIR") - psf_model_dir = get_last_dir(run_dirs["run_log"], module) - psf_model_pattern = config.get(module_config_sec, "PSF_MODEL_PATTERN") + if config.has_option(module_config_sec, "ME_DOT_PSF_EXP_DIR"): + exp_base_dir = config.getexpanded( + module_config_sec, "ME_DOT_PSF_EXP_DIR" + ) + if len(input_file_list) < 3: + raise ValueError( + "ME_DOT_PSF_EXP_DIR requires the exposure-numbers file" + " as a third input; add 'exp_numbers' to FILE_PATTERN" + f" and FILE_EXT in the [{module_config_sec}] config section." + ) + exp_numbers_file = input_file_list[2] + model_runner = config.get( + module_config_sec, + "ME_DOT_PSF_RUNNER", + fallback="mccd_fit_val_runner", + ) + psf_model_dir = get_exp_output_dirs( + exp_base_dir, exp_numbers_file, model_runner, w_log + ) + psf_model_pattern = config.get( + module_config_sec, "ME_DOT_PSF_PATTERN" + ) + else: + module = config.getexpanded(module_config_sec, "PSF_MODEL_DIR") + psf_model_dir = get_last_dir(run_dirs["run_log"], module) + psf_model_pattern = config.get( + module_config_sec, "PSF_MODEL_PATTERN" + ) galcat_path = input_file_list[0] # The WCS log is supplied as a positional input (via FILE_PATTERN) # in MULTI-EPOCH mode, not by the decorator default. diff --git a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py index a01ca095d..b6183f4e1 100644 --- a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py +++ b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py @@ -407,7 +407,9 @@ def process_me(self, dot_psf_dir, dot_psf_pattern, f_wcs_path): Path to the log file containing the WCS for each CCD """ - self._dot_psf_dir = dot_psf_dir + self._dot_psf_dir = ( + [dot_psf_dir] if isinstance(dot_psf_dir, str) else dot_psf_dir + ) self._dot_psf_pattern = dot_psf_pattern self._f_wcs_file = SqliteDict(f_wcs_path) @@ -467,14 +469,19 @@ def _interpolate_me(self): # dot_psf_path = self._dot_psf_dir + '/' +\ # self._dot_psf_pattern + '-' + exp_name + '-' + str(ccd) +\ # '.psf' - mccd_model_path = ( - self._dot_psf_dir - + "/" - + self._dot_psf_pattern - + "-" - + exp_name - + ".npy" - ) + found = False + for dot_psf_dir in self._dot_psf_dir: + mccd_model_path = ( + f"{dot_psf_dir}/{self._dot_psf_pattern}-{exp_name}.npy" + ) + if os.path.exists(mccd_model_path): + found = True + break + if not found: + self._w_log.info( + f"No .npy file found for exposure {exp_name}" + ) + continue ind_obj = np.where(cat.get_data(hdu_index)["CCD_N"] == ccd)[0] obj_id = all_id[ind_obj] diff --git a/workflow/README.md b/workflow/README.md index 6f48d3352..bf721a2cf 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -25,6 +25,8 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' # Edit workflow/config.yaml: tile_list, inputs.tiles/exposures, outputs.run_dir, # outputs.products_dir/star_cats/index_db, and container. +# `psf_model` is `psfex` or `mccd`; mccd is wired but unvalidated here, while psfex is exercised by smk-g4 through smk-g6. + # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. workflow/bin/sp run # bring products on disk up to date with the tile list diff --git a/workflow/Snakefile b/workflow/Snakefile index aa20e0bcc..47498908b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -71,6 +71,13 @@ if _kind == "none": container: _image +PSF_MODELS = {"psfex", "mccd"} +PSF_MODEL = config.get("psf_model", "psfex") +if PSF_MODEL not in PSF_MODELS: + raise WorkflowError( + f"Invalid psf_model={PSF_MODEL!r}; expected one of " + f"{sorted(PSF_MODELS)}.") + # --- paths ----------------------------------------------------------------- # RUN_DIR is the scratch root: bulk intermediates, sized so a batch finishes # inside the purge window. PRODUCTS_DIR is the persistent root @@ -513,6 +520,10 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, the committed config dir, and ``star_cat_exp`` is a real per-unit directory built by the ``exp_star_cat`` rule, not a symlink into a shared pool. + It also exports the configured input roots as ``SP_INPUT_TILES`` and + ``SP_INPUT_EXPOSURES`` and the PSF choice as ``SP_PSF`` for the committed + ini chain. + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees stale products (D2: the job clears its run dir at start). @@ -530,6 +541,7 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, f"export SP_CONFIG='{CONFIG_DIR}'", f"export SP_INPUT_TILES='{INPUTS['tiles']}'", f"export SP_INPUT_EXPOSURES='{INPUTS['exposures']}'", + f"export SP_PSF='{PSF_MODEL}'", # Also set via apptainer-args in the profile; kept here so a hand-run of # this same line outside snakemake behaves identically. f"export {_THREAD_CAPS}", diff --git a/workflow/config.yaml b/workflow/config.yaml index 0fc3fc9c7..43c9a3a2c 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -28,6 +28,9 @@ inputs: # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif +# PSF model used by the exposure and tile interpolation stages. +psf_model: psfex + # THE TWO ROOTS (D5). # # run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk diff --git a/workflow/config/cfis/config_MCCD.ini b/workflow/config/cfis/config_MCCD.ini new file mode 100644 index 000000000..bf5e6852e --- /dev/null +++ b/workflow/config/cfis/config_MCCD.ini @@ -0,0 +1,120 @@ +# Configuration file for the MCCD method + +[INPUTS] +INPUT_DIR = . +PREPROCESSED_OUTPUT_DIR = ./output +OUTPUT_DIR = ./output +INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits +INPUT_SEPARATOR = - +MIN_N_STARS = 20 +OUTLIER_STD_MAX = 100. +USE_SNR_WEIGHTS = False + +[INSTANCE] +N_COMP_LOC = 8 +D_COMP_GLOB = 8 +KSIG_LOC = 0.00 +KSIG_GLOB = 0.00 +FILTER_PATH = None +D_HYB_LOC = 2 +MIN_D_COMP_GLOB = None +RMSE_THRESH = 1.25 +CCD_STAR_THRESH = 0.15 +FP_GEOMETRY = CFIS + +[FIT] +LOC_MODEL = hybrid +PSF_SIZE = 6.2 +PSF_SIZE_TYPE = R2 +N_EIGENVECTS = 5 +N_ITER_RCA = 1 +N_ITER_GLOB = 2 +N_ITER_LOC = 2 +NB_SUBITER_S_LOC = 300 +NB_SUBITER_A_LOC = 400 +NB_SUBITER_S_GLOB = 100 +NB_SUBITER_A_GLOB = 200 + +[VALIDATION] +VAL_DATA_INPUT_DIR = . +VAL_PREPROCESSED_OUTPUT_DIR = ./output +VAL_MODEL_INPUT_DIR = ./output +VAL_OUTPUT_DIR = ./output +VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits +VAL_SEPARATOR = - +APPLY_DEGRADATION = True +MCCD_DEBUG = False +GLOBAL_POL_INTERP = False + + +# Parameter description: +# +# +# [INPUTS] +# INPUT_DIR : (Required) Must be a valid directory containing the input +# MCCD files. +# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should +# follow regex (regular expression) standards. +# INPUT_SEPARATOR : Separator of the different fields in the filename, +# ie sexcat[SEP]catalog_id[SEP]CCD_id.fits +# MIN_N_STARS : Minimum number of stars to keep a CCD for the training. +# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection. +# Should not be too low as a hihg quantity of low quality +# stars will be rejected. ie 9 is a conservative rejection. +# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will +# be used. +# For now, it needs the SNR estimations from SExtractor. +# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the +# preprocessed input files. +# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. +# The constructed models will be saved. +# +# +# [INSTANCE] +# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly, +# will be the max degree D of the polynomial. +# D_COMP_GLOB : Max degree of the global polynomial model. +# KSIG_LOC : Denoising parameter of the local model. +# ie 1 is a normal denoising, 3 is a hard denoising. +# KSIG_GLOB : Denoising parameter of the global model. +# ie 1 is a normal denoising, 3 is a hard denoising. +# FILTER_PATH : Path for predefined filters. +# +# +# [FIT] +# LOC_MODEL : Defines the type of local model to use, it can be: 'rca', +# 'poly' or 'hybrid'. +# When the poly model is used, N_COMP_LOC should be used +# as the D_LOC (max degree of the poly model) +# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways. +# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma +# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint +# construction. +# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between +# global and local estimations. +# N_ITER_GLOB : Number of epochs for each global optimization. Alternations +# between A_GLOB and S_GLOB. +# N_ITER_LOC : Number of epochs for each local optimization. Alternations +# between the different A_LOC and S_LOC. +# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC. +# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC. +# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB. +# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB. +# +# +# [VALIDATION] +# MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the +# saved trained models. +# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the +# validation input data (test dataset). +# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. +# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. +# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the +# validation outputs, test PSFs and interpolated PSFs. +# APPLY_DEGRADATION : Whether the PSF models should be degraded +# (sampling/shifts/flux) to match stars; use True if you +# plan on making pixel-based comparisons (residuals etc.). +# MCCD_DEBUG : Debug mode. Returns the local and global contributions. +# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model +# instead of RBF kernel interpolation. +# diff --git a/workflow/config/cfis/config_exp_mccd.ini b/workflow/config/cfis/config_exp_mccd.ini new file mode 100644 index 000000000..635952ca7 --- /dev/null +++ b/workflow/config/cfis/config_exp_mccd.ini @@ -0,0 +1,237 @@ +# ShapePipe configuration file for single-exposures, MCCD PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, setools_runner, + mccd_preprocessing_runner, mccd_fit_val_runner, + merge_starcat_runner, mccd_plots_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# Somehow this works but not +# - omitting +# - $SP_RUN/output +#INPUT_DIR = . + +# Input from two modules +INPUT_MODULE = split_exp_runner, mask_runner + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_MODULE = last:sextractor_runner + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat_sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[MCCD_PREPROCESSING_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +INPUT_DIR = last:setools_runner + +# Input are individual CCDs, thus single-exposure single-HDU images +NUMBERING_SCHEME = -0000000-0 + +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 + +FILE_EXT = .fits, .fits + + +[MCCD_FIT_VAL_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +NUMBERING_SCHEME = -0000000 + + +[MCCD_MERGE_STARCAT_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +NUMBERING_SCHEME = -0000000 + + +[MCCD_PLOTS_RUNNER] + +# Path to MCCD config file +CONFIG_PATH = $SP_CONFIG/config_MCCD.ini + +MODE = FIT_VALIDATION + +VERBOSE = False + +# Now MCCD has created a focal-plane PSF model, including all CCDS per images, +# thus single-exposure files +NUMBERING_SCHEME = -0000000 + +PLOT_MEANSHAPES = True + +# X_GRID, Y_GRID: correspond to the number of bins in each direction of each +# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. +X_GRID = 5 +Y_GRID = 10 + +PLOT_HISTOGRAMS = True + +# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape +# before drawing the plots. +REMOVE_OUTLIERS = False + + + +[MCCD_INTERP_RUNNER] + +# MODE: Define the way the MCCD interpolation will run. +# CLASSIC for classical run. +# MULTI-EPOCH for multi epoch. +MODE = CLASSIC + +# Position parameter names +# for multi-epoch XWIN_WORLD,YWIN_WORLD +# For classical XWIN_IMAGE,YWIN_IMAGE: +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Get PSF shapes calculated and saved on the output dict +GET_SHAPES = True + +# Directory with PSF models +PSF_MODEL_DIR = $SP_RUN/output + +# PSF model patterns +PSF_MODEL_PATTERN = fitted_model + +# PSF model separator +PSF_MODEL_SEPARATOR = - diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 0af871d8e..2431e8682 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -9,8 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SxSePsfPi -#RUN_NAME = run_sp_exp_SxSePsf +RUN_NAME = run_sp_exp_SxSePsf # Add date and time to RUN_NAME, optional, default: True RUN_DATETIME = False @@ -128,7 +127,7 @@ MAKE_POST_PROCESS = FALSE [SETOOLS_RUNNER] -INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsf/sextractor_runner/output # Note: Make sure this doe not match the SExtractor background images # (sexcat_background*) diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 4bcc6d6bf..5920d0a91 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -57,10 +57,10 @@ TIMEOUT = 96:00:00 # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN # The psfex_interp store is NODE-LOCAL: it is the vignette-store run this -# tile's fused group job wrote to node-local storage (see config_tile_PiViVi.ini +# tile's fused group job wrote to node-local storage (see config_tile_PiViVi_.ini # and workflow/rules/tile.smk). $NGMIX_VIGNET_DIR names it; the other two # inputs and this run's own output stay on $SP_RUN. -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_Ms/merge_sep_cats_runner/output +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $SP_RUN/output/run_sp_tile_Ms/merge_sep_cats_runner/output # Input file pattern(s), list of strings with length matching number of expected input file types # Cannot contain wild cards diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index dff44a3ca..04606636d 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -88,7 +88,7 @@ TIMEOUT = 96:00:00 # with everything else already node-local. tile_local() copies it beside the # store and points this variable at the copy. Set it to # $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output to read in place. -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers diff --git a/workflow/config/cfis/config_tile_PiViVi_mccd.ini b/workflow/config/cfis/config_tile_PiViVi_mccd.ini new file mode 100644 index 000000000..d312ca494 --- /dev/null +++ b/workflow/config/cfis/config_tile_PiViVi_mccd.ini @@ -0,0 +1,178 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# MCCD PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = mccd_interp_runner, + +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MCCD_INTERP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +# MCCD models are discovered in the per-exposure stores listed by SP_EXP. +ME_DOT_PSF_EXP_DIR = $SP_EXP +ME_DOT_PSF_RUNNER = mccd_fit_val_runner +ME_DOT_PSF_PATTERN = fitted_model + +# Multi-epoch mode parameters + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi_psfex.ini similarity index 98% rename from workflow/config/cfis/config_tile_PiViVi.ini rename to workflow/config/cfis/config_tile_PiViVi_psfex.ini index 2e1547f3e..27a759528 100644 --- a/workflow/config/cfis/config_tile_PiViVi.ini +++ b/workflow/config/cfis/config_tile_PiViVi_psfex.ini @@ -21,7 +21,7 @@ RUN_DATETIME = False # Module name, single string or comma-separated list of valid module runner names #MODULE = psfex_interp_runner, -MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 8ddbbe193..0cce45a76 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -285,9 +285,9 @@ rule exp_mask: shell: sp_shell("exp_mask", "config_exp_Ma.ini") -# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. -# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's -# :warn on psfex_interp_runner. +# SExtractor -> setools star selection -> the configured PSF model and +# interpolation, per CCD. setools may reject a sparse CCD (~0.2% attrition) — +# tolerated by the PSF-specific completeness table. rule exp_psf: input: rules.exp_mask.output.manifest @@ -310,7 +310,7 @@ rule exp_psf: mem_mb = lambda wc, attempt: 16000 * attempt, runtime = 240 shell: - sp_shell("exp_psf", "config_exp_psfex.ini") + sp_shell("exp_psf", f"config_exp_{PSF_MODEL}.ini") # --- reclamation (D5) ------------------------------------------------------- diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index debe148ff..599f7f173 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -78,7 +78,7 @@ across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). # dirs merge_sep_cats gathers, and final_cat on the PERSISTENT root. Only the # bulk intra-tile intermediate is node-local. That split is possible because # ShapePipe's configs set input and output paths independently -- see -# config_tile_PiViVi.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and +# config_tile_PiViVi_.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and # config_tile_Ng_template.ini and config_tile_Mc.ini ($NGMIX_VIGNET_DIR), and # config_tile_Ng_template.ini alone for $SP_WCS_DIR. # @@ -468,8 +468,8 @@ rule tile_detect: shell: sp_shell("tile_detect", "config_tile_Sx.ini") -# PSFEx interpolation to galaxies + vignet postage stamps: the last stage that -# reads exposure products, and the bulk intra-tile intermediate. The store it +# Configured PSF interpolation to galaxies + vignet postage stamps: the last +# stage that reads exposure products, and the bulk intra-tile intermediate. The store it # writes is node-local (see TILE_LOCAL above). rule tile_vignets: group: TILE_GROUP @@ -478,7 +478,7 @@ rule tile_vignets: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, psf = tile_exp_psf, - # config_tile_PiViVi.ini reads run_sp_tile_Fe output — same reason as + # config_tile_PiViVi_.ini reads run_sp_tile_Fe output — same reason as # tile_merge_headers above. fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: @@ -521,7 +521,7 @@ rule tile_vignets: shell: # The completeness check is pointed at the NODE-LOCAL run root; see # sp_shell's check_args for what the two flags do. - sp_shell("tile_vignets", "config_tile_PiViVi.ini", + sp_shell("tile_vignets", f"config_tile_PiViVi_{PSF_MODEL}.ini", check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') # ngmix shape measurement — N chunks per tile (D4). Each chunk LOOKS UP its own @@ -774,7 +774,7 @@ rule tile_merge_cats: sp_shell("tile_merge_cats", "config_tile_Ms.ini") # The run's science product. make_cat also reads the vignette store's -# psfex_interp output, so it — not ngmix — is the store's last reader. +# configured PSF-interpolation output, so it — not ngmix — is the store's last reader. # # No protected(): the full default rerun-triggers govern, and protected() only # ever forced people through a `--forcerun` detour. @@ -782,8 +782,8 @@ rule tile_make_cat: group: TILE_GROUP input: # No store input: it is node-local, written by tile_vignets in this same - # group job. make_cat reads its psfex_interp output through - # $NGMIX_VIGNET_DIR (config_tile_Mc.ini). + # group job. make_cat reads its configured PSF-interpolation output + # through $NGMIX_VIGNET_DIR (config_tile_Mc.ini). ms = rules.tile_merge_cats.output.manifest, output: manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 2053881eb..075fccb28 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -69,6 +69,7 @@ from pathlib import Path # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +# exp_psf and tile_vignets are selected by $SP_PSF at check time. COMPLETENESS = { # --- tile prepare (phase A) --- # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 @@ -87,22 +88,49 @@ # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree # AND the bash baseline (both 120/exposure). "exp_psf": { - "sextractor_runner": dict(expect=120, floor=2), - "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), - "psfex_runner": dict(expect=80, floor=2), - "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + "psfex": { + "sextractor_runner": dict(expect=120, floor=2), + "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), + "psfex_runner": dict(expect=80, floor=2), + "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + }, + # MCCD counts are derived from config_exp_mccd.ini and its per-CCD + # runners, but this chain has not been exercised through this workflow. + # Keep the expected counts visible while making the unverified branch + # warning-only until a real campaign validates its floors. + "mccd": { + "sextractor_runner": dict(expect=120, floor=0, warn=True), + "setools_runner": dict(expect=80, floor=0, warn=True, + subpath="rand_split"), + "mccd_preprocessing_runner": dict(expect=80, floor=0, warn=True), + # Fit/validation is exposure-wide: one model and one validation + # catalogue, unlike the per-CCD preprocessing outputs. + "mccd_fit_val_runner": dict(expect=2, floor=0, warn=True), + "merge_starcat_runner": dict(expect=1, floor=0, warn=True), + # config_exp_mccd enables the ten meanshape and six histogram plots. + "mccd_plots_runner": dict(expect=16, floor=0, warn=True), + }, }, # --- tile post --- "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, "tile_vignets": { - "psfex_interp_runner": dict(expect=1, floor=1), - "vignetmaker_runner_run_1": dict(expect=1, floor=1), - # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); - # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern - # (expect=floor: all-or-nothing, every vignette feeds ngmix). - "vignetmaker_runner_run_2": dict(expect=5, floor=5), + "psfex": { + "psfex_interp_runner": dict(expect=1, floor=1), + "vignetmaker_runner_run_1": dict(expect=1, floor=1), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern + # (expect=floor: all-or-nothing, every vignette feeds ngmix). + "vignetmaker_runner_run_2": dict(expect=5, floor=5), + }, + # MCCD is wired but unvalidated here; retain the expected runner names + # and counts as warnings until a workflow campaign exercises them. + "mccd": { + "mccd_interp_runner": dict(expect=1, floor=0, warn=True), + "vignetmaker_runner_run_1": dict(expect=1, floor=0, warn=True), + "vignetmaker_runner_run_2": dict(expect=5, floor=0, warn=True), + }, }, "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, @@ -145,6 +173,14 @@ def check_floor(stage, run_dir): ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. """ table = COMPLETENESS[stage] + if stage in ("exp_psf", "tile_vignets"): + psf_model = os.environ.get("SP_PSF", "psfex") + try: + table = table[psf_model] + except KeyError as exc: + raise ValueError( + f"Invalid SP_PSF={psf_model!r}; expected one of psfex, mccd." + ) from exc details, ok = [], True for runner, spec in table.items(): n = count_products(run_dir, runner, spec) @@ -175,7 +211,7 @@ def check_floor(stage, run_dir): "exp_get_images": ("exp", "run_sp_exp_Gie"), "exp_split": ("exp", "run_sp_exp_Sp"), "exp_mask": ("exp", "run_sp_exp_Ma"), - "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "exp_psf": ("exp", "run_sp_exp_SxSePsf"), "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), "tile_detect": ("tile", "run_sp_tile_Sx"), "tile_vignets": ("tile", "run_sp_tile_PiViVi"), From 4e130a8e8f80b513bb265d82905f8e73a609b656 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 11:42:41 -0400 Subject: [PATCH 096/100] Remove completeness count floors --- tests/helpers/artifacts.py | 2 +- tests/science/test_additive_null.py | 4 +- tests/science/test_resolution_ladder.py | 4 +- tests/unit/test_run_report_tombstones.py | 2 +- workflow/README.md | 12 +- workflow/Snakefile | 2 +- workflow/config.yaml | 2 +- workflow/rules/exposure.smk | 2 +- workflow/rules/prepare.smk | 2 +- workflow/rules/tile.smk | 4 +- workflow/scripts/completeness.py | 147 ++++++++++++----------- workflow/scripts/run_report.py | 8 +- workflow/scripts/star_cats.py | 2 +- 13 files changed, 99 insertions(+), 94 deletions(-) diff --git a/tests/helpers/artifacts.py b/tests/helpers/artifacts.py index 25cbbc8ff..d96d02242 100644 --- a/tests/helpers/artifacts.py +++ b/tests/helpers/artifacts.py @@ -297,7 +297,7 @@ def _m_cell(r): f"{rows}\n" f"As the galaxy shrinks toward the PSF, |m| grows negative (shear washes " f"out) and R11 dips through a ~0.92 minimum then rises toward the " - f"point-source floor. Only resolved rungs (ratio >= " + f"point-source limit. Only resolved rungs (ratio >= " f"{summary['resolved_ratio_threshold']}) are asserted.\n" ) diff --git a/tests/science/test_additive_null.py b/tests/science/test_additive_null.py index 552738b55..c8440071b 100644 --- a/tests/science/test_additive_null.py +++ b/tests/science/test_additive_null.py @@ -14,7 +14,7 @@ m-bias path — the galaxy is round (``shear=(0, 0)``) and the PSF is sheared (``psf_shear=(0.05, 0)``). With the deconvolution wired correctly the true PSF is fed as the model, so the deconvolution is unbiased and the recovered ``c`` sits -at the ~few x 1e-5 noise floor per seed. A regression that breaks the +at the ~few x 1e-5 noise level per seed. A regression that breaks the deconvolution (deconvolves by a fitted model instead of the PSF image, drops it, or shears the galaxy rather than the PSF) leaks the 0.05 PSF ellipticity straight into ``c``, blowing past the tolerance in seconds. @@ -49,7 +49,7 @@ def test_additive_bias_consistent_with_zero(): A round galaxy through an ``e1 = 0.05`` PSF should yield ``c ~ 0`` once the deconvolution removes the PSF shape (the true PSF is fed as the model, so - the deconvolution is unbiased and only the noise floor remains). Asserts the + the deconvolution is unbiased and only the noise level remains). Asserts the ensemble-mean ``|c|`` below the twin's published ``C_TOL``; a deconvolution regression that leaks PSF ellipticity pushes ``c`` toward 0.05 and trips it. """ diff --git a/tests/science/test_resolution_ladder.py b/tests/science/test_resolution_ladder.py index 0e04f1d46..c1e0a6c2b 100644 --- a/tests/science/test_resolution_ladder.py +++ b/tests/science/test_resolution_ladder.py @@ -27,7 +27,7 @@ ``~+0.0004`` at ratio 1.2 through zero to ``~-0.012`` at ratio 0.15. The response ``R11`` does something less obvious: it does **not** slide monotonically. It dips through a shallow ~0.92 minimum at mid-resolution and then **rises** back -toward 1 as the object approaches the point-source floor (a true point source is +toward 1 as the object approaches the point-source limit (a true point source is pathological — the round Gaussian fit collapses and R stiffens). So the JSON carries the whole shape; the assertions pin only what a *resolved* survey galaxy must satisfy. @@ -35,7 +35,7 @@ The guardrail: on the **resolved** rungs (``ratio >= 0.5``) the response correction must leave ``|m| < 5e-3`` — the same few-x-1e-3 scale as ``test_mbias`` (observed max ``0.0036``). The unresolved rungs are recorded but -**not** asserted: they document where the estimator's floor lives, not a +**not** asserted: they document where the estimator's limit lives, not a requirement. A response mis-scale (wrong metacal step, R not applied, a constant factor on R) shifts ``m`` by a roughly constant offset across every rung — a +5% R error moves each resolved ``m`` by ~0.05, tripping the tripwire loudly (the diff --git a/tests/unit/test_run_report_tombstones.py b/tests/unit/test_run_report_tombstones.py index 6403d0233..ccb791d17 100644 --- a/tests/unit/test_run_report_tombstones.py +++ b/tests/unit/test_run_report_tombstones.py @@ -50,7 +50,7 @@ def _manifest(stage, status="complete", found=1, expect=1): return { "stage": stage, "level": "tile", "unit": TILE, "status": status, "runners": {f"{stage}_runner": { - "found": found, "expect": expect, "floor": found, + "found": found, "expect": expect, "status": status, "warn": status == "warn"}}, "failures": [], } diff --git a/workflow/README.md b/workflow/README.md index bf721a2cf..18c5ac3ac 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -159,10 +159,10 @@ workflow/ exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat scripts/ - sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) + sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count check) build_index.py prepare-phase run_index.sqlite builder (plain script) build_forest.py per-tile exposure symlink forest (group-compatible shell) - completeness.py the ported count-floor table (shared by sp_rule + run_report) + completeness.py the ported count table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) container.py image layers + the resolution order behind `sp container` (stdlib-only) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) @@ -178,7 +178,7 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee millions of paths. - **Manifests are the DAG's currency, and they are success-only.** `completeness.py check` writes its full verdict — per-runner counts against - floors, scraped failure reasons, the `shapepipe_run` exit status when nonzero + expected counts, scraped failure reasons, the `shapepipe_run` exit status when nonzero — to the rule's `log:` (`/logs/.json`) on *every* run, and additionally to the declared `.json` manifest only when that verdict is a success. So `.json` on disk means "this stage succeeded", and a @@ -187,10 +187,10 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee output natively and never touches its log, which is why the profile runs *without* `keep-incomplete`. `sp report` reads both dirs — the manifest for success, the log for failure — and a unit with neither ran nothing. -- **Completeness is a count floor, not a taxonomy.** After a run, +- **Completeness is an exact-count check, not a taxonomy.** After a run, `sp_rule.py` counts products per mandatory runner against - `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition - between floor and `expect` is tolerated. No 3-class taxonomy, no + `completeness.py`'s expected count and exits nonzero below it. A shortfall + below `expect` fails unless `warn` is set. No 3-class taxonomy, no error-signature whitelist. `--keep-going` isolates a failure to its own DAG cone. - **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` diff --git a/workflow/Snakefile b/workflow/Snakefile index 47498908b..aa5ce6c72 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -584,7 +584,7 @@ def sp_shell(stage, config_name, *, check_args="", post=""): The rc is CAPTURED rather than ``&&``-ed onto, so the check still runs when shapepipe_run failed, and it is passed back as ``--job-rc "$rc"`` so the - verdict composes the count floors with shapepipe_run's own exit status. Both + verdict composes the count checks with shapepipe_run's own exit status. Both halves are argued in completeness.py's docstring. """ return ( diff --git a/workflow/config.yaml b/workflow/config.yaml index 43c9a3a2c..63314d775 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -162,5 +162,5 @@ ngmix_chunks: 8 # The container's installed shapepipe is overridden by the prod worktree via # --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. -# build_index.py's missing-tile fraction floor is passed by workflow/bin/sp +# build_index.py's missing-tile fraction threshold is passed by workflow/bin/sp # (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 0cce45a76..07792e0e4 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -21,7 +21,7 @@ neighbours the next time a tile is appended. GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under -the 15-minute floor Alliance policy asks us to bundle away. The composition +the 15-minute runtime limit Alliance policy asks us to bundle away. The composition rules are in prepare.smk's docstring; this chain is linear too, so the group asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 07405407f..80ec58831 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -10,7 +10,7 @@ tier). All three rules carry ``group: "tile_prep"``, so one tile's whole chain is ONE sbatch instead of three (medians 0:41 / ~0:40 / 0:15 — all far under the -15-minute floor Alliance policy asks us to bundle away, and at DR6 scale three +15-minute runtime limit Alliance policy asks us to bundle away, and at DR6 scale three submissions per tile is a scheduler load out of all proportion to the work). Group membership is per connected DAG component and distinct tiles share no edge, so this is exactly one group job per tile, never a cross-tile bundle. diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 599f7f173..a9dd27179 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -677,7 +677,7 @@ rule tile_ngmix: # memory.current, which under cgroup v2 CHARGES PAGE CACHE to the job. # Cache is reclaimable — the kernel evicts it before it kills anything — # so the cgroup high-water is an upper bound on what the job NEEDS, not - # a floor. What a tight reservation can still do is squeeze the cache + # a hard requirement. What a tight reservation can still do is squeeze the cache # that keeps the node-local store resident, which is part of why the # fused tile is fast. # @@ -707,7 +707,7 @@ rule tile_ngmix: # fairshare target is ~250 CE, which buys 9 tiles in flight at 28 and 25 # at 10, and DR6's wall clock is (tiles / tiles-in-flight) x elapsed. # - # 4000 is the floor and is NOT recommended yet: it would take the group + # 4000 is the current lower bound and is NOT recommended yet: it would take the group # to 32000 MiB, where tile_vignets' own 32000 becomes the binding term # and the group finally bills its 8 real cores — but that is a 1.14x # margin over the worst tile measured, and the first thing to give would diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 075fccb28..63cd4d4bc 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -1,12 +1,18 @@ #!/usr/bin/env python3 -"""The count-floor completeness table — the single failure policy. +"""The count-based completeness table — the single failure policy. This is the ported ``complete_check`` count table from the v2.0 bash layer -(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). It is the *only* -failure policy in the design: there is no 3-class taxonomy and no error-signature -whitelist. A stage is a real failure iff a mandatory runner produced fewer than -its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) -sits between ``floor`` and ``expect`` and is tolerated. +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). Across smk-g6 +(127 exposures, 64 tiles, and 512 ngmix chunks), every non-warning runner +produced exactly its ``expect`` count; the v2.0 layer likewise used exact counts, +with only ``psfex_interp`` marked ``:warn``. The formerly ported lower bounds +(and the unsupported ~0.2% setools-attrition claim) therefore have no basis: +setools produced 80/80 in all 127 exposures. ``split_exp`` is also structurally +all-or-nothing because it raises on an HDU-count mismatch. + +A runner below ``expect`` fails its unit unless it has ``warn=True``; such a +shortfall gives the unit status ``warn``. There is no 3-class taxonomy and no +error-signature whitelist. This file is also the ``check`` CLI, the second half of every rule's shell line (PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing @@ -19,15 +25,14 @@ exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory -runner is below its floor OR ``--job-rc`` is nonzero — the verdict is COMPOSED of +runner is below ``expect`` OR ``--job-rc`` is nonzero — the verdict is COMPOSED of the counts and shapepipe_run's own exit status, because a runner can raise after the counted ones have written their files. -The verdict is written to two files with different -different jobs: +The verdict is written to two files with different jobs: * the LOG (``--log``, the rule's snakemake ``log:``) gets the full verdict on - EVERY run, success or failure — counts against floors, per-runner detail, + EVERY run, success or failure — counts against ``expect``, per-runner detail, scraped failure reasons, ``job_rc`` when nonzero. Snakemake never deletes a log file, so it survives the failed job that wrote it and is the post-mortem evidence ``run_report.py`` reads. @@ -50,10 +55,9 @@ ``--forcerun``. Per-runner fields: - expect nominal file count for a fully complete unit (report yardstick) - floor the fail-loud minimum (below this the job exits nonzero) - warn if True the runner never fails the unit at all (bash ``:warn`` — - e.g. psfex_interp on tiles missing some epochs) + expect nominal file count for a fully complete unit; below it fails + warn if True a shortfall warns instead of failing the unit (bash + ``:warn`` — e.g. psfex_interp on tiles missing some epochs) subpath count files in ``/output//`` instead of ``/output/`` (bash ``:rand_split`` — setools split cats) @@ -68,7 +72,7 @@ import sys from pathlib import Path -# stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +# stage -> {runner_subdir: {expect, [warn], [subpath]}} # exp_psf and tile_vignets are selected by $SP_PSF at check time. COMPLETENESS = { # --- tile prepare (phase A) --- @@ -76,65 +80,66 @@ # for the canfar vos flavor; the nibi symlink configs produce one file per # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), # verified against the p3-batch1 baseline tree (100 files / 50 tiles). - "tile_get_images": {"get_images_runner": dict(expect=2, floor=2)}, - "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, - "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, + "tile_get_images": {"get_images_runner": dict(expect=2)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1)}, # --- exposure chain --- - "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, - "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, - "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + "exp_get_images": {"get_images_runner": dict(expect=3)}, + "exp_split": {"split_exp_runner": dict(expect=121)}, + "exp_mask": {"mask_runner": dict(expect=40)}, # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree # AND the bash baseline (both 120/exposure). "exp_psf": { "psfex": { - "sextractor_runner": dict(expect=120, floor=2), - "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), - "psfex_runner": dict(expect=80, floor=2), - "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + "sextractor_runner": dict(expect=120), + "setools_runner": dict(expect=80, subpath="rand_split"), + "psfex_runner": dict(expect=80), + "psfex_interp_runner": dict(expect=40, warn=True), }, # MCCD counts are derived from config_exp_mccd.ini and its per-CCD # runners, but this chain has not been exercised through this workflow. # Keep the expected counts visible while making the unverified branch - # warning-only until a real campaign validates its floors. + # warning-only until a real campaign validates its counts. "mccd": { - "sextractor_runner": dict(expect=120, floor=0, warn=True), - "setools_runner": dict(expect=80, floor=0, warn=True, + "sextractor_runner": dict(expect=120, warn=True), + "setools_runner": dict(expect=80, warn=True, subpath="rand_split"), - "mccd_preprocessing_runner": dict(expect=80, floor=0, warn=True), + "mccd_preprocessing_runner": dict(expect=80, warn=True), # Fit/validation is exposure-wide: one model and one validation # catalogue, unlike the per-CCD preprocessing outputs. - "mccd_fit_val_runner": dict(expect=2, floor=0, warn=True), - "merge_starcat_runner": dict(expect=1, floor=0, warn=True), + "mccd_fit_val_runner": dict(expect=2, warn=True), + "merge_starcat_runner": dict(expect=1, warn=True), # config_exp_mccd enables the ten meanshape and six histogram plots. - "mccd_plots_runner": dict(expect=16, floor=0, warn=True), + "mccd_plots_runner": dict(expect=16, warn=True), }, }, # --- tile post --- - "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, - "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, + "tile_merge_headers": {"merge_headers_runner": dict(expect=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2)}, "tile_vignets": { "psfex": { - "psfex_interp_runner": dict(expect=1, floor=1), - "vignetmaker_runner_run_1": dict(expect=1, floor=1), + "psfex_interp_runner": dict(expect=1), + "vignetmaker_runner_run_1": dict(expect=1), # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); - # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern - # (expect=floor: all-or-nothing, every vignette feeds ngmix). - "vignetmaker_runner_run_2": dict(expect=5, floor=5), + # v2.0's 4 was the canfar flavor. every vignette feeds ngmix, so the expected count is all-or-nothing. + "vignetmaker_runner_run_2": dict(expect=5), }, # MCCD is wired but unvalidated here; retain the expected runner names # and counts as warnings until a workflow campaign exercises them. "mccd": { - "mccd_interp_runner": dict(expect=1, floor=0, warn=True), - "vignetmaker_runner_run_1": dict(expect=1, floor=0, warn=True), - "vignetmaker_runner_run_2": dict(expect=5, floor=0, warn=True), + "mccd_interp_runner": dict(expect=1, warn=True), + "vignetmaker_runner_run_1": dict(expect=1, warn=True), + "vignetmaker_runner_run_2": dict(expect=5, warn=True), }, }, - "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, - "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, - "tile_make_cat": {"make_cat_runner": dict(expect=1, floor=1)}, + # One check runs inside run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u per chunk, + # so expect=1 is the correct per-chunk count. + "tile_ngmix": {"ngmix_runner": dict(expect=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1)}, } @@ -167,10 +172,10 @@ def count_products(run_dir, runner, spec): return n -def check_floor(stage, run_dir): - """Return (ok, details). ok is False iff a mandatory runner is below floor. +def check_counts(stage, run_dir): + """Return (ok, details); mandatory shortfalls below expect make ok false. - ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. + ``details`` is a list of (runner, n_found, expect, warn) tuples. """ table = COMPLETENESS[stage] if stage in ("exp_psf", "tile_vignets"): @@ -184,9 +189,9 @@ def check_floor(stage, run_dir): details, ok = [], True for runner, spec in table.items(): n = count_products(run_dir, runner, spec) - details.append((runner, n, spec["floor"], spec["expect"], - spec.get("warn", False))) - if not spec.get("warn", False) and n < spec["floor"]: + warn = spec.get("warn", False) + details.append((runner, n, spec["expect"], warn)) + if not warn and n < spec["expect"]: ok = False return ok, details @@ -225,7 +230,7 @@ def check_floor(stage, run_dir): # Lines worth showing a human who asks "why is this runner short?". Deliberately # crude: the point is a pointer into the logs, not a taxonomy (there is no error -# whitelist in this design — the count floor is the policy). +# whitelist in this design — the count policy is the policy). _ERROR_RE = re.compile( r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" r"killed|out of memory|oom|segmentation fault|bad chi2", @@ -286,7 +291,7 @@ def scrape_reasons(stage_dir, runner): def build_manifest(stage, run_dir, unit, stage_subdir=None): """Count, classify and (on shortfall) scrape. Returns (manifest, ok). - Stages absent from the table fall back to the zero-output floor: any product + Stages absent from the table fall back to a zero-output check: any product anywhere under the stage dir passes, nothing at all fails. """ level, subdir = STAGE_DIR.get(stage, (None, None)) @@ -309,25 +314,25 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): manifest["n_products"] = len(produced) if not ok: manifest["failures"].append( - {"runner": None, "found": 0, "floor": 1, - "reasons": [f"zero output under {stage_dir}"]}) + {"runner": None, "found": 0, "expect": 1, "warn": False, + "status": "failed", "reasons": [f"zero output under {stage_dir}"]}) return manifest, ok - ok, details = check_floor(stage, stage_dir) + ok, details = check_counts(stage, stage_dir) short = False - for runner, n, floor, expect, warn in details: - below = n < floor - if n < expect: + for runner, n, expect, warn in details: + below = n < expect + if below: short = True + status = "complete" if not below else "warn" if warn else "failed" manifest["runners"][runner] = { - "found": n, "expect": expect, "floor": floor, "warn": warn, - "status": ("complete" if n >= expect else - "warn" if (warn or not below) else "below_floor"), + "found": n, "expect": expect, "warn": warn, "status": status, } if below: manifest["failures"].append({ - "runner": runner, "found": n, "floor": floor, "expect": expect, - "warn": warn, "reasons": scrape_reasons(stage_dir, runner), + "runner": runner, "found": n, "expect": expect, + "warn": warn, "status": status, + "reasons": scrape_reasons(stage_dir, runner), }) manifest["status"] = "failed" if not ok else ("warn" if short else "complete") return manifest, ok @@ -379,10 +384,10 @@ def main(argv=None) -> int: manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) - # The verdict is COMPOSED of two independent statements: the count floors + # The verdict is COMPOSED of two independent statements: the count checks # (above) and shapepipe_run's own exit status (here). Counts alone are not # enough — a runner can raise AFTER the counted runners have written their - # files, so the floors are met while the job died. Without this, such a job + # files, so the counts are met while the job died. Without this, such a job # publishes a SUCCESS manifest that snakemake then deletes as a failed job's # output — and the log would claim "complete" for a stage with no manifest, # which reads as a bookkeeping bug rather than as the failure it is. @@ -394,10 +399,10 @@ def main(argv=None) -> int: manifest["status"] = "failed" manifest["job_rc"] = args.job_rc manifest["failures"].append({ - "runner": "shapepipe_run", "found": 0, "floor": 1, "expect": 1, - "warn": False, + "runner": "shapepipe_run", "found": 0, "expect": 1, + "warn": False, "status": "failed", "reasons": [f"shapepipe_run exited {args.job_rc} " - f"(counts above floor)"], + f"(counts met expect)"], }) text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" @@ -413,9 +418,9 @@ def main(argv=None) -> int: write_if_changed(args.manifest, text) for runner, r in manifest["runners"].items(): - tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} + tag = {"complete": "OK", "warn": "warn", "failed": "<-- BELOW expect"} print(f"[completeness] {runner}: {r['found']}/{r['expect']} " - f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) + f"{tag[r['status']]}", file=sys.stderr) print(f"[completeness] {args.stage} {unit}: {manifest['status']} " f"-> {args.log}" + (f" + {args.manifest}" if ok else ""), file=sys.stderr) for f in manifest["failures"]: diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 0149f6bc9..2b4032177 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -13,7 +13,7 @@ tile->exposure edges that let an exposure failure be blamed on the tiles it blocks; * the **verdicts** written by ``completeness.py check`` — per-runner - found/expect/floor and log-scraped failure reasons — in the unit's + found/expect and log-scraped failure reasons — in the unit's ``logs/`` (every run) and ``manifests/`` (successes only; completeness.py argues the split). @@ -206,8 +206,8 @@ def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, def shortfalls(m: dict) -> dict: - """``{runner: (found, expect, floor)}`` for every runner under expect.""" - return {r: (d["found"], d["expect"], d["floor"]) + """``{runner: (found, expect)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"]) for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} @@ -215,7 +215,7 @@ def reasons(m: dict) -> list: """Flattened failure reasons, runner-tagged, for the report's why column.""" out = [] for f in m.get("failures", []): - head = f"{f['runner']} {f['found']}/{f.get('expect', '?')} (floor {f['floor']})" + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')}" out += [f"{head}: {r}" for r in f["reasons"]] or [head] return out diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py index 0c8ea5ce8..0ae9bf4ac 100644 --- a/workflow/scripts/star_cats.py +++ b/workflow/scripts/star_cats.py @@ -224,7 +224,7 @@ def write_manifest(args, tile_ids, needed, n_fetched) -> None: """The rule's declared output. Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and - has no per-runner count floors, so there is nothing to compose and no + has no per-runner count checks, so there is nothing to compose and no separate ``log:`` — under ``set -euo pipefail`` the job either completes or aborts at the failing query, and snakemake's captured stderr is the evidence. The manifest keeps the workflow's "one rule, one manifest" currency: written From c7509af62906620aa413077398b5274fd791f6e7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 5 Sep 2026 11:47:13 -0400 Subject: [PATCH 097/100] clean_exposure: update exp_psf run name in docstring --- workflow/scripts/clean_exposure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 728f6c527..7f2ae52d8 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,7 +7,7 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its +run_sp_exp_Gie/Sp/Ma/SxSePsf), its ``manifests/`` and its ``logs/``, and its star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted here like every other, so the exposure's chain must read as unbuilt, and 40 From e32c4fb9fa789ea5d6a3f021a3103928c99382b5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 06:53:08 -0400 Subject: [PATCH 098/100] example/cfis: retire configs superseded by the Snakemake workflow The eight shared parameter files in workflow/config/cfis (default.*, final_cat.param, star_selection.setools) were symlinks into example/cfis; they become real files there, which is the directory $SP_CONFIG points at. example/cfis keeps only the PSF-validation and post-processing configs the workflow does not cover (Ms/MsPl/Pl/valjoint, exp_Pi, tile_Uc, tile_Git_cat_vos, stats), with a README; the per-module INIs the workflow carries as edited copies, the bash-layer batching/vos variants, and defunct/ are removed. Test and script paths repointed to workflow/config/cfis. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011C7UQydnTDDVLAo7yrvfCN --- docs/source/post_processing.md | 7 +- example/cfis/README.md | 17 ++ example/cfis/config_MCCD.ini | 120 -------- example/cfis/config_exp_Gie_vos.ini | 99 ------- example/cfis/config_exp_Sp.ini | 74 ----- example/cfis/config_exp_mccd.ini | 271 ------------------ example/cfis/config_exp_psfex.ini | 218 -------------- example/cfis/config_get_tiles_vos_headers.ini | 95 ------ example/cfis/config_make_cat_mccd.ini | 74 ----- example/cfis/config_make_cat_psfex.ini | 83 ------ example/cfis/config_make_cat_psfex_nosm.ini | 76 ----- .../cfis/config_merge_sep_cats_template.ini | 72 ----- example/cfis/config_tile_Fe.ini | 72 ----- example/cfis/config_tile_Git_vos.ini | 93 ------ example/cfis/config_tile_Mh_exp.ini | 72 ----- .../cfis/config_tile_Ng_batch_psfex_uc.ini | 79 ----- example/cfis/config_tile_Ng_template.ini | 112 -------- .../cfis/config_tile_Ng_template_batch.ini | 81 ------ example/cfis/config_tile_PiViVi_canfar_sx.ini | 204 ------------- example/cfis/config_tile_PiViVi_canfar_uc.ini | 166 ----------- example/cfis/config_tile_Sx.ini | 119 -------- example/cfis/config_tile_Uz.ini | 69 ----- example/cfis/default.conv | 5 - example/cfis/default.param | 68 ----- example/cfis/default.psfex | 85 ------ example/cfis/default_exp.sex | 133 --------- example/cfis/default_noimaflags.param | 65 ----- example/cfis/default_tile.sex | 133 --------- example/cfis/defunct/config_Gie_symlink.ini | 97 ------- .../cfis/defunct/config_GitFeGie_symlink.ini | 151 ---------- example/cfis/defunct/config_GitFeGie_vos.ini | 155 ---------- example/cfis/defunct/config_GitFe_symlink.ini | 105 ------- example/cfis/defunct/config_exp_SpMh.ini | 85 ------ example/cfis/defunct/config_tile_PiViSmVi.ini | 184 ------------ .../defunct/config_tile_PiViSmVi_canfar.ini | 184 ------------ example/cfis/final_cat.param | 117 -------- example/cfis/star_selection.setools | 131 --------- scripts/python/create_final_cat.py | 2 +- tests/module/test_psf_grammar_properties.py | 10 +- workflow/config/cfis/default.conv | 6 +- workflow/config/cfis/default.param | 69 ++++- workflow/config/cfis/default.psfex | 86 +++++- workflow/config/cfis/default_exp.sex | 134 ++++++++- workflow/config/cfis/default_noimaflags.param | 66 ++++- workflow/config/cfis/default_tile.sex | 134 ++++++++- workflow/config/cfis/final_cat.param | 118 +++++++- workflow/config/cfis/star_selection.setools | 132 ++++++++- 47 files changed, 761 insertions(+), 3967 deletions(-) create mode 100644 example/cfis/README.md delete mode 100644 example/cfis/config_MCCD.ini delete mode 100644 example/cfis/config_exp_Gie_vos.ini delete mode 100644 example/cfis/config_exp_Sp.ini delete mode 100644 example/cfis/config_exp_mccd.ini delete mode 100644 example/cfis/config_exp_psfex.ini delete mode 100644 example/cfis/config_get_tiles_vos_headers.ini delete mode 100644 example/cfis/config_make_cat_mccd.ini delete mode 100644 example/cfis/config_make_cat_psfex.ini delete mode 100644 example/cfis/config_make_cat_psfex_nosm.ini delete mode 100644 example/cfis/config_merge_sep_cats_template.ini delete mode 100644 example/cfis/config_tile_Fe.ini delete mode 100644 example/cfis/config_tile_Git_vos.ini delete mode 100644 example/cfis/config_tile_Mh_exp.ini delete mode 100644 example/cfis/config_tile_Ng_batch_psfex_uc.ini delete mode 100644 example/cfis/config_tile_Ng_template.ini delete mode 100644 example/cfis/config_tile_Ng_template_batch.ini delete mode 100644 example/cfis/config_tile_PiViVi_canfar_sx.ini delete mode 100644 example/cfis/config_tile_PiViVi_canfar_uc.ini delete mode 100644 example/cfis/config_tile_Sx.ini delete mode 100644 example/cfis/config_tile_Uz.ini delete mode 100644 example/cfis/default.conv delete mode 100644 example/cfis/default.param delete mode 100644 example/cfis/default.psfex delete mode 100644 example/cfis/default_exp.sex delete mode 100644 example/cfis/default_noimaflags.param delete mode 100644 example/cfis/default_tile.sex delete mode 100644 example/cfis/defunct/config_Gie_symlink.ini delete mode 100644 example/cfis/defunct/config_GitFeGie_symlink.ini delete mode 100644 example/cfis/defunct/config_GitFeGie_vos.ini delete mode 100644 example/cfis/defunct/config_GitFe_symlink.ini delete mode 100644 example/cfis/defunct/config_exp_SpMh.ini delete mode 100644 example/cfis/defunct/config_tile_PiViSmVi.ini delete mode 100644 example/cfis/defunct/config_tile_PiViSmVi_canfar.ini delete mode 100644 example/cfis/final_cat.param delete mode 100644 example/cfis/star_selection.setools mode change 120000 => 100644 workflow/config/cfis/default.conv mode change 120000 => 100644 workflow/config/cfis/default.param mode change 120000 => 100644 workflow/config/cfis/default.psfex mode change 120000 => 100644 workflow/config/cfis/default_exp.sex mode change 120000 => 100644 workflow/config/cfis/default_noimaflags.param mode change 120000 => 100644 workflow/config/cfis/default_tile.sex mode change 120000 => 100644 workflow/config/cfis/final_cat.param mode change 120000 => 100644 workflow/config/cfis/star_selection.setools diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md index e8a3768f4..f6af7f8e2 100644 --- a/docs/source/post_processing.md +++ b/docs/source/post_processing.md @@ -9,11 +9,6 @@ catalogue via _metacalibration_), a joint star catalogue, and PSF diagnostic plo ---- - -If main ShapePipe processing happened at the old canfar VM system (e.g. CFIS v0 and v1), go -[here](vos_retrieve.md) for details how to retrieve the ShapePipe output files. - --- ```{note} @@ -76,6 +71,6 @@ The following steps were used for pre-v1.4 runs performed on the canfar VM syste ``` Choose as input directory `input_dir` the `make_cat` output of the runs being combined. A default parameter file `` is - `/path/to/shapepipe/example/cfis/final_cat.param`. + `/path/to/shapepipe/workflow/config/cfis/final_cat.param`. On success, the file `./final_cat.npy` is created. Depending on the number of input tiles, this file can be several tens of Gb large. diff --git a/example/cfis/README.md b/example/cfis/README.md new file mode 100644 index 000000000..700f31e02 --- /dev/null +++ b/example/cfis/README.md @@ -0,0 +1,17 @@ +# CFIS PSF-validation and post-processing configs + +These configuration files cover PSF validation (`Ms`, `Pl`, `MsPl`, +`valjoint`) and assorted post-processing / statistics steps that are **not +(yet) covered by the Snakemake workflow**. They are kept here for reference +and manual use with `shapepipe_run -c `. + +The per-module pipeline configurations driven by the Snakemake workflow live +in [`workflow/config/cfis/`](../../workflow/config/cfis) instead; see +[`workflow/README.md`](../../workflow/README.md). + +The shared SExtractor / PSFEx / catalogue-parameter files (`default.sex` +variants, `default.param`, `default.psfex`, `default.conv`, `final_cat.param`, +`star_selection.setools`) also live in `workflow/config/cfis/`, which is what +`$SP_CONFIG` points at. None of the configs kept here reference them by path, +but if you add one that does, point it at `workflow/config/cfis/` rather than +copying the file back into this directory. diff --git a/example/cfis/config_MCCD.ini b/example/cfis/config_MCCD.ini deleted file mode 100644 index bf5e6852e..000000000 --- a/example/cfis/config_MCCD.ini +++ /dev/null @@ -1,120 +0,0 @@ -# Configuration file for the MCCD method - -[INPUTS] -INPUT_DIR = . -PREPROCESSED_OUTPUT_DIR = ./output -OUTPUT_DIR = ./output -INPUT_REGEX_FILE_PATTERN = star_split_ratio_80-*-*.fits -INPUT_SEPARATOR = - -MIN_N_STARS = 20 -OUTLIER_STD_MAX = 100. -USE_SNR_WEIGHTS = False - -[INSTANCE] -N_COMP_LOC = 8 -D_COMP_GLOB = 8 -KSIG_LOC = 0.00 -KSIG_GLOB = 0.00 -FILTER_PATH = None -D_HYB_LOC = 2 -MIN_D_COMP_GLOB = None -RMSE_THRESH = 1.25 -CCD_STAR_THRESH = 0.15 -FP_GEOMETRY = CFIS - -[FIT] -LOC_MODEL = hybrid -PSF_SIZE = 6.2 -PSF_SIZE_TYPE = R2 -N_EIGENVECTS = 5 -N_ITER_RCA = 1 -N_ITER_GLOB = 2 -N_ITER_LOC = 2 -NB_SUBITER_S_LOC = 300 -NB_SUBITER_A_LOC = 400 -NB_SUBITER_S_GLOB = 100 -NB_SUBITER_A_GLOB = 200 - -[VALIDATION] -VAL_DATA_INPUT_DIR = . -VAL_PREPROCESSED_OUTPUT_DIR = ./output -VAL_MODEL_INPUT_DIR = ./output -VAL_OUTPUT_DIR = ./output -VAL_REGEX_FILE_PATTERN = star_split_ratio_20-*-*.fits -VAL_SEPARATOR = - -APPLY_DEGRADATION = True -MCCD_DEBUG = False -GLOBAL_POL_INTERP = False - - -# Parameter description: -# -# -# [INPUTS] -# INPUT_DIR : (Required) Must be a valid directory containing the input -# MCCD files. -# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should -# follow regex (regular expression) standards. -# INPUT_SEPARATOR : Separator of the different fields in the filename, -# ie sexcat[SEP]catalog_id[SEP]CCD_id.fits -# MIN_N_STARS : Minimum number of stars to keep a CCD for the training. -# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection. -# Should not be too low as a hihg quantity of low quality -# stars will be rejected. ie 9 is a conservative rejection. -# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will -# be used. -# For now, it needs the SNR estimations from SExtractor. -# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the -# preprocessed input files. -# OUTPUT_DIR : (Required) Must be a valid directory to write the output files. -# The constructed models will be saved. -# -# -# [INSTANCE] -# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly, -# will be the max degree D of the polynomial. -# D_COMP_GLOB : Max degree of the global polynomial model. -# KSIG_LOC : Denoising parameter of the local model. -# ie 1 is a normal denoising, 3 is a hard denoising. -# KSIG_GLOB : Denoising parameter of the global model. -# ie 1 is a normal denoising, 3 is a hard denoising. -# FILTER_PATH : Path for predefined filters. -# -# -# [FIT] -# LOC_MODEL : Defines the type of local model to use, it can be: 'rca', -# 'poly' or 'hybrid'. -# When the poly model is used, N_COMP_LOC should be used -# as the D_LOC (max degree of the poly model) -# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways. -# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma -# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint -# construction. -# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between -# global and local estimations. -# N_ITER_GLOB : Number of epochs for each global optimization. Alternations -# between A_GLOB and S_GLOB. -# N_ITER_LOC : Number of epochs for each local optimization. Alternations -# between the different A_LOC and S_LOC. -# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC. -# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC. -# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB. -# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB. -# -# -# [VALIDATION] -# MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the -# saved trained models. -# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the -# validation input data (test dataset). -# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation. -# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation. -# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the -# validation outputs, test PSFs and interpolated PSFs. -# APPLY_DEGRADATION : Whether the PSF models should be degraded -# (sampling/shifts/flux) to match stars; use True if you -# plan on making pixel-based comparisons (residuals etc.). -# MCCD_DEBUG : Debug mode. Returns the local and global contributions. -# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model -# instead of RBF kernel interpolation. -# diff --git a/example/cfis/config_exp_Gie_vos.ini b/example/cfis/config_exp_Gie_vos.ini deleted file mode 100644 index 26f608b53..000000000 --- a/example/cfis/config_exp_Gie_vos.ini +++ /dev/null @@ -1,99 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Gie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get exposures -[GET_IMAGES_RUNNER] - -INPUT_DIR = last:find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = vos - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem - -#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/example/cfis/config_exp_Sp.ini b/example/cfis/config_exp_Sp.ini deleted file mode 100644 index 066c21e54..000000000 --- a/example/cfis/config_exp_Sp.ini +++ /dev/null @@ -1,74 +0,0 @@ -# ShapePipe configuration file for single-exposures, -# split images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Sp - -# Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = split_exp_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SPLIT_EXP_RUNNER] - -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = image, weight, flag - -# Matches compressed single-exposure files -FILE_EXT = .fitsfz, .fitsfz, .fitsfz - -NUMBERING_SCHEME = -0000000 - -# OUTPUT_SUFFIX, actually file name prefixes. -# Expected keyword "flag" will lead to a behavior where the data are saved as int. -# The code also expects the image data to use the "image" suffix -# (default value in the pipeline). -OUTPUT_SUFFIX = image, weight, flag - -# Number of HDUs/CCDs of mosaic -N_HDU = 40 diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini deleted file mode 100644 index be3f2690c..000000000 --- a/example/cfis/config_exp_mccd.ini +++ /dev/null @@ -1,271 +0,0 @@ -# ShapePipe configuration file for single-exposures, MCCD PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, mask_query_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . - -# The split CCDs, and nothing else: ShapePipe generates no masks -INPUT_MODULE = split_exp_runner - -# Read the instrument flag image split_exp wrote per CCD -FILE_PATTERN = image, weight, flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -# True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND, BACKGROUND_RMS - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[MASK_QUERY_RUNNER] - -INPUT_MODULE = last:sextractor_runner - -# Note: Make sure this does not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat_sexcat - -NUMBERING_SCHEME = -0000000-0 - -# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by -# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), -# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag -# objects for the final catalogue, they do not reject PSF stars (mask-force -# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by -# adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets MASK_EXT. Comma-separated. -# -# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue -# is passed through with no MASK_EXT column, the same gating make_cat gives -# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning -# the query on is uncommenting one line and never editing the chain. -# -# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive -# (instrument flags only) and the column is carried for transparency and -# measurement — see star_selection.setools' header for the one-line change -# that would impose it. -; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp - -# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, -# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map -# per bit — ignore it, which is why the diet above is a path list and not a -# bit mask. -; MASK_BITS = 4 - - -[SETOOLS_RUNNER] - -INPUT_MODULE = last:mask_query_runner - -FILE_PATTERN = sexcat_ext - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[MCCD_PREPROCESSING_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -INPUT_DIR = last:setools_runner - -# Input are individual CCDs, thus single-exposure single-HDU images -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - - -[MCCD_FIT_VAL_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -NUMBERING_SCHEME = -0000000 - - -[MCCD_MERGE_STARCAT_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -NUMBERING_SCHEME = -0000000 - - -[MCCD_PLOTS_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -# Now MCCD has created a focal-plane PSF model, including all CCDS per images, -# thus single-exposure files -NUMBERING_SCHEME = -0000000 - -PLOT_MEANSHAPES = True - -# X_GRID, Y_GRID: correspond to the number of bins in each direction of each -# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. -X_GRID = 5 -Y_GRID = 10 - -PLOT_HISTOGRAMS = True - -# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape -# before drawing the plots. -REMOVE_OUTLIERS = False - - - -[MCCD_INTERP_RUNNER] - -# MODE: Define the way the MCCD interpolation will run. -# CLASSIC for classical run. -# MULTI-EPOCH for multi epoch. -MODE = CLASSIC - -# Position parameter names -# for multi-epoch XWIN_WORLD,YWIN_WORLD -# For classical XWIN_IMAGE,YWIN_IMAGE: -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Get PSF shapes calculated and saved on the output dict -GET_SHAPES = True - -# Directory with PSF models -PSF_MODEL_DIR = /Users/tliaudat/Documents/PhD/codes/venv_p3/MCCD_pipeline_integration/test_val_data/fitted_model/ - -# PSF model patterns -PSF_MODEL_PATTERN = fitted_model - -# PSF model separator -PSF_MODEL_SEPARATOR = - diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini deleted file mode 100644 index fe3ecad43..000000000 --- a/example/cfis/config_exp_psfex.ini +++ /dev/null @@ -1,218 +0,0 @@ -# ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after splitting, from star detection to PSF model. -# ShapePipe generates no masks: SExtractor reads the instrument flag image -# delivered with the exposure, and mask_query flags detections against the -# external healsparse maps (see [MASK_QUERY_RUNNER] below). - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SxSePsfPi -#RUN_NAME = run_sp_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -# The split CCDs, and nothing else: ShapePipe generates no masks -INPUT_DIR = last:split_exp_runner - -# Read the instrument flag image split_exp wrote per CCD -FILE_PATTERN = image, weight, flag - -# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on -# the decorator's 4-entry FILE_EXT default (length check fails at startup) -FILE_EXT = .fits, .fits, .fits - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -# True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND, BACKGROUND_RMS - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[MASK_QUERY_RUNNER] - -INPUT_DIR = last:sextractor_runner - -# Note: Make sure this does not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by -# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), -# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag -# objects for the final catalogue, they do not reject PSF stars (mask-force -# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by -# adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets MASK_EXT. Comma-separated. -# -# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue -# is passed through with no MASK_EXT column, the same gating make_cat gives -# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning -# the query on is uncommenting one line and never editing the chain. -# -# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive -# (instrument flags only) and the column is carried for transparency and -# measurement — see star_selection.setools' header for the one-line change -# that would impose it. -; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp - -# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, -# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map -# per bit — ignore it, which is why the diet above is a path list and not a -# bit mask. -; MASK_BITS = 4 - - -[SETOOLS_RUNNER] - -INPUT_DIR = last:mask_query_runner - -FILE_PATTERN = sexcat_ext - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[PSFEX_RUNNER] - -# Use 80% sample for PSF model -FILE_PATTERN = star_split_ratio_80 - -NUMBERING_SCHEME = -0000000-0 - -# Path to executable for the PSF model (optional) -EXEC_PATH = psfex - -# Default psfex configuration file -DOT_PSFEX_FILE = $SP_CONFIG/default.psfex - -[PSFEX_INTERP_RUNNER] - -# Use 20% sample for PSF validation -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat - -FILE_EXT = .psf, .fits, .cat - -NUMBERING_SCHEME = -0000000-0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = VALIDATION - -# Column names of position parameters -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# If True, measure and store ellipticity of the PSF (using moments) -GET_SHAPES = True - -# Minimum number of stars per CCD for PSF model to be computed -STAR_THRESH = 22 - -# Maximum chi^2 for PSF model to be computed on CCD -CHI2_THRESH = 2 diff --git a/example/cfis/config_get_tiles_vos_headers.ini b/example/cfis/config_get_tiles_vos_headers.ini deleted file mode 100644 index 4b79a2493..000000000 --- a/example/cfis/config_get_tiles_vos_headers.ini +++ /dev/null @@ -1,95 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Git - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[GET_IMAGES_RUNNER] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/tiles_DR5 - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r - -# Input file extensions -INPUT_FILE_EXT = .fits - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image- -#, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = vos - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Copy command options, optional -RETRIEVE_OPTIONS = --head diff --git a/example/cfis/config_make_cat_mccd.ini b/example/cfis/config_make_cat_mccd.ini deleted file mode 100644 index 860f2020a..000000000 --- a/example/cfis/config_make_cat_mccd.ini +++ /dev/null @@ -1,74 +0,0 @@ -# ShapePipe post-run configuration file: create final catalogs - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Mc - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = make_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = last:sextractor_runner_run_1, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner - -# Output directory -OUTPUT_DIR = ./output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MAKE_CAT_RUNNER] - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .sqlite, .fits - -# Numbering convention, string that exemplifies a numbering pattern. -# Matches input single exposures (with 'p' removed) -# Needs to be given in this section, will be updated in module -# sections below -NUMBERING_SCHEME = -000-000 - -SM_DO_CLASSIFICATION = True -SM_STAR_THRESH = 0.003 -SM_GAL_THRESH = 0.01 - -SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis/config_make_cat_psfex.ini b/example/cfis/config_make_cat_psfex.ini deleted file mode 100644 index c83fcfad3..000000000 --- a/example/cfis/config_make_cat_psfex.ini +++ /dev/null @@ -1,83 +0,0 @@ -# ShapePipe post-run configuration file: create final catalogs - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Mc - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = make_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = ./output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MAKE_CAT_RUNNER] - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:spread_model_runner, last:psfex_interp_runner, last:merge_sep_cats_runner - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .sqlite, .fits - -# Numbering convention, string that exemplifies a numbering pattern. -# Matches input single exposures (with 'p' removed) -# Needs to be given in this section, will be updated in module -# sections below -NUMBERING_SCHEME = -000-000 - -SM_DO_CLASSIFICATION = True -SM_STAR_THRESH = 0.003 -SM_GAL_THRESH = 0.01 - -SHAPE_MEASUREMENT_TYPE = ngmix - -# Optional per-band external healsparse mask lookup (band:path pairs, -# comma-separated). All bits of each map are queried at each object's -# (RA, Dec) and written verbatim to a MASK_ column; no filtering -# is applied here. Absent this key, the step is a no-op. -; MASK_EXT_PATHS = r:$SP_CONFIG/mask_r.hsp, u:$SP_CONFIG/mask_u.hsp diff --git a/example/cfis/config_make_cat_psfex_nosm.ini b/example/cfis/config_make_cat_psfex_nosm.ini deleted file mode 100644 index 50f5c0612..000000000 --- a/example/cfis/config_make_cat_psfex_nosm.ini +++ /dev/null @@ -1,76 +0,0 @@ -# ShapePipe post-run configuration file: create final catalogs, with -# no spread model on input - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Mc - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = make_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = ./output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MAKE_CAT_RUNNER] - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:merge_sep_cats_runner - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = sexcat, galaxy_psf, ngmix - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .sqlite, .fits - -# Numbering convention, string that exemplifies a numbering pattern. -# Matches input single exposures (with 'p' removed) -# Needs to be given in this section, will be updated in module -# sections below -NUMBERING_SCHEME = -000-000 - -SM_DO_CLASSIFICATION = False - -SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis/config_merge_sep_cats_template.ini b/example/cfis/config_merge_sep_cats_template.ini deleted file mode 100644 index 96c4557f5..000000000 --- a/example/cfis/config_merge_sep_cats_template.ini +++ /dev/null @@ -1,72 +0,0 @@ -# ShapePipe post-run configuration file: merge separated catalogues - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ms - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = merge_sep_cats_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output - -# Output directory -OUTPUT_DIR = ./output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MERGE_SEP_CATS_RUNNER] - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = ngmix - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits - -# Numbering convention, string that exemplifies a numbering pattern. -NUMBERING_SCHEME = -000-000 - -# WARNING (optional, default is 'error'). Use 'always'/'ignore' to -# display/ignore warnings, and not raise error -WARNING = always - -# Maximum number of separated catalogues per input -N_SPLIT_MAX = X diff --git a/example/cfis/config_tile_Fe.ini b/example/cfis/config_tile_Fe.ini deleted file mode 100644 index 2661259d7..000000000 --- a/example/cfis/config_tile_Fe.ini +++ /dev/null @@ -1,72 +0,0 @@ -# ShapePipe configuration file for: find exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Fe - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = find_exposures_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[FIND_EXPOSURES_RUNNER] - -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p - diff --git a/example/cfis/config_tile_Git_vos.ini b/example/cfis/config_tile_Git_vos.ini deleted file mode 100644 index 52fbdc15b..000000000 --- a/example/cfis/config_tile_Git_vos.ini +++ /dev/null @@ -1,93 +0,0 @@ -# ShapePipe configuration file for: get tile images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Git - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/tiles_DR6, vos:cfis/tiles_DR6 - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = vos - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Copy command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem - -#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/example/cfis/config_tile_Mh_exp.ini b/example/cfis/config_tile_Mh_exp.ini deleted file mode 100644 index 0d3f9f8b3..000000000 --- a/example/cfis/config_tile_Mh_exp.ini +++ /dev/null @@ -1,72 +0,0 @@ -# ShapePipe configuration file for merging per-exposure WCS headers -# at the tile level. Input is the exp_numbers file produced by -# find_exposures_runner; EXP_BASE_DIR tells the runner where to find -# the per-exposure split_exp_runner header .npy files. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Mh_exp - -# Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = merge_headers_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MERGE_HEADERS_RUNNER] - -# Input: exp_numbers txt file from find_exposures_runner -INPUT_DIR = last:find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# Tile numbering scheme (RA-Dec, e.g. -301-279) -NUMBERING_SCHEME = -000-000 - -# Root directory containing all per-exposure work directories. -# The runner will walk this tree to collect headers-.npy files. -EXP_BASE_DIR = $SP_EXP diff --git a/example/cfis/config_tile_Ng_batch_psfex_uc.ini b/example/cfis/config_tile_Ng_batch_psfex_uc.ini deleted file mode 100644 index f2696bd8f..000000000 --- a/example/cfis/config_tile_Ng_batch_psfex_uc.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ng - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 24 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner,last:psfex_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as -# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for -# every tile (missing file -> error, no per-tile fallback); omit the option -# entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -SAVE_BATCH = 1000 - -ID_OBJ_MIN = -1 -ID_OBJ_MAX = -1 diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini deleted file mode 100644 index d928cc1be..000000000 --- a/example/cfis/config_tile_Ng_template.ini +++ /dev/null @@ -1,112 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_NgXu - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as -# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for -# every tile (missing file -> error, no per-tile fallback); omit the option -# entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -# CENTROID_SOURCE (optional): how to place the galaxy Jacobian origin for the -# centroid prior. "wcs" (default) uses the catalog sky position projected -# through the WCS, trusting the astrometry — the recommended choice. "hsm" -# re-centers on the HSM adaptive-moment centroid (legacy, being phased out; -# noisy for stars and flagged as incorrect — see #767). -CENTROID_SOURCE = wcs - -# BLEND_HANDLING (optional): neighbour treatment. "noisefill" (default, -# historical) replaces a neighbour's pixels with a noise realisation; -# "uberseg" hard-masks (weight -> 0) every pixel closer to a neighbour's -# segmentation footprint than to the central object. "uberseg" REQUIRES -# SEG_VIGNET_PATH below. -# BLEND_HANDLING = uberseg - -# SEG_VIGNET_PATH (optional): coadd-frame SExtractor segmentation vignets -# (the CLASSIC-mode VIGNETMAKER_RUNNER_RUN_3 seg output), row-aligned to the -# tile catalogue and on the same 51x51 grid as the coadd VIGNET. Required for -# BLEND_HANDLING = uberseg; when set, the file must exist for every tile -# (missing file -> error). Omit for the noise-fill path. -# SEG_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_3/output/seg_vignet{file_number_string}.fits - -# DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the uberseg -# neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay offset. -# Ignored unless BLEND_HANDLING = uberseg. Default 1 (~one pixel); 0 recovers -# the pure Sheldon UberSeg mask. -# DILATE_NEIGHBOUR = 1 - -# METACAL_PSF (optional): the metacal reconvolution-kernel scheme -# (metacal_pars['psf']). "fitgauss" (default) fits a Gaussian to the PSF and -# rounds it; "gauss" reconvolves with a fixed round Gaussian sized from the -# PSF; "dilate" dilates the original PSF; "azgauss" (ngmix >= 2.4.1) is a -# noise-robust variant of "gauss". Sets the round PSF metacal reconvolves -# with after shearing, so it moves the metacal response. -METACAL_PSF = fitgauss - -ID_OBJ_MIN = X -ID_OBJ_MAX = X diff --git a/example/cfis/config_tile_Ng_template_batch.ini b/example/cfis/config_tile_Ng_template_batch.ini deleted file mode 100644 index 655397156..000000000 --- a/example/cfis/config_tile_Ng_template_batch.ini +++ /dev/null @@ -1,81 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_NgXu - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as -# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for -# every tile (missing file -> error, no per-tile fallback); omit the option -# entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite - -# Number of objects to batch save during processing, optional. Omit or set -# to -1 for no batch saving -SAVE_BATCH = 1000 - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = X -ID_OBJ_MAX = X diff --git a/example/cfis/config_tile_PiViVi_canfar_sx.ini b/example/cfis/config_tile_PiViVi_canfar_sx.ini deleted file mode 100644 index 71e4b17e3..000000000 --- a/example/cfis/config_tile_PiViVi_canfar_sx.ini +++ /dev/null @@ -1,204 +0,0 @@ -# ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_PiViVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -#MODULE = psfex_interp_runner, - -MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner, vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[PSFEX_INTERP_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner - -FILE_PATTERN = sexcat, log_exp_headers, exp_numbers - -FILE_EXT = .fits, .sqlite, .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -# Number of stars threshold -STAR_THRESH = 22 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR -# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered -# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. -ME_DOT_PSF_EXP_DIR = $SP_EXP - -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - - -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner - -FILE_PATTERN = sexcat, log_exp_headers, exp_numbers - -FILE_EXT = .fits, .sqlite, .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -PREFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for -# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. -ME_IMAGE_EXP_DIR = $SP_EXP -ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner -ME_IMAGE_PATTERN = flag, image, weight, background, background_rms - - -[VIGNETMAKER_RUNNER_RUN_3] - -# Cut per-object coadd-frame segmentation stamps from the tile SExtractor -# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, -# SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode -# guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN -# centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg -# stamp directly for uberseg neighbour masking (shapepipe#776). - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner - -FILE_PATTERN = sexcat, segmentation - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# CLASSIC: cut stamps at object positions in the coadd (tile) frame. -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels — MUST equal the coadd VIGNET size (51) so the seg -# stamp overlays the galaxy stamp on an identical grid. -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = seg diff --git a/example/cfis/config_tile_PiViVi_canfar_uc.ini b/example/cfis/config_tile_PiViVi_canfar_uc.ini deleted file mode 100644 index b33361b6d..000000000 --- a/example/cfis/config_tile_PiViVi_canfar_uc.ini +++ /dev/null @@ -1,166 +0,0 @@ -# ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_PiViVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[PSFEX_INTERP_RUNNER] - -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner - -FILE_PATTERN = sexcat, log_exp_headers, exp_numbers - -FILE_EXT = .fits, .sqlite, .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -# Number of stars threshold -STAR_THRESH = 22 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR -# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered -# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. -ME_DOT_PSF_EXP_DIR = $SP_EXP - -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - - -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner, last:uncompress_fits_runner - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner - -FILE_PATTERN = sexcat, log_exp_headers, exp_numbers - -FILE_EXT = .fits, .sqlite, .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -PREFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for -# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. -ME_IMAGE_EXP_DIR = $SP_EXP -ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner -ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini deleted file mode 100644 index 80350eead..000000000 --- a/example/cfis/config_tile_Sx.ini +++ /dev/null @@ -1,119 +0,0 @@ -# ShapePipe configuration file for tile detection -# -# No flag image: ShapePipe generates no tile masks, and tiles have no -# instrument flag image of their own. Sky-fixed masks reach the catalogue as -# MASK_ columns, queried per object by make_cat. Hence -# default_noimaflags.param and FLAG_IMAGE = False below. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers - -FILE_EXT = .fits, .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = False - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND, SEGMENTATION - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis/config_tile_Uz.ini b/example/cfis/config_tile_Uz.ini deleted file mode 100644 index 3521f0aca..000000000 --- a/example/cfis/config_tile_Uz.ini +++ /dev/null @@ -1,69 +0,0 @@ -# ShapePipe configuration file for: uncompress FITS image - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Uz - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = uncompress_fits_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[UNCOMPRESS_FITS_RUNNER] - -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = CFIS_weight - -FILE_EXT = .fitsfz - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input HDU of image data, optional, default=0 -HDU_DATA = 1 - -# Output file pattern -OUTPUT_PATTERN = CFIS_weight diff --git a/example/cfis/default.conv b/example/cfis/default.conv deleted file mode 100644 index 2590b9cba..000000000 --- a/example/cfis/default.conv +++ /dev/null @@ -1,5 +0,0 @@ -CONV NORM -# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. -1 2 1 -2 4 2 -1 2 1 diff --git a/example/cfis/default.param b/example/cfis/default.param deleted file mode 100644 index 09ad8405e..000000000 --- a/example/cfis/default.param +++ /dev/null @@ -1,68 +0,0 @@ -NUMBER #Running object number -EXT_NUMBER #FITS extension number - -FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] -FLUXERR_AUTO #RMS error for AUTO flux [count] -MAG_AUTO #Kron-like elliptical aperture magnitude [mag] -MAGERR_AUTO #RMS error for AUTO magnitude [mag] -FLUX_WIN #Gaussian-weighted flux [count] -FLUXERR_WIN #RMS error for WIN flux [count] -MAG_WIN #Gaussian-weighted magnitude [mag] -MAGERR_WIN #RMS error for MAG_WIN [mag] -FLUX_APER(1) -FLUXERR_APER(1) - -FLUX_RADIUS #Fraction-of-light radii [pixel] - -SNR_WIN #Gaussian-weighted SNR - -BACKGROUND #Background at centroid position [count] -THRESHOLD #Detection threshold above background [count] - -X_IMAGE #Object position along x [pixel] -Y_IMAGE #Object position along y [pixel] - -X_WORLD #Barycenter position along world x axis [deg] -Y_WORLD #Barycenter position along world y axis [deg] - -X2_IMAGE #Variance along x [pixel**2] -Y2_IMAGE #Variance along y [pixel**2] -XY_IMAGE #Covariance between x and y [pixel**2] -ERRX2_IMAGE #Variance of position along x [pixel**2] -ERRY2_IMAGE #Variance of position along y [pixel**2] -ERRXY_IMAGE #Covariance of position between x and y [pixel**2] - -XWIN_IMAGE #Windowed position estimate along x [pixel] -YWIN_IMAGE #Windowed position estimate along y [pixel] - -XWIN_WORLD #Windowed position along world x axis [deg] -YWIN_WORLD #Windowed position along world y axis [deg] - -X2WIN_IMAGE #Windowed variance along x [pixel**2] -Y2WIN_IMAGE #Windowed variance along y [pixel**2] -XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] -ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] -ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] -ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] - -MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] -MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] - -FLAGS #Extraction flags -FLAGS_WIN #Flags for WINdowed parameters - -# The following flag requires a flag image -IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! - -FWHM_IMAGE #FWHM assuming a gaussian core [pixel] -FWHM_WORLD #FWHM assuming a gaussian core [deg] -ELONGATION #A_IMAGE/B_IMAGE -ELLIPTICITY #1 - B_IMAGE/A_IMAGE - -VIGNET(51,51) #Pixel data around detection [count] - -# For GaaP photometry -A_WORLD -B_WORLD -THETA_J2000 - diff --git a/example/cfis/default.psfex b/example/cfis/default.psfex deleted file mode 100644 index a9d1a906c..000000000 --- a/example/cfis/default.psfex +++ /dev/null @@ -1,85 +0,0 @@ -# Default configuration file for PSFEx 3.17.1 -# EB 2017-11-30 -# - -#-------------------------------- PSF model ---------------------------------- - -BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE -BASIS_NUMBER 20 # Basis number or parameter -BASIS_NAME basis.fits # Basis filename (FITS data-cube) -BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter -NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT - # or PCA_COMMON -NEWBASIS_NUMBER 8 # Number of new basis vectors -PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) -PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units -PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values -PSF_SIZE 51,51 # Image size of the PSF model -PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? -MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON - -#------------------------- Point source measurements ------------------------- - -CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering -PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. -PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error - -#----------------------------- PSF variability ------------------------------- - -PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params -PSFVAR_GROUPS 1,1 # Group tag for each context key -PSFVAR_DEGREES 2 # Polynom degree for each group -PSFVAR_NSNAP 9 # Number of PSF snapshots per axis -HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON -STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE - -#----------------------------- Sample selection ------------------------------ - -SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? - -BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? -BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed - -#----------------------- PSF homogeneisation kernel -------------------------- - -HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE -HOMOBASIS_NUMBER 10 # Kernel basis number or parameter -HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter -HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF -HOMOKERNEL_DIR # Where to write kernels (empty=same as input) -HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels - -#----------------------------- Output catalogs ------------------------------- - -OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC - -#------------------------------- Check-plots ---------------------------------- - -CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, - # JPEG, AQT, PDF or SVG -CHECKPLOT_RES 0 # Check-plot resolution (0 = default) -CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? -CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS -CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS - # or NONE -CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi - -#------------------------------ Check-Images --------------------------------- - -# Note: Check-image types can be set the ShapePipe config file, psfex_runner section -####### -CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS - # or MOFFAT,-MOFFAT,-SYMMETRICAL -#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits - # Check-image filenames -#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? - -#----------------------------- Miscellaneous --------------------------------- - -PSF_SUFFIX .psf # Filename extension for output PSF filename -VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL -WRITE_XML N # Write XML file (Y/N)? - -NTHREADS 1 # Number of simultaneous threads for - # the SMP version of PSFEx - # 0 = automatic diff --git a/example/cfis/default_exp.sex b/example/cfis/default_exp.sex deleted file mode 100644 index b87275ecb..000000000 --- a/example/cfis/default_exp.sex +++ /dev/null @@ -1,133 +0,0 @@ -# Default configuration file for SExtractor 2.19.5 -# EB 2017-11-30 -# - -#-------------------------------- Catalog ------------------------------------ - -CATALOG_TYPE FITS_LDAC - -PARAMETERS_NAME default.param - -#------------------------------- Extraction ---------------------------------- - -DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) -DETECT_MINAREA 5 # min. # of pixels above threshold -DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) -THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) - # or ABSOLUTE (in ADUs) -DETECT_THRESH 1.5 # or , in mag.arcsec-2 -ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 - -FILTER Y # apply filter for detection (Y or N)? -FILTER_NAME default.conv -FILTER_THRESH # Threshold[s] for retina filtering - -DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds -DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending - -CLEAN Y # Clean spurious detections? (Y or N)? -CLEAN_PARAM 1.0 # Cleaning efficiency - -MASK_TYPE CORRECT # type of detection MASKing: can be one of - # NONE, BLANK or CORRECT - -#-------------------------------- WEIGHTing ---------------------------------- - -WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, - # MAP_RMS, MAP_VAR or MAP_WEIGHT -RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? -WEIGHT_IMAGE weight.fits # weight-map filename -WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) -WEIGHT_THRESH # weight threshold[s] for bad pixels - -#-------------------------------- FLAGging ----------------------------------- - -FLAG_IMAGE flag.fits # filename for an input FLAG-image -FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX - # or MOST - -#------------------------------ Photometry ----------------------------------- - -PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels -PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , -PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , - # -PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures - # for MAG_AUTO and MAG_PETRO -PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS - -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) - -MAG_ZEROPOINT 30.0 # magnitude zero-point -MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) - -GAIN_KEY GAIN # keyword for detector gain in e-/ADU -PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) - -#------------------------- Star/Galaxy Separation ---------------------------- - -SEEING_FWHM 0.6 # stellar FWHM in arcsec -STARNNW_NAME default.nnw - -#------------------------------ Background ----------------------------------- - -BACK_TYPE AUTO # AUTO or MANUAL -BACK_VALUE 0.0 # Default background value in MANUAL mode -BACK_SIZE 64 # Background mesh: or , -BACK_FILTERSIZE 3 # Background filter: or , - -BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL -BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus -BACK_FILTTHRESH 0.0 # Threshold above which the background- - # map filter operates - -#------------------------------ Check Image ---------------------------------- - -####### -## AG : This parameter is set in pipeline config file. -####### -# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND -# can be NONE, BACKGROUND, BACKGROUND_RMS, - # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, - # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, - # or APERTURES -# CHECKIMAGE_NAME check.fits,back.fits -# Filename for the check-image - -#--------------------- Memory (change with caution!) ------------------------- - -MEMORY_OBJSTACK 3000 # number of objects in stack -MEMORY_PIXSTACK 300000 # number of pixels in stack -MEMORY_BUFSIZE 1024 # number of lines in buffer - -#------------------------------- ASSOCiation --------------------------------- - -ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate -ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) -ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] -ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD -ASSOC_RADIUS 2.0 # cross-matching radius (pixels) -ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, - # MAG_MEAN, SUM, MAG_SUM, MIN or MAX -ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED - -#----------------------------- Miscellaneous --------------------------------- - -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -HEADER_SUFFIX .head # Filename extension for additional headers -WRITE_XML N # Write XML file (Y/N)? - -NTHREADS 1 # 1 single thread - -FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? -INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation -INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation -INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL - -#--------------------------- Experimental Stuff ----------------------------- - -#PSF_NAME default.psf # File containing the PSF model -#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously -#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, - # RINGS-HARMONICS or GAUSS-LAGUERRE -#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/example/cfis/default_noimaflags.param b/example/cfis/default_noimaflags.param deleted file mode 100644 index b251f5b74..000000000 --- a/example/cfis/default_noimaflags.param +++ /dev/null @@ -1,65 +0,0 @@ -NUMBER #Running object number -EXT_NUMBER #FITS extension number - -FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] -FLUXERR_AUTO #RMS error for AUTO flux [count] -MAG_AUTO #Kron-like elliptical aperture magnitude [mag] -MAGERR_AUTO #RMS error for AUTO magnitude [mag] -FLUX_WIN #Gaussian-weighted flux [count] -FLUXERR_WIN #RMS error for WIN flux [count] -MAG_WIN #Gaussian-weighted magnitude [mag] -MAGERR_WIN #RMS error for MAG_WIN [mag] -FLUX_APER(1) -FLUXERR_APER(1) - -FLUX_RADIUS #Fraction-of-light radii [pixel] - -SNR_WIN #Gaussian-weighted SNR - -BACKGROUND #Background at centroid position [count] -THRESHOLD #Detection threshold above background [count] - -X_IMAGE #Object position along x [pixel] -Y_IMAGE #Object position along y [pixel] - -X_WORLD #Barycenter position along world x axis [deg] -Y_WORLD #Barycenter position along world y axis [deg] - -X2_IMAGE #Variance along x [pixel**2] -Y2_IMAGE #Variance along y [pixel**2] -XY_IMAGE #Covariance between x and y [pixel**2] -ERRX2_IMAGE #Variance of position along x [pixel**2] -ERRY2_IMAGE #Variance of position along y [pixel**2] -ERRXY_IMAGE #Covariance of position between x and y [pixel**2] - -XWIN_IMAGE #Windowed position estimate along x [pixel] -YWIN_IMAGE #Windowed position estimate along y [pixel] - -XWIN_WORLD #Windowed position along world x axis [deg] -YWIN_WORLD #Windowed position along world y axis [deg] - -X2WIN_IMAGE #Windowed variance along x [pixel**2] -Y2WIN_IMAGE #Windowed variance along y [pixel**2] -XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] -ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] -ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] -ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] - -MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] -MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] - -FLAGS #Extraction flags -FLAGS_WIN #Flags for WINdowed parameters - -FWHM_IMAGE #FWHM assuming a gaussian core [pixel] -FWHM_WORLD #FWHM assuming a gaussian core [deg] -ELONGATION #A_IMAGE/B_IMAGE -ELLIPTICITY #1 - B_IMAGE/A_IMAGE - -VIGNET(51,51) #Pixel data around detection [count] - -# For GaaP photometry -A_WORLD -B_WORLD -THETA_J2000 - diff --git a/example/cfis/default_tile.sex b/example/cfis/default_tile.sex deleted file mode 100644 index ff3b25213..000000000 --- a/example/cfis/default_tile.sex +++ /dev/null @@ -1,133 +0,0 @@ -# Default configuration file for SExtractor 2.19.5 -# EB 2017-11-30 -# - -#-------------------------------- Catalog ------------------------------------ - -CATALOG_TYPE FITS_LDAC - -PARAMETERS_NAME default.param - -#------------------------------- Extraction ---------------------------------- - -DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) -DETECT_MINAREA 5 # min. # of pixels above threshold -DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) -THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) - # or ABSOLUTE (in ADUs) -DETECT_THRESH 1.5 # or , in mag.arcsec-2 -ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 - -FILTER Y # apply filter for detection (Y or N)? -FILTER_NAME default.conv -FILTER_THRESH # Threshold[s] for retina filtering - -DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds -DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending - -CLEAN Y # Clean spurious detections? (Y or N)? -CLEAN_PARAM 1.0 # Cleaning efficiency - -MASK_TYPE CORRECT # type of detection MASKing: can be one of - # NONE, BLANK or CORRECT - -#-------------------------------- WEIGHTing ---------------------------------- - -WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, - # MAP_RMS, MAP_VAR or MAP_WEIGHT -RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? -WEIGHT_IMAGE weight.fits # weight-map filename -WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) -WEIGHT_THRESH # weight threshold[s] for bad pixels - -#-------------------------------- FLAGging ----------------------------------- - -FLAG_IMAGE flag.fits # filename for an input FLAG-image -FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX - # or MOST - -#------------------------------ Photometry ----------------------------------- - -PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels -PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , -PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , - # -PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures - # for MAG_AUTO and MAG_PETRO -PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS - -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) - -MAG_ZEROPOINT 30.0 # magnitude zero-point -MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) - -GAIN_KEY GAIN # keyword for detector gain in e-/ADU -PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) - -#------------------------- Star/Galaxy Separation ---------------------------- - -SEEING_FWHM 0.6 # stellar FWHM in arcsec -STARNNW_NAME default.nnw - -#------------------------------ Background ----------------------------------- - -BACK_TYPE MANUAL # AUTO or MANUAL -BACK_VALUE 0.0 # Default background value in MANUAL mode -BACK_SIZE 64 # Background mesh: or , -BACK_FILTERSIZE 3 # Background filter: or , - -BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL -BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus -BACK_FILTTHRESH 0.0 # Threshold above which the background- - # map filter operates - -#------------------------------ Check Image ---------------------------------- - -####### -## AG : This parameter is set in pipeline config file. -####### -# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND -# can be NONE, BACKGROUND, BACKGROUND_RMS, - # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, - # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, - # or APERTURES -# CHECKIMAGE_NAME check.fits,back.fits -# Filename for the check-image - -#--------------------- Memory (change with caution!) ------------------------- - -MEMORY_OBJSTACK 3000 # number of objects in stack -MEMORY_PIXSTACK 300000 # number of pixels in stack -MEMORY_BUFSIZE 1024 # number of lines in buffer - -#------------------------------- ASSOCiation --------------------------------- - -ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate -ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) -ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] -ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD -ASSOC_RADIUS 2.0 # cross-matching radius (pixels) -ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, - # MAG_MEAN, SUM, MAG_SUM, MIN or MAX -ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED - -#----------------------------- Miscellaneous --------------------------------- - -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -HEADER_SUFFIX .head # Filename extension for additional headers -WRITE_XML N # Write XML file (Y/N)? - -NTHREADS 1 # 1 single thread - -FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? -INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation -INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation -INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL - -#--------------------------- Experimental Stuff ----------------------------- - -#PSF_NAME default.psf # File containing the PSF model -#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously -#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, - # RINGS-HARMONICS or GAUSS-LAGUERRE -#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/example/cfis/defunct/config_Gie_symlink.ini b/example/cfis/defunct/config_Gie_symlink.ini deleted file mode 100644 index e6fa49f4c..000000000 --- a/example/cfis/defunct/config_Gie_symlink.ini +++ /dev/null @@ -1,97 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Gie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get exposures -[GET_IMAGES_RUNNER] - -INPUT_DIR = last:find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem diff --git a/example/cfis/defunct/config_GitFeGie_symlink.ini b/example/cfis/defunct/config_GitFeGie_symlink.ini deleted file mode 100644 index 6f84d1482..000000000 --- a/example/cfis/defunct/config_GitFeGie_symlink.ini +++ /dev/null @@ -1,151 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_GitFeGie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner, find_exposures_runner, get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER_RUN_1] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink - -# Copy command options, optional -RETRIEVE_OPTIONS = -L - - -[FIND_EXPOSURES_RUNNER] - -INPUT_MODULE = get_images_runner_run_1 - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p - -# Get exposures -[GET_IMAGES_RUNNER_RUN_2] - -INPUT_MODULE = find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = -L diff --git a/example/cfis/defunct/config_GitFeGie_vos.ini b/example/cfis/defunct/config_GitFeGie_vos.ini deleted file mode 100644 index 75044bf44..000000000 --- a/example/cfis/defunct/config_GitFeGie_vos.ini +++ /dev/null @@ -1,155 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_GitFeGie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner, find_exposures_runner, get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER_RUN_1] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/tiles_DR5, vos:cfis/tiles_DR5 - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = vos - -# Copy command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem - -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output - -[FIND_EXPOSURES_RUNNER] - -INPUT_MODULE = get_images_runner_run_1 - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p - - -# Get exposures -[GET_IMAGES_RUNNER_RUN_2] - -INPUT_DIR = last:find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags -# LSB images: -#INPUT_PATH = vos:cfis/lsb_individual, vos:cfis/weights, vos:cfis/flags - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag -# LSB images -#INPUT_FILE_PATTERN = 000000s, 000000p.weight, 000000p.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = vos - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem diff --git a/example/cfis/defunct/config_GitFe_symlink.ini b/example/cfis/defunct/config_GitFe_symlink.ini deleted file mode 100644 index 4e7ce75cb..000000000 --- a/example/cfis/defunct/config_GitFe_symlink.ini +++ /dev/null @@ -1,105 +0,0 @@ -# ShapePipe configuration file for: get images and find exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_GitFe - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner, find_exposures_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink - -# Copy command options, optional -RETRIEVE_OPTIONS = -L - - -[FIND_EXPOSURES_RUNNER] - -INPUT_MODULE = get_images_runner - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p diff --git a/example/cfis/defunct/config_exp_SpMh.ini b/example/cfis/defunct/config_exp_SpMh.ini deleted file mode 100644 index cfe51ccfc..000000000 --- a/example/cfis/defunct/config_exp_SpMh.ini +++ /dev/null @@ -1,85 +0,0 @@ -# ShapePipe configuration file for single-exposures, -# split images, merge headers - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SpMh - -# Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = split_exp_runner, merge_headers_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SPLIT_EXP_RUNNER] - -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = image, weight, flag - -# Matches compressed single-exposure files -FILE_EXT = .fitsfz, .fitsfz, .fitsfz - -NUMBERING_SCHEME = -0000000 - -# OUTPUT_SUFFIX, actually file name prefixes. -# Expected keyword "flag" will lead to a behavior where the data are saved as int. -# The code also expects the image data to use the "image" suffix -# (default value in the pipeline). -OUTPUT_SUFFIX = image, weight, flag - -# Number of HDUs/CCDs of mosaic -N_HDU = 40 - - -[MERGE_HEADERS_RUNNER] - -FILE_PATTERN = headers - -FILE_EXT = .npy - -# Single-exposure numbering scheme -NUMBERING_SCHEME = -0000000 - diff --git a/example/cfis/defunct/config_tile_PiViSmVi.ini b/example/cfis/defunct/config_tile_PiViSmVi.ini deleted file mode 100644 index 03ebf5fbb..000000000 --- a/example/cfis/defunct/config_tile_PiViSmVi.ini +++ /dev/null @@ -1,184 +0,0 @@ -# ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_PsViSmVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -#MODULE = psfex_interp_runner, - -MODULE = psfex_interp_runner, vignetmaker_runner, spread_model_runner, - vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[PSFEX_INTERP_RUNNER] - -INPUT_DIR = last:sextractor_runner_run_1, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -# Number of stars threshold -STAR_THRESH = 20 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -ME_DOT_PSF_DIR = psfex_runner - -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - - -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = last:sextractor_runner_run_1, last:uncompress_fits_runner - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[SPREAD_MODEL_RUNNER] - -INPUT_DIR = last:sextractor_runner_run_1, last:psfex_interp_runner, last:vignetmaker_runner_run_1 - -FILE_PATTERN = sexcat, galaxy_psf, weight_vignet - -FILE_EXT = .fits, .sqlite, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Pixel scale in arcsec -PIXEL_SCALE = 0.186 - -# Output mode: -# new: create a new catalog with: [number, mag, sm, sm_err] -# add: create a copy of the input SExtractor with the column sm and sm_err -OUTPUT_MODE = new - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_DIR = last:sextractor_runner_run_1, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -PREFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs -ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2 -ME_IMAGE_PATTERN = flag, image, weight, background diff --git a/example/cfis/defunct/config_tile_PiViSmVi_canfar.ini b/example/cfis/defunct/config_tile_PiViSmVi_canfar.ini deleted file mode 100644 index ceb356855..000000000 --- a/example/cfis/defunct/config_tile_PiViSmVi_canfar.ini +++ /dev/null @@ -1,184 +0,0 @@ -# ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_PsViSmVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -#MODULE = psfex_interp_runner, - -MODULE = psfex_interp_runner, vignetmaker_runner, spread_model_runner, - vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[PSFEX_INTERP_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -# Number of stars threshold -STAR_THRESH = 20 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -ME_DOT_PSF_DIR = all:psfex_runner - -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - - -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[SPREAD_MODEL_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:vignetmaker_runner_run_1 - -FILE_PATTERN = sexcat, galaxy_psf, weight_vignet - -FILE_EXT = .fits, .sqlite, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Pixel scale in arcsec -PIXEL_SCALE = 0.186 - -# Output mode: -# new: create a new catalog with: [number, mag, sm, sm_err] -# add: create a copy of the input SExtractor with the column sm and sm_err -OUTPUT_MODE = new - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -PREFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs -ME_IMAGE_DIR = all:split_exp_runner, all:split_exp_runner, all:split_exp_runner, all:sextractor_runner -ME_IMAGE_PATTERN = flag, image, weight, background diff --git a/example/cfis/final_cat.param b/example/cfis/final_cat.param deleted file mode 100644 index 00bcb3f73..000000000 --- a/example/cfis/final_cat.param +++ /dev/null @@ -1,117 +0,0 @@ -# coordinates -XWIN_WORLD -YWIN_WORLD - -# tile ID, for plot of tile-dependent additive bias. -# Can maybe be removed. -TILE_ID - -# flags -FLAGS -IMAFLAGS_ISO -NGMIX_MCAL_FLAGS - -# PSF ellipticity (original image PSF) -NGMIX_G1_PSF_ORIG_NOSHEAR -NGMIX_G2_PSF_ORIG_NOSHEAR - -# spread class -#SPREAD_CLASS - -# spread model flag and error -#SPREAD_MODEL -#SPREADERR_MODEL - -# Number of epochs (exposures) -N_EPOCH -NGMIX_N_EPOCH - -# Blend flag: coadd seg stamp held a non-central footprint (shapepipe#776) -NGMIX_NEIGHBOUR_FLAG - -## Shape measurement outputs -## Ngmix: model fitting - -# galaxy ellipticity -NGMIX_G1_1M -NGMIX_G2_1M -NGMIX_G1_1P -NGMIX_G2_1P -NGMIX_G1_2M -NGMIX_G2_2M -NGMIX_G1_2P -NGMIX_G2_2P -NGMIX_G1_NOSHEAR -NGMIX_G2_NOSHEAR -#NGMIX_G1_ERR_1M -#NGMIX_G2_ERR_1M -#NGMIX_G1_ERR_1P -#NGMIX_G2_ERR_1P -#NGMIX_G1_ERR_2M -#NGMIX_G2_ERR_2M -#NGMIX_G1_ERR_2P -#NGMIX_G2_ERR_2P -NGMIX_G1_ERR_NOSHEAR -NGMIX_G2_ERR_NOSHEAR - -# flags -NGMIX_FLAGS_1M -NGMIX_FLAGS_1P -NGMIX_FLAGS_2M -NGMIX_FLAGS_2P -NGMIX_FLAGS_NOSHEAR - -# size and error -NGMIX_T_1M -NGMIX_T_1P -NGMIX_T_2M -NGMIX_T_2P -NGMIX_T_NOSHEAR -NGMIX_T_ERR_1M -NGMIX_T_ERR_1P -NGMIX_T_ERR_2M -NGMIX_T_ERR_2P -NGMIX_T_ERR_NOSHEAR -NGMIX_T_PSF_RECONV_1M -NGMIX_T_PSF_RECONV_1P -NGMIX_T_PSF_RECONV_2M -NGMIX_T_PSF_RECONV_2P -NGMIX_T_PSF_RECONV_NOSHEAR - -# flux and error -NGMIX_FLUX_1M -NGMIX_FLUX_1P -NGMIX_FLUX_2M -NGMIX_FLUX_2P -NGMIX_FLUX_NOSHEAR -NGMIX_FLUX_ERR_1M -NGMIX_FLUX_ERR_1P -NGMIX_FLUX_ERR_2M -NGMIX_FLUX_ERR_2P -NGMIX_FLUX_ERR_NOSHEAR - -# magnitudes -MAG_AUTO -MAGERR_AUTO -MAG_WIN -MAGERR_WIN -FLUX_AUTO -FLUXERR_AUTO -FLUX_APER -FLUXERR_APER -FLUX_RADIUS - -# SNR from SExtractor -SNR_WIN - -FWHM_IMAGE -FWHM_WORLD - -# PSF size measured on original image -NGMIX_T_PSF_ORIG_NOSHEAR - -# PSF size measured on reconvolved image -# NGMIX_T_PSF_RECONV_NOSHEAR - -# ngmix moment failure flag -NGMIX_MOM_FAIL diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools deleted file mode 100644 index 32197b466..000000000 --- a/example/cfis/star_selection.setools +++ /dev/null @@ -1,131 +0,0 @@ -## SETools configuration file for star/galaxy separation based on size/mag properties -## -## ONE mask cut, and it is the instrument flags: -## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation) -## delivered with the exposure and read by SExtractor. -## -## That is deliberate, and it follows from what the two kinds of mask MEAN. -## An instrument flag marks a CORRUPTED MEASUREMENT — the pixels carry no -## usable signal — so a flagged star is not a star we could model badly, it is -## one we cannot model at all. The healsparse masks are sky-fixed LOCATION -## flags (a star halo, a manual region, a band with no data): they say where an -## object sits, not that its pixels are broken, so whether one disqualifies a -## PSF star is a judgement, not a fact about the data. -## -## So the star selection deliberately starts from OUTLIER REJECTION ALONE for -## that judgement. mask_query still queries the external masks per detection -## and writes MASK_EXT into this catalogue — carried for transparency and -## measurement, so their effect on the star sample can be measured before it is -## imposed. Flag transparently, cut downstream. -## -## MASK_EXT is the configurable pickup if outlier rejection proves -## insufficient. To impose it, add -## -## MASK_EXT == 0 -## -## beside each IMAFLAGS_ISO line below — one line per mask block, and that is -## the whole change. Which maps reach MASK_EXT is mask_query's MASK_PATHS -## config, which ships commented out; SETools has no bitwise operators, so the -## bit selection happens there and this file only ever tests for zero. - -[MASK:preselect] -MAG_AUTO > 0 -MAG_AUTO < 21 -FWHM_IMAGE > 0.3 / 0.187 -FWHM_IMAGE < 1.5 / 0.187 -FLAGS == 0 -IMAFLAGS_ISO == 0 -NO_SAVE - -[MASK:flag] -FLAGS == 0 -IMAFLAGS_ISO == 0 -NO_SAVE - - -[MASK:star_selection] -# Star selection using the FWHM mode -MAG_AUTO > 18. -MAG_AUTO < 22. -FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 -FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 -FLAGS == 0 -IMAFLAGS_ISO == 0 - -[MASK:fwhm_mag_cut] -FWHM_IMAGE > 0 -FWHM_IMAGE < 40 -MAG_AUTO < 35 -FLAGS == 0 -IMAFLAGS_ISO == 0 -NO_SAVE - -# Split the 'star_selection' sample into -# two random sub-samples with ratio 80/20 -[RAND_SPLIT:star_split] -RATIO = 20 -MASK = star_selection - -# The following selection is only used for plotting - -[PLOT:size_mag] -TYPE = plot -FORMAT = png -X_1 = FWHM_IMAGE{fwhm_mag_cut} -Y_1 = MAG_AUTO{fwhm_mag_cut} -X_2 = FWHM_IMAGE{star_selection} -Y_2 = MAG_AUTO{star_selection} -MARKER_1 = + -MARKER_2 = . -MARKERSIZE_1 = 3 -MARKERSIZE_2 = 3 -LABEL_1 = All -LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" -TITLE = "Stellar locus" -XLABEL = "FWHM (pix)" -YLABEL = Mag - -[PLOT:hist_mag_stars] -TYPE = hist -FORMAT = png -Y = MAG_AUTO{star_selection} -BIN = 20 -LABEL = "stars" -XLABEL = "Magnitude" -YLABEL = "Number" -TITLE = "Magnitude of stars" - -[PLOT:fwhm_field] -TYPE = scatter -FORMAT = png -X = X_IMAGE{star_selection} -Y = Y_IMAGE{star_selection} -SCATTER = FWHM_IMAGE{star_selection}*0.186 -MARKER = . -LABEL = "FWHM (arcsec)" -TITLE = "FWHM of stars" -XLABEL = "X (pix)" -YLABEL = "Y (pix)" - -[PLOT:mag_star_field] -TYPE = scatter -FORMAT = png -X = X_IMAGE{star_selection} -Y = Y_IMAGE{star_selection} -SCATTER = MAG_AUTO{star_selection} -MARKER = . -LABEL = "Magnitude" -TITLE = "Magnitude of stars" -XLABEL = "X (pix)" -YLABEL = "Y (pix)" - -[STAT:star_stat] -"Nb objects full cat" = len(FWHM_IMAGE) -"Nb objects not masked" = len(FWHM_IMAGE{flag}) -"Nb stars" = len(FWHM_IMAGE{star_selection}) -"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. -"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 -"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 -"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 -"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 -"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 24ebd6903..2b583b857 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -6,7 +6,7 @@ ShapePipe module ``make_catalogue_runner``. Supercedes `merge_final_cat.py`. Usage: in parent dir of patches: -create_final_cat.py -p ~/shapepipe/example/cfis/final_cat.param -i . -P 7 -v -m final_cat_P7.hdf5 +create_final_cat.py -p ~/shapepipe/workflow/config/cfis/final_cat.param -i . -P 7 -v -m final_cat_P7.hdf5 :Author: Martin Kilbinger diff --git a/tests/module/test_psf_grammar_properties.py b/tests/module/test_psf_grammar_properties.py index b0ac1ee47..6bf9e4713 100644 --- a/tests/module/test_psf_grammar_properties.py +++ b/tests/module/test_psf_grammar_properties.py @@ -19,7 +19,7 @@ group is optional, so a regressed ``GAL`` token is rejected, not absorbed); (c) every ``NGMIX_*`` token the shipped param file - (``example/cfis/final_cat.param``) names is a column the writer can + (``workflow/config/cfis/final_cat.param``) names is a column the writer can produce — writer/param-file consistency; (d) the FULL frozen grammar (shapepipe#761) — ``ESTIMATOR_COMPONENT[_ERR]_ OBJECT[_metacaltype]`` — holds across both estimator families: both @@ -183,10 +183,10 @@ def _run_save_ngmix(ngmix_path, obj_ids, cat_size_target=None): # The shipped final-catalogue param files, two levels up from tests/module/. # Both are consumer contracts updated to the new grammar, so both are checked. -_EXAMPLE = Path(__file__).resolve().parents[2] / "example" +_ROOT = Path(__file__).resolve().parents[2] PARAM_PATHS = [ - _EXAMPLE / "cfis" / "final_cat.param", - _EXAMPLE / "unions_800" / "cat_matched.param", + _ROOT / "workflow" / "config" / "cfis" / "final_cat.param", + _ROOT / "example" / "unions_800" / "cat_matched.param", ] # The one param-file NGMIX token outside the _save_ngmix_data grammar: the @@ -357,7 +357,7 @@ def test_emitted_column_names_match_grammar(obj_ids, tmp_path_factory): def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): """Every NGMIX_* token the param file names is a column the writer produces. - Each shipped final-catalogue param file (``example/cfis/final_cat.param`` + Each shipped final-catalogue param file (``workflow/config/cfis/final_cat.param`` and ``example/unions_800/cat_matched.param``) is a consumer contract for the final catalogue; ``create_final_cat`` keeps only the listed columns, so a token it names that the writer cannot emit is a silent, empty column diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv deleted file mode 120000 index e755942e7..000000000 --- a/workflow/config/cfis/default.conv +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/default.conv \ No newline at end of file diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param deleted file mode 120000 index bb469cf56..000000000 --- a/workflow/config/cfis/default.param +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/default.param \ No newline at end of file diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 100644 index 000000000..09ad8405e --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1,68 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +# The following flag requires a flag image +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex deleted file mode 120000 index 501adb090..000000000 --- a/workflow/config/cfis/default.psfex +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/default.psfex \ No newline at end of file diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 100644 index 000000000..a9d1a906c --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1,85 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2017-11-30 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +BASIS_NAME basis.fits # Basis filename (FITS data-cube) +BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter +NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT + # or PCA_COMMON +NEWBASIS_NUMBER 8 # Number of new basis vectors +PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) +PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 51,51 # Image size of the PSF model +PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? +MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 2 # Polynom degree for each group +PSFVAR_NSNAP 9 # Number of PSF snapshots per axis +HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON +STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? + +BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? +BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed + +#----------------------- PSF homogeneisation kernel -------------------------- + +HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE +HOMOBASIS_NUMBER 10 # Kernel basis number or parameter +HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter +HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF +HOMOKERNEL_DIR # Where to write kernels (empty=same as input) +HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels + +#----------------------------- Output catalogs ------------------------------- + +OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS +CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +# Note: Check-image types can be set the ShapePipe config file, psfex_runner section +####### +CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames +#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? + +#----------------------------- Miscellaneous --------------------------------- + +PSF_SUFFIX .psf # Filename extension for output PSF filename +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex deleted file mode 120000 index 4fcfdabc2..000000000 --- a/workflow/config/cfis/default_exp.sex +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/default_exp.sex \ No newline at end of file diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 100644 index 000000000..b87275ecb --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE AUTO # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param deleted file mode 120000 index e2bf3398c..000000000 --- a/workflow/config/cfis/default_noimaflags.param +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/default_noimaflags.param \ No newline at end of file diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 100644 index 000000000..b251f5b74 --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1,65 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex deleted file mode 120000 index 5a2469c92..000000000 --- a/workflow/config/cfis/default_tile.sex +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/default_tile.sex \ No newline at end of file diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 100644 index 000000000..ff3b25213 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param deleted file mode 120000 index 93e6e1f20..000000000 --- a/workflow/config/cfis/final_cat.param +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/final_cat.param \ No newline at end of file diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 100644 index 000000000..00bcb3f73 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1,117 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# flags +FLAGS +IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# spread class +#SPREAD_CLASS + +# spread model flag and error +#SPREAD_MODEL +#SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +# Blend flag: coadd seg stamp held a non-central footprint (shapepipe#776) +NGMIX_NEIGHBOUR_FLAG + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +#NGMIX_G1_ERR_1M +#NGMIX_G2_ERR_1M +#NGMIX_G1_ERR_1P +#NGMIX_G2_ERR_1P +#NGMIX_G1_ERR_2M +#NGMIX_G2_ERR_2M +#NGMIX_G1_ERR_2P +#NGMIX_G2_ERR_2P +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_T_PSF_RECONV_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL diff --git a/workflow/config/cfis/star_selection.setools b/workflow/config/cfis/star_selection.setools deleted file mode 120000 index a355664cf..000000000 --- a/workflow/config/cfis/star_selection.setools +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/star_selection.setools \ No newline at end of file diff --git a/workflow/config/cfis/star_selection.setools b/workflow/config/cfis/star_selection.setools new file mode 100644 index 000000000..32197b466 --- /dev/null +++ b/workflow/config/cfis/star_selection.setools @@ -0,0 +1,131 @@ +## SETools configuration file for star/galaxy separation based on size/mag properties +## +## ONE mask cut, and it is the instrument flags: +## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation) +## delivered with the exposure and read by SExtractor. +## +## That is deliberate, and it follows from what the two kinds of mask MEAN. +## An instrument flag marks a CORRUPTED MEASUREMENT — the pixels carry no +## usable signal — so a flagged star is not a star we could model badly, it is +## one we cannot model at all. The healsparse masks are sky-fixed LOCATION +## flags (a star halo, a manual region, a band with no data): they say where an +## object sits, not that its pixels are broken, so whether one disqualifies a +## PSF star is a judgement, not a fact about the data. +## +## So the star selection deliberately starts from OUTLIER REJECTION ALONE for +## that judgement. mask_query still queries the external masks per detection +## and writes MASK_EXT into this catalogue — carried for transparency and +## measurement, so their effect on the star sample can be measured before it is +## imposed. Flag transparently, cut downstream. +## +## MASK_EXT is the configurable pickup if outlier rejection proves +## insufficient. To impose it, add +## +## MASK_EXT == 0 +## +## beside each IMAFLAGS_ISO line below — one line per mask block, and that is +## the whole change. Which maps reach MASK_EXT is mask_query's MASK_PATHS +## config, which ships commented out; SETools has no bitwise operators, so the +## bit selection happens there and this file only ever tests for zero. + +[MASK:preselect] +MAG_AUTO > 0 +MAG_AUTO < 21 +FWHM_IMAGE > 0.3 / 0.187 +FWHM_IMAGE < 1.5 / 0.187 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +[MASK:flag] +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + + +[MASK:star_selection] +# Star selection using the FWHM mode +MAG_AUTO > 18. +MAG_AUTO < 22. +FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 +FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 +FLAGS == 0 +IMAFLAGS_ISO == 0 + +[MASK:fwhm_mag_cut] +FWHM_IMAGE > 0 +FWHM_IMAGE < 40 +MAG_AUTO < 35 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +# Split the 'star_selection' sample into +# two random sub-samples with ratio 80/20 +[RAND_SPLIT:star_split] +RATIO = 20 +MASK = star_selection + +# The following selection is only used for plotting + +[PLOT:size_mag] +TYPE = plot +FORMAT = png +X_1 = FWHM_IMAGE{fwhm_mag_cut} +Y_1 = MAG_AUTO{fwhm_mag_cut} +X_2 = FWHM_IMAGE{star_selection} +Y_2 = MAG_AUTO{star_selection} +MARKER_1 = + +MARKER_2 = . +MARKERSIZE_1 = 3 +MARKERSIZE_2 = 3 +LABEL_1 = All +LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" +TITLE = "Stellar locus" +XLABEL = "FWHM (pix)" +YLABEL = Mag + +[PLOT:hist_mag_stars] +TYPE = hist +FORMAT = png +Y = MAG_AUTO{star_selection} +BIN = 20 +LABEL = "stars" +XLABEL = "Magnitude" +YLABEL = "Number" +TITLE = "Magnitude of stars" + +[PLOT:fwhm_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = FWHM_IMAGE{star_selection}*0.186 +MARKER = . +LABEL = "FWHM (arcsec)" +TITLE = "FWHM of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[PLOT:mag_star_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = MAG_AUTO{star_selection} +MARKER = . +LABEL = "Magnitude" +TITLE = "Magnitude of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[STAT:star_stat] +"Nb objects full cat" = len(FWHM_IMAGE) +"Nb objects not masked" = len(FWHM_IMAGE{flag}) +"Nb stars" = len(FWHM_IMAGE{star_selection}) +"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. +"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 +"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 +"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 +"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 +"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 From 3840fee6d1879d6eb7e34e8415bd7024f19c4f0e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 06:53:26 -0400 Subject: [PATCH 099/100] docs: add Snakemake workflow page, drop obsolete VOspace retrieval page workflow.md points at workflow/README.md and the config layout; the bash job_sp tutorial stays until the bash layer's fate is decided. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011C7UQydnTDDVLAo7yrvfCN --- docs/source/toc.rst | 1 + docs/source/vos_retrieve.md | 51 ------------------------------------- docs/source/workflow.md | 24 +++++++++++++++++ 3 files changed, 25 insertions(+), 51 deletions(-) delete mode 100644 docs/source/vos_retrieve.md create mode 100644 docs/source/workflow.md diff --git a/docs/source/toc.rst b/docs/source/toc.rst index 6724b2b42..01c5a1596 100644 --- a/docs/source/toc.rst +++ b/docs/source/toc.rst @@ -25,6 +25,7 @@ basic_execution configuration testing + workflow pipeline_tutorial .. toctree:: diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md deleted file mode 100644 index 3606752c8..000000000 --- a/docs/source/vos_retrieve.md +++ /dev/null @@ -1,51 +0,0 @@ -## Retrieve files from VOspace - -This page describes how ShapePipe output files can be retrieved via the Virtual Observatory Space -on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete. - -1. Retrieve ShapePipe result files - - For a local run on the same machine as for post-processing, nothing needs to be done. In some cases, the run was carried out on a remote machine or cluster, and the resulting ShapePipe output files - need to be retrieved. - - In the specific case of canfar_avail_results.py, this is done as follows. - - 1. Check availability of results - - A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check - which tiles have finished running, and whose results have been uploaded, use - ```bash - canfar_avail_results -i tile_numbers.txt -v -p PSF --input_path INPUT_PATH - ``` - where PSF is one in [`psfex`|`mccd`], and INPUT_PATH the input path on vos, default `vos:cfis/cosmostat/kilbinger/results`. - See `-h` for all options. - - 2. Download results - - All results files will be downloaded with - ```bash - canfar_download_results -i tile_numbers.txt -v -p PSF --input_vos INPUT_VOS - ``` - Use the same options as for same as for `canfar_avail_results`. - - This command can be run in the same directory at subsequent times, to complete an ongoing run: Only newer files will be downloaded - from the `vos` directory. This also assures that partially downloaded or corrupt files will be replaced. - - Checking the `vos` directorty can be slow for large patches. - To only download files that are not yet present locally (in `.`), first write the missing ones to an ascii file, using again the - script `canfar_avail_results`, but this time with `.` as input path: - ```bash - canfar_avail_results -i tile_numbers.txt --input_path . -p PSF -v -o missing.txt - ``` - Then, download only the missing files with - ```bash - canfar_download_results -i missing.txt --input_vos cosmostat/kilbinger/results_mccd_oc2 -p mccd -v - ``` - - 3. Un-tar results - ```bash - untar_results -p PSF - ``` - On success, `ShapePipe` output `fits` and `log` files will be now in various subdirs of the `output` directory. - -At this step all required `ShapePipe` resulting output files are available in the current working directory. diff --git a/docs/source/workflow.md b/docs/source/workflow.md new file mode 100644 index 000000000..624a9819d --- /dev/null +++ b/docs/source/workflow.md @@ -0,0 +1,24 @@ +# Snakemake Workflow + +The end-to-end CFIS production run is driven by a +[Snakemake](https://snakemake.readthedocs.io) workflow that lives in the +`workflow/` directory of the repository. It replaces the older bit-coded bash +job chain (`job_sp`) that this documentation previously described. + +The workflow owns: + +- the per-module ShapePipe configuration files, in `workflow/config/cfis/`, + which is what `$SP_CONFIG` points at; +- the rule graph and job dispatch, in `workflow/Snakefile` and + `workflow/rules/`; +- the run-level settings, in `workflow/config.yaml`. + +Full usage — dependencies, profiles, how to launch a run, and how the rules map +onto the ShapePipe modules — is documented in `workflow/README.md` in the +repository. + +```{note} +The PSF-validation and post-processing configurations that the workflow does +not yet cover are kept separately in `example/cfis/`; see the README there and +[Post-processing](post_processing.md). +``` From 7e2036d38c6be71ee51fca1e7d3dce6ffa5cfdab Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 06:53:26 -0400 Subject: [PATCH 100/100] scripts: remove unreferenced canfar/vos helper scripts Nothing in the repository, pyproject, or docs invokes these; the live canfar path (canfar_submit.py -> init_run_exclusive_canfar.sh / *_v2.0.bash) and the post-processing drivers (post_proc_sp.bash, combine_runs.bash) stay. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011C7UQydnTDDVLAo7yrvfCN --- scripts/python/canfar_avail_results.py | 413 ------------------------ scripts/python/stats_headless_canfar.py | 29 -- scripts/sh/canfar_download_results.bash | 128 -------- scripts/sh/canfar_submit_selection.sh | 119 ------- scripts/sh/curl_canfar_local.sh | 303 ----------------- scripts/sh/run_scratch_local.sh | 178 ---------- scripts/sh/stats_jobs_canfar.sh | 119 ------- 7 files changed, 1289 deletions(-) delete mode 100755 scripts/python/canfar_avail_results.py delete mode 100755 scripts/python/stats_headless_canfar.py delete mode 100755 scripts/sh/canfar_download_results.bash delete mode 100755 scripts/sh/canfar_submit_selection.sh delete mode 100755 scripts/sh/curl_canfar_local.sh delete mode 100755 scripts/sh/run_scratch_local.sh delete mode 100755 scripts/sh/stats_jobs_canfar.sh diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py deleted file mode 100755 index cb9afcb0b..000000000 --- a/scripts/python/canfar_avail_results.py +++ /dev/null @@ -1,413 +0,0 @@ -#!/usr/bin/env python - -"""Script canfar_avail_results.py - -Check whether results files are available on vos. - -:Author: Martin Kilbinger - -:Date: 07/2020 -""" - -import re -import os -import sys -import glob -import copy -import io -from contextlib import redirect_stdout - -from optparse import OptionParser - -from cs_util.canfar import dir_list - -from shapepipe.utilities import cfis - - -def params_default(): - """Set default parameter values. - - Parameters - ---------- - None - - Returns - ------- - p_def: class cfis.param - parameter values - """ - - p_def = cfis.param( - input_IDs=".", - input_path="vos:cfis/cosmostat/kilbinger/results", - psf="mccd", - extension="tgz", - ) - - return p_def - - -def parse_options(p_def): - """Parse command line options. - - Parameters - ---------- - p_def: class cfis.param - parameter values - - Returns - ------- - options: tuple - Command line options - args: string - Command line string - """ - - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - # I/O - parser.add_option( - "-i", - "--input_IDs", - dest="input_IDs", - type="string", - default=p_def.input_IDs, - help="input tile ID file(s) or directory path, default='{}'".format( - p_def.input_IDs - ), - ) - parser.add_option( - "", - "--input_path", - dest="input_path", - type="string", - default=p_def.input_path, - help="input path, local or vos url, default='{}'".format( - p_def.input_path - ), - ) - parser.add_option( - "-o", - "--output_not_avail", - dest="output_not_avail", - type="string", - help="output file for not-available IDs, default no output", - ) - - parser.add_option( - "-p", - "--psf", - dest="psf", - type="string", - default=p_def.psf, - help="PSF model, one in ['psfex'|'mccd'], default='{}'".format( - p_def.psf - ), - ) - - parser.add_option( - "-f", - "--final_only", - dest="final_only", - action="store_true", - help="only check final catalogues", - ) - parser.add_option( - "-x", - "--extension", - dest="extension", - type="string", - default=p_def.extension, - help=f"file extension, default='{p_def.extension}'", - ) - parser.add_option( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose output", - ) - - options, args = parser.parse_args() - - return options, args - - -def check_options(options): - """Check command line options. - - Parameters - ---------- - options: tuple - Command line options - - Returns - ------- - erg: bool - Result of option check. False if invalid option value. - """ - - if options.psf not in ["psfex", "mccd"]: - print("Invalid PSF model '{}'".format(options.psf)) - return False - - return True - - -def update_param(p_def, options): - """Return default parameter, updated and complemented according to options. - - Parameters - ---------- - p_def: class param - parameter values - optiosn: tuple - command line options - - Returns - ------- - param: class param - updated paramter values - """ - - param = copy.copy(p_def) - - # Update keys in param according to options values - for key in vars(param): - if key in vars(options): - setattr(param, key, getattr(options, key)) - - # Add remaining keys from options to param - for key in vars(options): - if not key in vars(param): - setattr(param, key, getattr(options, key)) - - # Do extra stuff if necessary - - return param - - -def read_input_files(input_path, verbose=False): - """Return list of ID files. - - Parameters - ---------- - input_path: string - list of files or directory name - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - ID_files: list of strings - file names with tile IDs - """ - - if os.path.isdir(input_path): - input_files = glob.glob("{}/*".format(input_path)) - else: - input_files = cfis.my_string_split(input_path, stop=True, sep=" ") - - ID_files = [] - for f in input_files: - if os.path.isdir(f): - if verbose: - print("Skipping directory '{}'".format(f)) - else: - ID_files.append(f) - - if verbose: - print("{} input files found".format(len(ID_files))) - - return ID_files - - -def check_results( - ID_files, - input_path, - result_base_names, - n_complete, - extension, - verbose=False, -): - """Count the number of result files uploaded to vos for each input ID file. - - Parameters - ---------- - ID_files: list of strings - file name with tile IDs - input_path: string - path input directory - result_base_names: list of strings - result file base names - n_complete: int - number of files for complete result set - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - n_found: dictionary - number of files found for each input file and each ID - n_IDs: dictionary - number of ID for each input file - IDs_not_avail: list - IDs that are not available on vos - """ - - m = re.match("vos:", input_path) - if m: - ls_tmp = dir_list(input_path) - else: - ls_tmp = glob.glob(f"{input_path}/*") - ls_out = [os.path.basename(path) for path in ls_tmp] - - n_found = {} - n_IDs = {} - IDs_not_avail = [] - - # Loop over all input files - for ID_list in ID_files: - with open(ID_list) as f: - if verbose: - print("Checking ID list file {}...".format(ID_list)) - n_found[ID_list] = {} - n_IDs[ID_list] = 0 - - # Loop over all lines = IDs in file - for line in f: - ID = line.rstrip() - n_found[ID_list][ID] = 0 - - if extension == "fits": - ID_fname = ID.replace(".", "-") - ID_fname = f"-{ID_fname}" - else: - ID_fname = f"_{ID}" - # Count how many result files are available - for base in result_base_names: - name = "{}{}.{}".format(base, ID_fname, extension) - if name in ls_out: - n_found[ID_list][ID] = n_found[ID_list][ID] + 1 - n_IDs[ID_list] = n_IDs[ID_list] + 1 - - # If not complete set found, add to not-avail list - if n_found[ID_list][ID] != n_complete: - IDs_not_avail.append(ID) - - return n_found, n_IDs, IDs_not_avail - - -def output_summary(n_found, n_IDs, n_complete): - """Create output with summary of result availability. - - Parameters - ---------- - n_found: dictionary - number of files found for each input file and each ID - n_IDs: dictionary - number of ID for each input file - n_complete: int - number of files of a complete set of results - """ - - for ID_list in n_found.keys(): - nf = sum(value == n_complete for value in n_found[ID_list].values()) - print( - "{}: {}/{} ({:.1f}%) complete".format( - os.path.basename(ID_list), - nf, - n_IDs[ID_list], - nf / n_IDs[ID_list] * 100, - ) - ) - - -def output_IDs(ID_list, output): - """Write IDs to file - - Parameters - ---------- - ID_list: list of string - IDs - output: string - output file name - """ - - f = open(output, "w") - for ID in ID_list: - print(ID, file=f) - f.close() - - -def main(argv=None): - - # Set default parameters - p_def = params_default() - - # Command line options - options, args = parse_options(p_def) - - if check_options(options) is False: - return 1 - - param = update_param(p_def, options) - - # Save calling command - cfis.log_command(argv) - if param.verbose: - cfis.log_command(argv, name="sys.stdout") - - ### Start main program ### - - if param.verbose: - print("Start of program {}".format(os.path.basename(argv[0]))) - - ID_files = read_input_files(param.input_IDs, verbose=param.verbose) - - if param.final_only: - result_base_names = ["final_cat"] - else: - result_base_names = [] - types = [ - "final_cat", - "logs", - "setools_mask", - "setools_stat", - "setools_plot", - ] - for t in types: - result_base_names.append(t) - - if param.psf == "psfex": - result_base_names.append("psfex_interp_exp") - elif param.psf == "mccd": - result_base_names.append("mccd_fit_val_runner") - - n_complete = len(result_base_names) - - n_found, n_IDs, IDs_not_avail = check_results( - ID_files, - param.input_path, - result_base_names, - n_complete, - param.extension, - verbose=param.verbose, - ) - - output_summary(n_found, n_IDs, n_complete) - - if param.output_not_avail: - output_IDs(IDs_not_avail, param.output_not_avail) - - ### End main program - - if param.verbose: - print("End of program {}".format(os.path.basename(argv[0]))) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/stats_headless_canfar.py b/scripts/python/stats_headless_canfar.py deleted file mode 100755 index 1f728949d..000000000 --- a/scripts/python/stats_headless_canfar.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -# Name: stats_headless_canfar.py - -# Caution: Does not show all running or pending -# headless jobs, for some reason. - -import sys -from skaha.session import Session - - -def main(argv=None): - - print( - "# Depreciated, does not show pending jobs; use stats_jobs_canfar.sh", - file=sys.stderr, - ) - - session = Session() - - n_headless = session.stats()["instances"]["headless"] - - print(n_headless) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/sh/canfar_download_results.bash b/scripts/sh/canfar_download_results.bash deleted file mode 100755 index 991af515a..000000000 --- a/scripts/sh/canfar_download_results.bash +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_download_results.bash -# Description: Download ShapePipe results (.tgz files) -# from canfar with vos -# Author: Martin Kilbinger -# Date: v1.0 05/2020 -# v1.1 01/2021 - -# Command line - -## Default parameters -INPUT_VOS="cosmostat/kilbinger/results" -VERBOSE=0 -psf="mccd" -only_mask=0 - - -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -i, --input_IDs ID_FILE\n - \tASCII file with tile IDs to download, default:\n - \tdownload all available IDs\n - --input_vos PATH\n - \tinput path on vos:cfis, default='$INPUT_VOS'\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m\tonly mask files\n - -v\tverbose output\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -i|--input_IDs) - IDs=(`cat $2`) - echo "Downloading ${#IDs[@]} ID(s)" - shift - ;; - --input_vos) - INPUT_VOS="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m) - only_mask=1 - ;; - -v) - VERBOSE=1 - ;; - *) - echo "Invalid command line argument '$1'" - echo -ne $usage - exit 1 - ;; - esac - shift -done - -## Check options -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psf' or 'mccd'" - exit 2 -fi - -## Paths -remote="vos:cfis/$INPUT_VOS" -local="." - -if [ $only_mask == 1 ]; then - NAMES=("pipeline_flag") -else - NAMES=( - "final_cat" - "logs" - "setools_mask" - "setools_stat" - "setools_plot" - "pipeline_flag" - ) - - if [ $psf == "psfex" ]; then - NAMES+=( - "psfex_interp_exp" - ) - else - NAMES+=( - "mccd_fit_val_runner" - ) - fi -fi - -if [ $VERBOSE == 1 ]; then - vflag="-v" -else - vflag="" -fi -export VCP="vcp $vflag" - - -### Start ### - -# Download files -for name in ${NAMES[@]}; do - if [ ${#IDs[@]} == 0 ]; then - cmd="$VCP $remote/$name*.tgz $local" - $cmd - else - for ID in ${IDs[@]}; do - cmd="$VCP $remote/${name}_$ID.tgz $local" - $cmd - done - fi -done - -# Check number of files -for name in ${NAMES[@]}; do - n_downl=(`ls -l $local/${name}_*.tgz | wc`) - echo "$n_downl '$name' result files downloaded from $remote" -done diff --git a/scripts/sh/canfar_submit_selection.sh b/scripts/sh/canfar_submit_selection.sh deleted file mode 100755 index 3698f0722..000000000 --- a/scripts/sh/canfar_submit_selection.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_submit_selection.sh -# Author: Martin Kilbinger, martin.kilbinger@cea.fr -# Date: 2020 -# Description: Submits jobs to canfar, each jobs processes -# one tile - - -# Variables - -# Job file for one tile -SP_ROOT=$HOME/shapepipe -sp_job="$SP_ROOT/scripts/sh/job_sp.bash" - - -# Functions - -function add_queue() { - local job_file=$1 - - echo "queue" >> $job_file - echo >> $job_file -} - -function create_job_file() { - local tile_ID=$1 - local job_file=$2 - - echo "executable = $sp_job" > $job_file - echo >> $job_file - - add_to_job_file $tile_ID $job_file - - finalize_job_file $job_file - add_queue $job_file -} - -function add_to_job_file () { - local tile_ID=$1 - local job_file=$2 - - echo "arguments = $tile_ID" >> $job_file - echo "output = log_canfar_sp_$tile_ID.out" >> $job_file - echo "error = log_canfar_sp_$tile_ID.err" >> $job_file - echo "log = log_canfar_sp_$tile_ID.log" >> $job_file -} - -function finalize_job_file() { - local job_file=$1 - - echo "request_cpus = 8" >> $job_file - - # Cannot be larger than VM available RAM - echo "request_disk = 100G" >> $job_file - - echo >> $job_file -} - -function usage () { - ex=$1 - - echo "Usage: canfar_submit_selection.sh ID_path image [-h] [-n]" - echo "Options:" - echo " ID_path ascii file with tile IDs" - echo " image VM image name, newest is ShapePipe2-mk-20200820" - echo " -n dry run" - echo " -h this message" - - exit $ex -} - -if [ "${!#}" == "-h" ]; then - usage 0 -fi -if [ "$#" -lt 2 ] ; then - usage 1 -fi - -if [ "${!#}" == "-n" ]; then - dry_run=1 - dry_str=" (dry run)" -else - dry_run=0 - dry_str="" -fi - - -## Start - -tile_ID_list=$1 -image=$2 - -# Create job file -job_file="job_tile.sh" -echo "executable = $sp_job" > $job_file -echo >> $job_file - -echo "output = log_canfar_sp_\$(arguments).out" >> $job_file -echo "error = log_canfar_sp_\$(arguments).err" >> $job_file -echo "log = log_canfar_sp_\$(arguments).log" >> $job_file -echo >> $job_file -finalize_job_file $job_file -echo >> $job_file -echo "queue arguments from (" >> $job_file - -# Go through tile IDs and add job -while read -r tile_ID; do - echo "$tile_ID" >> $job_file -done < $tile_ID_list - echo ")" >> $job_file - -# Submit -cmd="canfar_submit $job_file $image c8-30gb-186" -echo "Running $cmd$dry_str" -if [ $dry_run == 0 ]; then - $cmd - condor_q -fi diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh deleted file mode 100755 index ada5dc4f0..000000000 --- a/scripts/sh/curl_canfar_local.sh +++ /dev/null @@ -1,303 +0,0 @@ -# Global variables -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session -IMAGE=images.canfar.net/unions/shapepipe -NAME=shapepipe - -# :TODO: Not working -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source $HOME/shapepipe/scripts/sh/functions.sh - -# Command line arguments - -## Default values -job=-1 -psf="psfex" -ID=-1 -file_IDs=-1 -N_SMP=1 -fix=0 -version="1.1" -cmd_remote="$HOME/shapepipe/scripts/sh/init_run_exclusive_canfar.sh" -batch=30 -batch_max=200 -dry_run=0 -mh_local=0 -sp_local=0 -test_only=0 -debug_out="-1" -scratch="-1" -sm=1 - -pat="- " - -## Help string -usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRunning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -f, --file_IDs path - \tfile containing IDs\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n - -s, --sp_local SP\n - \tsplit local run local (SP=1) or global (SP=0); default is SP=$sp_local\n - --sm SM\n - \tWith (SM=1; default) or without (SM=0) spread model input\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default=$N_SMP\n - -F, --fix FIX\n - \tfix missing data (re-download tile, unzip) for FIX=1; default is $fix\n - -V, --version\n - \tversion of docker image, default='$version'\n - -C, --command_remote\n - \tremote command to run on canfar, default='$cmd_remote'\n - -S, --scratch\n - \tprocessing scratch directory, default is None ($scratch)\n - -b, --batch_max\n - \tmaximum batch size = number of jobs run simultaneously, default=$batch_max\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n - -n, --dry_run LEVEL\n - \tdry run, from LEVEL=2 (no processing) to 0 (full run; default)\n - --test\n - \ttest mode, no processing\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -s|--sp_local) - sp_local="$2" - shift - ;; - --sm) - sm="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -f|--file_IDs) - file_IDs="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -F|--fix) - fix="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -V|--version) - version="$2" - shift - ;; - -B|--batch) - batch="$2" - shift - ;; - -b|--batch_max) - batch_max="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - --test) - test_only=1 - ;; - esac - shift -done - - -## Check options - -if [ "$test_only" == "1" ]; then - test_arg="--test" -else - test_arg="" -fi - -if [ "$job" == "-1" ]; then - echo "No job indicated, use option -j" - exit 2 -fi - -if [ "$ID" == "-1" ] && [ "$file_IDs" == "-1" ]; then - echo "No image ID(s) indicated, use option -e ID or -f file_IDs" - exit 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 4 -fi - -if [ "$dry_run" != 0 ] && [ "$dry_run" != 1 ] && [ "$dry_run" != 2 ]; then - echo "Invalid dry_run option, allowed are 0, 1, and 2" - exit 5 -fi - -if [ "$debug_out" != "-1" ]; then - echo "${pat}Starting $(basename "$0") $test_arg" >> $debug_out - echo "${pat}curl ID=$ID" >> $debug_out - echo ${pat}`date` >> $debug_out -fi - -source activate shapepipe -if [ "$debug_out" != "-1" ]; then - echo "${pat}conda prefix = ${CONDA_PREFIX}" >> $debug_out - echo "${pat}script version = ${script_version}" >> $debug_out -fi - -# command line arguments for remote script: -# collect into string - - -RESOURCES="ram=4&cores=$N_SMP" -dir=`pwd` - - -function submit_batch() { - path=$1 - - for ID in `cat $path`; do - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - done -} - -batch=50 -if [ "$batch" -ge "$batch_max" ]; then - ((batch=batch_max/2)) - echo "Reducing batch size to $batch" -fi -sleep=75 - -((n_thresh=batch_max-batch)) - - -if [ "$dry_run" == 2 ]; then - - # Do not call curl (dry run = 2) - echo "Running command dry run:" - - if [ "$ID" == "-1" ]; then - - - # Submit file (dry run = 2) - for ID in `cat $file_IDs`; do - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - done - - else - - # Submit image (dry run = 2) - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - - fi - -else - - # Call curl - rm -rf session_IDs.txt session_image_IDs.txt - - if [ "$ID" == "-1" ]; then - - # Submit file - n_jobs=`cat $file_IDs | wc -l` - if [ "$n_jobs" -gt "$batch_max" ]; then - - # Split into batches - prefix="${file_IDs}_split_" - split -d -l $batch $file_IDs $prefix - n_split=`ls -l $prefix* | wc -l` - echo "Split '$file_IDs' into $n_split batches of size $batch" - - count=1 - n_queued=`stats_jobs_canfar.sh -w all` - for batch in $prefix*; do - echo "Number of queued jobs = $n_queued" - echo "Submitting batch $batch ($count/$n_split)" - echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007" - submit_batch $batch - ((count=count+1)) - - n_queued=`stats_jobs_canfar.sh -w all` - - while [ "$n_queued" -gt "$n_thresh" ]; do - echo "Wait for #jobs = $n_queued jobs to go < $n_thresh ..." - sleep $sleep - n_queued=`stats_jobs_canfar.sh -w all` - done - - done - - else - - # Submit entire file (single batch) - echo "Submit '$file_IDs' in single batch" - submit_batch $file_IDs - - fi - - else - - # Submit image - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - - fi - -fi - -echo "Done $(basename "$0")" - -if [ "$debug_out" != "-1" ]; then - echo "${pat}End $(basename "$0") $test_arg" >> $debug_out -fi diff --git a/scripts/sh/run_scratch_local.sh b/scripts/sh/run_scratch_local.sh deleted file mode 100755 index 1223e6683..000000000 --- a/scripts/sh/run_scratch_local.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash - -# Command line arguments -## Default values -job=-1 -ID=-1 -N_SMP=1 -dry_run=0 -dir=`pwd` -debug_out=-1 -scratch=/n17data/`whoami`/scratch -exec_path=$HOME/shapepipe/scripts/sh -slurm=1 - -# mh_local is 0 (1) if merge_header_runner is run on all exposures, -# which is standard so far (run on exposures of given tile only; new) -mh_local=0 - -# sp_local is 0 (1) is split_headers_runner and mask_runner is run -# on all exposures (locally). Not 100% automatic yet. -sp_local=1 -VERBOSE=1 - -pat="-- " - -# Help string -usage="Usage: $(basename "$0") -j JOB -e ID -k KIND [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRUnning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default from original config files\n - -d, --directory\n - \trun directory, default is pwd ($dir)\n - -S, --scratch\n - \tprocessing scratch directory, default is $scratch\n - -n, --dry_run LEVEL\n - \tdry run, no actuall processing\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -d|--directory) - dir="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - esac - shift -done - -## Check options -if [ "$job" == "-1" ]; then - echo "No job indicated, use option -j" - exit 2 -fi - -if [ "$exclusive" == "-1" ]; then - echo "No image ID indicated, use option -e" - exit 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 4 -fi - - -source $HOME/shapepipe/scripts/sh/functions.sh - - -kind=$(get_kind_from_job $job) - - -# Load common functions -source $HOME/shapepipe/scripts/sh/functions.sh - - -# Start script - -if [ "$scratch" != "-1" ]; then - - command "mkdir -p $scratch/${kind}_runs" $dry_run - command "cp -R ${kind}_runs/$ID $scratch/${kind}_runs" $dry_run - command "cd $scratch" $dry_run - -fi - - if [ "$slurm" == "0" ]; then - command "init_run_exclusive_canfar.sh -j $job -p $psf -m $mh_local -N $N_SMP -e $ID" $dry_run - else - STATUS=$(sbatch --output=./sbatch-$ID.out --partition=comp --job-name="j${job}_${ID}" --ntasks-per-node=$N_SMP --time=32:00:00 --mem=64G $exec_path/init_run_exclusive_canfar.sh -j $job -p $psf -m $mh_local -N $N_SMP -e $ID) - - JOB_ID=$(echo $STATUS | cut -d ' ' -f 4) - echo "JOB_ID=$JOB_ID" - - # Wait for the job to finish - while true; do - STATUS=$(squeue -j "$JOB_ID" -h -o "%T") - if [[ -z "$STATUS" ]]; then - echo "job $JOB_ID no longer in the queue" - break - fi - - echo "Waiting for job $JOB_ID in state '$STATUS' to complete..." - sleep 10 - done - - echo "Job $JOB_ID has completed. Proceeding with the script..." - fi - -if [ "$scratch" != "-1" ]; then - - if [ "$job" == "32" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_exp_SxSe* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "64" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_PsViSm* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "128" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_ngmix_* $dir/${kind}_runs/$ID/output" $dry_run - fi - - command "rm -rf ${kind}_runs/$ID" $dry_run - command "cd $dir/${kind}_runs/$ID" $dry_run - # Gave Input/Output python error - #command "update_runs_log_file.py" $dry_run - command "cd $dir" $dry_run - -fi diff --git a/scripts/sh/stats_jobs_canfar.sh b/scripts/sh/stats_jobs_canfar.sh deleted file mode 100755 index f9c51101a..000000000 --- a/scripts/sh/stats_jobs_canfar.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# Name: stats_jobs_canfar.sh -# Author: Martin Kilbinger -# Description: Handles headless jobs on canfar - - -# Global variables - -## Temporary files -tmpfile_jobs="jobinfo.txt" -tmpfile_ids="ids.txt" -tmpfile_running="jobs_running.txt" - -## curl options -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session - - -# Command line arguments - -## Default values -mode="count" -debug=0 - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -m, --mode MODE\n - \tmode, allowed are 'count' (default), 'delete'\n - -d, --debug\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -m|--mode) - mode="$2" - shift - ;; - -d|--debug) - debug=1 - ;; - esac - shift -done - -## Check options -case $mode in - "count"|"delete") - # valid option - ;; - *) - echo "Invalid mode $mode" - exit 1 - ;; -esac - - -# Main program - -# Get all instances -if [ "$debug" == "1" ]; then - curl -E $SSL $SESSION - echo curl -E $SSL $SESSION - exit 0 -else - curl -E $SSL $SESSION &> /dev/null > $tmpfile_jobs -fi -res=$? - -if [ "$res" == "0" ]; then - - # Get headless job IDs - cat $tmpfile_jobs | grep headless -B 4 -A 2 | grep Running -A 1 > $tmpfile_ids - - # Number of jobs - n_headless=`cat $tmpfile_ids | grep Running | wc -l` - - # Get running job info - cat $tmpfile_ids | grep name | perl -F\- -ane 'chomp; $F[4] =~ s/[",]//g; print "$F[3].$F[4]"' > $tmpfile_running - -else - - # Failure: set to very high number - n_headless=10000 - -fi - - -if [ "$mode" == "count" ]; then - - echo $n_headless - -elif [ "$mode" == "delete" ]; then - - echo -n "Delete $n_headless jobs? [y|n] " - read answer - if [ "$answer" == "y" ]; then - cat $tmpfile_jobs | grep headless -B 34 -A 6 | grep Running -A 34 | grep id | grep -v user | perl -F\" -ane 'print "$F[3]\n"' > $tmpfile_ids - for ID in `cat $tmpfile_ids`; do - echo $ID - # Delete headless jobs - echo "curl -X DELETE -E $SSL $SESSION/$ID" - curl -X DELETE -E $SSL $SESSION/$ID - echo $? - done - fi - -fi - - -# Remove temporary files -#rm -f $tmpfile_jobs $tmpfile_ids