diff --git a/.gitignore b/.gitignore index db20f3086..4c4b6d0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ code # .felt here is a machine-local symlink into it. Never track it in this repo. /.felt/ /.felt +.snakemake/ diff --git a/CLAUDE.md b/CLAUDE.md index 8ee90c2cc..cae6d99eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,19 +80,36 @@ Full detail: `docs/source/installation.md` and `docs/source/container.md`. - `src/shapepipe/` — the package (src-layout). `modules/` holds the pipeline modules and their `*_runner.py` wrappers; `pipeline/` is execution and file - I/O; `utilities/`; `canfar/` is CANFAR/cluster job orchestration. Console entry - points (`shapepipe_run`, `summary_run`, `canfar_*`) are defined under + I/O; `utilities/`; `canfar/` monitors CANFAR sessions. Console entry + points (`shapepipe_run`, `canfar_*`, `plot_coverage_map`) are defined under `[project.scripts]`. - `tests/` — the whole test suite, one discovery root: `module/` (per-module unit/property/integration tests), `unit/` (structural), `science/` (fast guardrails), `cluster/` (candide-only), `helpers/` (shared library code). See `tests/README.md`. +- `workflow/` — **the production orchestration**: a Snakemake workflow over the + same `shapepipe_run` calls. `bin/sp` is the entry point (`run`, `report`, + `container`, `cancel`); `Snakefile` plus `rules/{prepare,exposure,tile, + coverage}.smk` are the rule graph; `scripts/` holds the plain Python each rule + shells out to; `config.yaml` declares one run (tile list, the scratch and + persistent roots, the container); `config/cfis/` holds the committed ini + chain; `profiles/nibi/config.yaml` is the SLURM executor profile. Deep + reference: `workflow/README.md`. User-facing: `docs/source/workflow.md`. - `example/` — a runnable example pipeline (`example/config.ini`) on a single CFIS tile; doubles as the CI smoke test. - `scripts/` — shell / Python / notebook helpers (`sh/`, `python/`, `jupyter/`), symlinked onto `$PATH` inside the image. - `docs/` — Sphinx sources; the API docs are generated from docstrings. +**There is no concept of a catalogue version in the code.** Nothing branches on +`v1.3`..`v1.6` or `v2.0`, and there are no sky patches (`P1`..`P9`): a campaign +is a tile list, and the version of a catalogue is the git tag of the code that +produced it. The CANFAR layers and the summary scrape that carried those +constructs were retired, last carried at `2ef07e45`. The pre-Snakemake bash job +layer (`scripts/sh/run_job_sp_canfar_v2.0.bash`, `job_sp_canfar_v2.0.bash`, +`job_list_help.bash`, `functions.sh`) stays — it is version-free, and +sp_validation's image-simulation workflow calls it. + ## Development workflow - **`develop` is the integration branch** — open PRs against it. `main` / diff --git a/Dockerfile b/Dockerfile index 0a549ae8f..fed629849 100644 --- a/Dockerfile +++ b/Dockerfile @@ -118,6 +118,7 @@ RUN chmod -R go+rwX /app && \ uv pip install --no-deps -e . && \ for ext in .py .sh .bash; do \ for script in /app/scripts/*/*$ext; do \ + [ -e "$script" ] || continue; \ link_name=$(basename $script $ext); \ ln -s $script /usr/local/bin/$link_name; \ done; \ @@ -169,6 +170,7 @@ RUN chmod -R go+rwX /app && \ uv pip install --no-deps -e . && \ for ext in .py .sh .bash; do \ for script in /app/scripts/*/*$ext; do \ + [ -e "$script" ] || continue; \ link_name=$(basename $script $ext); \ ln -s $script /usr/local/bin/$link_name; \ done; \ diff --git a/README.rst b/README.rst index e27b4028d..01303fe4e 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,8 @@ to start: - `Installation `_ — getting ShapePipe onto your machine or cluster. - `Basic execution `_ and `configuration `_ — running ``shapepipe_run`` and writing pipeline configs. - `Container workflow `_ — the two image targets and the ``pyproject.toml`` / ``uv.lock`` / ``Dockerfile`` layers. -- `Running on a cluster `_ — pulling the image and submitting jobs, with worked candide (SLURM) and CANFAR examples. +- `Running on a cluster `_ — pulling the image and submitting jobs, with a worked candide (SLURM) example. +- `The Snakemake workflow `_ — the production orchestration for a whole tile list, driven by ``workflow/bin/sp``. If you use ShapePipe in academic work, please cite Guinot et al. (2022) and Farrens et al. (2022). diff --git a/bin/canfar_submit_job.py b/bin/canfar_submit_job.py deleted file mode 100755 index 202e9d274..000000000 --- a/bin/canfar_submit_job.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python -from shapepipe.canfar_run import run_job -run_job() diff --git a/bin/summary_run.py b/bin/summary_run.py deleted file mode 100755 index a59689e41..000000000 --- a/bin/summary_run.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python -from shapepipe.summary_run import main -main() diff --git a/docs/source/clusters.md b/docs/source/clusters.md index 41d32a0bf..521e51bb3 100644 --- a/docs/source/clusters.md +++ b/docs/source/clusters.md @@ -2,9 +2,9 @@ ShapePipe runs the same way on every cluster: **through the container**. You pull the slim `runtime` image once, bind-mount your clone, and run -`shapepipe_run` (or the CANFAR submission tooling) inside it — there is no -environment to install or activate on the host. This page covers the shared -pattern, then the specifics for each supported machine. +`shapepipe_run` inside it — there is no environment to install or activate on +the host. This page covers the shared pattern, then the specifics for each +supported machine. For what is *inside* the image and how it is built, see [Container Workflow](container.md). @@ -67,32 +67,17 @@ over unchanged. ## CANFAR -CANFAR submission does not go through a batch scheduler. Instead you submit -container jobs to CANFAR's headless system with the `canfar_submit_job` console -script (backed by the `canfar` library), and watch them with `canfar_monitor` / -`canfar_monitor_log`. Pipeline steps are **bit-coded** through `-j` (the same -scheme as `scripts/sh/job_sp_canfar.bash`), the PSF model is chosen with -`-p psfex|mccd`, and `-V` selects the image version: +CANFAR submission has been retired. The `canfar_submit_job` console script and +the bash layers underneath it (`curl_canfar_local.sh`, +`init_run_exclusive_canfar.sh`, `job_sp_canfar.bash`) were last carried at +`2ef07e45`; the bit-coded `-j` production walkthrough went with them. -```bash -# Submit pipeline step(s) for the configured tiles (bit-coded -j). -canfar_submit_job -j 1 -p psfex -V 1.1 - -# Monitor sessions/jobs and stream logs. -canfar_monitor -canfar_monitor_log -``` +Production orchestration is now the Snakemake workflow in `workflow/` — see +`workflow/README.md`. It is SLURM-only and has no CANFAR execution mode, so a +CANFAR campaign means moving to a SLURM site (nibi, candide). -The full production run — input preparation, the per-step `-j` table, and -post-processing — is documented in the -[CANFAR production walkthrough](pipeline_canfar.md). - -```{note} -The CANFAR production submission scripts (`scripts/sh/job_sp_canfar*.bash`) still -run under the pre-container environment and are slated for the same -container-first cleanup the candide scripts received. Treat the walkthrough as -the current-but-evolving production procedure. -``` +`canfar_monitor` and `canfar_monitor_log` stay: they list, filter and destroy +CANFAR sessions, independently of how those sessions were started. ## ccin2p3 diff --git a/docs/source/conf.py b/docs/source/conf.py index eea9b8f43..d52836d43 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -104,6 +104,10 @@ myst_enable_extensions = ["html_image"] +# Generate slug anchors for h1-h3 so in-page links like +# `[Mask images](#mask-images)` resolve instead of warning. +myst_heading_anchors = 3 + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/source/dependencies.md b/docs/source/dependencies.md index 9378bfe00..2a44d9700 100644 --- a/docs/source/dependencies.md +++ b/docs/source/dependencies.md @@ -35,8 +35,7 @@ Data access & infrastructure (CANFAR / UNIONS): | Package | Purpose | |---------|---------| | [vos](https://github.com/opencadc/vostools) | CADC / CANFAR VOSpace access | -| [skaha](https://github.com/shinybrar/skaha) | CANFAR Science Platform sessions | -| canfar | CANFAR container-job submission | +| canfar | CANFAR session monitoring | | [astroquery](https://astroquery.readthedocs.io/) | external catalogue queries | | [cs_util](https://github.com/CosmoStat/cs_util) | shared CosmoStat utilities | | [sqlitedict](https://github.com/RaRe-Technologies/sqlitedict) | on-disk pipeline state | diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md deleted file mode 100644 index cfdd16507..000000000 --- a/docs/source/pipeline_canfar.md +++ /dev/null @@ -1,571 +0,0 @@ -# Running `ShapePipe` processing and post-processing pipelines on CANFAR - -Documentation to create ShapePipe output products for catalogues v1.x. - -## Initial Setup - -### CANFAR Login - -Login to the canfar system with - -```bash -canfar auth login -``` - -This can be done from at notebook or terminal within the canfar science portal, -or any remote terminal that has the canfar library installed. - -Check authentication status with - -```bash -canfar auth list -``` - -If not on "default", run - -```bash -canfar auth switch default -``` - -### Set variables (optional) - -Set the current patch in the shell as - -```bash -patch=P[1-9] -``` - -For convenience, the current PSF model can be set as environment variable, e.g.: - -```bash -psf="psfex" -``` - -Allowed are `psfex` and `mccd`. - -Setting the terminal title to display the patch can be useful for long jobs, to keep track of which terminal -runs which patch: - -```bash -echo -ne "\033]0;$patch\007" -``` - -### Prepare run directory - -First, go to the dedicated directory with - -```bash -cd /path/to/version/$patch -``` - -Next, set links to the tile number list and configuration directory: - -```bash -ln -s ~/shapepipe/auxdir/CFIS/tiles_202106/tiles_$patch.txt tile_numbers.txt -ln -s ~/shapepipe/example/cfis -``` - -Create output and debug log directories - -```bash -mkdir -p output -mkdir -p debug -``` - -Finally, create and link to central image storage directories for tiles and exposures: - -```bash -mkdir -p ~/cosmostat/v2/data_tiles/$patch -ln -s ~/cosmostat/v2/data_tiles/$patch data_tiles -mkdir -p ~/cosmostat/v2/data_exp/$patch -ln -s ~/cosmostat/v2/data_tiles/$patch data_exp -``` - -## `ShapePipe` processing - -Now, everything should be ready to start running `ShapePipe` for the weak lensing processing. The following -details all necessary steps. - -### Get Images - -We first download images, and in a second run create symbolic links with the proper pipeine naming scheme. - -#### Download and move tiles - -When running the main `ShapePipe` script `shapepipe_run`, the following env variable needs to point -to the current working directory - -```bash -export SP_RUN=`pwd` -``` - -Now we run the first module (`get_images_runner`) to download the tile images together with the weight files. -This run can get interrupted by VOSpace I/O or connection errors. In that case, -we move new files to the image storage directory, remove the previous (now void of images) run directory, -and update the run log file. We also check the number of previous and new tiles. - -```bash -shapepipe_run -c cfis/config_Git_vos.ini -ls -l data_tiles/ | wc -mv -i output/run_sp_Git_*/get_images_runner/output/CFIS.???.???.*fits* data_tiles -ls -l data_tiles/ | wc -rm -rf output/run_sp_Git_* -update_runs_log_file.py -``` - -Repeat the above block as needed. - -### Find Exposures - -With all tile images (= stacks) downloaded, we can inquire their headers to identify the exposures that were used -to create the stacks. This call to the pipeline also creates the symbolic links to the downloaded tile images. - -```bash -shapepipe_run -c cfis/config_GitFe_symlink.ini -``` - -(One could also run `Fe` alone.) - -### Download and Move Exposures - -The last module create exposure lists on output. These are now used to download all exposures. As for the tile downloads, -we have to account for VOSpace errors. - -```bash -shapepipe_run -c cfis/config_Gie_vos.ini -mv -i output/run_sp_Gie_*/get_images_runner/output/*.fits*fz data_exp -rm -rf output/run_sp_Gie_* -update_runs_log_file.py -``` - -Repeat the above by hand, or peform it in an automatic loop: - -```bash -while true; do - shapepipe_run -c cfis/config_Gie_vos.ini - ls -l data_exp/ | wc - mv -i output/run_sp_Gie_*/get_images_runner/output/*.fits*fz data_exp - ls -l data_exp/ | wc - rm -rf output/run_sp_Gie_* - update_runs_log_file.py -done -``` - -**Note:** Make sure that after all images are downloaded there is no `Gie` run in the output directory. -This would mess up later modules since `last:get_image_runner` could point to this run. - -### Create tile links again (necessary?) - -If necessary, e.g. because a previous `Git` run is no longer valid, re-create the symbolic links to the downloaded tiles with - -```bash -job_sp_canfar.bash -p $psf `cat tile_numbers.txt` -j 1 -r symlink -``` - -### Uncompress tile weights - -The downloaded tile weights are compressed. The following call uncompresses all. - -```bash -shapepipe_run -c cfis/config_tile_Uz.ini -``` - -### Mask tiles - -This step is done globally for all tiles. There might be job failures or interruptions. The following -command to the `ShapePipe` job script can be run repeatedly; already created masks will be skipped. - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 4 -``` - -If masks were created in more than one run, i.e. situated in more than one output directory, these have to be -combined for subsequent pipeline module runs. This is done by creating a new output directory with symbolic -links, using the script - -```bash -combine_runs.bash -c flag_tile -``` - -## Tile detection - -We can finally run our first module using the canfar submission system. First, determine the number of optimal jobs such that at a given -time the allowed maximum of 512 running jobs is not exceeded. -Set as `N_PAR` (number of parallel jobs) a number between 1 and 8. - -```bash -canfar_submit_job -j 16 -f tile_numbers.txt -P N_PAR -v -s -``` - -Now, run the previous command with that number `JMAX` - -```bash -canfar_submit_job -j 16 -f tile_numbers.txt -P N_PAR -v -J JMAX -``` - -### Exposure Processing - -#### Option 0: Global split and exp masks (deprecated; used for earlier v1.x patch runs) - -For this option, set `sp_local=0`. - -**TODO: ** Split Uz and SpMh - -For `sp_local=-` both `mh_local` (0, 1) are ok: - -```bash -export mh_local=0 -``` - -#### Option 0: Mask exposures (deprecated) - -Run repeatedly if necessary: - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 8 -``` - -Combine all runs: - -```bash -combine_runs.bash -c flag_exp -``` - -### Option 1: Local split and mask exposures (recommended) - -Optional: Enable flags for local split processing and merge header runs as - -```bash -export sp_local=1 -export mh_local=1 -``` - -These flags are automatically set to 1 in the new job scripts. - - -Get single-HDU single-exposure IDs file (from missing 32 job): - -```bash -summary_run P$patch 32 -cp summary/missing_job_32_all.txt exp_shdu.txt -``` - -### Split exposures - -First, determine the number of maximum jobs with the option `-s` (see above). Then, submit with - -```bash -canfar_submit_job -j 2 -v -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - -### Mask exposures - -```bash -canfar_submit_job -j 8 -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - -### Exposure detection - -```bash -canfar_submit_job -j 32 -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - -### Tile preparation - -```bash -canfar_submit_job -j 64 -f tile_numbers.txt -``` - -### Tile shape measurement - -```bash -canfar_submit_job -j 128 -f tile_numbers.txt -``` - -### Merge sub-catalogues - -```bash -canfar_submit_job -j 256 -f tile_numbers.txt -``` - -### Create final catalogues - -```bash -canfar_submit_job -j 512 -f tile_numbers.txt -``` - -This was the last `ShapePipe` module to run for main processing. - - -### Merge all final catalogues - -The last step of `ShapePipe` processing is, per patch, to merget all final catalogues. This is done via a python script, as follows. -First, change to parent directory `/path/to/version` and run the following command for all patches - -```bash -patchnum=`tr $patch P ''` -create_final_cat.py -m final_cat_$patch.hdf5 -i . -p $patch/cfis/final_cat.param \ - -P $patchnum -o $patch/n_tiles_final.txt -v -``` - -## Additional `ShapePipe` processing - - -### Create star Catalogue - -We can additionaly create a combined star catalogue, with star shapes projecte from detector to world coordinates. -This is useful for validation and galaxy-PSF/star correlation diagnostics. - -#### Combine all PSF runs - -In each patch directory /path/to/version/$patch, run - -```bash -combine_runs.bash -p $psf -c psf -``` - -to create a single output directory of PSF files (symbolic links). - -Optionally, to create and plot results for this patch only: - -```bash -shapepipe_run -c $SP_CONFIG/config_Ms_$psf.ini -shapepipe_run -c $SP_CONFIG/config_Pl_$psf.ini -``` - -#### Collate star catalogues - -Collate all input validation PSF files into star catalogues, gathering positions -(X/Y/RA/DEC) and the MCCD CCD id. - -Note: HSM shapes are no longer rotated into world coordinates at this step. The -PSF/star ellipticities and sizes are now measured directly in sky coordinates -during PSF interpolation (galsim `FindAdaptiveMom(use_sky_coords=True)`), so -`collate_star_cat.py` only collates and passes the shapes through. - -> **v2.0 is patch-less.** Runs up to `v1.6` are organised in sky patches -> `P1`..`P`, each patch a run directory. `v2.0` removes the patch concept: a -> single run root, outputs directly under it. `v2.0` is the default; select an -> older layout with `-V` (e.g. `-V v1.6`). For `v2.0` the patch loop and the -> `-P` option no longer apply, and the patch token drops from the output -> filename (`validation_psf_conv-.fits` instead of -> `validation_psf_conv--.fits`). - -```bash -cd /path/to/version -mkdir star_cat -cd star_cat -``` - -For `v2.0` (the default), run once against the patch-less run root, producing -files `validation_psf_conv-.fits`: - -```bash -collate_star_cat.py -i .. -v -``` - -For `v1.x`, pass the version explicitly and run once per patch, creating a -directory per patch `P?` and producing files -`validation_psf_conv--.fits` (for the v1.4 setup only one file): - -```bash -collate_star_cat.py -i .. -V v1.6 -P $patchnum -v -``` - -Combine previously created files as links within one ShapePipe run directory (for the v1.4 setup only one link). -First (and optiohnal), create a subdir for a run and link to the input patches: - -```bash -cd /path/to/version/star_cat -mkdir v1.6 -ln -s ../P1 -ln -s ../P2 -... -``` - -Next, create links to all `validation_conv` runs: - -```bash -combine_runs.bash -p psfex -c psf_conv -``` - -Merge all converted star catalogues and create `final-starcat.fits`: - -```bash -export SP_RUN=`pwd` -shapepipe_run -c ~/shapepipe/example/cfis/config_Ms_psfex_conv.ini -``` - -Rename to general PSF and star catalogue used for all ("a") sub-versions: - - -```bash -cp output/run_sp_Ms/merge_starcat_runner/output/full_starcat-0000000.fits \ - unions_shapepipe_psf_2024_v1.6.a.fits -``` - -The FITS file `CATTYPE` (newer version) should be `validation_psf_conf`. - -## Post-processing - -The following post-processing steps are performed with the library `sp_validation`. - -### Extract Information - -First, we extract all information from the final catalogue, per patch. We copy -the parameter file and set links to the catalogues and `ShapePipe` config directory. - -```bash -cd /path/to/version/$patch -cp ~/astro/repositories/github/sp_validation/notebooks/params.py . -ln -s /path/to/final_cat_$patchnum.hdf5 # not relative path ../final_cat_P$patchnum.hdf5 ! -ln -s output/run_sp_MsPl/mccd_merge_starcat_runner/output/full_starcat-0000000.fits -ln -s ~/astro/repositories/github/shapepipe/example/cfis -``` - -Then edit `params.py`: Set patch name; set `wrap_ra` for P2. - -Now we can run the script, recommended via job submission on candide. For large patches, -this requies a job with a large memory, e.g. with `mem=380000` - - -```bash -[squeue] python ~/astro/repositories/github/sp_validation/notebooks/extract_info.py -``` - -This creates a patch-wise comprehensive catalogue. - -### Create global comprehensive catalogues - -```bash -cd /patch/to/version -[squeue] python ~/astro/repositories/github/sp_validation/scripts/create_joint_comprehensive_cat.py \ - -v v1.6.c -v -p P1+P2+P3+P4+P5+P6+P7+P8+P9 -``` - -This creates the file `unions_shapepipe_comprehensive_2024_v1.6.c.hdf5`. - - -### Apply structural masks - -First, edit the Python script `~/astro/repositories/github/sp_validation/notebooks/demo_apply_hsp_masks.py` -to match catalogue name. Check the coverage mask input file (see below). -Run the script to apply the healsparse structural masks: - -```bash -[squeue] python ~/astro/repositories/github/sp_validation/notebooks/demo_apply_hsp_masks.py -``` - -This creates the file `unions_shapepipe_comprehensive_struct_2024_v1.6.c.hdf5`. - - -### Define sample, calibrate catalogue - -We are close to finally perform the last post-processing step, which is the calibration. First, the final galaxy sample -in question needs to be defined, with masks and cuts to apply from a `yaml` config file. A number of pre-defined files -can be found in `~/astro/repositories/github/sp_validation/calibration`. - -For example, to create `v1.6.6`, the steps are: - -```bash -cd /path/to/version -mkdir -p v1.6.6 -cd v1.6.6 -ln -s ~/astro/repositories/github/sp_validation/calibration/mask_v1.X.6.yaml config_mask.yaml -ln -s ..//unions_shapepipe_comprehensive_struct_2024_v1.6.c.hdf5 unions_shapepipe_comprehensive_struct_2024_v1.X.c.hdf5 -[squeue] python ~/astro/repositories/github/sp_validation/calibrate_comprehensive_cat.py -``` - -calibrate_comprehensive - - -```{note} -The matched-star-catalogue diagnostic (`merge_psf_cat.py`) has moved into -[`sp_validation`](https://github.com/CosmoStat/sp_validation) / `cosmo_val`; -see that repository for its current equivalent. The coverage-mask helpers -(`get_ccds_with_psf`, `download_headers`, `extract_field_corners`, -`build_coverage_map`, `plot_coverage_map`, and `summary_run`) are shipped by -ShapePipe and documented below. -``` - -### Create matched star catalogue - -For diagnostics, a catalogue with multi-epoch shapes measured by ngmix matched with the validation star catalogue is used. -This is created as follows: - -```bash -cd /path/to/version -merge_psf_cat.py [-V v1.6|-P P1+P2+...] -v -``` - -This creates the joint catalogue unions_shapepipe_star_2024_v1.6.a.fits . - -### Create coverage mask - -First, on canfar, move to the directory that has the patch subdirectories. - -```bash -cd /path/to/version -``` - -#### Get exposure numbers - -If the file `$patch/exp_numbers.txt` does not exist for a given patch, create it with the summary program - -```bash -summary_run $patch 1 -``` - -Now, create the list of single-CCD footprints that have a valid PSF model. -Only these CCDs are stamped into the mask, so the accumulated map counts, per -sky pixel, the number of exposures with a valid PSF. The list is written to -`ccds_with_psf_v1.6.txt`: - -```bash -get_ccds_with_psf -v -V v1.6 -o ccds_with_psf_v1.6.txt -``` - -Next, download the exposure headers; indicate (with `-d`) a directory of -already downloaded headers, which are linked so duplicate downloads are -skipped: - -```bash -download_headers -i ccds_with_psf_v1.6.txt -o headers_v1.6 -d headers_v1.3 -v -``` - -From the headers, the per-CCD corner coordinates are extracted. Pass the -CCD list with `-l` so only CCDs with a valid PSF are written: - -```bash -extract_field_corners -i headers_v1.6 -l ccds_with_psf_v1.6.txt -o exp_ra_dec_v1.6.txt -v -``` - -Then, build the healsparse coverage mask file as -```bash -build_coverage_map -i exp_ra_dec_v1.6.txt -o coverage_v1.6.x.hsp -c 128 -n 131072 -v -``` - -The healsparse resolutions (128, 131072) match the bit masks. - - -Use `plot_coverage_map` to plot a region of the coverage mask. For example, -the SGC region with a colorbar clipped to the 1–5 exposure range: - -```bash -plot_coverage_map -i coverage_v1.6.x.hsp -o coverage_v1.6_SGC.png \ - -C -R -20 -r 45 -D 18 -d 40 -m 1 -M 5 -v -``` - -Here `-R`/`-r` and `-D`/`-d` set the RA and Dec plot limits, `-m`/`-M` the -colorbar range, and `-C` adds the colorbar. Building and plotting for a range -of versions (SGC and NGC regions) is automated by -`build_and_plot_coverage_maps.sh`. - - -## Extra Utilities - -### Run in Terminal in Parallel - -```bash -cat IDs.txt | xargs -I {} -P 16 bash -c 'init_run_exclusive_canfar.sh -j 512 -e {}' -``` diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 95a0aa94a..cb701c6b8 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -2,13 +2,17 @@ ## Quick start -Run the entire pipeline on a single example CFIS image with tile ID 246.290: -1. [Install](installation.md) `ShapePipe` — the recommended path is the container image, which bundles everything needed to run the pipeline. -3. Run the job script +Production runs go through the [Snakemake workflow](workflow.md). Put the tile +IDs in the file named by `tile_list` in `workflow/config.yaml`, one +`IDra.IDdec` per line, and run + ```bash -job_sp 246.290 -j 127 +workflow/bin/sp run ``` +A single tile is a legal tile list and is the smallest useful end-to-end run. +The sections below describe what that produces, stage by stage. + ## Introduction The `ShapePipe` pipeline processes single-exposure images and stacked images. Input images have to be calibrated beforehand for astrometry and photometry. This tutorial of an entire `ShapePipe` run covers specifically images from CFIS, the Canada-France Imaging Survey. CFIS stacks are so-called tiles, which are the co-adds of on average three exposures in the r-band. @@ -69,21 +73,21 @@ Naming and numbering of the input files can closely follow the original image na sheared images. This information is used in post-processing to compute calibrated shear estimates via metacalibration. - Summary statistic files - The `SETools` module that creates samples of objects according to some user-defined selection criteria (see [Select stars](#select-stars)) also outputs ASCII + The `SETools` module that creates samples of objects according to some user-defined selection criteria (see [Detect objects](#detect-objects-on-tiles-and-process-stars-on-single-exposures)) also outputs ASCII files with user-defined summary statistics for each CCD, for example the number of selected stars, or mean and standard deviation of their FWHM. Example: `star_stat-2366993-18.txt` - Tile ID list - ASCII file with a tile number on each line. Used for the `get_image_runner` module to download CFIS images (see [Download tiles](#download-tiles)). + ASCII file with a tile number on each line. Used for the `get_image_runner` module to download CFIS images (see [Retrieve input images](#retrieve-input-images)). - Single-exposure name list ASCII file with a single-exposure name on each line. Produced by the `find_exposure_runner` module to identify single exposures that were used to create - a given tile. See [Find exposures](#find-exposures)). + a given tile. See [Retrieve input images](#retrieve-input-images)). - Plots The `SETools` module can also produce plots of the objects properties that were selected for a given CCD. The type of plot (histogram, scatter plot, ...) and quantities to plot as well as plot decorations can be specified in the - selection criteria config file (see [Select stars](#select-stars)). + selection criteria config file (see [Detect objects](#detect-objects-on-tiles-and-process-stars-on-single-exposures)). Example: `hist_mag_stars-2104133-5.png` - Log files @@ -94,7 +98,7 @@ Naming and numbering of the input files can closely follow the original image na `ShapePipe` splits the processing of CFIS images into several parts: These are the retrieval and preparation of input images, processing of single exposures, -processing of tile images, creation and upload (optional) of _final_ shape catalogues. +processing of tile images, and creation of _final_ shape catalogues. The following flowchart visualised the processing parts and steps. @@ -104,46 +108,58 @@ Below, the individual processing steps are described in detail. ### Input and output paths -All required paths are automatically set in the job script `job_sp`. +The workflow's rules export every path a config interpolates before each +`shapepipe_run` call, so nothing here needs setting by hand for a normal run. -If an example config file is run outside this script, -the following path variables might need to be defined. +If a config file is run outside the workflow, the following variables need to be +defined. - `$SP_RUN`: Run directory of `ShapePipe`. In general this is just `pwd`, and can be set via ```bash export SP_RUN=`pwd` ``` but on a cluster this directory might be different. - `$SP_CONFIG`: Path to configuration files. In our example this is `$SP_BASE/example/cfis`. +- `$SP_UNIT_NUM`: the tile or exposure this run is restricted to, in ShapePipe's + image-number convention — a leading dash, and dots replaced by dashes (tile + `210.282` becomes `-210-282`, exposure `2605805` becomes `-2605805`). The + configs interpolate it into `NUMBER_LIST`. In addition, the output path `$SP_RUN/output` needs to be created by the user before running `ShapePipe`. -### Job and pipeline scripts +### The rule chain -The job script to run the pipeline in its entity or in parts is `job_sp[.bash]`. Type -```bash -job_sp -h -``` -for all options. +There is no monolithic job script. Every processing step below is a **rule**, and +every rule instance is one `shapepipe_run` call on one unit — one tile, or one +single exposure — inside that unit's own directory: -This script creates the subdirectory `$SP_RUN/output` to store all pipeline outputs -(log files, diagnostics, statistics, output images, catalogues, single-exposure headers with WCS information). +| level | rule chain | +| --- | --- | +| tile, prepare phase | `tile_get_images` → `tile_uncompress` → `tile_find_exposures` | +| exposure | `exp_get_images` → `exp_star_cat` → `exp_split` → `exp_mask` → `exp_psf` → `exp_persist` → `exp_footprint` → `clean_exposure` | +| tile, compute phase | `tile_exp_forest` → `tile_merge_headers` → `tile_detect` → `tile_vignets` → `tile_ngmix` → `tile_merge_cats` → `tile_make_cat` → `clean_tile` | -Optionally, the subdir `output_star_cat` is created by the used to store the external star catalogues for masking. This is only necessary if the pipeline is run on a cluster without internet connection to access star catalogues. In that case, the star catalogues need to be retrieved outside the pipeline, for example on a login node, and copied to `output_star_cat`. +Each rule calls -The job script automaticall performs a number of subsequent calls to the `ShapePipe` executable `shapepipe_run`, as ```bash shapepipe_run -c $SP_CONFIG/.ini ``` -The config file `.ini` contains the configuration for one or more modules. -See the main `ShapePipe` readme for more details. -The user specifies which steps are run with the command line option `-j JOB`. The integer value `JOB` -is bit-coded such that arbitrary combinations of steps can be run with a single call to `job_sp`. For -example, to run steps #1 and #2, type `job_sp -j 3`. +on a config file committed under `workflow/config/cfis/`, containing the +configuration for one or more modules. See [Configuration](configuration.md) for +what goes in one. To run a single rule and its prerequisites and nothing else, +name it as a snakemake target: `sp exp_psf ...`. + +Each unit directory holds an `output/` subdirectory with all pipeline outputs +(log files, diagnostics, statistics, output images, catalogues, single-exposure +headers with WCS information), beside the `manifests/` and `logs/` that are the +workflow's own bookkeeping. ### Select tiles -To run the job script, one or more CFIS tiles need to be chosen. If the tile IDs are known, they are provided to `job_sp` on the command line. +The tiles to process are the lines of the tile list. The campaign grows by +appending to that file: the tile-to-exposure index accumulates across +invocations, so adding tiles later changes which jobs exist without invalidating +completed work. If the tile IDs are not known a priori, they can be selected via sky coordinates, with the script `cfis_field_select`. For example, to find the tile number for a Planck cluster at R.A.=213.68 deg, dec=57.79 deg, run: @@ -153,54 +169,57 @@ cfis_field_select -i /path/to/shapepipe/auxdir/CFIS/tiles_202007/tiles_all_order The input text file (provide via the flag `-i`) contains a list of CFIS tiles, this can also be directory containing the tile FITS files. -The following sections describe the different steps that are performed with `job_sp`. +The following sections describe the steps the rule chain performs. ## Run the pipeline ### Retrieve input images -The command -```bash -job_sp TILE_ID -j 1 -``` -retrieves the image and weight corresponding to TILE_ID using the module `get_images`. -It then identifies the exposures that were used to create the tile image via the `find_exposures` runner. -Finally, another call to `get_images` retrieves the exposure images, weights, and flag files. +`tile_get_images` retrieves the image and weight corresponding to a tile ID +using the module `get_images`. `tile_find_exposures` then identifies the +exposures that were used to create the tile image via the `find_exposures` +runner, and `exp_get_images` retrieves each of those exposures' image, weight +and flag files. -For the retrieval method the user can choose betwen -- download from VOspace (`-r vos`); -- create symbolic link to existing file on disk (`-r symlink`). +For the retrieval method the user can choose between +- download from VOspace (`RETRIEVE = vos` in the config); +- create symbolic link to existing file on disk (`RETRIEVE = symlink`). Note that internet access is required for this step if the download method is `vos`. -An output directory `run_sp_GitFeGie` (in `output`) is created containing the results of `get_images` for tiles (`Git`), -`find_exposures` (`Fe`), and `get_images` for exposures (`Gie`). +These three rules are the workflow's PREPARE phase. The exposure half of the run +cannot be scheduled before them, because which exposures exist is something +`find_exposures` has to say first — which is why one run is two snakemake +invocations over one Snakefile (see [the workflow page](workflow.md)). ## Prepare input images -With -```bash -job_sp TILE_ID -j 2 -``` -the compressed tile weight image is uncompressed via the `uncompress_fits` module. Then, the single-exposure images, weight, and flags are split into single-exposure single-CCD file -(one FITS file per CCD) with `split_exp`. -Finally, the headers of all single-exposure single-CCD files are merged into a single `sqlite` file, to store the WCS information of the input exposures. +`tile_uncompress` uncompresses the compressed tile weight image via the +`uncompress_fits` module. + +`exp_split` splits each single-exposure image, weight and flag into +single-exposure single-CCD files (one FITS file per CCD) with `split_exp`, and +writes that exposure's WCS information to `headers-.npy`. -Two output directories are created, `run_sp_Uz` for `uncompress_fits`, and `run_sp_exp_SpMh` for the output of the modules -`split_exp` (`Sp`) and `merge_headers` (`Mh`). +`tile_merge_headers` then merges the headers of the exposures overlapping one +tile into a single `sqlite` file, so the tile-side modules can look up the WCS of +any of them. ## Mask images -Run -```bash -job_sp TILE_ID -j 4 -``` -to mask tile and single-exposure single-CCD images. Both tasks are performed by two calls to the `mask` runner. +`exp_mask` masks the single-exposure single-CCD images with the `mask` runner. -Note that internet access is required for this step, since a reference star catalogue is downloaded. +Masking needs a reference star catalogue, and no compute node needs internet +access to get one: `star_catalogue` makes one Vizier cone query per HEALPix +chunk of the campaign footprint into a run-independent cache, and `exp_star_cat` +cuts each exposure's catalogue out of that cache with no network at all. Both +are `localrule`s, so the queries run in the head process. Network cost therefore +scales with sky area rather than with exposure count. -The output of both masking runs are stored in the output directory `run_sp_MaMa`, with run 1 (2) of -`mask` corresponding to tiles (exposures). +There is no `tile_mask` rule: the committed config chain is the unmasked +`tile_detect` variant. Adding the masked variant is a config plus one rule, and +needs a tile-side analogue of `exp_star_cat` — tile star catalogues key on tile +ID, so they are a separate cache namespace. **Diagnostics:** Open a single-exposure single-CCD image and the corresponding pipeline flag in `ds9`, and display both frames next to each other. Example @@ -218,23 +237,23 @@ have a zero-padded pixel border, which is not accounted for by `ds9`. ## Detect objects on tiles and process stars on single exposures -The call -```bash -job_sp TILE_ID -j 8 -``` -performs a number of steps. First, objects on the tiles are deteced with the `sextractor` runner. -Next, the following tasks are run on the single-exposure single-CCD images: -- Objects are deteced with `sextractor`. +`tile_detect` detects objects on the tile with the `sextractor` runner. + +`exp_psf` runs the single-exposure single-CCD chain: +- Objects are detected with `sextractor`. - Star candidates are selected via `setools`. -- The PSF model is created, either with `psfex` for PSFex, or - with `mccd_preprocessing` and `mccd_fit_val` for MCCD. -- The PSF model is interpolated to star positions for validation. For the PSFEx model, this is done - via a call to `psfex_interp`. For MCCD, the modules `merge_starcat`, `mccd_plots`, and - `mccd_interp` are called. +- The PSF model is created with `psfex`. +- The PSF model is interpolated to star positions for validation, via a call to + `psfex_interp`. + +The output directory is `run_sp_exp_SxSePsfPi`, holding the output of SExtractor +on the exposures (`Sx`), `setools` (`Se`), the PSF model (`Psf`) and the +validation interpolation (`Pi`). -The output directory for both the `mccd` and `psfex` options is `run_sp_tile_Sx_exp_SxSePsf`. -This stores the output of SExtractor on the tiles (`tile_Sx`), on the exposures (`exp_Sx`), -`setools` (`Se`), and the Psf model (`Psf`). +`exp_persist` then copies the products named by `persist_exp:` in +`workflow/config.yaml` — by default the `validation_psf-*.fits` that the rho and +tau statistics are computed from — off scratch onto the persistent root, before +the purge or `clean_exposure` can take them. The following plots show an example of a single CCD, in the center of the focal plane. @@ -282,60 +301,40 @@ several runs. ## Galaxy selection -The focus of the next step, -```bash -job_sp TILE_ID -j 16 -``` -is the selection of galaxies as extended objects compared to the PSF. -First, the PSF model is interpolated to galaxy positions, according to the PSF model -with `psfex_interp` or `mccd_interp`. Next, postage stamps around galaxies -of the weights maps are created via `vignetmaker`. Then, the spread model -is computed by the `spread_model` module. Finally, postage stamps -around galaxies of single-exposure data is extracted with another call -to `vignetmaker`. - -The output directory is -- `run_sp_MiViSmVi` if the PSF model is `mccd`; -- `run_sp_tile_PsViSmVi` for the `PSFEx` PSF model. +`tile_vignets` selects galaxies as extended objects compared to the PSF. +First, the PSF model is interpolated to galaxy positions with `psfex_interp`. +Next, postage stamps around galaxies of the weight maps are created via +`vignetmaker`. Then the spread model is computed by the `spread_model` module. +Finally, postage stamps around galaxies of single-exposure data are extracted +with another call to `vignetmaker` — `Pi`, `Vi`, `Sm`, `Vi`, hence the config +name `config_tile_PiViVi.ini`. -This corresponds to the MCCD/PSFex interpolation (`Mi`/`Pi`), `vignetmaker` (`Vi`), `spread_model` (`Sm`), and the -second call to `vignetmaker` (`Vi`). +`tile_vignets` reads the exposure products through the symlink forest that +`tile_exp_forest` built for this tile, and is the last stage that touches them: +once every consuming tile has its vignets, an exposure store becomes +reclaimable. ## Shape measurement -The call -```bash -job_sp TILE_ID -j 32 -``` -computes galaxy shapes using the multi-epoch model-fitting method `ngmix`. At the same time, -shapes of artifically sheared galaxies are obtained for metacalibration. +`tile_ngmix` computes galaxy shapes using the multi-epoch model-fitting method +`ngmix`. At the same time, shapes of artificially sheared galaxies are obtained +for metacalibration. -Shape measurement is performed in parallel for each tile, the number of processes can be specified -by the user with the option `--nsh_jobs NJOB`. This creates `NJOB` output directories `run_sp_tile_ngmix_Ngu`. -with `X` = 1 ... `NJOB` containing the result of `ngmix`. +Shape measurement is split into parallel chunks: `ngmix_chunks:` in +`workflow/config.yaml` sets how many, and each chunk is its own rule instance +writing its own `run_sp_tile_ngmix_Ngu` output directory. ## Paste catalogues -The last real processing step is -```bash -job_sp TILE_ID -j 64 -``` -This task first merges the `NJOB` parallel `ngmix` output files from the previous step into -one output file. Then, previously obtained information are pasted into a _final_ shape catalogue via `make_cat`. -Included are galaxy detection and basic measurement parameters, the PSF model at -galaxy positions, the spread-model classification, and the shape measurement. - -Two output directories are created. -The first one is `run_sp_Ms` for the `merge_sep` run. -The second is `run_sp_Mc` for the `make_cat` task; the name is the same for both the `MCCD` and `PSFEx` PSF model. - - -## Upload results - -Optionally, after the pipeline is finished, results can be uploaded to VOspace via -```bash -job_sp TILE_ID -j 128 -``` +`tile_merge_cats` merges the parallel `ngmix` output files from the previous +step into one file with `merge_sep`, in `run_sp_Ms`. +`tile_make_cat` then pastes the previously obtained information into a _final_ +shape catalogue via `make_cat`, in `run_sp_Mc`. Included are galaxy detection +and basic measurement parameters, the PSF model at galaxy positions, the +spread-model classification, and the shape measurement. The rule publishes that +catalogue onto the persistent root as +`/tiles///final_cat-.fits` — the campaign's +product, and also the marker that says this tile is finished. diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md deleted file mode 100644 index e8a3768f4..000000000 --- a/docs/source/post_processing.md +++ /dev/null @@ -1,81 +0,0 @@ -# Post-processing - -This page shows all required steps of post-processing the results from one or -more `ShapePipe` runs. Post-processing combines various individual `ShapePipe` -output files, and creates joint results, for example combining individual tile -catalogues into a large sky area. The output of post-processing is a joint _shape -catalogue_, containing all required information to create a calibrated shear -catalogue via _metacalibration_), a joint star catalogue, and PSF diagnostic plots. - - - ---- - -If main ShapePipe processing happened at the old canfar VM system (e.g. CFIS v0 and v1), go -[here](vos_retrieve.md) for details how to retrieve the ShapePipe output files. - ---- - -```{note} -This page documents the **legacy** post-processing used for pre-v1.4 runs on the -canfar VM system. PSF validation and the scale-dependent diagnostics -(ρ-statistics) now live in -[`sp_validation`](https://github.com/CosmoStat/sp_validation) rather than in -ShapePipe; the steps below are retained for reference and for reproducing older -runs. -``` - -The following steps were used for pre-v1.4 runs performed on the canfar VM system. - -1. Optional: Split output into sub-samples - - An optional intermediate step is to create directories for sub-samples, for example one directory - for each patch on the sky. This will create symbolic links to the results `.tgz` files downloaded in - the previous step. For example, to create the subdir `tiles_W3` with links to result files to `all` for - those tiles contained in the list `tiles_W3.txt`, do: - ```bash - create_sample_results --input_IDs tiles_W3.txt -i . all -o tiles_W3 -v - ``` - The following steps will then be done in the directory `tiles_W3`. - -2. Run PSF diagnostics, create merged catalogue - - Type - ```bash - post_proc_sp -p PSF - ``` - to automatically perform a number of post-processing steps. Choose the PSF model with the option - `-p psfex|mccd`. In detail, these are (and can also be done individually - by hand): - - 1. Analyse psf validation files - - ```bash - combine_runs -t psf -p PSF - ``` - with options as for `post_proc_sp`. - This script creates a new combined psf run in the ShapePipe `output` directory, by identifying all psf validation files - and creating symbolic links. The run log file is updated. - - 3. Merge the individual PSF validation files into one catalogue, and create - plots of the PSF and its residuals in the focal plane as a diagnostic of - the overall PSF model: - ```bash - shapepipe_run -c /path/to/shapepipe/example/cfis/config_MsPl_PSF.ini - ``` - The scale-dependent PSF diagnostics that propagate to the shear correlation - function (the ρ-statistics, {cite:p}`rowe:10`, {cite:p}`jarvis:16`) are no - longer computed here — they have moved to - [`sp_validation`](https://github.com/CosmoStat/sp_validation). - - 4. Merge final output files - - Create a single main shape catalogue: - ```bash - merge_final_cat -i -p -v - ``` - Choose as input directory `input_dir` the `make_cat` output of the runs - being combined. A default parameter file `` is - `/path/to/shapepipe/example/cfis/final_cat.param`. - On success, the file `./final_cat.npy` is created. Depending on the number of - input tiles, this file can be several tens of Gb large. diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md index a930d40ed..031a3b22b 100644 --- a/docs/source/random_cat.md +++ b/docs/source/random_cat.md @@ -3,28 +3,25 @@ This section describes how to create tile-based random catalogues and healpix masks, and combined randoms and masks for a selection of tiles. -The masked regions are obtained on input from ShapePipe pixel mask ("pipeline flag") -files. +The masked regions are obtained on input from ShapePipe pixel mask ("pipeline +flag") files. ```{note} -Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see -[VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The `random_cat` module itself is current; -the input-staging and joint-mask steps now overlap with -[`sp_validation`](https://github.com/CosmoStat/sp_validation). The steps are -retained for reference. +The `random_cat` module itself is current. The staging and joint-mask steps +below predate the Snakemake workflow and are retained for reference; the +helpers they used to call (`prepare_tiles_for_final`, `merge_final_cat`, +`canfar_avail_results`, `canfar_download_results`) are no longer shipped, and +where a step named one it now says what the step has to achieve instead. ``` ## Set up ### ID file and shell variables -First, if if does not exist already, create the file ``tile_numbers.txt`` containing a list of tile IDs, -one per line. This is the same format as the input file to ``get_images_runner``. -For example, link to a patch ID list, -```bash -ln -s tiles_PX.txt tile_numbers.txt -``` +First, if it does not exist already, create the file ``tile_numbers.txt``: a +tile list, one ID per line. This is the same format as the input file to +``get_images_runner``. + Next, set the run and config paths, ```bash export SP_RUN=. @@ -33,56 +30,23 @@ export SP_CONFIG=/path/to/config-files ### Get images or image headers -We need to footprint of the image tiles. If they have been downloaded for a ``ShapePipe`` run, -check that they are accessible as last run of the ``get_images_runner`` module. +We need the footprint of the image tiles. If they have been downloaded for a +``ShapePipe`` run, check that they are accessible as last run of the +``get_images_runner`` module. If not, we can just download the headers to gain significant download time. ```bash shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini ``` -### Check pixel mask files - -Make sure that all pixel mask files are present. If they have been downloaded from ``vos`` as ``.tgz`` files, -type -```bash -canfar_avail_results -i tile_numbers.txt --input_path . -v -m -o missing_mask.txt -``` -In case of missing mask files, check whether they are present in the ``vos`` remote directory, -```bash -canfar_avail_results -i tile_numbers.txt --input_path vos:cfis/vos-path/to/results -v -m -``` -If missing on ``vos``, process those tiles. If processing only up the the mask is necessary, -the following steps can be carried out, -```bash -job_sp -j 7 TILE_ID -job_sp -j 128 TILE_ID -``` -The first command processes the tile up to the mask; the second line uploads the mask files -to ``vos``. +### Collect the pixel mask files -Now, download the missing masks with -```bash -canfar_download_results -i missing_mask.txt --input_vos vos-path/to/results -m -v -``` -Untar .tgz files if required, -```bash -while read p; do tar xvf pipeline_flag_$p.tgz; done ` on the config chains described in +[Configuration](configuration.md); the workflow supplies the scheduling, the +per-unit isolation, the bookkeeping and the reclamation around them. Reading +[Basic execution](basic_execution.md) first is still the right order. + +This page is the orientation. The deep reference — design rationale, every +rule, the profile, the two-roots argument — is +[`workflow/README.md`](https://github.com/CosmoStat/shapepipe/blob/develop/workflow/README.md) +in the repository, and the living design record is +[issue #848](https://github.com/CosmoStat/shapepipe/issues/848). + +## Quick start + +```bash +# One-time, on a SHARED filesystem: the SLURM executor re-invokes this python +# inside every job, so a node-local path (/tmp) will not do. +uv venv /project///snakemake-env --python 3.12 +source /project///snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' + +# Edit workflow/config.yaml: tile_list, run_dir, products_dir, container, +# star_cats. + +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # the success/failure tables, any time, mid-run is fine +workflow/bin/sp container status # which image the jobs will run inside +workflow/bin/sp cancel # scancel this workflow's jobs +``` + +`workflow/bin/sp` is the entry point for everything. It loads the apptainer +module and the venv, snapshots the code it is about to launch, and sets the +state directory, the SLURM profile and `SP_PHASE`. **Bare `snakemake` outside +`sp` is unsupported** — it skips all of that, including the phase variable the +Snakefile needs to build its index at parse time. Any verb `sp` does not +recognise passes straight through to snakemake with the right profile and state +directory, which is the escape hatch for `sp --unlock`, `sp --dag` or +`sp exp_psf ...`. + +## The two static invocations + +The exposure job set is *derived from data the run itself produces* — a tile's +`find_exposures` output — so it cannot live in the same static DAG that produces +it. `sp run` is therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** (`snakemake prepare_all_tiles`) — the per-tile static chain + `tile_get_images → tile_uncompress → tile_find_exposures`, with `keep-going` + so tile failures stay independent. A nonzero exit is a warning, not a fatal + error: tiles that lost their exposure list are dropped at the compute parse, + and `SP_MISSING_THRESHOLD` is the real gate on how many may be lost. +2. **COMPUTE** (`snakemake all`) — this invocation's *parse* builds the + tile↔exposure index (`run_index.sqlite`) and the DAG then runs the full + exposure and tile chains: + + | level | chain | + | --- | --- | + | exposure | `exp_get_images → exp_star_cat → exp_split → exp_mask → exp_psf → exp_persist → exp_footprint → clean_exposure` | + | tile | `tile_exp_forest → tile_merge_headers → tile_detect → tile_vignets → tile_ngmix → tile_merge_cats → tile_make_cat → clean_tile` | + | campaign | `star_catalogue`, `coverage_map` | + +`sp run` chains both and fails if either phase did. The index **accumulates** +across invocations, so appending tiles to `tile_list` later changes which jobs +exist without invalidating completed work. + +Every rule instance is its own SLURM job (`executor: slurm` in +`profiles/nibi/config.yaml`) carrying that rule's own attempt-scaled resources, +and runs inside the container via snakemake's software-deployment method — the +workflow never calls apptainer itself. + +## Manifests are the currency + +The atom is **one rule == one `shapepipe_run` on one unit** (one tile, or one +exposure), in its own sharded store: `/tiles/<2-char prefix>//` or +`/exp///`. + +A rule's single declared output is not its product files but that unit's +**manifest**, `/manifests/.json`. Per-CCD declaration would +mean millions of paths at DR6 scale, and a missing CCD is frequently +legitimate. + +Manifests are *success-only*. After each run, `completeness.py check` writes its +full verdict — per-runner counts against floors, scraped failure reasons, the +`shapepipe_run` exit status — to the rule's `log:` +(`/logs/.json`) **every** time, and additionally to the +declared manifest **only** when that verdict is a success. So a manifest on disk +means "this stage succeeded", and a resume after an unclean death can never +schedule downstream work on top of a failure. + +Completeness is a **count floor, not a taxonomy**: a stage is a real failure iff +a mandatory runner produced fewer products than its floor. Per-CCD attrition +between the floor and the expected count is tolerated and recorded. There is no +error-signature whitelist. + +## `sp report` + +`sp report` reads the index and those two directories — the manifest for +success, the log for failure, a unit with neither ran nothing — and prints the +per-stage tables plus a `run_report.json` beside the index. It is deliberately +**not** a DAG node: a report rule declaring every tile's output would be a +descendant of every job, so one hard failure under `--keep-going` would poison +its cone and the report would never run in exactly the situation it exists for. +It is a plain script, runnable at any time including mid-run, and the COMPUTE +invocation's `onsuccess`/`onerror` hooks fire it automatically so every run ends +with one. + +## `sp container` + +`sp container` owns which image the jobs run inside, in one resolution order +shared by the CLI and the workflow: **sandbox → your cached SIF → the +`container:` path in `config.yaml`**. + +```bash +sp container status # layers present, active one, revision vs HEAD +sp container pull # ghcr.io/cosmostat/shapepipe:develop-runtime +sp container sandbox # unpack the SIF writable (opt-in) +sp container exec --writable pip install +sp container resolve # just the path the workflow will run +``` + +The cached SIF is a private pull, so nobody else's refresh moves the ground +under your running jobs; the sandbox is the escape hatch for work needing a +package the image does not carry yet. `status` places the image's +`org.opencontainers.image.revision` label against this checkout's HEAD. +`pull` needs the network — run it on a login node, never from a batch job. + +See [Container workflow](container.md) for how the images are built. + +## Durable products + +Two roots. `run_dir` is scratch: the bulk per-unit stores, sized to finish +inside the purge window. `products_dir` is persistent and holds the low-volume +things worth keeping: + +- **Final catalogues** — `/tiles///final_cat-.fits`, + mirroring the scratch shard structure. This file is also the *tile-finished + marker* that lets a completed tile stop pinning its exposures, which is the + second reason it cannot live on scratch. +- **PSF products** — `exp_persist` packs the files named by `persist_exp:` in + `config.yaml` (by default the psfex_interp `validation_psf-*.fits`, the rho/tau + statistics input) into one uncompressed tar per exposure, plus a manifest + recording the members. It answers the scratch purge, not the workflow's own + reclamation, so it runs whether or not `clean:` is on. +- **The index and the report** — `run_index.sqlite`, `missing.json`, + `run_report.json`. +- **Coverage** — see below. + +Exposure stores are reclaimed by the in-DAG `clean_exposure` rule rather than by +`temp()`: exposures overlap tiles, so `temp()` would cascade destructive reruns +whenever a tile is appended. Reclamation leaves a `cleaned.json` tombstone +holding the absorbed manifests, so a reclaimed exposure still reports its +per-runner counts and blocks no tile. + +## Coverage + +Sky coverage is a workflow product, built from records the DAG already writes +rather than by scraping a finished campaign. + +`exp_footprint` writes one JSON per exposure to +`/exp///manifests/exp_footprint.json` giving the +four sky corners of every CCD that got a PSF model. The valid-PSF CCD set comes +off `exp_persist.json`'s tar members — exact, because `psfex_interp` returns +*without* writing `validation_psf-*.fits` on NOT_ENOUGH_STARS, BAD_CHI2 or +FILE_NOT_FOUND — and the WCS off the `headers-.npy` written by `exp_split`. +That makes `validation_psf-*` in `persist_exp:` a precondition of the chain, not +a preference. + +Set `coverage: {enabled: true}` and one further job, `coverage_map`, stamps every +footprint into `/coverage/coverage.hsp` — a HealSparse map counting, +per sky pixel, the exposures with a valid PSF there. It is **campaign-cumulative**: +its declared inputs are the in-scope footprints, but the script reads every record +on the products root, reclaimed exposures included, so appending tiles grows the map +instead of replacing it. `nside` is set in `config.yaml` to the production +128/131072 pair, chosen to align pixel-wise with the UNIONS bit masks — nothing +defaults to it, and a coarser map would look plausible and silently fail to align. + +Plotting stays out of the DAG, as a human act on a durable product: + +```bash +plot_coverage_map -i /coverage/coverage.hsp ... +``` + +with the sky windows under `coverage.plot` in `config.yaml`. diff --git a/example/cfis/config_tile_PiViVi_canfar_sx.ini b/example/cfis/config_tile_PiViVi_canfar_sx.ini index cb72e1c11..134ebf0b9 100644 --- a/example/cfis/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis/config_tile_PiViVi_canfar_sx.ini @@ -79,16 +79,16 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 # Multi-epoch mode parameters -# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR -# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered -# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +# Root directory of per-exposure work directories. psfex_runner/output/ +# dirs are discovered by scanning $SP_EXP for the exposures listed in the +# exp_numbers input file. ME_DOT_PSF_EXP_DIR = $SP_EXP # Input psf file pattern @@ -161,8 +161,8 @@ STAMP_SIZE = 51 PREFIX = # Additional parameters for path and file pattern corresponding to single-exposure -# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for -# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +# run outputs. Output dirs are discovered by scanning $SP_EXP for the +# exposures listed in the exp_numbers input file. ME_IMAGE_EXP_DIR = $SP_EXP ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/example/cfis/config_tile_PiViVi_canfar_uc.ini b/example/cfis/config_tile_PiViVi_canfar_uc.ini index d59af1c4a..cc05969a1 100644 --- a/example/cfis/config_tile_PiViVi_canfar_uc.ini +++ b/example/cfis/config_tile_PiViVi_canfar_uc.ini @@ -77,16 +77,16 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 # Multi-epoch mode parameters -# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR -# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered -# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +# Root directory of per-exposure work directories. psfex_runner/output/ +# dirs are discovered by scanning $SP_EXP for the exposures listed in the +# exp_numbers input file. ME_DOT_PSF_EXP_DIR = $SP_EXP # Input psf file pattern @@ -159,8 +159,8 @@ STAMP_SIZE = 51 PREFIX = # Additional parameters for path and file pattern corresponding to single-exposure -# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for -# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +# run outputs. Output dirs are discovered by scanning $SP_EXP for the +# exposures listed in the exp_numbers input file. ME_IMAGE_EXP_DIR = $SP_EXP ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/example/cfis/config_valjoint_Pl_mccd.ini b/example/cfis/config_valjoint_Pl_mccd.ini index 086690815..81c444b55 100644 --- a/example/cfis/config_valjoint_Pl_mccd.ini +++ b/example/cfis/config_valjoint_Pl_mccd.ini @@ -39,7 +39,7 @@ INPUT_DIR = ./SP OUTPUT_DIR = ./output # FILE_PATTERN (opional) list of string patterns to identify input files -FILE_PATTERN = unions_shapepipe_psf_2022_v1.0.2 +FILE_PATTERN = unions_shapepipe_psf_2022 # FILE_EXT (opional) list of string extensions to identify input files FILE_EXT = .fits diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini index 28ed565af..164f6e69f 100644 --- a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -120,8 +120,8 @@ STAMP_SIZE = 51 PREFIX = # Additional parameters for path and file pattern corresponding to single-exposure -# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for -# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +# run outputs. Output dirs are discovered by scanning $SP_EXP for the +# exposures listed in the exp_numbers input file. ME_IMAGE_EXP_DIR = $SP_EXP ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner ME_IMAGE_PATTERN = flag, image, weight diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml new file mode 100644 index 000000000..23d385bd3 --- /dev/null +++ b/profiles/nibi/config.yaml @@ -0,0 +1,157 @@ +# Snakemake profile for the Nibi cluster (Digital Research Alliance). +# +# SLURM-EXECUTOR mode (PRD #848 D-profile): one SLURM job per rule instance, +# each carrying its rule's own attempt-scaled resources (cpus_per_task, +# mem_mb, runtime). Snakemake feeds the queue as jobs finish, so DR6's ~170k +# total jobs never queue at once, and multi-node scaling is inherent — this +# supersedes the earlier one-allocation/local-scheduler mode. +# +# Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). +# software-deployment-method wraps every job's shell in `apptainer exec` — the +# user never types apptainer. WHICH image is resolved in the Snakefile through +# workflow/scripts/container.py, whose docstring documents the layers and the +# resolution order; `sp container status` prints which layer is live. + +executor: slurm + +# Per-user submit cap, queried 2026-07-30 on nibi: +# sacctmgr show assoc user=cdaley format=Account,MaxSubmitPU -P +# -> def-mjhudson_cpu 1000, def-mjhudson_gpu 1000 (MaxSubmitPU; no site-wide +# MaxSubmitJobs in `scontrol show config`, so the association limit governs) +# Set to ~80% of that (1000) so this workflow never starves other submissions +# under the same account. Re-query if the association limits change. +jobs: 800 + +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: def-mjhudson_cpu + +software-deployment-method: [apptainer] +# Explicit container environment (finding 2/3): the apptainer SDM otherwise drops +# the proven-recipe env and hardcodes --home , hiding ~/.ssl/cadcproxy.pem. +# --cleanenv strict host-env isolation (APPTAINERENV_*/SINGULARITYENV_* survive) +# OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) +# MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) +# --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# >>> EDIT THIS: THE ONE NON-PORTABLE LINE IN THIS FILE. <<< +# The apptainer-args string below hardcodes one user's home +# (--home /home/cdaley). A different user must edit it by hand. +# IT CANNOT BE FIXED HERE: YAML cannot splice an env var, and +# snakemake escapes a literal `$` before the string reaches +# any shell (see the $SLURM_TMPDIR post-mortem below, which +# cost a whole campaign). Nor can bin/sp inject it through +# --apptainer-args: that REPLACES the profile value rather +# than appending to it, so the launcher would have to restate +# this whole string -- and workflow/scripts/container.py reads +# THIS LINE to give `sp container exec` the same environment +# jobs get, so a second definition is exactly the divergence +# that mechanism exists to prevent. +# THE PYTHONPATH PIN IS THE ONE EXCEPTION, and it is not +# edited by hand: `sp run` copies this whole file into the +# campaign's code snapshot and rewrites that ONE path to the +# snapshot's src/ (bin/sp, "the launch code snapshot"). Still +# one source of truth for the flags -- the copy is generated, +# never edited -- and jobs stop reading a live checkout. +# --bind /local NODE-LOCAL NVME. $SLURM_TMPDIR on nibi is +# /local/scratch/..0, and the mount is /local +# (/dev/nvme0n1, ext4, 3.5 TB) -- binding /localscratch is +# NOT enough, the real path is under /local (verified, +# probe job 20794182). The fused `tile_shape` group job +# WRITES its 5.6 GB vignette store there and never puts it +# on scratch at all (workflow/rules/tile.smk); every other +# rule simply ignores the bind. Adding it does NOT +# invalidate anything: the `software-env` rerun trigger +# hashes only job.container_img_url (+ conda env, env +# modules) -- +# snakemake/persistence/__init__.py::_software_stack_hash, +# v9.23.1 -- and apptainer-args is not in that hash. +# +# HOW the in-container shell LEARNS its node-local dir: +# IT DOES NOT. It DERIVES the path from the tile wildcard +# (tile.smk::tile_local -> /local/scratch/sp-), and +# that is the only mechanism that survives contact with +# snakemake. Three alternatives were tried; all fail, and +# the third failed IN PRODUCTION, so read this before +# reaching for any of them again: +# * {resources.tmpdir} is NOT $SLURM_TMPDIR. Snakemake +# defers that resource to the job and evaluates it as +# tempfile.gettempdir(); nibi's Slurm does not export +# TMPDIR, so it lands on /tmp -- which on a compute +# node is a 378 GB RAM-backed tmpfs whose pages are +# charged to the job's memory cgroup. Staging 6 GB +# there would eat the job's RAM, not use the NVMe. +# (Measured, job 20794656.) +# * APPTAINERENV_SLURM_TMPDIR would have to be exported +# by something that already knows the JOB's value; +# bin/sp runs on the login node, where it is unset. +# * SPLICING "$SLURM_TMPDIR" INTO apptainer-args AND +# LETTING THE JOB'S SHELL EXPAND IT DOES NOT WORK. +# This was the shipped design for exactly one campaign +# and it killed every tile of run 20798193 in eight +# minutes: snakemake ESCAPES the `$` before the string +# reaches any shell, so the container receives the +# literal characters and never a path. NOTHING can be +# carried into the container this way -- not a tmpdir, +# not anything else. The bind below is static, and the +# VALUE is derived job-side from a wildcard instead. +# Every tile_shape member still REFUSES TO RUN if the +# derived directory is not real (tile.smk), which is the +# only reason that campaign cost minutes and not a night: +# a broken node-local path is a loud failure, never a +# silent slide back onto NFS, and never a 5.6 GB write +# into the RAM-backed /tmp tmpfs. +# PYTHONPATH SETTLED CALL 3 — pins THIS branch's src/, which is +# develop@97e16d50 plus the four commits that genuinely need +# module code: ngmix chunk fields + position-seeded RNG, +# merge_sep_cats chunk paths, the vizier star-cat helpers, +# and the cherry-picked #873 (seeded setools split). +# NOT shapepipe-prod (drifted to a PR branch mid-run, and the +# live p3-batch1 job reads it) and not the sif default +# (frozen pre-#843, and pre-#873). Production later rebuilds +# the sif at the validated commit and DROPS this --env line. +# The path written HERE is the checkout's, and is what +# `sp container exec` uses; the value JOBS see is the +# snapshot's src/, substituted per the paragraph above. So a +# running campaign holds its launch code even after this +# line (or the tree it names) changes. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch --bind /local" + +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true + +# No `keep-incomplete` here. The declared output IS the manifest, so letting +# snakemake delete a failed job's output is exactly the manifest-vs-log +# semantics this workflow wants (completeness.py's docstring argues it), and +# `show-failed-logs` above surfaces the surviving log on the console. + +# rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still +# propagate — completeness.py writes the manifest only on change, so mtimes move +# only when reality moves. +# +# `input` is dropped because reclamation needs the tile->exposure edge to be +# CONDITIONAL; with the trigger on, that conditional itself reruns every +# finished tile against a reclaimed exposure store. tile.smk's tile_finished +# commentary carries the mechanism and the fixture-t4 job counts. +# +# Nothing this workflow relied on is lost. The two genuinely data-derived input +# sets are covered another way: a changed exposure list arrives through the +# (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides +# in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute +# + this profile), so a bare snakemake invocation cannot quietly recombine +# reclamation with the `input` trigger. +rerun-triggers: [mtime, params, code, software-env] + +# NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a +# rule's own values (verified snakemake 9.23), which would kill the +# attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +# ngmix thread count, and would flatten the group resource composition. The +# RULES own threads and resources; this profile only sets defaults for rules +# that state nothing (default-resources above). +# +# `group:` fusion of the short rules (PRD D-profile) lives in the rule files, +# each label documented in its file's docstring: `tile_prep` (prepare.smk), +# `exp_short` (exposure.smk) and `tile_shape` (tile.smk). diff --git a/pyproject.toml b/pyproject.toml index 5333aa91c..3d468ee89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ "pyqtgraph", "reproject>=0.19", "sf_tools>=2.0.4", - "skaha>=1.7", "skyproj", "sqlitedict>=2.0", "termcolor", @@ -84,14 +83,8 @@ dev = ["shapepipe[doc,jupyter,lint,plot,release,test,fitsio]"] [project.scripts] shapepipe_run = "shapepipe.shapepipe_run:main" -summary_run = "shapepipe.summary_run:main" -canfar_submit_job = "shapepipe.canfar_run:run_job" canfar_monitor = "shapepipe.canfar_run:run_log" canfar_monitor_log = "shapepipe.canfar_run:run_monitor_log" -get_ccds_with_psf = "shapepipe.get_ccds_run:run_ccd_psf_handler" -download_headers = "shapepipe.coverage_run:run_download_headers" -extract_field_corners = "shapepipe.coverage_run:run_extract_corners" -build_coverage_map = "shapepipe.coverage_run:run_build_coverage" plot_coverage_map = "shapepipe.coverage_run:run_plot_coverage" [tool.uv] diff --git a/scripts/README.rst b/scripts/README.rst index 1d3e265a6..e1f3d0bec 100644 --- a/scripts/README.rst +++ b/scripts/README.rst @@ -1,28 +1,80 @@ Scripts directory ================= -This directory contain scripts that are used on pipeline outputs or to prepare -files. They are not run throught the pipeline framework. For more details on how -to run each them see below. +Helpers that operate on pipeline inputs and outputs but are **not** pipeline +modules: nothing here is run by ``shapepipe_run``, and nothing here is part of +the Snakemake workflow's rule graph (that is ``workflow/scripts/``). -Python scripts -============== +Everything under ``scripts/*/`` with a ``.py``, ``.sh`` or ``.bash`` extension is +symlinked onto ``$PATH`` inside the container image under its bare name, so +``create_star_cat`` and ``python scripts/python/create_star_cat.py`` are the same +command. -1. `create_log_exp_headers`_ -2. `create_star_cat`_ +Each script's own ``--help`` and module docstring is the reference; this file is +only a map. Run any of them with ``-h`` first. -create_log_exp_headers -====================== +``python/`` — pipeline inputs +----------------------------- -This as to run after the module `split_exp_runner` it will create a master log -file containing all the WCS information for each CCDs of each single exposures. -To run the script : -`python create_log_exp_headers.py path/to/split_exp_runner/output path/to/srcipt/output_dir` +- ``cfis_field_select.py`` — select CFIS tiles or exposures by sky coordinates + or by name, and emit the ID list. This is how a tile list for + ``workflow/config.yaml`` is made. +- ``create_star_cat.py`` — build reference star catalogues for the mask module. + Superseded for exposures by the workflow's ``exp_star_cat`` rule; its + ``-k tile`` mode is the prerequisite for a future tile-mask rule. +- ``check_tile_coverage.py`` — flag input simulation tiles whose weight maps are + mostly empty, as a YAML exclude list. Called from sp_validation's image-sims + orchestration, not from here. -create_star_cat -=============== +``python/`` — pipeline outputs +------------------------------ -This script create all the star catalogs required to run the mask module for a -computational node without internet access. -To run the script : -`python create_star_cat.py path/to/image_dir path/to/script/output_dir` +- ``collate_star_cat.py`` — collate the per-exposure PSF validation catalogues + into the input ``merge_starcat`` wants for the rho/tau statistics. +- ``create_final_cat.py`` — merge per-tile final catalogues into one HDF5 file + for sp_validation. +- ``stats_global.py`` — histograms and tables from the SETools per-CCD summary + statistics of a whole run. + +``python/`` — image simulations +------------------------------- + +The image-simulation chain is still driven by the pre-Snakemake bash layer, +which sp_validation's ``image_sims.smk`` calls into. These exist for it and are +not used by real-data runs. + +- ``init_run_v2.0.py`` — lay out a run directory for that layer. +- ``update_runs_log_file.py`` — rebuild ``log_run_sp.txt`` from the run + directories on disk, which that layer needs because it deletes run dirs. +- ``test_tile_det.py`` + ``test_tile_det.cfg`` — drive tile detection over it. + +``python/`` — the ngmix status dashboard +---------------------------------------- + +Orthogonal to everything above: ``build_status.py``, ``build_history.py``, +``plot_trends.py``, ``run_breakdown_grid.py``, ``plot_breakdown_grid.py`` and +``plot_s4_ablations.py`` build the shape-measurement status page and its +trend/ablation figures. + +``sh/`` +------- + +- ``shapepipe_run_example.sh`` — run the bundled example pipeline against a + writable copy of ``/app/example``. This is the container's CI smoke test. +- ``apptainer_noslurm.sh`` — ``apptainer exec`` with the SLURM/PMI/PMIX/OMPI + environment stripped, for MPI inside a SLURM job. +- ``run_job_sp_canfar_v2.0.bash``, ``job_sp_canfar_v2.0.bash``, + ``job_list_help.bash``, ``functions.sh`` — the bit-coded bash job layer. Kept + only for image simulations (above); real-data runs go through ``workflow/``. + +``jupyter/`` +------------ + +Scratch notebooks: ``wcs.ipynb``, ``test_centroid_shift.py``. + +``validation/`` +--------------- + +``centroid/`` — the metacalibration multiplicative-bias validation of the ngmix +centroid handling: ``run_all.sh`` drives ``centroid_bias_v2.py`` over the three +cases in ``configs/``. diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb deleted file mode 100644 index 44f2f2545..000000000 --- a/scripts/jupyter/summary_run.ipynb +++ /dev/null @@ -1,399 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "130112a4-f2ca-4d26-b884-d8b054676f9c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 3\n", - "%reload_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "196beca5-10a1-4cf5-9462-be145167cc70", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'shapepipe'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mshapepipe\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mshapepipe\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutilities\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msummary\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'shapepipe'" - ] - } - ], - "source": [ - "import shapepipe\n", - "from shapepipe.utilities.summary import *" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ca63c72d-212c-463e-a792-71efbac0b908", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Setting\n", - "patch = \"P7\"\n", - "\n", - "verbose = False" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "dcb5604c-d61f-4705-8295-63875455cadb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Load parameters\n", - "%run ~/shapepipe/scripts/python/summary_params_pre_v2" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "e69b7dab-1fea-4fcc-a8d9-0720e1d628c3", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P7\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Set job info for patch P7\n" - ] - } - ], - "source": [ - "jobs, list_tile_IDs = set_jobs_v2_pre_v2(patch, verbose)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2c3a9dde-cf88-493f-926e-7ae7e8e10916", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Initialize runtime dicionary.\n", - "par_runtime = init_par_runtime(list_tile_IDs)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "1e9c1487-3cec-4394-9fcf-c12e92a0f984", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# No effect in notebook\n", - "#print_par_runtime(par_runtime, verbose=verbose)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "b7c63a22-ead1-4d6a-b081-a74ade515439", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "module expected found miss_expl missing uniq_miss fr_found\n", - "====================================================================================================\n" - ] - } - ], - "source": [ - "# Start program\n", - "job_data.print_stats_header()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "4720ae18-0633-4646-b392-b1b24e0294c3", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " (Job 1)\n", - "get_images_runner_run_1 462 462 0 0 0.0 100.0%\n", - "find_exposures_runner 231 231 0 0 0.0 100.0%\n", - "get_images_runner_run_2 537 0 0 537 179.0 0.0%\n" - ] - } - ], - "source": [ - "for key in \"1\":\n", - " jobs[key].print_intro()\n", - " jobs[key].check_numbers(par_runtime=par_runtime, indices=[0, 1])\n", - "\n", - " all_exposures = get_all_exposures(jobs[key]._paths_in_dir[1], verbose=verbose)\n", - " par_runtime[\"n_exposures\"] = len(all_exposures)\n", - " par_runtime[\"list_exposures\"] = all_exposures\n", - "\n", - " jobs[key].check_numbers(par_runtime, indices=[2])" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "f149f404-64e7-4d92-8f54-f300ed620130", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Update runtime dictionary with numbers of exposures\n", - "par_runtime = update_par_runtime_after_find_exp(par_runtime, all_exposures)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "78a9065f-8983-41cf-a34c-21892fc52dd2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Get all keys after \"1\"\n", - "keys = sorted(jobs.keys(), key=int)\n", - "_ = keys.pop(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "79e39954-1155-4ca3-b0b2-64bc5670db53", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " (Job 2)\n", - "uncompress_fits_runner 1268 1268 0 0 0.0 100.0%\n", - "merge_headers_runner 0 0 0 0 0.0 100.0%\n", - "split_exp_runner 137940 137940 0 0 0.0 100.0%\n", - " (Job 4)\n", - "mask_runner 1268 1268 0 0 0.0 100.0%\n", - " (Job 8)\n", - "mask_runner 45600 45600 0 0 0.0 100.0%\n", - " (Job 16)\n", - "sextractor_runner 2536 2536 0 0 0.0 100.0%\n", - " (Job 32)\n", - "sextractor_runner 91200 91200 0 0 0.0 100.0%\n", - "setools_runner 91200 91032 0 168 84.0 99.8%\n", - "psfex_runner 91200 91032 0 168 84.0 99.8%\n", - " (Job 64)\n", - "psfex_interp_runner 1268 1268 0 0 0.0 100.0%\n", - "vignetmaker_runner_run_1 1268 1268 0 0 0.0 100.0%\n", - "spread_model_runner 1268 1268 0 0 0.0 100.0%\n", - "vignetmaker_runner_run_2 5072 5072 0 0 0.0 100.0%\n", - " (Job 128)\n", - "ngmix_runner 1268 1225 0 43 43.0 96.6%\n", - "ngmix_runner 1268 1216 0 52 52.0 95.9%\n", - "ngmix_runner 1268 1216 0 52 52.0 95.9%\n", - "ngmix_runner 1268 1217 0 51 51.0 96.0%\n", - "ngmix_runner 1268 1228 0 40 40.0 96.8%\n", - "ngmix_runner 1268 1216 0 52 52.0 95.9%\n", - "ngmix_runner 1268 1216 0 52 52.0 95.9%\n", - "ngmix_runner 1268 1216 0 52 52.0 95.9%\n", - " (Job 256)\n", - "merge_sep_cats_runner 1268 0 0 1268 1268.0 0.0%\n", - "make_cat_runner 1268 0 0 1268 1268.0 0.0%\n", - " (Job 1024)\n", - "psfex_interp_runner 45600 41132 0 4468 4468.0 90.2%\n" - ] - } - ], - "source": [ - "for key in keys:\n", - " jobs[key].print_intro()\n", - " jobs[key].check_numbers(par_runtime=par_runtime)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "id": "b3d51a05-ecca-420b-b8b3-1fb2b1ec9fe3", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " (Job 128)\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1265 0 3 3.0 99.8%\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1266 0 2 2.0 99.8%\n", - "ngmix_runner 1268 1268 0 0 0.0 100.0%\n", - "ngmix_runner 1268 1266 0 2 2.0 99.8%\n" - ] - } - ], - "source": [ - "## Update some runs\n", - "for key in [\"128\"]:\n", - " jobs[key].print_intro()\n", - " jobs[key].check_numbers(par_runtime=par_runtime)" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "id": "67b50a61-e3cc-4559-941d-f39c6a200294", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " (Job 128)\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1265 0 3 3.0 99.8%\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1267 0 1 1.0 99.9%\n", - "ngmix_runner 1268 1266 0 2 2.0 99.8%\n", - "ngmix_runner 1268 1268 0 0 0.0 100.0%\n", - "ngmix_runner 1268 1266 0 2 2.0 99.8%\n" - ] - } - ], - "source": [ - "for key in [\"128\"]:\n", - " jobs[key].print_intro()\n", - " \n", - " \n", - " jobs[key].check_numbers(par_runtime=par_runtime)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "affa8293-daf9-4d2b-9215-fe19f8e2c1e2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "session = Session()" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "269496d1-cd89-4d13-a5e4-41b897669e22", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "ids = [session[\"id\"] for session in session.fetch(kind=\"headless\")]" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "80af8dff-98c7-4db4-8bcc-06936e1875cf", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "This event loop is already running", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_69/559116804.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdestroy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/.local/lib/python3.7/site-packages/skaha/session.py\u001b[0m in \u001b[0;36mdestroy\u001b[0;34m(self, id)\u001b[0m\n\u001b[1;32m 264\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"url\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserver\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[0mloop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_event_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 267\u001b[0m \u001b[0mresponses\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 268\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midentity\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/asyncio/base_events.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m 561\u001b[0m \"\"\"\n\u001b[1;32m 562\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 563\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_runnung\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 564\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 565\u001b[0m \u001b[0mnew_task\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfuture\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/asyncio/base_events.py\u001b[0m in \u001b[0;36m_check_runnung\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 521\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_runnung\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 522\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 523\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'This event loop is already running'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 524\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_running_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 525\u001b[0m raise RuntimeError(\n", - "\u001b[0;31mRuntimeError\u001b[0m: This event loop is already running" - ] - } - ], - "source": [ - "session.destroy(ids[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "66a3ed14-8aaf-4028-b933-10ecb7376d68", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py deleted file mode 100755 index 62fbf0564..000000000 --- a/scripts/python/canfar_avail_results.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python - -"""Script canfar_avail_results.py - -Check whether results files are available on vos. - -:Author: Martin Kilbinger - -:Date: 07/2020 -""" - -import re -import os -import sys -import glob -import copy -import io -from contextlib import redirect_stdout - -from optparse import OptionParser - -from cs_util.canfar import dir_list - -from shapepipe.utilities import cfis - - -def params_default(): - """Set default parameter values. - - Parameters - ---------- - None - - Returns - ------- - p_def: class cfis.param - parameter values - """ - - p_def = cfis.param( - input_IDs=".", - input_path="vos:cfis/cosmostat/kilbinger/results", - psf="mccd", - extension="tgz", - ) - - return p_def - - -def parse_options(p_def): - """Parse command line options. - - Parameters - ---------- - p_def: class cfis.param - parameter values - - Returns - ------- - options: tuple - Command line options - args: string - Command line string - """ - - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - # I/O - parser.add_option( - "-i", - "--input_IDs", - dest="input_IDs", - type="string", - default=p_def.input_IDs, - help="input tile ID file(s) or directory path, default='{}'".format( - p_def.input_IDs - ), - ) - parser.add_option( - "", - "--input_path", - dest="input_path", - type="string", - default=p_def.input_path, - help="input path, local or vos url, default='{}'".format( - p_def.input_path - ), - ) - parser.add_option( - "-o", - "--output_not_avail", - dest="output_not_avail", - type="string", - help="output file for not-available IDs, default no output", - ) - - parser.add_option( - "-p", - "--psf", - dest="psf", - type="string", - default=p_def.psf, - help="PSF model, one in ['psfex'|'mccd'], default='{}'".format( - p_def.psf - ), - ) - - parser.add_option( - "-f", - "--final_only", - dest="final_only", - action="store_true", - help="only check final catalogues", - ) - parser.add_option( - "-m", - "--mask_only", - dest="mask_only", - action="store_true", - help="only check mask files (pipeline_flag)", - ) - parser.add_option( - "-x", - "--extension", - dest="extension", - type="string", - default=p_def.extension, - help=f"file extension, default='{p_def.extension}'", - ) - parser.add_option( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose output", - ) - - options, args = parser.parse_args() - - return options, args - - -def check_options(options): - """Check command line options. - - Parameters - ---------- - options: tuple - Command line options - - Returns - ------- - erg: bool - Result of option check. False if invalid option value. - """ - - if options.psf not in ["psfex", "mccd"]: - print("Invalid PSF model '{}'".format(options.psf)) - return False - - if options.final_only and options.mask_only: - print("One one of the options '-f' or '-m' can be given") - return False - - return True - - -def update_param(p_def, options): - """Return default parameter, updated and complemented according to options. - - Parameters - ---------- - p_def: class param - parameter values - optiosn: tuple - command line options - - Returns - ------- - param: class param - updated paramter values - """ - - param = copy.copy(p_def) - - # Update keys in param according to options values - for key in vars(param): - if key in vars(options): - setattr(param, key, getattr(options, key)) - - # Add remaining keys from options to param - for key in vars(options): - if not key in vars(param): - setattr(param, key, getattr(options, key)) - - # Do extra stuff if necessary - - return param - - -def read_input_files(input_path, verbose=False): - """Return list of ID files. - - Parameters - ---------- - input_path: string - list of files or directory name - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - ID_files: list of strings - file names with tile IDs - """ - - if os.path.isdir(input_path): - input_files = glob.glob("{}/*".format(input_path)) - else: - input_files = cfis.my_string_split(input_path, stop=True, sep=" ") - - ID_files = [] - for f in input_files: - if os.path.isdir(f): - if verbose: - print("Skipping directory '{}'".format(f)) - else: - ID_files.append(f) - - if verbose: - print("{} input files found".format(len(ID_files))) - - return ID_files - - -def check_results( - ID_files, - input_path, - result_base_names, - n_complete, - extension, - verbose=False, -): - """Count the number of result files uploaded to vos for each input ID file. - - Parameters - ---------- - ID_files: list of strings - file name with tile IDs - input_path: string - path input directory - result_base_names: list of strings - result file base names - n_complete: int - number of files for complete result set - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - n_found: dictionary - number of files found for each input file and each ID - n_IDs: dictionary - number of ID for each input file - IDs_not_avail: list - IDs that are not available on vos - """ - - m = re.match("vos:", input_path) - if m: - ls_tmp = dir_list(input_path) - else: - ls_tmp = glob.glob(f"{input_path}/*") - ls_out = [os.path.basename(path) for path in ls_tmp] - - n_found = {} - n_IDs = {} - IDs_not_avail = [] - - # Loop over all input files - for ID_list in ID_files: - with open(ID_list) as f: - if verbose: - print("Checking ID list file {}...".format(ID_list)) - n_found[ID_list] = {} - n_IDs[ID_list] = 0 - - # Loop over all lines = IDs in file - for line in f: - ID = line.rstrip() - n_found[ID_list][ID] = 0 - - if extension == "fits": - ID_fname = ID.replace(".", "-") - ID_fname = f"-{ID_fname}" - else: - ID_fname = f"_{ID}" - # Count how many result files are available - for base in result_base_names: - name = "{}{}.{}".format(base, ID_fname, extension) - if name in ls_out: - n_found[ID_list][ID] = n_found[ID_list][ID] + 1 - n_IDs[ID_list] = n_IDs[ID_list] + 1 - - # If not complete set found, add to not-avail list - if n_found[ID_list][ID] != n_complete: - IDs_not_avail.append(ID) - - return n_found, n_IDs, IDs_not_avail - - -def output_summary(n_found, n_IDs, n_complete): - """Create output with summary of result availability. - - Parameters - ---------- - n_found: dictionary - number of files found for each input file and each ID - n_IDs: dictionary - number of ID for each input file - n_complete: int - number of files of a complete set of results - """ - - for ID_list in n_found.keys(): - nf = sum(value == n_complete for value in n_found[ID_list].values()) - print( - "{}: {}/{} ({:.1f}%) complete".format( - os.path.basename(ID_list), - nf, - n_IDs[ID_list], - nf / n_IDs[ID_list] * 100, - ) - ) - - -def output_IDs(ID_list, output): - """Write IDs to file - - Parameters - ---------- - ID_list: list of string - IDs - output: string - output file name - """ - - f = open(output, "w") - for ID in ID_list: - print(ID, file=f) - f.close() - - -def main(argv=None): - - # Set default parameters - p_def = params_default() - - # Command line options - options, args = parse_options(p_def) - - if check_options(options) is False: - return 1 - - param = update_param(p_def, options) - - # Save calling command - cfis.log_command(argv) - if param.verbose: - cfis.log_command(argv, name="sys.stdout") - - ### Start main program ### - - if param.verbose: - print("Start of program {}".format(os.path.basename(argv[0]))) - - ID_files = read_input_files(param.input_IDs, verbose=param.verbose) - - if param.final_only: - result_base_names = ["final_cat"] - elif param.mask_only: - result_base_names = ["pipeline_flag"] - else: - result_base_names = [] - types = [ - "final_cat", - "pipeline_flag", - "logs", - "setools_mask", - "setools_stat", - "setools_plot", - ] - for t in types: - result_base_names.append(t) - - if param.psf == "psfex": - result_base_names.append("psfex_interp_exp") - elif param.psf == "mccd": - result_base_names.append("mccd_fit_val_runner") - - n_complete = len(result_base_names) - - n_found, n_IDs, IDs_not_avail = check_results( - ID_files, - param.input_path, - result_base_names, - n_complete, - param.extension, - verbose=param.verbose, - ) - - output_summary(n_found, n_IDs, n_complete) - - if param.output_not_avail: - output_IDs(IDs_not_avail, param.output_not_avail) - - ### End main program - - if param.verbose: - print("End of program {}".format(os.path.basename(argv[0]))) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/clear_ngmix_prev.py b/scripts/python/clear_ngmix_prev.py deleted file mode 100644 index a1a7e802c..000000000 --- a/scripts/python/clear_ngmix_prev.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python3 -""" -Clean up previous ngmix run directories for completed tiles. -Removes *_prev directories when the current run has finished. -""" - -import argparse -import os -import re -import shutil -import glob -from pathlib import Path -from astropy.io import fits - - -def get_fits_info(fits_file): - """Get number of objects (NAXIS2 in HDU 1) and number of HDUs.""" - try: - with fits.open(fits_file) as hdul: - n_hdus = len(hdul) - # Get NAXIS2 from HDU 1 (second HDU, index 1) - if len(hdul) > 1: - n_objects = hdul[1].header.get("NAXIS2", "N/A") - else: - n_objects = "N/A (no HDU 1)" - return n_objects, n_hdus - except Exception as e: - return f"Error: {e}", "N/A" - - -def check_finished(log_file): - """Check if 'finished' appears in the log file.""" - try: - with open(log_file, "r") as f: - content = f.read() - return "finished" in content - except Exception as e: - print(f"Warning: Could not read {log_file}: {e}") - return False - - -def main(): - parser = argparse.ArgumentParser( - description="Clean up previous ngmix run directories for completed tiles." - ) - parser.add_argument( - "-n", "--dry-run", - action="store_true", - help="Show what would be removed without actually removing files" - ) - args = parser.parse_args() - - if args.dry_run: - print("dry run mode: No files will be removed\n") - - base_dir = Path("tile_runs") - - # Pattern to match tile IDs like 123.456 - tile_id_pattern = re.compile(r"^\d+\.\d+$") - - # Find all tile ID directories - if not base_dir.exists(): - print(f"Error: {base_dir} does not exist") - return - - tile_dirs = [ - d - for d in base_dir.iterdir() - if d.is_dir() and tile_id_pattern.match(d.name) - ] - - print(f"Found {len(tile_dirs)} tile directories") - - removed_count = 0 - finished_count = 0 - - for tile_dir in sorted(tile_dirs): - tile_id = tile_dir.name - - # Construct path to ngmix_runner directory - ngmix_dir = ( - tile_dir / "output" / "run_sp_tile_ngmix_Ng1u" / "ngmix_runner" - ) - - if not ngmix_dir.exists(): - continue - - # Check for process-*.log files - log_pattern = str(ngmix_dir / "logs" / "process-*.log") - log_files = glob.glob(log_pattern) - - if not log_files: - continue - - # Check if any log file contains "finished" - is_finished = any(check_finished(log_file) for log_file in log_files) - - if is_finished: - finished_count += 1 - - # Path to the _prev directory to remove - prev_dir = tile_dir / "output" / "run_sp_tile_ngmix_Ng1u_prev" - - if prev_dir.exists(): - - # File info of new - ngmix_out_dir = ngmix_dir / "output" - fits_files = list(ngmix_out_dir.glob("ngmix-*.fits")) - if fits_files: - for fits_file in fits_files: - n_objects, n_hdus = get_fits_info(fits_file) - else: - n_objects, n_hdus = (-1, -1) - - # File info of prev - prev_out_dir = prev_dir / "output" - fits_files = list(prev_out_dir.glob("ngmix-*.fits")) - if fits_files: - for fits_file in fits_files: - n_objects_prev, n_hdus_prev = get_fits_info(fits_file) - else: - n_objects_prev, n_hdus_prev = (-1, -1) - - action = "Would remove" if args.dry_run else "Removing" - print( - f"Tile {tile_id}: {action} {prev_dir} (new: {n_objects}" - + f" {n_hdus}) (prev: {n_objects_prev} {n_hdus_prev})" - ) - - if n_objects < n_objects_prev: - print(f"Warning: {tile_id}") - - if not args.dry_run: - try: - shutil.rmtree(prev_dir) - removed_count += 1 - except Exception as e: - print(f"Error removing {prev_dir}: {e}") - else: - removed_count += 1 - else: - pass - else: - print(f"Tile {tile_id}: Not finished") - - print(f"\nSummary:") - print(f" Tiles with finished runs: {finished_count}") - if args.dry_run: - print(f" Previous directories that would be removed: {removed_count}") - else: - print(f" Previous directories removed: {removed_count}") - - -if __name__ == "__main__": - main() diff --git a/scripts/python/collate_star_cat.py b/scripts/python/collate_star_cat.py index f7ab25b41..a3ccc2a2e 100755 --- a/scripts/python/collate_star_cat.py +++ b/scripts/python/collate_star_cat.py @@ -2,29 +2,25 @@ """COLLATE STAR CATALOGUES. -Collate the per-exposure PSF validation catalogues into star catalogues: gather -positions (X/Y/RA/DEC), assign the MCCD focal-plane CCD id, and merge into -``validation_psf_conv`` FITS files. - -Catalogue layout depends on the major version (``-V``). Runs up to ``v1.6`` are -organised in sky patches (``P1``..``P``): input runs live under -``/P/output`` and outputs are named -``validation_psf_conv--.fits``. ``v2.0`` removes the patch concept: -input runs live under a single ``/output`` root and outputs drop the patch -token (``validation_psf_conv-.fits``). - -HSM ellipticities and sizes are no longer rotated here. PSFEx and the in-repo -MCCD interpolation now measure adaptive moments directly in world coordinates -(galsim ``FindAdaptiveMom(use_sky_coords=True)``), so the WCS-Jacobian shape -rotation this script used to perform is redundant and has been removed. - -Caveat: the MCCD ``PSF_MOM_LIST``/``STAR_MOM_LIST`` columns are produced by the -external ``mccd`` fit-validation code (``mccd.auxiliary_fun.mccd_validation``), -which still measures HSM moments in the pixel frame. Those shapes are therefore -still rotated into world coordinates here, via the WCS-Jacobian rotation, until -``mccd`` itself adopts ``use_sky_coords``. This is the one branch that keeps the -rotation; the in-repo PSFEx and MCCD-interpolation paths measure adaptive -moments directly in world coordinates upstream and pass them straight through. +Collate the per-exposure PSFEx validation catalogues into star catalogues: +gather positions (X/Y/RA/DEC) and the HSM PSF and star shapes, tag each row +with its CCD number, and merge one ``validation_psf_conv-.fits`` per +single-exposure run. + +HSM ellipticities and sizes are not rotated here. ``psfex_interp`` measures +adaptive moments directly in world coordinates (galsim +``FindAdaptiveMom(use_sky_coords=True)``), so the WCS-Jacobian shape rotation +this script used to perform is redundant and has been removed; only positions +are collated. + +Caveat -- no workflow rule produces these catalogues yet. The input layout +below is the pre-workflow one: a link farm of ``psfex_interp`` run directories +under a single input root. The Snakemake workflow instead persists +``validation_psf--.fits`` into the per-exposure ``exp_persist`` +tars on the products root, and ``merge_starcat_runner`` still wants the +collated ``validation_psf_conv-*`` files for the rho/tau statistics. +Repointing this script at the persist tars (or replacing it with a rule) is a +follow-up, and a bigger change than this strip. """ import sys @@ -37,51 +33,27 @@ import numpy as np from astropy.io import fits -import galsim from cs_util import args as cs_args from cs_util import logging -def collate_paths(input_base_dir, output_base_dir, patch): - """Collate Paths. +# Input layout: single-exposure run directories matching ``EXP_RUN_GLOB`` under +# ``/output``, each holding the PSF interpolation output in +# ``PSF_INTERP_SUBDIR``. +EXP_RUN_GLOB = "run_sp_combined_psf*" +PSF_INTERP_SUBDIR = "psfex_interp_runner/output" - Return the ``(input run dir, output dir)`` for a patch. ``patch`` is None - for the patch-less v2.0 layout, which drops the ``P`` token; v1.x - passes the patch number. - Parameters - ---------- - input_base_dir : str - input base directory - output_base_dir : str - output base directory - patch : str or None - patch number, or None for the patch-less v2.0 layout - - Returns - ------- - tuple - input run directory and output directory - - """ - if patch is None: - return f"{input_base_dir}/output/", output_base_dir - return f"{input_base_dir}/P{patch}/output/", f"{output_base_dir}/P{patch}" - - -def output_filename(file_pattern, patch, idx): +def output_filename(file_pattern, idx): """Output Filename. - Build the collated catalogue filename. ``patch`` is None for the patch-less - v2.0 layout, which drops the patch token. + Build the collated catalogue filename. Parameters ---------- file_pattern : str input file pattern (e.g. ``validation_psf``) - patch : str or None - patch number, or None for the patch-less v2.0 layout idx : int exposure run index @@ -91,340 +63,7 @@ def output_filename(file_pattern, patch, idx): output catalogue file name """ - patch_token = "" if patch is None else f"{patch}-" - return f"{file_pattern}_conv-{patch_token}{idx}.fits" - - -def transform_shape(mom_list, jac): - """Transform Shape. - - Transform shape (ellipticity and size) using a Jacobian. - - Parameters - ---------- - mom_list : list - input moment measurements; each list element contains - first and second ellipticity component and size - jac : galsim.JacobianWCS - Jacobian transformation matrix information - - Returns - ------- - list - transformed shape parameters, which are - first and second ellipticity component and size - - """ - scale, shear, theta, flip = jac.getDecomposition() - - sig_tmp = mom_list[2] * scale - shape = galsim.Shear(g1=mom_list[0], g2=mom_list[1]) - if flip: - # The following output is not observed - print("FLIP!") - shape = galsim.Shear(g1=-shape.g1, g2=shape.g2) - shape = galsim.Shear(g=shape.g, beta=shape.beta + theta) - shape = shear + shape - - return shape.g1, shape.g2, sig_tmp - - -class Loc2Glob(object): - r"""Change from local to global coordinates. - - Class to pass from local coordinates to global coordinates under - CFIS (CFHT) MegaCam instrument. The geometrical informcation of the - instrument is encoded in this function. - - Parameters - ---------- - x_gap : int - Gap between the CCDs along the horizontal direction; - default is ``70`` (MegaCam value) - y_gap : int - Gap between the CCDs along the vertical direction; - Default is ``425`` (MegaCam value) - x_npix : int - Number of pixels per CCD along the horizontal direction; - default is ``2048`` (MegaCam value) - y_npix : int - Number of pixels per CCD along the vertical direction; - default to ``4612`` (MegaCam value) - ccd_tot : int - Total number of CCDs; - default to ``40`` (MegaCam value) - - Notes - ----- - This is the geometry of MegaCam. Watch out with the conventions ba,ab that means where - is the local coordinate system origin for each CCD. - For more info check out MegaCam's instrument webpage. - - Examples - -------- - 'COMMENT (North on top, East to the left)', - 'COMMENT --------------------------', - 'COMMENT ba ba ba ba ba ba ba ba ba', - 'COMMENT 00 01 02 03 04 05 06 07 08', - 'COMMENT --------------------------------', - 'COMMENT ba ba ba ba ba ba ba ba ba ba ba', - 'COMMENT 36 09 10 11 12 13 14 15 16 17 37', - 'COMMENT --------------*-----------------', - 'COMMENT 38 18 19 20 21 22 23 24 25 26 39', - 'COMMENT ab ab ab ab ab ab ab ab ab ab ab', - 'COMMENT --------------------------------', - 'COMMENT 27 28 29 30 31 32 33 34 35', - 'COMMENT ab ab ab ab ab ab ab ab ab', - 'COMMENT __________________________' - """ - - def __init__( - self, x_gap=70, y_gap=425, x_npix=2048, y_npix=4612, ccd_tot=40 - ): - r"""Initialize with instrument geometry.""" - self.x_gap = x_gap - self.y_gap = y_gap - self.x_npix = x_npix - self.y_npix = y_npix - self.ccd_tot = ccd_tot - - def loc2glob_img_coord(self, ccd_n, x_coor, y_coor): - """loc2glob Img Coord. - - Go from the local to the global img (pixel) coordinate system. - - Global system with (0,0) in the intersection of ccds [12,13,21,22]. - - Parameters - ---------- - ccd_n: int - CCD number of the considered positions - x_coor: float - Local coordinate system hotizontal value - y_coor: float - Local coordinate system vertical value - - Returns - ------- - glob_x_coor: float - Horizontal position in global coordinate system - glob_y_coor: float - Vertical position in global coordinate system - - """ - # Flip axes - x_coor, y_coor = self.flip_coord(ccd_n, x_coor, y_coor) - - # Calculate the shift - x_shift, y_shift = self.shift_coord(ccd_n) - - # Return new coordinates - return x_coor + x_shift, y_coor + y_shift - - def flip_coord(self, ccd_n, x_coor, y_coor): - r"""Change of coordinate convention. - - So that all of them are coherent on the global coordinate system. - So that the origin is on the south-west corner. - Positive: South to North ; West to East. - """ - if ccd_n < 18 or ccd_n in [36, 37]: - x_coor = self.x_npix - x_coor + 1 - y_coor = self.y_npix - y_coor + 1 - else: - pass - - return x_coor, y_coor - - def x_coord_range(self): - r"""Return range of the x coordinate.""" - max_x = self.x_npix * 6 + self.x_gap * 5 - min_x = self.x_npix * (-5) + self.x_gap * (-5) - return min_x, max_x - - def y_coord_range(self): - r"""Return range of the y coordinate.""" - max_y = self.y_npix * 2 + self.y_gap * 1 - min_y = self.y_npix * (-2) + self.y_gap * (-2) - return min_y, max_y - - def shift_coord(self, ccd_n): - r"""Provide the shifting. - - It is needed to go from the local coordinate - system origin to the global coordinate system origin. - """ - if ccd_n < 9: - # first row - x_shift = (ccd_n - 4) * (self.x_gap + self.x_npix) - y_shift = self.y_gap + self.y_npix - return x_shift, y_shift - - elif ccd_n < 18: - # second row, non-ears - x_shift = (ccd_n - 13) * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 27: - # third row non-ears - x_shift = (ccd_n - 22) * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 36: - # fourth row - x_shift = (ccd_n - 31) * (self.x_gap + self.x_npix) - y_shift = -2.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 37: - # ccd= 36 ears, second row - x_shift = (-5.0) * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 38: - # ccd= 37 ears, second row - x_shift = 5.0 * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 39: - # ccd= 38 ears, third row - x_shift = (-5.0) * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 40: - # ccd= 39 ears, third row - x_shift = 5.0 * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - -class Glob2CCD(object): - r"""Get the CCD ID number from the global coordinate position. - - The Loc2Glob() object as input is the one that defines the instrument's - geometry. - - Parameters - ---------- - loc2glob: Loc2Glob object - Object with the desired focal plane geometry. - with_gaps: bool - If add the gaps to the CCD area. - """ - - def __init__(self, loc2glob, with_gaps=True): - # Save loc2glob object - self.loc2glob = loc2glob - self.with_gaps = with_gaps - self.ccd_list = np.arange(self.loc2glob.ccd_tot) - # Init edges defininf the CCDs - self.edge_x_list, self.edge_y_list = self.build_all_edges() - - def build_all_edges(self): - """Build the edges for all the CCDs in the focal plane.""" - edge_xy_list = [] - for idx in (0, 1): - edge_list = np.array( - [self.build_edge(ccd_n)[idx] for ccd_n in self.ccd_list] - ) - edge_xy_list.append(edge_list) - - return edge_xy_list - - def build_edge(self, ccd_n): - """Build the edges of the `ccd_n` in global coordinates.""" - if self.with_gaps: - corners = np.array( - [ - [-self.loc2glob.x_gap / 2, -self.loc2glob.y_gap / 2], - [ - self.loc2glob.x_npix + self.loc2glob.x_gap / 2, - -self.loc2glob.y_gap / 2, - ], - [ - -self.loc2glob.x_gap / 2, - self.loc2glob.y_npix + self.loc2glob.y_gap / 2, - ], - [ - self.loc2glob.x_npix + self.loc2glob.x_gap / 2, - self.loc2glob.y_npix + self.loc2glob.y_gap / 2, - ], - ] - ) - else: - corners = np.array( - [ - [0, 0], - [self.loc2glob.x_npix, 0], - [0, self.loc2glob.y_npix], - [self.loc2glob.x_npix, self.loc2glob.y_npix], - ] - ) - - glob_corners = np.array( - [ - self.loc2glob.loc2glob_img_coord(ccd_n, pos[0], pos[1]) - for pos in corners - ] - ) - - edge_xy = [] - for idx in (0, 1): - edge = np.array( - [np.min(glob_corners[:, idx]), np.max(glob_corners[:, idx])] - ) - edge_xy.append(edge) - - return edge_xy - - def is_inside(self, x, y, edge_x, edge_y): - """Is the position inside the edges. - - Return True if the position is within the rectangle - defined by the edges. - - Parameters - ---------- - x: float - Horizontal position in global coordinate system. - y: float - Vertical position in global coordinate system. - edge_x: np.ndarray - Edge defined as `np.array([min_x, max_x])`. - edge_y: np.ndarray - Edge defined as `np.array([min_y, max_y])`. - """ - if ( - (x > edge_x[0]) - and (x < edge_x[1]) - and (y > edge_y[0]) - and (y < edge_y[1]) - ): - return True - else: - return False - - def get_ccd_n(self, x, y): - """Returns the CCD number from the position `(x, y)`. - - Returns `None` if the position is not found. - """ - bool_list = np.array( - [ - self.is_inside(x, y, edge_x, edge_y) - for edge_x, edge_y in zip(self.edge_x_list, self.edge_y_list) - ] - ) - - try: - return self.ccd_list[bool_list][0] - except Exception: - return None + return f"{file_pattern}_conv-{idx}.fits" class Convert(object): @@ -457,38 +96,25 @@ def params_default(self): self._params = { "input_base_dir": ".", "output_base_dir": ".", - "version_cat": "v2.0", "mode": "merge", - "patches": "", - "psf": "psfex", "file_pattern_psfint": "validation_psf", } self._short_options = { "input_base_dir": "-i", - "version_cat": "-V", "mode": "-m", - "psf": "-p", - "patches": "-P", } self._types = {} self._help_strings = { "input_base_dir": ( - "input base dir; for v1.x runs are expected in" - + " /P/output, for v2.0 (patch-less) in" + "input base dir; single-exposure runs are expected in" + " /output; default is {}" ), - "version_cat": ( - "catalogue major version, allowed are v1.3, v1.4, v1.5, v1.6," - + " v2.0; v2.0 is patch-less; default is {}" - ), "mode": ( "run mode, allowed are 'merge', 'test'; default is" + " '{}'" ), - "psf": "PSF model, allowed are 'psfex' and 'mccd'; default is {}", - "patches": "(list of) input patches; ignored for v2.0", } # Output column names with types @@ -508,127 +134,67 @@ def params_default(self): ("CCD_NB", int), ] - # Extra columns for MCCD:737 - self._dt_mccd = self._dt.copy() - self._dt_mccd.append(("GLOB_X", float)) - self._dt_mccd.append(("GLOB_Y", float)) - - def update_params(self): - """Update Params. - - Update parameters. - - """ - if self._params["psf"] == "psfex": - #self._params["sub_dir_pattern"] = "run_sp_exp_202" - self._params["sub_dir_pattern"] = "run_sp_combined_psf" - self._params["sub_dir_psfint"] = "psfex_interp_runner" - elif self._params["psf"] == "mccd": - self._params["sub_dir_pattern"] = "run_sp_exp_SxSePsf_202" - self._params["sub_dir_psfint"] = "mccd_fit_val_runner" - self._params["sub_dir_setools"] = "setools_runner/output/mask" - else: - raise ValueError(f"Invalid PSF model {self._params['psf']}") - self._params["sub_dir_psfint"] = ( - f"{self._params['sub_dir_psfint']}/output" - ) - def run(self): """Run. Main processing function. """ - # Guard against a mistyped version silently falling through to the - # v1.x patch loop (e.g. ``-V v2`` or ``-V 2.0``). - allowed_versions = ("v1.3", "v1.4", "v1.5", "v1.6", "v2.0") - if self._params["version_cat"] not in allowed_versions: - raise ValueError( - f"Invalid version {self._params['version_cat']}; allowed are" - + f" {', '.join(allowed_versions)}" - ) - - # v2.0 removes the patch concept: a single patch-less run root. For - # v1.x, iterate over the requested sky patches as before. ``patch`` is - # None in the patch-less case, which drops the patch token from the - # input path and the output filename. - if self._params["version_cat"] == "v2.0": - patch_nums = [None] - elif self._params["mode"] == "test": - patch_nums = ["3", "4"] - else: - patch_nums = cs_args.my_string_split(self._params["patches"]) - do_parallel = True - # Loop over patches - for patch in patch_nums: + output_dir = self._params["output_base_dir"] + if not os.path.isdir(output_dir): + os.makedirs(output_dir, exist_ok=True) - patch_dir, output_dir = collate_paths( - self._params["input_base_dir"], - self._params["output_base_dir"], - patch, - ) - print("Running patch-less (v2.0)" if patch is None else f"Running patch: {patch}") - - if not os.path.isdir(output_dir): - os.makedirs(output_dir, exist_ok=True) + subdirs = f"{self._params['input_base_dir']}/output/{EXP_RUN_GLOB}" + exp_run_dirs = glob.glob(subdirs) + n_exp_runs = len(exp_run_dirs) + print(f"Found {n_exp_runs} input single-exposure run(s) ({subdirs})") - subdirs = f"{patch_dir}/{self._params['sub_dir_pattern']}*" - exp_run_dirs = glob.glob(subdirs) + if self._params["mode"] == "test": + exp_run_dirs = exp_run_dirs[:2] n_exp_runs = len(exp_run_dirs) print( - f"Found {n_exp_runs} input single-exposure run(s) for patch" - + f" {patch_dir} ({subdirs})" + f"test mode: only using {n_exp_runs} input single-exposure" + + f" runs" ) - if self._params["mode"] == "test": - exp_run_dirs = exp_run_dirs[:2] - n_exp_runs = len(exp_run_dirs) - print( - f"test mode: only using {n_exp_runs} input single-exposure" - + f" runs" + # Loop over exposure runs + if not do_parallel: + for idx_exp, exp_run_dir in tqdm( + enumerate(exp_run_dirs), + total=n_exp_runs, + disable=self._params["verbose"], + ): + self.transform_exposures(output_dir, idx_exp, exp_run_dir) + else: + res = Parallel(n_jobs=-1, backend="loky")( + delayed(self.transform_exposures)( + output_dir, idx_exp, exp_run_dir ) - - # Loop over exposure runs - if not do_parallel: for idx_exp, exp_run_dir in tqdm( enumerate(exp_run_dirs), total=n_exp_runs, disable=self._params["verbose"], - ): - self.transform_exposures( - output_dir, patch, idx_exp, exp_run_dir - ) - else: - res = Parallel(n_jobs=-1, backend="loky")( - delayed(self.transform_exposures)( - output_dir, patch, idx_exp, exp_run_dir - ) - for idx_exp, exp_run_dir in tqdm( - enumerate(exp_run_dirs), - total=n_exp_runs, - disable=self._params["verbose"], - ) ) + ) - def transform_exposures(self, output_dir, patch, idx, exp_run_dir): - """Transform exposures. + def transform_exposures(self, output_dir, idx, exp_run_dir): + """Transform Exposures. - Transform shapes for exposure for a given run (input exp run dir). + Collate the PSF validation catalogues of one single-exposure run + (input exp run dir) into one output catalogue. """ output_path = ( f"{output_dir}/" - + output_filename( - self._params["file_pattern_psfint"], patch, idx - ) + + output_filename(self._params["file_pattern_psfint"], idx) ) if os.path.exists(output_path): print(f"Skipping transform_exposures, file {output_path} exists") return - psf_dir = f"{exp_run_dir}/{self._params['sub_dir_psfint']}" + psf_dir = f"{exp_run_dir}/{PSF_INTERP_SUBDIR}" try: all_files = os.listdir(psf_dir) if self._params["verbose"]: @@ -645,11 +211,7 @@ def transform_exposures(self, output_dir, patch, idx, exp_run_dir): tmp = re.findall(r"\d+", file_name) - if self._params["psf"] == "psfex": - exp_name, ccd_id = int(tmp[0]), int(tmp[1]) - elif self._params["psf"] == "mccd": - exp_name = int(tmp[0]) - ccd_id = -1 + exp_name, ccd_id = int(tmp[0]), int(tmp[1]) if self._params["verbose"]: print("Match found ", exp_name, ccd_id) @@ -657,194 +219,42 @@ def transform_exposures(self, output_dir, patch, idx, exp_run_dir): psf_file_path = f"{psf_dir}/{file_name}" try: - if self._params["psf"] == "psfex": - psf_file_hdus = fits.open(psf_file_path, memmap=False) - psf_file = psf_file_hdus[2].data - psf_file_hdus.close() - mod = "RA" - else: - psf_file = fits.getdata(psf_file_path, 1, memmap=True) - mod = "RA_LIST" + psf_file_hdus = fits.open(psf_file_path, memmap=False) + psf_file = psf_file_hdus[2].data + psf_file_hdus.close() except Exception: continue - if self._params["psf"] == "psfex": - # HSM ellipticities and sizes are measured directly in world - # coordinates upstream (FindAdaptiveMom use_sky_coords=True), so - # they are passed straight through; only positions are collated. - exp_cat = np.array( - list( - map( - tuple, - np.array( - [ - psf_file["X"], - psf_file["Y"], - psf_file["RA"], - psf_file["DEC"], - psf_file["E1_PSF_HSM"], - psf_file["E2_PSF_HSM"], - psf_file["SIGMA_PSF_HSM"], - psf_file["FLAG_PSF_HSM"], - psf_file["E1_STAR_HSM"], - psf_file["E2_STAR_HSM"], - psf_file["SIGMA_STAR_HSM"], - psf_file["FLAG_STAR_HSM"], - np.ones_like(psf_file["RA"], dtype=int) - * ccd_id, - ] - ).T.tolist(), - ) - ), - dtype=self._dt, - ) - cat_list.append(exp_cat) - - else: - l2g = Loc2Glob() - g2c = Glob2CCD(l2g) - new_ccd_id = np.array( - [ - int( - g2c.get_ccd_n( - psf_file["GLOB_POSITION_IMG_LIST"][ii, 0], - psf_file["GLOB_POSITION_IMG_LIST"][ii, 1], - ) - ) - for ii in range(len(psf_file)) - ] - ) - - # Local-to-CCD position: subtract each CCD's focal-plane shift. - new_x = np.zeros_like(psf_file[mod]) - new_y = np.zeros_like(psf_file[mod]) - - # The MCCD PSF_MOM_LIST/STAR_MOM_LIST columns come from the - # external mccd fit-validation code, which still measures HSM - # moments in the pixel frame; rotate them into world coordinates - # via the per-CCD WCS Jacobian. This rotation stays until mccd - # itself adopts use_sky_coords (see the module docstring). The - # in-repo PSFEx / MCCD-interpolation paths are already in world - # coordinates and are passed through unrotated. - new_e1_psf = np.zeros_like(psf_file[mod]) - new_e2_psf = np.zeros_like(psf_file[mod]) - new_sig_psf = np.zeros_like(psf_file[mod]) - new_e1_star = np.zeros_like(psf_file[mod]) - new_e2_star = np.zeros_like(psf_file[mod]) - new_sig_star = np.zeros_like(psf_file[mod]) - new_flag_psf = np.zeros_like(psf_file[mod]) - new_flag_star = np.zeros_like(psf_file[mod]) - for ccd_id in range(40): - m_ccd_id = new_ccd_id == ccd_id - if sum(m_ccd_id) == 0: - continue - - x_shift, y_shift = l2g.shift_coord(ccd_id) - - new_x[m_ccd_id] = ( - psf_file["GLOB_POSITION_IMG_LIST"][:, 0][m_ccd_id] - - x_shift + # HSM ellipticities and sizes are measured directly in world + # coordinates upstream (FindAdaptiveMom use_sky_coords=True), so + # they are passed straight through; only positions are collated. + exp_cat = np.array( + list( + map( + tuple, + np.array( + [ + psf_file["X"], + psf_file["Y"], + psf_file["RA"], + psf_file["DEC"], + psf_file["E1_PSF_HSM"], + psf_file["E2_PSF_HSM"], + psf_file["SIGMA_PSF_HSM"], + psf_file["FLAG_PSF_HSM"], + psf_file["E1_STAR_HSM"], + psf_file["E2_STAR_HSM"], + psf_file["SIGMA_STAR_HSM"], + psf_file["FLAG_STAR_HSM"], + np.ones_like(psf_file["RA"], dtype=int) + * ccd_id, + ] + ).T.tolist(), ) - new_y[m_ccd_id] = ( - psf_file["GLOB_POSITION_IMG_LIST"][:, 1][m_ccd_id] - - y_shift - ) - - header_file_path = ( - self._params["sub_dir_setools"] - + self._params["file_pattern_psfint"] - + f"{exp_name}-{ccd_id}.fits" - ) - try: - header_file = fits.getdata(header_file_path, 1) - except Exception: - continue - header = fits.Header.fromstring( - "\n".join(header_file[0][0]), sep="\n" - ) - wcs = galsim.AstropyWCS(header=header) - - g1_psf_tmp_l = [] - g2_psf_tmp_l = [] - sig_psf_tmp_l = [] - g1_star_tmp_l = [] - g2_star_tmp_l = [] - sig_star_tmp_l = [] - flag_psf_tmp_l = [] - flag_star_tmp_l = [] - - for obj in psf_file[m_ccd_id]: - try: - jac = wcs.jacobian( - world_pos=galsim.CelestialCoord( - ra=obj["RA_LIST"] * galsim.degrees, - dec=obj["DEC_LIST"] * galsim.degrees, - ) - ) - except Exception: - flag_star_tmp_l.append(16) - flag_psf_tmp_l.append(16) - g1_psf_tmp_l.append(0) - g2_psf_tmp_l.append(0) - sig_psf_tmp_l.append(0) - g1_star_tmp_l.append(0) - g2_star_tmp_l.append(0) - sig_star_tmp_l.append(0) - continue - g1_psf_tmp, g2_psf_tmp, sig_psf_tmp = transform_shape( - obj["PSF_MOM_LIST"], jac - ) - - g1_psf_tmp_l.append(g1_psf_tmp) - g2_psf_tmp_l.append(g2_psf_tmp) - sig_psf_tmp_l.append(sig_psf_tmp) - flag_psf_tmp_l.append(obj["PSF_MOM_LIST"][3]) - - g1_star_tmp, g2_star_tmp, sig_star_tmp = ( - transform_shape(obj["STAR_MOM_LIST"], jac) - ) - g1_star_tmp_l.append(g1_star_tmp) - g2_star_tmp_l.append(g2_star_tmp) - sig_star_tmp_l.append(sig_star_tmp) - flag_star_tmp_l.append(obj["STAR_MOM_LIST"][3]) - - new_e1_psf[m_ccd_id] = g1_psf_tmp_l - new_e2_psf[m_ccd_id] = g2_psf_tmp_l - new_sig_psf[m_ccd_id] = sig_psf_tmp_l - new_flag_psf[m_ccd_id] = flag_psf_tmp_l - new_e1_star[m_ccd_id] = g1_star_tmp_l - new_e2_star[m_ccd_id] = g2_star_tmp_l - new_sig_star[m_ccd_id] = sig_star_tmp_l - new_flag_star[m_ccd_id] = flag_star_tmp_l - - exp_cat = np.array( - list( - map( - tuple, - np.array( - [ - new_x, - new_y, - psf_file["RA_LIST"], - psf_file["DEC_LIST"], - new_e1_psf, - new_e2_psf, - new_sig_psf, - psf_file["PSF_MOM_LIST"][:, 3], - new_e1_star, - new_e2_star, - new_sig_star, - psf_file["STAR_MOM_LIST"][:, 3], - new_ccd_id, - psf_file["GLOB_POSITION_IMG_LIST"][:, 0], - psf_file["GLOB_POSITION_IMG_LIST"][:, 1], - ] - ).T.tolist(), - ) - ), - dtype=self._dt_mccd, - ) - cat_list.append(exp_cat) + ), + dtype=self._dt, + ) + cat_list.append(exp_cat) del psf_file @@ -852,10 +262,10 @@ def transform_exposures(self, output_dir, patch, idx, exp_run_dir): return # Finalize catalogue - patch_cat = np.concatenate(cat_list) + star_cat = np.concatenate(cat_list) hdul = fits.HDUList() hdul.append(fits.PrimaryHDU()) - hdul.append(fits.BinTableHDU(patch_cat)) + hdul.append(fits.BinTableHDU(star_cat)) # Write catalogue hdul.writeto( @@ -874,7 +284,6 @@ def run_convert(*args): obj = Convert() obj.set_params_from_command_line(args) - obj.update_params() obj.run() diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py index 0e3586f88..05e3d3bb9 100755 --- a/scripts/python/create_star_cat.py +++ b/scripts/python/create_star_cat.py @@ -19,14 +19,10 @@ from cs_util import args as cs_args from cs_util import logging as cs_logging -import numpy as np - -from astropy.coordinates import SkyCoord -from astropy.wcs import WCS from astropy.io import fits -from astropy import units as u -from astropy.table import Table +from shapepipe.utilities.file_io import write_atomic +from shapepipe.utilities.focal_plane import ccd_center_and_radius, focal_plane_disc from shapepipe.utilities.vizier import query_vizier as _query_vizier @@ -34,86 +30,6 @@ CDS_CAT_ID = "I/305/out" -def _get_wcs(header): - """Get WCS. - - Compute the astropy WCS from header manually. - (The purpose of this is to avoid possible incompatibility on distortion - convention) - - Parameters - ---------- - header : astropy.header - Image header - - Returns - ------- - astropy.wcs.WCS - WCS object - - """ - final_wcs = WCS(naxis=2) - final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except: - final_wcs.wcs.cunit = ["deg", "deg"] - final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - final_wcs.wcs.cd = [ - [header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]], - ] - - return final_wcs - - -def _sphere_dist_arcmin(ra1, dec1, ra2, dec2): - """Compute angular distance between two sky positions in arcmin.""" - c1 = SkyCoord(ra=ra1 * u.deg, dec=dec1 * u.deg) - c2 = SkyCoord(ra=ra2 * u.deg, dec=dec2 * u.deg) - return c1.separation(c2).arcmin - - -def _ccd_center_and_radius(header): - """Return (ra, dec, radius_arcmin) for a single CCD.""" - w = _get_wcs(header) - nx, ny = header["NAXIS1"], header["NAXIS2"] - cx, cy = nx / 2.0, ny / 2.0 - ra_c, dec_c = w.all_pix2world([[cx, cy]], 1)[0] - # radius = half-diagonal to CCD corner - ra_corner, dec_corner = w.all_pix2world([[0, 0]], 1)[0] - radius = _sphere_dist_arcmin(ra_c, dec_c, ra_corner, dec_corner) - return ra_c, dec_c, radius - - -def _focal_plane_center_and_radius(f, n_ccd=40): - """Return (ra, dec, radius_arcmin) covering all CCDs of an exposure.""" - ras, decs, radii = [], [], [] - for ind in range(1, n_ccd + 1): - h = fits.getheader(f, ind) - ra, dec, r = _ccd_center_and_radius(h) - ras.append(ra) - decs.append(dec) - radii.append(r) - - ras = np.array(ras) - decs = np.array(decs) - radii = np.array(radii) - - ra_center = np.mean(ras) - dec_center = np.mean(decs) - - # Radius = max distance from focal plane center to any CCD center + that CCD's half-diagonal - dists = np.array([ - _sphere_dist_arcmin(ra_center, dec_center, ras[i], decs[i]) - for i in range(len(ras)) - ]) - radius = np.max(dists + radii) - - return ra_center, dec_center, radius - - def query_vizier(ra, dec, radius_arcmin): return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) @@ -129,34 +45,24 @@ def main(input_dir, output_dir, kind): img_number = re.split("image", os.path.splitext(f)[0])[1] fpath = os.path.join(input_dir, f) + output_name = f"{output_dir}/star_cat{img_number}.fits" + if os.path.isfile(output_name): + continue + if kind == "exp": # One query covering the full MegaCam focal plane - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - ra, dec, radius = _focal_plane_center_and_radius(fpath) + ra, dec, radius_deg = focal_plane_disc(fpath) + radius = radius_deg * 60.0 print( f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" ) - table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) - else: - h = fits.getheader(fpath, 0) - w = _get_wcs(h) - nx, ny = h["NAXIS1"], h["NAXIS2"] - cx, cy = nx / 2.0, ny / 2.0 - ra, dec = w.all_pix2world([[cx, cy]], 1)[0] - ra_corner, dec_corner = w.all_pix2world([[0, 0]], 1)[0] - radius = _sphere_dist_arcmin(ra, dec, ra_corner, dec_corner) - - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) + # A single image: its own centre and half-diagonal. + ra, dec, radius_deg = ccd_center_and_radius(fits.getheader(fpath, 0)) + radius = radius_deg * 60.0 + + table = query_vizier(ra, dec, radius) + write_atomic(table, output_name) return 0 diff --git a/scripts/python/distribute_tiles.py b/scripts/python/distribute_tiles.py deleted file mode 100755 index 338286002..000000000 --- a/scripts/python/distribute_tiles.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python -""" -distribute_tiles.py - -Uses canfar.helpers.distributed.chunk to automatically distribute tiles -across replicas, then processes each tile assigned to this replica. -""" - -import os -import sys -import subprocess -from multiprocessing import Pool -import fcntl -from canfar.helpers.distributed import chunk - - -def parse_arguments(args): - """ - Extract the file IDs from command line arguments. - - Args: - args (list of str): Original command line arguments (e.g., sys.argv[1:]) - - Returns: - str or None: The value passed with -f, or None if not present - """ - # Default values - parsed = {"dry_run": 0, "parallel_jobs": 1} - - i = 0 - while i < len(args): - if args[i] == '-f' and i + 1 < len(args): - parsed['file_ids'] = args[i + 1] - i += 2 - elif args[i] == '--batch_num' and i + 1 < len(args): - parsed['batch_num'] = int(args[i + 1]) - i += 2 - elif args[i] == '--batch_tot' and i + 1 < len(args): - parsed['batch_tot'] = int(args[i + 1]) - i += 2 - elif args[i] == '--batch_size' and i + 1 < len(args): - parsed['batch_size'] = int(args[i + 1]) - i += 2 - elif args[i] in ('-n', '--dry_run') and i + 1 < len(args): - parsed['dry_run'] = int(args[i + 1]) - i += 2 - elif args[i] == '--parallel_jobs' and i + 1 < len(args): - parsed['parallel_jobs'] = int(args[i + 1]) - i += 2 - elif args[i] == "--debug_out" and i + 1 < len(args): - parsed["debug_out"] = args[i + 1] - i += 2 - else: - i += 1 - - return parsed - - -def get_my_tiles(all_tiles, batch_num, batch_tot, batch_size): - """ - Calculate which tiles this replica should process based on global position. - Args: - all_tiles: List of all tile IDs - batch_num: Current batch number (1-indexed) - batch_tot: Total number of batches - batch_size: Number of replicas per batch - Returns: - list: Tiles assigned to this replica - - """ - # Get local replica info from environment - local_replica_id = int(os.environ.get('REPLICA_ID', 1)) - - start_idx = (batch_num - 1) * batch_size - end_idx = min(batch_num * batch_size, len(all_tiles)) - - batch_tiles = all_tiles[start_idx:end_idx] - print(f"Batch {batch_num}/{batch_tot}, local replica {local_replica_id}/{batch_size}") - print(f"Batch processes tiles {start_idx + 1} to {end_idx} ({len(batch_tiles)} tiles)") - - # Use chunk() to distribute this batch's tiles among local replicas - # chunk() will use REPLICA_ID and REPLICA_COUNT automatically - my_tiles = list(chunk(batch_tiles)) - - return my_tiles - -def get_tile_list(file_ids): - """Read all tile IDs from file. - - Args: - file_ids: Path to file containing tile IDs - - Returns: - list: List of tile ID strings - """ - with open(file_ids, 'r') as f: - tiles = [line.strip() for line in f if line.strip()] - return tiles - - -def build_process_command(tile_id, original_args): - """ - Build command for a single tile by replacing -f with -e . - - Args: - tile_id (str): Tile ID to process - original_args (list of str): Original CLI arguments - - Returns: - list of str: Command ready for subprocess - """ - cmd = [f"{os.environ['HOME']}/shapepipe/scripts/sh/init_run_exclusive_canfar.sh"] - - # Transform arguments: replace -f with -e , - # skip batch arguments, parallel_jobs, and dry_run if not 0, 1 - i = 0 - while i < len(original_args): - if original_args[i] == '-f' and i + 1 < len(original_args): - cmd.extend(['-e', tile_id]) - i += 2 - elif original_args[i] in ('--batch_num', '--batch_tot', '--batch_size', '--parallel_jobs'): - i += 2 - elif original_args[i] in ('-n', '--dry_run'): - if i + 1 < len(original_args) and original_args[i + 1] in ("0", "1"): - cmd.extend(["-n", original_args[i + 1]]) - i += 2 - else: - cmd.append(original_args[i]) - i += 1 - - return cmd - - -def process_single_tile(args_tuple): - """ - Process a single tile (designed for multiprocessing). - - Args: - args_tuple: Tuple of (tile_id, tile_num, total_tiles, original_args, dry_run, msg_batch) - - Returns: - tuple: (tile_id, success, error_message) - """ - tile_id, tile_num, total_tiles, original_args, dry_run, msg_batch = args_tuple - - print(f"{'='*5} Processing {msg_batch}tile num/total={tile_num}/{total_tiles}: ID={tile_id} {'='*5}") - - # Build command - cmd = build_process_command(tile_id, original_args) - print(f"Command: {' '.join(cmd)}") - - # Execute command - try: - if dry_run != 2: - result = subprocess.run(cmd, check=True, capture_output=True, text=True) - print(f"✓ Successfully processed tile {tile_id}") - return (tile_id, True, None) - else: - print(f"dry_run=2") - return (tile_id, True, None) - except subprocess.CalledProcessError as e: - error_msg = f"Failed to process tile {tile_id}: {e}" - print(f"✗ {error_msg}", file=sys.stderr) - return (tile_id, False, error_msg) - except Exception as e: - error_msg = f"Unexpected error processing tile {tile_id}: {e}" - print(f"✗ {error_msg}", file=sys.stderr) - return (tile_id, False, error_msg) - - -def print_debug(pat, tile_list, out_path, verbose=False): - - local_replica_id = int(os.environ.get('REPLICA_ID', 1)) - - with open(out_path, "a") as f: - # Exclusive lock for save parallel use - fcntl.flock(f.fileno(), fcntl.LOCK_EX) - try: - # Build output string - output = f"{pat} distribute_tiles REPLICA_ID={local_replica_id}, tiles=" - output += " ".join(tile_list) + "\n" - - # Write to file - f.write(output) - - # Optionally write to stdout - if verbose: - print(output, end="") - finally: - # Unlock - fcntl.flock(f.fileno(), fcntl.LOCK_UN) - - -def main(): - - # Debug file line pattern - pat = "- " - - # Parse arguments - args = parse_arguments(sys.argv[1:]) - - if not "file_ids" in args: - print("Error: -f must be provided", file=sys.stderr) - sys.exit(1) - else: - file_ids = args["file_ids"] - - # Read all tiles - print(f"Reading tile list from {file_ids}") - all_tiles = get_tile_list(file_ids) - print(f"Total tiles in file: {len(all_tiles)}") - - print( - "REPLICA_ID, REPLICA_COUNT=", - os.environ.get('REPLICA_ID'), - os.environ.get('REPLICA_COUNT') - ) - - - # Check if we're in multi-batch mode - if "batch_num" in args and "batch_tot" in args and "batch_size" in args: - # Multi-batch mode: calculate global distribution - my_tile_list = get_my_tiles( - all_tiles, - args['batch_num'], - args['batch_tot'], - args['batch_size'] - ) - msg_batch = f"batch {args['batch_num']}/{args['batch_tot']} " - - else: - # Use chunk() to get tiles for this replica - print("Using chunk() to determine tiles for this replica...") - my_tile_list = list(chunk(all_tiles)) - msg_batch = "" - - if "debug_out" in args: - print_debug(f"{pat}{msg_batch}", my_tile_list, args["debug_out"], verbose=True) - - print(f"This replica assigned {len(my_tile_list)} tiles") - print(f"Parallel jobs: {args['parallel_jobs']}") - - if len(my_tile_list) > 0: - print(f"First tile: {my_tile_list[0]}") - if len(my_tile_list) > 1: - print(f"Last tile: {my_tile_list[-1]}") - else: - print("Only one tile") - - # Process each tile assigned to this replica - success_count = 0 - failure_count = 0 - - # Prepare arguments for parallel processing - process_args = [ - (tile_id, i, len(my_tile_list), sys.argv[1:], args["dry_run"], msg_batch) - for i, tile_id in enumerate(my_tile_list, 1) - ] - - if args['parallel_jobs'] > 1: - # Parallel processing - print(f"Processing tiles in parallel with {args['parallel_jobs']} workers") - with Pool(processes=args['parallel_jobs']) as pool: - results = pool.map(process_single_tile, process_args) - else: - # Sequential processing (original behavior) - print("Processing tiles sequentially") - results = [process_single_tile(arg) for arg in process_args] - - # Count successes and failures - for tile_id, success, error_msg in results: - if success: - success_count += 1 - else: - failure_count += 1 - - # Summary - print( - f"{'='*5} Processing completed tot/suc/fail=" - + f"{len(my_tile_list)}/{success_count}/{failure_count}" - ) - - # Exit with error if any failures - if failure_count > 0: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/scripts/python/get_ccds_with_psf.py b/scripts/python/get_ccds_with_psf.py deleted file mode 100755 index e284e2640..000000000 --- a/scripts/python/get_ccds_with_psf.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 - -"""GET_CCDS_WITH_PSF - -Obtain list of CCDs (single-exposure single-HDU files) for which valid PSF information -is available. This can serve to create a footprint coverage mask. - -Author: Martin Kilbinger - -""" - -import sys - -from shapepipe.utilities.ccd_psf_handler import CcdPsfHandler - - -def run_ccd_psf_handler(args=None): - """Run CCD PSF Handler. - - Create instance and run the CCD PSF handler. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code - - """ - # Create instance - obj = CcdPsfHandler() - - return obj.run(args=args) - - -def main(argv=None): - """Main. - - Main program. - - Parameters - ---------- - argv : list, optional - command line arguments - - Returns - ------- - int - exit code - - """ - return run_ccd_psf_handler(args=argv) - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/get_number_objects.py b/scripts/python/get_number_objects.py deleted file mode 100755 index 98422457c..000000000 --- a/scripts/python/get_number_objects.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script get_number_objects.py - -Get number of objects in a (last-run SExtractor) catalogue. - -:Author: Martin Kilbinger - -""" - -import sys -import copy -import glob - -from optparse import OptionParser -from astropy.io import fits - -from shapepipe.pipeline.run_log import get_last_dir, get_all_dirs -from shapepipe.utilities import cfis - - -class param: - """General class to store (default) variables""" - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def print(self, **kwds): - print(self.__dict__) - - def var_list(self, **kwds): - return vars(self) - - -def params_default(): - """Set default parameter values. - - Returns - ------- - class param - parameter values - - """ - p_def = param( - input_path=".", - input_name_base="final_cat", - hdu_num=1, - ) - - return p_def - - -def parse_options(p_def): - """Parse command line options. - - Parameters - ---------- - p_def: class param - parameter values - - Returns - ------- - list - command line options - command line str - - """ - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - # IO - parser.add_option( - "-i", - "--input_path", - dest="input_path", - type="string", - default=p_def.input_path, - help=f"input path, default='{p_def.input_path}'", - ) - parser.add_option( - "-n", - "--input_name_base", - dest="input_name_base", - type="string", - default=p_def.input_name_base, - help=f"input name base, default='{p_def.input_name_base}'", - ) - parser.add_option( - "-l", - "--list_tile_ID_path", - dest="tile_ID_list_path", - type="string", - default=None, - help=f"tile ID list, default: Use all data in input files", - ) - - # Control - parser.add_option( - "-p", - "--param_path", - dest="param_path", - type="string", - default=None, - help="parameter file path, default=None", - ) - - parser.add_option( - "", - "--hdu_num", - dest="hdu_num", - type="int", - default=p_def.hdu_num, - help=f"input HDU number, default='{p_def.hdu_num}'", - ) - - parser.add_option( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose output", - ) - - options, args = parser.parse_args() - - return options, args - - -def check_options(options): - """Check command line options. - - Parameters - ---------- - options: tuple - Command line options - - Returns - ------- - bool - Result of option check. False if invalid option value. - - """ - return True - - -def update_param(p_def, options): - """Return default parameter, updated and complemented according to options. - - Parameters - ---------- - p_def: class param - parameter values - optiosn: tuple - command line options - - Returns - ------- - class param - updated paramter values - - """ - param = copy.copy(p_def) - - # Update keys in param according to options values - for key in vars(param): - if key in vars(options): - setattr(param, key, getattr(options, key)) - - # Add remaining keys from options to param - for key in vars(options): - if not key in vars(param): - setattr(param, key, getattr(options, key)) - - # Do extra stuff if necessary - - return param - - -def main(argv=None): - - # Set default parameters - p_def = params_default() - - # Command line options - options, args = parse_options(p_def) - - if check_options(options) is False: - return 1 - - param = update_param(p_def, options) - - # Save command line arguments to log file - f_log = cfis.log_command(argv, close_no_return=False) - - pattern = "sexcat" - run_log_file = "output/log_run_sp.txt" - - # For v1 - # module = 'sextractor_runner_run_1' - - # For v2 - module = "sextractor_runner" - - # Get all sextractor output directories - all_dir = get_all_dirs(run_log_file, module) - paths = [] - - # Find tile runs - for path in all_dir: - if "run_sp_tile_Sx" in path: - paths.append(path) - paths = sorted(paths) - - if len(paths) == 0: - # No previous tile SExtractor run found - return -1 - - # Get latest run - last_dir = paths[-1] - - # Get all output SExtractor catalogues - file_list = glob.glob(f"{last_dir}/{pattern}*.fits") - if len(file_list) == 0: - raise ValueError(f"No files {last_dir}/{pattern}*.fits found") - - # Add up number of objects over all catalogues - n_obj = 0 - hdu_no = -1 - for fpath in file_list: - hdu_list = fits.open(fpath) - header = hdu_list[-1].header - n_obj += int(header["NAXIS2"]) - - # Compute average - n_obj = int(n_obj / len(file_list)) - - print(n_obj) - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py deleted file mode 100755 index 5b2a5f0e2..000000000 --- a/scripts/python/link_to_exp_for_tile.py +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script link_to_exp_for_tile.py - -:Description: Link to exposure and PSF catalogue - for a given tile. - -:Author: Martin Kilbinger - -""" - -import os -import sys -import re -import copy - -from optparse import OptionParser - - -class param: - """General class to store (default) variables""" - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def print(self, **kwds): - print(self.__dict__) - - def var_list(self, **kwds): - return vars(self) - - -def params_default(): - """Params Default. - - Set default parameter values. - - Returns - ------- - class param - parameter values - - """ - p_def = param( - tile_base_dir=".", - exp_base_dir=".", - sp_local=0, - ) - - return p_def - - -def parse_options(p_def): - """Parse Options. - - Parse command line options. - - Parameters - ---------- - p_def: class param - parameter values - - Returns - ------- - list - command line options - command line str - - """ - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - # IO - parser.add_option( - "-i", - "--input_tile_dir", - dest="tile_base_dir", - type="string", - default=p_def.tile_base_dir, - help=f"input tile base directory, default='{p_def.tile_base_dir}'", - ) - parser.add_option( - "-t", - "--tile_ID", - dest="tile_ID", - type="string", - help=f"input tile ID", - ) - parser.add_option( - "-I", - "--input_exp_dir", - dest="exp_base_dir", - type="string", - default=p_def.exp_base_dir, - help=f"input exposure base directory, default='{p_def.exp_base_dir}'", - ) - parser.add_option( - "-s", - "--sp_local", - dest="sp_local", - type="int", - default=p_def.sp_local, - help=f"local runi of split_exposure_runner, default='{p_def.sp_local}'", - ) - parser.add_option( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose output", - ) - - options, args = parser.parse_args() - - return options, args - - -def check_options(options): - """Check Options. - - Check command line options. - - Parameters - ---------- - options: tuple - Command line options - - Returns - ------- - bool - Result of option check. False if invalid option value. - - """ - return True - - -def update_param(p_def, options): - """Update Param. - - Return default parameter, updated and complemented according to options. - - Parameters - ---------- - p_def: class param - parameter values - optiosn: tuple - command line options - - Returns - ------- - class param - updated paramter values - - """ - param = copy.copy(p_def) - - # Update keys in param according to options values - for key in vars(param): - if key in vars(options): - setattr(param, key, getattr(options, key)) - - # Add remaining keys from options to param - for key in vars(options): - if not key in vars(param): - setattr(param, key, getattr(options, key)) - - # Do extra stuff if necessary - - return param - - -# TODO: move to cs_util -def matching_subdirs(base_dir, pattern): - """Matching Subdirs. - - Return all subdirectories whose name start with a given string pattern. - - Parameters - ---------- - base_dir: str - base directory - pattern: str - pattern to match beginning of subdir name - - Returns - ------- - list - matched subdirectory names - - """ - subdirs = [] - if os.path.exists(base_dir): - # Loop over directory entries - for entry in os.listdir(base_dir): - full_path = os.path.join(base_dir, entry) - # Append if match - if os.path.isdir(full_path) and entry.startswith(pattern): - subdirs.append(full_path) - else: - print(f"Warning: {base_dir} does not exist, continuing...") - - # Sort according to creation date - subdirs.sort(key=os.path.getctime) - - return subdirs - - -def get_tile_out_dir(tile_base_dir, tile_ID): - """Get Tile Out Dir. - - Return output directory path for a given tile ID. - - Parameters - ---------- - tile_base_dir: str - base directory of tile runs - tile_ID: str - tile ID - - Returns - ------- - str - path - - """ - return f"{tile_base_dir}/{tile_ID}/output" - - -def get_exp_IDs(tile_base_dir, tile_ID, verbose=False): - """Get Exp IDs. - - Return exposure IDs used for given tile. - - Parameters - ---------- - tile_base_dir: str - base directory of tile runs - tile_ID: str - tile ID - verbose: bool, optional - verbose output if ``True``; default is ``False`` - - Returns - ------- - list - exposure IDs - - """ - # Get tile output path - tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID) - - # Get subdirectories with runs "Fe" (find exposures) - pattern = "run_sp_GitFeGie" - subdirs = matching_subdirs(tile_out_dir, pattern) - - # Raise error if not exactly one run - if len(subdirs) == 0: - raise IOError( - f"No matching directory '{pattern}' in {tile_out_dir} found" - ) - if len(subdirs) != 1: - raise IOError( - f"Exactly one directory natching {pattern} in {tile_out_dir} " - + f"expected, not {len(subdirs)}" - ) - - # Replace dot with dash in tile ID - tile_ID_sp = re.sub(r"\.", "-", tile_ID) - # Get output file of find_exposure_runner for this tile ID - exp_ID_file = ( - f"{subdirs[0]}/find_exposures_runner/output/" - + f"exp_numbers-{tile_ID_sp}.txt" - ) - - exp_IDs = [] - # Read file to get exposure IDs - with open(exp_ID_file) as f_in: - for line in f_in: - name = line.strip() - # Remove any letter - ID = re.sub("[a-zA-Z]", "", name) - exp_IDs.append(ID) - - if verbose: - print("Exposures: ", exp_IDs) - return exp_IDs - - -def get_exp_single_HDU_IDs(exp_IDs, n_CPU): - """Get Exp Single HDU IDs. - - Return all single-HDU IDs for given exposures. - - Parameters - ---------- - exp_IDs: list - input exposure IDs, list of str - n_CPU: int - number of CPUs (=HDUs) of an exposure; n_CPU=40 for MegaCAM - - Returns - ------- - list - list of single-HDU exposure IDs - - """ - exp_shdu_IDs = [] - # Loop over input exposure IDs - for exp_ID in exp_IDs: - # Append int up to n_CPU to each - for idx in range(n_CPU): - ID = f"{exp_ID}-{idx}" - exp_shdu_IDs.append(ID) - - return exp_shdu_IDs - - -def get_paths(exp_base_dir, exp_shdu_IDs, pattern): - """Get Paths. - - Return (newest) subdirectory for each input single-exposure HDU ID that matches pattern. - - Parameters - ---------- - exp_base_dir: str - base directory for (single-HDU) exposure runs - exp_shdu_IDs: list - single-HDU exposure IDs; list of str - pattern: str - pattern to match beginning of subdir names - Returns - ------- - list - matching paths; list of str, one entry for each input single-exp ID - - """ - paths = [] - # Loop over single-HDU exposure IDs - for exp_shdu_ID in exp_shdu_IDs: - - # output path of runs - name = f"{exp_base_dir}/{exp_shdu_ID}/output" - path = os.path.abspath(name) - - # get matching subdirs - subdirs = matching_subdirs(path, pattern) - n_subdirs = len(subdirs) - - if n_subdirs != 1: - msg = ( - f"Exactly one directory matching {pattern} in {path} expected," - + f" not {n_subdirs}" - ) - - # If more than one found: sort by name = sort by date - subdirs = sorted(subdirs) - - # No match - if n_subdirs == 0: - continue - - # Append matching subdir; if more than one append newest - paths.append(f"{subdirs[-1]}") - - return paths - - -def create_links_paths(tile_base_dir, tile_ID, paths, verbose=False): - """Create Links Paths. - - Create links to paths. - - Parameters - ---------- - tile_base_dir: str - base directory for tile runs - tile_ID: str - tile ID - paths: list - paths; list of str - verbose: bool, optional - verbose output if ``True``; default is ``False`` - - """ - # Get tile output path - tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID) - - # Loop over paths - for path in paths: - - # Get destination = tile output dir + path tail (part of path after last slash) - head, tail = os.path.split(path) - src = path - dst = f"{tile_out_dir}/{tail}" - - if os.path.exists(dst): - - src_existing = os.readlink(dst) - if src_existing == src: - # destination already points to source: skip - if verbose: - print( - f"Warning: {src} <- {dst} already exists, no link created" - ) - continue - else: - # destination points to different source: create links with added index to distinguish - # from existing one(s) - idx = 1 - dst_orig = dst - while True: - # Find destination name with lowest appended index that does not exist - dst = f"{dst_orig}_{idx}" - if os.path.exists(dst): - idx += 1 - else: - # Found: create new link - if verbose: - print(f"link {src} <- {dst}") - os.symlink(src, dst) - break - else: - - # destination does not exist: create link - if verbose: - print(f"link {src} <- {dst}") - os.symlink(src, dst) - - -def main(argv=None): - - # Set default parameters - p_def = params_default() - - # Command line options - options, args = parse_options(p_def) - - if check_options(options) is False: - return 1 - - param = update_param(p_def, options) - - tile_base_dir = param.tile_base_dir - exp_base_dir = param.exp_base_dir - tile_ID = param.tile_ID - n_CPU = 40 - verbose = param.verbose - - exp_IDs = get_exp_IDs(tile_base_dir, tile_ID, verbose=verbose) - exp_shdu_IDs = get_exp_single_HDU_IDs(exp_IDs, n_CPU) - - # Note: psfex P3 is mostly run_sp_exp_SxSePsf - patterns = ["run_sp_exp_SxSePsfPi"] # , "run_sp_exp_Pi"] - if param.sp_local == 1: - patterns.append("run_sp_exp_Sp_shdu") - for pattern in patterns: - paths = get_paths(exp_base_dir, exp_shdu_IDs, pattern) - - create_links_paths(tile_base_dir, tile_ID, paths, verbose=verbose) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py deleted file mode 100755 index 9d9c10d21..000000000 --- a/scripts/python/merge_final_cat.py +++ /dev/null @@ -1,396 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script merge_final_cat.py - -Merge all final catalogues, created by ShapePipe module -``make_catalogue_runner``, into a joined numpy binary file. - -:Authors: Axel Guinot, Martin Kilbinger - -""" - -from astropy.io import fits -import numpy as np -import os -import sys -import re -import copy - -from optparse import OptionParser - -from shapepipe.utilities import cfis - - -class param: - """General class to store (default) variables""" - - def __init__(self, **kwds): - self.__dict__.update(kwds) - - def print(self, **kwds): - print(self.__dict__) - - def var_list(self, **kwds): - return vars(self) - - -def params_default(): - """Params Default. - - Set default parameter values. - - Returns - ------- - class param - parameter values - - """ - p_def = param( - input_path=".", - input_name_base="final_cat", - hdu_num=1, - ) - - return p_def - - -def parse_options(p_def): - """Parse Options. - - Parse command line options. - - Parameters - ---------- - p_def: class param - parameter values - - Returns - ------- - list - command line options - command line str - - """ - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - # IO - parser.add_option( - "-i", - "--input_path", - dest="input_path", - type="string", - default=p_def.input_path, - help=f"input path, default='{p_def.input_path}'", - ) - parser.add_option( - "-n", - "--input_name_base", - dest="input_name_base", - type="string", - default=p_def.input_name_base, - help=f"input name base, default='{p_def.input_name_base}'", - ) - parser.add_option( - "-l", - "--list_tile_ID_path", - dest="tile_ID_list_path", - type="string", - default=None, - help=f"tile ID list, default: Use all data in input files", - ) - - # Control - parser.add_option( - "-p", - "--param_path", - dest="param_path", - type="string", - default=None, - help="parameter file path, default=None", - ) - - parser.add_option( - "", - "--hdu_num", - dest="hdu_num", - type="int", - default=p_def.hdu_num, - help=f"input HDU number, default='{p_def.hdu_num}'", - ) - - parser.add_option( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose output", - ) - - options, args = parser.parse_args() - - return options, args - - -def check_options(options): - """Check Options. - - Check command line options. - - Parameters - ---------- - options: tuple - Command line options - - Returns - ------- - bool - Result of option check. False if invalid option value. - - """ - return True - - -def update_param(p_def, options): - """Update Param. - - Return default parameter, updated and complemented according to options. - - Parameters - ---------- - p_def: class param - parameter values - optiosn: tuple - command line options - - Returns - ------- - class param - updated paramter values - - """ - param = copy.copy(p_def) - - # Update keys in param according to options values - for key in vars(param): - if key in vars(options): - setattr(param, key, getattr(options, key)) - - # Add remaining keys from options to param - for key in vars(options): - if not key in vars(param): - setattr(param, key, getattr(options, key)) - - # Do extra stuff if necessary - - return param - - -# MKDEBUG TODO: remove this function, duplicate in create_final_cat.py -def read_param_file(path, verbose=False): - """Read Param File. - - Return parameter list read from file. - - Parameters - ---------- - path: str - input file name - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - list of str - parameter names - - """ - param_list = [] - - if path: - - with open(path) as f: - for line in f: - if line.startswith("#"): - continue - entry = line.rstrip() - if not entry or entry == "": - continue - param_list.append(entry) - - if verbose: - if len(param_list) > 0: - print(f"Copying {len(param_list)} columns", end="") - else: - print("Copying all columns", end="") - print(" into merged catalogue") - - # Check for multiples - multiples = [] - for param in param_list: - if param_list.count(param) > 1: - multiples.append(param) - - if len(multiples) > 0: - print( - "The following parameters are more than one times " - "in the parameter file: ", - end="", - ) - for m in multiples: - print(m, end=" ") - print() - raise ValueError("Multiple identical keys found") - - return param_list - - -def get_data(path, hdu_num, param_list): - """Get Data. - - Return data of selected columns from FITS file. - - Parameters - ---------- - path: str - input file name - hdu_num: int - HDU number - param_list: list of str - parameters to be extracted. If none, copy - all columns - - Returns - ------- - numpy array - data columns - - """ - hdu_list = fits.open(path) - hdu = hdu_list[hdu_num] - - if param_list: - cols = [] - for p in param_list: - cols.append(hdu.columns[p]) - coldefs = fits.ColDefs(cols) - hdu_new = fits.BinTableHDU.from_columns(coldefs) - d = hdu_new.data - else: - d = hdu.data - - return d - - -def main(argv=None): - - - # Set default parameters - p_def = params_default() - - # Command line options - options, args = parse_options(p_def) - - if check_options(options) is False: - return 1 - - param = update_param(p_def, options) - - if param.verbose: - print("Start") - - # Save command line arguments to log file - f_log = cfis.log_command(argv, close_no_return=False) - - path = param.input_path - - if param.verbose: - print("Read parameter file") - param.param_list = read_param_file(param.param_path, verbose=param.verbose) - - # read (optional) input tile ID file - if param.tile_ID_list_path: - if param.verbose: - print("Read tile ID list") - tile_ID_list = cfis.read_list(param.tile_ID_list_path) - - if param.verbose: - print("Find input catalogue FITS files") - l = os.listdir(path=path) - ext = "fits" - lpath = [] - for this_l in l: - - add_this_l = False - - # mark to add if correct extension, matches input pattern, not `.npy` file - if ( - this_l.endswith(ext) - and (f"{param.input_name_base}" in this_l) - and (".npy" not in this_l) - ): - add_this_l = True - - # unmark to add if no in (optional) input tile ID file - if param.tile_ID_list_path: - nix, niy = cfis.get_tile_number(this_l) - tile_ID = f"{nix}.{niy}" - if tile_ID not in tile_ID_list: - add_this_l = False - if add_this_l: - lpath.append(os.path.join(path, this_l)) - - if param.verbose: - print(f"{len(lpath)} files files to merge found") - - count = 0 - - # Determine number of columns and keys from first catalogue file - d_tmp = get_data(lpath[0], param.hdu_num, param.param_list) - d = np.zeros(d_tmp.shape, dtype=d_tmp.dtype) - for key in d_tmp.dtype.names: - d[key] = d_tmp[key] - count = count + 1 - if param.verbose: - print(f"File '{lpath[0]}' copied ({count}/{len(lpath)})") - - # merge remaining catalogue files - for fname in lpath[1:]: - - try: - d_tmp = get_data(fname, param.hdu_num, param.param_list) - dd = np.zeros(d_tmp.shape, dtype=d.dtype) - - for key in d_tmp.dtype.names: - dd[key] = d_tmp[key] - - count = count + 1 - if param.verbose: - print(f"File '{fname}' copied ({count}/{len(lpath)})") - - d = np.concatenate((d, dd)) - except: - print( - f"Error while adding file '{fname}', {len(dd)} objects" - " not in final cat" - ) - - # Save merged catalogue as numpy binary file - if param.verbose: - print("Saving merged catalogue") - np.save(f"{param.input_name_base}.npy", d) - - msg = f"{count} catalog files merged with success" - if param.verbose: - print(msg) - print(msg, file=f_log) - - f_log.close() - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/stats_headless_canfar.py b/scripts/python/stats_headless_canfar.py deleted file mode 100755 index 1f728949d..000000000 --- a/scripts/python/stats_headless_canfar.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -# Name: stats_headless_canfar.py - -# Caution: Does not show all running or pending -# headless jobs, for some reason. - -import sys -from skaha.session import Session - - -def main(argv=None): - - print( - "# Depreciated, does not show pending jobs; use stats_jobs_canfar.sh", - file=sys.stderr, - ) - - session = Session() - - n_headless = session.stats()["instances"]["headless"] - - print(n_headless) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/summary_tiles.py b/scripts/python/summary_tiles.py deleted file mode 100755 index 9addbb017..000000000 --- a/scripts/python/summary_tiles.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python - -import sys -import os -import re -import numpy as np - -from tqdm import tqdm - - -def get_input_IDs_patch(tiles, patch): - - # Read in files from summary output with missing CCDs. - # Create and update CCD name set. - fname = f"{patch}/tile_numbers.txt" - with open(fname) as f: - lines = f.readlines() - for line in lines: - ID = line.rstrip("\n") - tiles.append( - { - "ID": ID, - "patch": patch, - } - ) - - return tiles - - -def check_zero_weight(tiles, patches): - - for patch in tqdm(patches, desc="check zero weight"): - - fname = f"{patch}/summary/special_job_16_sextractor_runner_5.txt" - - if os.path.exists(fname): - - with open(fname) as f: - lines = f.readlines() - for line in lines: - m = re.search("process-(\d{3})-(\d{3})\.log", line) - if m: - ID = f"{m[1]}.{m[2]}" - my_tile = next((tile for tile in tiles if tile.get("ID") == ID), None) - my_tile["zero_weight"] = 1 - - return tiles - - -def check_exceptions(tiles, patches): - - for patch in tqdm(patches, desc="check processing exceptions"): - - fname = f"{patch}/exceptions.txt" - - if os.path.exists(fname): - - with open(fname) as f: - lines = f.readlines() - for line in lines: - m = re.search("(\d{3})\.(\d{3})\s+\t+(.*)", line) - if m: - ID = f"{m[1]}.{m[2]}" - my_tile = next((tile for tile in tiles if tile.get("ID") == ID), None) - my_tile[m[3]] = 1 - - return tiles - - -def get_final(tiles, patches): - - n_final = {} - - for patch in tqdm(patches, desc="check processing exceptions"): - - n_final[patch] = 0 - - fname = f"{patch}/n_tiles_final.txt" - if os.path.exists(fname): - with open(fname) as f: - lines = f.readlines() - for line in lines: - num = line.rstrip("\n") - n_final[patch] = int(num) - - n_final["all"] = sum(n_final.values()) - - return n_final - - -def fill_True(tiles, keys): - - - for tile in tiles: - for key in keys: - tile[key] = 0 - - return tiles - - -def summary(patches, keys): - - tiles = [] - - n_final = get_final(tiles, patches) - - # Loop over patches - for patch in tqdm(patches, desc="read input tile lists"): - tiles = get_input_IDs_patch(tiles, patch) - - tiles = fill_True(tiles, keys) - - tiles = check_zero_weight(tiles, patches) - - tiles = check_exceptions(tiles, patches) - - return tiles, n_final - -def print_tile(tile, f_out, keys): - - for key in keys: - print(f"{tile[key]}", file=f_out, end=" ") - print(file=f_out) - - -def output_tiles(tiles, path, keys): - - with open(path, "w") as f_out: - - for tile in tiles: - print_tile(tile, f_out, keys) - - -def print_summary(tiles, patches, n_final, keys, keys_sum, nform): - - for key in keys_sum: - print(key.rjust(nform, " "), end="") - print("tot_miss".rjust(nform, " ")) - - for patch in patches: - tiles_patch = [entry for entry in tiles if entry.get("patch") == patch] - print_summary_patch(tiles_patch, patch, n_final[patch], keys, nform) - print_summary_patch(tiles, "all", n_final["all"], keys, nform) - -def print_summary_patch(tiles, patch, n_final, keys, nform): - - n_tiles = len(tiles) - print(patch.rjust(nform, " "), end="") - print(str(n_tiles).rjust(nform, " "), end="") - print(str(n_final).rjust(nform, " "), end="") - n_all = 0 - for key in keys: - n = sum(v[key] for v in tiles) - print(str(n).rjust(nform, " "), end="") - n_all += n - print(str(n_all).rjust(nform, " "), f"{n_all / n_tiles:10.2%}", end="") - print() - - -def main(argv): - - n_patch = 8 - patches = [f'P{x}' for x in np.arange(n_patch) + 1] - - keys = ["zero_weight", "aborted", "large_num_det"] - - nform = 15 - - tiles, n_final = summary(patches, keys) - - output_path = "tile_summary.txt" - - keys_ext = ["ID", "patch"] + keys - output_tiles(tiles, output_path, keys_ext) - - keys_sum = ["patch", "n_tiles", "n_final"] + keys - print_summary(tiles, patches, n_final, keys, keys_sum, nform) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/sh/build_and_plot_coverage_maps.sh b/scripts/sh/build_and_plot_coverage_maps.sh deleted file mode 100755 index b7ea9c8b2..000000000 --- a/scripts/sh/build_and_plot_coverage_maps.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Build and plot per-CCD coverage (nexp) maps for a range of catalogue -# versions. Each map counts, per sky pixel, the number of exposures with a -# valid PSF model. The per-version flow is: -# 0. get_ccds_with_psf -> ccds_with_psf_.txt (valid CCD IDs) -# 1. download_headers -> per-exposure header text files -# 2. extract_field_corners-> exp_ra_dec_.txt (per-CCD corners) -# 3. build_coverage_map -> coverage_.x.hsp -# 4. plot_coverage_map -> SGC / NGC plots -# Steps 0-2 are expected to have been run already on canfar (they need -# VOSpace access); this script rebuilds from the extracted per-CCD corners and -# plots. Uncomment the marked lines to run the full chain. - -# Configuration variables - -## Catalogue versions -VERSIONS=("v1.3" "v1.4" "v1.5" "v1.6") - -## Output directory of plots -OUTPUT_DIR="coverage_map_plots" - -# healsparse resolution. nside=131072 gives ~0.1" per pixel, chosen to -# match the UNIONS bit-mask resolution so coverage and mask align pixel- -# wise. CoverageMapBuilder defaults to nside=2048 for lighter-weight -# offline use; override here for the production build. -BUILD_NSIDE=131072 -BUILD_CHANNELS=128 - -# Common parameters -VERBOSE="-v" - -## Colorbar -PLOT_COLORBAR="-C" -PLOT_MIN=1 -PLOT_MAX=5 - -# Plot parameters -## SGC region -SGC_RA_MIN=-20 -SGC_RA_MAX=45 -SGC_DEC_MIN=18 -SGC_DEC_MAX=40 - -## NGC region -NGC_RA_MIN=110 -NGC_RA_MAX=270 -NGC_DEC_MIN=28 -NGC_DEC_MAX=90 - -# Create output directory if it doesn't exist -mkdir -p "${OUTPUT_DIR}" - -# Loop over versions -for VERSION in "${VERSIONS[@]}"; do - echo "Processing ${VERSION}..." - - # Define file paths - CCD_LIST="ccds_with_psf_${VERSION}.txt" - HEADER_DIR="headers_${VERSION}" - INPUT_CORNERS="exp_ra_dec_${VERSION}.txt" - COVERAGE_MAP="coverage_${VERSION}.x.hsp" - - # Step 0-2: build the per-CCD corner file (canfar / VOSpace only). - # Uncomment to run the full chain from scratch. - # get_ccds_with_psf -V "${VERSION}" -o "${CCD_LIST}" ${VERBOSE} - # download_headers -i "${CCD_LIST}" -o "${HEADER_DIR}" ${VERBOSE} - # extract_field_corners -i "${HEADER_DIR}" -l "${CCD_LIST}" \ - # -o "${INPUT_CORNERS}" ${VERBOSE} - - # Build coverage map - echo " Building coverage map from ${INPUT_CORNERS}..." - CMD="build_coverage_map -i ${INPUT_CORNERS} -o ${COVERAGE_MAP} -c ${BUILD_CHANNELS} -n ${BUILD_NSIDE} ${VERBOSE}" - echo "$CMD" - $CMD - - # Plot SGC region - echo " Plotting SGC region..." - CMD="plot_coverage_map -i ${COVERAGE_MAP} -o ${OUTPUT_DIR}/coverage_${VERSION}_SGC.png ${VERBOSE} ${PLOT_COLORBAR} -R ${SGC_RA_MIN} -r ${SGC_RA_MAX} -D ${SGC_DEC_MIN} -d ${SGC_DEC_MAX} -m ${PLOT_MIN} -M ${PLOT_MAX}" - echo "$CMD" - $CMD - - # Plot NGC region - echo " Plotting NGC region..." - CMD="plot_coverage_map -i ${COVERAGE_MAP} -o ${OUTPUT_DIR}/coverage_${VERSION}_NGC.png ${VERBOSE} ${PLOT_COLORBAR} -R ${NGC_RA_MIN} -r ${NGC_RA_MAX} -D ${NGC_DEC_MIN} -d ${NGC_DEC_MAX} -m ${PLOT_MIN} -M ${PLOT_MAX}" - echo "$CMD" - $CMD - - echo " Done with ${VERSION}" - echo -done - -echo "All versions processed successfully!" diff --git a/scripts/sh/canfar_async_job.sh b/scripts/sh/canfar_async_job.sh deleted file mode 100755 index 7128e7734..000000000 --- a/scripts/sh/canfar_async_job.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -source $HOME/shapepipe/scripts/sh/functions.sh - - -# 1. Extract the line from -f file -# Save original arguments -ARGS=("$@") - -# Find -f argument and get its value; remove -f and -S -FILE_IDS="" -NEW_ARGS=() -i=0 -while [[ $i -lt $# ]]; do - arg="${ARGS[i]}" - case "$arg" in - -f) - ((i++)) - FILE_IDS="${ARGS[i]}" - ;; - -S) - ((i++)) # skip next argument (value of -S) - ;; - *) - NEW_ARGS+=("$arg") - ;; - esac - ((i++)) -done - -if [[ -z "$FILE_IDS" ]]; then - echo "Error: -f must be provided" - exit 1 -fi - -# Get the line for this replica -LINE=$(get_line_from_file -f "$FILE_IDS") - -# 2. Call init_run.sh with remaining args + -e line -$HOME/shapepipe/scripts/sh/init_run_exclusive_canfar.sh "${NEW_ARGS[@]}" -e "$LINE" diff --git a/scripts/sh/canfar_download_results.bash b/scripts/sh/canfar_download_results.bash deleted file mode 100755 index 991af515a..000000000 --- a/scripts/sh/canfar_download_results.bash +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_download_results.bash -# Description: Download ShapePipe results (.tgz files) -# from canfar with vos -# Author: Martin Kilbinger -# Date: v1.0 05/2020 -# v1.1 01/2021 - -# Command line - -## Default parameters -INPUT_VOS="cosmostat/kilbinger/results" -VERBOSE=0 -psf="mccd" -only_mask=0 - - -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -i, --input_IDs ID_FILE\n - \tASCII file with tile IDs to download, default:\n - \tdownload all available IDs\n - --input_vos PATH\n - \tinput path on vos:cfis, default='$INPUT_VOS'\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m\tonly mask files\n - -v\tverbose output\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -i|--input_IDs) - IDs=(`cat $2`) - echo "Downloading ${#IDs[@]} ID(s)" - shift - ;; - --input_vos) - INPUT_VOS="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m) - only_mask=1 - ;; - -v) - VERBOSE=1 - ;; - *) - echo "Invalid command line argument '$1'" - echo -ne $usage - exit 1 - ;; - esac - shift -done - -## Check options -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psf' or 'mccd'" - exit 2 -fi - -## Paths -remote="vos:cfis/$INPUT_VOS" -local="." - -if [ $only_mask == 1 ]; then - NAMES=("pipeline_flag") -else - NAMES=( - "final_cat" - "logs" - "setools_mask" - "setools_stat" - "setools_plot" - "pipeline_flag" - ) - - if [ $psf == "psfex" ]; then - NAMES+=( - "psfex_interp_exp" - ) - else - NAMES+=( - "mccd_fit_val_runner" - ) - fi -fi - -if [ $VERBOSE == 1 ]; then - vflag="-v" -else - vflag="" -fi -export VCP="vcp $vflag" - - -### Start ### - -# Download files -for name in ${NAMES[@]}; do - if [ ${#IDs[@]} == 0 ]; then - cmd="$VCP $remote/$name*.tgz $local" - $cmd - else - for ID in ${IDs[@]}; do - cmd="$VCP $remote/${name}_$ID.tgz $local" - $cmd - done - fi -done - -# Check number of files -for name in ${NAMES[@]}; do - n_downl=(`ls -l $local/${name}_*.tgz | wc`) - echo "$n_downl '$name' result files downloaded from $remote" -done diff --git a/scripts/sh/canfar_submit_selection.sh b/scripts/sh/canfar_submit_selection.sh deleted file mode 100755 index 3698f0722..000000000 --- a/scripts/sh/canfar_submit_selection.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_submit_selection.sh -# Author: Martin Kilbinger, martin.kilbinger@cea.fr -# Date: 2020 -# Description: Submits jobs to canfar, each jobs processes -# one tile - - -# Variables - -# Job file for one tile -SP_ROOT=$HOME/shapepipe -sp_job="$SP_ROOT/scripts/sh/job_sp.bash" - - -# Functions - -function add_queue() { - local job_file=$1 - - echo "queue" >> $job_file - echo >> $job_file -} - -function create_job_file() { - local tile_ID=$1 - local job_file=$2 - - echo "executable = $sp_job" > $job_file - echo >> $job_file - - add_to_job_file $tile_ID $job_file - - finalize_job_file $job_file - add_queue $job_file -} - -function add_to_job_file () { - local tile_ID=$1 - local job_file=$2 - - echo "arguments = $tile_ID" >> $job_file - echo "output = log_canfar_sp_$tile_ID.out" >> $job_file - echo "error = log_canfar_sp_$tile_ID.err" >> $job_file - echo "log = log_canfar_sp_$tile_ID.log" >> $job_file -} - -function finalize_job_file() { - local job_file=$1 - - echo "request_cpus = 8" >> $job_file - - # Cannot be larger than VM available RAM - echo "request_disk = 100G" >> $job_file - - echo >> $job_file -} - -function usage () { - ex=$1 - - echo "Usage: canfar_submit_selection.sh ID_path image [-h] [-n]" - echo "Options:" - echo " ID_path ascii file with tile IDs" - echo " image VM image name, newest is ShapePipe2-mk-20200820" - echo " -n dry run" - echo " -h this message" - - exit $ex -} - -if [ "${!#}" == "-h" ]; then - usage 0 -fi -if [ "$#" -lt 2 ] ; then - usage 1 -fi - -if [ "${!#}" == "-n" ]; then - dry_run=1 - dry_str=" (dry run)" -else - dry_run=0 - dry_str="" -fi - - -## Start - -tile_ID_list=$1 -image=$2 - -# Create job file -job_file="job_tile.sh" -echo "executable = $sp_job" > $job_file -echo >> $job_file - -echo "output = log_canfar_sp_\$(arguments).out" >> $job_file -echo "error = log_canfar_sp_\$(arguments).err" >> $job_file -echo "log = log_canfar_sp_\$(arguments).log" >> $job_file -echo >> $job_file -finalize_job_file $job_file -echo >> $job_file -echo "queue arguments from (" >> $job_file - -# Go through tile IDs and add job -while read -r tile_ID; do - echo "$tile_ID" >> $job_file -done < $tile_ID_list - echo ")" >> $job_file - -# Submit -cmd="canfar_submit $job_file $image c8-30gb-186" -echo "Running $cmd$dry_str" -if [ $dry_run == 0 ]; then - $cmd - condor_q -fi diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash deleted file mode 100755 index 8cbf9bbdf..000000000 --- a/scripts/sh/combine_runs.bash +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env bash - -# Name: combine_runs.bash -# Description: Create new shapepipe run directory with -# links to source files from combined existing runs -# Author: Martin Kilbinger - - -# Command line arguments - -## Default values -cat='final' -psf="mccd" - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -p, --psf MODEL\n - \tPSF model, allowed are 'psfex', 'mccd', 'setools', default='$psf'\n - -c, --cat TYPE\n - \tCatalogue type, allowed are 'final', 'flag_tile', 'flag_exp', \n - \t'psf', 'psf_conv', 'image', 'shdu', default='$cat'\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -c|--cat) - cat="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - *) - echo -ne $usage - exit 1 - ;; - esac - shift -done - - -## Check options -if [ "$cat" != "final" ] \ - && [ "$cat" != "flag_tile" ] \ - && [ "$cat" != "tile_detection" ] \ - && [ "$cat" != "flag_exp" ] \ - && [ "$cat" != "psf" ] \ - && [ "$cat" != "psf_conv" ] \ - && [ "$cat" != "image" ] \ - && [ "$cat" != "shdu" ]; then - echo "cat (option -c) needs to be 'final', 'tile_detection', 'flag_tile', 'flag_exp', 'psf', 'psf_conv', 'shdu', or 'image'" - exit 2 -fi - -## Check options -if [ "$psf" != "psfex" ] \ - && [ "$psf" != "mccd" ] \ - && [ "$psf" != "setools" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 2 -fi - - -## Functions -function link_s () { - target=$1 - link_name=$2 - - if [ -L "$link_name" ]; then - echo "link with name $link_name already exists, skipping..." - let "n_skipped+=1" - else - echo "create link $target <- $link_name" - ln -s $target $link_name - let "n_created+=1" - fi -} - - -# Start program - -n_skipped=0 -n_created=0 - -pwd=`pwd` -out_base="output" - -# Set paths: -## run_out: target output new run directory -## run_in: source input run base directory -## module: source input module runner sub-directory -## pattern: source file pattern - -run_out="run_sp_combined_$cat" - -if [ "$cat" == "final" ]; then - - # v1 - #run_in="$pwd/$out_base/run_sp_Mc_*" - # v2 - run_in="$pwd/tile_runs/*/$out_base/run_sp_Mc_*" - - module="make_catalog_runner" - pattern="final_cat-*" - -elif [ "$cat" == "tile_detection" ]; then - - run_in="$pwd/P?/tile_runs/*/$out_base/run_sp_tile_Sx_*" - module="sextractor_runner" - pattern="sexcat-*" - -elif [ "$cat" == "flag_tile" ]; then - - # v1 - #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1" - # v2 - run_in="$pwd/$out_base/run_sp_tile_Ma_*" - run_out="run_sp_Ma_tile" - - module="mask_runner" - pattern="pipeline_flag-*" - -elif [ "$cat" == "flag_exp" ]; then - - # v1 - #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_2" - # v2 - run_in="$pwd/$out_base/run_sp_exp_Ma_*" - run_out="run_sp_Ma_exp" - - module="mask_runner" - pattern="pipeline_flag-*" - -elif [ "$cat" == "image" ]; then - - run_in="$pwd/$out_base/run_sp_Git_*" - module="get_images_runner" - pattern="CFIS_image-*" - -elif [ "$cat" == "psf" ]; then - - #MKDEBUG TODO: add option - # v1 - #run_in="$pwf/$out_base/run_sp_exp_Pi_*" - # v2 - #run_in="$pwd/exp_runs/*/$out_base/run_sp_exp_Pi_*" - run_in="$pwd/exp_runs/*/$out_base/run_sp_exp_SxSePsfPi_*" - - pattern="validation_psf-*" - if [ "$psf" == "psfex" ]; then - module="psfex_interp_runner" - elif [ "$psf" == "setools" ]; then - module="setools_runner" - else - module="mccd_interp_runner" - fi - -elif [ "$cat" == "psf_conv" ]; then - - #run_in="$pwd/../P?" - run_in="$pwd" - pattern="validation_psf_conv-*" - module="psfex_interp_runner" - -elif [ "$cat" == "shdu" ]; then - - run_in="$pwd/$out_base/run_sp_exp_Sp_shdu_*" - module="split_exp_runner" - pattern="headers-*" - -else - - echo "Invalid catalogue type $cat" - exit 2 - -fi - - -OUTPUT="$pwd/$out_base/$run_out" -mkdir -p $OUTPUT - - -# Create links - -## target directory -outdir=$OUTPUT/$module/output -mkdir -p $outdir - -## identify source files - -# The following can result in an "Argument list too long" error -#FILES=(`find $run_in -type f -name "$pattern" -print0 | xargs -0 echo`) - -i=0 -for dir in $run_in; do - FILES=(`find -L $dir -type f -name "$pattern" -print0 | xargs -0 echo`) - - #echo "$dir $pattern" - - ## Look over source files - for file in ${FILES[@]}; do - - target=$file - link_name=$outdir/`basename $file` - link_s $target $link_name - ((i=i+1)) - - done - -done - -#echo " $n_files target files, $i links created/skipped" -echo " $i total, "$n_skipped skipped, "$n_created links created" - -# Update log file -update_runs_log_file.py diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh deleted file mode 100755 index ada5dc4f0..000000000 --- a/scripts/sh/curl_canfar_local.sh +++ /dev/null @@ -1,303 +0,0 @@ -# Global variables -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session -IMAGE=images.canfar.net/unions/shapepipe -NAME=shapepipe - -# :TODO: Not working -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source $HOME/shapepipe/scripts/sh/functions.sh - -# Command line arguments - -## Default values -job=-1 -psf="psfex" -ID=-1 -file_IDs=-1 -N_SMP=1 -fix=0 -version="1.1" -cmd_remote="$HOME/shapepipe/scripts/sh/init_run_exclusive_canfar.sh" -batch=30 -batch_max=200 -dry_run=0 -mh_local=0 -sp_local=0 -test_only=0 -debug_out="-1" -scratch="-1" -sm=1 - -pat="- " - -## Help string -usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRunning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -f, --file_IDs path - \tfile containing IDs\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n - -s, --sp_local SP\n - \tsplit local run local (SP=1) or global (SP=0); default is SP=$sp_local\n - --sm SM\n - \tWith (SM=1; default) or without (SM=0) spread model input\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default=$N_SMP\n - -F, --fix FIX\n - \tfix missing data (re-download tile, unzip) for FIX=1; default is $fix\n - -V, --version\n - \tversion of docker image, default='$version'\n - -C, --command_remote\n - \tremote command to run on canfar, default='$cmd_remote'\n - -S, --scratch\n - \tprocessing scratch directory, default is None ($scratch)\n - -b, --batch_max\n - \tmaximum batch size = number of jobs run simultaneously, default=$batch_max\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n - -n, --dry_run LEVEL\n - \tdry run, from LEVEL=2 (no processing) to 0 (full run; default)\n - --test\n - \ttest mode, no processing\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -s|--sp_local) - sp_local="$2" - shift - ;; - --sm) - sm="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -f|--file_IDs) - file_IDs="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -F|--fix) - fix="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -V|--version) - version="$2" - shift - ;; - -B|--batch) - batch="$2" - shift - ;; - -b|--batch_max) - batch_max="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - --test) - test_only=1 - ;; - esac - shift -done - - -## Check options - -if [ "$test_only" == "1" ]; then - test_arg="--test" -else - test_arg="" -fi - -if [ "$job" == "-1" ]; then - echo "No job indicated, use option -j" - exit 2 -fi - -if [ "$ID" == "-1" ] && [ "$file_IDs" == "-1" ]; then - echo "No image ID(s) indicated, use option -e ID or -f file_IDs" - exit 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 4 -fi - -if [ "$dry_run" != 0 ] && [ "$dry_run" != 1 ] && [ "$dry_run" != 2 ]; then - echo "Invalid dry_run option, allowed are 0, 1, and 2" - exit 5 -fi - -if [ "$debug_out" != "-1" ]; then - echo "${pat}Starting $(basename "$0") $test_arg" >> $debug_out - echo "${pat}curl ID=$ID" >> $debug_out - echo ${pat}`date` >> $debug_out -fi - -source activate shapepipe -if [ "$debug_out" != "-1" ]; then - echo "${pat}conda prefix = ${CONDA_PREFIX}" >> $debug_out - echo "${pat}script version = ${script_version}" >> $debug_out -fi - -# command line arguments for remote script: -# collect into string - - -RESOURCES="ram=4&cores=$N_SMP" -dir=`pwd` - - -function submit_batch() { - path=$1 - - for ID in `cat $path`; do - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - done -} - -batch=50 -if [ "$batch" -ge "$batch_max" ]; then - ((batch=batch_max/2)) - echo "Reducing batch size to $batch" -fi -sleep=75 - -((n_thresh=batch_max-batch)) - - -if [ "$dry_run" == 2 ]; then - - # Do not call curl (dry run = 2) - echo "Running command dry run:" - - if [ "$ID" == "-1" ]; then - - - # Submit file (dry run = 2) - for ID in `cat $file_IDs`; do - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - done - - else - - # Submit image (dry run = 2) - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - - fi - -else - - # Call curl - rm -rf session_IDs.txt session_image_IDs.txt - - if [ "$ID" == "-1" ]; then - - # Submit file - n_jobs=`cat $file_IDs | wc -l` - if [ "$n_jobs" -gt "$batch_max" ]; then - - # Split into batches - prefix="${file_IDs}_split_" - split -d -l $batch $file_IDs $prefix - n_split=`ls -l $prefix* | wc -l` - echo "Split '$file_IDs' into $n_split batches of size $batch" - - count=1 - n_queued=`stats_jobs_canfar.sh -w all` - for batch in $prefix*; do - echo "Number of queued jobs = $n_queued" - echo "Submitting batch $batch ($count/$n_split)" - echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007" - submit_batch $batch - ((count=count+1)) - - n_queued=`stats_jobs_canfar.sh -w all` - - while [ "$n_queued" -gt "$n_thresh" ]; do - echo "Wait for #jobs = $n_queued jobs to go < $n_thresh ..." - sleep $sleep - n_queued=`stats_jobs_canfar.sh -w all` - done - - done - - else - - # Submit entire file (single batch) - echo "Submit '$file_IDs' in single batch" - submit_batch $file_IDs - - fi - - else - - # Submit image - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - - fi - -fi - -echo "Done $(basename "$0")" - -if [ "$debug_out" != "-1" ]; then - echo "${pat}End $(basename "$0") $test_arg" >> $debug_out -fi diff --git a/scripts/sh/functions.sh b/scripts/sh/functions.sh index f6763c5b0..777f1a557 100644 --- a/scripts/sh/functions.sh +++ b/scripts/sh/functions.sh @@ -1,97 +1,12 @@ -# Global variables -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session -IMAGE=images.canfar.net/unions/shapepipe -NAME=shapepipe +# Shell helper sourced by run_job_sp_canfar_v2.0.bash. +# +# command() reads $VERBOSE, $debug_out, $pat and $STOP as free variables set +# by its caller; the two defaults below cover callers that set neither. -version="2.0" -cmd_remote="$HOME/shapepipe/scripts/sh/run_job_sp_canfar_v2.0.bash" pat="---- " STOP=0 -get_line_from_file() { - local file="" - local OPTIND opt - - # Parse options: look for -f - while getopts ":f:" opt; do - case $opt in - f) - file="$OPTARG" - ;; - \?) - echo "Unknown option: -$OPTARG", ", continuing" - #return 1 - ;; - :) - echo "Option -$OPTARG requires an argument." - return 1 - ;; - esac - done - - # Check that file and REPLICA_ID are set - if [[ -z "$file" ]]; then - echo "Error: -f option not provided." - return 1 - fi - if [[ -z "$REPLICA_ID" ]]; then - echo "Error: REPLICA_ID not set." - return 1 - fi - - # Check file exists - if [[ ! -f "$file" ]]; then - echo "Error: file '$file' not found." - return 1 - fi - - # Extract the line corresponding to REPLICA_ID (1-based index) - sed -n "${REPLICA_ID}p" "$file" -} - - - -# Add session and image IDs to log files -function update_session_logs() { - echo $my_session >> session_IDs.txt - echo "$my_session $ID" >> session_image_IDs.txt -} - -function call_curl() { - my_name=$1 - my_job=$2 - my_psf=$3 - my_ID=$4 - my_N_SMP=$5 - my_dry_run=$6 - my_dir=$7 - my_debug_out=$8 - my_scratch=$9 - my_test_arg=${10} - - my_arg="-j $my_job -p $my_psf -e $my_ID -N $my_N_SMP -n $my_dry_run -d $my_dir --debug_out $my_debug_out -S $my_scratch $my_test_arg" - - if [ "$my_dry_run" == "0" ]; then - my_session=`curl -E $SSL "$SESSION?$RESOURCES" -d "image=$IMAGE:$version" -d "name=${my_name}" -d "cmd=$cmd_remote" --data-urlencode "args=${my_arg[@]}"` - fi - - cmd=("curl" "-E" "$SSL" "$SESSION?$RESOURCES" "-d" "image=$IMAGE:$version" "-d" "name=${my_name}" "-d" "cmd=$cmd_remote" "--data-urlencode" "args=\"${my_arg}\"") - - if [ -n "$my_debug_out" ]; then - echo "${pat}call_curl $my_name $my_arg" >> $my_debug_out - echo "${pat}Running ${cmd[@]} (dry_run=$my_dry_run)" >> $my_debug_out - fi - echo "${cmd[@]} (dry_run=$my_dry_run)" - - - # Running $cmd does not work due to unknown problems with passing of args - - update_session_logs -} - - ## Print string, executes command, and prints return value. function command () { cmd=$1 @@ -142,57 +57,3 @@ function command () { fi fi } - - -function get_kind_from_job() { - my_job=$1 - - job_to_test=2 - kind="none" - - # loop over possible job numbers - while [ $job_to_test -le 1024 ]; do - - (( do_job = $job & $job_to_test )) - if [[ $do_job != 0 ]]; then - - if [ $job_to_test == 32 ]; then - if [ "$kind" == "tile" ]; then - echo "Error: Invalid job $job. mixing tile and exp kinds" - exit 6 - fi - - # job=32 -> set kind to exp - kind="exp" - elif [ $job_to_test == 2 ]; then - if [ "$kind" == "tile" ]; then - echo "Error: Invalid job $job. mixing tile and exp kinds" - exit 6 - fi - - kind="exp" - elif [ $job_to_test == 8 ]; then - if [ "$kind" == "tile" ]; then - echo "Error: Invalid job $job. mixing tile and exp kinds" - exit 6 - fi - - kind="exp" - else - if [ "$kind" == "exp" ]; then - echo "Error: Invalid job $job. mixing tile and exp kinds" - exit 6 - fi - - # job != 32 -> set kind to tile - kind="tile" - fi - - fi - - # Multiply job number by two to get next bitwise number - job_to_test=$((job_to_test * 2)) - done - - echo $kind -} diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh deleted file mode 100755 index 27efadc11..000000000 --- a/scripts/sh/init_run_exclusive_canfar.sh +++ /dev/null @@ -1,642 +0,0 @@ -#!/bin/bash - -# init_run_exclusive_canfar.sh - -# Command line arguments -## Default values -job=-1 -ID=-1 -N_SMP=1 -dry_run=0 -dir=`pwd` -debug_out=-1 -scratch=-1 -fix=0 -test_only=0 -sm=1 - -# mh_local is 0 (1) if merge_header_runner is run on all exposures, -# which is standard so far (run on exposures of given tile only; new) -mh_local=1 - -# sp_local is 0 (1) is split_headers_runner and mask_runner is run -# on all exposures (locally). Not 100% automatic yet. -sp_local=1 -VERBOSE=1 - -pat="-- " - - -## Help string -usage="Usage: $(basename "$0") -j JOB -e ID [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRUnning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerge header file local (MH=1) or global (MH=0); default is $mh_local\n - -s, --sp_local SP\n - \tsplit local run local (SP=1) or global (SP=0); default is $sp_local\n - --sm SM\n - \tWith (SM=1; default) or without (SM=0) spread model input\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default from original config files\n - -d, --directory\n - \trun directory, default is pwd ($dir)\n - -S, --scratch\n - \tprocessing scratch directory, default is None ($scratch)\n - -F, --fix FIX\n - \tfix missing data (re-download tile, unzip) for FIX=1; default is $fix\n - -n, --dry_run LEVEL\n - \tdry run (LEVEL=1), no actual processing; default is $dry_run\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n - --test\n - \ttest mode, no processing\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -s|--sp_local) - sp_local="$2" - shift - ;; - --sm) - sm="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -d|--directory) - dir="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - -F|--fix) - fix="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - --test) - test_only=1 - ;; - esac - shift -done - - -# functions -function message() { - msg=$1 - my_debug_out=$2 - my_exit=$3 - - echo $msg - if [ "$my_debug_out" != "-1" ]; then - echo ${pat}$msg >> $my_debug_out - fi - - if [ "$my_exit" != "-1" ]; then - echo "${pat}exiting with code $my_exit" >> $my_debug_out - exit $my_exit - fi -} - -# Write an updated copy of a shapepipe config with NUMBER_LIST set to the -# given image ID, expressed in the numbering scheme (leading dash, dots -> -# dashes). Replaces the retired shapepipe_run -e/--exclusive flag (#746). -function set_config_number_list() { - local config_orig=$1 - local config_upd=$2 - local _id=$3 - - local number="-$(echo $_id | tr '.' '-')" - local config_tmp="${config_upd}.tmp" - - if grep -q "^NUMBER_LIST" "$config_orig"; then - perl -pe 's/^NUMBER_LIST\s*=.*/NUMBER_LIST = '$number'/' "$config_orig" > "$config_tmp" - else - perl -pe 's/^\[FILE\][ \t]*$/[FILE]\nNUMBER_LIST = '$number'/' "$config_orig" > "$config_tmp" - fi - if ! grep -q "^NUMBER_LIST = $number$" "$config_tmp"; then - echo "set_config_number_list: failed to set NUMBER_LIST in $config_orig" >&2 - exit 1 - fi - mv "$config_tmp" "$config_upd" -} - - -# Init message -message "test=$test_only" $debug_out -1 -if [ "$test_only" == "1" ]; then - echo "MKDEBUG test" - msg="init_run_exclusive.py script test mode, exiting." - ex=0 -else - msg="init_run_exclusive.py script processing mode, starting." - ex=-1 -fi -message "$msg" $debug_out $ex - - -## Check options -message "checking options" $debug_out -1 - -if [ "$job" == "-1" ]; then - message "No job indicated, use option -j" $debug_out 2 -fi - -if [ "$ID" == "-1" ]; then - message "No image ID indicated, use option -e" $debug_out 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - message "PSF (option -p) needs to be 'psfex' or 'mccd'" $debug_out 4 -fi - -if [ "$mh_local" != "0" ] && [ "$mh_local" != "1" ]; then - message "mh_local (option -m) needs to be 0 or 1" $debug_out 5 -fi - -if [ "$sp_local" != "0" ] && [ "$sp_local" != "1" ]; then - message "sp_local (option -m) needs to be 0 or 1" $debug_out 6 -fi - -if [ "$dry_run" != "0" ] && [ "$dry_run" != 1 ]; then - message "dry_run must be 0 or 1, not $dry_run" $debug_out 8 -fi - - -# Start script - -source $HOME/shapepipe/scripts/sh/functions.sh - -msg="Starting $(basename "$0") `date` ID=$ID" -message "$msg" $debug_out -1 -#message "`date`" $debug_out -1 -#message "ID=$ID" $debug_out -1 - -# Set kind -kind=$(get_kind_from_job $job) - -if [ "$kind" == "none" ]; then - message "Error: invalid job $job" $debug_out 5 -fi - -message "kind=$kind" $debug_out -1 - - -if [ "$dry_run" == "1" ]; then - message "running in dry run mode" $debug_out -1 -else - message "not running in dry run mode" $debug_out -1 -fi - -CONDA_PREFIX=$HOME/.conda/envs/shapepipe -PATH=$PATH:$CONDA_PREFIX/bin - -cd $dir - -if [ ! -d ${kind}_runs ]; then - command "mkdir ${kind}_runs" $dry_run -fi - -if [ "$fix" == "1" ]; then - message "Fixing missing data" $debug_out -1 - - message "Download tile images" $debug_out -1 - cd data_tiles - file_name=CFIS.$ID.r.fits - if [ ! -e $file_name ]; then - message "Downloading $file_name" $debug_out -1 - vcp vos:cfis/tiles_DR5/$file_name . - else - message "File $file_name exists, skipping" $debug_out -1 - fi - file_name=CFIS.$ID.r.weight.fits.fz - if [ ! -e $file_name ]; then - message "Downloading $file_name" $debug_out -1 - vcp vos:cfis/tiles_DR5/$file_name . - else - message "File $file_name exists, skipping" $debug_out -1 - fi - cd .. - - message "link to tiles..." $debug_out -1 - cd output/run_sp_GitFeGie_202*/get_images_runner_run_1/output - IDt=`echo $ID | tr "." "-"` - path_tiles=/arc/home/kilbinger/cosmostat/v2/pre_v2/$psf/$patch/data_tiles - link_name=CFIS_image-$IDt.fits - if [ ! -e $link_name ]; then - message "Creating link $link_name" $debug_out -1 - # force to overwrite broken link - ln -sf $path_tiles/CFIS.$ID.r.fits $link_name - else - message "Link $link_name exists, skipping" $debug_out -1 - fi - - link_name=CFIS_weight-$IDt.fitsfz - if [ ! -e $link_name ]; then - message "Creating link $link_name" $debug_out -1 - ln -sf $path_tiles/CFIS.$ID.r.weight.fits.fz $link_name - else - message "Link $link_name exists, skipping" $debug_out -1 - fi - - command "cd ../../../.." $dry_run - - message "Unzip weight ($dry_run)" $debug_out -1 - command "cd tile_runs/$ID" $dry_run - export SP_RUN=`pwd` - command "set_config_number_list cfis/config_tile_Uz.ini config_tile_Uz_upd.ini $ID" $dry_run - command "shapepipe_run -c config_tile_Uz_upd.ini" $dry_run - - cd $dir -else - message "Not fixing missing data" $debug_out -1 -fi - - -cd ${kind}_runs - -if [ ! -d "$ID" ]; then - command "mkdir $ID" $dry_run -fi - -cd $ID -pwd - -# Point cfis to local link, to be independent of platform -ln -sf ~/shapepipe/example/cfis - - -if [ ! -d "output" ]; then - command "mkdir output" $dry_run -fi - -cd output - - -# Update links to global run directories (GiFeGie) -# New: 27/11/2024: Remove link to Uz, conflict with fix -for my_dir in $dir/output/run_sp_[G]*; do - command "ln -sf $my_dir" $dry_run -done - -# The following could be done also with fix=1 -if [ "$fix" == "0" ] && [ "$kind" == "tile" ]; then - if [ ! -e "run_sp_Uz*" ]; then - - message "previous uncompress run not found..." $debug_out -1 - - # Remove broken links - for my_dir in run_sp_Uz*; do - if [ -L $my_dir ]; then - command "rm $my_dir" $dry_run - message "remove broken link $my_dir" $debug_out -1 - fi - done - - # Create working link - for my_dir in $dir/output/run_sp_Uz*; do - command "ln -sf $my_dir" $dry_run - message "create valid link $my_dir" $debug_out -1 - done - cd .. - command "update_runs_log_file.py" $dry_run - cd output - fi -fi - - -# Combined flags - -## Tiles -command "ln -sf $dir/output/run_sp_Ma_tile" $dry_run - -if [ "$sp_local" == "0" ]; then - # Exposures - command "ln -sf $dir/output/run_sp_Ma_exp" $dry_run -#else - #command "rm -f $dir/output/run_sp_Ma_exp" $dry_run -fi - -# Check for existing exp SpMh dir -dir_SpMh="run_sp_exp_SpMh" -if [[ -d "$dir/output/$dir_SpMh" ]]; then - # create link - command "ln -sf $dir/output/$dir_SpMh" $dry_run -else - message "No global SpMh directory found" $debug_out -1 - if [ -L "$dir_SpMh" ] && [ ! -e "$dir_SpMh" ]; then - message "Remove broken link $dir_SpMh" $debug_out -1 - rm $dir_SpMh - fi -fi - -(( do_job = $job & 2 )) -if [ $do_job != 0 ] && [ "$sp_local" == "1" ]; then - message "run local sp" $debug_out -1 - command "rm -rf run_sp_Gie*" $dry_run - command "rm -rf run_sp_exp_Sp*" $dry_run - - # Create new get_image dir - new_dir="run_sp_Gie/get_images_runner/output" - command "rm -rf $new_dir" $dry_run - command "mkdir -p $new_dir" $dry_run - - # Link to image, weight, and flag file of current exposure - - # Remove HDU extension - command "cd $new_dir" $dry_run - exp_ID=$(echo "$ID" | sed 's/-[0-9]\{1,2\}//') - for file in $dir/output//run_sp_GitFeGie_20*/get_images_runner_run_2/output/*${exp_ID}.* ; do - echo $file - command "ln -s $file" $dry_run - done - command "cd ../../.." $dry_run - - # Run Sp - command "cd .." $dry_run - if [ "$scratch" != "-1" ]; then - command "cd ../.." $dry_run - command "mkdir -p $scratch/exp_runs" $dry_run - command "cp -R exp_runs/$ID $scratch/exp_runs" $dry_run - command "cd $scratch/exp_runs/$ID" $dry_run - fi - command "update_runs_log_file.py" $dry_run - export SP_RUN=`pwd` - command "set_config_number_list cfis/config_exp_Sp.ini config_exp_Sp_upd.ini $exp_ID" $dry_run - command "shapepipe_run -c config_exp_Sp_upd.ini" $dry_run - - # Only keep CCD of this ID - command "mkdir -p output/run_sp_exp_Sp_shdu/split_exp_runner/output" $dry_run - command "mv output/run_sp_exp_Sp/split_exp_runner/output/*$ID.* output/run_sp_exp_Sp_shdu/split_exp_runner/output" $dry_run - command "mv output/run_sp_exp_Sp/split_exp_runner/output/headers* output/run_sp_exp_Sp_shdu/split_exp_runner/output" $dry_run - command "rm -rf output/run_sp_exp_Sp" $dry_run - if [ "$scratch" != "-1" ]; then - command "mv output/run_sp_exp_Sp_shdu $dir/exp_runs/$ID/output" $dry_run - command "cd .." $dry_run - command "rm -rf $ID" $dry_run - command "cd $dir" $dry_run - fi - command "update_runs_log_file.py" $dry_run - cd output - - if [ "$job" == "2" ]; then - msg="Finishing $(basename "$0") after job=2 `date` ID=$ID" - message "$msg" $debug_out 0 - fi - -fi - -if [ "$kind" == "tile" ] && [ "$sp_local" == "1" ]; then - echo "New (for P9): skipping link_to_exp_for_tile.py" - cd ../../.. - #command "link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs -s $sp_local" $dry_run - cd tile_runs/$ID - #command "combine_runs.bash -p psfex -c shdu" $dry_run - cd output -fi - - -if [ "$mh_local" == "0" ]; then - if [ ! -f log_exp_headers.sqlite ]; then - # Global Mh and file does not exist -> symlink to - # gllobal mh file - message "creating global link to exp headers (mh_local=0)" $debug_out -1 - command "ln -s $dir/output/log_exp_headers.sqlite" $dry_run - else - message "global link to exp headers (mh_local=0) exists" $debug_out -1 - fi -else - # Local Mh - message "not creating global link to exp headers (mh_local=1)" $debug_out -1 - if [ "$ID" == "-1" ]; then - message "ID needs to be given (option -e) for mh_local" $debug_out 6 - fi - - # Check and remove symbolic (global) mh file link - if [ -L log_exp_headers.sqlite ]; then - # Local Mh and symlink -> remove previous link to - # (potentially incomplete) global mh file - message "Removing previous mh sym link" $debug_out -1 - command "rm log_exp_headers.sqlite" $dry_run - else - message "no mh link found" $debug_out -1 - fi - - # Check size of existing header file - if [ -e log_exp_headers.sqlite ]; then - size=$(stat -c %s log_exp_headers.sqlite) - if (( size > 15000 )); then - message "Found valid local mh file, continuing" $debug_out -1 - else - message "Existing local mh file looks invalid, deleting" $debug_out -1 - rm -f log_exp_headers.sqlite - fi - fi - - if [ ! -e log_exp_headers.sqlite ]; then - message "Creating local mh file" $debug_out -1 - - cd .. - command "update_runs_log_file.py" $dry_run - export SP_RUN=`pwd` - command "shapepipe_run -c cfis/config_exp_Mh.ini" $dry_run - cd output - else - message "Found local mh file, continuing" $debug_out -1 - fi -fi - - -(( do_job = $job & 8 )) -if [ $do_job != 0 ] && [ "$sp_local" == "1" ]; then - # Remove previous local Ma runs - message "cdsclient = $CDSCLIENT" $debug_out -1 - command "rm -rf run_sp_exp_Ma*" $dry_run -fi - -(( do_job = $job & 16 )) -if [[ $do_job != 0 ]]; then - # Remove previous Sx runs - command "rm -rf run_sp_tile_Sx_*" $dry_run -fi - -# Update links to exposure run directories, which were created in job 32 -(( do_job = $job & 64 )) -if [[ $do_job != 0 ]]; then - if [ "$kind" == "tile" ]; then - - # Remove previous runs of this job - rm -rf run_sp_tile_PsViSmVi* - fi -fi - -(( do_job = $job & 128 )) -if [[ $do_job != 0 ]]; then - - echo - - cat_ngmix="run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-*.fits" - dir_ngmix_prev="run_sp_tile_ngmix_Ng1u_prev/ngmix_runner/output" - cat_ngmix_prev="$dir_ngmix_prev/ngmix-*.fits" - - # Remove if empty - if [ ! -s $cat_ngmix ]; then - echo "Removing empty file $cat_ngmix" - rm $cat_ngmix - fi - if [ ! -s $cat_ngmix_prev ]; then - echo "Removing empty file $cat_ngmix_prev" - rm $cat_ngmix_prev - fi - - # Check whether ngmix output exists - if [ -e $cat_ngmix ]; then - message "ngmix output catalogue exists" $debug_out -1 - - # Check whether previous ngmix directory and output cat exist - exists="1" - if [ ! -d $dir_ngmix_prev ]; then - exists="0" - elif [ ! -e "$cat_ngmix_prev" ]; then - exists="1" - fi - if [ "$exists" == "0" ]; then - message "Moving to previous batch-save dir (does not exist yet)" $debug_out -1 - command "mkdir -p $dir_ngmix_prev" $dry_run - command "mv $cat_ngmix $dir_ngmix_prev" $dry_run - else - # Compare file sizes - size_cat_ngmix=$(stat -c%s $cat_ngmix) - size_cat_ngmix_prev=$(stat -c%s $cat_ngmix_prev) - if [ "$size_cat_ngmix" -gt "$size_cat_ngmix_prev" ]; then - message "Moving to batch-save dir, overwriting smaller batch-save cat" $debug_out -1 - command "mv $cat_ngmix $dir_ngmix_prev" $dry_run - else - message "Previous batch-save dir not smaller, removing ngmix output" $debug_out -1 - command "rm $cat_ngmix" $dry_run - fi - fi - else - # Whether or not previous ngmix exists, job_sp_canfar will handle it - message "No ngmix output exists, continuing..." $debug_out -1 - fi - - echo -fi - -(( do_job = $job & 256 )) -if [[ $do_job != 0 ]]; then - - # Remove previous runs of this job - rm -rf run_sp_Ms_20??-* - -fi - -(( do_job = $job & 512 )) -if [[ $do_job != 0 ]]; then - - # Remove previous runs of this job - rm -rf run_sp_Mc_20??-* - -fi - - -cd .. - -# Update log file -command update_runs_log_file.py $dry_run - -echo -n "pwd: " -pwd - -echo -n "environment: " -echo $CONDA_PREFIX - -# To avoid (new?) qt error with setools (-j 32) -export DISPLAY=:1.0 - - -if [ "$scratch" != "-1" ]; then - # Copy inputs to scratch - command "mkdir -p $scratch/${kind}_runs" $dry_run - cd ../.. - command "pwd" $dry_run - command "cp -R ${kind}_runs/$ID $scratch/${kind}_runs" $dry_run - command "cd $scratch/${kind}_runs/$ID" $dry_run -fi - -command "job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP --nsh_jobs $N_SMP --debug_out $debug_out --sm $sm " $dry_run - -if [ "$scratch" != "-1" ]; then - cd ../.. - if [ "$job" == "16" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_Sx_* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "32" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_exp_SxSe* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "64" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_PsViSm** $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "128" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_ngmix* $dir/${kind}_runs/$ID/output" $dry_run - else - echo "scratch mode with job=$job not implemented yet, exiting..." - exit 8 - fi - - command "rm -rf ${kind}_runs/$ID" $dry_run - command "cd $dir/${kind}_runs/$ID" $dry_run - command "update_runs_log_file.py" $dry_run - command "cd $dir" $dry_run -fi - -cd $dir - -#msg="End $(basename "$0")" - -msg="Finished $(basename "$0") `date` ID=$ID" -message "$msg" $debug_out -1 diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash deleted file mode 100755 index c99435823..000000000 --- a/scripts/sh/job_sp_canfar.bash +++ /dev/null @@ -1,610 +0,0 @@ -#!/bin/bash - -# Name: job_sp_canfar.bash -# Description: General script to process one or more tiles -# with all contributing exposures. -# This works as job submission script for -# the canfar batch system. -# called in interactive mode on a virtual -# machine. -# Author: Martin Kilbinger - - -# Command line arguments -## Default values -job=255 -config_dir=$HOME/shapepipe/example/cfis -psf='mccd' -retrieve='vos' -star_cat_for_mask='onthefly' -exclusive='' -results='cosmostat/kilbinger/results_v2' -n_smp=-1 -nsh_jobs=8 -debug_out=-1 -sm=1 - -pat="--- " - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] [TILE_ID] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRunning JOB, bit-coded\n - \t 1: retrieve images (online if method=vos)\n - \t 2: prepare images (offline)\n - \t 4: mask tiles (online if star_cat_for_mask=onthefly)\n - \t 8: mask exposures (online if star_cat_for_mask=onthefly)\n - \t 16: detection of galaxies on tiles (offline)\n - \t 32: processing of stars on exposures (offline)\n - \t 64: galaxy selection on tiles (offline)\n - \t 128: shapes and morphology (offline)\n - \t 256: paste catalogues (offline)\n - -c, --config_dir DIR\n - \t config file directory, default='$config_dir'\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -r, --retrieve METHOD\n - \tmethod to retrieve images, allowed are 'vos', 'symlink', default='$retrieve'\n - -s, --star_cat_for_mask\n - \tcatalogue for masking bright stars, allowed are 'onthefly', 'save',\n - \tdefault is '${star_cat_for_mask}'\n - --sm SM\n - \tWith (SM=1; default) or without (SM=0) spread model input\n - -e, --exclusive ID\n - \texclusive input filer number string ID (default: None)\n - -o, --output_dir\n - \toutput (upload) directory on vos:cfis, default='$results'\n - -n, --n_smp N_SMP\n - \tnumber of jobs (SMP mode only), default from original config files\n - --nsh_jobs NJOB\n - \tnumber of objects per parallel shape module call, \n - \tdefault: optimal number is computed\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n - TILE_ID_i\n - \ttile ID(s), e.g. 283.247 214.242, only with '-j 1'\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -c|--config_dir) - config_dir="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -r|--retrieve) - retrieve="$2" - shift - ;; - -s|--star_cat_for_mask) - star_cat_for_mask="$2" - shift - ;; - --sm) - sm="$2" - shift - ;; - -e|--exclusive) - exclusive="$2" - shift - ;; - -o|--output_dir) - results="$2" - shift - ;; - -n|--n_smp) - n_smp="$2" - shift - ;; - --nsh_jobs) - nsh_jobs="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - esac - shift -done - -## Check options -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 2 -fi - -if [ "$star_cat_for_mask" != "onthefly" ] && [ "$star_cat_for_mask" != "save" ]; then - echo "Star cat for mask (option -s) needs to be 'onthefly' or 'save'" - exit 4 -fi - -if [ "$retrieve" != "vos" ] && [ "$retrieve" != "symlink" ]; then - echo "method to retrieve images (option -r) needs to be 'vos' or 'symlink'" - exit 5 -fi - -if [ "$debug_out" != "-1" ]; then - echo $pat`date` >> $debug_out - echo "${pat}Starting $(basename "$0")" >> $debug_out -fi - -CONDA_PREFIX=/arc/home/kilbinger/.conda/envs/shapepipe -PATH=$PATH:$CONDA_PREFIX/bin - -# For tar archives. TODO: Should be unique to each job -export ID="test" - -## Paths - -## Path variables used in shapepipe config files - -# Run path and location of input image directories -export SP_RUN=`pwd` - -# Config file path -export SP_CONFIG=$SP_RUN/cfis -export SP_CONFIG_MOD=$SP_RUN/cfis_mod - -## Other variables - -# Output -OUTPUT=$SP_RUN/output - -# For tar archives -output_rel=`realpath --relative-to=. $OUTPUT` - -# Stop on error, default=1 -STOP=1 - -# Verbose mode (1: verbose, 0: quiet) -VERBOSE=1 - -# VCP options -export CERTFILE=$HOME/.ssl/cadcproxy.pem -export VCP="vcp --certfile=$CERTFILE" - - -## Functions - -# Print string, executes command, and prints return value. -function command () { - cmd=$1 - str=$2 - - RED='\033[0;31m' - GREEN='\033[0;32m' - NC='\033[0m' # No Color - # Color escape characters show up in log files - #RED='' - #GREEN='' - #NC='' - - - if [ "$debug_out" != "-1" ]; then - echo "${pat}pwd = `pwd`" >> $debug_out - echo "${pat}SP_RUN = $SP_RUN" >> $debug_out - echo "${pat}SP_CONFIG = $SP_CONFIG" >> $debug_out - fi - - if [ $# == 2 ]; then - if [ $VERBOSE == 1 ]; then - echo "$str: running '$cmd'" - fi - if [ "$debug_out" != "-1" ]; then - echo "${pat}Running $cmd" >> $debug_out - fi - - $cmd - - else - if [ $VERBOSE == 1 ]; then - echo "$str: running '$cmd $4 \"$5 $6\"'" - fi - if [ "$debug_out" != "-1" ]; then - echo "${pat}Running $cmd $4 \"$5 $6\"" >> $debug_out - fi - - $cmd $4 "$5 $6" - - fi - - res=$? - - if [ "$debug_out" != "-1" ]; then - echo "${pat}exit code = $res" >> $debug_out - fi - - if [ $VERBOSE == 1 ]; then - if [ $res == 0 ]; then - echo -e "${GREEN}success, return value = $res${NC}" - else - echo -e "${RED}error, return value = $res${NC}" - if [ $STOP == 1 ]; then - echo "${RED}exiting 'canfar_sp.bash', error in command '$cmd'${NC}" - exit $res - else - echo "${RED}continuing 'canfar_sp.bash', error in command '$cmd'${NC}" - fi - fi - fi -} - -# Run shapepipe command. If error occurs, upload sp log files before stopping script. -function command_sp() { - local cmd=$1 - local str=$2 - - command "$1" "$2" -} - -# Set up config file and call shapepipe_run -function command_cfg_shapepipe() { - local config_name=$1 - local str=$2 - local _n_smp=$3 - local _exclusive=$4 - - config_upd=$(set_config_n_smp $config_name $_n_smp) - - # Run a single image ID via NUMBER_LIST in an updated config copy; - # replaces the retired shapepipe_run -e/--exclusive flag (#746) - if [ "$_exclusive" != "" ]; then - set_config_number_list "$config_upd" "$SP_CONFIG_MOD/$config_name" "$_exclusive" - config_upd="$SP_CONFIG_MOD/$config_name" - fi - - local cmd="shapepipe_run -c $config_upd" - command_sp "$cmd" "$str" -} - -# Tar and upload files to vos -function upload() { - base=$1 - shift - ID=$1 - shift - verbose=$1 - shift - upl=("$@") - - echo "Counting upload files" - n_upl=(`ls -l ${upl[@]} | wc`) - if [ $n_upl == 0 ]; then - if [ $STOP == 1 ]; then - echo "Exiting script, no file found for '$base' tar ball" - exit 3 - fi - fi - tar czf ${base}_${ID}.tgz ${upl[@]} - command "$VCP ${base}_${ID}.tgz vos:cfis/$results" "Upload tar ball" -} - -# Upload log files -function upload_logs() { - id=$1 - verbose=$2 - - upl="$output_rel/*/*/logs $output_rel/*/logs" - upload "logs" "$id" "$verbose" "${upl[@]}" -} - -function set_config_n_smp() { - local config_name=$1 - local _n_smp=$2 - - local config_orig="$SP_CONFIG/$config_name" - - if [[ $_n_smp != -1 ]]; then - # Update SMP batch size - local config_upd="$SP_CONFIG_MOD/$config_name" - update_config $config_orig $config_upd "SMP_BATCH_SIZE" $_n_smp - else - # Keep original config file - local config_upd=$config_orig - fi - - # Set "return" value (stdout) - echo "$config_upd" -} - -# Update config file -function update_config() { - local config_orig=$1 - local config_upd=$2 - local key=$3 - local val_upd=$4 - - cat $config_orig \ - | perl -ane 's/'$key'\s+=.+/'$key' = '$val_upd'/; print' > $config_upd -} - -# Write an updated copy of a shapepipe config with NUMBER_LIST set to the -# given image ID, expressed in the numbering scheme (leading dash, dots -> -# dashes). Replaces the retired shapepipe_run -e/--exclusive flag (#746). -function set_config_number_list() { - local config_orig=$1 - local config_upd=$2 - local _id=$3 - - local number="-$(echo $_id | tr '.' '-')" - local config_tmp="${config_upd}.tmp" - - if grep -q "^NUMBER_LIST" "$config_orig"; then - perl -pe 's/^NUMBER_LIST\s*=.*/NUMBER_LIST = '$number'/' "$config_orig" > "$config_tmp" - else - perl -pe 's/^\[FILE\][ \t]*$/[FILE]\nNUMBER_LIST = '$number'/' "$config_orig" > "$config_tmp" - fi - if ! grep -q "^NUMBER_LIST = $number$" "$config_tmp"; then - echo "set_config_number_list: failed to set NUMBER_LIST in $config_orig" >&2 - exit 1 - fi - mv "$config_tmp" "$config_upd" -} - -### Start ### - -echo "Start processing" - -# Create input and output directories -mkdir -p $SP_RUN -cd $SP_RUN -mkdir -p $OUTPUT -mkdir -p $SP_CONFIG_MOD - -# Processing - - -### Retrieve config files -if [[ $config_dir == *"vos:"* ]]; then - command_sp "$VCP $config_dir ." "Retrieve shapepipe config files" -else - if [[ ! -L cfis ]]; then - command_sp "ln -s $config_dir cfis" "Retrieve shapepipe config files" - fi -fi - - -## Retrieve config files and images (online if retrieve=vos) -## Retrieve and save star catalogues for masking (if star_cat_for_mask=save) -(( do_job = $job & 1 )) -if [[ $do_job != 0 ]]; then - - ### Retrieve files - command_cfg_shapepipe \ - "config_GitFeGie_$retrieve.ini" \ - "Retrieve images" \ - -1 \ - $exclusive - - ### Retrieve and save star catalogues for masking - if [ "$star_cat_for_mask" == "save" ]; then - #### For tiles - mkdir $SP_RUN/star_cat_tiles - command_sp \ - "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_1/output $SP_RUN/star_cat_tiles" \ - "Save star cats for masking (tile)" - - #### For single-exposures - mkdir $SP_RUN/star_cat_exp - command_sp \ - "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_2/output $SP_RUN/star_cat_exp exp" \ - "Save star cats for masking (exp)" - fi - -fi - -## Prepare images (offline) -(( do_job = $job & 2 )) -if [[ $do_job != 0 ]]; then - - ### Uncompress tile weights - command_cfg_shapepipe "config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" $n_smp $exclusive - - ### Split images into single-HDU files, merge headers for WCS info - command_cfg_shapepipe \ - "config_exp_SpMh.ini" \ - "Run shapepipe (split images, merge headers)" \ - $n_smp \ - $exclusive - -fi - -## Mask tiles: add star, halo, and Messier object masks (online if "star_cat_for_mask" is "onthefly") -(( do_job = $job & 4 )) -if [[ $do_job != 0 ]]; then - - ### Mask tiles - command_cfg_shapepipe \ - "config_tile_Ma_$star_cat_for_mask.ini" \ - "Run shapepipe (mask tiles)" \ - $n_smp \ - $exclusive - -fi - -## Mask exposures: add star, halo, and Messier object masks (online if "star_cat_for_mask" is "onthefly") -(( do_job = $job & 8 )) -if [[ $do_job != 0 ]]; then - - ### Mask exposures - command_cfg_shapepipe \ - "config_exp_Ma_$star_cat_for_mask.ini" \ - "Run shapepipe (mask exposures)" \ - $n_smp \ - $exclusive - -fi - - -## Remaining exposure processing (offline) -(( do_job = $job & 16 )) -if [[ $do_job != 0 ]]; then - - ### Object detection on tiles - command_cfg_shapepipe \ - "config_tile_Sx.ini" \ - "Run shapepipe (tile detection)" \ - $n_smp \ - $exclusive - -fi - -## Exposure processing (offline) -(( do_job = $job & 32 )) -if [[ $do_job != 0 ]]; then - - ### Star detection, selection, PSF model. setools can exit with an error for CCD with insufficient stars, - ### the script should continue - STOP=0 - command_cfg_shapepipe \ - "config_exp_${psf}.ini" \ - "Run shapepipe (exp $psf)" \ - $n_smp \ - $exclusive - STOP=1 - -fi - -## Process tiles up to shape measurement -(( do_job = $job & 64 )) -if [[ $do_job != 0 ]]; then - - ### PSF model letter: 'P' (psfex) or 'M' (mccd) - letter=${psf:0:1} - Letter=${letter^} - command_cfg_shapepipe \ - "config_tile_${Letter}iViSmVi_canfar.ini" \ - "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" \ - $n_smp \ - $exclusive - -fi - -## Shape measurement (offline) -(( do_job = $job & 128 )) -if [[ $do_job != 0 ]]; then - - ### Prepare config files - n_min=0 - n_obj=`get_number_objects` - if [ "$n_obj" == "-1" ]; then - echo "No tile SExtractor run found, exiting after et_number_objects call" - exit 10 - fi - nsh_step=`echo "$(($n_obj/$nsh_jobs))"` - - n_max=$((nsh_step - 1)) - for k in $(seq 1 $nsh_jobs); do - cat $SP_CONFIG/config_tile_Ng_template_batch.ini | \ - perl -ane \ - 's/(ID_OBJ_MIN =) X/$1 '$n_min'/; s/(ID_OBJ_MAX =) X/$1 '$n_max'/; s/NgXu/Ng'$k'u/; s/X_interp/'$psf'_interp/g; print' \ - > $SP_CONFIG_MOD/config_tile_Ng${k}u.ini - n_min=$((n_min + nsh_step)) - if [ "$k" == $((nsh_jobs - 1)) ]; then - n_max=-1 - else - n_max=$((n_min + nsh_step - 1)) - fi - done - - ### Shapes, run $nsh_jobs parallel processes - VERBOSE=0 - for k in $(seq 1 $nsh_jobs); do - - # if output dir for subrun exists but no output: re-run - ngmix_run=$OUTPUT/"run_sp_tile_ngmix_Ng${k}u/ngmix_runner" - if [ -e "$ngmix_run" ]; then - ngmix_out="$ngmix_run/output" - n_out=`ls -rlt $ngmix_out | wc -l` - if [ "$n_out" -lt 2 ]; then - command \ - "rm -rf $OUTPUT/run_sp_tile_ngmix_Ng${k}u" \ - "Re-running existing empty ngmix subrun $k" - command_sp \ - "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" \ - "Run shapepipe (tile: ngmix $k)" & - else - echo "Skipping existing non-empty ngmix subrun $k" - fi - else - command_sp \ - "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" \ - "Run shapepipe (tile: ngmix $k)" & - fi - done - wait - VERBOSE=1 - -fi - -## Create final catalogues (offline) -(( do_job = $job & 256 )) -if [[ $do_job != 0 ]]; then - - cat $SP_CONFIG/config_merge_sep_cats_template.ini | \ - perl -ane \ - 's/(N_SPLIT_MAX =) X/$1 '$nsh_jobs'/; print' \ - > $SP_CONFIG_MOD/config_merge_sep_cats.ini - - ### Merge separated shapes catalogues - command_sp \ - "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" \ - "Run shapepipe (tile: merge sep cats)" \ - "$VERBOSE" \ - "$ID" -fi - -(( do_job = $job & 512 )) -if [[ $do_job != 0 ]]; then - - # spread_model suffix for config file with or without SM input - if [ "$sm" == "0" ]; then - suff_sm="_nosm" - else - suff_sm="" - fi - - ### Merge all relevant information into final catalogue - command_cfg_shapepipe \ - "config_make_cat_$psf${suff_sm}.ini" \ - "Run shapepipe (tile: create final cat $psf)" \ - $n_smp \ - $exclusive - -fi - -# MKDEBUG: Putting Mh at the end for now, could be integrated before 16. -(( do_job = $job & 1024 )) -if [[ $do_job != 0 ]]; then - - command_cfg_shapepipe \ - "config_exp_Mh.ini" \ - "Run shapepipe (merge exp headers)" \ - $n_smp \ - $exclusive - -fi - -if [ "$debug_out" != "-1" ]; then - echo "${pat}End $(basename "$0") ID=$exclusive success" >> $debug_out -fi diff --git a/scripts/sh/job_sp_canfar_v2.0.bash b/scripts/sh/job_sp_canfar_v2.0.bash index 592cae70d..e0fb28a01 100755 --- a/scripts/sh/job_sp_canfar_v2.0.bash +++ b/scripts/sh/job_sp_canfar_v2.0.bash @@ -1,6 +1,6 @@ #!/bin/bash -# Name: job_sp_canfar.bash +# Name: job_sp_canfar_v2.0.bash # Description: General script to process one or more tiles # with all contributing exposures. # This works as job submission script for diff --git a/scripts/sh/post_proc_sp.bash b/scripts/sh/post_proc_sp.bash deleted file mode 100755 index 930d5b79d..000000000 --- a/scripts/sh/post_proc_sp.bash +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# Name: post_proc_sp.bash -# Description: Post-process downloaded canfar results and -# creates final merge catalog. -# Author: Martin Kilbinger -# Date: 06/2020 -# Package: shapepipe - -# The shapepipe python virtual environment needs to -# be active to run this script. - - -# Command line arguments - -## Default values -psf='mccd' - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -p|--psf) - psf="$2" - shift - ;; - *) - echo -ne $usage - exit 1 - ;; - esac - shift -done - -## Check options -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 2 -fi - -# Paths -export SP_RUN=. -SP_BASE=$HOME/astro/repositories/github/shapepipe -SP_CONFIG=$SP_BASE/example/cfis - - -# To download results from canfar, use -# -# canfar_download_results.sh -# -# On candide this needs to be done on -# the login node. - -# To Un-tar all .tgz results files, use -# -# $SP_BASE/scripts/sh/untar_results.sh - - -# PSF - -## Collect all psfinterp results -combine_runs -p $psf -t psf - -## Merge all psfinterp results and compute PSF residuals -shapepipe_run -c $SP_CONFIG/config_MsPl_$psf.ini - - -# Galaxies - -## Prepare output directory with links to all 'final_cat' result files -combine_runs - -## Merge final output files to single mother catalog -input_final=output/run_sp_combined/make_catalog_runner/output -merge_final_cat -i $input_final -p $SP_CONFIG/final_cat.param -v diff --git a/scripts/sh/remove_duplicates_tiles.sh b/scripts/sh/remove_duplicates_tiles.sh deleted file mode 100755 index 6f789fe6e..000000000 --- a/scripts/sh/remove_duplicates_tiles.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Script to remove duplicate tile runs, keeping only the newest -# Run from directory containing P?/tile_runs subdirectories - -count=0 -total_removed=0 - -echo "Starting duplicate tile run removal..." -echo "Looking for patches in P?/tile_runs structure..." -echo "" - -# Process each patch directory -for patch_dir in P*/tile_runs; do - if [ ! -d "$patch_dir" ]; then - continue - fi - - patch_name=$(dirname "$patch_dir") - echo "Processing patch: $patch_name" - - # Process each tile directory within the patch - for dir in "$patch_dir"/*; do - if [ ! -d "$dir" ]; then - continue - fi - - # Check each of the three run types - for run_type in run_sp_tile_PsViSmVi run_sp_tile_Mc run_sp_tile_Ms; do - if compgen -G "$dir/output/${run_type}*" > /dev/null; then - n=$(ls -dt "$dir/output/${run_type}"* 2>/dev/null | wc -l) - if [ "$n" -gt 1 ]; then - ((n_remove=n-1)) - echo " Found $n copies of $run_type in $(basename "$dir"), removing $n_remove oldest" - - # Remove all but the newest - ls -dt "$dir/output/${run_type}"* | tail -n "$n_remove" | while read old_run; do - echo " Removing: $(basename "$old_run")" - rm -rf "$old_run" - ((total_removed++)) - done - - ((count++)) - fi - fi - done - done - echo "" -done - -echo "========================================" -echo "Summary:" -echo " Processed tile directories: $count" -echo " Total run directories removed: $total_removed" -echo "========================================" diff --git a/scripts/sh/run_scratch_local.sh b/scripts/sh/run_scratch_local.sh deleted file mode 100755 index 1223e6683..000000000 --- a/scripts/sh/run_scratch_local.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash - -# Command line arguments -## Default values -job=-1 -ID=-1 -N_SMP=1 -dry_run=0 -dir=`pwd` -debug_out=-1 -scratch=/n17data/`whoami`/scratch -exec_path=$HOME/shapepipe/scripts/sh -slurm=1 - -# mh_local is 0 (1) if merge_header_runner is run on all exposures, -# which is standard so far (run on exposures of given tile only; new) -mh_local=0 - -# sp_local is 0 (1) is split_headers_runner and mask_runner is run -# on all exposures (locally). Not 100% automatic yet. -sp_local=1 -VERBOSE=1 - -pat="-- " - -# Help string -usage="Usage: $(basename "$0") -j JOB -e ID -k KIND [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRUnning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default from original config files\n - -d, --directory\n - \trun directory, default is pwd ($dir)\n - -S, --scratch\n - \tprocessing scratch directory, default is $scratch\n - -n, --dry_run LEVEL\n - \tdry run, no actuall processing\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -d|--directory) - dir="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - esac - shift -done - -## Check options -if [ "$job" == "-1" ]; then - echo "No job indicated, use option -j" - exit 2 -fi - -if [ "$exclusive" == "-1" ]; then - echo "No image ID indicated, use option -e" - exit 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 4 -fi - - -source $HOME/shapepipe/scripts/sh/functions.sh - - -kind=$(get_kind_from_job $job) - - -# Load common functions -source $HOME/shapepipe/scripts/sh/functions.sh - - -# Start script - -if [ "$scratch" != "-1" ]; then - - command "mkdir -p $scratch/${kind}_runs" $dry_run - command "cp -R ${kind}_runs/$ID $scratch/${kind}_runs" $dry_run - command "cd $scratch" $dry_run - -fi - - if [ "$slurm" == "0" ]; then - command "init_run_exclusive_canfar.sh -j $job -p $psf -m $mh_local -N $N_SMP -e $ID" $dry_run - else - STATUS=$(sbatch --output=./sbatch-$ID.out --partition=comp --job-name="j${job}_${ID}" --ntasks-per-node=$N_SMP --time=32:00:00 --mem=64G $exec_path/init_run_exclusive_canfar.sh -j $job -p $psf -m $mh_local -N $N_SMP -e $ID) - - JOB_ID=$(echo $STATUS | cut -d ' ' -f 4) - echo "JOB_ID=$JOB_ID" - - # Wait for the job to finish - while true; do - STATUS=$(squeue -j "$JOB_ID" -h -o "%T") - if [[ -z "$STATUS" ]]; then - echo "job $JOB_ID no longer in the queue" - break - fi - - echo "Waiting for job $JOB_ID in state '$STATUS' to complete..." - sleep 10 - done - - echo "Job $JOB_ID has completed. Proceeding with the script..." - fi - -if [ "$scratch" != "-1" ]; then - - if [ "$job" == "32" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_exp_SxSe* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "64" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_PsViSm* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "128" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_ngmix_* $dir/${kind}_runs/$ID/output" $dry_run - fi - - command "rm -rf ${kind}_runs/$ID" $dry_run - command "cd $dir/${kind}_runs/$ID" $dry_run - # Gave Input/Output python error - #command "update_runs_log_file.py" $dry_run - command "cd $dir" $dry_run - -fi diff --git a/scripts/sh/stats_jobs_canfar.sh b/scripts/sh/stats_jobs_canfar.sh deleted file mode 100755 index f9c51101a..000000000 --- a/scripts/sh/stats_jobs_canfar.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# Name: stats_jobs_canfar.sh -# Author: Martin Kilbinger -# Description: Handles headless jobs on canfar - - -# Global variables - -## Temporary files -tmpfile_jobs="jobinfo.txt" -tmpfile_ids="ids.txt" -tmpfile_running="jobs_running.txt" - -## curl options -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session - - -# Command line arguments - -## Default values -mode="count" -debug=0 - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -m, --mode MODE\n - \tmode, allowed are 'count' (default), 'delete'\n - -d, --debug\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -m|--mode) - mode="$2" - shift - ;; - -d|--debug) - debug=1 - ;; - esac - shift -done - -## Check options -case $mode in - "count"|"delete") - # valid option - ;; - *) - echo "Invalid mode $mode" - exit 1 - ;; -esac - - -# Main program - -# Get all instances -if [ "$debug" == "1" ]; then - curl -E $SSL $SESSION - echo curl -E $SSL $SESSION - exit 0 -else - curl -E $SSL $SESSION &> /dev/null > $tmpfile_jobs -fi -res=$? - -if [ "$res" == "0" ]; then - - # Get headless job IDs - cat $tmpfile_jobs | grep headless -B 4 -A 2 | grep Running -A 1 > $tmpfile_ids - - # Number of jobs - n_headless=`cat $tmpfile_ids | grep Running | wc -l` - - # Get running job info - cat $tmpfile_ids | grep name | perl -F\- -ane 'chomp; $F[4] =~ s/[",]//g; print "$F[3].$F[4]"' > $tmpfile_running - -else - - # Failure: set to very high number - n_headless=10000 - -fi - - -if [ "$mode" == "count" ]; then - - echo $n_headless - -elif [ "$mode" == "delete" ]; then - - echo -n "Delete $n_headless jobs? [y|n] " - read answer - if [ "$answer" == "y" ]; then - cat $tmpfile_jobs | grep headless -B 34 -A 6 | grep Running -A 34 | grep id | grep -v user | perl -F\" -ane 'print "$F[3]\n"' > $tmpfile_ids - for ID in `cat $tmpfile_ids`; do - echo $ID - # Delete headless jobs - echo "curl -X DELETE -E $SSL $SESSION/$ID" - curl -X DELETE -E $SSL $SESSION/$ID - echo $? - done - fi - -fi - - -# Remove temporary files -#rm -f $tmpfile_jobs $tmpfile_ids diff --git a/src/shapepipe/canfar/canfar_submit.py b/src/shapepipe/canfar/canfar_submit.py deleted file mode 100644 index 685584b99..000000000 --- a/src/shapepipe/canfar/canfar_submit.py +++ /dev/null @@ -1,576 +0,0 @@ -# canfar_submit_job.py - -# Submit job with the canfar client - -import os -import sys -import asyncio -import math - -from canfar.sessions import Session -from canfar.sessions import AsyncSession -from datetime import datetime - -from cs_util import args as cs_args -from cs_util import logging - - -class Job(object): - """Class Job. - - Handles job submission with the canfar system. - - """ - - def __init__(self): - - self.params_default() - - self._patch = os.environ["patch"] if "patch" in os.environ else "P0" - - # Maximum replicas per batch (CANFAR limit is 512) - self._max_replicas_per_batch = 512 - if self._max_replicas_per_batch != 512: - print(f"Setting the maximum number of replicas to the non-standard {self._max_replicas_per_batch}") - - # Default number of cores, can be overwritten by -c or -P (parallel jobs) options - self._cores_default = 2 - - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - ) - self._params = options - - # Save calling command - logging.log_command(args) - - def params_default(self): - """Params Default. - - Set default parameters. - - """ - self._params = { - "job": -1, - "exclusive": None, - "file_IDs": None, - "psf": "psfex", - "n": "0", - "debug_out": None, - "dry_run": 0, - "log": "log_jobs.txt", - "sync": "async", - "cores": 2, - "ram": 4, - "parallel_jobs": 1, - "jobs_per_session": 0, - "stats": False, - "version": "1.1", - } - - self._short_options = { - "job": "-j", - "exclusive": "-e", - "file_IDs": "-f", - "psf": "-p", - "dry_run": "-n", - "log": "-l", - "sync": "-S", - "cores": "-c", - "ram": "-r", - "parallel_jobs": "-P", - "jobs_per_session": "-J", - "stats": "-s", - "version": "-V", - } - - self._types = { - "job": "int", - "cores": "int", - "ram": "int", - "parallel_jobs": "int", - "jobs_per_session": "int", - "stats": "bool", - } - - self._help_strings = { - "job": "Running JOB, bit-coded, default is {}", - "exclusive": "Run exclusively given image", - "file_IDs": "file containing IDs", - "psf": "PSF model, allowed are 'psfex' and 'mccd'; default is {}", - "dry_run": "if dry run > 0 no actual processing; allowed are 2, 1, 0; default is {}", - "debug_out": "debug output file path, default:set automatically", - "log": "output log file with job and tile IDs", - "sync": "job submission mode, allowed are async, sync; default is {}", - "cores": "number of CPU cores per replica; default is {} or overwritten by -P", - "ram": "RAM in GB per replica; default is {}", - "parallel_jobs": "number of tiles to process in parallel per replica; default is {}", - "jobs_per_session": ( - "number of jobs (tiles) per session; if 0, create one session per job (default is {});" - + " Number of replicas is #FILE_IDs / JOBS_PER_SESSION" - ), - "stats": "print stats of sessions/batches/replicas and exit", - "version": "shapepipe image version; default is {}", - } - - def update_params(self): - """Update Params. - - Update parameters. - """ - - # Set version and image - version = self._params["version"] - self._image = f"images.canfar.net/unions/shapepipe:{version}" - - # If not given, set number of cores to number of parallel jobs; - # else set to default. - if self._params["cores"] == -1: - if self._params["parallel_jobs"] != 1: - self._params["cores"] = self._params["parallel_jobs"] - else: - self._params["cores"] = self._cores_default - - def check_params(self): - """Check Params. - - Check command line arguments and parameters. - - Returns - ------- - bool - status of checks; True (False) if passed (failed) - - """ - if self._params["job"] == -1: - raise ValueError(f"need to specify job (option -j)") - - if self._params["sync"] == "async" and not self._params["file_IDs"]: - raise ValueError("asynchronous mode only possible when tile_IDs file given (-f)") - - if not self._params["exclusive"] and not self._params["file_IDs"]: - raise ValueError("No image ID(s) indicated, use option -e ID or -f file_IDs") - - def set_options_base(self): - """Set Options Base. - - Set basic options from command line options and parameters. - - """ - opt = {} - - cwd = os.getcwd() - - # asynchronous mode - if self._params["sync"] == "async": - opt["-f"] = f"-f {cwd}/{self._params['file_IDs']}" - - # Options - opt["j"] = f"-j {self._params['job']}" - opt["p"] = f"-p {self._params['psf']}" - - # Set identifier for debug file name - if self._params["exclusive"]: - deb_ID = self._params["exclusive"] - else: - deb_ID = os.path.basename(self._params["file_IDs"]) - - if self._params['debug_out']: - opt["debug_out"] = f"--debug_out {cwd}/debug/{self._params['debug_out']}" - else: - opt["debug_out"] = f"--debug_out {cwd}/debug/debug_{deb_ID}.txt" - - opt["d"] = f"-d {cwd}" - opt["m_s"] = "-m 1 -s 1" if self._patch in ("P8", "P9") else "" - if self._params["dry_run"] != 0: - opt["n"] = f"-n {self._params['dry_run']}" - if self._params["parallel_jobs"] > 1: - opt["P"] = f"--parallel_jobs {self._params['parallel_jobs']}" - - options = "" - for key in opt: - options = f"{options} {opt[key]}" - - self._options_base = options - - def set_job_name(self, suf=None): - """Set Job Name. - - Set job name as it will appear on the canfar submission system. - - Parameters - ---------- - suf : str, optional - suffix, default is ``None`` - - """ - if suf is None: - suf = self._params["exclusive"] - - suf1 = suf.replace(".", "-") - - job_name = f"sp-{self._patch}-j{self._params['job']}-{suf1}" - - return job_name - - def set_command(self, mode): - """Set Command. - - Set shell command to run. - - Parameters - ---------- - mode: str - job submission mode, allowed are "async_single" (for asynchronous - submission of single job), "async_bulk" (asynchronous submission of many jobs), - "sync" (synchronous submission of one or more jobs) - - """ - if mode == "async_single": - self._cmd = f"{os.environ['HOME']}/shapepipe/scripts/sh/canfar_async_job.sh" - elif mode == "async_bulk": - self._cmd = f"{os.environ['HOME']}/shapepipe/scripts/python/distribute_tiles.py" - elif mode == "sync": - self._cmd = f"{os.environ['HOME']}/shapepipe/scripts/sh/init_run_exclusive_canfar.sh" - - def get_tile_IDs(self): - """Get Tile IDs. - - Return tile ID information. - - Returns - ------- - str - tile_ID(s) - int - length of tile ID list - str - suffix - - """ - if self._params["exclusive"]: - tile_IDs = [self._params["exclusive"]] - suf = None - else: - with open(self._params["file_IDs"], "r") as f: - tile_IDs = [line.strip() for line in f] - suf = "" - - return tile_IDs, len(tile_IDs), suf - - async def run_async(self): - """Run Async. - - Run asynchronous job submission. - - """ - async with AsyncSession() as session: - - _, total_n, suf = self.get_tile_IDs() - - # Calculate number of replicas (sessions) based on jobs_per_session - if self._params["jobs_per_session"] > 0: - # Each replica processes multiple jobs - num_replicas = math.ceil(total_n / self._params["jobs_per_session"]) - print(f"Distributing {total_n} jobs across {num_replicas} sessions") - print(f"Each session will process ~{self._params['jobs_per_session']} jobs") - else: - # Old behavior: one replica per job - num_replicas = total_n - print(f"Creating {num_replicas} sessions (one per job)") - - # Calculate number of batches needed - num_batches = math.ceil(num_replicas / self._max_replicas_per_batch) - - if num_batches == 1: - # Submit single batch - self.set_command(mode="async_bulk") - print(f"Submitting {num_replicas} replicas in a single batch") - return await self._submit_single_batch(num_replicas, total_n) - else: - # Submit multiple batches as chunks - self.set_command(mode="async_bulk") - print(f"Splitting into {num_batches} batches (max {self._max_replicas_per_batch} replicas per batch)") - return await self._submit_multiple_batches(num_replicas, total_n, num_batches) - - async def _submit_single_batch(self, num_replicas, total_n): - """Submit Single Batch. - - Submit a single batch of jobs. - - Parameters - ---------- - num_replicas: int - number of replicas (sessions) to create - total_n: int - total number of jobs (tiles) to process - - Returns - ------- - list - IDs of submitted sessions - """ - print(f"Submitting {num_replicas} replicas to process {total_n} jobs") - - job_name = self.set_job_name(suf="") - options = self._options_base.lstrip() - print(f"Running '{self._cmd} {options}'") - - async with AsyncSession() as session: - try: - sessions = await session.create( - name=job_name, - image=self._image, - cmd=self._cmd, - args=options, - replicas=num_replicas, - cores=self._params["cores"], - ram=self._params["ram"], - ) - print(f"✓ Batch submitted successfully: {len(sessions)} sessions created") - print(f"Each session will process ~{math.ceil(total_n / num_replicas)} jobs using chunk()") - print("Sessions = ", sessions) - except Exception as e: - print(f"❌ CANFAR session.create() failed: {type(e).__name__}: {e}") - raise - - return sessions - - async def _submit_multiple_batches(self, num_replicas, total_n, num_batches): - """Submit Multiple Batches. - - Submit multiple batches of jobs. - - Parameters - ---------- - num_replicas: int - total number of replicas (sessions) to create - total_n: int - total number of jobs (tiles) to process - num_batches: int - number of batches to split replicas into - - Each batch creates a subset of replicas. chunk() automatically - distributes all tiles across all replicas. - - Returns - ------- - list - IDs of submitted sessions - - """ - all_sessions = [] - - async with AsyncSession() as session: - for batch_num in range(1, num_batches + 1): - # Calculate batch size (number of replicas in this batch) - if batch_num < num_batches: - batch_size = self._max_replicas_per_batch - else: - # Last batch gets the remainder - batch_size = num_replicas - (num_batches - 1) * self._max_replicas_per_batch - - print(f"\n--- Batch {batch_num}/{num_batches} ---") - print(f"Submitting {batch_size} replicas") - print(f"Total {num_replicas} replicas will process {total_n} jobs") - print(f"chunk() will distribute jobs across all replicas") - - job_name = self.set_job_name(suf=f"b{batch_num}") - options = ( - f"{self._options_base} --batch_num {batch_num}" - + f" --batch_tot {num_batches}" - + f" --batch_size {batch_size}" - ) - options = options.lstrip() - print(f"Running '{self._cmd} {options}'") - - try: - sessions = await session.create( - name=job_name, - image=self._image, - cmd=self._cmd, - args=options, - replicas=batch_size, - cores=self._params["cores"], - ram=self._params["ram"], - - ) - print(f"✓ Batch {batch_num} submitted: {len(sessions)} sessions created") - print(sessions) - all_sessions.extend(sessions) - - # Small delay between batches to avoid overwhelming the system - if batch_num < num_batches: - await asyncio.sleep(1) - - except Exception as e: - print(f"❌ Batch {batch_num} failed: {type(e).__name__}: {e}") - # Continue with other batches even if one fails - continue - - print(f"\n=== Submission complete ===") - print(f"Total sessions created: {len(all_sessions)} replicas (for {total_n} jobs)") - print(f"Each replica will process ~{math.ceil(total_n / num_replicas)} jobs") - - return all_sessions - - def run_no_async(self): - """Run No Async. - - Run synchronous job submission. - - Returns - ------- - list - IDs of submitted jobs - - """ - session = Session() - - self.set_command(mode="sync") - - tile_IDs, _, _ = self.get_tile_IDs() - - job_ids = [] - for tile_ID in tile_IDs: - - job_name = self.set_job_name(tile_ID) - - options = f"{self._options_base} -e {tile_ID}" - - # Remove leading whitespace, problem in passing args below - options = options.lstrip() - - print(f"Running '{self._cmd} {options}'") - - # Submit flexible job (default - auto-scaling) - job_id = session.create( - name=job_name, - image=self._image, - cmd=self._cmd, - args=options, - ) - - print(f"Submitted job: {job_id}") - job_ids.append(job_id) - - return job_ids - - def write_job_tile_IDs(self, job_ids): - """Write Job Tile IDs - - Write tile IDs of jobs to log file. - - Parameters - ---------- - jobs_ids: list - IDs of submitted jobs. - - """ - tile_IDs, _, _ = self.get_tile_IDs() - - cwd = os.getcwd() - log_path = f"{cwd}/{self._params['log']}" - - print(f"Writing job and tile ID log file {log_path}") - with open(log_path, "w") as log: - for job_id, tile_ID in zip(job_ids, tile_IDs): - print(job_id, tile_ID, file=log) - - def print_stats(self): - """Print Stats. - - Print statistics of (to be submitted) jobs, helpfule to fine-tune parameters - before the actual submission. - - """ - # Total number of jobs (= length of input ID file) - _, total_n, _ = self.get_tile_IDs() - - if self._params["jobs_per_session"] == 0: - # Compute max jobs per session to fit in single batch - self._params["jobs_per_session"] = math.ceil(total_n / self._max_replicas_per_batch) - print(f"Max jobs per session (-J) = {self._params['jobs_per_session']}") - - # Number of sessions - n_session = math.ceil(total_n / self._params["jobs_per_session"]) - - # Number of batches - n_batch = math.ceil(n_session / self._max_replicas_per_batch) - - # Number of jobs per session - n_jobs = math.ceil(self._params["jobs_per_session"] / n_batch) - - n_jobs_serial = max( - math.ceil(n_jobs / self._params["parallel_jobs"]), - 1, - ) - - print( - f"Number of jobs = {total_n}\t\t(#{self._params['file_IDs']})") - print( - f"Number of sessions = {n_session:5d}\t\t({total_n} /" - + f" {self._params['jobs_per_session']})" - ) - print( - f"Number of batches = {n_batch:5d}\t\t({n_session} /" - + f" {self._max_replicas_per_batch})" - ) - print( - f"Number of jobs per session = {n_jobs:5d}" - + f"\t\t({self._params['jobs_per_session']} / {n_batch}" - ) - print( - f"Number of serial jobs = {n_jobs_serial:5d}\t\t({n_jobs}" - + f" / {self._params['parallel_jobs']})" - ) - - def run(self, args=None): - """Run. - - Run instance. - - Parameters - ---------- - args: list, optional - command line arguments, default is ``None`` - - """ - obj = self - - if args is None: - args = sys.argv - obj.set_params_from_command_line(args) - obj.update_params() - obj.check_params() - - obj.set_options_base() - - if self._params["stats"]: - obj.print_stats() - sys.exit(0) - - # Initialize session manager - self._session = Session() - - if obj._params["sync"] == "async": - print("Async mode") - job_ids = asyncio.run(obj.run_async()) - else: - print("Sync mode") - job_ids = obj.run_no_async() - print(f"Submitting jobs: done") - - obj.write_job_tile_IDs(job_ids) diff --git a/src/shapepipe/canfar_run.py b/src/shapepipe/canfar_run.py index 8330b8013..8f54c08ef 100644 --- a/src/shapepipe/canfar_run.py +++ b/src/shapepipe/canfar_run.py @@ -4,25 +4,7 @@ # executables are created by pyproject.toml. import sys -from shapepipe.canfar import canfar_submit, canfar_monitor, canfar_log_monitor - -def run_job(args=None): - """Run Job. - - Handles job submission with the canfar library - - Parameters - ---------- - args : list, optional - command line arguments, default is ``None`` - - """ - # Create instance - obj = canfar_submit.Job() - - # Run instance - obj.run(args=args) - +from shapepipe.canfar import canfar_monitor, canfar_log_monitor def run_log(args=None): """Run Log. diff --git a/src/shapepipe/coverage_run.py b/src/shapepipe/coverage_run.py index 15aaa1a37..9a167d8d6 100644 --- a/src/shapepipe/coverage_run.py +++ b/src/shapepipe/coverage_run.py @@ -1,79 +1,20 @@ """COVERAGE_RUN -Call coverage processing classes. +Console entry point for plotting a coverage map. + +Building the map is the Snakemake ``coverage_map`` rule's job; plotting a +finished ``.hsp`` is a human act on a durable product, so it stays a +hand-run command, on the same argument that keeps ``run_report.py`` out of the +DAG. The plot windows for the UNIONS SGC and NGC fields are in +``workflow/config.yaml``'s ``coverage:`` block. Author: Martin Kilbinger """ -import sys - -from shapepipe.utilities.header_downloader import HeaderDownloader -from shapepipe.utilities.field_corners_extractor import FieldCornersExtractor -from shapepipe.utilities.coverage_map_builder import CoverageMapBuilder from shapepipe.utilities.coverage_plotter import CoveragePlotter -def run_download_headers(args=None): - """Run Download Headers. - - Download FITS headers from VOSpace for exposures in a CCD list. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code - - """ - obj = HeaderDownloader() - return obj.run(args=args) - - -def run_extract_corners(args=None): - """Run Extract Corners. - - Extract per-CCD sky-footprint corner coordinates from FITS headers. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code - - """ - obj = FieldCornersExtractor() - return obj.run(args=args) - - -def run_build_coverage(args=None): - """Run Build Coverage. - - Build HealSparse coverage maps from per-CCD corner coordinates. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code - - """ - obj = CoverageMapBuilder() - return obj.run(args=args) - - def run_plot_coverage(args=None): """Run Plot Coverage. @@ -92,13 +33,3 @@ def run_plot_coverage(args=None): """ obj = CoveragePlotter() return obj.run(args=args) - - -def main(argv=None): - """Main. - - Main program. - - """ - # Scripts to call coverage classes are created by pyproject.toml - return 0 diff --git a/src/shapepipe/get_ccds_run.py b/src/shapepipe/get_ccds_run.py deleted file mode 100644 index fb18b0a74..000000000 --- a/src/shapepipe/get_ccds_run.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -"""GET_CCDS_WITH_PSF - -Obtain list of CCDs (single-exposure single-HDU files) for which valid PSF information -is available. This can serve to create a footprint coverage mask. - -Author: Martin Kilbinger - -""" - -import sys - -from shapepipe.utilities.ccd_psf_handler import CcdPsfHandler - - -def run_ccd_psf_handler(args=None): - """Run CCD PSF Handler. - - Create instance and run the CCD PSF handler. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code - - """ - # Create instance - - obj = CcdPsfHandler() - - return obj.run(args=args) - - -def main(argv=None): - """Main. - """ - # A scripts to call the ccd psf class is created by pyproject.toml - return 0 diff --git a/src/shapepipe/modules/merge_headers_package/__init__.py b/src/shapepipe/modules/merge_headers_package/__init__.py index ed8416790..cd47d6a3d 100644 --- a/src/shapepipe/modules/merge_headers_package/__init__.py +++ b/src/shapepipe/modules/merge_headers_package/__init__.py @@ -4,9 +4,10 @@ :Author: Axel Guinot -:Parent module: ``split_exp_runner`` +:Parent module: ``find_exposures_runner`` -:Input: Numpy binary files (``.npy``) with single-exposure header information +:Input: An ``exp_numbers`` text file listing the tile's exposures; the + per-exposure header files (``.npy``) are found under ``EXP_BASE_DIR`` :Output: Single SQL file with combined header information @@ -20,8 +21,10 @@ Module-specific config file entries =================================== -OUTPUT_PATH : str, optional - Overrides the default module output directory under ``[FILE]:OUTPUT_DIR`` +EXP_BASE_DIR : str + Root of the per-exposure work directories; the header ``.npy`` files are + collected from ``///`` for every exposure + listed in the input ``exp_numbers`` file """ diff --git a/src/shapepipe/modules/merge_headers_runner.py b/src/shapepipe/modules/merge_headers_runner.py index d7b0bd035..21a2ec08b 100644 --- a/src/shapepipe/modules/merge_headers_runner.py +++ b/src/shapepipe/modules/merge_headers_runner.py @@ -34,38 +34,32 @@ def merge_headers_runner( output_dir = run_dirs["output"] w_log.info(f"output_dir = {output_dir}") - if config.has_option(module_config_sec, "EXP_BASE_DIR"): - # Tile-level mode: input is an exp_numbers txt file; collect header - # files from each per-exposure work directory via get_exp_output_files. - exp_base_dir = config.getexpanded(module_config_sec, "EXP_BASE_DIR") + # The input is an exp_numbers txt file (find_exposures_runner); the header + # files themselves are collected from each per-exposure work directory + # under EXP_BASE_DIR via get_exp_output_files. + exp_base_dir = config.getexpanded(module_config_sec, "EXP_BASE_DIR") - # In serial mode several tiles' exp_numbers files can arrive in a - # single call; merge each tile into its own per-tile sqlite file. - for exp_numbers_file in (item[0] for item in input_file_list): - w_log.info( - f"Tile-level merge: collecting headers from {exp_base_dir} " - f"using {exp_numbers_file}" - ) - headers_file_list = get_exp_output_files( - exp_base_dir, - exp_numbers_file, - "split_exp_runner", - "headers", - ".npy", - w_log=w_log, - ) - # Extract tile number from the exp_numbers filename, e.g. - # "exp_numbers-284.272-1.000.txt" -> "-284.272-1.000" - base = os.path.splitext(os.path.basename(exp_numbers_file))[0] - tile_number = re.sub(r"^exp_numbers", "", base) - merge_headers(headers_file_list, output_dir, tile_number) - w_log.info( - f"Merged {len(headers_file_list)} exposure header files" - ) - else: - # Per-exposure mode: input_file_list already contains the header files. - merge_headers(input_file_list, output_dir) - w_log.info(f"Merged {len(input_file_list)} input file headers") + # In serial mode several tiles' exp_numbers files can arrive in a + # single call; merge each tile into its own per-tile sqlite file. + for exp_numbers_file in (item[0] for item in input_file_list): + w_log.info( + f"Collecting headers from {exp_base_dir} " + f"using {exp_numbers_file}" + ) + headers_file_list = get_exp_output_files( + exp_base_dir, + exp_numbers_file, + "split_exp_runner", + "headers", + ".npy", + w_log=w_log, + ) + # Extract tile number from the exp_numbers filename, e.g. + # "exp_numbers-284.272-1.000.txt" -> "-284.272-1.000" + base = os.path.splitext(os.path.basename(exp_numbers_file))[0] + tile_number = re.sub(r"^exp_numbers", "", base) + merge_headers(headers_file_list, output_dir, tile_number) + w_log.info(f"Merged {len(headers_file_list)} exposure header files") # No return objects return None, None diff --git a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py index 5ecdb6fb0..b0be88944 100644 --- a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py +++ b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py @@ -7,7 +7,6 @@ """ import os -import re import warnings import numpy as np @@ -16,6 +15,48 @@ from shapepipe.pipeline import file_io +def chunk_path(input_file, n): + """Chunk Path. + + Derive chunk ``n``'s input path from chunk 1's. + + The separate catalogues live in ShapePipe run directories whose names differ + only in the chunk number (``run_sp_tile_ngmix_Ng1u`` -> + ``run_sp_tile_ngmix_Ng2u``). The substitution is confined to that + run-directory component: replacing the first "1" found anywhere in the path + breaks for absolute paths whose parent directories carry digits, e.g. a + sharded store ``.../tiles/21/210.282/output/run_..._Ng1u/...``. + + Parameters + ---------- + input_file : str + Path to chunk 1's catalogue + n : int + Chunk number + + Returns + ------- + str + Path to chunk ``n``'s catalogue + + Raises + ------ + ValueError + If no run-directory component of the path carries a chunk number + + """ + parts = input_file.split(os.sep) + for idx in reversed(range(len(parts))): + if parts[idx].startswith("run_") and "1" in parts[idx]: + parts[idx] = parts[idx].replace("1", str(n), 1) + return os.sep.join(parts) + + raise ValueError( + f"Cannot derive chunk {n}'s path from '{input_file}': no 'run_*' " + + "directory component contains a chunk number '1'" + ) + + class MergeSep(object): """Merge Sep. @@ -79,8 +120,7 @@ def process(self): input_path_n = [] input_path_n.append(input_file) for n in range(2, self._n_split_max + 1): - res = re.sub("1", str(n), input_file, 1) - input_path_n.append(res) + input_path_n.append(chunk_path(input_file, n)) # Open first catalogue, read number of extensions and columns cat0 = file_io.FITSCatalogue(input_file, SEx_catalogue=True) diff --git a/src/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py index 927c79b76..6e99cf3ec 100644 --- a/src/shapepipe/modules/merge_sep_cats_runner.py +++ b/src/shapepipe/modules/merge_sep_cats_runner.py @@ -28,7 +28,7 @@ def merge_sep_cats_runner( ): """Define The Merge SEP Catalogues Runner.""" # Get config entries - n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") + n_split_max = int(config.getexpanded(module_config_sec, "N_SPLIT_MAX")) file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") file_ext = config.getlist(module_config_sec, "FILE_EXT") diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 89727caff..ef05bed76 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -43,15 +43,25 @@ (no batch saving) ID_OBJ_MIN : int ID of first galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, so an orchestrator can + set the object range per chunk, for example + ``ID_OBJ_MIN = $SP_NGMIX_ID_OBJ_MIN``. ID_OBJ_MAX : int ID of last galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, as for ``ID_OBJ_MIN``. BKG_RMS_VIGNET_PATH : str, optional Path to a ``background_rms_vignet*.sqlite`` file produced by ``vignetmaker_runner``. The string may contain ``{file_number_string}``, which is replaced by the current tile ID. +Random number generation +======================== + +Each object gets its own random number stream, seeded from its sky position +and CCD (``ngmix.position_seed``, which argues what that buys). It is the only +mode: the config option ``SEED_FROM_POSITION`` is obsolete, and setting it to +``False`` raises rather than silently changing the RNG. + """ __all__ = ["ngmix"] diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index e07e0dd7b..5ca65b9bc 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -137,7 +137,8 @@ def position_seed(ra, dec, ccd): difference, inflating ``sigma_m``. Seeding the per-object RNG from sky position instead makes the same object draw the same added noise (and the same fit guesses) in every branch, so both cancel and the m-bias error - shrinks. Off in production; a knob for the sim path only. + shrinks. It also makes the result independent of how the tile is split into + ``ID_OBJ_MIN``/``ID_OBJ_MAX`` chunks, which is why it is now the only mode. Box math (kept exactly as Fabian's issue #796):: @@ -299,8 +300,7 @@ def __init__( self.ra = [] self.dec = [] # CCD number of the first epoch, used only to build the per-object - # position seed (see :func:`position_seed`, ngmix#796). ``None`` when - # position seeding is off, so the field costs nothing on the hot path. + # position seed (see :func:`position_seed`). self.ccd = None self.bkg_sub = bkg_sub self.megacam_flip = megacam_flip @@ -439,13 +439,11 @@ class Ngmix(object): dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. - seed_from_position : bool, optional - If ``True``, replace the tile-level RNG with a per-object RNG seeded - from the object's sky position (:func:`position_seed`) inside the - object loop, so metacal's ``fixnoise`` counter-noise and the fit - guesses cancel across Pujol image-simulation branches (ngmix#796). The - default ``False`` leaves the production path byte-identical. See - :func:`position_seed` for the physics and the seed construction. + + Notes + ----- + The RNG is always per object and seeded from that object's sky position; + :func:`position_seed` says what that buys. Raises ------ @@ -474,7 +472,6 @@ def __init__( blend_handling="noisefill", seg_cat_path=None, dilate_neighbour=1, - seed_from_position=False, metacal_psf="fitgauss", ): @@ -544,15 +541,14 @@ def __init__( self._blend_handling = blend_handling self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour - self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf self._w_log = w_log - # Initiatlise random generator - seed = int(''.join(re.findall(r'\d+', self._file_number_string))) - self._rng = np.random.RandomState(seed) - self._w_log.info(f'Random generator initialisation seed = {seed}') + self._w_log.info( + 'Per-object RNG seeded from sky position (ngmix#796): results are' + ' invariant to how the tile is split into object chunks' + ) # Pixel scale: an explicit positive PIXEL_SCALE overrides; otherwise # derive it from the image WCS so it can never drift from the pixels @@ -573,12 +569,6 @@ def __init__( f'PIXEL_SCALE from config = {self._pixel_scale:.6f} arcsec' ) - if self._seed_from_position: - self._w_log.info( - 'SEED_FROM_POSITION on: per-object RNG seeded from sky position' - ' for Pujol noise cancellation (image sims, ngmix#796)' - ) - @classmethod def MegaCamFlip(self, vign, ccd_nb): """Flip for MegaCam. @@ -607,18 +597,6 @@ def MegaCamFlip(self, vign, ccd_nb): # swap y axis so origin is on bottom-left return vign - def get_prior(self, T_range=None, F_range=None): - """Get Prior. - - Returns - ------- - ngmix.joint_prior.PriorSimpleSep - """ - return get_prior( - self._pixel_scale, self._rng, - T_range=T_range, F_range=F_range, - ) - def compile_results(self, results): """Compile Results. @@ -1001,7 +979,6 @@ def process(self): vignet_cat = self._vignet_cat final_res = [] - prior = self.get_prior() count = 0 n_empty_cat = 0 @@ -1023,36 +1000,39 @@ def process(self): id_last = obj_id count += 1 - # Skip objects with no multi-epoch PSF or vignet data - if (vignet_cat.psf_vign_cat[str(obj_id)] == 'empty' - or vignet_cat.gal_vign_cat[str(obj_id)] == 'empty'): + # Skip objects with no multi-epoch PSF or vignet data. + # Read each store once here and pass the dicts down: every + # sqlitedict access unpickles the object's whole all-epoch dict. + psf_obj = vignet_cat.psf_vign_cat[str(obj_id)] + gal_obj = vignet_cat.gal_vign_cat[str(obj_id)] + if psf_obj == 'empty' or gal_obj == 'empty': n_empty_cat += 1 continue - stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat, self._bkg_sub) + stamp = prepare_postage_stamps( + vignet_cat, + obj_id, + i_tile, + tile_cat, + self._bkg_sub, + psf_obj, + gal_obj, + ) if len(stamp.gals) == 0: n_no_epoch += 1 continue - # Position-seeded per-object RNG for Pujol noise cancellation in - # image sims (ngmix#796): the same object gets the same fixnoise - # counter-noise and fit guesses in every shear branch, so both - # cancel in the branch difference. The prior is rebuilt from the - # same per-object RNG because the guesser draws its initial guess - # via prior.sample() (ngmix guessers.py), which consumes the RNG the - # prior was CONSTRUCTED with — so a per-object rng alone would leave - # the guess drawing from the shared tile stream and break - # cancellation. Off in production, where the single tile-level - # self._rng and the tile-level prior carry the whole loop. - if self._seed_from_position: - obj_rng = np.random.RandomState( - position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) - ) - obj_prior = get_prior(self._pixel_scale, obj_rng) - else: - obj_rng = self._rng - obj_prior = prior + # Per-object RNG, seeded from (ra, dec, ccd) — see + # :func:`position_seed`. The prior is rebuilt from that same RNG + # because the guesser draws its initial guess via prior.sample() + # (ngmix guessers.py), which consumes the RNG the prior was + # CONSTRUCTED with: a per-object rng alone would leave the guess + # drawing from a shared stream and break the invariance. + obj_rng = np.random.RandomState( + position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) + ) + obj_prior = get_prior(self._pixel_scale, obj_rng) try: flux_guess = ( @@ -1165,25 +1145,49 @@ def process(self): # Log mean ellipticity statistics self.log_mean_ellipticity() -def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): +def prepare_postage_stamps( + vignet, + obj_id, + i_tile, + tile_cat, + bkg_sub=True, + psf_obj=None, + gal_obj=None, +): # define per-object lists of individual exposures to go into ngmix stamp = Postage_stamp(bkg_sub=bkg_sub) + # Read each store's per-object dict ONCE: every sqlitedict access + # unpickles the object's whole all-epoch dict, so keeping these out of + # the epoch loop below saves O(n_epoch) full unpickles per store. + if psf_obj is None: + psf_obj = vignet.psf_vign_cat[str(obj_id)] + if gal_obj is None: + gal_obj = vignet.gal_vign_cat[str(obj_id)] + bkg_obj = ( + vignet.bkg_vign_cat[str(obj_id)] + if stamp.bkg_sub and vignet.bkg_vign_cat is not None + else None + ) + flag_obj = vignet.flag_vign_cat[str(obj_id)] + weight_obj = vignet.weight_vign_cat[str(obj_id)] + bkg_rms_obj = ( + vignet.bkg_rms_vign_cat[str(obj_id)] + if vignet.bkg_rms_vign_cat is not None + else None + ) + wcs_cache = {} #identify exposure and ccd number from psf catalog - psf_expccd_names = list(vignet.psf_vign_cat[str(obj_id)].keys()) + psf_expccd_names = list(psf_obj.keys()) for expccd_name in psf_expccd_names: exp_name, ccd_n = re.split('-', expccd_name) - gal_vign = ( - vignet.gal_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + gal_vign = gal_obj[expccd_name]['VIGNET'] if np.all(gal_vign == 0): continue if stamp.bkg_sub: - bkg_vign = ( - vignet.bkg_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + bkg_vign = bkg_obj[expccd_name]['VIGNET'] gal_vign_sub_bkg = background_subtract( gal_vign, bkg_vign @@ -1216,9 +1220,7 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): if stamp.megacam_flip and tile_seg is not None: tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) - flag_vign = ( - vignet.flag_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + flag_vign = flag_obj[expccd_name]['VIGNET'] if tile_vign is not None: flag_vign[np.where(tile_vign == -1e30)] = 2**10 v_flag_tmp = flag_vign.ravel() @@ -1226,23 +1228,27 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): if len(np.where(v_flag_tmp != 0)[0]) / v_flag_tmp.size > 1 / 3.0: continue - weight_vign = vignet.weight_vign_cat[str(obj_id)][expccd_name]['VIGNET'] + weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( - vignet.bkg_rms_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - if vignet.bkg_rms_vign_cat is not None + bkg_rms_obj[expccd_name]['VIGNET'] + if bkg_rms_obj is not None else None ) - epoch_wcs = vignet.f_wcs_file[exp_name][int(ccd_n)]['WCS'] + # One unpickle per exposure (all CCDs), reused across this object's + # epochs; the cache is per-call, so bounded by the object's exposures + if exp_name not in wcs_cache: + wcs_cache[exp_name] = vignet.f_wcs_file[exp_name] + ccd_wcs = wcs_cache[exp_name][int(ccd_n)] + + epoch_wcs = ccd_wcs['WCS'] jacob = get_galsim_jacobian( epoch_wcs, tile_cat.ra[i_tile], tile_cat.dec[i_tile] ) - header = fits.Header.fromstring( - vignet.f_wcs_file[exp_name][int(ccd_n)]['header'] - ) + header = fits.Header.fromstring(ccd_wcs['header']) # rescale by relative zero-points ( @@ -1258,9 +1264,7 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): # gather postage stamps in all of the epochs stamp.gals.append(gal_vign_scaled) - stamp.psfs.append( - vignet.psf_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + stamp.psfs.append(psf_obj[expccd_name]['VIGNET']) stamp.weights.append(weight_vign_scaled) stamp.flags.append(flag_vign) stamp.bkg_rms.append(bkg_rms_vign_scaled) @@ -1272,8 +1276,8 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): stamp.ra.append(tile_cat.ra[i_tile]) stamp.dec.append(tile_cat.dec[i_tile]) # CCD of the first surviving epoch — Fabian's coord_list[0] convention - # for the position seed (ngmix#796). All epochs of one object share the - # ra/dec above, so first-epoch CCD pins one deterministic seed stream. + # for the position seed. All epochs of one object share the ra/dec + # above, so first-epoch CCD pins one deterministic seed stream. if stamp.ccd is None: stamp.ccd = int(ccd_n) @@ -1574,8 +1578,8 @@ def prepare_ngmix_weights( weight : numpy.ndarray flag : numpy.ndarray rng : numpy.random.RandomState - Random state for the noise realisations (seeded per tile for - reproducibility). + Random state for the noise realisations (seeded per object; see + :func:`position_seed`). bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. If supplied, unmasked pixels use ``1 / bkg_rms**2`` as the ngmix inverse variance. @@ -1703,8 +1707,8 @@ def make_ngmix_observation( wcs : galsim.BaseWCS Local WCS Jacobian at the object position. rng : numpy.random.RandomState - Random state for the noise realisations (seeded per tile for - reproducibility). + Random state for the noise realisations (seeded per object; see + :func:`position_seed`). bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. centroid_source : {"hsm", "wcs"}, optional diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 16403a543..072288e00 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -109,9 +109,12 @@ def ngmix_runner( # No batch saving save_batch = -1 - # First and last galaxy ID to process - id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") - id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") + # First and last galaxy ID to process. Read via ``getexpanded`` so an + # orchestrator can drive the chunk bounds from environment variables + # (``$SP_NGMIX_ID_OBJ_MIN`` and friends); ``getexpanded`` is the only + # accessor in ShapePipe's config that expands ``$VAR``. + id_obj_min = int(config.getexpanded(module_config_sec, "ID_OBJ_MIN")) + id_obj_max = int(config.getexpanded(module_config_sec, "ID_OBJ_MAX")) # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) @@ -139,16 +142,17 @@ def ngmix_runner( else: dilate_neighbour = 1 - # Seed the per-object RNG from sky position instead of per tile, so - # metacal's fixnoise counter-noise (and the fit guesses) cancel across - # Pujol image-simulation shear branches (ngmix#796). Default False leaves - # the production path byte-identical. + # Position-seeded RNG is the only mode (see ngmix.position_seed). Old + # configs that disable it must fail loudly, not silently change the RNG. if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - seed_from_position = config.getboolean( - module_config_sec, "SEED_FROM_POSITION" - ) - else: - seed_from_position = False + if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): + raise ValueError( + "SEED_FROM_POSITION = False is no longer supported: the" + " tile-seeded RNG mode has been retired because it makes" + " results depend on the object chunking. Remove the" + " SEED_FROM_POSITION entry from the ngmix config section" + " (position-seeded RNG is now the only mode)." + ) # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. @@ -204,7 +208,6 @@ def ngmix_runner( blend_handling=blend_handling, seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, - seed_from_position=seed_from_position, metacal_psf=metacal_psf, ) diff --git a/src/shapepipe/modules/psfex_interp_package/__init__.py b/src/shapepipe/modules/psfex_interp_package/__init__.py index 1c3b51ca4..2b6a161b1 100644 --- a/src/shapepipe/modules/psfex_interp_package/__init__.py +++ b/src/shapepipe/modules/psfex_interp_package/__init__.py @@ -32,14 +32,18 @@ Threshold of stars under which the PSF is not interpolated CHI2_THRESH : int Threshold for chi squared (:math:`\chi^2`) -ME_DOT_PSF_DIR : str - Module name of last run producing PSFEx PSF model files, for multi-epoch - processing. The specifier "last:" is not required +ME_DOT_PSF_EXP_DIR : str + Root of the per-exposure work directories, for multi-epoch processing; + the ``psfex_runner`` output directory of every exposure listed in the + input ``exp_numbers`` file is resolved beneath it ME_DOT_PSF_PATTERN : str Input file name pattern for PSFEx PSF model files, for multi-epoch processing -ME_LOG_WCS : str - Path to world coordinate system log file (``*sqlite``) + +In ``MULTI-EPOCH`` mode the input file list is positional and carries three +entries -- the galaxy catalogue, the world-coordinate-system log +(``*sqlite``) and the ``exp_numbers`` file -- set through ``FILE_PATTERN`` +and ``FILE_EXT``, not through module-specific keys. """ diff --git a/src/shapepipe/modules/psfex_interp_runner.py b/src/shapepipe/modules/psfex_interp_runner.py index 67c21adfc..5627d5d20 100644 --- a/src/shapepipe/modules/psfex_interp_runner.py +++ b/src/shapepipe/modules/psfex_interp_runner.py @@ -10,7 +10,6 @@ from shapepipe.modules.psfex_interp_package import psfex_interp from shapepipe.pipeline.exp_utils import get_exp_output_dirs -from shapepipe.pipeline.run_log import get_last_dir, get_all_dirs @module_runner( @@ -62,38 +61,22 @@ def psfex_interp_runner( # Run in MULTI-EPOCH mode elif mode == "MULTI-EPOCH": - # Fetch multi-epoch parameters - if config.has_option(module_config_sec, "ME_DOT_PSF_EXP_DIR"): - # v2.0: locate psfex_runner output dirs via the $SP_EXP tree - exp_base_dir = config.getexpanded( - module_config_sec, "ME_DOT_PSF_EXP_DIR" - ) - if len(input_file_list) < 3: - raise ValueError( - "ME_DOT_PSF_EXP_DIR requires the exposure-numbers file" - + " as a third input; add 'exp_numbers' to FILE_PATTERN" - + f" and FILE_EXT in the [{module_config_sec}] config" - + " section." - ) - exp_numbers_file = input_file_list[2] - dot_psf_dirs = get_exp_output_dirs( - exp_base_dir, exp_numbers_file, "psfex_runner", w_log - ) - else: - module = config.getexpanded( - module_config_sec, - "ME_DOT_PSF_DIR", + # Fetch multi-epoch parameters. The psfex_runner output dirs are + # located through the per-exposure work tree. + exp_base_dir = config.getexpanded( + module_config_sec, "ME_DOT_PSF_EXP_DIR" + ) + if len(input_file_list) < 3: + raise ValueError( + "ME_DOT_PSF_EXP_DIR requires the exposure-numbers file" + + " as a third input; add 'exp_numbers' to FILE_PATTERN" + + f" and FILE_EXT in the [{module_config_sec}] config" + + " section." ) - module_name = module.split(":")[-1] - if "last" in module: - dot_psf_dirs = [get_last_dir(run_dirs["run_log"], module_name)] - elif "all" in module: - dot_psf_dirs = get_all_dirs(run_dirs["run_log"], module_name) - else: - raise ValueError( - "Expected qualifier 'last:' or 'all' before module" - + f" '{module}' in config entry 'ME_DOT_PSF_DIR'" - ) + exp_numbers_file = input_file_list[2] + dot_psf_dirs = get_exp_output_dirs( + exp_base_dir, exp_numbers_file, "psfex_runner", w_log + ) dot_psf_pattern = config.get( module_config_sec, diff --git a/src/shapepipe/modules/setools_package/setools.py b/src/shapepipe/modules/setools_package/setools.py index c430b52b1..33d8efa8b 100644 --- a/src/shapepipe/modules/setools_package/setools.py +++ b/src/shapepipe/modules/setools_package/setools.py @@ -658,13 +658,18 @@ def _make_rand_split(self): cat_size = len(np.where(mask)[0]) n_keep = int(np.ceil(cat_size * ratio)) - mask_ratio = [] - mask_left = list(range(0, cat_size)) - while len(mask_ratio) != n_keep: - idx = np.random.randint(0, len(mask_left)) - mask_ratio.append(mask_left.pop(idx)) - mask_ratio = np.array(mask_ratio) - mask_left = np.array(mask_left) + # Deterministic split, seeded from the unit's file number: the + # train/validation assignment is a pure function of the input + # catalogue, so the PSF star sample (and everything downstream + # of the PSF model) is reproducible run-to-run. An unseeded + # np.random here made the shear catalogue non-reproducible + # upstream of ngmix's own position seeding. + seed = int( + re.sub(r"\D", "", self._file_number_string) or 0 + ) % (2 ** 32) + perm = np.random.RandomState(seed).permutation(cat_size) + mask_ratio = perm[:n_keep] + mask_left = np.sort(perm[n_keep:]) self.rand_split[key]["mask"] = mask self.rand_split[key][f"ratio_{int(ratio * 100)}"] = mask_ratio self.rand_split[key][f"ratio_{100 - int(ratio * 100)}"] = mask_left diff --git a/src/shapepipe/modules/vignetmaker_package/__init__.py b/src/shapepipe/modules/vignetmaker_package/__init__.py index b24f8397e..27c679c23 100644 --- a/src/shapepipe/modules/vignetmaker_package/__init__.py +++ b/src/shapepipe/modules/vignetmaker_package/__init__.py @@ -35,15 +35,22 @@ Run mode for module, options are ``CLASSIC`` or ``MULTI-EPOCH`` PREFIX : str or list Output file name prefix(es) -ME_IMAGE_DIR : list - Module names of last run producing single-exposure flags, images, weights, - and SExtractor background images, for multi-epoch processing. The specifier - "last:" is not required +ME_IMAGE_EXP_DIR : str + Root of the per-exposure work directories, for multi-epoch processing; + each runner output directory is resolved beneath it for every exposure + listed in the input ``exp_numbers`` file +ME_IMAGE_EXP_RUNNERS : list + Names of the runners producing the single-exposure flags, images, weights + and SExtractor background images, in the same order as + ``ME_IMAGE_PATTERN`` ME_IMAGE_PATTERN : list Input file name patterns for flag, image, weight, and SExtractor background files, for multi-epoch processing -ME_LOG_WCS : str - Path to world coordinate system log file (``*sqlite``) + +In ``MULTI-EPOCH`` mode the input file list is positional and carries three +entries -- the galaxy catalogue, the world-coordinate-system log +(``*sqlite``) and the ``exp_numbers`` file -- set through ``FILE_PATTERN`` +and ``FILE_EXT``, not through module-specific keys. """ diff --git a/src/shapepipe/modules/vignetmaker_runner.py b/src/shapepipe/modules/vignetmaker_runner.py index 2d9661faa..bad5a22df 100644 --- a/src/shapepipe/modules/vignetmaker_runner.py +++ b/src/shapepipe/modules/vignetmaker_runner.py @@ -10,7 +10,6 @@ from shapepipe.modules.vignetmaker_package import vignetmaker as vm from shapepipe.pipeline.exp_utils import get_exp_output_dirs -from shapepipe.pipeline.run_log import get_last_dir, get_all_dirs @module_runner( @@ -105,44 +104,27 @@ def vignetmaker_runner( elif mode == "MULTI-EPOCH": # Multi-epoch exposures - if config.has_option(module_config_sec, "ME_IMAGE_EXP_DIR"): - # v2.0: locate runner output dirs via the $SP_EXP tree - exp_base_dir = config.getexpanded( - module_config_sec, "ME_IMAGE_EXP_DIR" + # Locate the runner output dirs through the per-exposure work tree. + exp_base_dir = config.getexpanded( + module_config_sec, "ME_IMAGE_EXP_DIR" + ) + if len(input_file_list) < 3: + raise ValueError( + "ME_IMAGE_EXP_DIR requires the exposure-numbers" + + " file as a third input; add 'exp_numbers' to" + + " FILE_PATTERN and FILE_EXT in the" + + f" [{module_config_sec}] config section." ) - if len(input_file_list) < 3: - raise ValueError( - "ME_IMAGE_EXP_DIR requires the exposure-numbers" - + " file as a third input; add 'exp_numbers' to" - + " FILE_PATTERN and FILE_EXT in the" - + f" [{module_config_sec}] config section." - ) - exp_numbers_file = input_file_list[2] - exp_runner_names = config.getlist( - module_config_sec, "ME_IMAGE_EXP_RUNNERS" + exp_numbers_file = input_file_list[2] + exp_runner_names = config.getlist( + module_config_sec, "ME_IMAGE_EXP_RUNNERS" + ) + image_dirs = [] + for runner_name in exp_runner_names: + dirs = get_exp_output_dirs( + exp_base_dir, exp_numbers_file, runner_name, w_log ) - image_dirs = [] - for runner_name in exp_runner_names: - dirs = get_exp_output_dirs( - exp_base_dir, exp_numbers_file, runner_name, w_log - ) - image_dirs.append(dirs) - else: - # v1: run-log based lookup via symlinked exposure run dirs - modules = config.getlist(module_config_sec, "ME_IMAGE_DIR") - image_dirs = [] - for module in modules: - module_name = module.split(":")[-1] - if "last" in module: - dirs = [get_last_dir(run_dirs["run_log"], module_name)] - elif "all" in module: - dirs = get_all_dirs(run_dirs["run_log"], module_name) - else: - raise ValueError( - "Expected qualifier 'last:' or 'all' before module" - + f" '{module}' in config entry 'ME_IMAGE_DIR'" - ) - image_dirs.append(dirs) + image_dirs.append(dirs) image_pattern = config.getlist( module_config_sec, diff --git a/src/shapepipe/pipeline/exp_utils.py b/src/shapepipe/pipeline/exp_utils.py index b934f0bb3..889ff31c1 100644 --- a/src/shapepipe/pipeline/exp_utils.py +++ b/src/shapepipe/pipeline/exp_utils.py @@ -1,11 +1,10 @@ """EXPOSURE UTILITIES. Utility functions for accessing per-exposure runner outputs from the -tile level. In the v2.0 pipeline, each single exposure is processed in -its own work directory ``///``. Tile-level -modules that need files produced by a per-exposure runner use the -functions here to discover those files by scanning the contributing -exposure directories. +tile level. Each single exposure is processed in its own work directory +``///``. Tile-level modules that need files +produced by a per-exposure runner use the functions here to discover those +files by scanning the contributing exposure directories. :Author: Martin Kilbinger @@ -40,7 +39,7 @@ def get_exp_output_files( ---------- exp_base_dir : str Root directory that contains all per-exposure work directories, - e.g. ``/arc/home/kilbinger/v2.0/exp`` + e.g. ``/exp`` exp_numbers_file : str Path to a text file listing one exposure ID per line, e.g. ``exp_numbers-301-279.txt`` @@ -91,7 +90,8 @@ def get_exp_output_files( missing = [] for exp_id in exp_ids: - # Directory structure mirrors run_job_canfar_v2.0.sh: + # Directory structure mirrors the workflow's per-exposure store, built + # by rule tile_exp_forest / workflow/scripts/build_forest.py: # exp_prefix = first 2 chars of exp_id (e.g. "21") # exp_base = exp_id without trailing letter if present (e.g. "2113864"), # or full exp_id for numeric-only ids (image sims) diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 962953ede..fee0dad3e 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -84,7 +84,7 @@ def _set_run_name(self): Set the name of the current pipeline run. """ - self._run_name = self.config.get("DEFAULT", "RUN_NAME") + self._run_name = self.config.getexpanded("DEFAULT", "RUN_NAME") if self.config.getboolean("DEFAULT", "RUN_DATETIME"): self._run_name += datetime.now().strftime("_%Y-%m-%d_%H-%M-%S") diff --git a/src/shapepipe/summary_run.py b/src/shapepipe/summary_run.py deleted file mode 100755 index 39d60ce08..000000000 --- a/src/shapepipe/summary_run.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -import argparse -import sys -import os - -from shapepipe.utilities import summary - -from shapepipe.utilities import summary_params_pre_v2 as summary_params - - -def run(patch, job_exclusive=None, verbose=False): - - jobs, list_tile_IDs_dot = summary_params.set_jobs_v2_pre_v2(patch, verbose) - - list_tile_IDs = summary.job_data.replace_dot_dash(list_tile_IDs_dot) - - # Numbers updated at runtime - par_runtime = summary.init_par_runtime(list_tile_IDs) - - summary.job_data.print_stats_header() - - exp_IDs_path = "exp_numbers.txt" - if os.path.exists(exp_IDs_path): - # Read exposure ID list if file exists - all_exposures = summary.get_IDs_from_file(exp_IDs_path) - par_runtime = summary.update_par_runtime_after_find_exp( - par_runtime, all_exposures - ) - - if ( - not os.path.exists(exp_IDs_path) - or not job_exclusive - or int(job_exclusive) & 1 - ): - # Run job 1 if exposure ID list file does not exist or - # job_exclusive is 1 or not set - key = "1" - jobs[key].print_intro() - jobs[key].check_numbers(par_runtime=par_runtime, indices=[0, 1]) - - all_exposures = summary.get_all_exposures( - jobs[key]._paths_in_dir[1], verbose=True - ) - par_runtime = summary.update_par_runtime_after_find_exp( - par_runtime, all_exposures - ) - - jobs[key].write_IDs_to_file("exp_numbers.txt", all_exposures) - - jobs[key].check_numbers(par_runtime, indices=[2]) - - summary.print_par_runtime(par_runtime, verbose=verbose) - - # Get all keys after "1" - keys = sorted(jobs.keys(), key=int) - _ = keys.pop(0) - - for key in keys: - if job_exclusive and not int(key) & int(job_exclusive): - continue - jobs[key].print_intro() - jobs[key].check_numbers(par_runtime=par_runtime) - - return 0 - - -def parse_args(argv=None): - parser = argparse.ArgumentParser( - description=( - 'Print summary of ShapePipe job status for a given UNIONS patch.' - ), - ) - parser.add_argument( - 'patch', - help='Patch identifier (e.g. P3, P9).', - ) - parser.add_argument( - 'job_exclusive', - nargs='?', - default=None, - help=( - 'Bitmask selecting which jobs to run; ' - + 'if omitted, all jobs run.' - ), - ) - parser.add_argument( - '-v', - '--verbose', - action='store_true', - help='Verbose output.', - ) - return parser.parse_args(argv) - - -def main(argv=None): - args = parse_args(argv) - run(args.patch, args.job_exclusive, args.verbose) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/shapepipe/utilities/__init__.py b/src/shapepipe/utilities/__init__.py index 5f0eab2d6..15e59739c 100644 --- a/src/shapepipe/utilities/__init__.py +++ b/src/shapepipe/utilities/__init__.py @@ -7,4 +7,4 @@ """ -__all__ = ["file_system", "cfis", "galaxy", "summary"] +__all__ = ["file_system", "cfis", "galaxy"] diff --git a/src/shapepipe/utilities/ccd_footprint.py b/src/shapepipe/utilities/ccd_footprint.py new file mode 100644 index 000000000..41cff7612 --- /dev/null +++ b/src/shapepipe/utilities/ccd_footprint.py @@ -0,0 +1,92 @@ +"""CCD_FOOTPRINT + +Project a single CCD onto the sky: image shape from a header, sky corners +from a WCS. + +The two functions here are the geometry the coverage mask rests on, and they +are all that survives of the pre-Snakemake header scrape. The ``exp_footprint`` +rule (``workflow/scripts/exp_footprint.py``) calls them once per CCD of an +exposure, reading the headers out of ``headers-.npy`` rather than out of +text files downloaded from VOSpace; the ``coverage_map`` rule then stamps the +corners into the campaign's HealSparse map. + +Author: Mike Hudson, Martin Kilbinger + +""" + + +def _image_shape(header): + """Return the CCD image ``(nx, ny)`` pixel shape from a header. + + For fpack tile-compressed HDUs (``ZIMAGE = T``), ``NAXIS1``/``NAXIS2`` + describe the compressed *binary table* (row width in bytes, row count), + not the image, and astropy's WCS never maps the true dimensions into + ``pixel_shape``. The true image size is carried by ``ZNAXIS1``/``ZNAXIS2``, + which are preferred here; a plain image HDU falls back to + ``NAXIS1``/``NAXIS2``. + + Both branches are live. ``headers-.npy`` carries the *decompressed* + header astropy hands back for a tile-compressed HDU, so it takes the plain + ``NAXIS`` path; a header read straight off a fpacked file takes the ``Z*`` + one. + + Parameters + ---------- + header : astropy.io.fits.Header + single-HDU header + + Returns + ------- + tuple + ``(nx, ny)`` image pixel dimensions + + Raises + ------ + ValueError + if neither the ``Z*`` nor the plain ``NAXIS`` dimensions are present + """ + if header.get("ZIMAGE", False): + keys = ("ZNAXIS1", "ZNAXIS2") + else: + keys = ("NAXIS1", "NAXIS2") + + if keys[0] not in header or keys[1] not in header: + raise ValueError( + f"Header is missing image dimensions ({keys[0]}/{keys[1]})" + ) + + return int(header[keys[0]]), int(header[keys[1]]) + + +def _ccd_corners(w, shape): + """Return (ra, dec) of the 4 corners of a single CCD. + + The polygon is built from the CCD's pixel bounds ``shape = (nx, ny)`` using + pixel edges ``(-0.5, n - 0.5)`` so the quadrilateral covers the full CCD + area rather than pixel centres. Corners are returned in a consistent + counterclockwise pixel order (bottom-left, bottom-right, top-right, + top-left); since ``pixel_to_world`` maps this order to a simple, + non-self-intersecting sky quadrilateral for a CCD-sized field, and + HealSparse ``Polygon`` is orientation-agnostic, the resulting polygon is a + valid convex footprint. + + Parameters + ---------- + w : astropy.wcs.WCS + WCS of a single CCD + shape : tuple + ``(nx, ny)`` CCD image pixel dimensions + + Returns + ------- + tuple + ``(ra_list, dec_list)`` each of length 4, in degrees + """ + nx, ny = shape + + # Pixel-edge corners, counterclockwise: BL, BR, TR, TL + px = [-0.5, nx - 0.5, nx - 0.5, -0.5] + py = [-0.5, -0.5, ny - 0.5, ny - 0.5] + + sky = w.pixel_to_world(px, py) + return list(sky.ra.deg), list(sky.dec.deg) diff --git a/src/shapepipe/utilities/ccd_psf_handler.py b/src/shapepipe/utilities/ccd_psf_handler.py deleted file mode 100644 index 2d0dc90eb..000000000 --- a/src/shapepipe/utilities/ccd_psf_handler.py +++ /dev/null @@ -1,387 +0,0 @@ -"""CCD_PSF_HANDLER - -Obtain list of CCDs (single-exposure single-HDU files) for which valid PSF -information is available. This can serve to create a footprint coverage mask. - -Author: Mike Hudson, Martin Kilbinger - -""" - -import glob -import os -import re -import sys - -import numpy as np - -from cs_util import args as cs_args -from cs_util import logging - -from shapepipe.utilities import summary - - -class CcdPsfHandler(object): - """CCD PSF Handler Class. - - Handles extraction of CCDs with valid PSF information from shapepipe - patches. - """ - - def __init__(self): - """Initialize the handler.""" - self.params_default() - - def params_default(self): - """Set default parameters and command line options.""" - - self._params = { - "version_cat": "v1.6", - "n_CCD": 40, - "output": None, - } - - self._short_options = { - "version_cat": "-V", - "n_CCD": "-n", - "output": "-o", - } - - self._types = { - "n_CCD": "int", - } - - self._help_strings = { - "version_cat": "catalogue major version, allowed are v1.3, v1.4, v1.5, v1.6; default is {}", - - "n_CCD": "number of CCDs per exposure; default is {}", - "output": "output file path; default is ccds_with_psf_.txt", - } - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - Parameters - ---------- - args : list - command line arguments - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - args=args, - ) - self._params = options - - # Save calling command - logging.log_command(args) - - def update_params(self): - """Update parameters. - - Set derived parameters based on input parameters. - """ - # Determine number of patches based on version - version = self._params["version_cat"] - if version in ("v1.3", "v1.4"): - n_patch = 7 - elif version == "v1.5": - n_patch = 8 - elif version == "v1.6": - n_patch = 9 - else: - raise ValueError(f"Invalid version {version}") - - self._params["n_patch"] = n_patch - self._params["patches"] = [f"P{x}" for x in np.arange(n_patch) + 1] - - # Set output file if not specified - if self._params["output"] is None: - self._params["output"] = f"ccds_with_psf_{version}.txt" - - def check_params(self): - """Check parameters for validity.""" - # Add any parameter validation logic here - pass - - def get_lines(self, fname): - """Get Lines. - - Return list of lines read from a text file. - - Parameters - ---------- - fname : str - input file name - - Returns - ------- - list - IDs - - """ - IDs = [] - with open(fname) as f: - lines = f.readlines() - for line in lines: - IDs.append(line.rstrip()) - - return IDs - - def get_exp_shdu_missing(self, patches): - """Get Exp Shdu Missing. - - Returns set of missing CCDs (single-exposure single-HDU IDs) from a list of patches. - - Parameters - ---------- - patches : list - input patches - - Returns - ------- - set - missing CCD IDs - - """ - exp_shdu_missing_all = set() - - for patch in patches: - - path_exp_shdu_missing = f"{patch}/summary/missing_job_32_all.txt" - exp_shdu_missing = self.get_lines(path_exp_shdu_missing) - - print( - f"Patch {patch}: Found {len(exp_shdu_missing)} missing ccds", - end="; ", - ) - - exp_shdu_missing_all.update(exp_shdu_missing) - - print(f"cumulative {len(exp_shdu_missing_all)} missing ccds") - - print() - - return exp_shdu_missing_all - - def get_exp(self, patches): - """Get Exp. - - Return set of exposures from a list of patches. - - Parameters - ---------- - patches : list - input patches - - Returns - ------- - set - exposure IDs - - """ - exp_all = set() - - for patch in patches: - - path_exp = f"{patch}/exp_numbers.txt" - exp = self.get_lines(path_exp) - - print(f"Patch {patch}: Found {len(exp)} exposures", end="; ") - - exp_all.update(exp) - - print(f"cumulative {len(exp_all)} exposures") - - print() - - return exp_all - - def get_ccds_with_psf(self, patches, n_CCD=40): - """Get CCDs With PSF. - - Return set of CCDs with valid PSF from a list of patches. A CCD has a - valid PSF if it appears among the exposures' single-HDU IDs but not in - the per-patch missing-CCD lists. - - Parameters - ---------- - patches : list - input patches - n_CCD : int - number of CCDs per exposure - - Returns - ------- - set - CCD IDs with valid PSF - - """ - # Get missing CCDs - print("=== get missing CCDs ===") - exp_shdu_missing_all = self.get_exp_shdu_missing(patches) - - # Get all exposures used in tiles - print("=== get exposures ===") - exp_all = self.get_exp(patches) - - # Turn exposures into exposure-single-HDU names (CCDs) - exp_shdu_all = set(summary.get_all_shdus(exp_all, n_CCD)) - - # Subtract the CCDs whose PSF model is missing - exp_shdu_with_psf = exp_shdu_all - exp_shdu_missing_all - - print( - f"Found {len(exp_shdu_all)} CCDs, " - f"{len(exp_shdu_missing_all)} missing, " - f"{len(exp_shdu_with_psf)} with valid PSF" - ) - - return exp_shdu_with_psf - - def get_ccds_with_psf_method_v1_3(self, patches, n_CCD=40): - """Get CCDs With PSF Method v1.3. - - Return set of CCDs with valid PSF by scanning run directories. - Finds star_selection files and checks corresponding stat files - for valid (non-nan) FWHM values. - - Parameters - ---------- - patches : list - input patches - n_CCD : int - number of CCDs per exposure (unused, kept for API consistency) - - Returns - ------- - set - CCD IDs with valid PSF - - """ - exp_shdu_all = set() - - for patch in patches: - # Find all star_selection files - mask_pattern = ( - f"{patch}/output/run_sp_exp_SxSePs*/" - f"setools_runner/output/mask/star_selection-*.fits" - ) - mask_files = glob.glob(mask_pattern) - - print( - f"Patch {patch}: Found {len(mask_files)} star_selection FITS" - + " files" - ) - - for mask_file in mask_files: - # Extract exp_shdu from filename - basename = os.path.basename(mask_file) - match = re.match(r"star_selection-(.+)\.fits", basename) - if not match: - print(f"Warning: Non-matching file name {basename}") - continue - exp_shdu = match.group(1) - - # Get corresponding stat file path - # Replace mask dir with stat/stat dir in the path - stat_file = mask_file.replace( - "/mask/star_selection-", - "/stat/star_stat-", - ).replace(".fits", ".txt") - - #print("MKDEBUG ", exp_shdu, stat_file) - - # Check if stat file exists and has valid FWHM - if not os.path.exists(stat_file): - print("MKDEBUG stat file ", stat_file, " does not exist") - continue - - # Read stat file and check for nan in "Mean star fwhm selected" - has_nan = False - with open(stat_file) as f: - for line in f: - if "Mean star fwhm selected" in line: - if "nan" in line.lower(): - has_nan = True - break - - if not has_nan: - #print("MKDEBUG append", exp_shdu) - exp_shdu_all.add(exp_shdu) - - print(f"After patch {patch}: {len(exp_shdu_all)} CCDs with valid PSF") - - print(f"Found {len(exp_shdu_all)} CCDs with valid PSF") - - return exp_shdu_all - - def save(self, IDs, path): - """Save. - - Save list of IDs to text file. - - Parameters - ---------- - IDs : set - input IDs - path : str - output file name - - """ - with open(path, "w") as f_out: - for ID in IDs: - print(ID, file=f_out) - - def run(self, args=None): - """Run. - - Main execution method. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code (0 for success) - - """ - if args is None: - args = sys.argv[1:] - - # Set parameters from command line - self.set_params_from_command_line(args) - self.update_params() - self.check_params() - - # Get parameters - patches = self._params["patches"] - version = self._params["version_cat"] - n_CCD = self._params["n_CCD"] - output = self._params["output"] - - print( - f"=== get_ccds_with_psf for version {version}, patches {patches} ===" - ) - - if self._params["version_cat"] == "v1.3": - print("=== method for 1.3: exp with PSF star sample === ") - exp_shdu_all = self.get_ccds_with_psf_method_v1_3(patches, n_CCD) - else: - print("=== method for >= 1.4: exp_list - missing === ") - exp_shdu_all = self.get_ccds_with_psf(patches, n_CCD) - - self.save(exp_shdu_all, output) - - print(f"Results saved to {output}") - - return 0 diff --git a/src/shapepipe/utilities/coverage_map_builder.py b/src/shapepipe/utilities/coverage_map_builder.py index c7144ef5c..659a0d71f 100644 --- a/src/shapepipe/utilities/coverage_map_builder.py +++ b/src/shapepipe/utilities/coverage_map_builder.py @@ -1,25 +1,27 @@ """COVERAGE_MAP_BUILDER -Build HealSparse coverage maps from per-CCD corner coordinates. +Stamp per-CCD sky footprints into a HealSparse coverage (nexp) map. -Each input row is one CCD footprint. Since the CCDs of a single exposure do -not overlap, stamping value 1 per CCD polygon and accumulating makes the map -count, per sky pixel, the number of exposures with a valid PSF model covering -that pixel. +Each polygon is one CCD footprint. Since the CCDs of a single exposure do not +overlap, stamping value 1 per CCD polygon and accumulating makes the map count, +per sky pixel, the number of exposures with a valid PSF model covering that +pixel. + +Arrays in, map out. The only caller is the ``coverage_map`` rule +(``workflow/scripts/coverage_map.py``), which reads the corners out of the +per-exposure ``exp_footprint.json`` records; the nine-column ``exp_ra_dec.txt`` +the pre-Snakemake chain appended to is gone, and so is the CLI that parsed it. +What is left here is survey geometry — the RA-seam guard, the pole guard, the +nside validation and the median smoothing — none of which depends on how the +corners arrived. Author: Mike Hudson, Martin Kilbinger """ -import sys -from os.path import exists - -import numpy as np import healsparse as hsp import hpgeom as hpg - -from cs_util import args as cs_args -from cs_util import logging +import numpy as np # Declination beyond which a polygon is considered too close to a pole for # HealSparse's planar polygon fill to be reliable. CCD footprints are ~10 @@ -52,162 +54,134 @@ def unwrap_ra(ra): return ra +def check_nside(nside_coverage, nside): + """Raise unless both HealSparse resolutions are positive powers of two. + + Checked before any polygon is stamped: healsparse itself complains only + once a map is being made, by which time the caller has already paid for + reading every footprint record on the products root. -class CoverageMapBuilder(object): - """Coverage Map Builder Class. + Parameters + ---------- + nside_coverage : int + HealSparse coverage nside + nside : int + HealSparse map nside + + Raises + ------ + ValueError + if either value is not a positive power of two + + """ + for name, n in (("nside_coverage", nside_coverage), ("nside", nside)): + if n <= 0 or n & (n - 1): + raise ValueError(f"{name} must be a power of 2, got {n}") + + +def build_map( + ccd_ids, ra, dec, nside_coverage, nside, verbose=False +): + """Stamp one polygon per CCD footprint into a HealSparse nexp map. + + Since the CCDs of a single exposure do not overlap, accumulating value 1 + per CCD polygon makes the map count, per sky pixel, the number of + exposures with a valid PSF model covering it. + + Parameters + ---------- + ccd_ids : array_like + length-N CCD IDs, used only in the skipped-polygon warnings + ra : array_like + ``(N, 4)`` polygon RA corners, in degrees + dec : array_like + ``(N, 4)`` polygon Dec corners, in degrees + nside_coverage : int + HealSparse coverage nside + nside : int + HealSparse map nside + verbose : bool, optional + print progress + + Returns + ------- + healsparse.HealSparseMap + the nexp map - Builds HealSparse coverage maps from field corner coordinates. """ + check_nside(nside_coverage, nside) - def __init__(self): - """Initialize the builder.""" - self.params_default() - - def params_default(self): - """Set default parameters and command line options.""" - - self._params = { - "input_file": "exp_ra_dec.txt", - "output_file": "coverage.hsp", - "nside_coverage": 32, - "nside": 2048, - "apply_median_filter": False, - "n_median_iterations": 2, - "create_boolean": False, - "boolean_threshold": 3, - "boolean_output": None, - "create_plot": False, - "plot_output": None, - "plot_region": None, - "verbose": False, - } - - self._short_options = { - "input_file": "-i", - "output_file": "-o", - "nside_coverage": "-c", - "nside": "-n", - "apply_median_filter": "-m", - "n_median_iterations": "-N", - "create_boolean": "-b", - "boolean_threshold": "-t", - "boolean_output": "-B", - "create_plot": "-p", - "plot_output": "-P", - "plot_region": "-g", - } - - self._types = { - "nside_coverage": "int", - "nside": "int", - "apply_median_filter": "bool", - "n_median_iterations": "int", - "create_boolean": "bool", - "boolean_threshold": "int", - "create_plot": "bool", - "verbose": "bool", - } - - self._help_strings = { - "input_file": "input file with field corners; default is {}", - "output_file": "output HealSparse map file; default is {}", - "nside_coverage": "HealSparse coverage nside; default is {}", - "nside": "HealSparse map nside; default is {}", - "apply_median_filter": "apply median filter to smooth map; default is {}", - "n_median_iterations": "number of median filter iterations; default is {}", - "create_boolean": "create boolean coverage map; default is {}", - "boolean_threshold": "threshold value for boolean map; default is {}", - "boolean_output": "output file for boolean map; default is _bool.hsp", - "create_plot": "create plot of coverage map; default is {}", - "plot_output": "output file for plot; default is .png", - "plot_region": "predefined region for plot (NGC, SGC, fullsky); default is {}", - } - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - Parameters - ---------- - args : list - command line arguments - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - args=args, + ra = np.atleast_2d(np.asarray(ra, dtype=float)) + dec = np.atleast_2d(np.asarray(dec, dtype=float)) + + if verbose: + print( + f"Creating HealSparse map (nside_coverage={nside_coverage}," + f" nside={nside})" ) - self._params = options - - # Save calling command - logging.log_command(args) - - def update_params(self): - """Update parameters. - - Set derived parameters based on input parameters. - """ - # Set boolean output filename if not specified - if self._params["boolean_output"] is None: - output_file = self._params["output_file"] - if output_file.endswith(".hsp"): - base = output_file[:-4] - else: - base = output_file - self._params["boolean_output"] = f"{base}_bool.hsp" - - # Set plot output filename if not specified - if self._params["plot_output"] is None: - output_file = self._params["output_file"] - if output_file.endswith(".hsp"): - base = output_file[:-4] - else: - base = output_file - self._params["plot_output"] = f"{base}.png" - - def check_params(self): - """Check parameters for validity.""" - if not exists(self._params["input_file"]): - raise FileNotFoundError( - f"Input file not found: {self._params['input_file']}" - ) + m = hsp.HealSparseMap.make_empty(nside_coverage, nside, np.uint16) + + if verbose: + print("Adding polygons to map") - # Check nside values are powers of 2 - nside_coverage = self._params["nside_coverage"] - nside = self._params["nside"] + n_added = 0 + n_skipped = 0 + for i in range(len(ccd_ids)): + dec_i = dec[i] - if not (nside_coverage & (nside_coverage - 1) == 0): - raise ValueError( - f"nside_coverage must be a power of 2, got {nside_coverage}" + # Pole guard: HealSparse's planar polygon fill degrades near the + # poles. CCD footprints never reach here, so warn and skip. + if np.any(np.abs(dec_i) >= _DEC_POLE_LIMIT): + print( + f"Warning: skipping CCD {ccd_ids[i]} with |dec| >= " + f"{_DEC_POLE_LIMIT} (too close to a pole)" ) + n_skipped += 1 + continue + + # RA-wrap guard: put corners on a common branch across the seam. + ra_i = unwrap_ra(ra[i]) - if not (nside & (nside - 1) == 0): - raise ValueError(f"nside must be a power of 2, got {nside}") + m += hsp.Polygon(ra=list(ra_i), dec=list(dec_i), value=1) + n_added += 1 - def median_filter(self, hsp_map): - """Median Filter. + if verbose and i % 1000 == 0: + print(f"{i:6d} / {len(ccd_ids):6d}") - Apply median filter to HealSparse map using neighbors. + print(f"Added {n_added} polygons to map") + if n_skipped > 0: + print(f"Skipped {n_skipped} polygons near a pole") - Parameters - ---------- - hsp_map : healsparse.HealSparseMap - input map + return m + + +def median_filter(hsp_map, n_iterations=1): + """Smooth an nexp map: each pixel becomes its neighbourhood median. + + The neighbourhood is a pixel and its eight HEALPix neighbours; out-of-map + neighbours read as the map's sentinel and so pull an edge pixel down, + which is the intended behaviour for a coverage map (a lone pixel is noise, + not depth). The ``coverage_map`` rule does NOT smooth — the map it writes + is the raw exposure count, which is what sp_validation's mask application + wants — so this is the offline step, for looking at a map rather than + applying one. + + Parameters + ---------- + hsp_map : healsparse.HealSparseMap + input nexp map + n_iterations : int, optional + number of smoothing passes - Returns - ------- - healsparse.HealSparseMap - filtered map + Returns + ------- + healsparse.HealSparseMap + filtered map - """ - nside = hsp_map.nside_sparse + """ + nside = hsp_map.nside_sparse + for _ in range(n_iterations): new_hsp = hsp_map.copy() pixs = hsp_map.valid_pixels n = hpg.neighbors(nside, pixs) @@ -217,156 +191,6 @@ def median_filter(self, hsp_map): mn = hsp_map[n] new = np.median(mn, axis=1).astype(np.uint16) new_hsp[pixs] = new + hsp_map = new_hsp - return new_hsp - - def run(self, args=None): - """Run. - - Main execution method. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code (0 for success) - - """ - if args is None: - args = sys.argv[1:] - - # Set parameters from command line - self.set_params_from_command_line(args) - self.update_params() - self.check_params() - - # Get parameters - input_file = self._params["input_file"] - output_file = self._params["output_file"] - nside_coverage = self._params["nside_coverage"] - nside = self._params["nside"] - apply_median_filter = self._params["apply_median_filter"] - n_median_iterations = self._params["n_median_iterations"] - create_boolean = self._params["create_boolean"] - boolean_threshold = self._params["boolean_threshold"] - boolean_output = self._params["boolean_output"] - create_plot = self._params["create_plot"] - plot_output = self._params["plot_output"] - plot_region = self._params["plot_region"] - verbose = self._params["verbose"] - - if verbose: - print(f"Reading per-CCD corners from: {input_file}") - - # Load per-CCD corner data. Column 0 is a "-" string - # ID; the remaining 8 columns are the 4 RA then 4 Dec corners. - ccd_ids = np.atleast_1d(np.loadtxt(input_file, usecols=(0), dtype=str)) - corners = np.atleast_2d( - np.loadtxt(input_file, usecols=range(1, 9), dtype=float) - ) - ra_all = corners[:, :4] - dec_all = corners[:, 4:] - - print(f"Loaded {len(ccd_ids)} CCD footprints") - - if verbose: - print( - f"Creating HealSparse map (nside_coverage={nside_coverage}, nside={nside})" - ) - - # Create empty map - m = hsp.HealSparseMap.make_empty(nside_coverage, nside, np.uint16) - - # Add one polygon per CCD footprint - if verbose: - print("Adding polygons to map") - - n_added = 0 - n_skipped = 0 - for i in range(len(ccd_ids)): - dec = dec_all[i] - - # Pole guard: HealSparse's planar polygon fill degrades near the - # poles. CCD footprints never reach here, so warn and skip. - if np.any(np.abs(dec) >= _DEC_POLE_LIMIT): - print( - f"Warning: skipping CCD {ccd_ids[i]} with |dec| >= " - f"{_DEC_POLE_LIMIT} (too close to a pole)" - ) - n_skipped += 1 - continue - - # RA-wrap guard: put corners on a common branch across the seam. - ra = unwrap_ra(ra_all[i]) - - m += hsp.Polygon(ra=list(ra), dec=list(dec), value=1) - n_added += 1 - - if verbose and i % 1000 == 0: - print(f"{i:6d} / {len(ccd_ids):6d}") - - print(f"Added {n_added} polygons to map") - if n_skipped > 0: - print(f"Skipped {n_skipped} polygons near a pole") - - # Apply median filter if requested - if apply_median_filter: - if verbose: - print( - f"Applying median filter ({n_median_iterations} iterations)" - ) - - for i in range(n_median_iterations): - m = self.median_filter(m) - if verbose: - print(f" Iteration {i+1}/{n_median_iterations} complete") - - # Write main coverage map - if verbose: - print(f"Writing coverage map to: {output_file}") - - m.write(output_file, clobber=True) - print(f"Coverage map saved to {output_file}") - - # Create and write boolean map if requested - if create_boolean: - if verbose: - print( - f"Creating boolean map (threshold={boolean_threshold})" - ) - - c = hsp.HealSparseMap.make_empty(nside_coverage, nside, bool) - c[m.valid_pixels] = m[m.valid_pixels] >= boolean_threshold - - if verbose: - print(f"Writing boolean map to: {boolean_output}") - - c.write(boolean_output, clobber=True) - print(f"Boolean map saved to {boolean_output}") - - # Create plot if requested - if create_plot: - if verbose: - print(f"Creating plot of coverage map") - - try: - from shapepipe.utilities.coverage_plotter import CoveragePlotter - plotter = CoveragePlotter() - plotter.plot_coverage_map( - m, - plot_output, - region=plot_region, - vmax=boolean_threshold if create_boolean else 3, - colorbar=True, - colorbar_label="Coverage depth", - ) - print(f"Plot saved to {plot_output}") - except ImportError as e: - print(f"Warning: Could not create plot: {e}") - print("Install the cs_util package for plotting support.") - - return 0 + return hsp_map diff --git a/src/shapepipe/utilities/field_corners_extractor.py b/src/shapepipe/utilities/field_corners_extractor.py deleted file mode 100644 index 7e3273456..000000000 --- a/src/shapepipe/utilities/field_corners_extractor.py +++ /dev/null @@ -1,493 +0,0 @@ -"""FIELD_CORNERS_EXTRACTOR - -Extract per-CCD sky-footprint corner coordinates from FITS headers. - -For each CCD (image HDU) in an exposure's multi-HDU header, the four corners -of the CCD are projected from pixel bounds to the sky, producing one row per -CCD keyed on its ``-`` ID. - -Author: Mike Hudson, Martin Kilbinger - -""" - -import sys -import os -import glob -import re -from os.path import exists -from multiprocessing import Pool, cpu_count - -import numpy as np -from astropy import wcs -from astropy.io.fits import Header - -from cs_util import args as cs_args -from cs_util import logging - -# Filename suffix convention for header files: <6+ digit exposure number>.txt -_EXPNUM_RE = re.compile(r'(\d+)\.txt$') - - -def _expnum_from_path(path): - """Extract exposure number from a header filename like ``1234567.txt``.""" - match = _EXPNUM_RE.search(path) - if match is None: - raise ValueError(f"Could not extract exposure number from {path!r}") - return int(match.group(1)) - - -def _image_shape(header): - """Return the CCD image ``(nx, ny)`` pixel shape from a header. - - For fpack tile-compressed HDUs (``ZIMAGE = T``), ``NAXIS1``/``NAXIS2`` - describe the compressed *binary table* (row width in bytes, row count), - not the image, and astropy's WCS never maps the true dimensions into - ``pixel_shape``. The true image size is carried by ``ZNAXIS1``/``ZNAXIS2``, - which are preferred here; a plain image HDU falls back to - ``NAXIS1``/``NAXIS2``. - - Parameters - ---------- - header : astropy.io.fits.Header - single-HDU header - - Returns - ------- - tuple - ``(nx, ny)`` image pixel dimensions - - Raises - ------ - ValueError - if neither the ``Z*`` nor the plain ``NAXIS`` dimensions are present - """ - if header.get("ZIMAGE", False): - keys = ("ZNAXIS1", "ZNAXIS2") - else: - keys = ("NAXIS1", "NAXIS2") - - if keys[0] not in header or keys[1] not in header: - raise ValueError( - f"Header is missing image dimensions ({keys[0]}/{keys[1]})" - ) - - return int(header[keys[0]]), int(header[keys[1]]) - - -def _parse_header_to_wcs(path): - """Parse a multi-HDU header text file into ``(wcs, (nx, ny))`` per HDU. - - The primary HDU is skipped. Each remaining HDU yields its WCS together with - the CCD image pixel shape (``ZNAXIS1/2`` for compressed HDUs, else - ``NAXIS1/2``); the shape is read from the header because the WCS drops the - ``Z*`` keywords. - """ - with open(path, "r") as f: - string = f.read() - tokens = re.split(r"^(END\s+)", string, flags=re.MULTILINE) - result = [] - for i in range(2, len(tokens) - 1, 2): - header = Header.fromstring(tokens[i] + tokens[i + 1], sep="\n") - result.append((wcs.WCS(header), _image_shape(header))) - return result - - -def _ccd_corners(w, shape): - """Return (ra, dec) of the 4 corners of a single CCD. - - The polygon is built from the CCD's pixel bounds ``shape = (nx, ny)`` using - pixel edges ``(-0.5, n - 0.5)`` so the quadrilateral covers the full CCD - area rather than pixel centres. Corners are returned in a consistent - counterclockwise pixel order (bottom-left, bottom-right, top-right, - top-left); since ``pixel_to_world`` maps this order to a simple, - non-self-intersecting sky quadrilateral for a CCD-sized field, and - HealSparse ``Polygon`` is orientation-agnostic, the resulting polygon is a - valid convex footprint. - - Parameters - ---------- - w : astropy.wcs.WCS - WCS of a single CCD - shape : tuple - ``(nx, ny)`` CCD image pixel dimensions - - Returns - ------- - tuple - ``(ra_list, dec_list)`` each of length 4, in degrees - """ - nx, ny = shape - - # Pixel-edge corners, counterclockwise: BL, BR, TR, TL - px = [-0.5, nx - 0.5, nx - 0.5, -0.5] - py = [-0.5, -0.5, ny - 0.5, ny - 0.5] - - sky = w.pixel_to_world(px, py) - return list(sky.ra.deg), list(sky.dec.deg) - - -class FieldCornersExtractor(object): - """Field Corners Extractor Class. - - Extracts RA/Dec coordinates of field corners from FITS headers. - """ - - def __init__(self): - """Initialize the extractor.""" - self.params_default() - - def params_default(self): - """Set default parameters and command line options.""" - - self._params = { - "input_dir": "header", - "output_file": "exp_ra_dec.txt", - "ccd_list": None, - "resume": False, - "n_processes": 1, - "verbose": False, - } - - self._short_options = { - "input_dir": "-i", - "output_file": "-o", - "ccd_list": "-l", - "resume": "-r", - "n_processes": "-n", - } - - self._types = { - "resume": "bool", - "n_processes": "int", - "verbose": "bool", - } - - self._help_strings = { - "input_dir": "input directory containing header files; default is {}", - "output_file": "output file for per-CCD corners; default is {}", - "ccd_list": "file of valid CCD IDs (output of get_ccds_with_psf); when given, only listed CCDs are written; on --resume, CCDs new to an expanded list are added without duplicating existing rows; default is all CCDs", - "resume": "resume from existing output file; default is {}", - "n_processes": f"number of parallel processes (1=serial, 0=auto={cpu_count()}); default is {{}}", - } - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - Parameters - ---------- - args : list - command line arguments - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - args=args, - ) - self._params = options - - # Save calling command - logging.log_command(args) - - def update_params(self): - """Update parameters. - - Set derived parameters based on input parameters. - """ - # Ensure input directory ends without trailing slash - if self._params["input_dir"].endswith("/"): - self._params["input_dir"] = self._params["input_dir"][:-1] - - def check_params(self): - """Check parameters for validity.""" - if not exists(self._params["input_dir"]): - raise FileNotFoundError( - f"Input directory not found: {self._params['input_dir']}" - ) - - if self._params["ccd_list"] is not None and not exists( - self._params["ccd_list"] - ): - raise FileNotFoundError( - f"CCD list file not found: {self._params['ccd_list']}" - ) - - # Set n_processes to cpu_count if 0 - if self._params["n_processes"] == 0: - self._params["n_processes"] = cpu_count() - - if self._params["n_processes"] < 0: - raise ValueError( - f"n_processes must be >= 0, got {self._params['n_processes']}" - ) - - @staticmethod - def load_ccd_list(path): - """Load CCD List. - - Read valid CCD IDs (one ``-`` per line) into a set. - - Parameters - ---------- - path : str - path to the CCD list file - - Returns - ------- - set - valid CCD IDs - - """ - with open(path) as f: - return {line.strip() for line in f if line.strip()} - - @staticmethod - def process_single_header(args): - """Process Single Header. - - Worker function to process a single header file into per-CCD corners. - Static method so it can be pickled for multiprocessing. - - Parameters - ---------- - args : tuple - ``(path, verbose, valid_ccds)`` where ``path`` is the header file - path, ``verbose`` is a bool, and ``valid_ccds`` is a set of CCD IDs - to keep (or ``None`` to keep all) - - Returns - ------- - list or None - list of ``(ccd_id, ra_list, dec_list)`` for the exposure's CCDs on - success, ``None`` on failure - - """ - path, verbose, valid_ccds = args - expnum = _expnum_from_path(path) - - try: - wcs_shapes = _parse_header_to_wcs(path) - except Exception as e: - if verbose: - print(f"Failed to process {expnum}: {e}") - return None - - rows = [] - for ccd_idx, (w, shape) in enumerate(wcs_shapes): - ccd_id = f"{expnum}-{ccd_idx}" - if valid_ccds is not None and ccd_id not in valid_ccds: - continue - try: - ra, dec = _ccd_corners(w, shape) - except Exception as e: - if verbose: - print(f"Failed to process CCD {ccd_id}: {e}") - continue - rows.append((ccd_id, ra, dec)) - - return rows - - def get_done_ccds(self): - """Get Done CCDs. - - Read the set of CCD IDs already present in the output file. Resume is - keyed on individual CCD IDs, not exposure numbers: a CCD counts as done - only if its own row is present. This keeps resume correct when a write - was interrupted mid-exposure (the missing CCDs are filled in) and when - a rerun uses an expanded ``--ccd_list`` (the newly requested CCDs are - added), and it never duplicates a row. - - Returns - ------- - set - CCD IDs already written - - """ - output_file = self._params["output_file"] - - if not exists(output_file): - return set() - - try: - ids = np.atleast_1d( - np.loadtxt(output_file, usecols=(0), dtype=str) - ) - return set(ids.tolist()) - except Exception as e: - if self._params["verbose"]: - print(f"Could not read existing output file: {e}") - return set() - - def run(self, args=None): - """Run. - - Main execution method. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code (0 for success) - - """ - if args is None: - args = sys.argv[1:] - - # Set parameters from command line - self.set_params_from_command_line(args) - self.update_params() - self.check_params() - - # Get parameters - input_dir = self._params["input_dir"] - output_file = self._params["output_file"] - resume = self._params["resume"] - verbose = self._params["verbose"] - - # Load the set of valid CCD IDs to keep, if given - valid_ccds = None - if self._params["ccd_list"] is not None: - valid_ccds = self.load_ccd_list(self._params["ccd_list"]) - print(f"{len(valid_ccds)} valid CCDs in {self._params['ccd_list']}") - - # Find all header files - paths = glob.glob(f"{input_dir}/*.txt") - n = len(paths) - - if n == 0: - print(f"No header files found in {input_dir}/") - return 1 - - print(f"{n} header files found") - - # On resume, read the CCD IDs already written so we can skip them - # per-CCD (not per-exposure): a partially written exposure is completed - # rather than skipped, and no row is ever duplicated. - done_ccds = set() - if resume: - done_ccds = self.get_done_ccds() - print(f"{len(done_ccds)} CCDs already done") - - # Every header still needs parsing (a header may hold both done and - # not-yet-done CCDs); the per-CCD filter below decides what is written. - todo = paths - - # Get n_processes - n_processes = self._params["n_processes"] - - # Process headers - if n_processes == 1: - # Serial processing - results = self._process_serial(todo, verbose, valid_ccds) - else: - # Parallel processing - print(f"Using {n_processes} parallel processes") - results = self._process_parallel( - todo, n_processes, verbose, valid_ccds - ) - - # Flatten per-exposure CCD lists (dropping failed exposures), then drop - # CCDs already present in the output. - rows = [ - row - for res in results if res is not None - for row in res - if row[0] not in done_ccds - ] - - # Sort by exposure number, then CCD index - rows.sort(key=lambda r: (int(r[0].split("-")[0]), - int(r[0].split("-")[1]))) - - # Write results to file: " ra1 ra2 ra3 ra4 dec1 dec2 dec3 dec4" - mode = "a" if resume else "w" - with open(output_file, mode, buffering=1) as f: - for ccd_id, ra, dec in rows: - f.write(f"{ccd_id} ") - np.savetxt(f, ra, fmt="%9.5f", newline=" ") - np.savetxt(f, dec, fmt="%9.5f", newline=" ") - f.write("\n") - - n_exp_success = sum(1 for res in results if res is not None) - n_failed = len(todo) - n_exp_success - - print(f"Processed {n_exp_success} exposures, {len(rows)} new CCDs") - if n_failed > 0: - print(f"Failed to process {n_failed} exposures") - - print(f"Results written to {output_file}") - - return 0 - - def _process_serial(self, todo, verbose, valid_ccds): - """Process Serial. - - Process headers serially. - - Parameters - ---------- - todo : list - list of header file paths to process - verbose : bool - verbose output - valid_ccds : set or None - CCD IDs to keep, or ``None`` to keep all - - Returns - ------- - list - list of per-exposure CCD-row lists (or ``None`` for failures) - - """ - results = [] - n_todo = len(todo) - - for i, p in enumerate(todo): - result = self.process_single_header((p, verbose, valid_ccds)) - results.append(result) - - if verbose and i % 100 == 0: - print(f"{i:6d} / {n_todo:6d}") - - return results - - def _process_parallel(self, todo, n_processes, verbose, valid_ccds): - """Process Parallel. - - Process headers in parallel using multiprocessing. - - Parameters - ---------- - todo : list - list of header file paths to process - n_processes : int - number of parallel processes - verbose : bool - verbose output - valid_ccds : set or None - CCD IDs to keep, or ``None`` to keep all - - Returns - ------- - list - list of per-exposure CCD-row lists (or ``None`` for failures) - - """ - # Prepare arguments for worker function - args = [(p, verbose, valid_ccds) for p in todo] - - # Create pool and process - with Pool(processes=n_processes) as pool: - results = pool.map(self.process_single_header, args) - - return results diff --git a/src/shapepipe/utilities/file_io.py b/src/shapepipe/utilities/file_io.py new file mode 100644 index 000000000..8a0964aa0 --- /dev/null +++ b/src/shapepipe/utilities/file_io.py @@ -0,0 +1,45 @@ +"""FILE I/O UTILITIES. + +Small, dependency-light helpers for publishing files the rest of the pipeline +treats as a cache. + +:Author: consolidated from workflow/scripts/star_cats.py and + scripts/python/create_star_cat.py + +""" + +import os + + +def write_atomic(table, path): + """Publish ``table`` at ``path`` all-or-nothing. + + Every caller's only cache test is existence (``Path.exists`` / + ``os.path.isfile``), so a write killed part-way -- job timeout, OOM, node + failure -- would otherwise leave a truncated FITS that every later run + trusts forever, and ``test -s`` passes on partial bytes. Writing to a temp + and renaming makes the visible file all-or-nothing: ``os.replace`` is atomic + within a directory. + + The temp keeps the target's suffix, because astropy picks its writer from + the extension. It is dot-prefixed and PID-tagged so it stays out of the + ``star_chunk-*`` / ``star_cat*`` globs the rules use, and two concurrent + writers cannot collide. + + Parameters + ---------- + table : astropy.table.Table + Table to write + path : str or pathlib.Path + Destination path + + """ + path = os.fspath(path) + directory, name = os.path.split(path) + tmp = os.path.join(directory or ".", f".tmp-{os.getpid()}-{name}") + try: + table.write(tmp, overwrite=True) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.remove(tmp) diff --git a/src/shapepipe/utilities/focal_plane.py b/src/shapepipe/utilities/focal_plane.py new file mode 100644 index 000000000..32770bd06 --- /dev/null +++ b/src/shapepipe/utilities/focal_plane.py @@ -0,0 +1,92 @@ +"""FOCAL PLANE GEOMETRY. + +The sky footprint of a MegaCam exposure, read from its image headers. Both +star-catalogue producers need exactly this and used to carry their own copy of +it -- ``workflow/scripts/star_cats.py`` (the HEALPix chunk store's ``cut``) and +``scripts/python/create_star_cat.py`` (the one-cone-per-exposure path the store +replaced). They must agree on which sky an exposure covers, so there is one +definition of it. + +:Author: consolidated from the two star-catalogue scripts + +""" + +import numpy as np +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS + + +def get_wcs(header): + """Build the WCS by hand, from the linear terms only. + + Deliberately NOT ``WCS(header)``: it sidesteps distortion-convention + incompatibilities between these headers and astropy, and a footprint needs + nothing finer than the linear terms. + + Parameters + ---------- + header : astropy.io.fits.Header + Image header + + Returns + ------- + astropy.wcs.WCS + WCS object + + """ + final_wcs = WCS(naxis=2) + final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] + try: + final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] + except KeyError: + final_wcs.wcs.cunit = ["deg", "deg"] + final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] + final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] + final_wcs.wcs.cd = [ + [header["CD1_1"], header["CD1_2"]], + [header["CD2_1"], header["CD2_2"]], + ] + + return final_wcs + + +def ccd_center_and_radius(header): + """Return ``(ra_deg, dec_deg, radius_deg)`` for a single CCD. + + The radius is the half-diagonal: centre to the ``(0, 0)`` corner. + + """ + w = get_wcs(header) + (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( + [[header["NAXIS1"] / 2.0, header["NAXIS2"] / 2.0], [0, 0]], 1) + center = SkyCoord(ra_c * u.deg, dec_c * u.deg) + radius = center.separation(SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg + return float(ra_c), float(dec_c), float(radius) + + +def focal_plane_disc(image, n_ccd=40): + """Return ``(ra_deg, dec_deg, radius_deg)`` covering all CCDs of one exposure. + + The centre is the mean of the CCD centres; the radius is the largest + centre-to-CCD-centre distance plus that CCD's own half-diagonal. + + ONE ``fits.open`` for all the extensions: ``fits.getheader(image, ext)`` + opens the file, walks the HDU list to ``ext`` and closes again, so a loop + over it costs ``n_ccd`` opens and O(n^2) header seeks. + + """ + centers, radii = [], [] + with fits.open(image) as hdul: + for ext in range(1, n_ccd + 1): + ra_c, dec_c, radius = ccd_center_and_radius(hdul[ext].header) + centers.append((ra_c, dec_c)) + radii.append(radius) + + ras = np.array([c[0] for c in centers]) + decs = np.array([c[1] for c in centers]) + center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) + seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg + return (float(center.ra.deg), float(center.dec.deg), + float(np.max(seps + np.array(radii)))) diff --git a/src/shapepipe/utilities/header_downloader.py b/src/shapepipe/utilities/header_downloader.py deleted file mode 100644 index 122327a58..000000000 --- a/src/shapepipe/utilities/header_downloader.py +++ /dev/null @@ -1,291 +0,0 @@ -"""HEADER_DOWNLOADER - -Download FITS headers from VOSpace for exposures listed in a CCD file. - -Author: Mike Hudson, Martin Kilbinger - -""" - -import sys -import os -from os.path import exists - -import numpy as np -import vos - -from cs_util import args as cs_args -from cs_util import logging - - -class HeaderDownloader(object): - """Header Downloader Class. - - Downloads FITS headers from VOSpace for exposures in a CCD list. - """ - - def __init__(self): - """Initialize the downloader.""" - self.params_default() - - def params_default(self): - """Set default parameters and command line options.""" - - self._params = { - "input_file": None, - "output_dir": "header", - "vospace_path": "vos:cfis/pitcairn", # UNIONS/CFIS default; override for other surveys - "overwrite": False, - "dir_for_links": None, - "verbose": False, - } - - self._short_options = { - "input_file": "-i", - "output_dir": "-o", - "vospace_path": "-p", - "overwrite": "-O", - "dir_for_links": "-d", - } - - self._types = { - "overwrite": "bool", - "verbose": "bool", - } - - self._help_strings = { - "input_file": "input CCD list file (txt or csv); required", - "output_dir": "output directory for headers; default is {}", - "vospace_path": "VOSpace base path; default is {}", - "overwrite": "overwrite existing header files; default is {}", - "dir_for_links": "directory to check for existing headers to link instead of download; default is {}", - } - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - Parameters - ---------- - args : list - command line arguments - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - args=args, - ) - self._params = options - - # Save calling command - logging.log_command(args) - - def update_params(self): - """Update parameters. - - Set derived parameters based on input parameters. - """ - # Ensure output directory ends without trailing slash - if self._params["output_dir"].endswith("/"): - self._params["output_dir"] = self._params["output_dir"][:-1] - - def check_params(self): - """Check parameters for validity.""" - if self._params["input_file"] is None: - raise ValueError("Input file is required (use -i or --input_file)") - - if not exists(self._params["input_file"]): - raise FileNotFoundError( - f"Input file not found: {self._params['input_file']}" - ) - - # Check if dir_for_links exists if specified - if self._params["dir_for_links"] is not None: - if not exists(self._params["dir_for_links"]): - raise FileNotFoundError( - f"Directory for links not found: {self._params['dir_for_links']}" - ) - if not os.path.isdir(self._params["dir_for_links"]): - raise ValueError( - f"Path is not a directory: {self._params['dir_for_links']}" - ) - - # Create output directory if it doesn't exist - if not exists(self._params["output_dir"]): - os.makedirs(self._params["output_dir"]) - if self._params["verbose"]: - print(f"Created output directory: {self._params['output_dir']}") - - def get_exposures(self, ccd_list_file): - """Get Exposures. - - Extract unique exposure numbers from CCD list file. - - Parameters - ---------- - ccd_list_file : str - path to CCD list file (txt or csv) - - Returns - ------- - np.array - unique exposure numbers - - """ - exps = [] - - # Check if CSV format - if ccd_list_file.endswith(".csv"): - import astropy.table - - t = astropy.table.Table.read(ccd_list_file) - expccd = t["CCD"].data - r = np.char.split(expccd, sep="-") - - for i, r1 in enumerate(r): - exp = int(r1[0]) - exps.append(exp) - else: - # Text format; atleast_1d so a single-line file (0-d array) is - # still iterable. - f = np.atleast_1d( - np.loadtxt(ccd_list_file, dtype="str", encoding="ascii") - ) - r = np.char.split(f, sep="-") - - for i, r1 in enumerate(r): - exp = int(r1[0]) - exps.append(exp) - - exps = np.array(exps) - uniq = np.unique(exps) - - return uniq - - def get_fits_header(self, expnum, client): - """Get FITS Header. - - Download FITS header from VOSpace, or create a symbolic link - if the header exists in dir_for_links. - - Parameters - ---------- - expnum : int - exposure number - client : vos.Client - VOSpace client - - Returns - ------- - bool - True if successful, False otherwise - - """ - vospace_path = self._params["vospace_path"] - output_dir = self._params["output_dir"] - overwrite = self._params["overwrite"] - dir_for_links = self._params["dir_for_links"] - - source = f"{vospace_path}/{expnum:d}p.fits.fz" - dest = f"{output_dir}/{expnum:d}.txt" - - if exists(dest) and not overwrite: - return True - - # Check if header exists in dir_for_links - if dir_for_links is not None: - link_source = os.path.abspath(f"{dir_for_links}/{expnum:d}.txt") - if exists(link_source): - try: - # Remove existing file/link if overwrite is True - if exists(dest): - os.remove(dest) - # Create symbolic link - os.symlink(link_source, dest) - return True - except Exception as e: - print(f"Could not create symlink from {link_source}: {e}") - # Fall through to download if symlink fails - - # Download from VOSpace atomically: copy to a temp file in the same - # directory, then rename on success. An interrupted transfer leaves - # only the temp file behind, so resume never treats a partial download - # as complete. - tmp_dest = f"{dest}.part" - try: - client.copy(source, tmp_dest, head=True) - os.rename(tmp_dest, dest) - return True - except Exception as e: - print(f"Could not copy {source}: {e}") - if exists(tmp_dest): - os.remove(tmp_dest) - return False - - def run(self, args=None): - """Run. - - Main execution method. - - Parameters - ---------- - args : list, optional - command line arguments - - Returns - ------- - int - exit code (0 for success) - - """ - if args is None: - args = sys.argv[1:] - - # Set parameters from command line - self.set_params_from_command_line(args) - self.update_params() - self.check_params() - - # Get parameters - input_file = self._params["input_file"] - output_dir = self._params["output_dir"] - verbose = self._params["verbose"] - - if verbose: - print(f"Reading CCD list from: {input_file}") - - # Extract unique exposures - exps = self.get_exposures(input_file) - - print(f"Found {len(exps)} unique exposures") - - if verbose: - print(f"Downloading headers to: {output_dir}/") - - # Initialize VOSpace client once - client = vos.Client() - - # Download headers - n_success = 0 - n_failed = 0 - - for i, exp in enumerate(exps): - success = self.get_fits_header(exp, client) - if success: - n_success += 1 - else: - n_failed += 1 - - if verbose and i % 100 == 0: - print(f"{i:6d} / {len(exps):6d}") - - print(f"Downloaded {n_success} headers") - if n_failed > 0: - print(f"Failed to download {n_failed} headers") - - return 0 diff --git a/src/shapepipe/utilities/summary.py b/src/shapepipe/utilities/summary.py deleted file mode 100755 index dd5009f3f..000000000 --- a/src/shapepipe/utilities/summary.py +++ /dev/null @@ -1,832 +0,0 @@ -"""SUMMARY - -Author: Martin Kilbinger - -""" - -import sys -import os -import re -import fnmatch - -import logging - -from collections import Counter - -from tqdm import tqdm - - -def init_par_runtime(list_tile_IDs): - - # Numbers updated at runtime - par_runtime = {} - - par_runtime["n_tile_IDs"] = len(list_tile_IDs) - par_runtime["list_tile_IDs"] = list_tile_IDs - - return par_runtime - - -def update_par_runtime_after_find_exp(par_runtime, all_exposures): - - # Single-exposure images - par_runtime["n_exposures"] = len(all_exposures) - par_runtime["list_exposures"] = all_exposures - - # Single-HDU single exposure images - n_CCD = 40 - par_runtime["n_shdus"] = get_par_runtime(par_runtime, "exposures") * n_CCD - par_runtime["list_shdus"] = get_all_shdus(all_exposures, n_CCD) - - return par_runtime - - -def get_IDs_from_file(path): - """Get IDs From File. - - Return IDs from text file. Removes letters and replaces - dots "." with dashes "-". - - Parameters - ---------- - path: str - input file path - - Returns - -------- - list - IDs - - """ - numbers = [] - with open(path) as f_in: - for line in f_in: - entry = line.rstrip() - number = re.sub("[a-zA-Z]", "", entry) - numbers.append(number) - - return numbers - - -def get_all_exposures(exp_number_file_list, verbose=False): - """Get All Exposures. - - Return all exposure names from a list of text files. - - Parameters - ---------- - exp_number_list: list - input file names - - """ - exposures = set() - for idx, path in enumerate(exp_number_file_list): - exps = get_IDs_from_file(path) - exposures.update(exps) - - return list(exposures) - - -def get_all_shdus(exposures, n_CCD): - """Get All SHDUs. - - Return all single-exposure single-HDU (CCD) IDs. - - Parameters - ---------- - exposures: list - exposure names - n_CCD: int - number of CCDs per exposure - - Returns - -------- - list - single-exposure single-HDU IDs - - """ - shdus = [] - for exposure in exposures: - for idx_CCD in range(n_CCD): - shdus.append(f"{exposure}-{idx_CCD}") - - return shdus - - -def set_as_list(item=None, n=None, default=1): - """Set As List. - - Return input as list. - - Parameters - ----------- - item: str, int, or list, optional - input item(s); default is None, in which - case the return is [1] * n - n: int, optional - number of list items to return, default is None, - in which case the number will be set to 1. If item and - n are not None, n has to be equal to len(item) - default: int, optional - value to return if item is not given; - default is 1 - - Raises - ------- - IndexError - if n != len(item) - - Returns - ------- - list - input item(s) as list - """ - my_n = n or 1 - - if not item: - result = [default] * my_n - elif not isinstance(item, list): - result = [item] * my_n - else: - result = item - if len(item) != my_n: - raise IndexError(f"item has length {len(item)} != {n}") - - return result - - -def check_special_one(module, path): - - ngmix_finished = False - - with open(path) as f_in: - lines = f_in.readlines() - for line in lines: - entry = line.rstrip() - - if module == "setools_runner": - m = re.search("Nb stars = (\S*)", line) - if m: - value = int(m[1]) - if value < 2: - code = 0 - msg = ( - f"Not enough stars for random split:" - + f" #stars = {value}" - ) - return msg, code - break - m = re.search("Mode computation failed", line) - if m: - code = 1 - msg = "Mode computation of stellar locus failed" - return msg, code - - if module == "psfex_interp_runner": - m = re.search("Key N_EPOCH not found", line) - if m: - code = 2 - msg = "N_EPOCH not in SEx cat, rerun job 16" - return msg, code - - m = re.search( - "ValueError: cannot reshape array of size 0 into shape", - line, - ) - if m: - code = 3 - msg = "found array of size 0" - return msg, code - - if module == "mask_runner": - m = re.search("Empty or corrupt FITS file", line) - if m: - code = 4 - msg = "empty or corrult FITS file" - return msg, code - - if module == "sextractor_runner": - m = re.search("sextracted 0", line) - if m: - code = 5 - msg = "No object detected (weight might be 0 everywhere)" - return msg, code - m = re.search("astropy\.wcs\.wcs\.NoConvergence", line) - if m: - code = 8 - msg = "WCS world2pix did not converge" - return msg, code - - if module == "ngmix_runner": - m = re.search("finished", line) - if m: - ngmix_finished = True - break - - if module == "merge_sep_cats_runner": - m = re.search("Input catalogue", line) - if m: - code = 7 - msg = "One or more ngmix catalogues not found for merge" - return msg, code - - if module == "ngmix_runner" and not ngmix_finished: - code = 6 - msg = "ngmix incomplete" - return msg, code - - - return None, None - - -class job_data(object): - """Job Data. - - Class to handle a job. - - Parameters - ---------- - bit: int - bit-coded job number - run_dir: str or list - run directory(ies) - modules: list - module names - key_expected: int or str - number of expected output files; if str: will be updated - with runtime value - n_mult: int or list, optional - multiplicity of output files, default `None`, in which - case it is set to 1 - pattern: list, optional - if not None, file pattern to match; defafult is `None` - path_main: str, optional - main (left-most) part of output directory, default is "." - path_left: str, optional - left (first) part of output directory, default is "./output" - output_subdirs: str, optional - output subdirectories if not `None`; default is `None` - path_right: str, optional - right (last) part of output subdir suffix if not `None`; - default is `None` - path_output: str, optional - module output path, default is "output" - output_path_missing_IDs: list, optional - output path of missing ID, if `None` (default) will be - given by job bit and module. - special: bool, optional - if True check output file content for special messages; - default is False - verbose: bool, optional - verbose output if True; default is False - - """ - - def __init__( - self, - bit, - run_dir, - modules, - key_expected, - n_mult=None, - pattern=None, - path_main=".", - path_left="output", - output_subdirs=None, - path_right=None, - path_output="output", - output_path_missing_IDs=None, - special=False, - verbose=False, - ): - self._bit = bit - self._run_dir = set_as_list(item=run_dir, n=len(modules)) - self._modules = modules - self._key_expected = set_as_list(item=key_expected, n=len(modules)) - self._n_mult = set_as_list(item=n_mult, n=len(modules)) - self._pattern = set_as_list(item=pattern, n=len(modules), default="") - self._path_main = path_main - self._path_left = path_left - self._output_subdirs = output_subdirs or [""] - self._path_right = set_as_list( - path_right, - len(modules), - default=".", - ) - self._path_output = set_as_list( - path_output, - len(modules), - default="output", - ) - self._output_path_missing_IDs = output_path_missing_IDs - self._special = set_as_list( - special, - len(modules), - default=False, - ) - self._path_right = set_as_list(path_right, len(modules), default=".") - self._output_path_missing_IDs = output_path_missing_IDs - self._verbose = verbose - - def print_intro(self): - """Print Intro. - - Print header line for job statistics. - - """ - logging.info(f" # Job {self._bit}:") - - @classmethod - def print_stats_header(self): - """Print Stats Header. - - Print overall header information for stats output. - - """ - logging.info( - "module expected found" - + " missing uniq_miss fr_found" - ) - logging.info("=" * 100) - - def print_stats( - self, - module, - n_expected, - n_found, - n_special, - n_missing, - idx, - ): - """Print Stats. - - Print output file statistics. - - Parameters - ---------- - module: str - module name - n_expected: int - number of expected files - n_found: int - number of found files - n_special: int - number of special cases - n_missing: int - number of missing files - idx: int - module index - - """ - module_str = module - - if not self._special[idx]: - if n_expected > 0: - fraction_found = n_found / n_expected - else: - fraction_found = 1 - - n_missing_per_mult = n_missing / self._n_mult[idx] - - else: - module_str = f"{module_str} (special)" - n_found = n_special - n_missing = -1 - n_missing_per_mult = -1 - fraction_found = n_found / n_expected - n_expected = -1 - - logging.info( - f"{module_str:30s} {n_expected:9d} {n_found:9d}" - + f" {n_missing:9d}" - + f" {n_missing_per_mult:9.1f} {fraction_found:9.1%}" - ) - - @classmethod - def is_ID_in_str(self, ID, path): - if ID in path: - return True - - @classmethod - def is_not_in_any(self, ID, list_str): - return not any(ID in string for string in list_str) - - @classmethod - def replace_dot_dash(self, numbers): - - results = [re.sub("\.", "-", number) for number in numbers] - - return results - - @classmethod - def replace_dash_dot_if_tile(self, numbers): - - pattern = re.compile(r"(\d{3})-(\d{3})") - results = [pattern.sub(r"\1.\2", number) for number in numbers] - - return results - - @classmethod - def get_unique(self, names): - n_all = len(names) - names_unique = list(set(names)) - n_unique = len(names_unique) - - if n_all != n_unique: - if True: # self._verbose: - logging.warning( - f"{n_all - n_unique} duplicates removed from {n_all} IDs" - ) - - return names_unique - - @classmethod - def write_IDs_to_file(self, output_path, IDs): - """Write IDs to file. - - Write list if image IDs to text file. - - Parameters - ---------- - output_path: str - output file path - IDs: list - image IDs - - """ - IDs_dot = self.replace_dash_dot_if_tile(IDs) - if len(IDs_dot) > 0: - # Write IDs to file - with open(output_path, "w") as f_out: - for ID in IDs_dot: - print(ID, file=f_out) - elif os.path.exists(output_path): - # Remove preivous obsolete ID file - os.unlink(output_path) - - def check_special(self, module, idx): - - messages = {} - - if self._special[idx]: - - # Loop over input file names and paths - for name, path in zip(self._names_in_dir[idx], self._paths_in_dir[idx]): - - # Check if special case is found - msg, code = check_special_one(module, path) - if msg: - # First time occurance: create empty list for this code - if code not in messages: - messages[code] = [] - - # Append file name, message, and code - messages[code].append(f"{name} {code} {msg}") - - if len(messages) > 0: - # Loop over codes = key in messages dict - for code in messages: - # Create output file for this code - output_path = ( - f"{self._path_main}/summary/special_job_{self._bit}" - + f"_{module}_{code}.txt" - ) - # Write all messages - with open(output_path, "w") as f_out: - for msg in messages[code]: - print(msg, file=f_out) - - # Count all special cases = sum of cases over all codes - n_all = sum([len(messages[code]) for code in messages]) - return n_all - - def output_missing( - self, - module, - idx, - par_runtime=None, - ): - """Output Missing. - - Writes IDs of missing images to disk. - - """ - key_expected = self._key_expected[idx] - names_in_dir = self._names_in_dir[idx] - paths_in_dir = self._paths_in_dir[idx] - n_mult = self._n_mult[idx] - - list_expected = get_par_runtime(par_runtime, key_expected, kind="list") - - # Count image IDs in names that were found earlier - - # Get file name pattern - if module != "split_exp_runner" or (self._bit != 2 and self._bit != 4096): - pattern = re.compile(r"(?:\d{3}-\d{3}|\d{7}-\d+|\d{7})") - else: - # split_exp_runner with sp_local=0: input is exp, output is shdu - # (images) and exp (header); ignore hdu number. - # If sp_local=1 set bit to != 2 - # Update 11/2025: No longer working for P9 4096. Solution: set n_mult=3. - pattern = re.compile( - r"(?:\d{3}-\d{3}|\d{7})" - ) - - ## Extract image IDs from names - IDs = [] - for name, path in zip(names_in_dir, paths_in_dir): - - match = pattern.search(name) - if match: - ID = match.group() - IDs.append(ID) - else: - msg = f"No ID found in {name}" - #raise ValueError(msg) - print(f"Warning: {msg}, continuing") - - # For split_exp_runner P8, IDs now contain exps and sdus, - # not matching mult. - - ## Count occurences - ID_counts = Counter(IDs) - - ## Add to missing if ocurence less than n_mult - missing_IDs = [] - for ID in list_expected: - if ID_counts[ID] < n_mult: - missing_IDs.append(ID) - - n_all = len(missing_IDs) - missing_IDs_unique = self.get_unique(missing_IDs) - - if not self._output_path_missing_IDs: - # Default name using bit and module - output_path = ( - f"{self._path_main}/summary/missing_job_{self._bit}" - + f"_{module}.txt" - ) - else: - # User-defined name (e.g. ngmix_runner_X) - output_path = self._output_path_missing_IDs[idx] - self.write_IDs_to_file(output_path, missing_IDs_unique) - - return missing_IDs_unique - - def output_missing_job(self): - output_path = ( - f"{self._path_main}/summary/missing_job_{self._bit}_all.txt" - ) - - missing_IDs_all = set(self._missing_IDs_job) - - self.write_IDs_to_file(output_path, missing_IDs_all) - - @classmethod - def get_last_full_path(self, base_and_subdir, matches): - """Get Last Full Path - - Return full path of last file in list. - - """ - # Sort according to creation time - matches_sorted = sorted( - matches, - key=lambda entry: entry.name, - ) - - # Get most recent one - last = matches_sorted[-1] - - # Get full path - full_path = os.path.join(base_and_subdir, last.name) - - return full_path - - @classmethod - def get_module_output_dir(self, full_path, module, path_output): - """Get Module Output Dir. - - Return output directory name for given module. - - """ - directory = f"{full_path}/{module}/{path_output}" - - return directory - - def get_matches_final(self, directory, idx): - - # Loop over files - # os.path.whether exists is twice faster than try/except - - if os.path.exists(directory): - pattern = f"{self._pattern[idx]}*" - for entry2 in os.scandir(directory): - if ( - entry2.is_file() - and (fnmatch.fnmatch(entry2.name, pattern)) - and entry2.stat().st_size > 0 - ): - # Append matching files - self._names_in_dir[idx].append(entry2.name) - self._paths_in_dir[idx].append( - os.path.join(directory, entry2.name) - ) - - def get_names_in_dir(self, iterable, module, idx): - - # Initialise output file names and paths - self._names_in_dir[idx] = [] - self._paths_in_dir[idx] = [] - - # Loop over subdirs - for jdx, subdir in enumerate(iterable): - base_and_subdir = ( - f"{self._path_main}/" - + f"{self._path_left}/{subdir}/" - + f"{self._path_right[idx]}" - ) - if self._verbose: - print(f"**** base_and_subdir {base_and_subdir}") - - if os.path.isdir(base_and_subdir): - - matches = [] - - # Loop over entries (files and dirs) - with os.scandir(base_and_subdir) as entries: - for entry in entries: - - # Append directory name if matches module - if ( - entry.name.startswith(self._run_dir[idx]) - and not entry.name.endswith("prev") - ): - matches.append(entry) - - # This entry does not match module -> next - if not matches: - continue - - if self._verbose: - print("**** Matching entries: ", end="") - for match in matches: - print(match.name) - - full_path = self.get_last_full_path( - base_and_subdir, - matches, - ) - - # Get module output directory - directory = self.get_module_output_dir( - full_path, - module, - self._path_output[idx], - ) - if self._verbose: - print(f"**** Output dir = {directory}") - - # Find matching file names and paths - self.get_matches_final(directory, idx) - else: - if self._verbose: - print(f"Directory {base_and_subdir} not found") - - def update_subdirs(self, par_runtime): - """Update Subdirs. - - Update subdir names with runtime information if required. - - """ - if not isinstance(self._output_subdirs, list): - self._output_subdirs = get_par_runtime( - par_runtime, self._output_subdirs, kind="list" - ) - - def check_numbers(self, par_runtime=None, indices=None): - """Check Numbers. - - Check output file numbers and IDs. - - Parameters - ---------- - par_runtime : dict, optional - runtime parameter. default is None - indices: list, optional - if not None (default), only check modules corresponding - to indices - - """ - # Update subdirs if not already set as list - self.update_subdirs(par_runtime) - - # Initialise variables - self._names_in_dir = {} - self._paths_in_dir = {} - self._missing_IDs_job = [] - n_missing_job = 0 - - # Loop over modules - for idx, module in enumerate(self._modules): - if indices is not None and idx not in indices: - continue - - if self._verbose: - print(f"** module {module}") - - # Look over subdirs - iterable = self._output_subdirs - if len(iterable) > 1 and self._verbose: - iterable = tqdm(iterable, desc="subdirs", leave=False) - - if self._verbose: - print(f"*** subdirs {self._output_subdirs}") - - # Get output file names and paths - self.get_names_in_dir( - iterable, - module, - idx, - ) - - # If expected is string: Update parameter with runtime value - # and set as integer - if isinstance(self._key_expected[idx], str): - n_expected_base = get_par_runtime( - par_runtime, self._key_expected[idx], kind="n" - ) - else: - n_expected_base = self._key_expected[idx] - - # Get some numbers - n_found = len(self._names_in_dir[idx]) - n_expected = n_expected_base * self._n_mult[idx] - n_missing = n_expected - n_found - - n_special = self.check_special( - module, - idx, - ) - - # Print statistics - self.print_stats( - module, - n_expected, - n_found, - n_special, - n_missing, - idx, - ) - - # Write missing IDs for module to file - if n_missing > 0: - missing_IDs = self.output_missing( - module, - idx, - par_runtime=par_runtime, - ) - n_missing_job += n_missing - self._missing_IDs_job.extend(missing_IDs) - - # Empty line after job - logging.info("") - - # Write missing IDs for entire job to file - # if n_missing_job > 0: - self.output_missing_job() - - -def get_par_runtime(par_runtime, key, kind="n"): - """Get Par RunTime. - - Return runtime parameter value. - - Parameters - ---------- - par_runtime: dict - runtime parameter - key: str - key - - """ - combined_key = f"{kind}_{key}" - - return par_runtime[combined_key] - - -def print_par_runtime(par_runtime, verbose=True): - # Print runtime parameter values - if True: - logging.info("") - logging.info("===========") - logging.info("par_runtime") - logging.info("-----------") - for key, value in par_runtime.items(): - if not key.startswith("list"): - logging.info(f"{key:30s} {value:6d}") - else: - # logging.info(f"{key:30s} {len(value):6d} entries") - pass - logging.info("===========") - logging.info("") diff --git a/src/shapepipe/utilities/summary_params_pre_v2.py b/src/shapepipe/utilities/summary_params_pre_v2.py deleted file mode 100644 index 2def4d33b..000000000 --- a/src/shapepipe/utilities/summary_params_pre_v2.py +++ /dev/null @@ -1,276 +0,0 @@ -# Parameters for summary run - -import logging -import os - -from shapepipe.utilities import summary - - -def set_jobs_v2_pre_v2(patch, verbose): - """Return information about shapepipe jobs""" - print(f"Set job info for patch {patch}") - - # Main input and output directory - path_main = f"{os.environ['HOME']}/cosmostat/v2/pre_v2/psfex/{patch}" - - # Logging - path = f"{path_main}/summary" - if not os.path.isdir(path): - os.mkdir(path) - log_file_name = f"{path}/summary_log.txt" - handlers = [ - logging.FileHandler(log_file_name, mode="w"), - logging.StreamHandler(), - ] - logging.basicConfig( - level=logging.INFO, format="%(message)s", handlers=handlers - ) - - logging.info(f"Checking main directory = {path_main}") - - # Tile IDs - tile_ID_path = f"{path_main}/tile_numbers.txt" - - ## Tile IDs with dots - list_tile_IDs_dot = summary.get_IDs_from_file(tile_ID_path) - - jobs = {} - - # Set the first job (retrieve images) - - # With "CFIS_" only the linked images are counted. The original - # ones do not match the IDdash pattern. - # If images were downloaded in several runs: - # - Only copy original images, then (re-)set links in SP numbering format - # - get_images_runner_run_[12] consistent - # - remove previous output dirs since only last is searched - jobs["1"] = summary.job_data( - 1, - "run_sp_GitFeGie", - [ - "get_images_runner_run_1", - "find_exposures_runner", - "get_images_runner_run_2", - ], - ["tile_IDs", "tile_IDs", "exposures"], - pattern=["CFIS_", "", ""], - n_mult=[2, 1, 3], - path_main=path_main, - path_left="output", - verbose=verbose, - ) - - if patch in ("P8", "P9"): - jobs["2"] = summary.job_data( - 2, - ["run_sp_Uz", "run_sp_exp_Sp_shdu"], - ["uncompress_fits_runner", "split_exp_runner"], - ["tile_IDs", "shdus"], - n_mult=[1, 3], - path_main=path_main, - path_left=["output", "exp_runs"], - output_subdirs=[None, "shdus"], - path_right=[None, "output"], - verbose=verbose, - ) - jobs["4096"] = summary.job_data( - 4096, - ["run_sp_exp_Sp_shdu"], - ["split_exp_runner"], - ["shdus"], - n_mult=4, - path_main=path_main, - path_left="exp_runs", - output_subdirs="shdus", - path_right="output", - verbose=verbose, - ) - else: - jobs["2"] = summary.job_data( - 2, - ["run_sp_Uz", "run_sp_exp_SpMh"], - ["uncompress_fits_runner", "split_exp_runner"], - ["tile_IDs", "shdus"], - n_mult=[1, 121], - path_main=path_main, - path_left="output", - verbose=verbose, - ) - - jobs["4"] = summary.job_data( - 4, - ["run_sp_Ma_tile"], - ["mask_runner"], - ["tile_IDs"], - path_main=path_main, - path_left="output", - verbose=verbose, - ) - - if patch not in ("P8", "P9"): - jobs["8"] = summary.job_data( - 8, - ["run_sp_Ma_exp"], - ["mask_runner"], - ["shdus"], - path_main=path_main, - path_left="output", - verbose=verbose, - ) - else: - jobs["8"] = summary.job_data( - 8, - ["run_sp_exp_Ma"], - ["mask_runner"], - ["shdus"], - n_mult=[1], - path_main=path_main, - path_left="exp_runs", - output_subdirs= "shdus", - path_right="output", - verbose=verbose, - ) - - - jobs["16"] = summary.job_data( - 16, - "run_sp_tile_Sx", - ["sextractor_runner", "sextractor_runner"], - "tile_IDs", - n_mult=[2, 1], - path_main=path_main, - path_left="tile_runs", - path_output=["output", "logs"], - output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot], - special=[False, True], - verbose=verbose, - ) - - jobs["32"] = summary.job_data( - 32, - [ - "run_sp_exp_SxSePsf", - "run_sp_exp_SxSePsf", - "run_sp_exp_SxSePsf", - "run_sp_exp_SxSePsf", - "run_sp_exp_SxSePsf", - ], - [ - "sextractor_runner", - "setools_runner", - "setools_runner", - "setools_runner", - "psfex_runner", - ], - "shdus", - n_mult=[2, 2, 1, 1, 2], - path_main=path_main, - path_left="exp_runs", - output_subdirs="shdus", - path_right="output", - path_output=[ - "output", - "output/rand_split", - "output/stat", - "logs", - "output", - ], - special=[False, False, True, True, False], - verbose=verbose, - ) - - jobs["64"] = summary.job_data( - "64", - "run_sp_tile_PsViSmVi", - [ - "psfex_interp_runner", - "psfex_interp_runner", - "vignetmaker_runner_run_1", - "spread_model_runner", - "vignetmaker_runner_run_2", - ], - "tile_IDs", - n_mult=[1, 1, 1, 1, 4], - path_main=path_main, - path_left="tile_runs", - output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot], - path_output=["output", "logs", "output", "output", "output"], - special=[False, True, False, False, False], - verbose=verbose, - ) - - if patch in ("P2", "P5", "P8", "P9"): - n_sh = 1 - else: - n_sh = 8 - run_dirs = [f"run_sp_tile_ngmix_Ng{idx+1}u" for idx in range(n_sh)] - - # Add special (unfinished run) - run_dirs.append("run_sp_tile_ngmix_Ng1u") - - output_path_missing_IDs = [ - f"{path_main}/summary/missing_job_128_ngmix_runner_{idx+1}.txt" - for idx in range(n_sh + 1) - ] - jobs["128"] = summary.job_data( - "128", - run_dirs, - ["ngmix_runner"] * (n_sh + 1), - "tile_IDs", - path_main=path_main, - path_left="tile_runs", - output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot], - path_output=["output"] * n_sh + ["logs"], - special=[False] * n_sh + [True], - output_path_missing_IDs=output_path_missing_IDs, - verbose=verbose, - ) - - jobs["256"] = summary.job_data( - "256", - "run_sp_Ms", - ["merge_sep_cats_runner"] * 2, - "tile_IDs", - path_main=path_main, - path_left="tile_runs", - path_output=["output", "logs"], - special=[False, True], - output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot], - verbose=verbose, - ) - - jobs["512"] = summary.job_data( - "512", - ["run_sp_Mc"], - ["make_cat_runner"], - "tile_IDs", - path_main=path_main, - path_left="tile_runs", - output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot], - verbose=verbose, - ) - - # Post-processing - jobs["1024"] = summary.job_data( - "1024", - ["run_sp_combined_final"], - ["make_catalog_runner"], - "tile_IDs", - path_main=path_main, - path_left="output", - verbose=verbose, - ) - - jobs["2048"] = summary.job_data( - "2048", - "run_sp_combined_psf", - ["psfex_interp_runner"], - "shdus", - path_main=path_main, - path_left="output", - verbose=verbose, - ) - - return jobs, list_tile_IDs_dot - - diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py index 1f4fd3472..0bf077a78 100644 --- a/src/shapepipe/utilities/vizier.py +++ b/src/shapepipe/utilities/vizier.py @@ -64,8 +64,16 @@ def query_vizier(ra, dec, radius_arcmin, cat_id): v = Vizier( row_limit=-1, timeout=timeout, vizier_server=server ) + # cache=False: astroquery otherwise pickles every HTTP response into + # $HOME/.astropy/cache/astroquery/Vizier, ~2 MB per query. The + # workflow already caches the RESULT as a FITS catalogue on scratch + # and skips the query when it hits, so the pickle is pure duplicate — + # and at campaign scale (~25k exposures) it is ~50 GB against a + # 50 GB home quota. Home is for source and config, not for a second + # copy of the survey. result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id + coord, radius=radius_arcmin * u.arcmin, catalog=cat_id, + cache=False, ) if len(result) > 0: print( diff --git a/tests/module/test_collate_star_cat.py b/tests/module/test_collate_star_cat.py deleted file mode 100644 index 20d52ce23..000000000 --- a/tests/module/test_collate_star_cat.py +++ /dev/null @@ -1,70 +0,0 @@ -"""UNIT TESTS FOR STAR-CATALOGUE COLLATION PATHS. - -Pin the patch vs patch-less (v2.0) path and filename convention of -``scripts/python/collate_star_cat.py``. Runs up to v1.6 carry a ``P`` -token in both the input run directory and the output filename; v2.0 is -patch-less (``patch is None``) and drops that token, reading from a single -``/output`` root and writing ``validation_psf_conv-.fits`` — the -name still matched by the downstream ``validation_psf_conv-*`` glob. -""" - -import importlib.util -from pathlib import Path - -import pytest - -# The collation script lives under scripts/python (not an importable package), -# so load it by path. -_SCRIPT = ( - Path(__file__).resolve().parents[2] - / "scripts" - / "python" - / "collate_star_cat.py" -) -_spec = importlib.util.spec_from_file_location("collate_star_cat", _SCRIPT) -collate_star_cat = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(collate_star_cat) - - -@pytest.mark.parametrize( - "patch, exp_input, exp_output", - [ - ("3", "in/P3/output/", "out/P3"), - (None, "in/output/", "out"), - ], -) -def test_collate_paths(patch, exp_input, exp_output): - """v1.x carries the P token; v2.0 (patch None) drops it.""" - assert collate_star_cat.collate_paths("in", "out", patch) == ( - exp_input, - exp_output, - ) - - -@pytest.mark.parametrize( - "patch, expected", - [ - ("3", "validation_psf_conv-3-0.fits"), - (None, "validation_psf_conv-0.fits"), - ], -) -def test_output_filename(patch, expected): - """The patch token is present for v1.x and absent for v2.0.""" - assert collate_star_cat.output_filename("validation_psf", patch, 0) == expected - - -def test_output_filename_matches_downstream_glob(): - """Both layouts stay under the downstream ``validation_psf_conv-*`` glob.""" - for patch in ("1", None): - assert collate_star_cat.output_filename( - "validation_psf", patch, 5 - ).startswith("validation_psf_conv-") - - -@pytest.mark.parametrize("bad", ["v2", "2.0", "v1.7", ""]) -def test_invalid_version_raises(bad): - """A mistyped -V is rejected rather than falling through to v1.x.""" - obj = collate_star_cat.Convert() - obj._params["version_cat"] = bad - with pytest.raises(ValueError): - obj.run() diff --git a/tests/module/test_coverage.py b/tests/module/test_coverage.py index ea48905c3..26d98a809 100644 --- a/tests/module/test_coverage.py +++ b/tests/module/test_coverage.py @@ -1,49 +1,35 @@ -"""UNIT / PROPERTY TESTS FOR THE COVERAGE-MASK FEATURE. - -Covers the pure and lightly-fixtured logic behind the per-CCD coverage nexp -masks: exposure-number parsing, the CCD-list -> unique-exposure reduction, the -handler's missing-CCD subtraction (pinned against the real ID format), per-CCD -corner extraction from plain and fpack-compressed multi-HDU headers, -``--ccd_list`` filtering, the per-CCD resume path, the builder's per-CCD row -parsing, the accumulated exposure-count (nexp) contract, the RA-wrap and pole -guards, the power-of-two ``nside`` validation, the atomic-download rename, and -the ``-h``/argv handling of the console runners. The heavy paths (real VOSpace -download, production-resolution map building, plotting, multiprocessing) are -exercised end to end by the pipeline, not here. +"""UNIT / PROPERTY TESTS FOR THE COVERAGE-MASK GEOMETRY. + +Covers the pure logic the coverage nexp mask rests on, on both sides of the +``exp_footprint`` -> ``coverage_map`` handover: per-CCD image-shape resolution +from plain and fpack-compressed headers, the sky-corner projection, and then +the accumulated exposure-count (nexp) contract, the RA-wrap and pole guards, +the power-of-two ``nside`` validation and the median smoothing of a finished +map. Everything here takes arrays or headers in memory; the way a record +reaches these functions is ``tests/unit/test_exp_footprint.py``'s subject, and +the heavy paths (production-resolution map building, plotting) are exercised +end to end by the pipeline. """ -import sys -from types import SimpleNamespace - +import healsparse as hsp +import hpgeom as hpg import numpy as np import numpy.testing as npt import pytest from astropy import wcs -from hypothesis import given -from hypothesis import strategies as st +from astropy.io.fits import Header -from shapepipe.utilities import summary -from shapepipe.utilities.ccd_psf_handler import CcdPsfHandler +from shapepipe.utilities.ccd_footprint import _ccd_corners, _image_shape from shapepipe.utilities.coverage_map_builder import ( - CoverageMapBuilder, + build_map, + check_nside, + median_filter, unwrap_ra, ) -from shapepipe.utilities.field_corners_extractor import ( - FieldCornersExtractor, - _ccd_corners, - _expnum_from_path, - _image_shape, - _parse_header_to_wcs, -) -from shapepipe.utilities.header_downloader import HeaderDownloader def _tan_wcs(crval_ra, crval_dec=0.0, nx=2080, ny=4612): - """Build a minimal 2-D TAN WCS with a populated pixel shape. - - ``nx``/``ny`` set ``pixel_shape`` so ``_header_text`` can emit matching - ``NAXIS1``/``NAXIS2`` keywords. - """ + """Build a minimal 2-D TAN WCS with a populated pixel shape.""" w = wcs.WCS(naxis=2) w.wcs.ctype = ["RA---TAN", "DEC--TAN"] w.wcs.crval = [crval_ra, crval_dec] @@ -53,244 +39,61 @@ def _tan_wcs(crval_ra, crval_dec=0.0, nx=2080, ny=4612): return w -def _header_text(wcs_list): - """Render a multi-HDU header text file: a primary HDU then one plain image - HDU per WCS, each carrying ``NAXIS1``/``NAXIS2``. - """ - blocks = ["SIMPLE = T"] - for w in wcs_list: - nx, ny = w.pixel_shape - header = w.to_header() - header["NAXIS"] = 2 - header["NAXIS1"] = nx - header["NAXIS2"] = ny - blocks.append("\n".join(str(card) for card in header.cards)) - return "".join(f"{block}\nEND \n" for block in blocks) +def _plain_header(w): + """A plain image-HDU header: ``NAXIS1``/``NAXIS2`` are the image.""" + nx, ny = w.pixel_shape + header = w.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = nx + header["NAXIS2"] = ny + return header -def _compressed_header_text(w): - """Render a one-CCD header mimicking an fpack tile-compressed HDU. +def _compressed_header(w): + """A header shaped like an fpack tile-compressed HDU. ``NAXIS1``/``NAXIS2`` describe the compressed binary table (byte width, row - count); the true image dimensions live in ``ZNAXIS1``/``ZNAXIS2``. This is - the shape of headers fetched from 'p.fits.fz' with ``head=True``. + count); the true image dimensions live in ``ZNAXIS1``/``ZNAXIS2``. """ nx, ny = w.pixel_shape - wcs_cards = "\n".join(str(card) for card in w.to_header().cards) - ccd = ( - "XTENSION= 'BINTABLE'\n" - "BITPIX = 8\n" - "NAXIS = 2\n" - "NAXIS1 = 8\n" - f"NAXIS2 = {ny}\n" - "PCOUNT = 1000000\n" - "GCOUNT = 1\n" - "TFIELDS = 1\n" - "ZIMAGE = T\n" - "ZBITPIX = -32\n" - "ZNAXIS = 2\n" - f"ZNAXIS1 = {nx}\n" - f"ZNAXIS2 = {ny}\n" - f"{wcs_cards}" - ) - return f"SIMPLE = T\nEND \n{ccd}\nEND \n" - - -# --------------------------------------------------------------------------- -# _expnum_from_path -# --------------------------------------------------------------------------- - -@pytest.mark.parametrize( - "path, expected", - [ - ("1234567.txt", 1234567), - ("/a/b/2143523.txt", 2143523), - ("headers/0000042.txt", 42), - ], -) -def test_expnum_from_path_extracts_trailing_number(path, expected): - """The trailing ``.txt`` is parsed as the exposure number.""" - assert _expnum_from_path(path) == expected - - -@pytest.mark.parametrize("path", ["no_number.txt", "1234567.fits", "abc.txt"]) -def test_expnum_from_path_raises_without_number(path): - """A filename without a trailing numeric stem raises ``ValueError``.""" - with pytest.raises(ValueError): - _expnum_from_path(path) - - -@given(st.integers(min_value=0, max_value=99999999)) -def test_expnum_from_path_roundtrips(expnum): - """Any exposure number round-trips through the filename convention.""" - assert _expnum_from_path(f"vos_headers/{expnum}.txt") == expnum - - -# --------------------------------------------------------------------------- -# HeaderDownloader.get_exposures -# --------------------------------------------------------------------------- - -def test_get_exposures_reduces_to_unique_exposures(tmp_path): - """A ``-`` CCD list collapses to its unique exposure numbers.""" - ccd_list = tmp_path / "ccds.txt" - ccd_list.write_text("2143523-0\n2143523-5\n2143524-3\n2143524-8\n") - - exps = HeaderDownloader().get_exposures(str(ccd_list)) - - npt.assert_array_equal(exps, np.array([2143523, 2143524])) - - -def test_get_exposures_single_line(tmp_path): - """A one-line CCD list (0-d loadtxt array) still yields one exposure.""" - ccd_list = tmp_path / "ccds.txt" - ccd_list.write_text("2143523-0\n") - - exps = HeaderDownloader().get_exposures(str(ccd_list)) - - npt.assert_array_equal(exps, np.array([2143523])) - - -def test_get_exposures_csv_matches_txt(tmp_path): - """The CSV and text code paths yield the same unique exposures.""" - txt = tmp_path / "ccds.txt" - txt.write_text("2143523-0\n2143523-5\n2143524-3\n") - csv = tmp_path / "ccds.csv" - csv.write_text("CCD\n2143523-0\n2143523-5\n2143524-3\n") - - dl = HeaderDownloader() - - npt.assert_array_equal( - dl.get_exposures(str(txt)), dl.get_exposures(str(csv)) - ) - - -# --------------------------------------------------------------------------- -# HeaderDownloader.get_fits_header — atomic rename -# --------------------------------------------------------------------------- - -def test_get_fits_header_writes_atomically(tmp_path): - """A successful download copies to ``.part`` then renames to the dest.""" - dl = HeaderDownloader() - dl._params["output_dir"] = str(tmp_path) - dl._params["overwrite"] = False - dl._params["dir_for_links"] = None - - dest = tmp_path / "42.txt" - tmp_dest = tmp_path / "42.txt.part" - - def fake_copy(source, target, head=True): - # The copy must land on the temp path, not the final destination. - assert target == str(tmp_dest) - with open(target, "w") as f: - f.write("HEADER") - - client = SimpleNamespace(copy=fake_copy) - - assert dl.get_fits_header(42, client) is True - assert dest.exists() - assert not tmp_dest.exists() - assert dest.read_text() == "HEADER" - - -def test_get_fits_header_failed_copy_leaves_no_dest(tmp_path): - """A failed download leaves no destination file (only, if any, ``.part``).""" - dl = HeaderDownloader() - dl._params["output_dir"] = str(tmp_path) - dl._params["overwrite"] = False - dl._params["dir_for_links"] = None - - def failing_copy(source, target, head=True): - raise RuntimeError("transfer interrupted") - - client = SimpleNamespace(copy=failing_copy) - - assert dl.get_fits_header(42, client) is False - assert not (tmp_path / "42.txt").exists() - assert not (tmp_path / "42.txt.part").exists() - - -# --------------------------------------------------------------------------- -# CcdPsfHandler.get_ccds_with_psf — missing-CCD subtraction -# --------------------------------------------------------------------------- - -def test_get_ccds_with_psf_subtracts_missing(monkeypatch): - """Valid CCDs are all exposure single-HDUs minus the missing set. - - The real ``summary.get_all_shdus`` is used so the cross-component - ``-`` ID format is pinned end to end. - """ - handler = CcdPsfHandler() - - # Two exposures, 3 CCDs each -> 6 candidate CCDs; two are missing. - monkeypatch.setattr(handler, "get_exp", lambda patches: {"100", "200"}) - monkeypatch.setattr( - handler, - "get_exp_shdu_missing", - lambda patches: {"100-1", "200-2"}, - ) - - result = handler.get_ccds_with_psf(["P1"], n_CCD=3) - - # get_all_shdus yields "-" for ccd in range(n_CCD). - assert result == {"100-0", "100-2", "200-0", "200-1"} - # Guard the assumption that the missing IDs share the produced format. - assert set(summary.get_all_shdus({"100"}, 3)) == {"100-0", "100-1", "100-2"} - - -@pytest.mark.parametrize( - ("version", "n_patch"), - [("v1.3", 7), ("v1.4", 7), ("v1.5", 8), ("v1.6", 9)], -) -def test_version_to_patch_count(version, n_patch): - """Each v1.x catalogue version maps to its patch count.""" - handler = CcdPsfHandler() - handler._params["version_cat"] = version - handler.update_params() - assert handler._params["n_patch"] == n_patch - assert len(handler._params["patches"]) == n_patch - - -def test_invalid_version_raises(): - """An unknown catalogue version fails loudly.""" - handler = CcdPsfHandler() - handler._params["version_cat"] = "v9.9" - with pytest.raises(ValueError, match="v9.9"): - handler.update_params() + header = w.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 8 + header["NAXIS2"] = ny + header["ZIMAGE"] = True + header["ZNAXIS"] = 2 + header["ZNAXIS1"] = nx + header["ZNAXIS2"] = ny + return header # --------------------------------------------------------------------------- # image-shape resolution (fpack ZNAXIS vs plain NAXIS) # --------------------------------------------------------------------------- -def test_image_shape_prefers_znaxis_for_compressed_header(tmp_path): +def test_image_shape_prefers_znaxis_for_compressed_header(): """A compressed HDU reports ZNAXIS dims, not the binary-table NAXIS.""" - w = _tan_wcs(100.0, 20.0, nx=2080, ny=4612) - path = tmp_path / "1234567.txt" - path.write_text(_compressed_header_text(w)) - - (parsed_w, shape), = _parse_header_to_wcs(str(path)) + header = _compressed_header(_tan_wcs(100.0, 20.0, nx=2080, ny=4612)) # WCS pixel_shape would wrongly report the compressed byte width (8). - assert parsed_w.pixel_shape == (8, 4612) + assert wcs.WCS(header).pixel_shape == (8, 4612) # _image_shape recovers the true image dimensions. - assert shape == (2080, 4612) + assert _image_shape(header) == (2080, 4612) -def test_image_shape_falls_back_to_naxis(tmp_path): - """A plain image HDU (no ZIMAGE) uses NAXIS1/NAXIS2.""" - w = _tan_wcs(100.0, 20.0, nx=2080, ny=4612) - path = tmp_path / "1234567.txt" - path.write_text(_header_text([w])) +def test_image_shape_falls_back_to_naxis(): + """A plain image HDU (no ZIMAGE) uses NAXIS1/NAXIS2. - (_, shape), = _parse_header_to_wcs(str(path)) + This is the live path for the workflow: ``headers-.npy`` carries the + decompressed header astropy hands back for a tile-compressed HDU. + """ + header = _plain_header(_tan_wcs(100.0, 20.0, nx=2080, ny=4612)) - assert shape == (2080, 4612) + assert _image_shape(header) == (2080, 4612) def test_image_shape_raises_without_dimensions(): """A header with no image dimensions raises a clear ``ValueError``.""" - from astropy.io.fits import Header - header = Header() header["CTYPE1"] = "RA---TAN" @@ -298,18 +101,14 @@ def test_image_shape_raises_without_dimensions(): _image_shape(header) -def test_compressed_header_corners_are_full_width(tmp_path): +def test_compressed_header_corners_are_full_width(): """Corners from a compressed header span the true CCD width, not 8 px.""" - w = _tan_wcs(100.0, 20.0, nx=2080, ny=4612) - path = tmp_path / "1234567.txt" - path.write_text(_compressed_header_text(w)) + header = _compressed_header(_tan_wcs(100.0, 20.0, nx=2080, ny=4612)) - (parsed_w, shape), = _parse_header_to_wcs(str(path)) - ra, dec = _ccd_corners(parsed_w, shape) + ra, dec = _ccd_corners(wcs.WCS(header), _image_shape(header)) # RA extent must reflect ~2080 px * 1e-5 deg/px * cos(dec), not 8 px. - ra_extent = max(ra) - min(ra) - assert ra_extent > 0.01 + assert max(ra) - min(ra) > 0.01 # --------------------------------------------------------------------------- @@ -332,257 +131,56 @@ def test_ccd_corners_returns_four_corners_around_centre(): npt.assert_allclose(max(dec) - min(dec), ny * 1e-5, rtol=1e-3) -def test_parse_header_to_wcs_returns_one_wcs_per_extension(tmp_path): - """One (wcs, shape) pair is returned per CCD HDU; the primary is skipped.""" - ccd_wcs = [_tan_wcs(10.0), _tan_wcs(20.0), _tan_wcs(30.0)] - - path = tmp_path / "1234567.txt" - path.write_text(_header_text(ccd_wcs)) - - result = _parse_header_to_wcs(str(path)) - - assert len(result) == len(ccd_wcs) - assert all(shape == (2080, 4612) for _, shape in result) - - -# --------------------------------------------------------------------------- -# FieldCornersExtractor.process_single_header — per-CCD rows and filtering -# --------------------------------------------------------------------------- - -def test_process_single_header_emits_one_row_per_ccd(tmp_path): - """Without a CCD list, every HDU yields a ``-`` row.""" - ccd_wcs = [_tan_wcs(10.0), _tan_wcs(20.0), _tan_wcs(30.0)] - path = tmp_path / "1234567.txt" - path.write_text(_header_text(ccd_wcs)) - - rows = FieldCornersExtractor.process_single_header( - (str(path), False, None) - ) - - assert [r[0] for r in rows] == [ - "1234567-0", - "1234567-1", - "1234567-2", - ] - assert all(len(r[1]) == 4 and len(r[2]) == 4 for r in rows) - - -def test_process_single_header_filters_to_ccd_list(tmp_path): - """A ``valid_ccds`` set keeps only the listed CCDs of the exposure.""" - ccd_wcs = [_tan_wcs(10.0), _tan_wcs(20.0), _tan_wcs(30.0)] - path = tmp_path / "1234567.txt" - path.write_text(_header_text(ccd_wcs)) - - rows = FieldCornersExtractor.process_single_header( - (str(path), False, {"1234567-0", "1234567-2"}) - ) - - assert [r[0] for r in rows] == ["1234567-0", "1234567-2"] - - -def test_load_ccd_list_reads_ids(tmp_path): - """The CCD list loader returns the stripped, non-blank IDs as a set.""" - path = tmp_path / "ccds.txt" - path.write_text("100-0\n100-1\n\n200-5\n") - - assert FieldCornersExtractor.load_ccd_list(str(path)) == { - "100-0", - "100-1", - "200-5", - } - - -def test_run_extract_writes_per_ccd_rows(tmp_path): - """End to end: run() writes one per-CCD row filtered by the CCD list.""" - header_dir = tmp_path / "headers" - header_dir.mkdir() - ccd_wcs = [_tan_wcs(10.0), _tan_wcs(20.0), _tan_wcs(30.0)] - (header_dir / "1234567.txt").write_text(_header_text(ccd_wcs)) - - ccd_list = tmp_path / "ccds.txt" - ccd_list.write_text("1234567-0\n1234567-2\n") - - out = tmp_path / "corners.txt" - - extractor = FieldCornersExtractor() - extractor.run( - args=[ - "-i", str(header_dir), - "-l", str(ccd_list), - "-o", str(out), - ] - ) - - lines = out.read_text().splitlines() - assert len(lines) == 2 - ids = [line.split()[0] for line in lines] - assert ids == ["1234567-0", "1234567-2"] - # Each row: 1 ID + 4 RA + 4 Dec = 9 columns. - assert all(len(line.split()) == 9 for line in lines) - - -# --------------------------------------------------------------------------- -# FieldCornersExtractor resume path (per-CCD done-set) -# --------------------------------------------------------------------------- - -def _write_headers(header_dir, expnums, n_ccd=3): - """Write one plain multi-HDU header per exposure into ``header_dir``.""" - header_dir.mkdir(exist_ok=True) - for j, expnum in enumerate(expnums): - ccd_wcs = [_tan_wcs(10.0 + j + i) for i in range(n_ccd)] - (header_dir / f"{expnum}.txt").write_text(_header_text(ccd_wcs)) - - -def test_get_done_ccds_reads_present_ids(tmp_path): - """The done-set is the exact set of CCD IDs already in the output.""" - out = tmp_path / "corners.txt" - out.write_text( - "1234567-0 10 10 10 10 20 20 20 20\n" - "1234567-2 10 10 10 10 20 20 20 20\n" - ) - - extractor = FieldCornersExtractor() - extractor._params["output_file"] = str(out) - - assert extractor.get_done_ccds() == {"1234567-0", "1234567-2"} - - -def test_resume_adds_new_exposure_without_duplicating(tmp_path): - """Resume appends a new exposure and leaves existing rows untouched.""" - header_dir = tmp_path / "headers" - _write_headers(header_dir, [1000001, 1000002]) - - out = tmp_path / "corners.txt" - # Pre-populate with exposure 1's three CCD rows. - pre = FieldCornersExtractor().process_single_header( - (str(header_dir / "1000001.txt"), False, None) - ) - with open(out, "w") as f: - for ccd_id, ra, dec in pre: - f.write(f"{ccd_id} " + " ".join(f"{v:.5f}" for v in ra + dec) + "\n") - - FieldCornersExtractor().run( - args=["-i", str(header_dir), "-o", str(out), "-r"] - ) - - ids = [line.split()[0] for line in out.read_text().splitlines()] - # No duplicate exposure-1 rows; exposure 2's three CCDs added. - assert ids.count("1000001-0") == 1 - assert sorted(ids) == [ - "1000001-0", "1000001-1", "1000001-2", - "1000002-0", "1000002-1", "1000002-2", - ] - - -def test_resume_completes_partial_exposure(tmp_path): - """An exposure interrupted mid-write is completed, not skipped.""" - header_dir = tmp_path / "headers" - _write_headers(header_dir, [1000001]) - - out = tmp_path / "corners.txt" - # Simulate an interrupt: only the first of exposure 1's CCDs was written. - pre = FieldCornersExtractor().process_single_header( - (str(header_dir / "1000001.txt"), False, None) - ) - ccd_id, ra, dec = pre[0] - with open(out, "w") as f: - f.write(f"{ccd_id} " + " ".join(f"{v:.5f}" for v in ra + dec) + "\n") - - FieldCornersExtractor().run( - args=["-i", str(header_dir), "-o", str(out), "-r"] - ) - - ids = [line.split()[0] for line in out.read_text().splitlines()] - # The missing CCDs are filled in; the present one is not duplicated. - assert sorted(ids) == ["1000001-0", "1000001-1", "1000001-2"] - assert ids.count("1000001-0") == 1 - - # --------------------------------------------------------------------------- -# CoverageMapBuilder.check_params (nside validation) +# nside validation # --------------------------------------------------------------------------- -def test_check_params_accepts_power_of_two_nside(tmp_path): +def test_check_nside_accepts_powers_of_two(): """Powers of two for both nside values pass validation.""" - infile = tmp_path / "corners.txt" - infile.write_text("100-0 0.0 1.0 1.0 0.0 0.0 0.0 1.0 1.0\n") - - builder = CoverageMapBuilder() - builder._params["input_file"] = str(infile) - builder._params["nside_coverage"] = 32 - builder._params["nside"] = 2048 - - builder.check_params() + check_nside(32, 2048) @pytest.mark.parametrize( - "nside_coverage, nside", [(33, 2048), (32, 100), (32, 3000)] + "nside_coverage, nside", [(33, 2048), (32, 100), (32, 3000), (0, 2048)] ) -def test_check_params_rejects_non_power_of_two_nside( - tmp_path, nside_coverage, nside -): +def test_check_nside_rejects_non_power_of_two(nside_coverage, nside): """A non-power-of-two nside raises ``ValueError``.""" - infile = tmp_path / "corners.txt" - infile.write_text("100-0 0.0 1.0 1.0 0.0 0.0 0.0 1.0 1.0\n") - - builder = CoverageMapBuilder() - builder._params["input_file"] = str(infile) - builder._params["nside_coverage"] = nside_coverage - builder._params["nside"] = nside - with pytest.raises(ValueError): - builder.check_params() - + check_nside(nside_coverage, nside) -def test_check_params_missing_input_file_raises(tmp_path): - """A missing input file raises ``FileNotFoundError``.""" - builder = CoverageMapBuilder() - builder._params["input_file"] = str(tmp_path / "does_not_exist.txt") - with pytest.raises(FileNotFoundError): - builder.check_params() +def test_build_map_validates_nside_before_stamping(): + """``build_map`` refuses a bad nside before it reaches healsparse.""" + with pytest.raises(ValueError, match="nside"): + build_map(["100-0"], [0.0, 0.1, 0.1, 0.0], [0.0, 0.0, 0.1, 0.1], + 32, 3000) # --------------------------------------------------------------------------- -# CoverageMapBuilder — parsing, nexp contract, RA-wrap and pole guards +# build_map — nexp contract, RA-wrap and pole guards # --------------------------------------------------------------------------- -def _read_map(path): - import healsparse as hsp - - return hsp.HealSparseMap.read(str(path)) - - -def test_build_map_single_row(tmp_path): - """A one-row corners file exercises the atleast_1d/2d parsing guards.""" - infile = tmp_path / "corners.txt" - infile.write_text("100-0 9.9 10.1 10.1 9.9 0.0 0.0 0.2 0.2\n") - out = tmp_path / "cov.hsp" - - CoverageMapBuilder().run( - args=["-i", str(infile), "-o", str(out), "-c", "32", "-n", "1024"] +def test_build_map_single_footprint(): + """One footprint given as flat corner lists exercises the shape guards.""" + m = build_map( + ["100-0"], [9.9, 10.1, 10.1, 9.9], [0.0, 0.0, 0.2, 0.2], 32, 1024 ) - m = _read_map(out) assert 0 < len(m.valid_pixels) < 200 assert m[m.valid_pixels].max() == 1 -def test_build_map_nexp_counts_overlapping_exposures(tmp_path): - """Two overlapping CCDs from different exposures give value 2 in overlap.""" +def test_build_map_nexp_counts_overlapping_exposures(): + """Two overlapping CCDs from two exposures give value 2 in the overlap.""" # Two 0.4x0.4 deg CCDs offset by 0.2 deg in RA -> a central overlap strip. - infile = tmp_path / "corners.txt" - infile.write_text( - "100-0 9.8 10.2 10.2 9.8 19.8 19.8 20.2 20.2\n" - "200-0 10.0 10.4 10.4 10.0 19.8 19.8 20.2 20.2\n" + m = build_map( + ["100-0", "200-0"], + [[9.8, 10.2, 10.2, 9.8], [10.0, 10.4, 10.4, 10.0]], + [[19.8, 19.8, 20.2, 20.2], [19.8, 19.8, 20.2, 20.2]], + 32, + 1024, ) - out = tmp_path / "cov.hsp" - CoverageMapBuilder().run( - args=["-i", str(infile), "-o", str(out), "-c", "32", "-n", "1024"] - ) - - m = _read_map(out) values = m[m.valid_pixels] # The overlap is covered by both exposures (value 2); the union edges by # one (value 1). Both must be present; nothing exceeds 2. @@ -606,30 +204,19 @@ def test_unwrap_ra_leaves_normal_polygon_unchanged(): ) -def test_build_map_invariant_under_ra_shift(tmp_path): +def test_build_map_invariant_under_ra_shift(): """A seam CCD's footprint matches an identical CCD shifted +10 deg in RA. - Rotating both the seam CCD (via +360/unwrap) and a reference CCD onto the - same RA and comparing pixel counts fails if the seam polygon were filling + Comparing the seam polygon's pixel count against a reference polygon of the + same size at the same declination fails if the seam polygon were filling the ~360 deg complement. This is the real RA-wrap regression guard. """ - # Seam CCD straddling RA=0, and the same CCD translated to RA~10. - seam = tmp_path / "seam.txt" - seam.write_text("100-0 359.9 0.1 0.1 359.9 20.0 20.0 20.2 20.2\n") - ref = tmp_path / "ref.txt" - ref.write_text("200-0 9.9 10.1 10.1 9.9 20.0 20.0 20.2 20.2\n") - - m_seam = tmp_path / "seam.hsp" - m_ref = tmp_path / "ref.hsp" - CoverageMapBuilder().run( - args=["-i", str(seam), "-o", str(m_seam), "-c", "32", "-n", "1024"] - ) - CoverageMapBuilder().run( - args=["-i", str(ref), "-o", str(m_ref), "-c", "32", "-n", "1024"] - ) + dec = [20.0, 20.0, 20.2, 20.2] + m_seam = build_map(["100-0"], [359.9, 0.1, 0.1, 359.9], dec, 32, 1024) + m_ref = build_map(["200-0"], [9.9, 10.1, 10.1, 9.9], dec, 32, 1024) - n_seam = len(_read_map(m_seam).valid_pixels) - n_ref = len(_read_map(m_ref).valid_pixels) + n_seam = len(m_seam.valid_pixels) + n_ref = len(m_ref.valid_pixels) # Same-size footprints at the same declination: pixel counts agree to # within a few boundary pixels, and are nowhere near a hemisphere. @@ -637,17 +224,14 @@ def test_build_map_invariant_under_ra_shift(tmp_path): assert 0 < n_seam < 200 -def test_build_map_pole_guard_skips_polygon(tmp_path, capsys): +def test_build_map_pole_guard_skips_polygon(capsys): """A polygon with |dec| near 90 deg is skipped with a warning.""" - infile = tmp_path / "corners.txt" - infile.write_text( - "100-0 10.0 10.2 10.2 10.0 89.5 89.5 89.7 89.7\n" - "200-0 10.0 10.2 10.2 10.0 20.0 20.0 20.2 20.2\n" - ) - out = tmp_path / "cov.hsp" - - CoverageMapBuilder().run( - args=["-i", str(infile), "-o", str(out), "-c", "32", "-n", "1024"] + build_map( + ["100-0", "200-0"], + [[10.0, 10.2, 10.2, 10.0], [10.0, 10.2, 10.2, 10.0]], + [[89.5, 89.5, 89.7, 89.7], [20.0, 20.0, 20.2, 20.2]], + 32, + 1024, ) captured = capsys.readouterr() @@ -656,19 +240,17 @@ def test_build_map_pole_guard_skips_polygon(tmp_path, capsys): # --------------------------------------------------------------------------- -# console-runner argv handling (regression guard for the -h entry points) +# median smoothing (offline; the coverage_map rule writes the raw nexp map) # --------------------------------------------------------------------------- -def test_run_help_flag_exits_cleanly(monkeypatch): - """``run()`` with no args reads ``sys.argv[1:]`` so ``-h`` exits 0. - - Guards the regression where the runners parsed the full ``sys.argv`` - (including ``argv[0]``), which made ``-h`` collide with the integer - ``-c`` option and exit non-zero. - """ - monkeypatch.setattr(sys, "argv", ["extract_field_corners", "-h"]) +def test_median_filter_fills_a_lone_hole(): + """A lone low pixel in a covered disc is pulled up to its neighbours.""" + nside = 1024 + m = hsp.HealSparseMap.make_empty(32, nside, np.uint16) + m[hpg.query_circle(nside, 10.0, 20.0, 0.1)] = 2 - with pytest.raises(SystemExit) as excinfo: - FieldCornersExtractor().run() + centre = hpg.angle_to_pixel(nside, 10.0, 20.0) + m[[centre]] = 1 + assert m[centre] == 1 - assert excinfo.value.code == 0 + assert median_filter(m)[centre] == 2 diff --git a/tests/module/test_coverage_map.py b/tests/module/test_coverage_map.py new file mode 100644 index 000000000..7ba895066 --- /dev/null +++ b/tests/module/test_coverage_map.py @@ -0,0 +1,158 @@ +"""The workflow's coverage map: exposure footprint records -> a HealSparse nexp map. + +``tests/module/test_coverage.py`` pins the nexp contract through the +``exp_ra_dec.txt`` CLI path. This module pins the SAME contract through the path +the Snakemake workflow actually takes — one ``exp_footprint.json`` per exposure +on the products root, globbed and fed to ``build_map`` as arrays — because that +path has two properties the text one does not and neither is visible in the map: + + * it is CAMPAIGN-CUMULATIVE by construction. The script globs every record + under ``/exp/*/*/manifests/``, sharded, including exposures + whose scratch stores were reclaimed. A regression that fed it only the + declared rule inputs would build a map of one batch and look fine. + * the records are RAW SKY. Unwrapping across RA=0 happens once, inside + ``build_map``; a record written pre-unwrapped, or unwrapped twice, silently + moves a footprint by 360 degrees. + +Two exposures, one CCD each, offset so they overlap — the same fixture geometry +as the CLI test, so the two routes are comparable by eye. +""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "coverage_map.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_coverage_map", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +coverage_map = _load() + + +def write_footprint(products, exp, ccds): + """One exposure's record, at the sharded path exp_footprint writes to.""" + path = (products / "exp" / exp[:2] / exp / "manifests" + / "exp_footprint.json") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({ + "stage": "exp_footprint", "level": "exp", "unit": exp, + "status": "complete", + "n_ccd_headers": len(ccds), "n_valid_psf": len(ccds), + "ccds": [{"id": f"{exp}-{i}", "ra": ra, "dec": dec} + for i, (ra, dec) in enumerate(ccds)], + "ccds_no_psf": [], + }, indent=2, sort_keys=True)) + return path + + +def box(ra_min, ra_max, dec_min, dec_max): + """A rectangular CCD footprint, corners counterclockwise from bottom-left.""" + return ([ra_min, ra_max, ra_max, ra_min], + [dec_min, dec_min, dec_max, dec_max]) + + +def run(products, tmp_path, monkeypatch, nside="1024"): + out = tmp_path / "coverage" / "coverage.hsp" + manifest = tmp_path / "coverage" / "manifests" / "coverage_map.json" + monkeypatch.setattr(sys, "argv", [ + "coverage_map.py", + "--products-dir", str(products), + "--out", str(out), "--manifest", str(manifest), + "--nside-coverage", "32", "--nside", nside]) + coverage_map.main() + import healsparse as hsp + return hsp.HealSparseMap.read(str(out)), json.loads(manifest.read_text()) + + +def test_two_exposures_accumulate_to_nexp(tmp_path, monkeypatch): + """Overlapping CCDs from two exposures give 2 in the overlap, 1 outside. + + The exposures live in DIFFERENT shard directories, which is the campaign + layout: a glob that assumed one shard would find only one of them. + """ + products = tmp_path / "products" + write_footprint(products, "1000001", [box(9.8, 10.2, 19.8, 20.2)]) + write_footprint(products, "2000001", [box(10.0, 10.4, 19.8, 20.2)]) + + m, manifest = run(products, tmp_path, monkeypatch) + + values = m[m.valid_pixels] + assert values.max() == 2 + assert values.min() == 1 + assert (values == 2).sum() > 0 + assert (values == 1).sum() > 0 + + assert manifest["n_exposures"] == 2 + assert manifest["exposures"] == ["1000001", "2000001"] + assert manifest["n_ccds"] == 2 + assert manifest["nside_coverage"] == 32 + + +def test_every_record_on_the_root_is_used(tmp_path, monkeypatch): + """The map is campaign-cumulative: nothing selects a subset of the records. + + A third exposure appears on the products root with no involvement from any + caller — the state a reclaimed or out-of-scope exposure is in — and it must + still be in the map. + """ + products = tmp_path / "products" + write_footprint(products, "1000001", [box(9.8, 10.2, 19.8, 20.2)]) + write_footprint(products, "2000001", [box(10.0, 10.4, 19.8, 20.2)]) + write_footprint(products, "3000001", [box(40.0, 40.4, 19.8, 20.2)]) + + m, manifest = run(products, tmp_path, monkeypatch) + + assert manifest["exposures"] == ["1000001", "2000001", "3000001"] + # The far-away exposure is real sky in the map, not just a row in the record. + assert m.get_values_pos(40.2, 20.0, lonlat=True) == 1 + + +def test_seam_record_is_unwrapped_by_the_builder(tmp_path, monkeypatch): + """A raw-sky record across RA=0 lands where it belongs, not 360 deg away.""" + products = tmp_path / "products" + write_footprint(products, "1000001", + [([359.8, 0.2, 0.2, 359.8], [19.8, 19.8, 20.2, 20.2])]) + + m, _ = run(products, tmp_path, monkeypatch) + + assert m.get_values_pos(0.0, 20.0, lonlat=True) == 1 + assert m.get_values_pos(359.9, 20.0, lonlat=True) == 1 + # Nothing was stamped on the far side of the sky. + assert m.get_values_pos(180.0, 20.0, lonlat=True) == 0 + + +def test_no_records_is_a_loud_failure(tmp_path, monkeypatch): + """An empty products root must not write an empty map and exit green.""" + products = tmp_path / "products" + products.mkdir() + with pytest.raises(SystemExit) as exc: + run(products, tmp_path, monkeypatch) + assert "nothing to build a map from" in str(exc.value) + + +def test_records_naming_no_ccd_is_a_loud_failure(tmp_path, monkeypatch): + """Records that name no CCD are the quieter empty map, and equally fatal. + + Every exposure on the root having lost every CCD is a broken PSF stage, not + a survey with no coverage — but the .hsp it would write is valid and + plausible, and its consumer would mask everything. + """ + products = tmp_path / "products" + write_footprint(products, "1000001", []) + write_footprint(products, "2000001", []) + + with pytest.raises(SystemExit) as exc: + run(products, tmp_path, monkeypatch) + assert "not one names a CCD with a PSF model" in str(exc.value) diff --git a/tests/unit/test_clean_tile_prune.py b/tests/unit/test_clean_tile_prune.py new file mode 100644 index 000000000..9830ff2d9 --- /dev/null +++ b/tests/unit/test_clean_tile_prune.py @@ -0,0 +1,167 @@ +"""``clean_tile`` deletes a tile's store without ever following a symlink out of it. + +``workflow/scripts/clean_tile.py`` reclaims a finished tile by deleting its whole +scratch directory bar four survivors. A finished tile holds NINE symlinks in two +classes, and the second is why this module exists: + + * ``exp_forest///output`` — 7 links into the exposure stores, each + shared with 7-10 other tiles; + * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` — + 2 links into ``/project/def-mjhudson/unions-wl/tiles``, 621 GB of staged + survey imaging across 2,536 files on the BACKED-UP, GROUP-SHARED filesystem. + +Both are handed wholesale to ``shutil.rmtree`` rather than caught by ``prune``'s +own ``is_symlink()`` test, so the safety argument rests on rmtree's semantics at +a depth ``prune`` never inspects. That is a fine thing to rely on and a bad thing +to leave unpinned: the failure mode is silent, immediate and unrecoverable, and +it would be introduced by an innocent-looking edit to ``prune``. + +Deliberately container-free — clean_tile.py is stdlib-only. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "clean_tile.py" + +TILE = "186.307" +SURVIVORS = ( + "cleaned.json", + "manifests/tile_vignets.json", + "manifests/tile_find_exposures.json", + "output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-186-307.txt", +) + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_clean_tile", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +clean_tile = _load() + + +@pytest.fixture +def store(tmp_path): + """A finished tile's store, plus the two off-store trees it links into. + + Mirrors the real layout of 186.307 (smk-g4) at the depths that matter: the + forest links sit three levels down, the Git links four, and ``prune`` + descends into ``output/`` only because the Fe survivor lives there. + """ + precious = {} + for name, rel in (("exposures", "exp/21/2114045/output"), + ("project_tiles", "unions-wl/tiles")): + d = tmp_path / name / rel + d.mkdir(parents=True) + (d / "DO_NOT_DELETE").write_text("survey data\n") + precious[name] = d + + tile_dir = tmp_path / "run" / "tiles" / TILE[:2] / TILE + for rel in SURVIVORS[1:]: # the tombstone is written by the job + p = tile_dir / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("{}" if rel.endswith(".json") else "2114045p\n") + + # bulk that must go + for rel in ("output/run_sp_tile_Sx/sextractor_runner/output/sexcat.fits", + "output/run_sp_tile_Uz/uncompress_fits_runner/output/image.fits", + "logs/tile_detect.json", "manifests/tile_detect.json", + "tile_numbers.txt"): + p = tile_dir / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("bulk") + + # class 1: the exposure forest (rmtree'd as a top-level entry) + forest = tile_dir / "exp_forest" / "21" / "2114045" + forest.mkdir(parents=True) + (forest / "output").symlink_to(precious["exposures"]) + + # class 2: the get_images links into /project (rmtree'd inside output/) + git = tile_dir / "output" / "run_sp_tile_Git" / "get_images_runner" / "output" + git.mkdir(parents=True) + (git / "CFIS_image-186-307.fits").symlink_to( + precious["project_tiles"] / "CFIS.186.307.r.fits") + (git / "CFIS_weight-186-307.fitsfz").symlink_to( + precious["project_tiles"] / "CFIS.186.307.r.weight.fits.fz") + + return tile_dir, precious + + +def test_prune_never_follows_a_symlink_out_of_the_store(store): + """The whole point: reclaiming one tile touches nothing outside that tile.""" + tile_dir, precious = store + before = {k: sorted(p.rglob("*")) for k, p in precious.items()} + + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + + for name, d in precious.items(): + assert d.is_dir(), f"{name} tree was removed" + assert sorted(d.rglob("*")) == before[name], f"{name} tree was modified" + assert (d / "DO_NOT_DELETE").read_text() == "survey data\n" + + +def test_prune_keeps_exactly_the_survivors(store): + """Ten inodes: the four survivors and the directories that carry them.""" + tile_dir, _ = store + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + left = sorted(str(p.relative_to(tile_dir)) for p in tile_dir.rglob("*")) + assert left == sorted([ + "manifests", "manifests/tile_vignets.json", + "manifests/tile_find_exposures.json", + "output", "output/run_sp_tile_Fe", + "output/run_sp_tile_Fe/find_exposures_runner", + "output/run_sp_tile_Fe/find_exposures_runner/output", + "output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-186-307.txt", + ]) + + +def test_prune_unlinks_a_dangling_link(store): + """A forest link whose exposure clean_exposure already reclaimed. + + ``exists()`` follows the link and is False for a dangling one, so an + exists()-first test would skip it and leave the link behind. + """ + tile_dir, precious = store + import shutil + shutil.rmtree(precious["exposures"]) + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + assert not (tile_dir / "exp_forest").exists() + + +def test_survivor_contract_refuses_before_deleting_anything(store): + """A missing survivor aborts with the store intact — it is a contract.""" + tile_dir, _ = store + (tile_dir / "manifests" / "tile_vignets.json").unlink() + survivors = clean_tile.survivor_paths(tile_dir, TILE) + with pytest.raises(SystemExit) as exc: + clean_tile.require_survivors(survivors, TILE) + assert "tile_vignets.json" in str(exc.value) + assert (tile_dir / "output" / "run_sp_tile_Sx").is_dir() + + +def test_absorption_is_additive_over_an_existing_tombstone(tmp_path): + """A second clean must not blank the record the first one saved. + + ``script_hash`` reruns this job on any edit to clean_tile.py, so a re-clean + over an already-pruned store is routine — and it finds only the surviving + manifests on disk. + """ + tomb = tmp_path / "cleaned.json" + tomb.write_text(json.dumps({ + "tile": TILE, + "manifests": {f"tile_ngmix_{k}": {"stage": "tile_ngmix"} for k in range(1, 9)}, + "benchmarks": {"tile_ngmix_1.benchmark.tsv": {"s": "6852.09"}}, + })) + manifests, benchmarks = clean_tile.previous_record(tomb) + assert len(manifests) == 8 + assert benchmarks["tile_ngmix_1.benchmark.tsv"]["s"] == "6852.09" diff --git a/tests/unit/test_exp_footprint.py b/tests/unit/test_exp_footprint.py new file mode 100644 index 000000000..5ee569413 --- /dev/null +++ b/tests/unit/test_exp_footprint.py @@ -0,0 +1,241 @@ +"""The exp_footprint record, and the CCD-index contract it rests on. + +``workflow/scripts/exp_footprint.py`` joins two records that name CCDs in two +different ways and never cross-check each other: + + * ``headers-.npy`` names a CCD by its POSITION in the array — the same + ``idx-1`` split_exp used for ``image--.fits``; + * ``exp_persist.json`` names a CCD inside a FILENAME, + ``validation_psf--.fits``, written by psfex_interp. + +The whole design depends on those being the same integer, and no code asserts +it: split_exp writes the image and the array element in one loop, psfex_interp +inherits the numbering through the file handler, and the coverage map would be +silently WRONG — right pixels, wrong exposure count — if they ever diverged by a +permutation. This module is where that contract is pinned. It inherits the role +of ``tests/module/test_coverage.py::test_get_all_shdus``, which pinned the same +``-`` format against the summary scrape, both now retired. + +The fixtures are real astropy WCSs, one per CCD with a DISTINCT centre, so a +permutation of the array shows up as corners on the wrong ``id`` rather than as +a passing test. One CCD straddles RA=0 deliberately: the record is raw sky, and +the seam is the map builder's business (``unwrap_ra``), not this script's. + +Deliberately not a module test: nothing here runs shapepipe, only its two +geometry helpers, so it belongs with the fast structural suite. +""" + +import importlib.util +import json +import sys +from pathlib import Path + +import numpy as np +import pytest +from astropy.io.fits import Header +from astropy.wcs import WCS + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "exp_footprint.py" + +EXP = "2605805" +# MegaCam-ish: 2048 x 4612 pixels at 0.187"/pixel, i.e. ~0.106 x 0.240 deg. +NX, NY = 2048, 4612 +PIXSCALE = 0.187 / 3600.0 + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_exp_footprint", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +exp_footprint = _load() + + +def ccd_header(ra_centre, dec_centre): + """A single-CCD header + WCS, as split_exp stores them. + + The header carries plain ``NAXIS1/2`` and no ``ZIMAGE``, which is what + astropy hands back for a tile-compressed HDU once it is decompressed — the + case ``_image_shape`` takes its second branch for. + """ + w = WCS(naxis=2) + w.wcs.ctype = ["RA---TAN", "DEC--TAN"] + w.wcs.crpix = [NX / 2.0, NY / 2.0] + w.wcs.crval = [ra_centre, dec_centre] + w.wcs.cdelt = [-PIXSCALE, PIXSCALE] + header = w.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = NX + header["NAXIS2"] = NY + return {"WCS": w, "header": header.tostring()} + + +def headers_array(centres): + """``headers-.npy``'s content: object array, one entry per CCD.""" + arr = np.zeros(len(centres), dtype="O") + for i, (ra, dec) in enumerate(centres): + arr[i] = ccd_header(ra, dec) + return arr + + +# Six CCDs on a row, each 0.3 deg from the last so no two share a footprint, and +# CCD 3 sitting on the RA=0 seam. +CENTRES = [(10.0, 30.0), (10.3, 30.0), (10.6, 30.0), + (0.0, 30.0), (11.2, 30.0), (11.5, 30.0)] + + +@pytest.fixture +def store(tmp_path): + """A scratch exposure store with a headers npy, and a products root.""" + npy_dir = tmp_path / "exp" / EXP / exp_footprint.HEADERS_DIR + npy_dir.mkdir(parents=True) + np.save(npy_dir / f"headers-{EXP}.npy", headers_array(CENTRES)) + return tmp_path + + +def persist_manifest(path, ccds, patterns=("validation_psf-*.fits",)): + """An ``exp_persist`` manifest naming exactly ``ccds`` as PSF-bearing. + + Shaped as persist_exp.py writes it, including the decoy member: a keep list + of several patterns packs files this script must ignore, and reading a CCD + index out of one of them would be a real bug. + """ + files = [{"name": f"validation_psf-{EXP}-{c}.fits", + "pattern": "validation_psf-*.fits", "bytes": 1} for c in ccds] + files.append({"name": f"{EXP}-0.psf", "pattern": "*.psf", "bytes": 1}) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps( + {"stage": "exp_persist", "unit": EXP, "status": "complete", + "patterns": list(patterns), "n_files": len(files), "files": files}, + indent=2, sort_keys=True)) + return path + + +def run(store, persist, out, monkeypatch): + """Invoke the script's CLI, as the rule's shell does.""" + monkeypatch.setattr(sys, "argv", [ + "exp_footprint.py", + "--exp-dir", str(store / "exp" / EXP), + "--exp", EXP, + "--persist-manifest", str(persist), + "--manifest", str(out)]) + exp_footprint.main() + return json.loads(out.read_text()) + + +def test_ccd_index_alignment(store, monkeypatch): + """npy index ``i`` == ``-`` == ``validation_psf--.fits``. + + The three CCDs with a PSF are non-adjacent on purpose: an off-by-one or a + "renumber the survivors 0..n" bug passes a contiguous fixture and fails + this one. + """ + persist = persist_manifest(store / "prod" / "exp_persist.json", [0, 2, 5]) + record = run(store, persist, store / "prod" / "exp_footprint.json", + monkeypatch) + + assert record["n_ccd_headers"] == len(CENTRES) + assert record["n_valid_psf"] == 3 + assert [c["id"] for c in record["ccds"]] == [ + f"{EXP}-0", f"{EXP}-2", f"{EXP}-5"] + assert record["ccds_no_psf"] == [f"{EXP}-1", f"{EXP}-3", f"{EXP}-4"] + + # The corners on `-i` are the corners of ARRAY ELEMENT i, not of the + # i-th surviving CCD: each fixture CCD has its own centre, so this catches + # any permutation. Checked against the same two helpers the script imports, + # which is the point — the contract under test is the INDEXING, not the + # projection arithmetic (tests/module/test_coverage.py owns that). + arr = headers_array(CENTRES) + for entry in record["ccds"]: + i = int(entry["id"].rsplit("-", 1)[1]) + shape = exp_footprint._image_shape(Header.fromstring(arr[i]["header"])) + ra, dec = exp_footprint._ccd_corners(arr[i]["WCS"], shape) + assert entry["ra"] == pytest.approx(list(ra)) + assert entry["dec"] == pytest.approx(list(dec)) + # ... and that centre really is CCD i's, not its neighbour's. + assert np.mean(entry["dec"]) == pytest.approx(CENTRES[i][1], abs=1e-3) + + +def test_record_is_raw_sky_across_the_ra_seam(store, monkeypatch): + """CCD 3 straddles RA=0 and the record says so, unwrapped by nobody. + + The seam is handled once, in ``coverage_map_builder.unwrap_ra``, at the + moment a polygon is stamped. Unwrapping here as well would put negative RA + into a durable record that other consumers read, and double-unwrapping is + not idempotent. + """ + persist = persist_manifest(store / "prod" / "exp_persist.json", [3]) + record = run(store, persist, store / "prod" / "exp_footprint.json", + monkeypatch) + + ra = record["ccds"][0]["ra"] + assert record["ccds"][0]["id"] == f"{EXP}-3" + assert min(ra) >= 0.0 and max(ra) < 360.0, "raw sky, not a shifted branch" + assert max(ra) - min(ra) > 180.0, "the fixture must actually cross RA=0" + # The four corners land on both sides of the seam, ~0.05 deg out. + assert sorted(round(r) for r in ra) == [0, 0, 360, 360] + + +def test_byte_stable_rerun_keeps_the_mtime(store, monkeypatch): + """A rerun over an unchanged store must not move the manifest's mtime. + + mtime is a rerun trigger, and this manifest is an input of clean_exposure: + an unconditional rewrite would make every downstream reclamation look out + of date once per invocation. Same contract as persist_exp.py's. + """ + persist = persist_manifest(store / "prod" / "exp_persist.json", [0, 2, 5]) + out = store / "prod" / "exp_footprint.json" + + first = run(store, persist, out, monkeypatch) + raw = out.read_bytes() + before = out.stat().st_mtime_ns + + second = run(store, persist, out, monkeypatch) + assert second == first + assert out.read_bytes() == raw + assert out.stat().st_mtime_ns == before + # No stray tmp left behind on the persistent root. + assert not list(out.parent.glob("*.tmp")) + + +def test_keep_list_without_the_psf_pattern_is_fatal(store, monkeypatch): + """A manifest that packs no validation_psf files names no valid-PSF set. + + Writing an empty footprint there would be indistinguishable from an + exposure that genuinely lost every CCD, and the map would silently lose a + whole exposure's worth of sky. + """ + persist = persist_manifest(store / "prod" / "exp_persist.json", [], + patterns=("*.psf",)) + with pytest.raises(SystemExit) as exc: + run(store, persist, store / "prod" / "exp_footprint.json", monkeypatch) + assert "persist_exp" in str(exc.value) + + +def test_psf_for_a_ccd_the_split_never_wrote_is_fatal(store, monkeypatch): + """The one disagreement the index alignment cannot absorb, made loud.""" + persist = persist_manifest(store / "prod" / "exp_persist.json", [0, 99]) + with pytest.raises(SystemExit) as exc: + run(store, persist, store / "prod" / "exp_footprint.json", monkeypatch) + assert "99" in str(exc.value) + + +def test_missing_headers_array_is_fatal(store, monkeypatch): + """A purged or unbuilt split store fails here, never against VOS. + + This is the other half of the rule declaring only its DURABLE input + (exposure.smk): a persist manifest outliving its scratch store is a real + state, and it must cost one error line rather than an exposure rebuild. + """ + npy = (store / "exp" / EXP / exp_footprint.HEADERS_DIR + / f"headers-{EXP}.npy") + npy.unlink() + persist = persist_manifest(store / "prod" / "exp_persist.json", [0]) + with pytest.raises(SystemExit) as exc: + run(store, persist, store / "prod" / "exp_footprint.json", monkeypatch) + assert "no WCS array" in str(exc.value) diff --git a/tests/unit/test_merge_sep_cats_paths.py b/tests/unit/test_merge_sep_cats_paths.py new file mode 100644 index 000000000..754bc76dd --- /dev/null +++ b/tests/unit/test_merge_sep_cats_paths.py @@ -0,0 +1,40 @@ +"""Chunk-path derivation in the merge_sep_cats module.""" + +import pytest + +from shapepipe.modules.merge_sep_cats_package.merge_sep_cats import chunk_path + + +REL = "./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-210-282.fits" +ABS = ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng1u" + "/ngmix_runner/output/ngmix-210-282.fits" +) + + +def test_relative_path_unchanged_behaviour(): + """The bash pipeline's relative INPUT_DIR still resolves as before.""" + assert chunk_path(REL, 3) == ( + "./output/run_sp_tile_ngmix_Ng3u/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_sharded_absolute_path(): + """Digits in the sharded parent dirs and in the file number are untouched.""" + assert chunk_path(ABS, 2) == ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng2u" + "/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_double_digit_chunk(): + assert "Ng12u" in chunk_path(ABS, 12) + + +def test_chunk_one_is_identity(): + assert chunk_path(ABS, 1) == ABS + + +def test_no_run_directory_raises(): + with pytest.raises(ValueError, match="run_"): + chunk_path("/scratch/run/tiles/21/210.282/ngmix-210-282.fits", 2) diff --git a/tests/unit/test_ngmix_range.py b/tests/unit/test_ngmix_range.py new file mode 100644 index 000000000..30027cd29 --- /dev/null +++ b/tests/unit/test_ngmix_range.py @@ -0,0 +1,273 @@ +"""The ngmix chunk split tiles ``[1, n_obj]`` exactly. + +Nothing downstream cross-checks the chunk ranges: merge_sep_cats concatenates +the chunk catalogues, so an overlap silently duplicates objects and a gap +silently drops them. The one invariant that catches both is that the ranges +partition ``1..n_obj`` — that is what this module asserts, over the epoch-weight +distributions the splitter actually has to cope with. + +``workflow/scripts/ngmix_range.py`` now runs ONCE per tile (tile_vignets writes +the whole partition to the group's node-local scratch) and is then only read +from, once per chunk. The second thing this module pins is that the trip through +JSON changes nothing: row ``k`` of the written file is exactly what asking the +splitter for chunk ``k`` used to return. + +Deliberately container-free: the range function takes a plain sequence of epoch +counts, so nothing here imports astropy, numpy, or shapepipe. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "ngmix_range.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_ngmix_range", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +ngmix_range = _load() + + +def assert_tiles(ranges, n_obj, n_chunks): + """Assert the partition invariant, in the form each failure mode takes.""" + assert len(ranges) == n_chunks + assert ranges[0][0] == 1 + assert ranges[-1][1] == n_obj + for (lo, hi), (prev_lo, prev_hi) in zip(ranges[1:], ranges): + assert lo == prev_hi + 1 + assert hi >= lo - 1, "an empty range is (hi + 1, hi), never narrower" + assert prev_hi >= prev_lo - 1 + covered = [i for lo, hi in ranges for i in range(lo, hi + 1)] + assert covered == list(range(1, n_obj + 1)) + # ID_OBJ_MAX <= 0 is ngmix's "unbounded" sentinel: a chunk emitting it + # re-measures the whole tile instead of nothing. + assert all(hi >= 1 for _, hi in ranges) + + +# --------------------------------------------------------------------------- # +# The property +# --------------------------------------------------------------------------- # + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=400), + n_chunks=st.integers(min_value=1, max_value=16), + data=st.data(), +) +def test_ranges_tile_the_catalogue(n_obj, n_chunks, data): + """Any epoch distribution, any shape: the ranges still partition 1..N.""" + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=40), + min_size=n_obj, + max_size=n_obj, + ) + ) + assert_tiles(ngmix_range.id_ranges(epochs, n_chunks), n_obj, n_chunks) + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=60), + n_chunks=st.integers(min_value=1, max_value=8), + data=st.data(), +) +def test_ranges_are_deterministic(n_obj, n_chunks, data): + """Same input, same ranges — the eight processes share nothing else. + + Integer weights make the split a pure function of the epoch list, with no + float accumulation whose order could matter; this pins that. + """ + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=99), + min_size=n_obj, + max_size=n_obj, + ) + ) + first = ngmix_range.id_ranges(epochs, n_chunks) + assert first == ngmix_range.id_ranges(list(epochs), n_chunks) + assert all(isinstance(b, int) for lo, hi in first for b in (lo, hi)) + + +# --------------------------------------------------------------------------- # +# Write-once / read-per-chunk is the same partition +# --------------------------------------------------------------------------- # + + +def _roundtrip(epochs, n_chunks): + """What a chunk shell sees: the doc as it comes back off disk.""" + return json.loads(json.dumps(ngmix_range.partition(epochs, n_chunks))) + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=200), + n_chunks=st.integers(min_value=1, max_value=12), + data=st.data(), +) +def test_written_rows_reproduce_the_per_chunk_computation(n_obj, n_chunks, data): + """THE INVARIANCE TEST for materialising the split. + + Writing the whole partition and reading chunk ``k``'s row back must give + byte-for-byte what the old per-chunk ``id_ranges(...)[k - 1]`` returned. If + this ever fails, the two mechanisms have drifted and a tile's coverage is + what pays. + """ + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=40), + min_size=n_obj, + max_size=n_obj, + ) + ) + doc = _roundtrip(epochs, n_chunks) + assert doc["n_obj"] == n_obj and doc["n_chunks"] == n_chunks + expected = ngmix_range.id_ranges(epochs, n_chunks) + got = [ngmix_range.chunk_range(doc, k) for k in range(1, n_chunks + 1)] + assert got == expected + assert_tiles(got, n_obj, n_chunks) + + +def test_chunk_index_outside_the_written_partition_is_fatal(): + """A chunk index the file does not hold must not index from the end.""" + doc = _roundtrip([3] * 40, 4) + for bad in (0, -1, 5): + with pytest.raises(SystemExit, match="outside 1..4"): + ngmix_range.chunk_range(doc, bad) + + +def test_reading_an_absent_ranges_file_is_fatal(tmp_path): + """NO FALLBACK: a missing file fails, it does not recompute. + + Recomputing per chunk is precisely the mechanism this design removed. + """ + with pytest.raises(SystemExit, match="no ranges file"): + ngmix_range.read_ranges(tmp_path / "ngmix_ranges.json") + + +# --------------------------------------------------------------------------- # +# Degenerate shapes — each pins a decision, not just an absence of crash +# --------------------------------------------------------------------------- # + + +def test_single_chunk_takes_everything(): + """n_chunks == 1: one range over the whole catalogue.""" + assert ngmix_range.id_ranges([3] * 17, 1) == [(1, 17)] + + +def test_one_object_per_chunk(): + """n_obj == n_chunks: one object each, however lopsided the weights.""" + assert ngmix_range.id_ranges([0, 9, 1, 40], 4) == [ + (1, 1), (2, 2), (3, 3), (4, 4) + ] + + +def test_fewer_objects_than_chunks_pads_with_empty_ranges(): + """DOCUMENTED: the surplus chunks get ``(n_obj + 1, n_obj)``, not (1, 0). + + The old equal-count split gave every surplus chunk ``(1, 0)``, and ngmix + reads ``ID_OBJ_MAX = 0`` as unbounded — so each of them would have measured + the entire tile rather than nothing. + """ + assert ngmix_range.id_ranges([2, 5, 1], 6) == [ + (1, 1), (2, 2), (3, 3), (4, 3), (4, 3), (4, 3) + ] + assert_tiles(ngmix_range.id_ranges([2, 5, 1], 6), 3, 6) + + +def test_zero_objects_is_fatal(): + """No split is meaningful, and (1, 0) is ngmix's unbounded sentinel.""" + with pytest.raises(ValueError, match="zero objects"): + ngmix_range.id_ranges([], 8) + + +def test_zero_chunks_is_fatal(): + """There is no zeroth chunk to hand a range to.""" + with pytest.raises(ValueError, match="n_chunks"): + ngmix_range.id_ranges([1, 2, 3], 0) + + +def test_equal_weights_reproduce_an_equal_count_split(): + """Uniform epochs: chunk sizes differ by at most one object.""" + ranges = ngmix_range.id_ranges([3] * 1000, 8) + assert_tiles(ranges, 1000, 8) + sizes = [hi - lo + 1 for lo, hi in ranges] + assert max(sizes) - min(sizes) <= 1 + assert sum(sizes) == 1000 + + +def test_zero_epoch_objects_still_weigh_something(): + """ALPHA is why: without it a 0-epoch tail is free and lands in one chunk. + + Ninety-six zero-epoch objects and four 1-epoch ones. Weighing only epochs + would let a single chunk swallow all ninety-six. + """ + ranges = ngmix_range.id_ranges([0] * 96 + [1] * 4, 4) + assert_tiles(ranges, 100, 4) + assert max(hi - lo + 1 for lo, hi in ranges) < 96 + + +def test_one_enormously_heavy_object_is_isolated(): + """The heavy object gets a chunk to itself; the tail still tiles.""" + epochs = [1] * 20 + [10_000] + [1] * 20 + ranges = ngmix_range.id_ranges(epochs, 4) + assert_tiles(ranges, 41, 4) + assert (21, 21) in ranges + + +# --------------------------------------------------------------------------- # +# The split is the optimal one, not merely a legal one +# --------------------------------------------------------------------------- # + + +def _brute_force_min_max(weights, n_chunks): + """Smallest achievable maximum chunk weight, by exhaustive enumeration.""" + n = len(weights) + best = {} + + def solve(start, chunks): + if chunks == 1: + return sum(weights[start:]) + if (start, chunks) not in best: + best[(start, chunks)] = min( + max(sum(weights[start:cut]), solve(cut, chunks - 1)) + for cut in range(start + 1, n - chunks + 2) + ) + return best[(start, chunks)] + + return solve(0, n_chunks) + + +@settings(deadline=None, max_examples=40) +@given( + epochs=st.lists( + st.integers(min_value=0, max_value=12), min_size=4, max_size=14 + ), + n_chunks=st.integers(min_value=2, max_value=4), +) +def test_slowest_chunk_is_minimal(epochs, n_chunks): + """The objective is the slowest chunk — check it against brute force.""" + if len(epochs) < n_chunks: + return + weights = [ + ngmix_range.MILLI_EPOCH * e + ngmix_range.ALPHA_MILLI_EPOCHS + for e in epochs + ] + ranges = ngmix_range.id_ranges(epochs, n_chunks) + loads = [sum(weights[lo - 1:hi]) for lo, hi in ranges] + assert max(loads) == _brute_force_min_max(weights, n_chunks) diff --git a/tests/unit/test_run_report_tombstones.py b/tests/unit/test_run_report_tombstones.py new file mode 100644 index 000000000..6403d0233 --- /dev/null +++ b/tests/unit/test_run_report_tombstones.py @@ -0,0 +1,171 @@ +"""Reclamation must not change what ``sp report`` says about a unit. + +``workflow/scripts/run_report.py`` reads a unit's verdict from two places with +the same shape and different lifetimes: the records on disk +(``manifests/*.json`` + ``logs/*.json``), and — once ``clean_tile`` or +``clean_exposure`` has deleted those — the copies absorbed into the unit's +``cleaned.json``. Both readers face the same collision, because several files +map to one ``(unit, stage)``: a stage's manifest and its byte-identical log, and +the eight ngmix chunks, which all carry ``stage: "tile_ngmix"``. + +If the two readers resolve that collision differently, RECLAMATION SILENTLY +EDITS THE REPORT. It did: absorption used first-key-wins, so only +``tile_ngmix_1`` survived and a ``warn`` on any other chunk became ``complete`` +the moment the tile was cleaned — and the tile dropped out of the "tiles not +complete" table. This module pins the invariant that catches that whole class: +**the tombstone path and the on-disk path agree, stage for stage.** + +Deliberately container-free — run_report.py is stdlib-only, so nothing here +imports astropy, numpy, or shapepipe. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "run_report.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; bin/sp calls it by path" + spec = importlib.util.spec_from_file_location("_run_report", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +run_report = _load() + +TILE = "186.307" +NGMIX_CHUNKS = 8 + + +def _manifest(stage, status="complete", found=1, expect=1): + """One completeness verdict, in the shape completeness.py writes.""" + return { + "stage": stage, "level": "tile", "unit": TILE, "status": status, + "runners": {f"{stage}_runner": { + "found": found, "expect": expect, "floor": found, + "status": status, "warn": status == "warn"}}, + "failures": [], + } + + +def _records(ngmix_status=None): + """A finished tile's manifests, keyed by file stem as the store keys them. + + ``ngmix_status`` is ``{chunk_number: status}``; unlisted chunks are complete. + """ + out = {s: _manifest(s) for s in run_report.TILE_STAGES if s != "tile_ngmix"} + for k in range(1, NGMIX_CHUNKS + 1): + out[f"tile_ngmix_{k}"] = _manifest( + "tile_ngmix", (ngmix_status or {}).get(k, "complete")) + return out + + +def _cleaned_store(root, records, survivors=("tile_vignets", "tile_find_exposures")): + """A reclaimed tile store on disk: the survivors, and the tombstone. + + Exactly what ``clean_tile`` leaves behind — ``logs/`` gone, ``manifests/`` + holding only the manifests other mechanisms own, every record absorbed into + ``cleaned.json``. + """ + tile_dir = root / "tiles" / TILE[:2] / TILE + (tile_dir / "manifests").mkdir(parents=True) + for stem in survivors: + (tile_dir / "manifests" / f"{stem}.json").write_text( + json.dumps(records[stem])) + (tile_dir / "cleaned.json").write_text(json.dumps({ + "tile": TILE, "manifests": records, "benchmarks": {}})) + return tile_dir + + +def _live_store(root, records): + """The same tile BEFORE reclamation: every manifest, and its identical log.""" + tile_dir = root / "tiles" / TILE[:2] / TILE + for sub in ("manifests", "logs"): + (tile_dir / sub).mkdir(parents=True) + for stem, m in records.items(): + (tile_dir / sub / f"{stem}.json").write_text(json.dumps(m)) + return tile_dir + + +def _stage_status(run_dir, cleaned_survivors=None): + """``{stage: status}`` as the report would tally it, from a store on disk.""" + manifests = run_report.load_manifests(run_dir, "tiles") + if cleaned_survivors is not None: + run_report.absorb_tombstones(run_dir, "tiles", manifests, + cleaned_survivors) + return {s: m.get("status") for s, m in manifests[TILE].items()} + + +@pytest.mark.parametrize("chunk", range(1, NGMIX_CHUNKS + 1)) +def test_a_warning_chunk_survives_reclamation(tmp_path, chunk): + """A warn on ANY chunk reads the same before and after the tile is cleaned. + + Parametrised over all eight because the bug was invisible on chunk 1: that + is the stem first-key-wins happened to keep, so the one chunk anybody would + test by hand was the one chunk that worked. + """ + records = _records({chunk: "warn"}) + + live = tmp_path / "live" + _live_store(live, records) + before = _stage_status(live) + + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, records) + after = _stage_status(cleaned, run_report.SURVIVING_TILE_STAGES) + + assert before["tile_ngmix"] == "warn" + assert after == before + + +def test_a_failed_chunk_survives_reclamation(tmp_path): + """Worst-status-wins, not merely warn: failed beats warn beats complete.""" + records = _records({3: "warn", 6: "failed"}) + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, records) + after = _stage_status(cleaned, run_report.SURVIVING_TILE_STAGES) + assert after["tile_ngmix"] == "failed" + + +def test_reclaimed_tile_reports_every_stage(tmp_path): + """The survivors must not read as a rebuilt chain. + + ``clean_tile`` cannot empty ``manifests/`` — two manifests are currency + other mechanisms own — so without the survivor set the "records on disk mean + a rebuilt chain" guard fires and a reclaimed tile reports as one that ran + two stages and stopped. + """ + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, _records()) + manifests = run_report.load_manifests(cleaned, "tiles") + got = run_report.absorb_tombstones(cleaned, "tiles", manifests, + run_report.SURVIVING_TILE_STAGES) + assert got == {TILE} + assert sorted(manifests[TILE]) == sorted(run_report.TILE_STAGES) + + +def test_rebuilt_chain_keeps_the_guard(tmp_path): + """A unit with a NON-survivor record on disk is a rebuilt chain: skip it. + + This is the default (empty survivor set) that the exposure side uses, and it + must keep behaving exactly as it did before tiles needed an argument here. + """ + cleaned = tmp_path / "cleaned" + tile_dir = _cleaned_store(cleaned, _records()) + (tile_dir / "manifests" / "tile_detect.json").write_text( + json.dumps(_manifest("tile_detect", "failed"))) + + manifests = run_report.load_manifests(cleaned, "tiles") + got = run_report.absorb_tombstones(cleaned, "tiles", manifests, + run_report.SURVIVING_TILE_STAGES) + assert got == set() + assert manifests[TILE]["tile_detect"]["status"] == "failed" + assert "tile_ngmix" not in manifests[TILE] diff --git a/uv.lock b/uv.lock index ca985ee4c..ccf54862c 100644 --- a/uv.lock +++ b/uv.lock @@ -14,7 +14,7 @@ name = "accessible-pygments" version = "0.0.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872", size = 1377899, upload-time = "2024-05-10T11:23:10.216Z" } wheels = [ @@ -62,8 +62,8 @@ name = "anyio" version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ @@ -75,7 +75,7 @@ name = "argon2-cffi" version = "25.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argon2-cffi-bindings", marker = "sys_platform == 'linux'" }, + { name = "argon2-cffi-bindings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } wheels = [ @@ -87,7 +87,7 @@ name = "argon2-cffi-bindings" version = "26.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "sys_platform == 'linux'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } wheels = [ @@ -127,8 +127,8 @@ name = "arrow" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "tzdata", marker = "sys_platform == 'linux'" }, + { name = "python-dateutil" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } wheels = [ @@ -140,11 +140,11 @@ name = "astropy" version = "8.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy-iers-data", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyerfa", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "astropy-iers-data" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyerfa" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/c4/21be4313ddfde5f60e0607fd307f367b9e0f0bf153a89b10cbd036dd8cfd/astropy-8.0.1.tar.gz", hash = "sha256:45ca31d5b91fa294cd590a4791a32db94de7f9c8a343155f4d5877baa82351da", size = 7152500, upload-time = "2026-07-05T07:24:48.482Z" } wheels = [ @@ -158,8 +158,8 @@ name = "astropy-healpix" version = "2.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/c1/aeb3fe3be2ee863708d625014267c71abfe20ddaa293b3d4ddb72ee1d6e9/astropy_healpix-2.0.1.tar.gz", hash = "sha256:0e3f1c94064c45da779900cb90c938df7aef99a924abb23eeb893b16540e77e6", size = 112256, upload-time = "2026-07-20T21:07:30.004Z" } wheels = [ @@ -185,13 +185,13 @@ name = "astroquery" version = "0.4.11" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "html5lib", marker = "sys_platform == 'linux'" }, - { name = "keyring", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pyvo", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "beautifulsoup4" }, + { name = "html5lib" }, + { name = "keyring" }, + { name = "numpy" }, + { name = "pyvo" }, + { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/32/48/273dbde090e071f9d264d084bc49193d126498d2906172b78febd9d62e28/astroquery-0.4.11.tar.gz", hash = "sha256:5537529bddc7fa07e773d5cd9baca593e3f5d93474edd1914f68e89506042b33", size = 12561055, upload-time = "2025-09-20T04:26:36.744Z" } wheels = [ @@ -239,8 +239,8 @@ name = "beautifulsoup4" version = "4.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "soupsieve", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "soupsieve" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } wheels = [ @@ -252,7 +252,7 @@ name = "bleach" version = "6.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "webencodings", marker = "sys_platform == 'linux'" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/3c/e12ac860709702bd5ebeb9b56a4fe334f1001246ee1b8f2b7ee28912df7d/bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452", size = 204857, upload-time = "2026-06-05T13:01:13.734Z" } wheels = [ @@ -261,7 +261,7 @@ wheels = [ [package.optional-dependencies] css = [ - { name = "tinycss2", marker = "sys_platform == 'linux'" }, + { name = "tinycss2" }, ] [[package]] @@ -269,8 +269,8 @@ name = "build" version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyproject-hooks", marker = "sys_platform == 'linux'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/b7/1db48a9ce2984842c8c886432ec8a2719613322e868a966ba82a28862f25/build-1.6.0.tar.gz", hash = "sha256:bd2c8afc603e7a2e0ce70e2ea85f0a6d02043bafbd307f5bada0f98669eca5af", size = 113825, upload-time = "2026-08-27T21:01:16.458Z" } wheels = [ @@ -282,12 +282,12 @@ name = "cadcutils" version = "1.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "distro", marker = "sys_platform == 'linux'" }, - { name = "lxml", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyopenssl", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "distro" }, + { name = "lxml" }, + { name = "packaging" }, + { name = "pyopenssl" }, + { name = "requests" }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fe/44/4964823c72c9e6a93e26de0118cead37fa3c3600b4677f0c06788c903c80/cadcutils-1.6.2.tar.gz", hash = "sha256:6e2d7822756d48a363bc5f857cd717f80aab760554df86518b957f8826a7a906", size = 94673, upload-time = "2026-06-22T17:35:47.303Z" } wheels = [ @@ -299,10 +299,10 @@ name = "camb" version = "2.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "sympy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "scipy" }, + { name = "sympy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/51/0b/856e8e329c088155509e3c1a999b02cd3f0333a31fc1fa37653455a4ec80/camb-2.0.4.tar.gz", hash = "sha256:2f1f5f3b3964a3746693dc88a54218dca4ea049a04726f8177664c11d2d2616e", size = 925860, upload-time = "2026-08-25T22:10:58.612Z" } wheels = [ @@ -315,19 +315,19 @@ name = "canfar" version = "1.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cadcutils", marker = "sys_platform == 'linux'" }, - { name = "click", marker = "sys_platform == 'linux'" }, - { name = "defusedxml", marker = "sys_platform == 'linux'" }, - { name = "httpx", extra = ["http2"], marker = "sys_platform == 'linux'" }, - { name = "humanize", marker = "sys_platform == 'linux'" }, - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "pydantic-settings", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "questionary", marker = "sys_platform == 'linux'" }, - { name = "rich", marker = "sys_platform == 'linux'" }, - { name = "segno", marker = "sys_platform == 'linux'" }, - { name = "toml", marker = "sys_platform == 'linux'" }, - { name = "typer", marker = "sys_platform == 'linux'" }, + { name = "cadcutils" }, + { name = "click" }, + { name = "defusedxml" }, + { name = "httpx", extra = ["http2"] }, + { name = "humanize" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyyaml" }, + { name = "questionary" }, + { name = "rich" }, + { name = "segno" }, + { name = "toml" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/24/d8/a457c57a8c83d6a6a4a96aa41fc46484489a83932bc5fe26180e1daef130/canfar-1.4.1.tar.gz", hash = "sha256:bbd7a37a21e6f9edf8790176797763187fe38a39758c21d652d77ad2fb76a3ee", size = 25743742, upload-time = "2026-06-11T20:39:05.07Z" } wheels = [ @@ -348,7 +348,7 @@ name = "cffi" version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy' and sys_platform == 'linux'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ @@ -538,7 +538,7 @@ name = "conda-inject" version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/a8/8dc86113c65c949cc72d651461d6e4c544b3302a85ed14a5298829e6a419/conda_inject-1.3.2.tar.gz", hash = "sha256:0b8cde8c47998c118d8ff285a04977a3abcf734caf579c520fca469df1cd0aac", size = 3635, upload-time = "2024-05-27T12:20:58.873Z" } wheels = [ @@ -565,7 +565,7 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -675,7 +675,7 @@ name = "cryptography" version = "50.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'linux'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } wheels = [ @@ -719,18 +719,18 @@ name = "cs-util" version = "0.2.2" source = { git = "https://github.com/CosmoStat/cs_util?branch=develop#1b15a5554e378226d98ae7bfe94e7db05573edf0" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "datetime", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "keyring", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "swig", marker = "sys_platform == 'linux'" }, - { name = "vos", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "camb" }, + { name = "datetime" }, + { name = "healpy" }, + { name = "healsparse" }, + { name = "keyring" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pyccl" }, + { name = "scipy" }, + { name = "swig" }, + { name = "vos" }, ] [[package]] @@ -747,13 +747,13 @@ name = "dask" version = "2026.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "sys_platform == 'linux'" }, - { name = "cloudpickle", marker = "sys_platform == 'linux'" }, - { name = "fsspec", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "partd", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "toolz", marker = "sys_platform == 'linux'" }, + { name = "click" }, + { name = "cloudpickle" }, + { name = "fsspec" }, + { name = "packaging" }, + { name = "partd" }, + { name = "pyyaml" }, + { name = "toolz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/a7/6b3c7ac32b642fbbe0821111654e0bd8cfbe88f68560bcf23cc78ab35c71/dask-2026.8.0.tar.gz", hash = "sha256:8a94c37b5de6d869343340dc26c3c3acca7ec48a3abdabe00ea3abb1125884d5", size = 11561752, upload-time = "2026-08-24T19:21:25.906Z" } wheels = [ @@ -762,7 +762,7 @@ wheels = [ [package.optional-dependencies] array = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] [[package]] @@ -770,11 +770,11 @@ name = "dask-image" version = "2026.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pims", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, + { name = "dask", extra = ["array"] }, + { name = "numpy" }, + { name = "pims" }, + { name = "scipy" }, + { name = "tifffile" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } wheels = [ @@ -786,8 +786,8 @@ name = "datetime" version = "6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pytz", marker = "sys_platform == 'linux'" }, - { name = "zope-interface", marker = "sys_platform == 'linux'" }, + { name = "pytz" }, + { name = "zope-interface" }, ] sdist = { url = "https://files.pythonhosted.org/packages/77/32/decbfd165e9985ba9d8c2d34a39afe5aeba2fc3fe390eb6e9ef1aab98fa8/datetime-6.0.tar.gz", hash = "sha256:c1514936d2f901e10c8e08d83bf04e6c9dbd7ca4f244da94fec980980a3bc4d5", size = 64167, upload-time = "2025-11-25T08:00:34.586Z" } wheels = [ @@ -838,7 +838,7 @@ name = "donfig" version = "0.8.1.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } wheels = [ @@ -877,7 +877,7 @@ name = "fitsio" version = "1.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/fd/53bfe8986f7b242fb5b66ddc9344fe67614102f44438415b0a8b32cf7f0a/fitsio-1.4.2.tar.gz", hash = "sha256:92a02f0e63d539d85ca5a185ae0cc8d40029270858275964ee2539ee0136f0c3", size = 4178786, upload-time = "2026-07-16T12:31:06.723Z" } wheels = [ @@ -948,11 +948,11 @@ name = "galsim" version = "2.8.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "lsstdesc-coord", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pybind11", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "lsstdesc-coord" }, + { name = "numpy" }, + { name = "pybind11" }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/94/41/358f965c0512531be47c49afe7bdc1ad264b95cf5b9dacb306cad3da82b0/galsim-2.8.5.tar.gz", hash = "sha256:e449bb0d30ece16c3ee7345846b01f0c54e482e37755b623c17174881cfe5ad3", size = 8590974, upload-time = "2026-08-05T03:40:06.337Z" } wheels = [ @@ -971,7 +971,7 @@ name = "gitdb" version = "4.0.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "smmap", marker = "sys_platform == 'linux'" }, + { name = "smmap" }, ] sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } wheels = [ @@ -983,7 +983,7 @@ name = "gitpython" version = "3.1.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "gitdb", marker = "sys_platform == 'linux'" }, + { name = "gitdb" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/61/3285044215fb596bf093e39ccb96ece0a1076a8ca57a61e069a6a33cdb1b/gitpython-3.1.61.tar.gz", hash = "sha256:f51c24d8c0f733a195447385f5774a5dfe8767f5acfd7994a33755644c6ecc95", size = 231680, upload-time = "2026-08-28T11:01:13.761Z" } wheels = [ @@ -1056,8 +1056,8 @@ name = "h2" version = "4.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "hpack", marker = "sys_platform == 'linux'" }, - { name = "hyperframe", marker = "sys_platform == 'linux'" }, + { name = "hpack" }, + { name = "hyperframe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } wheels = [ @@ -1069,7 +1069,7 @@ name = "h5py" version = "3.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } wheels = [ @@ -1096,8 +1096,8 @@ name = "healpy" version = "1.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3f/54/dd2fa7e6b12f8d3bcc34d61c72cbc1bbbad9b3142583d40803825abea2f6/healpy-1.20.0.tar.gz", hash = "sha256:03b0e1551ae235c7290e9bfdf69fa9cdc194f8a2be51198c1924b15d826102ed", size = 66326598, upload-time = "2026-07-23T20:15:51.281Z" } wheels = [ @@ -1120,8 +1120,8 @@ name = "healsparse" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "hpgeom" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/d6/c299485e7bc16779e33824d838395a7339565d193e017ca9aecbfc0a8269/healsparse-1.14.0.tar.gz", hash = "sha256:60e94c8a12ca3af80cfbba7b7b11c30be1c6b5585132dd9f8ea93c7935887198", size = 133778, upload-time = "2026-08-21T04:33:43.11Z" } wheels = [ @@ -1142,7 +1142,7 @@ name = "hpgeom" version = "1.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/72/27640de7d85a566b8ecd27e7894f371a19669c56c268756e18111a137df8/hpgeom-1.5.4.tar.gz", hash = "sha256:85ac73e267c11f3f248920d3541a0e6ffab821cae54f0d44414d1939dfb91200", size = 153906, upload-time = "2026-02-24T19:03:01.954Z" } wheels = [ @@ -1168,8 +1168,8 @@ name = "html5lib" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, - { name = "webencodings", marker = "sys_platform == 'linux'" }, + { name = "six" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" } wheels = [ @@ -1181,8 +1181,8 @@ name = "httpcore" version = "1.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "h11", marker = "sys_platform == 'linux'" }, + { name = "certifi" }, + { name = "h11" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ @@ -1194,10 +1194,10 @@ name = "httpx" version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform == 'linux'" }, - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "httpcore", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ @@ -1206,7 +1206,7 @@ wheels = [ [package.optional-dependencies] http2 = [ - { name = "h2", marker = "sys_platform == 'linux'" }, + { name = "h2" }, ] [[package]] @@ -1241,7 +1241,7 @@ name = "hypothesis" version = "6.167.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "sortedcontainers", marker = "sys_platform == 'linux'" }, + { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8cee74c1390b2932406faaab76980f18946f258fa5a8afca17189b3bc655/hypothesis-6.167.1.tar.gz", hash = "sha256:62eefcb4d2791423626e9901c3027a6e0c5ffda2ac0b44b3c7e797ab9d2d5a4c", size = 505849, upload-time = "2026-08-30T19:53:09.05Z" } wheels = [ @@ -1291,7 +1291,7 @@ name = "id" version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "urllib3", marker = "sys_platform == 'linux'" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/04/c2156091427636080787aac190019dc64096e56a23b7364d3c1764ee3a06/id-1.6.1.tar.gz", hash = "sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069", size = 18088, upload-time = "2026-02-04T16:19:41.26Z" } wheels = [ @@ -1312,8 +1312,8 @@ name = "imageio" version = "2.37.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "pillow" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/62/aa770a9307508d2a2a2c62d536a49347bffe9e55322db27838d3c93d0b07/imageio-2.37.4.tar.gz", hash = "sha256:e45cbc5e83502047fb138f7f585f7f105a136a57eea5f4b3cfc6ce1b52720bd3", size = 390173, upload-time = "2026-07-20T05:26:11.369Z" } wheels = [ @@ -1350,7 +1350,7 @@ name = "importlib-metadata" version = "9.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "sys_platform == 'linux'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -1371,18 +1371,18 @@ name = "ipykernel" version = "7.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "comm", marker = "sys_platform == 'linux'" }, - { name = "debugpy", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, - { name = "nest-asyncio2", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio2" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/c4/e4a38f579de4225a561305666f7541cdabb30075def2aa1ac17bd73c1fb5/ipykernel-7.3.0.tar.gz", hash = "sha256:9acaaaf97d16355166e4085afe9d225bfbdf2b7ef520f9df3be8f2b248275e09", size = 184899, upload-time = "2026-06-10T08:41:25.481Z" } wheels = [ @@ -1394,15 +1394,15 @@ name = "ipython" version = "9.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipython-pygments-lexers", marker = "sys_platform == 'linux'" }, - { name = "jedi", marker = "sys_platform == 'linux'" }, - { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, - { name = "pexpect", marker = "sys_platform == 'linux'" }, - { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "stack-data", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect" }, + { name = "prompt-toolkit" }, + { name = "psutil" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/bc/e05ae123712ce4e1fde4408eedca1791fc1ff832684565132ea1dc646092/ipython-9.17.0.tar.gz", hash = "sha256:1dc69e6966b270fb259f676c71a21450e63607729b14a672b942914a54e8b730", size = 4538547, upload-time = "2026-08-28T09:00:58.233Z" } wheels = [ @@ -1414,7 +1414,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1426,7 +1426,7 @@ name = "isoduration" version = "20.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "arrow", marker = "sys_platform == 'linux'" }, + { name = "arrow" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" } wheels = [ @@ -1438,7 +1438,7 @@ name = "jaraco-classes" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools", marker = "sys_platform == 'linux'" }, + { name = "more-itertools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } wheels = [ @@ -1459,7 +1459,7 @@ name = "jaraco-functools" version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools", marker = "sys_platform == 'linux'" }, + { name = "more-itertools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280", size = 20837, upload-time = "2026-07-14T01:28:02.544Z" } wheels = [ @@ -1471,7 +1471,7 @@ name = "jedi" version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "parso", marker = "sys_platform == 'linux'" }, + { name = "parso" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } wheels = [ @@ -1492,7 +1492,7 @@ name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe", marker = "sys_platform == 'linux'" }, + { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ @@ -1531,10 +1531,10 @@ name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "sys_platform == 'linux'" }, - { name = "jsonschema-specifications", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "rpds-py", marker = "sys_platform == 'linux'" }, + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ @@ -1543,15 +1543,15 @@ wheels = [ [package.optional-dependencies] format-nongpl = [ - { name = "fqdn", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "isoduration", marker = "sys_platform == 'linux'" }, - { name = "jsonpointer", marker = "sys_platform == 'linux'" }, - { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3987-syntax", marker = "sys_platform == 'linux'" }, - { name = "uri-template", marker = "sys_platform == 'linux'" }, - { name = "webcolors", marker = "sys_platform == 'linux'" }, + { name = "fqdn" }, + { name = "idna" }, + { name = "isoduration" }, + { name = "jsonpointer" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator" }, + { name = "rfc3987-syntax" }, + { name = "uri-template" }, + { name = "webcolors" }, ] [[package]] @@ -1559,7 +1559,7 @@ name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "referencing", marker = "sys_platform == 'linux'" }, + { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ @@ -1571,8 +1571,8 @@ name = "jupyter-builder" version = "1.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/e1/e4ef07e2be228271b59e011a746d97feef69577af1f465b1cff0f1d7c760/jupyter_builder-1.2.2.tar.gz", hash = "sha256:b6cea88f58e44b2c5eba96f28d2e0d16fd453d3ca6dc9c4492ff8a1f2e97f601", size = 981074, upload-time = "2026-08-07T06:47:18.742Z" } wheels = [ @@ -1584,12 +1584,12 @@ name = "jupyter-client" version = "8.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/2a/906772148a06e48885039e0250c340b770a55bbf37b08d6ee0449df369c3/jupyter_client-8.10.0.tar.gz", hash = "sha256:9f7116294dca55f1785be880057d44544db9b1567718d92cb33c58886afb9497", size = 360653, upload-time = "2026-08-28T12:17:10.854Z" } wheels = [ @@ -1601,8 +1601,8 @@ name = "jupyter-core" version = "5.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "platformdirs" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } wheels = [ @@ -1614,14 +1614,14 @@ name = "jupyter-events" version = "0.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonschema", extra = ["format-nongpl"], marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "python-json-logger", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "jsonschema", extra = ["format-nongpl"] }, + { name = "packaging" }, + { name = "python-json-logger" }, + { name = "pyyaml" }, + { name = "referencing" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/f8/475c4241b2b75af0deaae453ed003c6c851766dbc44d332d8baf245dc931/jupyter_events-0.12.1.tar.gz", hash = "sha256:faff25f77218335752f35f23c5fe6e4a392a7bd99a5939ccb9b8fbf594636cf3", size = 62854, upload-time = "2026-04-20T23:17:50.66Z" } wheels = [ @@ -1633,7 +1633,7 @@ name = "jupyter-lsp" version = "2.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server" }, ] sdist = { url = "https://files.pythonhosted.org/packages/36/ff/1e4a61f5170a9a1d978f3ac3872449de6c01fc71eaf89657824c878b1549/jupyter_lsp-2.3.1.tar.gz", hash = "sha256:fdf8a4aa7d85813976d6e29e95e6a2c8f752701f926f2715305249a3829805a6", size = 55677, upload-time = "2026-04-02T08:10:06.749Z" } wheels = [ @@ -1645,23 +1645,23 @@ name = "jupyter-server" version = "2.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform == 'linux'" }, - { name = "argon2-cffi", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyter-events", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server-terminals", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "prometheus-client", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "send2trash", marker = "sys_platform == 'linux'" }, - { name = "terminado", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "websocket-client", marker = "sys_platform == 'linux'" }, + { name = "anyio" }, + { name = "argon2-cffi" }, + { name = "jinja2" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "jupyter-events" }, + { name = "jupyter-server-terminals" }, + { name = "nbconvert" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "prometheus-client" }, + { name = "pyzmq" }, + { name = "send2trash" }, + { name = "terminado" }, + { name = "tornado" }, + { name = "traitlets" }, + { name = "websocket-client" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cf/e0/63b481d5b21f81fe23173dfc9eb25629fa1bc174fdc4a2c19d09c1ff8124/jupyter_server-2.21.0.tar.gz", hash = "sha256:70d9a1883f57d3576ea17f4ce061ec1a7aad7ef388d00428cfb7f5e4f0022271", size = 760357, upload-time = "2026-08-27T16:06:34.049Z" } wheels = [ @@ -1673,7 +1673,7 @@ name = "jupyter-server-terminals" version = "0.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "terminado", marker = "sys_platform == 'linux'" }, + { name = "terminado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } wheels = [ @@ -1685,19 +1685,19 @@ name = "jupyterlab" version = "4.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "async-lru", marker = "sys_platform == 'linux'" }, - { name = "httpx", marker = "sys_platform == 'linux'" }, - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-builder", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyter-lsp", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-server", marker = "sys_platform == 'linux'" }, - { name = "notebook-shim", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "async-lru" }, + { name = "httpx" }, + { name = "ipykernel" }, + { name = "jinja2" }, + { name = "jupyter-builder" }, + { name = "jupyter-core" }, + { name = "jupyter-lsp" }, + { name = "jupyter-server" }, + { name = "jupyterlab-server" }, + { name = "notebook-shim" }, + { name = "packaging" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/00/c0/45934229995d4c6e38192ca118d93185f60808f64157e3df3c5c28f8c5fd/jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498", size = 28319771, upload-time = "2026-08-10T18:50:57.947Z" } wheels = [ @@ -1718,13 +1718,13 @@ name = "jupyterlab-server" version = "2.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "json5", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "babel" }, + { name = "jinja2" }, + { name = "json5" }, + { name = "jsonschema" }, + { name = "jupyter-server" }, + { name = "packaging" }, + { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d6/2c/90153f189e421e93c4bb4f9e3f59802a1f01abd2ac5cf40b152d7f735232/jupyterlab_server-2.28.0.tar.gz", hash = "sha256:35baa81898b15f93573e2deca50d11ac0ae407ebb688299d3a5213265033712c", size = 76996, upload-time = "2025-10-22T13:59:18.37Z" } wheels = [ @@ -1736,11 +1736,11 @@ name = "keyring" version = "25.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jaraco-classes", marker = "sys_platform == 'linux'" }, - { name = "jaraco-context", marker = "sys_platform == 'linux'" }, - { name = "jaraco-functools", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney" }, + { name = "secretstorage" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } wheels = [ @@ -1848,7 +1848,7 @@ name = "lazy-loader" version = "0.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "packaging" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/ac/21a1f8aa3777f5658576777ea76bfb124b702c520bbe90edf4ae9915eafa/lazy_loader-0.5.tar.gz", hash = "sha256:717f9179a0dbed357012ddad50a5ad3d5e4d9a0b8712680d4e687f5e6e6ed9b3", size = 15294, upload-time = "2026-03-06T15:45:09.054Z" } wheels = [ @@ -1885,8 +1885,8 @@ name = "lsstdesc-coord" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "future", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "future" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/e0/6806df4cfa54927a2d8e425407611b4fc031dd125bc952b4a31deb743305/lsstdesc_coord-1.3.1.tar.gz", hash = "sha256:60f878c29e1f30a9b50bf60dca3c466dc9cfb2cbc71f0a27c575ced969de57ab", size = 41647, upload-time = "2026-02-13T18:37:32.611Z" } wheels = [ @@ -1978,7 +1978,7 @@ name = "markdown-it-py" version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl", marker = "sys_platform == 'linux'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ @@ -2028,15 +2028,15 @@ name = "matplotlib" version = "3.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy", marker = "sys_platform == 'linux'" }, - { name = "cycler", marker = "sys_platform == 'linux'" }, - { name = "fonttools", marker = "sys_platform == 'linux'" }, - { name = "kiwisolver", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "pyparsing", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/64/f9a391af28f518b11ad45a8a712353c94a0aefce09d3703200e5c54b610a/matplotlib-3.11.1.tar.gz", hash = "sha256:69647db5746941c793d6e445a4cd349323ffb87d9cc958c2ad84a659b4832d30", size = 32612045, upload-time = "2026-07-18T03:39:46.63Z" } wheels = [ @@ -2062,7 +2062,7 @@ name = "matplotlib-inline" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } wheels = [ @@ -2074,11 +2074,11 @@ name = "mccd" version = "1.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "modopt", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "python-pysap", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "modopt" }, + { name = "numpy" }, + { name = "python-pysap" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/77/b4/2d5618bff92bcc1b928dcc356cbce19cd4ade689d046ba0267da10247f38/mccd-1.2.4.tar.gz", hash = "sha256:e146c84867f4f97e58a0d3de6ead25feceb46f58a1a24e29d102007723e8719f", size = 75762, upload-time = "2023-02-17T19:09:17.039Z" } wheels = [ @@ -2090,7 +2090,7 @@ name = "mdit-py-plugins" version = "0.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, + { name = "markdown-it-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } wheels = [ @@ -2120,10 +2120,10 @@ name = "modopt" version = "1.7.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "importlib-metadata", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tqdm", marker = "sys_platform == 'linux'" }, + { name = "importlib-metadata" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/cf/7665c4990b8623b9e27733edd17a69c63e2d35c5f05d1a253d5ed2f98e52/modopt-1.7.2.tar.gz", hash = "sha256:d5e8edd935b813c3677beeed3245ef4894e7ac09e180150368eda044914488b4", size = 778069, upload-time = "2024-04-12T10:22:11.24Z" } wheels = [ @@ -2173,12 +2173,12 @@ name = "myst-parser" version = "5.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/dc/603751677fff302f34396e206b610f556a59d7fe58b9a2145f54e96b48e8/myst_parser-5.1.0.tar.gz", hash = "sha256:ab69322dc6719dcc7f296479dbb70181b66df6ed315064f92dbc85c0e1bf2f02", size = 101182, upload-time = "2026-05-13T09:38:19.361Z" } wheels = [ @@ -2199,10 +2199,10 @@ name = "nbclient" version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbformat" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } wheels = [ @@ -2214,20 +2214,20 @@ name = "nbconvert" version = "7.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "bleach", extra = ["css"], marker = "sys_platform == 'linux'" }, - { name = "defusedxml", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-pygments", marker = "sys_platform == 'linux'" }, - { name = "markupsafe", marker = "sys_platform == 'linux'" }, - { name = "mistune", marker = "sys_platform == 'linux'" }, - { name = "nbclient", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pandocfilters", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "beautifulsoup4" }, + { name = "bleach", extra = ["css"] }, + { name = "defusedxml" }, + { name = "jinja2" }, + { name = "jupyter-core" }, + { name = "jupyterlab-pygments" }, + { name = "markupsafe" }, + { name = "mistune" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "pandocfilters" }, + { name = "pygments" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } wheels = [ @@ -2239,10 +2239,10 @@ name = "nbformat" version = "5.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fastjsonschema", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/31/72/b3446efab8756e7df4b8ec587f8e611cb5a7249e4323db480802f1d3be04/nbformat-5.11.1.tar.gz", hash = "sha256:32d4521c68c6e7d5b29c76defaeed9f42ea733142b9b19f88277ce10390b9c4d", size = 147775, upload-time = "2026-08-17T08:10:51.942Z" } wheels = [ @@ -2303,9 +2303,9 @@ name = "nibabel" version = "5.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/01/3d2cc510c616bc8e27be17a063070d9126f69407961594a9ae734ea51121/nibabel-5.4.2.tar.gz", hash = "sha256:d5f4b9076a13178ae7f7acf18c8dbd503ee1c4d5c0c23b85df7be87efcbb49da", size = 4663132, upload-time = "2026-03-11T13:31:52.42Z" } wheels = [ @@ -2317,7 +2317,7 @@ name = "notebook-shim" version = "0.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server" }, ] sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } wheels = [ @@ -2329,8 +2329,8 @@ name = "numba" version = "0.67.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "llvmlite", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "llvmlite" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/90/2544f4e3a61e501d6c9a5418fd4b905323222693d54a02cab0106a0af865/numba-0.67.0.tar.gz", hash = "sha256:cd75aa535b33fa05d9d930b1ae8af9f97a2881e96d72dfb38ec9b78284d9f851", size = 2836515, upload-time = "2026-08-11T23:04:00.174Z" } wheels = [ @@ -2349,8 +2349,8 @@ name = "numcodecs" version = "0.16.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } wheels = [ @@ -2399,7 +2399,7 @@ name = "numpydoc" version = "1.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } wheels = [ @@ -2420,8 +2420,8 @@ name = "pandas" version = "3.0.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } wheels = [ @@ -2467,8 +2467,8 @@ name = "partd" version = "1.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "locket", marker = "sys_platform == 'linux'" }, - { name = "toolz", marker = "sys_platform == 'linux'" }, + { name = "locket" }, + { name = "toolz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } wheels = [ @@ -2480,7 +2480,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "sys_platform == 'linux'" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -2533,11 +2533,11 @@ name = "pims" version = "0.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "imageio", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "slicerator", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, + { name = "imageio" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "slicerator" }, + { name = "tifffile" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } @@ -2564,7 +2564,7 @@ name = "progressbar2" version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-utils", marker = "sys_platform == 'linux'" }, + { name = "python-utils" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0c/1e/5bd376abe32a392e74fffd82feb42ee1472d263bd358d628af20663d0346/progressbar2-4.6.0.tar.gz", hash = "sha256:fe48c8955a84428af77bff2642ba47041e1b8f7c867a5b7cc94f8bc255a8f0cf", size = 542464, upload-time = "2026-08-14T01:07:30.008Z" } wheels = [ @@ -2585,7 +2585,7 @@ name = "prompt-toolkit" version = "3.0.53" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wcwidth", marker = "sys_platform == 'linux'" }, + { name = "wcwidth" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } wheels = [ @@ -2658,8 +2658,8 @@ name = "pybtex" version = "0.26.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "latexcodec", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "latexcodec" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/f5/f30da9c93f0fa6d619332b2f69597219b625f35780473a05164a9981fd9a/pybtex-0.26.1.tar.gz", hash = "sha256:2e5543bea424e60e9e42eef70bff597be48649d8f68ba061a7a092b2477d5464", size = 692991, upload-time = "2026-04-03T13:05:39.014Z" } wheels = [ @@ -2671,8 +2671,8 @@ name = "pybtex-docutils" version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "pybtex", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "pybtex" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/84/796ea94d26188a853660f81bded39f8de4cfe595130aef0dea1088705a11/pybtex-docutils-1.0.3.tar.gz", hash = "sha256:3a7ebdf92b593e00e8c1c538aa9a20bca5d92d84231124715acc964d51d93c6b", size = 18348, upload-time = "2023-08-22T18:47:54.833Z" } wheels = [ @@ -2684,10 +2684,10 @@ name = "pyccl" version = "3.3.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d5/76/dc091056f1af48e6cb56d651dac6eb8494dd2515547d9543c0ce5365b873/pyccl-3.3.6.tar.gz", hash = "sha256:fd6c0181381eb57345ec9b56bdcf9300c9a31a42014aa9c8ced95f31ace16795", size = 16790489, upload-time = "2026-07-28T09:29:17.738Z" } wheels = [ @@ -2710,10 +2710,10 @@ name = "pydantic" version = "2.13.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-types", marker = "sys_platform == 'linux'" }, - { name = "pydantic-core", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, - { name = "typing-inspection", marker = "sys_platform == 'linux'" }, + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } wheels = [ @@ -2725,7 +2725,7 @@ name = "pydantic-core" version = "2.46.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } wheels = [ @@ -2778,9 +2778,9 @@ name = "pydantic-settings" version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "python-dotenv", marker = "sys_platform == 'linux'" }, - { name = "typing-inspection", marker = "sys_platform == 'linux'" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } wheels = [ @@ -2792,14 +2792,14 @@ name = "pydata-sphinx-theme" version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "accessible-pygments", marker = "sys_platform == 'linux'" }, - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "accessible-pygments" }, + { name = "babel" }, + { name = "beautifulsoup4" }, + { name = "docutils" }, + { name = "jinja2" }, + { name = "pygments" }, + { name = "requests" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/8e/add936feaaa9dade7d5b87c6852566d85e518b38ad32224dea32ef958984/pydata_sphinx_theme-0.20.0.tar.gz", hash = "sha256:0da172d41e19a66de875f4002f7054b385372ec65763852193791e658d50bb4a", size = 5004756, upload-time = "2026-07-09T09:09:14.693Z" } wheels = [ @@ -2811,7 +2811,7 @@ name = "pyerfa" version = "2.0.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/39/63cc8291b0cf324ae710df41527faf7d331bce573899199d926b3e492260/pyerfa-2.0.1.5.tar.gz", hash = "sha256:17d6b24fe4846c65d5e7d8c362dcb08199dc63b30a236aedd73875cc83e1f6c0", size = 818430, upload-time = "2024-11-11T15:22:30.852Z" } wheels = [ @@ -2834,8 +2834,8 @@ name = "pyopenssl" version = "26.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } wheels = [ @@ -2865,8 +2865,8 @@ name = "pyqt5" version = "5.15.11" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyqt5-qt5", marker = "sys_platform == 'linux'" }, - { name = "pyqt5-sip", marker = "sys_platform == 'linux'" }, + { name = "pyqt5-qt5" }, + { name = "pyqt5-sip" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/07/c9ed0bd428df6f87183fca565a79fee19fa7c88c7f00a7f011ab4379e77a/PyQt5-5.15.11.tar.gz", hash = "sha256:fda45743ebb4a27b4b1a51c6d8ef455c4c1b5d610c90d2934c7802b5c1557c52", size = 3216775, upload-time = "2024-07-19T08:39:57.756Z" } wheels = [ @@ -2897,8 +2897,8 @@ name = "pyqtgraph" version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "colorama" }, + { name = "numpy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/32/36/4c242f81fdcbfa4fb62a5645f6af79191f4097a0577bd5460c24f19cc4ef/pyqtgraph-0.14.0-py3-none-any.whl", hash = "sha256:7abb7c3e17362add64f8711b474dffac5e7b0e9245abdf992e9a44119b7aa4f5", size = 1924755, upload-time = "2025-11-16T19:43:22.251Z" }, @@ -2909,10 +2909,10 @@ name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "iniconfig", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pluggy", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ @@ -2924,9 +2924,9 @@ name = "pytest-cov" version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coverage", marker = "sys_platform == 'linux'" }, - { name = "pluggy", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } wheels = [ @@ -2938,7 +2938,7 @@ name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, + { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ @@ -2968,16 +2968,16 @@ name = "python-pysap" version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "modopt", marker = "sys_platform == 'linux'" }, - { name = "nibabel", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "progressbar2", marker = "sys_platform == 'linux'" }, - { name = "pywavelets", marker = "sys_platform == 'linux'" }, - { name = "scikit-image", marker = "sys_platform == 'linux'" }, - { name = "scikit-learn", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "matplotlib" }, + { name = "modopt" }, + { name = "nibabel" }, + { name = "numpy" }, + { name = "progressbar2" }, + { name = "pywavelets" }, + { name = "scikit-image" }, + { name = "scikit-learn" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f1/90/29b10f180876a322c7cfbadd2552efef34374dceb618fdc3f48ef792666e/python_pysap-0.3.0.tar.gz", hash = "sha256:55cdfec05eacc410723f31df754b67728224935aef7d4332b89ef6e90f2aab97", size = 775664, upload-time = "2026-01-06T10:12:58.287Z" } @@ -2986,7 +2986,7 @@ name = "python-utils" version = "4.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/71/ec6665d4ce42ee5a59fffd31a4d5164f92da15ccb8c758dba13d2419ea53/python_utils-4.0.1.tar.gz", hash = "sha256:4e8e8ecaba3862f843a60c1982c99cda23b522f417006a807996a876c18beb8d", size = 43765, upload-time = "2026-08-30T19:26:04.591Z" } wheels = [ @@ -3007,8 +3007,8 @@ name = "pyvo" version = "1.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/35/12c0f4fa0879316837ac56f275942f006f0735e5aedba529b4449dddc36f/pyvo-1.9.1.tar.gz", hash = "sha256:2f26c99af7c32f3c34b919e2d14eaf1a95914176d693fb7769773f3ab0b7999d", size = 2167800, upload-time = "2026-06-11T14:44:01.349Z" } wheels = [ @@ -3020,7 +3020,7 @@ name = "pywavelets" version = "1.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/75/50581633d199812205ea8cdd0f6d52f12a624886b74bf1486335b67f01ff/pywavelets-1.9.0.tar.gz", hash = "sha256:148d12203377772bea452a59211d98649c8ee4a05eff019a9021853a36babdc8", size = 3938340, upload-time = "2025-08-04T16:20:04.978Z" } wheels = [ @@ -3079,7 +3079,7 @@ name = "pyzmq" version = "27.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy' and sys_platform == 'linux'" }, + { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e7/8d/5b3d5631c2f4b4b8862f64cd0c9eb777b5710eeb5125b4be8dd0a200a4c0/pyzmq-27.2.0.tar.gz", hash = "sha256:54d4259d1bfae24ecdb5ca79f7acc2eac6c286a02d6a0ae617797cb45f0726d3", size = 292316, upload-time = "2026-08-20T19:08:21.19Z" } wheels = [ @@ -3109,7 +3109,7 @@ name = "questionary" version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, + { name = "prompt-toolkit" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f6/45/eafb0bba0f9988f6a2520f9ca2df2c82ddfa8d67c95d6625452e97b204a5/questionary-2.1.1.tar.gz", hash = "sha256:3d7e980292bb0107abaa79c68dd3eee3c561b83a0f89ae482860b181c8bd412d", size = 25845, upload-time = "2025-08-28T19:00:20.851Z" } wheels = [ @@ -3121,9 +3121,9 @@ name = "readme-renderer" version = "46.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "nh3", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "nh3" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/d7/9309494fad74ee831d4546f69325b5519f37c6dfb2d9ba495db8c6d4f4ca/readme_renderer-46.0.tar.gz", hash = "sha256:af3e964914f6310a33ff67b72a4bdd940bed8d7c3bdecd2d14f40edf284bfe90", size = 38382, upload-time = "2026-08-28T15:18:32.49Z" } wheels = [ @@ -3135,9 +3135,9 @@ name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "sys_platform == 'linux'" }, - { name = "rpds-py", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -3149,16 +3149,16 @@ name = "reproject" version = "0.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "astropy-healpix", marker = "sys_platform == 'linux'" }, - { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, - { name = "dask-image", marker = "sys_platform == 'linux'" }, - { name = "fsspec", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "pyavm", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "zarr", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "astropy-healpix" }, + { name = "dask", extra = ["array"] }, + { name = "dask-image" }, + { name = "fsspec" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "pyavm" }, + { name = "scipy" }, + { name = "zarr" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } wheels = [ @@ -3171,10 +3171,10 @@ name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "charset-normalizer", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "urllib3", marker = "sys_platform == 'linux'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -3186,7 +3186,7 @@ name = "requests-toolbelt" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } wheels = [ @@ -3198,7 +3198,7 @@ name = "rfc3339-validator" version = "0.1.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, + { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } wheels = [ @@ -3228,7 +3228,7 @@ name = "rfc3987-syntax" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "lark", marker = "sys_platform == 'linux'" }, + { name = "lark" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz", hash = "sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d", size = 14239, upload-time = "2025-07-18T01:05:05.015Z" } wheels = [ @@ -3240,8 +3240,8 @@ name = "rich" version = "15.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "markdown-it-py" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } wheels = [ @@ -3350,14 +3350,14 @@ name = "scikit-image" version = "0.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "imageio", marker = "sys_platform == 'linux'" }, - { name = "lazy-loader", marker = "sys_platform == 'linux'" }, - { name = "networkx", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, + { name = "imageio" }, + { name = "lazy-loader" }, + { name = "networkx" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "scipy" }, + { name = "tifffile" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/b4/2528bb43c67d48053a7a649a9666432dc307d66ba02e3a6d5c40f46655df/scikit_image-0.26.0.tar.gz", hash = "sha256:f5f970ab04efad85c24714321fcc91613fcb64ef2a892a13167df2f3e59199fa", size = 22729739, upload-time = "2025-12-20T17:12:21.824Z" } wheels = [ @@ -3388,11 +3388,11 @@ name = "scikit-learn" version = "1.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "joblib", marker = "sys_platform == 'linux'" }, - { name = "narwhals", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "threadpoolctl", marker = "sys_platform == 'linux'" }, + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -3411,7 +3411,7 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3442,8 +3442,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -3482,9 +3482,9 @@ name = "sf-tools" version = "2.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "future", marker = "sys_platform == 'linux'" }, - { name = "modopt", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "future" }, + { name = "modopt" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/41/6281d12b863867fc7d7db0aeb55db7ec94fb8f35873b9940edeeb3c29d0f/sf_tools-2.0.4.tar.gz", hash = "sha256:607204b7369cb381c02ba95f49f13d2733877a625026fac340dcddcca6700d24", size = 10426, upload-time = "2019-08-22T08:41:46.961Z" } @@ -3493,87 +3493,86 @@ name = "shapepipe" version = "1.1.0" source = { editable = "." } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "astroquery", marker = "sys_platform == 'linux'" }, - { name = "canfar", marker = "sys_platform == 'linux'" }, - { name = "cs-util", marker = "sys_platform == 'linux'" }, - { name = "galsim", marker = "sys_platform == 'linux'" }, - { name = "h5py", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "joblib", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "mccd", marker = "sys_platform == 'linux'" }, - { name = "modopt", marker = "sys_platform == 'linux'" }, - { name = "mpi4py", marker = "sys_platform == 'linux'" }, - { name = "ngmix", marker = "sys_platform == 'linux'" }, - { name = "numba", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "pyqt5", marker = "sys_platform == 'linux'" }, - { name = "pyqtgraph", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "python-pysap", marker = "sys_platform == 'linux'" }, - { name = "reproject", marker = "sys_platform == 'linux'" }, - { name = "sf-tools", marker = "sys_platform == 'linux'" }, - { name = "skaha", marker = "sys_platform == 'linux'" }, - { name = "skyproj", marker = "sys_platform == 'linux'" }, - { name = "sqlitedict", marker = "sys_platform == 'linux'" }, - { name = "termcolor", marker = "sys_platform == 'linux'" }, - { name = "tqdm", marker = "sys_platform == 'linux'" }, - { name = "vos", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "astroquery" }, + { name = "canfar" }, + { name = "cs-util" }, + { name = "galsim" }, + { name = "h5py" }, + { name = "healsparse" }, + { name = "hpgeom" }, + { name = "joblib" }, + { name = "matplotlib" }, + { name = "mccd" }, + { name = "modopt" }, + { name = "mpi4py" }, + { name = "ngmix" }, + { name = "numba" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pyqt5" }, + { name = "pyqtgraph" }, + { name = "python-dateutil" }, + { name = "python-pysap" }, + { name = "reproject" }, + { name = "sf-tools" }, + { name = "skyproj" }, + { name = "sqlitedict" }, + { name = "termcolor" }, + { name = "tqdm" }, + { name = "vos" }, ] [package.optional-dependencies] dev = [ - { name = "build", marker = "sys_platform == 'linux'" }, - { name = "fitsio", marker = "sys_platform == 'linux'" }, - { name = "hypothesis", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "myst-parser", marker = "sys_platform == 'linux'" }, - { name = "numpydoc", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, - { name = "pytest-cov", marker = "sys_platform == 'linux'" }, - { name = "ruff", marker = "sys_platform == 'linux'" }, - { name = "skyproj", marker = "sys_platform == 'linux'" }, - { name = "snakemake", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, - { name = "sphinx-book-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, - { name = "twine", marker = "sys_platform == 'linux'" }, + { name = "build" }, + { name = "fitsio" }, + { name = "hypothesis" }, + { name = "ipython" }, + { name = "jupyterlab" }, + { name = "matplotlib" }, + { name = "myst-parser" }, + { name = "numpydoc" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, + { name = "skyproj" }, + { name = "snakemake" }, + { name = "sphinx" }, + { name = "sphinx-book-theme" }, + { name = "sphinxcontrib-bibtex" }, + { name = "twine" }, ] doc = [ - { name = "myst-parser", marker = "sys_platform == 'linux'" }, - { name = "numpydoc", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, - { name = "sphinx-book-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, + { name = "myst-parser" }, + { name = "numpydoc" }, + { name = "sphinx" }, + { name = "sphinx-book-theme" }, + { name = "sphinxcontrib-bibtex" }, ] fitsio = [ - { name = "fitsio", marker = "sys_platform == 'linux'" }, + { name = "fitsio" }, ] jupyter = [ - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "snakemake", marker = "sys_platform == 'linux'" }, + { name = "ipython" }, + { name = "jupyterlab" }, + { name = "snakemake" }, ] lint = [ - { name = "ruff", marker = "sys_platform == 'linux'" }, + { name = "ruff" }, ] plot = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "skyproj", marker = "sys_platform == 'linux'" }, + { name = "matplotlib" }, + { name = "skyproj" }, ] release = [ - { name = "build", marker = "sys_platform == 'linux'" }, - { name = "twine", marker = "sys_platform == 'linux'" }, + { name = "build" }, + { name = "twine" }, ] test = [ - { name = "hypothesis", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, - { name = "pytest-cov", marker = "sys_platform == 'linux'" }, + { name = "hypothesis" }, + { name = "pytest" }, + { name = "pytest-cov" }, ] [package.metadata] @@ -3613,7 +3612,6 @@ requires-dist = [ { name = "ruff", marker = "extra == 'lint'" }, { name = "sf-tools", specifier = ">=2.0.4" }, { name = "shapepipe", extras = ["doc", "jupyter", "lint", "plot", "release", "test", "fitsio"], marker = "extra == 'dev'" }, - { name = "skaha", specifier = ">=1.7" }, { name = "skyproj" }, { name = "skyproj", marker = "extra == 'plot'" }, { name = "snakemake", marker = "extra == 'jupyter'", specifier = ">=9.22.0" }, @@ -3646,34 +3644,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "skaha" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "defusedxml", marker = "sys_platform == 'linux'" }, - { name = "httpx", marker = "sys_platform == 'linux'" }, - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "rich", marker = "sys_platform == 'linux'" }, - { name = "toml", marker = "sys_platform == 'linux'" }, - { name = "typer", marker = "sys_platform == 'linux'" }, - { name = "vos", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/62/a733508ff88200c41459f5cdb72a797eb8795c833b4e9d51d31b5efe4073/skaha-1.7.0.tar.gz", hash = "sha256:ef9d69e7a4da8653cdbde3e62f18caef137130064ccb6fabed57c7c6f8bf8ef9", size = 54414, upload-time = "2025-05-28T21:17:03.631Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/d3/a3c1d569ae714ed9985adccefdfe2d6dd9f8ad7a297bd8fa4c7ed30c587b/skaha-1.7.0-py3-none-any.whl", hash = "sha256:68b0d3c925b98bf145c5f695237474fe1cea07cf9b69b4a9bae9910375dfa01a", size = 40795, upload-time = "2025-05-28T21:17:02.296Z" }, -] - [[package]] name = "skyproj" version = "2.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "healsparse" }, + { name = "hpgeom" }, + { name = "matplotlib" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/90/cf/5158151ae6fc60459f430451b612a4bf2da07ebff2a9f52a37e73715a5ee/skyproj-2.5.0.tar.gz", hash = "sha256:cb8d5115927ca43cacdb6d92f00bb4fcc8563ee71028f755203ce1752e67c140", size = 8521587, upload-time = "2026-06-26T19:51:15.378Z" } wheels = [ @@ -3695,7 +3675,7 @@ name = "smart-open" version = "7.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt", marker = "sys_platform == 'linux'" }, + { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/c6/22e7a2acd5d27941e85e0d7ede398da5abe2e4677d2265c924157247c32e/smart_open-7.7.1.tar.gz", hash = "sha256:9414ba5733e28309f29b28a303b0f1054ad23fe0275f1a1b600c80a724f4bd1a", size = 54952, upload-time = "2026-06-26T07:56:35.309Z" } wheels = [ @@ -3716,37 +3696,37 @@ name = "snakemake" version = "9.26.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "conda-inject", marker = "sys_platform == 'linux'" }, - { name = "configargparse", marker = "sys_platform == 'linux'" }, - { name = "connection-pool", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "dpath", marker = "sys_platform == 'linux'" }, - { name = "gitpython", marker = "sys_platform == 'linux'" }, - { name = "humanfriendly", marker = "sys_platform == 'linux'" }, - { name = "immutables", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "platformdirs", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pulp", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "smart-open", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-executor-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-logger-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-report-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-scheduler-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-storage-plugins", marker = "sys_platform == 'linux'" }, - { name = "sqlmodel", marker = "sys_platform == 'linux'" }, - { name = "tabulate", marker = "sys_platform == 'linux'" }, - { name = "tenacity", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, - { name = "wrapt", marker = "sys_platform == 'linux'" }, - { name = "yte", marker = "sys_platform == 'linux'" }, + { name = "conda-inject" }, + { name = "configargparse" }, + { name = "connection-pool" }, + { name = "docutils" }, + { name = "dpath" }, + { name = "gitpython" }, + { name = "humanfriendly" }, + { name = "immutables" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "psutil" }, + { name = "pulp" }, + { name = "pyyaml" }, + { name = "referencing" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "snakemake-interface-common" }, + { name = "snakemake-interface-executor-plugins" }, + { name = "snakemake-interface-logger-plugins" }, + { name = "snakemake-interface-report-plugins" }, + { name = "snakemake-interface-scheduler-plugins" }, + { name = "snakemake-interface-storage-plugins" }, + { name = "sqlmodel" }, + { name = "tabulate" }, + { name = "tenacity" }, + { name = "throttler" }, + { name = "wrapt" }, + { name = "yte" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3e/37/f999e99357a9b30c6f36c92027ab3b6f64cade4ec54840197853297d2e04/snakemake-9.26.1.tar.gz", hash = "sha256:091b5d480c0c5eb2ef75b2568e9885091f3e2f11372de358da98ef66ef66e1e0", size = 6814489, upload-time = "2026-08-28T05:44:51.852Z" } wheels = [ @@ -3758,9 +3738,9 @@ name = "snakemake-interface-common" version = "1.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "configargparse", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "argparse-dataclass" }, + { name = "configargparse" }, + { name = "packaging" }, ] sdist = { url = "https://files.pythonhosted.org/packages/89/c3/592f832f6e5d2d31f749392e48e8401b7625dec668d3d365d8d28f2b6c30/snakemake_interface_common-1.23.0.tar.gz", hash = "sha256:6ed14531a461417659364a0dd0acc51b786af4e26fc15cc5e00ff3d9fcaffacc", size = 13960, upload-time = "2026-03-08T21:54:29.251Z" } wheels = [ @@ -3772,9 +3752,9 @@ name = "snakemake-interface-executor-plugins" version = "9.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, + { name = "argparse-dataclass" }, + { name = "snakemake-interface-common" }, + { name = "throttler" }, ] sdist = { url = "https://files.pythonhosted.org/packages/54/50/de06b284c45a8e94fb8e4a12d5235065e78b49b8f84329dc10fe39f4b7dd/snakemake_interface_executor_plugins-9.4.0.tar.gz", hash = "sha256:9d4138897beacbaadaedad94b63f948eaeb604b7fc78f9cf65ac57f090f2c066", size = 16549, upload-time = "2026-03-08T17:04:02.644Z" } wheels = [ @@ -3786,7 +3766,7 @@ name = "snakemake-interface-logger-plugins" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/0c/3fa5d592663c65669a867526604aadc6fbc235fb9284e94b49c0ef59aa41/snakemake_interface_logger_plugins-2.1.0.tar.gz", hash = "sha256:c89a00d2a398490cecd91b6dc6db8049cba93712d82e1d8f3000f3040bf3791c", size = 15917, upload-time = "2026-05-20T15:12:35.259Z" } wheels = [ @@ -3798,7 +3778,7 @@ name = "snakemake-interface-report-plugins" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/d6/6160ed98de665d6871dd356597dbf726688cc786e88668359ca37b7d9f54/snakemake_interface_report_plugins-1.3.0.tar.gz", hash = "sha256:fc9495298bec4e69721ab8afe6d6d88a86966fda2eeb003db56b9a88b86d5934", size = 4283, upload-time = "2025-10-31T10:52:36.55Z" } wheels = [ @@ -3810,7 +3790,7 @@ name = "snakemake-interface-scheduler-plugins" version = "2.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common" }, ] sdist = { url = "https://files.pythonhosted.org/packages/88/d9/d480807d2cfc2d132bc760d877d45ec8fbe620a24200ec4d2697c4a26031/snakemake_interface_scheduler_plugins-2.0.2.tar.gz", hash = "sha256:2797e8fa9019d983132c2b403f14d6fcd3c5ad4c8d8a66b984b4740a71cacc46", size = 8642, upload-time = "2025-10-20T13:58:12.988Z" } wheels = [ @@ -3822,11 +3802,11 @@ name = "snakemake-interface-storage-plugins" version = "4.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "tenacity", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, - { name = "wrapt", marker = "sys_platform == 'linux'" }, + { name = "humanfriendly" }, + { name = "snakemake-interface-common" }, + { name = "tenacity" }, + { name = "throttler" }, + { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/6e/f3c5b2d621fd6a6b78d8cfc01fef6b926fe2c277f5ed77c5e4deeacb94eb/snakemake_interface_storage_plugins-4.4.1.tar.gz", hash = "sha256:b2b5bf05318af36955ebf2ce76c921c0fb06904ca98fb30e1657d88b7b7b6945", size = 14924, upload-time = "2026-03-16T11:16:01.075Z" } wheels = [ @@ -3865,22 +3845,22 @@ name = "sphinx" version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "alabaster", marker = "sys_platform == 'linux'" }, - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "imagesize", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "roman-numerals", marker = "sys_platform == 'linux'" }, - { name = "snowballstemmer", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-applehelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-devhelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-htmlhelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-jsmath", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-qthelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-serializinghtml", marker = "sys_platform == 'linux'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -3892,8 +3872,8 @@ name = "sphinx-book-theme" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydata-sphinx-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "pydata-sphinx-theme" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/8a/cb008736682b3974cae6925c54939792fe56f5a167e37984cb45d8a85d8a/sphinx_book_theme-1.4.0.tar.gz", hash = "sha256:fee3cd70573d9a8cd4ab380a3cb1f11139c24dcaa2cf47d06d7f924f1b44eaff", size = 406113, upload-time = "2026-07-19T21:23:54.183Z" } wheels = [ @@ -3914,10 +3894,10 @@ name = "sphinxcontrib-bibtex" version = "2.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "pybtex", marker = "sys_platform == 'linux'" }, - { name = "pybtex-docutils", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "pybtex" }, + { name = "pybtex-docutils" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/6a/8e0b2c2420286389e7fed78ff361ec30e2f1d58c8560af8d64df5e7b61e0/sphinxcontrib_bibtex-2.7.0.tar.gz", hash = "sha256:fee700f7aae29bb8f654c62913f00d34ac44fc0b8ca0fa67ac922ff4453addee", size = 120669, upload-time = "2026-05-06T09:29:24.935Z" } wheels = [ @@ -3974,8 +3954,8 @@ name = "sqlalchemy" version = "2.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "greenlet", marker = "(platform_machine == 'AMD64' and sys_platform == 'linux') or (platform_machine == 'WIN32' and sys_platform == 'linux') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'amd64' and sys_platform == 'linux') or (platform_machine == 'ppc64le' and sys_platform == 'linux') or (platform_machine == 'win32' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } wheels = [ @@ -4005,8 +3985,8 @@ name = "sqlmodel" version = "0.0.37" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "sqlalchemy", marker = "sys_platform == 'linux'" }, + { name = "pydantic" }, + { name = "sqlalchemy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fb/26/1d2faa0fd5a765267f49751de533adac6b9ff9366c7c6e7692df4f32230f/sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352", size = 85527, upload-time = "2026-02-21T16:39:47.038Z" } wheels = [ @@ -4018,9 +3998,9 @@ name = "stack-data" version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "asttokens", marker = "sys_platform == 'linux'" }, - { name = "executing", marker = "sys_platform == 'linux'" }, - { name = "pure-eval", marker = "sys_platform == 'linux'" }, + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } wheels = [ @@ -4054,7 +4034,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath", marker = "sys_platform == 'linux'" }, + { name = "mpmath" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -4093,8 +4073,8 @@ name = "terminado" version = "0.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "os_name != 'nt' and sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "ptyprocess", marker = "os_name != 'nt'" }, + { name = "tornado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } wheels = [ @@ -4124,7 +4104,7 @@ name = "tifffile" version = "2026.8.23" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/07/90078f49e60718d414d5440dc498301f11ff049458e65cf8d27c62a5c9d1/tifffile-2026.8.23.tar.gz", hash = "sha256:bd3c816f166f85c93329a54a0c9a1eccc9968a6a78f91d63b65d7b17675915f2", size = 446179, upload-time = "2026-08-23T18:45:05.976Z" } wheels = [ @@ -4136,7 +4116,7 @@ name = "tinycss2" version = "1.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "webencodings", marker = "sys_platform == 'linux'" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/ae/2ca4913e5c0f09781d75482874c3a95db9105462a92ddd303c7d285d3df2/tinycss2-1.5.1.tar.gz", hash = "sha256:d339d2b616ba90ccce58da8495a78f46e55d4d25f9fd71dfd526f07e7d53f957", size = 88195, upload-time = "2025-11-23T10:29:10.082Z" } wheels = [ @@ -4196,15 +4176,15 @@ name = "twine" version = "6.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "id", marker = "sys_platform == 'linux'" }, - { name = "keyring", marker = "platform_machine != 'ppc64le' and platform_machine != 's390x' and sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "readme-renderer", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "requests-toolbelt", marker = "sys_platform == 'linux'" }, - { name = "rfc3986", marker = "sys_platform == 'linux'" }, - { name = "rich", marker = "sys_platform == 'linux'" }, - { name = "urllib3", marker = "sys_platform == 'linux'" }, + { name = "id" }, + { name = "keyring", marker = "platform_machine != 'ppc64le' and platform_machine != 's390x'" }, + { name = "packaging" }, + { name = "readme-renderer" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "rfc3986" }, + { name = "rich" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/a8/949edebe3a82774c1ec34f637f5dd82d1cf22c25e963b7d63771083bbee5/twine-6.2.0.tar.gz", hash = "sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf", size = 172262, upload-time = "2025-09-04T15:43:17.255Z" } wheels = [ @@ -4216,9 +4196,9 @@ name = "typer" version = "0.27.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-doc", marker = "sys_platform == 'linux'" }, - { name = "rich", marker = "sys_platform == 'linux'" }, - { name = "shellingham", marker = "sys_platform == 'linux'" }, + { name = "annotated-doc" }, + { name = "rich" }, + { name = "shellingham" }, ] sdist = { url = "https://files.pythonhosted.org/packages/16/f7/57713ba479fd405eb76de31404b2c744c289e336b2d999511ebf51e496f7/typer-0.27.2.tar.gz", hash = "sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945", size = 204045, upload-time = "2026-08-28T10:26:55.046Z" } wheels = [ @@ -4239,7 +4219,7 @@ name = "typing-inspection" version = "0.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } wheels = [ @@ -4278,9 +4258,9 @@ name = "vos" version = "3.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aenum", marker = "sys_platform == 'linux'" }, - { name = "cadcutils", marker = "sys_platform == 'linux'" }, - { name = "html2text", marker = "sys_platform == 'linux'" }, + { name = "aenum" }, + { name = "cadcutils" }, + { name = "html2text" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2f/71/35b07ebde34232b4f3deca8cb320c3ad615b15818fbb3b10a5209c14aadc/vos-3.7.tar.gz", hash = "sha256:073cde0ec3ffe52f283ba6505870e760492a73743986797686e7dc57d30ca563", size = 103538, upload-time = "2026-08-06T23:47:46.099Z" } wheels = [ @@ -4373,9 +4353,9 @@ name = "yte" version = "1.9.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "dpath", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "argparse-dataclass" }, + { name = "dpath" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/f5/7e44620e6e077bfe624b9a17c329b8e0d0159e176e1f1a93c2790428ab2c/yte-1.9.4.tar.gz", hash = "sha256:86a47e6d722cec9419a7ac88be57d0d6c4ce28f02860393b71a66f2c674069f6", size = 8101, upload-time = "2025-11-27T12:55:00.85Z" } wheels = [ @@ -4387,12 +4367,12 @@ name = "zarr" version = "3.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "donfig", marker = "sys_platform == 'linux'" }, - { name = "google-crc32c", marker = "sys_platform == 'linux'" }, - { name = "numcodecs", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "donfig" }, + { name = "google-crc32c" }, + { name = "numcodecs" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/34/15/436cb1d3bbe86173bd44ce7a34ecb210d0c0416946e337858149a905ef5a/zarr-3.3.0.tar.gz", hash = "sha256:cd0c8cf738b4bb4807815bc1255acad5bdf1a7b7264b606c5a1bc0d0392a306b", size = 943626, upload-time = "2026-07-30T16:35:10.491Z" } wheels = [ diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 000000000..74d1279c8 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,311 @@ +# ShapePipe Snakemake orchestration + +Snakemake workflow that orchestrates real-data ShapePipe runs. It is the one +production orchestration. It replaced the bit-coded bash layers +(`run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash`) for real data, and +retired the CANFAR submission front end that drove them +(`curl_canfar_local.sh`, `canfar_submit_job`) and the per-site sbatch +reimplementations — those were last carried at `2ef07e45`. The two Gen-2 bash +scripts are still in the tree and still installed (`pyproject.toml`'s +`[tool.setuptools] script-files`): sp_validation's image-simulation workflow +calls `run_job_sp_canfar_v2.0.bash` by path, so they retire when that chain is +ported or parked, not with this one. **Module code is untouched**: rules call +`shapepipe_run -c ` on the existing config chains. Design and rationale: +[CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +(the living PRD). + +**There is no concept of a catalogue version in the code.** No path branches on +`v1.3`..`v1.6` or `v2.0`, and there are no sky patches: a campaign is a tile +list, and the version of a catalogue is the git tag of the code that produced +it. What that retirement removed, and where each piece was last carried, is in +the PR that made this sentence true. + +Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is +unsupported: `sp` sets the state directory, the SLURM profile, and +`SP_PHASE`, which the Snakefile needs to build the tile/exposure index at +parse time. Running snakemake directly skips all of that. + +## Quick start (nibi) + +```bash +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which +# is node-local; the SLURM executor re-invokes this python inside every job). +uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 +source /project/def-mjhudson/cdaley/snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' + +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the +# star-catalogue cache root). + +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a +# fresh shell always has the right state. +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp cancel # scancel this workflow's jobs +workflow/bin/sp container status # which image the jobs will run +``` + +Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, +queried 2026-07-30): `snakemake==9.23.1`, `snakemake-executor-plugin-slurm==2.7.1`. +Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The +v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became +plugins, and full `rerun-triggers` became the default. + +Anything other than `run`, `report`, `container`, `cancel` passes straight through to +snakemake with the workflow's profile and state dir — the escape hatch for +`sp --unlock`, `sp --dag`, `sp exp_psf ...`. + +## The container image + +`sp container` owns which image the jobs run inside. Two layers, and the second +only exists if you ask for one: + +* your **cached SIF** (`~/.cache/shapepipe/shapepipe.sif`, `SP_CACHE_DIR` or + `SP_CONTAINER` to move it) — a pristine pull of the published image, private + to you, so nobody else's refresh moves the ground under your running jobs; +* an optional **sandbox** (`~/.cache/shapepipe/sandbox/`, `SP_SANDBOX`) — the + same image unpacked writable, so a `pip install` into it sticks. The escape + hatch for work needing a package the image does not carry yet. + +The Snakefile's `container:` is that resolution, in one order shared by the CLI +and the workflow: **sandbox → cached SIF → the `container:` path in +`config.yaml`**. With an empty cache — the normal case — that lands on the +shared `/project` `.sif` the workflow has always used, so this changes nothing +until you opt in. + +```bash +sp container status # layers present, active one, revision vs HEAD +sp container pull # ghcr.io/cosmostat/shapepipe:develop-runtime +sp container pull --tag docker://... # some other image +sp container sandbox # unpack the SIF writable (opt-in) +sp container exec --writable pip install +sp container exec python -c 'import shapepipe' +sp container resolve # just the path the workflow will run +``` + +`status` reads the image's OCI labels and places its +`org.opencontainers.image.revision` against this checkout's HEAD: +in-sync / behind / ahead / diverged, or unknown when the image carries no label +or the commit was never fetched here. + +**`pull` needs the network.** Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an `salloc` allocation — never from a +batch job. `pull` and `sandbox` both stage to a sibling path and swap it in, so +an in-flight job never sees a half-written image and a failed rebuild leaves the +one you had intact. + +## Execution: two static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, +so it cannot live in the same static DAG that produces it. `sp run` is +therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** — `snakemake prepare_all_tiles`: per-tile static DAG + (`Git_vos → Uz → Fe`), `keep-going` so tile failures are independent. A + nonzero exit here is not fatal to the run — tiles that lost their + exposure list are dropped at the compute parse — but it is a warning: + `SP_MISSING_THRESHOLD` (default 0.0) is the real gate. +2. **COMPUTE** — `snakemake all`: this invocation's *parse* builds the + tile↔exposure index (`build_index.py`, imported at parse time, not a DAG + node) and runs the full tile/exposure compute chain. The index + accumulates across invocations, so appending tiles later changes which + jobs exist without invalidating completed work. + +`sp run` chains both so the UX is one command; both exit codes are checked +and the run fails if either phase failed. + +## The launch code snapshot + +`sp run` copies the code it is about to launch — `workflow/` (config symlinks +dereferenced), `src/` and the profile — into `/code`, records HEAD +plus a dirty flag in `/code/snapshot.json`, and runs the campaign +entirely out of that copy. It matters because a campaign is not one process: the +SLURM executor re-invokes snakemake on every job's node, so jobs re-parse the +Snakefile and read `workflow/scripts/*` and the ini chain hours after launch. +**Editing the checkout while a campaign runs is therefore harmless; a change +takes effect on the next `sp run`.** + +Everything workflow-internal hangs off `workflow.basedir`, which *is* the +snapshot, so it follows for free. The one exception is the profile's `PYTHONPATH` +pin, which YAML cannot interpolate: `sp run` rewrites that single path in the +snapshot's copy of the profile and launches `--profile` at the copy. The snapshot +is refreshed wholesale on every `sp run` — a new `sp run` *is* the relaunch — and +the other verbs run out of the existing snapshot, `sp container` excepted (it is +about the image you are working with now, not about a campaign). The mechanism +and its rationale live in one place: `bin/sp`. + +## Execution mode: one SLURM job per rule + +The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule +instance becomes its own SLURM job carrying that rule's own attempt-scaled +resources (`cpus_per_task = threads`, `mem_mb`, `runtime`) and runs inside +`apptainer exec` via the profile's software-deployment method — the workflow +never calls apptainer directly. Snakemake feeds the queue as jobs finish, so +the full campaign's job count never needs to queue at once, and multi-node +scaling is inherent. `jobs:` in the profile caps concurrent submissions at a +fraction of the cluster's per-user submit limit (queried, not invented — see +the comment in the profile file for the query and date). + +The profile intentionally sets no `set-resources` / `set-threads` overrides: +those replace a rule's own values wholesale, which would kill the +attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +ngmix thread count. Rules own their own resources; the profile only supplies +defaults for rules that state nothing. + +`group:` labels that fuse short rules (uncompress, merges) into their chunky +neighbours (queue-latency amortization, per the PRD) are **not yet wired**: +they require labels in `workflow/rules/*.smk`, out of scope for this +profile-only pass. + +## Layout + +``` +workflow/ + Snakefile parse-time index load; global container:; onsuccess/onerror report hooks + config.yaml the run: tile list, paths, container, chunk count + bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) + rules/ + prepare.smk tile get_images/uncompress/find_exposures + exposure.smk per-exposure: get_images, star_cat, split, mask, psf, persist, footprint (no temp()) + tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + coverage.smk campaign-level: the HealSparse nexp mask from the exposure footprints + scripts/ + build_index.py prepare-phase run_index.sqlite builder (plain script) + build_forest.py per-tile exposure symlink forest (group-compatible shell) + completeness.py the ported count-floor table + the `check` every rule ends with + run_report.py standalone report (NOT a DAG node; run_report hooks call it) + container.py image layers + the resolution order behind `sp container` (stdlib-only) + star_cats.py the campaign's GSC 2.3 sky store + the per-exposure cuts from it + ngmix_range.py the ngmix chunk partition: written once by tile_vignets, read by each chunk + persist_exp.py ONE exposure's keepable PSF products -> one tar on products_dir (the exp_persist rule) + exp_footprint.py ONE exposure's per-CCD sky corners, for the CCDs with a PSF (the exp_footprint rule) + coverage_map.py every exposure footprint on products_dir -> coverage.hsp (the coverage_map rule) + clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) + clean_tile.py ONE finished tile's store -> tombstone (the clean_tile rule) +profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going +``` + +## How it works + +- **The atom is one rule == one `shapepipe_run` on one unit.** Its single + declared output is that unit's manifest + (`/manifests/.json`), not its product files — a missing + CCD is often legitimate, and at DR6 scale per-CCD declaration means + millions of paths. +- **Manifests are the DAG's currency, and they are success-only.** + `completeness.py check` writes its full verdict — per-runner counts against + floors, scraped failure reasons, the `shapepipe_run` exit status when nonzero + — to the rule's `log:` (`/logs/.json`) on *every* run, and + additionally to the declared `.json` manifest only when that verdict is + a success. So `.json` on disk means "this stage succeeded", and a + resume after an unclean death cannot schedule downstream work on top of a + failure. Nothing unlinks anything: Snakemake deletes a failed job's declared + output natively and never touches its log, which is why the profile runs + *without* `keep-incomplete`. `sp report` reads both dirs — the manifest for + success, the log for failure — and a unit with neither ran nothing. +- **Completeness is a count floor, not a taxonomy.** There is no per-unit + wrapper script: the Snakefile's `unit_pre()` builds the unit's furniture and + clears the stage's run dir as bash inlined into the rule's `params`, and + `sp_shell()` composes every rule's shell as *prologue, one `shapepipe_run`, + one `completeness.py check`*. That check counts products per mandatory runner + against `completeness.py`'s floor and exits nonzero below it. Per-CCD + attrition between floor and `expect` is tolerated. No 3-class taxonomy, no + error-signature whitelist. `--keep-going` isolates a failure to its own + DAG cone. +- **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` + in `tiles/<2-char prefix>//` or `exp///`. Configs are + committed under `workflow/config/cfis/` and version with the rules that set + the env vars they interpolate — there is no `config_src` knob, and no per-unit + config symlink; `$SP_CONFIG` points straight at the committed directory. +- **Mask star catalogues are built in the DAG.** `exp_star_cat` runs one Vizier + cone query per exposure into the run-independent cache at `star_cats:`, then + fans it out into a real per-unit `star_cat_exp/` directory of 40 per-CCD + symlinks, which `exp_mask` consumes. The directory must be per-unit and real: + the file handler intersects the image numbers it finds across a config's + `INPUT_DIR`s, so a symlink to the whole cache contributes every other + exposure's numbers and the intersection comes out empty. It is a `localrule`, + so the queries run serially in the head process — CDS is never hammered, and + the scheduler never sees a six-second job. The cache makes reruns and later + campaigns free. +- **The index is parse-time data, never a rule input.** Appending tiles + changes which jobs exist without invalidating completed work. +- **Exposure products are not `temp()`.** Exposures overlap tiles, so + `temp()` would cascade destructive reruns when a tile is appended later. + Reclamation is the in-DAG `clean_exposure` rule instead: one job per + exposure, taking every consuming tile's `tile_vignets` manifest as input + (the campaign-wide consumer set comes from the accumulating index), which + deletes the store *and* the exposure's `manifests/` and `logs/`, and leaves a + `cleaned.json` tombstone. Deleting the manifests is what makes a late append + correct: the + appended tile finds an unbuilt chain and regenerates it. The `clean:` flag + in `config.yaml` gates it; flipping it on later reclaims retroactively, + since the missing tombstones schedule exactly the outstanding clean jobs. + The tombstone is written *before* anything is deleted, so a crash can cost + disk but never the record. +- **A finished tile declares no reclaimed exposures.** Deleting an exposure's + manifests would otherwise rerun every other tile that reads it, and those + reruns spread across the exposure-overlap component. So a tile whose + `final_cat` exists drops the exposure manifests that are gone from its input + list, and holds the rest through `ancient()`. This is why the profile runs + with `rerun-triggers: [mtime, params, code, software-env]`: the `input` + trigger reads that cut as a reason to rerun the very tiles it protects. + Know the consequence — `--forcerun` on a tile whose `final_cat` exists will + not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **PSF products leave scratch before the purge does.** `exp_persist` packs + the files named by `persist_exp:` in `config.yaml` (default: the psfex_interp + `validation_psf-*.fits`, the rho/tau statistics input) from the exposure's + scratch store into ONE uncompressed tar, + `/exp///psf/.tar` (inodes, not bytes, bind + on /project), and writes ONE manifest beside it recording the patterns, the + members and their sizes. The + threat it answers is the /scratch purge, not `clean_exposure` — the store goes + in 60 days whether or not the workflow reclaimed it — so it runs even with + `clean: false`, requested directly by `rule all`. `clean_exposure` takes its + manifest as an input, so reclamation can never overtake the copy. It is a + rule of its own rather than a `cp` on the end of `exp_psf` because the keep + list rides on `params`: adding a pattern reruns seconds of packing, not four + hours of PSF fitting per exposure. A pattern that matches nothing is a + recorded warning (setools rejects sparse CCDs); matching nothing at all is a + failure. A `localrule`, like `exp_star_cat` and for the same arithmetic. +- **Coverage is a workflow product, built from records the DAG already + writes.** `exp_footprint` writes one JSON per exposure to + `/exp///manifests/exp_footprint.json`, giving the + four sky corners of every CCD that got a PSF model. It reads the valid-PSF CCD + set off `exp_persist.json`'s tar members — exact, because `psfex_interp` + returns *without* writing `validation_psf-*.fits` on NOT_ENOUGH_STARS, + BAD_CHI2 or FILE_NOT_FOUND — and the WCS off `headers-.npy`, written by + `exp_split`. That makes `validation_psf-*` in `persist_exp:` a **precondition** + of the whole chain, not a preference. Like `exp_persist` it runs whatever + `clean:` and `coverage:` say, because its input is on /scratch and the purge + takes it; `clean_exposure` takes its manifest as an input. Set + `coverage: {enabled: true}` and one further job, `coverage_map`, stamps every + footprint into `/coverage/coverage.hsp` — a HealSparse map + counting, per sky pixel, the exposures with a valid PSF there. That job is + **campaign-cumulative**: its declared inputs are the in-scope footprints, but + the script reads *every* record on the products root, reclaimed exposures + included, so appending tiles grows the map instead of replacing it. `nside` is + set in `config.yaml` to the production 128/131072 pair, ~0.1"/pixel, chosen to + align pixel-wise with the UNIONS bit masks; nothing defaults to it, and a + coarser map would look plausible and not align. Plotting stays out of the DAG: run `plot_coverage_map -i + /coverage/coverage.hsp ...` by hand, with the sky windows under + `coverage.plot` in `config.yaml`. sp_validation consumes the map in + `notebooks/demo_apply_hsp_masks.py`. +- **A dead tile can be told to stop pinning exposures.** An exposure is + cleanable only once every consuming tile has its vignets, so one + permanently-failed tile holds its ~80 exposures for the life of the + campaign. List it under `clean_ignore_tiles:` in `config.yaml` and it leaves + the consumer sets. Retrying an ignored tile later is legal and expensive: + its exposure chains are gone and rebuild from scratch. +- **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the + absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its + per-runner counts and blocks no tile. The logs go with the manifests — a log + claiming `complete` for a store that is gone would contradict the unbuilt + chain the DAG must now see, and its content duplicates the manifest anyway. + The `exp_psf` benchmark tsv lives beside both dirs, not inside either, so + reclamation does not eat the memory-sizing data. +- **Failure is a report, not a gate.** `run_report.py` disk-scans the trees + against the count table and enumerates shortfalls (whole-unit absence vs + per-CCD attrition). It runs standalone — a DAG report node would itself be + poisoned by the failures it must enumerate — and fires automatically from + the COMPUTE invocation's `onsuccess`/`onerror` hooks, or on demand via + `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 000000000..1757ceaf0 --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,736 @@ +"""ShapePipe real-data orchestration — Snakemake workflow. + +Design / rationale: CosmoStat/shapepipe#848 (the living PRD), D1-D5. + +`sp run` is TWO snakemake invocations over this one Snakefile: + + SP_PHASE=prepare snakemake prepare_all_tiles # Git -> Uz -> Fe, per tile + SP_PHASE=compute snakemake all # everything else + +They are two because the exposure job set is *data-derived*: it comes from the +tiles' find_exposures output, and a Snakemake DAG is fixed at parse time. The +join between them is the tile<->exposure index, built HERE at parse time of +invocation 2 (build_index.build(), imported — there is no `sp index` verb) and +loaded into plain dicts. The index is never a rule input, so appending tiles and +rebuilding it changes which jobs exist without invalidating completed work; it +ACCUMULATES across invocations, so a later clean_exposure (S5) sees every +consuming tile of the whole campaign, not just this tile list. + +Neither invocation reads the checkout: `sp run` snapshots the code into the +campaign's state dir and points snakemake at THAT tree, so `workflow.basedir` +below is the snapshot and mid-campaign edits are inert. The mechanism and its +rationale live in bin/sp. + +The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared +output is its MANIFEST (`/manifests/.json`), written by +`completeness.py check`. Product files are not declared — a missing CCD is often +legitimate, and at DR6 scale per-CCD declaration means millions of paths. + +Failure evidence rides on the rule's `log:` (`/logs/.json`): +the manifest says "this stage succeeded", the log says "here is what happened" +(the contract is argued in completeness.py's docstring). +""" + +import functools +import hashlib +import json +import os +import sqlite3 +import sys +from fnmatch import fnmatch +from pathlib import Path + +from snakemake.exceptions import WorkflowError + +# Resolved relative to THIS file, not the working directory: snakemake runs with +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project +# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +configfile: str(Path(workflow.snakefile).parent / "config.yaml") + +# Every job's shell runs inside this container (apptainer software-deployment in +# the profile); the user never types apptainer. WHICH image is the one resolution +# order `sp container` exposes — this user's writable sandbox if they built one, +# else their cached SIF if they pulled one, else `container:` above. An empty +# cache therefore lands on exactly the shared /project image the workflow has +# always run, and a package installed into a sandbox reaches the jobs too. +sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import container as _container # noqa: E402 + +# Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to +# resolve at all (empty cache AND no `container:` key). The empty string the +# resolver returns for "none" is a valid-looking container directive that passes +# dry-run and only fails once jobs reach a node. +try: + _image, _kind = _container.resolve_image() +except _container.ContainerError as _exc: + raise WorkflowError(str(_exc)) +if _kind == "none": + raise WorkflowError( + f"No container image resolved: no sandbox, no cached SIF, and no " + f"'{_container.CONFIG_KEY}:' key in {_container.CONFIG_FILE}. " + f"Run `sp container pull`, or restore the key.") + +container: _image + +# --- paths ----------------------------------------------------------------- +# TWO ROOTS (D5). RUN_DIR is the scratch root: bulk intermediates, sized so a +# batch finishes inside the purge window. PRODUCTS_DIR is the persistent root +# for the durable, low-volume products — the final catalogues, the index, the +# report. Snakemake's own state is the one durable-looking thing that stays on +# scratch (bin/sp explains why: tens of thousands of small, hot metadata writes +# against a backed-up, file-count-limited filesystem). +RUN_DIR = Path(config["run_dir"]) +# Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no +# second path: one root, exactly the pre-D5 layout. +PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) +# Run-independent root for the mask star catalogues: the HEALPix chunk store the +# star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it +# (config.yaml explains the placement). +STAR_CATS = Path(config["star_cats"]) +INDEX_DB = Path(config["index_db"]) +SCRIPTS = Path(workflow.basedir) / "scripts" +# The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the +# configs and the rules that set the env vars they interpolate are one artefact +# and must version together. Hence no `config_src` knob. +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" + +sys.path.insert(0, str(SCRIPTS)) +import build_index # noqa: E402 +from completeness import STAGE_DIR # noqa: E402 + +# SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two +# invocations of `sp run`, `passthrough` on the escape hatch (`sp --unlock`, `sp +# --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index +# build and the report hooks — so that a passthrough parse never mutates durable +# state or dies on a threshold it was not asked about. +# +# Requiring it to be SET is the point of the check below: a bare `snakemake` +# would otherwise parse as a passthrough, find `rule all` gated on an index it +# never built, and exit 0 on an empty DAG. Jobs re-parse this file under the +# slurm executor and inherit the head process's environment, so a submitted job +# always carries the launching phase. +PHASE = os.environ.get("SP_PHASE", "") +if not PHASE: + raise WorkflowError( + "SP_PHASE is not set — run the workflow through workflow/bin/sp " + "(`sp run`), which exports it. A bare snakemake invocation would " + "silently build an empty DAG.") +if PHASE not in ("prepare", "compute", "passthrough"): + raise WorkflowError( + f"SP_PHASE={PHASE!r} is not one of prepare, compute, passthrough.") + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- ngmix scatter (D4) ---------------------------------------------------- +# Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one +# ngmix job per tile. We take the count and drive our own integer `chunk` +# wildcard rather than snakemake's `{scatteritem}` ("3-of-8") token, because the +# chunk number is not ours alone: it names the run dir +# (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and +# merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the +# chunk number in that run-directory name — which only works for bare integers. +scattergather: + ngmix=int(config.get("ngmix_chunks", 8)) + +NGMIX_CHUNKS = workflow._scatter["ngmix"] + +# --- parse-time index build + load (D1) ------------------------------------ +# The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY +# there — no "some Fe output exists" guard. That guard used to make a +# totally-failed prepare produce an empty index, an empty DAG and a green exit 0; +# with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the +# SP_MISSING_THRESHOLD gate and fails loudly, as it should. +# +# In every other phase (prepare, or unset for a passthrough invocation) the parse +# builds NOTHING and only loads whatever index is already on disk. +# +# is_main_process matters as much as PHASE: the SLURM executor re-invokes +# snakemake inside every job, that re-invocation parses this file again, and it +# inherits SP_PHASE=compute from the submitting environment. Without the guard, +# every one of the run's jobs re-runs the build — hundreds of concurrent sqlite +# writers on Lustre, which is exactly the "database is locked" storm that killed +# the first real run (2026-07-31). Job parses only LOAD the index below. +if PHASE == "compute" and workflow.is_main_process: + build_index.build( + TILES, RUN_DIR, INDEX_DB, + missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) + +# EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes +# verbatim into the fabricated per-unit exp_numbers list so get_images matches +# .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. +# EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure +# is keyed on (D5). +# +# SLURPED ONLY IN THE HEAD PROCESS. The executor re-parses this file inside every +# job (~800 per run), and each of those parses needs at most its own unit's edges +# — reading the whole campaign's tile_exposures table there is pure waste that +# grows with the campaign rather than with the job. Job parses get the same three +# lookups backed by memoised single-row queries instead (the table's primary key +# indexes both), so every input function returns exactly what it would have. +EXP, TILE_EXP, EXP_TILES = {}, {}, {} +if INDEX_DB.exists() and workflow.is_main_process: + # timeout=60: Lustre lock handoffs are slow; the default 5 s trips on + # nothing more sinister than a reader in another job's parse. + _con = sqlite3.connect(INDEX_DB, timeout=60) + EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) + for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(_tile, []).append(_exp) + EXP_TILES.setdefault(_exp, []).append(_tile) + _con.close() + + +if workflow.is_main_process: + + def exp_name(exp): + """The exposure's original name (get_images matches .fits.fz).""" + return EXP[exp] + + def tile_exposures(tile): + """This tile's exposure base-ids, sorted as the index stores them.""" + return TILE_EXP.get(tile, []) + + def exp_consumers(exp): + """Every tile in the campaign that reads this exposure (D5).""" + return EXP_TILES.get(exp, []) + +else: + # One connection for the whole parse, and lru_cache over it: a job parse asks + # about a handful of units, each of them several times (params, input, log). + _JOB_CON = sqlite3.connect(INDEX_DB, timeout=60) if INDEX_DB.exists() else None + + @functools.lru_cache(maxsize=None) + def exp_name(exp): + row = _JOB_CON.execute( + "SELECT name FROM exposures WHERE exp_id = ?", (exp,)).fetchone() + if row is None: + raise KeyError(exp) # same failure as EXP[exp] would give + return row[0] + + @functools.lru_cache(maxsize=None) + def tile_exposures(tile): + if _JOB_CON is None: + return [] + # ORDER BY rowid, not by exp_id: the eager slurp above is a table scan, + # so its lists are in INSERTION order, and the two paths must hand the + # DAG the same list and not merely the same set. + return [r[0] for r in _JOB_CON.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id = ? ORDER BY rowid", + (tile,))] + + @functools.lru_cache(maxsize=None) + def exp_consumers(exp): + if _JOB_CON is None: + return [] + # Unindexed on exp_id (the PK leads with tile_id), so this is a scan — + # harmless because the only caller, clean_consumers, is reachable from + # the head process alone (clean_exposure is a localrule). + return [r[0] for r in _JOB_CON.execute( + "SELECT tile_id FROM tile_exposures WHERE exp_id = ? ORDER BY rowid", + (exp,))] + +# Tiles this run can actually compute: declared AND indexed. The index spans the +# campaign, so it is intersected with the declared list, not used as it. +TILES_READY = [t for t in TILES if tile_exposures(t)] +READY_SET = set(TILES_READY) + +# A compute invocation with nothing to compute is never a success. Without this, +# an empty intersection yields `rule all` with no inputs, an empty DAG and exit +# 0 — the silent green run the threshold gate exists to prevent. +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} tiles " + f"are ready to compute (index {INDEX_DB}). Run the prepare phase first " + f"(`sp run`), or check {INDEX_DB.parent / 'missing.json'}.") + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", + chunk = r"\d+", + +# --- the sharded stores (D2) ---------------------------------------------- +# tiles/<2-char prefix>// and exp/<2-char prefix>// — no directory +# exceeds ~1k entries at full-UNIONS scale. Rules carry the shard as its own +# wildcard because an output pattern cannot compute it; every path the DAG uses +# is built by these helpers, so a mismatched (shard, id) pair is never requested. +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") +# The persistent root mirrors the scratch one, shard for shard, so the two trees +# read as the same campaign seen from two filesystems. +PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") +PROD_EXP_DIR = str(PRODUCTS_DIR / "exp" / "{shard}" / "{exp}") + +def tile_dir(tile): + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" + +def exp_dir(exp): + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def prod_exp_dir(exp): + """The exposure's dir on the PERSISTENT root — where exp_persist writes. + + Sharded identically to the scratch one, so the two trees read as the same + campaign seen from two filesystems, exposure side as well as tile side.""" + return f"{PRODUCTS_DIR}/exp/{exp[:2]}/{exp}" + +def prod_exp_manifest(exp, stage): + """A manifest that must SURVIVE reclamation, so it is not in the exposure's + scratch manifests/ dir (clean_exposure deletes that wholesale).""" + return f"{prod_exp_dir(exp)}/manifests/{stage}.json" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def final_cat(tile): + """The campaign's science product, and its tile-finished marker. + + It lives on the PERSISTENT root, not the scratch one, and both halves of + that sentence are load-bearing. As a product: it is what the campaign is + for, and a 60-day purge must not eat it. As a marker: tile_finished() keys + on this path to cut a finished tile's exposure edges (D5), so if a purge + could remove it, finished tiles would re-declare inputs against exposure + stores that reclamation deleted — the rerun avalanche the cut exists to + prevent, arriving by way of the purge instead. + """ + return f"{PRODUCTS_DIR}/tiles/{tile[:2]}/{tile}/final_cat-{tile}.fits" + +def unit_num(unit): + """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading + dash (tile ``210.282`` -> ``-210-282``; exposure ``2605805`` -> ``-2605805``). + The RULES do this transform; the configs just interpolate $SP_UNIT_NUM into + NUMBER_LIST (the `set_config_number_list` mechanism that replaced the retired + -e/--exclusive flag, #746).""" + return "-" + unit.replace(".", "-") + +# Content hash of completeness.py, computed once at parse time and carried as a +# param on every rule: the default rerun-triggers' `code` trigger hashes only the +# rule's own shell string, NOT external scripts it calls — without this, a fix to +# the count table silently leaves stale manifests in place (bitten live). Scoped +# to completeness.py alone, the one script every shell line runs; build_forest.py +# gets its own hash, on the forest rule only. +def script_hash(name): + """The 12-hex fingerprint of one script under workflow/scripts/.""" + return hashlib.md5((SCRIPTS / name).read_bytes()).hexdigest()[:12] + +SCRIPT_HASH = script_hash("completeness.py") +FOREST_HASH = script_hash("build_forest.py") +CLEAN_HASH = script_hash("clean_exposure.py") +CLEAN_TILE_HASH = script_hash("clean_tile.py") +PERSIST_HASH = script_hash("persist_exp.py") +FOOTPRINT_HASH = script_hash("exp_footprint.py") +# Same argument for star_cats.py, which both star-cat rules call: their params +# otherwise fingerprint nothing but paths, so an edit to the chunking or the cut +# would never rerun them. ONE hash for both rules because it is one script — and +# that is also why fetch and cut live in one module (they must agree on which +# pixel holds which star). The hash does NOT key the store path — see +# config.yaml's star_cats block on clearing the store after a semantic change. +STAR_CAT_HASH = script_hash("star_cats.py") +# ngmix_range.py earns a hash for a stronger reason than the others. What it +# emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are +# a PARTITION of its object IDs: resume a tile across an edit to the split and +# the chunks that already succeeded keep the old ranges while the reruns take the +# new ones, so within one tile some objects are measured twice and others by +# nobody. merge_sep_cats concatenates whatever it is handed, so the tile +# completes green with a corrupt catalogue and no error anywhere. +# +# ACROSS TIME IS ALL THIS GUARDS NOW. The within-group version of the same +# corruption — eight sibling chunks each computing the split and disagreeing — is +# structurally closed: the partition is materialised ONCE per tile, by +# tile_vignets, and the chunks only look their row up (tile.smk's +# TILE_NGMIX_RANGES, which also keeps the incident transcript). So this hash buys +# REPRODUCIBILITY of the file across attempts and resumes, not agreement between +# siblings. It therefore fingerprints the script that PRODUCES the file, and +# rides on tile_vignets as well as tile_ngmix — both rules, deliberately; the +# one-rule version of this param is the dangerous one, argued at tile.smk. +# +# Land this hash, and any later edit to the split, at a campaign boundary on a +# fresh root: a resume across it re-measures the tile, and the failure modes a +# mid-campaign params change can reach are catalogued at tile.smk's range_hash. +NGMIX_RANGE_HASH = script_hash("ngmix_range.py") + +# --- exposure reclamation (D5, S5) ----------------------------------------- + + +def flag(value, default=False): + """Truthiness for a config value that may arrive as a STRING. + + `--config clean=false` delivers the string "false", and every non-empty + string is truthy in Python — a plain bool() read that as ON and scheduled + the deletions the user had just switched off. YAML booleans pass through + unchanged; only strings are parsed, and an unparseable one is an error, not + a guess. + """ + if value is None: + return default + if isinstance(value, str): + v = value.strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off", ""): + return False + raise WorkflowError(f"Cannot read {value!r} as a boolean (use true/false).") + return bool(value) + + +# `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no +# jobs at all (nothing requests a tombstone); flipping it on later reclaims +# RETROACTIVELY, because the exposures already cleaned are exactly the ones with +# a tombstone, so the missing tombstones schedule exactly the clean jobs. +# Only ever active under SP_PHASE=compute: the prepare and passthrough parses +# read no index of their own and must schedule no deletions. +CLEAN = flag(config.get("clean", False)) and PHASE == "compute" + +# Reclamation REQUIRES the `input` rerun-trigger to be off (see tile.smk's +# tile_finished cascade note). The gate is structural rather than asserted: CLEAN +# needs SP_PHASE=compute, only bin/sp sets it, and bin/sp always launches with +# profiles/nibi — so the incompatible combination is unreachable. A runtime +# assertion is impossible (workflow.dag_settings is None until the DAG is built). + +# Tiles that must not pin an exposure's store. See config.yaml: a permanently +# failed tile otherwise holds every exposure it touches (~80) forever, because +# its vignets manifest will never exist and its exposures are therefore never +# eligible. Listing it here drops it from the consumer sets. +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) + + +def tombstone(exp): + """The clean_exposure output. Lives BESIDE manifests/ and logs/, not inside + either: the clean job deletes both wholesale, and `sp report` scans it.""" + return f"{exp_dir(exp)}/cleaned.json" + + +def clean_consumers(exp): + """Every tile in the campaign that reads this exposure — the set whose + vignets must all exist before the store may go — minus the ignored tiles.""" + return sorted(t for t in exp_consumers(exp) + if t not in CLEAN_IGNORE_TILES) + + +def clean_targets(): + """Which exposures this invocation may clean. + + An exposure is eligible only when every consuming tile is either in this + run's scope or has already produced its vignets on disk. Without that test, + requesting a tombstone for an exposure shared with a LATER batch would drag + that batch's whole tile chain into this DAG through the clean rule's input — + scope expansion by cleanup, which is not a trade anyone asked for. Ineligible + exposures are simply skipped; the invocation that finishes their last + consumer picks them up. Deferral, never loss. + + Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in + `clean_ignore_tiles` neither gates eligibility nor appears in the job's + input — which is the whole point of that list. + + HEAD PROCESS ONLY. This feeds `rule all` at module level, so it runs on every + parse — including the ~800 per-job re-parses under the slurm executor, none of + which can ever schedule `all`. Each of those was stat()ing the whole + campaign's tile_vignets manifests for nothing. The empty list a job parse + returns costs it exactly the target it could not have used anyway. + """ + if not CLEAN or not workflow.is_main_process: + return [] + out = [] + for exp, raw in EXP_TILES.items(): + if not raw: + continue + # An empty set AFTER filtering means every consumer is ignored: nothing + # is left that could ever read this exposure, so it is eligible now. + # That is the whole point of clean_ignore_tiles — all() of an empty set + # is True, and it is true here in the intended sense. + tiles = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() + for t in tiles): + out.append(tombstone(exp)) + return sorted(out) + +# --- persisted exposure products (D5) -------------------------------------- +# The keep list is config, not a rule input, and it is READ HERE so that exactly +# one place converts it into the form the rule carries. An empty list is a +# deliberate "keep nothing" and produces no jobs at all. +PERSIST_EXP = list(config.get("persist_exp") or []) + + +def persist_targets(): + """Which exposures this invocation must pack PSF products off scratch for. + + `rule all` requests these DIRECTLY rather than reaching them only through + clean_exposure. Persistence and reclamation are different concerns — the + /scratch purge takes the store whether or not `clean:` is on — and hanging + the copy off the clean rule alone would mean a campaign run with clean:false + persists nothing and loses everything at the purge. + + Scope is the ready tiles' exposures, which `all` already builds through the + tile chain, so nothing new is pulled into the DAG by asking. + + EXCEPT A CLEANED EXPOSURE. Its exp_psf manifest was deleted by + clean_exposure, so requesting its persist manifest would make the DAG + rebuild the whole exposure chain from VOS — the avalanche tile.smk's + reclaimed-edge cut exists to prevent, arriving through a new target instead. + A tombstone means the copy already happened (clean_exposure cannot run + before exp_persist), so there is nothing to ask for. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. + """ + if not PERSIST_EXP or not workflow.is_main_process: + return [] + exps = {e for t in TILES_READY for e in tile_exposures(t)} + return sorted(prod_exp_manifest(e, "exp_persist") for e in exps + if not Path(tombstone(e)).exists()) + +# --- per-CCD sky footprints (the coverage map's raw material) --------------- +# `exp_footprint` reads the valid-PSF CCD set off exp_persist.json's MEMBERS, so +# `persist_exp:` naming the psfex_interp product is that rule's PRECONDITION, not +# a preference: with those files unpacked the manifest carries no valid-PSF set +# and there is nothing to build a footprint from. Asked by matching a +# hypothetical member name rather than by string-matching the pattern, so any +# spelling that would have packed them counts (exp_footprint.py asks the same +# question of the manifest it actually reads). +# +# Absent the pattern, the footprint rule is simply OFF — no jobs, no error. It is +# `coverage:` being on that turns the same missing pattern into a parse-time +# failure (coverage.smk), because there the user asked for a map. +PSF_PRODUCT_PROBE = "validation_psf-0000000-0.fits" +PERSIST_HAS_PSF = any(fnmatch(PSF_PRODUCT_PROBE, pat) for pat in PERSIST_EXP) + + +def footprint_targets(): + """Which exposures this invocation must record a per-CCD sky footprint for. + + REQUESTED BY `rule all` DIRECTLY, AND NOT GATED ON `coverage:`. The record is + derived from `headers-.npy`, which lives on /scratch and which the + 60-day purge takes whether or not this campaign ever builds a map — so + building it only when a map is asked for means that turning coverage on a + month later finds the WCS gone and no way back short of re-downloading the + exposures. This is persist_targets' argument above, applied to the other + scratch-only input coverage needs: milliseconds and a few KB now, against an + irrecoverable loss later. + + Scope, and the tombstone exclusion, are persist_targets' exactly: a cleaned + exposure's footprint was written before its store went (clean_exposure takes + this manifest as an input), and asking again would rebuild the chain from VOS. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. + """ + if not PERSIST_HAS_PSF or not workflow.is_main_process: + return [] + exps = {e for t in TILES_READY for e in tile_exposures(t)} + return sorted(prod_exp_manifest(e, "exp_footprint") for e in exps + if not Path(tombstone(e)).exists()) + +# --- tile reclamation (D5) -------------------------------------------------- +# A SEPARATE FLAG from `clean:`, deliberately (config.yaml carries the full +# argument): exposure reclamation costs nothing but a rebuild if a tile is +# appended later, while tile reclamation destroys the per-tile AUDIT TRAIL that +# the campaign's cost model was derived from. They are not the same decision and +# must not be made by the same switch. Gated on SP_PHASE=compute exactly as +# CLEAN is, and read through flag() for the same reason (`--config +# clean_tiles=false` arrives as the string "false"). +CLEAN_TILES = flag(config.get("clean_tiles", False)) and PHASE == "compute" + + +def tile_tombstone(tile): + """The clean_tile output. Beside manifests/ and logs/, not inside either — + same placement and same reason as the exposure tombstone() above.""" + return f"{tile_dir(tile)}/cleaned.json" + + +def clean_tile_targets(): + """Which tiles this invocation may clean: the in-scope ones, and no test. + + No eligibility test, because a tile's store has no consumer outside that + tile (clean_tile.py argues the asymmetry with clean_targets()). Every ready + tile is eligible the moment its own final_cat exists, and the rule's input + is exactly that. + + The restriction to TILES_READY is therefore scope containment, not + eligibility, and it runs the opposite way from the exposure case: an + OUT-OF-SCOPE tile's tombstone would drag that tile's final_cat, its + tile_make_cat job and the whole fused tile_shape group into this DAG, where + the `params` rerun-trigger can find a finished tile out of date and rerun it + against reclaimed exposure stores (see the TILE_LOCAL warning in tile.smk). + In scope the edge is free: `rule all` already requests the same final_cat. + Nothing is lost by skipping an out-of-scope tile — the tile list only ever + grows, so a later invocation has it in scope. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. + """ + if not CLEAN_TILES or not workflow.is_main_process: + return [] + return sorted(tile_tombstone(t) for t in TILES_READY) + +# --- the shell every rule runs (D2) ---------------------------------------- + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1), + ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) + + +def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, + pre_run=()): + """The unit-furniture + environment prologue, as bash. + + Returned as a rule ``params`` value, NEVER inlined into the ``shell:`` + string: snakemake formats a shell string ONCE, so a ``{output}``/``{threads}`` + placeholder inside a params value would survive literally — and, conversely, + the literal ``${SP_NGMIX_CHUNK}`` braces this prologue needs would blow up + that formatting if they lived in the shell string. Params values are + substituted after formatting, so both hazards go away together. + + What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT + -e/--exclusive): + * ``output/``, ``manifests/`` and ``logs/`` — the last two are a SIBLING + pair, not one dir with two naming conventions, because clean_exposure + deletes both wholesale (clean_exposure.py says why). The benchmark tsv + is deliberately outside both: it measures the job, not the store, so + reclamation must not eat it. + * tile: ``tile_numbers.txt`` (dot format — what get_images reads); + * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the + ORIGINAL exposure name from the index (``2605805p``), so get_images + matches ``.fits.fz`` in the store — the bare base id matches + nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale + pre-fix file with the bare id. + There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at + the committed config dir, and ``star_cat_exp`` is a real per-unit directory + built by the ``exp_star_cat`` rule, not a symlink into a shared pool. + + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's + FileHandler raises on an existing run dir, and it is how a rerun never sees + stale products (D2: the job clears its run dir at start). + """ + # The stage table already knows both halves of "where this stage writes": + # its level and its run dir. Taking them from there rather than from an + # argument means a caller cannot disagree with completeness.py about which + # store a stage belongs to. + level, subdir = STAGE_DIR[stage] + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_UNIT_NUM='{unit_num(unit)}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + # Also set via apptainer-args in the profile; kept here so a hand-run of + # this same line outside snakemake behaves identically. + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + for k, v in (env or {}).items(): + lines.append(f"export {k}='{v}'") + + if level == "tile": + lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + else: + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + + lines += list(pre_run) + lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] + return "\n".join(lines) + + +def sp_shell(stage, config_name, *, check_args="", post=""): + """The rule's shell string: prologue, one shapepipe_run, one completeness check. + + EVERY rule that runs shapepipe_run goes through here, including the two that + need a little more than the plain body — they pass it rather than restating + it, so the rc composition below has exactly one definition: + * ``check_args`` — extra flags for the completeness check. tile_vignets + points it at the NODE-LOCAL run root (``--run-dir "$SP_LOCAL"``) and keeps + the manifest's unit field the tile ID rather than the local root's + basename (``--unit``), which also keeps the manifest content independent + of an ephemeral path the mtime rerun-trigger reads. + * ``post`` — bash between the check and the ``exit``. tile_make_cat + publishes its final_cat there, guarded on ``rc``. + + ``{threads}``, ``{output}`` and ``{log}`` are placeholders HERE and nowhere + else (see unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task + one number by construction (D4). + + The rc is CAPTURED rather than ``&&``-ed onto, so the check still runs when + shapepipe_run failed, and it is passed back as ``--job-rc "$rc"`` so the + verdict composes the count floors with shapepipe_run's own exit status. Both + halves are argued in completeness.py's docstring. + """ + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" + f"{check_args}" + " --log {log} --job-rc \"$rc\" || rc=1\n" + f"{post}" + "exit $rc\n" + ) + + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" +include: "rules/coverage.smk" + +# --- top-level targets ------------------------------------------------------ +# The aggregation targets, clean_exposure, clean_tile, star_catalogue, +# exp_star_cat and exp_persist run in the head process. The two clean rules are seconds of rmtree +# and hang off `all`; exp_star_cat is seconds of local FITS work; all three would +# otherwise be ~20k (clean_exposure, exp_star_cat) or ~23k (clean_tile) sbatch +# submissions at DR6 scale for work shorter than the scheduling latency. +# star_catalogue is one job either way, and local keeps its CDS concurrency the +# explicit number its thread pool sets (see exposure.smk). +# +# exp_persist joins them for the same arithmetic — a few MB of `tar` per exposure, +# ~20k of them at DR6 scale, each far shorter than the scheduling latency that +# would submit it (exposure.smk argues the placement in full). exp_footprint sits +# beside it and is smaller still: one pickle load and ~160 pixel_to_world calls. +# coverage_map is deliberately NOT one — it is a single submitted job that stamps +# ~1M polygons at nside=131072 (coverage.smk). +# +# star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out +# of any future `group:` label: a local job cannot be fused into a submitted group. +# exp_persist sits between exp_psf and clean_exposure, both of which are outside +# every group already (exp_psf is heavy, clean_exposure is local), so it adds no +# new constraint. The two clean rules are DAG leaves and have none either. +localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat, exp_persist, exp_footprint + +rule all: + input: + [final_cat(t) for t in TILES_READY], + persist_targets(), + footprint_targets(), + coverage_targets(), + clean_targets(), + clean_tile_targets(), + +# Invocation 1 — the static per-tile DAG, known from the tile list alone. +# keep-going makes tile failures independent; the ones that lose their exposure +# list are dropped by the index build at invocation 2's parse. +rule prepare_all_tiles: + input: + [tile_manifest(t, "tile_find_exposures") for t in TILES] + +# --- report hooks ----------------------------------------------------------- +# run_report is NOT a DAG node (a descendant of every job would be poisoned by +# any hard failure — the exact case it exists for). It is a standalone script, +# emitted automatically at the end of the COMPUTE invocation, runnable any time +# via `sp report`. +def _report(status): + shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " + f"--index {INDEX_DB} --status {status} || true") + +if PHASE == "compute": + + onsuccess: + _report("success") + + onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp new file mode 100755 index 000000000..9b5550bea --- /dev/null +++ b/workflow/bin/sp @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). +# +# Three verbs, nothing else: +# +# sp run [ARGS...] bring the products on disk up to date with the tile list. +# Snapshots the code into the state dir first (see "the +# launch code snapshot" below) and runs out of the copy, so +# editing the checkout mid-campaign cannot reach the jobs. +# Two snakemake invocations over one Snakefile: +# 1. PREPARE snakemake prepare_all_tiles +# 2. COMPUTE snakemake all <- its PARSE builds the index +# ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# sp report [ARGS...] emit run_report.json now (mid-run is fine). +# sp container VERB manage the image every job runs inside: pull, status, +# sandbox, exec, resolve. `sp container --help` documents +# the layers and the resolution order. +# +# Anything else is passed straight through to snakemake with the same profile and +# state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). +# +# It also loads the apptainer module (snakemake resolves `apptainer` via PATH at +# job runtime) and activates the snakemake venv on the shared /project FS (the +# executor re-invokes it inside jobs, so it cannot live on a node-local path). +# One entry point, so a fresh tmux or a restart after a crash always launches +# with the right state. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ +REPO="$(dirname "$HERE")" +VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" +SCRIPTS="$HERE/scripts" +CONFIG="$HERE/config.yaml" + +module load apptainer/1.4.5 2>/dev/null || true +# shellcheck disable=SC1091 +source "$VENV/bin/activate" + +# Scalar reader for workflow/config.yaml. The venv is active by this point and +# snakemake depends on PyYAML, so this parses the file rather than pattern-matching +# it -- quotes, inline comments and nesting are the parser's problem, not ours. +cfg() { python -c 'import sys, yaml +print(yaml.safe_load(open(sys.argv[1])).get(sys.argv[2]) or "")' "$CONFIG" "$1"; } +RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" + +# Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO +# THE RUN on /scratch, never on the persistent root — the one exception to D5's +# "durable, low-volume products go on /project". It is neither: one small, hot +# metadata file per output, rewritten on every job, which at DR6 is >170k files +# of churn against a backed-up filesystem with a 1M-inode group quota. It is also +# reconstructible — losing it costs a re-parse, not a re-run. (A hard 27/27 TiB +# group quota killed a metadata write mid-run, live. The quota has since eased; +# the placement is right on its own merits.) +# --directory only moves state: all data paths are absolute, and the Snakefile +# resolves its own configfile. +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" + +# --- the launch code snapshot ---------------------------------------------- +# THE ONE HOME for this concept; everything else points here. +# +# WHY. A campaign is not one process. The SLURM executor re-invokes snakemake on +# every job's node, so each job RE-PARSES the Snakefile, and each shell reads +# workflow/scripts/*.py and workflow/config/cfis/* when it STARTS -- hours after +# launch, out of whatever the checkout contains THEN. Editing the checkout +# mid-campaign therefore fed different code to different jobs of the same run +# (the 186.307 partition incident: an edit landed four seconds into a tile split +# it into two disagreeing halves). +# +# WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and +# runs the campaign entirely out of that copy: the Snakefile, the rules, the +# scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points +# into example/, and the copy must be self-contained), src/, and the profile. +# Every workflow-internal path hangs off `workflow.basedir`, which IS the +# snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the +# one that cannot (YAML splices nothing) and is rewritten below. +# +# CONSEQUENCE, and the reason the old "never edit workflow/ while a campaign +# runs" warnings are gone: mid-campaign edits to the checkout are inert. A change +# takes effect on the next `sp run`, and not before. +# +# LIFECYCLE: one snapshot per campaign state dir, REFRESHED WHOLESALE on every +# `sp run` -- a new `sp run` IS the relaunch, and snakemake's params/code rerun +# triggers decide what the new code invalidates. Nothing accumulates, and the +# path is stable, so a later `sp --unlock` reaches the same workflow that took +# the lock. +SNAPSHOT="$STATE_DIR/code" +snapshot_code() { + mkdir -p "$SNAPSHOT" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ + "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + else + rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" + cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + + fi + + # The profile's `apptainer-args:` hardcodes one checkout's src/ in its PYTHONPATH + # pin and CANNOT interpolate anything (snakemake escapes a literal `$` before the + # string reaches any shell -- the post-mortem is on that line). So the pin is + # rewritten HERE, in the snapshot's copy, and `sm` points --profile at that copy. + # The checked-in profile keeps its literal checkout path and stays the source of + # truth for the FLAGS; exactly one path is substituted, by one line of sed-work. + python - "$SNAPSHOT/profiles/nibi/config.yaml" "$SNAPSHOT/src" <<'PY' +import pathlib, re, sys +f, src = pathlib.Path(sys.argv[1]), sys.argv[2] +text, n = re.subn(r'(--env PYTHONPATH=)\S+', lambda m: m.group(1) + src, + f.read_text(), count=1) +if not n: + sys.exit("sp: no --env PYTHONPATH= found in the profile's apptainer-args; " + "the snapshot would run against the live checkout's src/") +f.write_text(text) +PY + + python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' +import json, pathlib, subprocess, sys, time +out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] +def git(*a): + try: + p = subprocess.run(["git", "-C", repo, *a], capture_output=True, + text=True, timeout=30) + except OSError: + return None + return p.stdout.strip() if p.returncode == 0 else None +status = git("status", "--porcelain") +out.write_text(json.dumps({ + "_comment": "Code snapshot taken by `sp run`; the campaign runs out of THIS " + "tree, never the checkout. Refreshed wholesale on every `sp run`.", + "source": repo, + "taken_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "head": git("rev-parse", "HEAD"), + "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(status), + "dirty_files": status.splitlines() if status else [], +}, indent=2) + "\n") +PY + echo "sp: code snapshot refreshed at $SNAPSHOT" >&2 +} + +# SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of +# declared tiles allowed to be missing their exposure list (default 0.0). +export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" + +# WHICH code a verb runs. `sp run` refreshes the snapshot and runs out of it; +# every other verb and the escape hatch run out of the EXISTING snapshot when the +# state dir has one -- `sp --unlock` and `sp report` must speak for the campaign +# that is actually on disk, not for a checkout that has moved since. Before the +# first `sp run` there is no snapshot and the live checkout is all there is. +# (`sp container` is the exception: it is interactive tooling about the image, +# not about a campaign, and stays on the checkout.) +code_root() { [ -f "$SNAPSHOT/workflow/Snakefile" ] && echo "$SNAPSHOT" || echo "$REPO"; } + +# --snakefile pins the workflow to one tree: sp must work from any cwd (an sbatch +# head job starts in the submission directory, not the repo), and the SLURM +# executor re-invokes snakemake with this same path inside every job -- which is +# exactly what makes the snapshot stick. +# +# SP_PHASE is REQUIRED by the Snakefile (a bare `snakemake` would build an empty +# DAG and exit 0). `sp run` sets prepare/compute on its two invocations; every +# other verb and the escape hatch fall through to `passthrough`, which parses the +# index without building it and schedules no side effects. +sm() { + local root; root="$(code_root)" + SP_PHASE="${SP_PHASE:-passthrough}" \ + snakemake --snakefile "$root/workflow/Snakefile" \ + --profile "$root/profiles/nibi" \ + --directory "$STATE_DIR" "$@" +} + +cmd="${1:-}" +case "$cmd" in + run) + shift + snapshot_code + # PREPARE failing is NOT fatal to the run: keep-going means a failed tile + # poisons only its own cone, and the tiles that lost their exposure list are + # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which + # the compute parse's index build enforces over the WHOLE tile list — so we + # record the failure and go on rather than letting `set -e` abort here. + prep_rc=0 + SP_PHASE=prepare sm prepare_all_tiles "$@" || prep_rc=$? + if [ "$prep_rc" -ne 0 ]; then + echo "" >&2 + echo "############################################################" >&2 + echo "## WARNING: the PREPARE phase exited $prep_rc." >&2 + echo "## Some tiles may be missing their exposure list and will be" >&2 + echo "## dropped from the compute DAG. Continuing to COMPUTE; the" >&2 + echo "## SP_MISSING_THRESHOLD gate (now $SP_MISSING_THRESHOLD) decides" >&2 + echo "## whether that is tolerable." >&2 + echo "############################################################" >&2 + echo "" >&2 + fi + + comp_rc=0 + SP_PHASE=compute sm all "$@" || comp_rc=$? + if [ "$prep_rc" -ne 0 ] || [ "$comp_rc" -ne 0 ]; then + exit "$([ "$comp_rc" -ne 0 ] && echo "$comp_rc" || echo "$prep_rc")" + fi + ;; + report) + # Out of the snapshot when there is one, so a manual report reads the trees + # with the same script the campaign's own onsuccess/onerror hooks use (they + # call it through the Snakefile's SCRIPTS, which is the snapshot). Read-only + # either way -- this is consistency, not safety. + shift + python "$(code_root)/workflow/scripts/run_report.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + --status manual "$@" + ;; + container) + # The image layer, outside snakemake entirely: the script is stdlib-only so + # it runs on the bare host, and the apptainer module is already loaded above. + # Deliberately the CHECKOUT's copy, not the snapshot's: `sp container` is + # about the image you are working with now (status compares against HEAD, and + # the snapshot has no .git), and `exec` should hand you the environment your + # NEXT run will have. It reads the checked-in profile's apptainer-args, so a + # running campaign whose snapshot predates an edit to that line gets the old + # value while `exec` shows the new one -- the only place the two can differ, + # and it differs in the direction that matters (jobs stay pinned). + shift + python "$SCRIPTS/container.py" "$@" + ;; + cancel) + # Kept only because it is two lines: scancel this workflow's jobs by name + # before an --unlock. Not part of the design surface. + run="${2:?usage: sp cancel }" + squeue --me --noheader --format='%i %j' \ + | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" + ;; + *) + sm "$@" + ;; +esac diff --git a/workflow/config.yaml b/workflow/config.yaml new file mode 100644 index 000000000..32e3ffedb --- /dev/null +++ b/workflow/config.yaml @@ -0,0 +1,275 @@ +# Run configuration for the ShapePipe Snakemake workflow. +# +# A "run" is declared by a tile list plus the paths below. Everything here is +# read at parse time; none of it is a rule input, so editing it (e.g. appending +# tiles) never invalidates completed work — it only changes which jobs exist. + +# The tile list that scopes this run (one "IDra.IDdec" per line). +# The campaign grows by appending to this file — parse-time config, so +# completed work is never invalidated. Current contents: smk-g5, ONE tile +# (186.307), the paired control for the two fixes landed after smk-g4 — the +# node-local WCS sqlite and tile_ngmix's mem_mb 14000 -> 5000. That tile ran +# ALONE as smk-g4 job 20799387 (elapsed 8063 s, chunk median 6813 s, chunk mean +# load 59%, MaxRSS 26.6 GiB), so the baseline carries no concurrency confound +# and the contrast is same tile, same solitude, two commits apart. +# The 34-tile smk-g4 set is preserved at smk-g4/tiles34.txt. +# (The 210/211 quad used earlier is unusable: its tile images are symlinks into +# anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are +# broken.) +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt + +# The container every job runs inside (apptainer software-deployment in the profile). +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif + +# THE TWO ROOTS (D5). +# +# run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk +# intermediates, sized so a batch finishes inside the 60-day purge window. The +# sharded per-unit stores live under it: +# /tiles/<2-char prefix>// and /exp/// +run_dir: /scratch/cdaley/shapepipe-output/smk-g6 + +# products_dir is the PERSISTENT root: the durable, low-volume products — the +# final catalogues (/tiles///final_cat-.fits, +# mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB +# per tile, so a full DR6 campaign is a few hundred GB. +# +# The final catalogue is also the tile-finished MARKER that cuts a finished +# tile's exposure edges (D5), which is the second reason it cannot sit on +# scratch: a purge would not merely lose a product, it would make every finished +# tile re-declare inputs against exposure stores reclamation already deleted. +# +# Unset means "one root": products land under run_dir, exactly the pre-D5 +# layout, which is what a fixture or smoke test wants. +# +# Snakemake's own state is the one durable-looking thing that stays on scratch +# (-state; bin/sp explains why). +products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 + +# There is no config_src knob: the config chain is workflow/config/cfis, resolved +# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / +# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules +# are one artefact and must version together, so the dir is fixed by construction. + +# Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). + +# The mask star-catalogue root — run-independent, shared by every campaign, and +# holding two things: +# /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 +# query per HEALPix chunk (~3.4 deg^2, ~25k rows), written by +# `star_catalogue` over the tile list's footprint and never fetched twice; +# /exp/star_cat-.fits the per-exposure cuts +# `exp_star_cat` makes from those chunks, with no network at all. +# Network therefore scales with SKY AREA, not exposure count: exposures overlap +# ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. +# +# On the PERSISTENT root: the sky store is a durable science product bought with +# ~1.5k catalogue-server queries at DR6 scale, and re-buying it after a scratch +# purge is the one cost in this workflow that cannot be paid with local compute. +# (It sat on scratch through smk-g3 only because def-mjhudson /project was then +# hard-full at 27/27 TiB.) +# +# THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to +# workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun +# both rules — the script's hash is a param on each — but a chunk already on disk +# is skipped and only re-cut. Clear the store by hand when the change must reach +# the data. Changing NSIDE or the catalogue ID is the exception: those name the +# directory, so a change there fetches into a new one beside the old. +star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache + +# The run index, and — sharing its directory — missing.json and run_report.json. +# On the persistent root with the catalogues (D5): the index is the record of +# which tile reads which exposure, so it is what a post-purge reconstruction +# would otherwise have to rebuild from tile headers. +index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite + +# Per-exposure PSF products to carry onto the persistent root before the scratch +# store goes (`exp_persist`, exposure.smk). A list of plain file-name globs, +# matched recursively under the PSF chain's four module output dirs +# (/exp///output/run_sp_exp_SxSePsfPi/*/output/ — +# sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner). +# Matches are packed, flat, into ONE uncompressed tar per exposure: +# /exp///psf/.tar, with a manifest listing the +# members beside it. One tar rather than loose copies because inodes, not bytes, +# bind on /project (~1 M-file group quota; loose copies would be ~200 files per +# exposure, ~2 M at DR6 scale). FITS members read straight from the tar: +# fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())). +# +# WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat. +# run_dir is /scratch and is PURGED on a 60-day window whether or not +# clean_exposure ever ran; products_dir is /project, backed up and not purged. +# The only way a per-exposure product outlives its campaign is to leave the +# filesystem. (Ordering is free: clean_exposure takes the exp_persist manifest +# as an input, so a store is never reclaimed before its keepers are written.) +# +# EDITING THIS LIST IS CHEAP. It rides on exp_persist's `params`, so a change +# reruns the packing (seconds) and NOT exp_psf (four hours per exposure). That +# separation is the whole reason exp_persist is a rule of its own. +# +# The default is the minimum: the psfex_interp VALIDATION catalogue, one per +# CCD, which is the input to the rho/tau statistics. Without it the PSF +# diagnostics cannot be recomputed after a purge without rebuilding the exposure +# chain from VOS. +# +# OPT-IN CANDIDATES, and what each buys. Sizes are per exposure (40 CCDs), +# measured on smk-m2 (127 exposures, 64 tiles); a 64-tile campaign with all of +# the measured ones on came to 7.2 GB: +# validation_psf-*.fits (the default) 2.0 MB +# *.psf the PSFEx model itself. Keeping it means the PSF +# can be re-interpolated at ANY position later +# without rebuilding the exposure chain — the +# single most capability-adding entry here. +# 2.8 MB +# psfex_cat-*.cat PSFEx's own output catalogue (FITS_LDAC): the +# per-star FLAGS_PSF / CHI2_PSF, i.e. WHICH stars +# outlier rejection clipped. Not recoverable from +# anything else (the .psf header keeps only the +# LOADED/ACCEPTED counts). unmeasured +# star_selection-*.fits the PRE-SPLIT selection (setools writes it under +# mask/). The only file that can answer "which +# stars were rejected by the selection cuts, and +# why" — the split samples have already lost the +# rejects. 24.5 MB +# star_split_ratio_80-*.fits setools' 80% TRAINING star sample, the set PSFEx +# actually fitted. Rows duplicate star_selection. +# 19.9 MB +# star_split_ratio_20-*.fits the 20% VALIDATION sample — the positions the +# validation_psf rows correspond to. Rows +# duplicate star_selection. 7.1 MB +# star_stat-*.txt setools' per-CCD STAT block (star counts, +# stars/deg^2, FWHM mode and cuts, under stat/): +# the selection's summary without its catalogue. +# unmeasured +# A production keep list is `validation_psf` + `*.psf` + `psfex_cat` (~5 MB per +# exposure); the star_split files are only worth it if star_selection is off. +# PSFEx residual/check images and its XML diagnostics are NOT candidates as the +# chain stands: the committed default.psfex sets CHECKIMAGE_TYPE NONE and +# WRITE_XML N, so nothing is emitted to match. They are a config change first, +# a pattern second. +# +# NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the tar +# then lands beside the store on the same filesystem and buys nothing, and the +# manifest sits in the exposure's own manifests/ dir, which clean_exposure +# deletes wholesale — so a one-root run re-persists after every reclamation. +# Harmless, and exactly the pre-D5 behaviour a one-root run asks for. +persist_exp: + - validation_psf-*.fits + +# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_exposure` job per exposure. It fires once every campaign tile that reads +# that exposure has its vignets, deletes the exposure's store AND its manifests, +# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them +# back out of the tombstone and reports the exposure as `cleaned`. OFF for smk-g5: +# this is a one-tile control, its ~8 exposure stores cost ~54 GiB against 979 GiB +# free, reclamation is already measured exactly by smk-g4, and keeping the stores +# means a re-measurement re-runs only the shape chain instead of the whole tile. +clean: true + +# Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_tile` job per in-scope tile, ordered after that tile's own final_cat, so +# a tile's scratch store goes as soon as its chain lands rather than at the end +# of the batch. It deletes the tile's whole /tiles/// tree +# except four things another mechanism owns — `cleaned.json`, +# `manifests/tile_vignets.json` (clean_exposure's eligibility currency), +# `manifests/tile_find_exposures.json` (prepare_all_tiles' target) and the Fe +# exposure list under output/ (build_index re-reads it at every compute parse). +# clean_tile.py argues each one. +# +# A SEPARATE SWITCH FROM `clean:` ABOVE, AND NOT AN OVERSIGHT. Exposure +# reclamation is reversible in the only sense that matters: a tile appended later +# rebuilds the exposure chain from VOS, expensively but completely. TILE +# reclamation DESTROYS THE PER-TILE AUDIT TRAIL and nothing rebuilds it. +# TWO tools read it: sp-products/tools/sp_tilecost.py and sp_costmodel.py. Both +# attribute the fused tile_shape group job's cost per tile from that tile's +# scratch store — the SExtractor catalogue (NAXIS2 of the sexcat is the object +# count, the cost model's independent variable; sp_costmodel also reads its +# EPOCH_k extensions for the geometric epoch count) and the eight tile_ngmix +# benchmark TSVs — and that model is how every mem_mb and runtime number in +# tile.smk was derived. The sexcat is ~380 MB per tile and cannot be kept; the +# benchmark ROWS are absorbed into the tombstone, so the measurements survive +# but not at the paths the tools read. +# +# (The third artifact they read, run_sp_tile_ngmix_Ngu/, is NOT lost to this +# flag: tile_ngmix declares its chunk dir temp(), so snakemake reclaims it as +# soon as tile_merge_cats runs. It is already absent from every finished tile, +# clean_tiles or not.) +# +# The honest one-line summary: REQUIRED FOR ANY BATCH OVER ~850 TILES, AND IT +# TURNS OFF PER-TILE COST ATTRIBUTION. The arithmetic: a finished tile leaves +# 1.19 GiB across 137 inodes (measured on 186.307), so a 1 TiB scratch quota is +# full at 859 tiles, and DR6's 23,114 tiles would want 26.9 TiB and 3.17M inodes +# against a 1M quota. Reclaimed, a tile costs 10 inodes and 12.9 KB — 231k inodes +# and ~300 MB for the whole of DR6. +# +# OFF here for the same reason `clean:` is: smk-g5 is a one-tile control, and its +# store is what a re-measurement would read. +clean_tiles: true + +# Tiles that may NOT pin an exposure store (default: empty). +# +# An exposure is eligible for cleaning only once EVERY consuming tile has its +# vignets. One permanently-failed tile therefore holds all its exposures +# (~8 measured on P3: 7.9 exposures/tile) for the life of the campaign. A tile listed here is dropped from the +# consumer sets, and its exposures become eligible. +# +# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its +# exposures' stores. If you later retry that tile, those exposure chains are +# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# exposure. That is correct, and expensive. Ignore a tile when you have decided +# it is dead, not while you are still debugging it. +clean_ignore_tiles: [] + +# ngmix within-tile chunking: static N chunks (closed ID ranges computed +# per-tile, in-job, from the tile's own sexcat). +ngmix_chunks: 8 + +# COVERAGE MASK. `enabled: true` adds ONE campaign-level job, `coverage_map` +# (workflow/rules/coverage.smk), which builds a HealSparse nexp map from every +# exposure footprint on the products root. OFF by default because the map is a +# campaign-END product: a half-finished campaign's map is a picture of how far it +# has got, not of the survey. Turning it on costs one job and no rebuild — the +# per-exposure records it reads are written all along, by `exp_footprint`, +# whether or not this block is on (that rule is gated on the keep list below, +# not on this flag: its input is on /scratch and the purge takes it). +# +# The map counts, per sky pixel, the number of exposures with a VALID PSF MODEL +# — only CCDs whose PSF model was actually written are stamped. In the v1.x +# chain that set was approximated by subtracting a summary-scrape file from all +# 40*N candidates; in the workflow it is exact, read off exp_persist.json's +# files[] (psfex_interp returns WITHOUT writing on NOT_ENOUGH_STARS / BAD_CHI2 / +# FILE_NOT_FOUND). That makes `validation_psf-*.fits` in `persist_exp:` above a +# PRECONDITION: empty that list and no valid-PSF record exists to build from. +# +# nside_coverage / nside: 128 / 131072 is the PRODUCTION pair, and NOTHING +# defaults to it — build_map takes both as required arguments, and this block is +# now the only place the pair is written down (the v1.x shell script that carried +# it is retired). nside=131072 is ~0.1"/pixel, CHOSEN TO MATCH THE UNIONS +# BIT-MASK RESOLUTION so that coverage and mask align pixel-wise. A coarser map +# looks entirely reasonable and does not align, and the consumer would not +# notice. +# +# Consumer: sp_validation applies these healsparse structural masks in +# notebooks/demo_apply_hsp_masks.py (it reads the coverage .hsp and writes the +# masked comprehensive catalogue). +# +# Plot windows are the two UNIONS sky regions, with the colourbar clipped to the +# 1-5 exposure range; plotting stays OUT of the DAG (a human act on a durable +# product), so these are defaults for `plot_coverage_map`, not rule params. +# +coverage: + enabled: false + nside_coverage: 128 + nside: 131072 + # Read by `plot_coverage_map`, not by any rule (plotting stays out of the DAG). + plot: + colorbar: true + n_exp_min: 1 + n_exp_max: 5 + regions: + SGC: {ra_min: -20, ra_max: 45, dec_min: 18, dec_max: 40} + NGC: {ra_min: 110, ra_max: 270, dec_min: 28, dec_max: 90} + +# The container's installed shapepipe is overridden by the prod worktree via +# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# build_index.py's missing-tile fraction floor is passed by workflow/bin/sp +# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini new file mode 100644 index 000000000..0c17cf19f --- /dev/null +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -0,0 +1,97 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini new file mode 100644 index 000000000..d5b521080 --- /dev/null +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -0,0 +1,86 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Input file patterns: image, weight, external flag, external star catalogue +FILE_PATTERN = image, weight, flag, star_cat + +FILE_EXT = .fits, .fits, .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +# True: the cat comes from $SP_RUN/star_cat_exp, the per-unit farm the +# exp_star_cat rule builds (40 per-CCD links to this exposure's one cat). +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +# Path to check for existing output mask files +CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_exp_Sp.ini b/workflow/config/cfis/config_exp_Sp.ini new file mode 100644 index 000000000..3b573cbc4 --- /dev/null +++ b/workflow/config/cfis/config_exp_Sp.ini @@ -0,0 +1,78 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the dashed +# exposure ID (the Snakefile's unit_num(), exported by unit_pre()). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fitsfz, .fitsfz, .fitsfz + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini new file mode 100644 index 000000000..0af871d8e --- /dev/null +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -0,0 +1,181 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsfPi +#RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# Input from two modules +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini new file mode 100644 index 000000000..9dda5cf0f --- /dev/null +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -0,0 +1,85 @@ +# ShapePipe post-run configuration file: merge separated catalogues + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Ms + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_sep_cats_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +# NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner +# derives the other chunks' paths itself from N_SPLIT_MAX and this pattern +# (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the +# same fixed output dir. +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_SEP_CATS_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Numbering convention, string that exemplifies a numbering pattern. +NUMBERING_SCHEME = -000-000 + +# WARNING (optional, default is 'error'). Use 'always'/'ignore' to +# display/ignore warnings, and not raise error +WARNING = always + +# Maximum number of separated catalogues per input. +# merge_sep_cats_runner.py reads this with getexpanded (runner .py line ~31), so +# $NGMIX_N_CHUNKS DOES expand here, exactly like ID_OBJ_MIN/MAX in the ngmix +# module. The tile_merge_cats rule exports it from the workflow's scattergather +# chunk count, which is the single source of truth; this value must equal that +# count, so do not replace it with a literal. +N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask new file mode 120000 index 000000000..9cd6a04b1 --- /dev/null +++ b/workflow/config/cfis/config_onthefly.mask @@ -0,0 +1 @@ +../../../example/cfis/config_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/config_tile_Fe.ini b/workflow/config/cfis/config_tile_Fe.ini new file mode 100644 index 000000000..9546f062e --- /dev/null +++ b/workflow/config/cfis/config_tile_Fe.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +COLNUM = 3 + +# Prefix to remove from exposure name +EXP_PREFIX = p + diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini new file mode 100644 index 000000000..72a0f0be7 --- /dev/null +++ b/workflow/config/cfis/config_tile_Git.ini @@ -0,0 +1,93 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/tiles, /project/def-mjhudson/unions-wl/tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini new file mode 100644 index 000000000..55daef12e --- /dev/null +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -0,0 +1,84 @@ +# ShapePipe post-run configuration file: create final catalogs, with +# no spread model on input + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +# The psfex_interp store is NODE-LOCAL: it is the vignette-store run this +# tile's fused group job wrote to node-local storage (see config_tile_PiViVi.ini +# and workflow/rules/tile.smk). $NGMIX_VIGNET_DIR names it; the other two +# inputs and this run's own output stay on $SP_RUN. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282) +NUMBER_LIST = $SP_UNIT_NUM + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/workflow/config/cfis/config_tile_Mh_exp.ini b/workflow/config/cfis/config_tile_Mh_exp.ini new file mode 100644 index 000000000..96512df1a --- /dev/null +++ b/workflow/config/cfis/config_tile_Mh_exp.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini new file mode 100644 index 000000000..f0e374b08 --- /dev/null +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -0,0 +1,130 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +# Per-chunk run dir: the workflow exports SP_NGMIX_CHUNK= for each chunk. +# RUN_NAME is env-expanded (run.py getexpanded); braces keep the trailing "u" +# out of the variable name. +RUN_NAME = run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +# INPUT and OUTPUT are independent here, and that is what makes node-local +# staging possible: [FILE] OUTPUT_DIR stays $SP_RUN/output (scratch, durable +# for the life of the job's tile), while the two directories ngmix actually +# READS AT RANDOM -- the psfex_interp galaxy_psf store and the vignetmaker +# run_2 image/background/weight/flag vignet stores, together the ~5.6 GB +# per-tile vignette store that all 8 chunks re-read -- come from +# $NGMIX_VIGNET_DIR. +# +# $NGMIX_VIGNET_DIR is the tile's run_sp_tile_PiViVi directory. tile_ngmix +# (workflow/rules/tile.smk) copies that directory to the job's node-local +# NVMe ($SLURM_TMPDIR) and points this variable at the copy; the outputs +# below are unaffected and still land on scratch. Set it to +# $SP_RUN/output/run_sp_tile_PiViVi to read the store in place. +# +# It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset $NGMIX_VIGNET_DIR is +# a loud error rather than a silent read of the wrong tree. +# +# $SP_WCS_DIR is the fourth input and the same idea applied to the file the +# vignette store left behind. log_exp_headers-.sqlite is 11.3 MB and +# ngmix reads it once per object PER EPOCH for the WCS; on NFS that is a +# network round trip per read, and thread-state sampling of the fused job +# measured every chunk 44% blocked in `rpc_wait_bit_killable` on this one file +# with everything else already node-local. tile_local() copies it beside the +# store and points this variable at the copy. Set it to +# $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output to read in place. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/psfex_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as +# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for +# every tile (missing file -> error, no per-tile fallback); omit the option +# entirely to fall back to the scalar sigma_mad noise estimate. +BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite + +# Number of objects to batch save during processing, optional. Omit or set +# to -1 for no batch saving. +# 250 (folded from runs/p3-batch1/cfis): worker RSS grows ~4.8 MB/object +# until the flush recycles it -> peak ~1.1 GB + 250 x 4.8 MB ~= 2.3 GB/worker +# (A/B test, job 17607877). Not superseded by -b {threads} (that sets fork +# width, this bounds per-worker memory). +SAVE_BATCH = 250 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# SEED_FROM_POSITION: per-object RNG seeded from sky position (ra, dec, ccd) +# instead of one ordered per-tile stream, so results are bit-identical under +# any chunking (D4/#796). Required for the chunked ngmix scatter/gather. +SEED_FROM_POSITION = True + +# ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, +# computed at execution time from the tile's own object count and expanded +# via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not +# plain getint). +ID_OBJ_MIN = $NGMIX_ID_MIN +ID_OBJ_MAX = $NGMIX_ID_MAX diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini new file mode 100644 index 000000000..c0793eed0 --- /dev/null +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -0,0 +1,188 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# PSFEx PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[PSFEX_INTERP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +# Number of stars threshold +STAR_THRESH = 22 + +# chi^2 threshold +CHI2_THRESH = 2 + +# Multi-epoch mode parameters + +# Root directory of per-exposure work directories. psfex_runner/output/ +# dirs are discovered by scanning $SP_EXP for the exposures listed in the +# exp_numbers input file. +ME_DOT_PSF_EXP_DIR = $SP_EXP + +# Input psf file pattern +ME_DOT_PSF_PATTERN = star_split_ratio_80 + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. Output dirs are discovered by scanning $SP_EXP for the +# exposures listed in the exp_numbers input file. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini new file mode 100644 index 000000000..0ce9ea226 --- /dev/null +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -0,0 +1,118 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uz.ini b/workflow/config/cfis/config_tile_Uz.ini new file mode 100644 index 000000000..fc8550af4 --- /dev/null +++ b/workflow/config/cfis/config_tile_Uz.ini @@ -0,0 +1,73 @@ +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +FILE_EXT = .fitsfz + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +HDU_DATA = 1 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask new file mode 120000 index 000000000..4743a87f0 --- /dev/null +++ b/workflow/config/cfis/config_tile_onthefly.mask @@ -0,0 +1 @@ +../../../example/cfis/config_tile_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 120000 index 000000000..e755942e7 --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1 @@ +../../../example/cfis/default.conv \ No newline at end of file diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 120000 index 000000000..bb469cf56 --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1 @@ +../../../example/cfis/default.param \ No newline at end of file diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 120000 index 000000000..501adb090 --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1 @@ +../../../example/cfis/default.psfex \ No newline at end of file diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 120000 index 000000000..4fcfdabc2 --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1 @@ +../../../example/cfis/default_exp.sex \ No newline at end of file diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 120000 index 000000000..e2bf3398c --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1 @@ +../../../example/cfis/default_noimaflags.param \ No newline at end of file diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 120000 index 000000000..5a2469c92 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1 @@ +../../../example/cfis/default_tile.sex \ No newline at end of file diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 120000 index 000000000..93e6e1f20 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1 @@ +../../../example/cfis/final_cat.param \ No newline at end of file diff --git a/workflow/config/cfis/mask_default b/workflow/config/cfis/mask_default new file mode 120000 index 000000000..0970152ab --- /dev/null +++ b/workflow/config/cfis/mask_default @@ -0,0 +1 @@ +../../../example/cfis/mask_default \ No newline at end of file diff --git a/workflow/config/cfis/star_selection.setools b/workflow/config/cfis/star_selection.setools new file mode 120000 index 000000000..a355664cf --- /dev/null +++ b/workflow/config/cfis/star_selection.setools @@ -0,0 +1 @@ +../../../example/cfis/star_selection.setools \ No newline at end of file diff --git a/workflow/rules/coverage.smk b/workflow/rules/coverage.smk new file mode 100644 index 000000000..48e908562 --- /dev/null +++ b/workflow/rules/coverage.smk @@ -0,0 +1,97 @@ +"""Coverage — the campaign's HealSparse nexp mask, from the exposure footprints. + + exp_footprint (per exposure, exposure.smk) -> coverage_map (per campaign) + +One rule, and it is the only campaign-level product besides the report. The +precedent it follows is ``star_catalogue`` (exposure.smk): a rule keyed by the +campaign rather than by a unit, whose inputs are the units' own records. + +CAMPAIGN-CUMULATIVE. The declared inputs are the IN-SCOPE, non-tombstoned +footprint manifests — ordering and rerun semantics for free, without dragging +out-of-scope tiles into the DAG through a new target. The SCRIPT then reads every +footprint record on the persistent root, tombstoned exposures included: their +records outlive their scratch stores and are still valid sky. So appending tiles +grows the map instead of replacing it, which is what a survey coverage mask +should do, and rebuilding is one job rather than a campaign. + +NOT A LOCALRULE, unlike every other rule that writes to the persistent root: at +DR6 scale this stamps ~1M polygons at nside=131072 in a Python loop +(coverage_map_builder.build_map). The resources below are a first sizing from +that count and not a measurement — the polygon loop is unmeasured above a few +thousand CCDs. + +PLOTS STAY OUT OF THE DAG. `plot_coverage_map -i /coverage/ +coverage.hsp ...`, by hand, with the windows in config.yaml's `coverage.plot` +block — the same argument that keeps run_report.py a standalone script: it is a +human act on a durable product. +""" + +# `coverage:` is OFF by default: the map is a campaign-end product, and a +# half-finished campaign's map is a picture of how far it has got rather than of +# the survey. flag() because `--config` delivers booleans as strings. +COVERAGE = config.get("coverage") or {} +COVERAGE_ENABLED = flag(COVERAGE.get("enabled", False)) +NSIDE_COVERAGE = int(COVERAGE.get("nside_coverage", 128)) +NSIDE = int(COVERAGE.get("nside", 131072)) + +COVERAGE_DIR = f"{PRODUCTS_DIR}/coverage" +COVERAGE_HSP = f"{COVERAGE_DIR}/coverage.hsp" +COVERAGE_MANIFEST = f"{COVERAGE_DIR}/manifests/coverage_map.json" +COVERAGE_HASH = script_hash("coverage_map.py") + +# PARSE-TIME GUARDS, and both of them fail before submission rather than after. +# +# The first is the precondition the whole chain rests on: exp_footprint reads its +# valid-PSF CCD set off exp_persist.json's members, so with `validation_psf-*` +# absent from `persist_exp:` no footprint job exists, no record exists, and +# `coverage:` on would otherwise mean a green run producing nothing — or, worse, +# a map built from whatever stale records happened to be on the products root. +if COVERAGE_ENABLED and not PERSIST_HAS_PSF: + raise WorkflowError( + f"coverage: is enabled but persist_exp: {PERSIST_EXP} packs no " + f"{PSF_PRODUCT_PROBE}-like files. The coverage map is built from the " + f"valid-PSF CCD set, which is read off exp_persist's tar members — " + f"without that pattern there is no such set. Add it, or turn coverage " + f"off.") + +# The second: healsparse requires powers of two, and it says so only once a job +# has reached a node. +for _key, _n in (("nside_coverage", NSIDE_COVERAGE), ("nside", NSIDE)): + if COVERAGE_ENABLED and (_n <= 0 or _n & (_n - 1)): + raise WorkflowError(f"coverage.{_key} must be a power of 2, got {_n}") + + +def coverage_targets(): + """The campaign map, when `coverage:` asks for one. + + HEAD PROCESS ONLY, for the reason clean_targets() gives: this feeds `rule + all` at module level, so it is evaluated on every per-job re-parse too, none + of which can schedule `all`. + """ + if not COVERAGE_ENABLED or not workflow.is_main_process: + return [] + return [COVERAGE_HSP, COVERAGE_MANIFEST] + + +rule coverage_map: + input: + footprint_targets() + output: + hsp = COVERAGE_HSP, + manifest = COVERAGE_MANIFEST + # No `log:`: this is one job with one verdict, and its stderr is the job's. + params: + products = PRODUCTS_DIR, + nside_coverage = NSIDE_COVERAGE, + nside = NSIDE, + script_hash = COVERAGE_HASH + threads: 1 + resources: + mem_mb = 32000, + runtime = 240 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/coverage_map.py" + " --products-dir '{params.products}'" + " --out {output.hsp} --manifest {output.manifest}" + " --nside-coverage {params.nside_coverage} --nside {params.nside}" diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk new file mode 100644 index 000000000..511d983d0 --- /dev/null +++ b/workflow/rules/exposure.smk @@ -0,0 +1,492 @@ +"""Exposure chain — per exposure, keyed by exp base id (dedup is structural). + + exp_get_images -> exp_split ---> exp_mask -> exp_psf -> exp_persist -> exp_footprint + -> exp_star_cat -/ + star_catalogue -------------/ + +``star_catalogue`` is campaign-level, not per-exposure: one fetch of the whole +footprint's stars, which every exposure's ``exp_star_cat`` then cuts locally. + +Each in the exposure's own sharded work dir, chained by manifests; every config +reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a +run log. There is no `prepare_exposures` aggregation target: these chains hang +off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). + +The last two rules write to the PERSISTENT root, and both exist because the +exposure store is on /scratch and the purge takes it. ``exp_persist`` packs the +PSF products named by `persist_exp:` into one tar per exposure; it is a separate +rule from exp_psf precisely so that editing that list costs a `tar` and not a +four-hour refit (workflow/scripts/persist_exp.py). ``exp_footprint`` then records +where on the sky the CCDs that got a PSF model are, which is the coverage mask's +raw material (workflow/scripts/exp_footprint.py, rules/coverage.smk). + +NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by +construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, +not over one invocation — reclamation here is clean_exposure's job (S5), driven +by the accumulating index. A temp() here would delete an exposure the moment +this invocation's readers finished and cascade destructive reruns across spatial +neighbours the next time a tile is appended. + +GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — +one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under +the 15-minute floor Alliance policy asks us to bundle away. The composition +rules are in prepare.smk's docstring; this chain is linear too, so the group +asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. + +The two rules NOT in it are structural, not taste: + * exp_psf is heavy (16 GB, 4 h) and never fuses with a short rule; + * exp_get_images cannot join, because ``exp_star_cat`` — a LOCALRULE, and so + ungroupable — sits between it and exp_mask. Pulling get_images in would make + the group both a dependency and a dependent of exp_star_cat, i.e. a cycle. + Starting the group at exp_split leaves star_cat's inputs entirely upstream + of it, so the group has one clean external edge. +Different exposures share no DAG edge, so this is one group job per exposure. + +NUMBER_LIST ($SP_UNIT_NUM, see unit_num in the Snakefile) is set only for +exp_split, whose numbering scheme IS the exposure id; never for get_images / +exp_mask / exp_psf, whose per-CCD or download numbering would turn tolerated +per-CCD attrition into a whole-exposure hard failure. It is a property of the +committed configs (config_exp_Sp.ini alone carries the entry). +""" + +rule exp_get_images: + output: + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" + log: + f"{EXP_DIR}/logs/exp_get_images.json" + params: + pre = lambda wc: unit_pre("exp_get_images", wc.exp, + exp_name=exp_name(wc.exp)), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("exp_get_images", "config_exp_Gie.ini") + +# --- mask star catalogues --------------------------------------------------- +# Two rules, and the split between them is the design: the NETWORK is a function +# of the campaign's sky area, the per-exposure catalogue is a local cut. +# +# `star_catalogue` fetches the footprint's GSC 2.3 stars once, one Vizier query +# per HEALPix chunk, into a run-independent chunk store under config `star_cats`. +# `exp_star_cat` then reads the chunks covering an exposure's focal plane and +# cuts them to it — no network at all. workflow/scripts/star_cats.py holds both +# halves, the geometry they must agree on, and the arithmetic that motivates the +# split; its docstring is the reference for chunking, padding and query counts. + +# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE +# pointing at a path that does not exist inside the image, so requests is pointed +# at the bundle explicitly (proven in the p3-batch1 bash precedent). +STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" + +# The rules run star_cats.py inside the container (healpy, astroquery, astropy) +# but call apptainer THEMSELVES rather than letting the SDM wrap them +# (`container: None` on both): the CA bundle above and the exposure rule's +# host-side farm loop both need the explicit exec. bin/sp has loaded the +# apptainer module. +# +# WHICH image and WHICH arguments are not this file's to decide, and hand-rolling +# them here was a real divergence: these were the only two rules that ignored a +# user's dev sandbox, because they read `config['container']` — the shared /project +# fallback — instead of the image the Snakefile resolved for everything else. +# `_image` is that resolution (sandbox -> cached SIF -> config), and the profile's +# own apptainer-args are the same string the SDM splices onto every other rule, +# PYTHONPATH pin for shapepipe.utilities.{vizier,cfis} included. +# container.profile_apptainer_args() exists precisely so this file can read them +# rather than restate them. Only the CA bundle is added on top, and only when the +# command touches the network. +def in_container(cmd, *, network=False): + args = list(_container.profile_apptainer_args()) + if not args: + # Silently falling back would run these two rules with no --cleanenv and + # no PYTHONPATH pin, i.e. against a different src/ than every other rule. + raise WorkflowError( + f"Could not read apptainer-args from {_container.PROFILE_FILE}; " + f"star_catalogue and exp_star_cat build their apptainer line from it.") + if network: + # The container's certifi bundle, one --env per variable (the profile's + # own PYTHONPATH entry uses the same one-assignment-per-flag form). + args += [a for k in ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", + "CURL_CA_BUNDLE") + for a in ("--env", f"{k}={STAR_CAT_CA}")] + return f"apptainer exec {' '.join(args)} '{_image}' {cmd}" + + +# The campaign's star catalogue: a first-class durable science product, keyed by +# sky rather than by run. Chunk-need is recomputed from the tile list on every +# run and only the missing chunks are fetched, so appending tiles costs exactly +# the chunks they add. +# +# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the +# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the +# per-exposure rule reached by accident through --local-cores, now an explicit +# number that does not scale with the head node's CPU count. +# +# `tile_list_hash` is what makes the incremental behaviour visible to the DAG. +# The tile list is parse-time config, not a rule input (and the profile drops the +# `input` rerun-trigger anyway), so appending tiles would otherwise leave this +# rule up to date against a footprint that has grown. Hashing the list into a +# param reruns it, and the rerun fetches only what is new. +STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" + + +rule star_catalogue: + output: + manifest = STAR_CAT_MANIFEST + # No `log:` — see write_manifest() in star_cats.py. + # `cmd` is a params value, so placeholders in it are not formatted (see + # unit_pre in the Snakefile). Hence the explicit manifest path. + params: + cmd = in_container( + f"python {SCRIPTS}/star_cats.py fetch" + f" --tile-list '{config['tile_list']}' --store '{STAR_CATS}'" + f" --manifest '{STAR_CAT_MANIFEST}'", network=True), + tile_list_hash = hashlib.md5( + Path(config["tile_list"]).read_bytes()).hexdigest()[:12], + script_hash = STAR_CAT_HASH + container: + None + threads: 4 + retries: 2 + resources: + mem_mb = 4000, + runtime = 720 + shell: + "set -euo pipefail\n{params.cmd}" + + +# The per-exposure catalogue and the 40 per-CCD symlinks the mask module's +# numbering scheme needs. Local: one header read for the focal-plane footprint, +# a load of the chunks covering it, a radial cut. +# +# A LOCALRULE, for the reason the Snakefile's localrules line gives. +# +# The per-unit farm is a REAL directory holding exactly this exposure's 40 +# numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR +# and the file handler INTERSECTS the numbers found across INPUT_DIRs, so a +# symlink to a shared whole-store pool contributes every other exposure's numbers +# and the intersection is empty ("numbers ... do not intersect", live). +# +# TWO declared outputs, and the second one is the point. +# +# The manifest keeps the "one rule, one manifest" currency of every other rule: +# written last, unique to this rule, a record of what the farm points at, and +# deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the +# chunk store at no network cost. +# +# But a manifest attests FOREVER, and the two things it attests to both live +# outside the unit's manifests/ dir: the cut catalogue on /scratch (60-day purge) +# and the farm itself. Either can vanish under a manifest that still says +# "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is +# declared too — one link stands for all 40, they are created by the same loop +# in the same instant, and declaring 40 buys nothing. Snakemake's existence test +# is os.path.exists, which FOLLOWS symlinks and is therefore False for a link +# whose target the purge removed. A purged cut or a deleted farm makes the rule +# out of date, it reruns, and it re-cuts or re-links as needed. +def star_cat_cmd(exp): + """The whole rule body, as bash — carried as a params value because it + contains literal ``{}`` (the manifest JSON); see unit_pre in the Snakefile.""" + cut_dir = f"{STAR_CATS}/exp" + cat = f"{cut_dir}/star_cat-{exp}.fits" + work = exp_dir(exp) + farm = f"{work}/star_cat_exp" + images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" + manifest = exp_manifest(exp, "exp_star_cat") + body = json.dumps({ + "stage": "exp_star_cat", "level": "exp", "unit": exp, + "status": "complete", "cat": cat, "link_dir": farm, "n_links": 40, + }, indent=2, sort_keys=True) + return "\n".join([ + "set -euo pipefail", + # LEGACY-SYMLINK HAZARD. Unit dirs built before this rule existed carry + # star_cat_exp as a SYMLINK into the old shared star-cat pool. `mkdir -p` + # is a no-op on an existing symlink-to-directory, so the 40-link loop + # below followed it and wrote this exposure's links INTO THE SHARED POOL + # (520 stray links found live). Replace the link — never `rm -rf` it, + # which would recurse into the pool, and never touch a real directory: + # a real farm is this rule's own output and `ln -sfn` refreshes it. + f"[ -L '{farm}' ] && rm -f '{farm}' || true", + f"mkdir -p '{cut_dir}' '{farm}' '{work}/manifests'", + in_container(f"python {SCRIPTS}/star_cats.py cut" + f" --images '{images}' --store '{STAR_CATS}'" + f" --out '{cat}'"), + f"test -s '{cat}'", + # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the + # one focal-plane catalogue (pattern from the p3-batch1 precedent). + f"for ccd in $(seq 0 39); do ln -sfn '{cat}' " + f"'{farm}/star_cat-{exp}-'\"$ccd\"'.fits'; done", + # Byte-stable, and written only after the links exist: an unconditional + # write would move the mtime, which is a rerun-trigger. + f"tmp='{manifest}.tmp'", + "cat > \"$tmp\" <<'SP_STAR_CAT_JSON'", + body, + "SP_STAR_CAT_JSON", + f"cmp -s \"$tmp\" '{manifest}' && rm -f \"$tmp\" || mv -f \"$tmp\" '{manifest}'", + ]) + + +rule exp_star_cat: + input: + rules.exp_get_images.output.manifest, + # The chunks this cut reads. star_cats.py fails loudly on a chunk that is + # missing anyway, but the edge is what makes the fetch happen first. + rules.star_catalogue.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", + # The sentinel: ccd-0 of the 40-link farm (see above). + link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" + # No `log:`, for the same reason as star_catalogue above. + params: + cmd = lambda wc: star_cat_cmd(wc.exp), + # star_cats.py is external to the shell string, so the `code` + # rerun-trigger does not see it — same reason SCRIPT_HASH exists. + script_hash = STAR_CAT_HASH + container: + None + threads: 1 + retries: 2 + resources: + mem_mb = 4000, + runtime = 10 + shell: + "{params.cmd}" + +# Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the +# tiles' merge_headers reads). +rule exp_split: + group: "exp_short" + input: + rules.exp_get_images.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_split.json" + log: + f"{EXP_DIR}/logs/exp_split.json" + params: + pre = lambda wc: unit_pre("exp_split", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("exp_split", "config_exp_Sp.ini") + +rule exp_mask: + group: "exp_short" + input: + # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs + # and this exposure's own star_cat_exp farm. + rules.exp_split.output.manifest, + rules.exp_star_cat.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_mask.json" + log: + f"{EXP_DIR}/logs/exp_mask.json" + params: + pre = lambda wc: unit_pre("exp_mask", wc.exp), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("exp_mask", "config_exp_Ma.ini") + +# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's +# :warn on psfex_interp_runner. +rule exp_psf: + input: + rules.exp_mask.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_psf.json" + log: + f"{EXP_DIR}/logs/exp_psf.json" + params: + pre = lambda wc: unit_pre("exp_psf", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + retries: 2 + benchmark: + # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ + # wholesale, and this tsv is the measured-memory feed for mem_mb sizing + # (D4). Inside manifests/ it died with the first reclamation and took + # the campaign's only record of exp_psf's real footprint with it. + f"{EXP_DIR}/exp_psf.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 240 + shell: + sp_shell("exp_psf", "config_exp_psfex.ini") + + +# --- persistence (D5) ------------------------------------------------------- +# The counterpart of reclamation, and it must come first in the DAG: this packs +# the exposure's keepable PSF products into one tar on the persistent root, and +# clean_exposure below takes its manifest as an input so the store is never +# reclaimed before the keepers have left /scratch. The purge would take them +# anyway — that, not clean_exposure, is what this rule exists for +# (persist_exp.py's docstring argues both halves, and config.yaml's +# `persist_exp:` block carries the keep list and its candidates). +# +# A LOCALRULE (declared in the Snakefile), by exactly the arithmetic that made +# exp_star_cat one: the body is a `tar` of a few MB from one shared filesystem +# to another, seconds of work, and one sbatch per exposure would be ~20k +# submissions at DR6 scale for jobs shorter than the scheduling latency. The +# grouping constraint that binds mid-chain localrules (this file's docstring) +# does not bite here: exp_persist's only neighbours are exp_psf, which is too +# heavy to ever fuse, and clean_exposure, which is local itself. +# +# ONE DECLARED OUTPUT, AND IT IS A MANIFEST, NOT THE TAR OR A directory(). The +# tar is not declared: a directory output would attest that a directory exists, +# where what we want written down is WHICH files were packed and how big each was — +# the provenance a rho-statistics run months from now needs in order to know +# what it is reading. The manifest is byte-stable, so a no-op rerun does not +# move its mtime and does not make clean_exposure look out of date. +# +# THE KEEP LIST RIDES ON params. That is the entire reason this is not three +# lines of tar appended to exp_psf's shell: `params` is a rerun trigger, so +# adding a pattern reruns the packing and leaves the PSF chain alone. +rule exp_persist: + input: + rules.exp_psf.output.manifest + output: + manifest = f"{PROD_EXP_DIR}/manifests/exp_persist.json" + # No `log:`: the script's only failure modes are "nothing matched" and a + # name collision, both of which it reports on stderr and neither of which + # has a per-CCD verdict worth a completeness record. + params: + patterns = " ".join(f"--pattern '{p}'" for p in PERSIST_EXP), + exp_dir = lambda wc: exp_dir(wc.exp), + dest = lambda wc: f"{prod_exp_dir(wc.exp)}/psf", + script_hash = PERSIST_HASH + threads: 1 + retries: 2 + resources: + mem_mb = 2000, + runtime = 10 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/persist_exp.py" + " --exp-dir '{params.exp_dir}' --exp {wildcards.exp}" + " --dest '{params.dest}' --manifest {output.manifest}" + " {params.patterns}" + + +# --- per-CCD sky footprints ------------------------------------------------- +# One JSON per exposure recording, for each CCD that HAS a PSF model, the four +# sky corners of that CCD. It is the raw material of the campaign's coverage mask +# (rules/coverage.smk), and it replaces the v1.x chain's VOSpace header download +# + summary-scrape subtraction with a local read of two things this workflow +# already wrote. workflow/scripts/exp_footprint.py argues both inputs and the +# CCD-index invariant tests/unit/test_exp_footprint.py pins. +# +# A LOCALRULE (declared in the Snakefile), by exp_persist's arithmetic and then +# some: one pickle load and ~160 pixel_to_world calls, milliseconds, against a +# scheduling latency of seconds and ~20k exposures at DR6 scale. +# +# ONE DECLARED INPUT, AND IT IS THE PERSIST MANIFEST — deliberately NOT +# exp_psf's as well, even though the WCS array this rule reads is written by +# exp_split and lives in the same scratch store. exp_persist already orders this +# rule after the whole PSF chain, so the second edge would buy no ordering; what +# it WOULD buy is a scratch manifest (the one clean_exposure deletes) in the +# input list of a rule whose output is durable. A persistent-root manifest +# outliving a purged scratch store is a real state, and in it that edge would +# schedule a four-hour VOS rebuild of the exposure to satisfy a few KB of +# provenance. Declaring only the durable input makes the same state a loud +# one-line failure from the script instead. +# +# WRITES TO THE PERSISTENT ROOT, beside exp_persist's manifest and for the same +# reason: the record must outlive both reclamation and the purge — a coverage +# map is campaign-cumulative, so a record written today is read by every map +# built after it. +rule exp_footprint: + input: + lambda wc: prod_exp_manifest(wc.exp, "exp_persist") + output: + manifest = f"{PROD_EXP_DIR}/manifests/exp_footprint.json" + # No `log:`, for exp_persist's reason: the failure modes are "no WCS array" + # and "the two stages disagree about the focal plane", both of which the + # script reports on stderr and neither of which has a per-CCD verdict. + params: + exp_dir = lambda wc: exp_dir(wc.exp), + persist = lambda wc: prod_exp_manifest(wc.exp, "exp_persist"), + script_hash = FOOTPRINT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = 2000, + runtime = 10 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/exp_footprint.py" + " --exp-dir '{params.exp_dir}' --exp {wildcards.exp}" + " --persist-manifest '{params.persist}'" + " --manifest {output.manifest}" + + +# --- reclamation (D5) ------------------------------------------------------- +# The one exception to "no reclamation in this file": clean_exposure OWNS +# exposure-level deletion, and it is a real job, not temp() bookkeeping, because +# an exposure's consumer set closes over the CAMPAIGN. The index supplies that +# set (EXP_TILES, accumulated across invocations); the input is every consuming +# tile's tile_vignets manifest — vignets is the last stage that reads exposure +# products, everything after it reads tile-level files. +# +# What the job deletes, and why a late append still behaves, is argued in +# clean_exposure.py's docstring; params.consumers is what makes a grown consumer +# set stale (same file). +# +# The tile side reads the exposure manifests through ancient() and cuts the +# reclaimed edges of finished tiles (see tile.smk), which is what keeps this +# deletion from rebuilding every neighbouring tile. This rule's OWN inputs are +# deliberately not ancient: a tile that really did rebuild its vignets must +# reschedule the cleans of the exposures it read. +# +# A localrule (declared in the Snakefile). Local execution serialises the cleans +# under local-cores, which costs nothing at rmtree speed and never blocks the +# compute chains (this rule is in none of them). +rule clean_exposure: + input: + # ONLY the consumers this invocation may actually build. A consumer that + # is out of scope had its vignets manifest checked for existence at parse + # time (clean_targets' eligibility test) — declaring it here as well would + # pull that finished tile's whole chain into the DAG, where a rebuilt + # shared exposure then reruns it. That is how one damaged tile reached its + # spatial neighbours. In-scope consumers keep their edge: they may run in + # this DAG, so the clean must be ordered after them. + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp) if t in READY_SET], + # The keepers must be off /scratch before the store goes. Unlike the + # consumer edges above, this edge does not depend on scope: it is the + # same exposure's own rule, so it drags nothing into the DAG that this + # exposure's chain did not already put there. It is conditional only on + # there being a keep list at all — with `persist_exp:` empty, "keep + # nothing" is a coherent instruction and must not become a dependency on + # a rule that would fail for having nothing to copy. + lambda wc: ([prod_exp_manifest(wc.exp, "exp_persist")] + if PERSIST_EXP else []), + # And the footprint, for the same ordering reason one layer further out: + # it is derived from headers-.npy, which lives in the store this job + # deletes. Reclamation must not overtake the read, and unlike the purge + # this deletion is ours to order. Conditional on the keep list naming the + # PSF products (Snakefile's PERSIST_HAS_PSF) because that is exactly when + # the footprint rule exists at all. + lambda wc: ([prod_exp_manifest(wc.exp, "exp_footprint")] + if PERSIST_HAS_PSF else []) + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)), + script_hash = CLEAN_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk new file mode 100644 index 000000000..07405407f --- /dev/null +++ b/workflow/rules/prepare.smk @@ -0,0 +1,92 @@ +"""Invocation 1 — PREPARE: one static chain per tile. + + tile_get_images -> tile_uncompress -> tile_find_exposures + +Known from the tile list alone, cheap, wide, idempotent. find_exposures parses +the tile FITS HISTORY header into ``exp_numbers--.txt`` — the +data-derived tile->exposure edge that invocation 2's parse aggregates into the +index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node +tier). + +All three rules carry ``group: "tile_prep"``, so one tile's whole chain is ONE +sbatch instead of three (medians 0:41 / ~0:40 / 0:15 — all far under the +15-minute floor Alliance policy asks us to bundle away, and at DR6 scale three +submissions per tile is a scheduler load out of all proportion to the work). +Group membership is per connected DAG component and distinct tiles share no +edge, so this is exactly one group job per tile, never a cross-tile bundle. +Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. + +Group resource composition (snakemake 9.23, ``GroupResources.basic_layered`` in +snakemake/resources.py): jobs are laid out per toposort level; within a level +non-additive resources (mem_mb, cpus) SUM — split into layers when a global +constraint is exceeded, the group's width being the widest layer — while the +additive resource ``runtime`` is maxed within a layer and SUMMED across layers. +This chain is strictly linear, one job per level, so the group asks for +max(mem_mb) = 8000*attempt, max(threads) = 4 and sum(runtime) = 150 min. +Attempt scaling survives grouping: ``GroupJob.attempt``'s setter clears the +cached group resources and re-sets ``attempt`` on every member (jobs.py), and +``GroupJob.restart_times`` is the max over members — so tile_get_images' +``retries: 2`` still governs. A retry re-runs the whole group, which is safe +because every rule ``rm -rf``s its own run dir at start. + +Star catalogues for masking are NOT a prepare-phase concern and not pre-run +input: the compute DAG fetches the campaign footprint's stars once +(``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in +exposure.smk, into a run-independent store. The tile side has no star-cat node +because it has no mask rule yet — see tile.smk. +""" + +# No NUMBER_LIST for get_images — a download stage has nothing on disk to +# validate against (see exposure.smk's docstring for the convention). +rule tile_get_images: + group: "tile_prep" + output: + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" + log: + f"{TILE_DIR}/logs/tile_get_images.json" + params: + pre = lambda wc: unit_pre("tile_get_images", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("tile_get_images", "config_tile_Git.ini") + +rule tile_uncompress: + group: "tile_prep" + input: + rules.tile_get_images.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" + log: + f"{TILE_DIR}/logs/tile_uncompress.json" + params: + pre = lambda wc: unit_pre("tile_uncompress", wc.tile), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 + shell: + sp_shell("tile_uncompress", "config_tile_Uz.ini") + +rule tile_find_exposures: + group: "tile_prep" + input: + rules.tile_uncompress.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" + log: + f"{TILE_DIR}/logs/tile_find_exposures.json" + params: + pre = lambda wc: unit_pre("tile_find_exposures", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 + shell: + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk new file mode 100644 index 000000000..74d985e2a --- /dev/null +++ b/workflow/rules/tile.smk @@ -0,0 +1,906 @@ +"""Tile post-chain — per tile: gather exposures, then detect / PSF / shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> tile_vignets -> tile_ngmix x N + -> tile_merge_cats -> tile_make_cat + +The DAG edge to the exposures is always the exposures' MANIFESTS, looked up +through the index (TILE_EXP). The per-tile exposure "forest" — a symlink view of +exactly this tile's exposures' products — exists only so the ShapePipe configs +have one deterministic ``$SP_EXP`` to glob; it is NEVER the edge. Its 2-char +shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires +``///output/run_sp_*`` into its glob, so a flat forest +makes every tile gather stage fail "No split_exp_runner output found". + +All rules are group-compatible (shell only, no mid-chain localrules), and the two +short regions are grouped, per the composition rules in prepare.smk's docstring. +Distinct tiles share no edge, so each is one group job per tile: + +* ``group: "tile_gather"`` — tile_exp_forest (2 GB, 20 min) and + tile_merge_headers (median 0:38; 8 GB, 4 threads, 120 min). The group asks max + mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also + consumes the forest, but a consumer OUTSIDE the group is just an ordinary DAG + edge on the group job — it does not pull tile_detect (16 GB) in. +* ``group: TILE_GROUP`` ("tile_shape") — tile_vignets -> 8 x tile_ngmix -> + tile_merge_cats -> tile_make_cat, THE WHOLE SHAPE CHAIN AS ONE JOB. This is + not a latency optimisation; it is what lets the 5.6 GB vignette store live on + node-local NVMe and never touch /scratch (see tile_local() below). + Composition, verified against snakemake 9.23.1 and a real sbatch: + cpus = max over levels of summed siblings = max(8, 8x1, 8, 8) = 8; + mem_mb = max(32000, 8x5000, 16000, 16000) = 40000, so nibi's + max(cores, mem_GB/4) bills the group 10 core-equivalents rather than the 28 + it billed at 8x14000 (see the mem_mb note on tile_ngmix for the 31-tile + measurement that sized it); tile_vignets' own 32000 is now the second term + and becomes binding if the chunks ever go below 4000; + runtime = sum along the chain of each level's MAX = 20 + 120 + 10 + 15 = 165. + 165 min is under the 180 min ceiling of ``cpubase_bycore_b1``, so the fused + job reaches the widest partition set (plus cpubackfill) — which is why each + member's runtime is measured p99 plus margin and not the old ceiling. + +The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor +run that the shape chain does not need co-scheduled, and folding it in would add +its runtime to a sum that has no room. + +Note there is no `tile_mask` rule: the committed config chain is the +"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask +run), and no tile-mask config was committed in the S2 sweep. Adding the masked +variant is a config + one rule, at the config selector the PRD describes. + +That rule also needs a tile-side analogue of ``exp_star_cat``: tile star cats key +on TILE id, so they are a separate cache namespace and a separate node, and the +earliest point it can run is after ``tile_uncompress`` (create_star_cat.py's +``-k tile`` mode reads the uncompressed tile image's primary header). The mask +config would then read a real per-unit ``$SP_RUN/star_cat_tiles`` directory, +built the same way and for the same reason (the file handler intersects numbers +across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). +""" + +# --- the node-local tile root (the I/O + scratch fix) ---------------------- +# +# MEASURED PROBLEM. tile_ngmix was I/O-bound, not CPU-bound: median 7h34m +# elapsed against 52 min of CPU (3% efficiency, 0.12 of 4 reserved cores). +# Each chunk read ~20 GB at 0.78 MB/s and all 8 chunks of a tile re-read the +# SAME 5.6 GB vignette store -- ~163 GB of small random sqlite preads per tile +# against /scratch, which on nibi is NFS (VAST, 4.5 PB, 95% full). What hurts +# is per-read LATENCY, not bandwidth: the sequential local-vs-scratch gap is +# only ~3x and is NOT the argument for this change. +# +# THE FIX IS THE FUSE. tile_vignets -> 8 x tile_ngmix -> tile_merge_cats -> +# tile_make_cat run as ONE group job on ONE node, and the vignette store is +# WRITTEN to that node's NVMe and never lands on /scratch at all. So the store +# is not copied, it is simply never remote: zero staging cost, zero NFS random +# reads, and -- the reason this was chosen over per-chunk staging -- the +# per-tile scratch high-water drops by the whole 5.6 GB store (~190 GiB across +# 34 concurrent tiles). Scratch quota, not speed, is what caps batch size. +# +# WHAT STAYS ON SHARED STORAGE, and it is everything that is DAG currency: +# every manifest (the success sentinels), every verdict log, the ngmix chunk +# dirs merge_sep_cats gathers, and final_cat on the PERSISTENT root. Only the +# bulk intra-tile intermediate is node-local. That split is possible because +# ShapePipe's configs set input and output paths independently -- see +# config_tile_PiViVi.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and +# config_tile_Ng_template.ini and config_tile_Mc.ini ($NGMIX_VIGNET_DIR), and +# config_tile_Ng_template.ini alone for $SP_WCS_DIR. +# +# TWO node-local stores, not one: the ~5-8 GB vignette store and the 11.3 MB +# WCS sqlite, the same fix applied at the two ends of the size distribution +# (tile_local()'s docstring carries the thread-state measurement). +# +# WHY IT NEEDS THE GROUP. The store is node-local, so it exists only on the +# machine that wrote it and only while that job lives. Unfused, tile_vignets and +# tile_ngmix are different jobs on (usually) different nodes, and the store would +# have to be on shared storage -- which is the whole problem. Fused, all members +# run in one allocation on one node, so the store the first member writes is +# simply there for the rest. (Verified live: group members share the directory.) +# +# HOW THE PATH GETS IN HERE. Not through the environment: profiles/nibi passes +# --bind /local and tile_local() below DERIVES the path from the tile wildcard. +# Why nothing can be communicated instead is on that profile line. +# +# THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, +# not one chunk, because the store dies with the job. At ~1 h per fused tile +# that is a cheap trade for the scratch it buys. Noted, not engineered around. +# +# It lives in each rule's `pre_run`, i.e. in THAT RULE's params.pre. The shared +# prologue (unit_pre) and completeness.py are untouched by design: both are +# fingerprinted into every rule, and changing either would invalidate the whole +# campaign, including the 117 finished exposure chains. +# +# EDITING THIS FUNCTION INVALIDATES FINISHED TILES. READ THIS BEFORE YOU DO. +# The returned string lands in `params.pre`, `params` is an active rerun trigger +# in profiles/nibi/config.yaml, and `params.pre` is a non-derived param (its +# lambda takes only `wc`), so snakemake records it and compares it. Any edit +# here therefore reschedules tile_vignets, all eight tile_ngmix chunks and +# tile_make_cat for EVERY tile the campaign already finished. +# +# On a fresh campaign that is free. On a RESUME it is destructive, and the +# mechanism is worth spelling out because it is not obvious: +# * clean_exposure has already reclaimed the exposure stores those tiles read, +# and tile_finished() deliberately drops the manifests of a finished tile's +# exposures, so the rerun is UNSATISFIABLE -- PiViVi runs against dangling +# symlinks and the group fails; +# * a failed group job's postprocess(error=True) fans out over every member in +# every toposort level and removes each member's EXISTING outputs. One of +# those is tile_make_cat's final_cat on the persistent root. So the failure +# deletes the science product of a tile that was finished and correct; +# * with final_cat gone, tile_finished() flips and the tile re-declares its +# whole exposure edge set, rebuilding the reclaimed chains from VOS. That is +# the rerun avalanche D5 exists to prevent, arriving through the params +# trigger instead of through inputs. +# So: land changes to this function BETWEEN campaigns. An edit cannot reach a +# RUNNING one (the launch code snapshot, bin/sp) -- the hazard is the `sp run` +# after it, which is a resume against finished tiles. If that resume is genuinely +# needed, run it once with +# `--rerun-triggers mtime code software-env`, accepting that clean_exposure's +# consumer-set staleness detection (which rides on params) is off for that +# invocation. +def tile_local(tile): + """The node-local prologue, as bash, for one tile. + + A FUNCTION of the tile rather than a constant, because the path has to be + literal by the time apptainer sees it. Nothing can be COMMUNICATED into the + container -- `$SLURM_TMPDIR`, `--env`, `APPTAINERENV_*` and + `{resources.tmpdir}` all fail, one of them in production; the post-mortem is + on the `--bind /local` line of profiles/nibi/config.yaml. + + So the path is derived instead. `/local/scratch` on nibi is + `drwxrwxrwt root:root` -- world-writable with the sticky bit, verified in + the container (probe job 20798618) -- and the tile id is a wildcard + snakemake substitutes at DAG time, so the shell string carries a concrete + path with no `$` left for anything to escape. One group job per tile means + the name cannot collide; the sticky bit means nobody else can remove it. + + What we give up is Slurm's own cleanup of `$SLURM_TMPDIR`. TILE_VIGNET_FRESH + reclaims a stale directory on the next attempt for the same tile, the trap + in the last group member removes it on the way out, and the sweep below + catches what a hard kill leaves behind -- on a shared node that last one is + manners, not housekeeping. + + NOT `/tmp`: inside the container that is a 378 GB tmpfs, i.e. RAM charged to + the job's cgroup, so a 5.6 GB store there would be paid for twice. + + TWO STORES, AND THE SECOND IS 11 MB. `$SP_WCS_DIR` holds one file, + `log_exp_headers-.sqlite`, and staging it is the rest of the same fix + rather than a refinement of it. Moving the 5.6 GB vignette store off NFS + bought 4.3x and did not solve the problem: thread-state sampling of the + fused job (3,200 samples across 8 chunks, campaign smk-g4 job 20799387) put + every chunk at 56% on CPU and 44% in `rpc_wait_bit_killable`, with 147 of + 176 D-state samples inside NFS RPC and exactly one hot NFS file still open. + ngmix reads that file once per object PER EPOCH for the WCS, so what costs + is the operation count and not the byte count -- a network round trip per + read. The two stores are the same intervention at the two ends of the size + distribution, and the small one is worth ~1.8x on the rule that is 99.3% of + tile-side core-hours. + + A copy, never a symlink: a link resolves straight back to NFS. The prologue + runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in + `tile_merge_cats`, which has no pre_run), so the staging is attempted nine + times per tile, eight of them concurrent siblings in one toposort level. It + is copy-to-a-temp-name plus `mv -f` -- the same all-or-nothing publish + shapepipe.utilities.file_io.write_atomic argues -- and it is UNCONDITIONAL, + no `cp -u` and no already-there test. An interrupted `cp` leaves a truncated + destination whose mtime is NEWER than the source, so `-u` would skip it + forever, and nothing downstream would catch it: TILE_VIGNET_FRESH and + TILE_VIGNET_REQUIRED both look only at the vignette store, and a truncated + WCS store reaches ngmix as wrong astrometry rather than as an error. Nine + unconditional copies of 11 MB per tile is not a cost worth reasoning about; + a staleness rule would be. Renaming over a file a sibling chunk already has + open is safe -- the open descriptor keeps the old inode, identical in + content. + + `tile_merge_headers` is upstream of the whole group, so the file exists by + the time any member runs; failing loudly if it does not is correct, because + the alternative is ngmix silently reading a different tree. + """ + # `log_exp_headers--.sqlite`, named from the tile in ShapePipe's + # dashed image-number form -- spelled out rather than globbed so a missing + # file is one precise error instead of an empty expansion. unit_num() owns + # that convention and supplies the SEPARATING DASH itself, so there is none + # in the literals below. + tile_num = unit_num(tile) + return f''' +if [ ! -d /local/scratch ]; then + echo "tile_shape: node-local storage unavailable." >&2 + echo " /local/scratch is not a directory inside the container." >&2 + echo " profiles/nibi/config.yaml apptainer-args must carry --bind /local" >&2 + exit 1 +fi +export SP_LOCAL="/local/scratch/sp-{tile}" +export SP_VIGNET_OUT="$SP_LOCAL/output" +export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" +export SP_WCS_DIR="$SP_LOCAL/wcs" +mkdir -p "$SP_VIGNET_OUT" "$SP_WCS_DIR" || {{ + echo "tile_shape: cannot create $SP_LOCAL on this node." >&2 + exit 1 +}} +# the WCS store -- see the docstring. Copy to a temp name and rename, because +# `cp` is not atomic and the eight chunks share this destination. +sp_wcs_src="$SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers{tile_num}.sqlite" +cp "$sp_wcs_src" "$SP_WCS_DIR/.log_exp_headers.$$" && \ +mv -f "$SP_WCS_DIR/.log_exp_headers.$$" \ + "$SP_WCS_DIR/log_exp_headers{tile_num}.sqlite" || {{ + rm -f "$SP_WCS_DIR/.log_exp_headers.$$" + echo "tile_shape: could not stage the WCS store." >&2 + echo " source: $sp_wcs_src" >&2 + echo " dest: $SP_WCS_DIR" >&2 + echo " check that tile_merge_headers ran, and that /local/scratch has room." >&2 + exit 1 +}} +find /local/scratch -maxdepth 1 -name 'sp-*.*' -user "$(id -u)" -mmin +1440 \ + -exec rm -rf {{}} + 2>/dev/null || true +''' + + +# Every member of a group job must agree on a STRING resource or snakemake +# raises "Resource slurm_extra is a string but not all jobs in group require +# the same value" (resources.py::_is_string_resource). So the tmp-disk request +# is one constant on all four members, not a property of tile_ngmix alone. +# +# --tmp is a node SELECTION FLOOR, not a reservation: Slurm matches it against +# the node's configured TmpDisk (nibi: 1.67-12 TB, TmpFS=/local) and does not +# decrement it per job. It buys exactly one thing, and it is the thing worth +# having -- the fused job can no longer land on a node with no usable local +# disk, which is the one configuration in which the whole design fails. 16 GB +# against a measured 5.6 GB store leaves room for an unusually rich tile, and +# filters no nibi node today, so it costs no queue time. Resources are not a +# rerun trigger, so adding it invalidates nothing. +TILE_SLURM_EXTRA = "--tmp=16000" + +# One label for the whole shape chain; the composition arithmetic is in this +# file's docstring. The group's declared runtime is a SUM along the chain and +# that sum gates partitions on nibi, so each member's runtime below is measured +# p99 plus margin, not the old defensive ceiling. +TILE_GROUP = "tile_shape" + +# Cleanup, on the LAST member only. $SLURM_TMPDIR is Slurm's to reclaim, but +# /local/scratch on nibi demonstrably carries stale directories from +# long-finished jobs, so the epilog is not on its own reliable. tile_make_cat is +# the last reader of the store, so its EXIT trap is the earliest moment the +# 5.6 GB can go. It must NOT be set by the earlier members: their shells exit +# while the store is still needed. Residual failure mode accepted: SIGKILL +# (OOM-kill, node death) skips the trap, and if the epilog also misses it the +# store leaks until the node drains -- 5.6 GB against a 3.3 TB disk. +TILE_CLEAN = r""" +trap 'rm -rf "$SP_LOCAL"' EXIT +""" + +# tile_vignets ONLY. unit_pre clears each stage's own run dir on the shared root +# ("the job clears its run dir at start" — ShapePipe's FileHandler raises on an +# existing one); the node-local root needs the same treatment, and only the rule +# that WRITES the store may do it. Putting this in TILE_LOCAL would have +# tile_ngmix delete the store it is about to read. +TILE_VIGNET_FRESH = r""" +rm -rf "$SP_VIGNET_OUT/run_sp_tile_PiViVi" +""" + +# THE SINGLE WRITER OF THE ngmix CHUNK PARTITION, and tile_vignets ONLY. +# +# The ranges are materialised once per tile, here, and every chunk then looks +# its own row up (TILE_NGMIX_RANGE_READ below). tile_vignets is the first member +# of the fused group and strictly precedes all chunks in its DAG, so this is the +# one place in the group where a single process can write something all chunks +# read — no flock, no first-one-wins race. The sexcat it reads is tile_detect's, +# upstream of the whole group and on the SHARED root ($SP_RUN); only the OUTPUT +# is node-local. +# +# Group-internal plumbing, deliberately not a rule output: the file lives and +# dies with the group job exactly as the vignette store does, so a chunk can +# only ever read what its own group job wrote. ngmix_range.py owns the rest. +NGMIX_RANGES = "$SP_LOCAL/ngmix_ranges.json" +TILE_NGMIX_RANGES = ( + f'python {SCRIPTS}/ngmix_range.py --run-dir "$SP_RUN" ' + f'--n-chunks {NGMIX_CHUNKS} --write "{NGMIX_RANGES}" || exit 1' +) + +# THE FUSE'S ONE SHARP EDGE, made loud rather than mysterious. +# +# The vignette store is not a declared output any more, so snakemake cannot see +# it. tile_vignets' manifest CAN be up to date while the store does not exist on +# this node — concretely, after a fused group job dies part-way: the manifest is +# a success and survives, so a LATER invocation re-plans the group with only the +# surviving members and the store is simply not there. (An in-flight `retries:` +# resubmission is safe: it reruns the same member set, tile_vignets included.) +# +# Recovery is one line, and the message says it: delete the tile's +# tile_vignets.json and resume — that puts tile_vignets back in the group and +# the store comes back with it. The alternative fixes are worse: temp()-ing the +# vignets manifest would break clean_exposure, which keys reclamation +# eligibility on exactly that file. +TILE_VIGNET_REQUIRED = r""" +if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output" ]; then + echo "tile_shape: the node-local vignette store is missing." >&2 + echo " expected: $NGMIX_VIGNET_DIR" >&2 + echo " This means tile_vignets did NOT run in this group job — its manifest" >&2 + echo " was already satisfied, most likely because a previous fused job for" >&2 + echo " this tile died after tile_vignets succeeded. The store lives and dies" >&2 + echo " with the job, so it cannot be inherited." >&2 + echo " FIX: rm \"\$SP_RUN/manifests/tile_vignets.json\" and resume." >&2 + exit 1 +fi +""" + + + + +def tile_exp(wc): + return tile_exposures(wc.tile) + +# --- the tile->exposure edge, and why it is cut for finished tiles ---------- +# +# THE cascade fix. clean_exposure deletes the exposure's manifests on purpose: +# that is what makes a tile appended later rebuild the chain instead of running +# against an empty store. But those manifests are the tile side's inputs, and an +# exposure is read by ~7-10 tiles. So the moment ONE tile's chain rebuilt an +# exposure, every other tile reading it saw "input files updated by another job" +# and reran — and that rerun rebuilt ITS exposures, which reran THEIR other +# consumers, propagating across the whole exposure-overlap connected component. +# On fixture t4, asking for one damaged tile scheduled all four tiles' chains. +# +# Two mechanisms, and only the second one actually cuts it: +# +# 1. ancient() on every exposure-manifest edge. Correct on its own terms — a +# tile has no business rerunning because an exposure manifest is NEWER — and +# it is what keeps a pure-mtime disturbance (a re-touched manifest, a +# restored backup) from waking finished tiles. But ancient() governs +# TIMESTAMPS only. Snakemake propagates "my input is produced by a job that +# will run in this DAG" separately, and ancient does not suppress it +# (measured: t4 counts were identical with ancient alone). +# +# 2. Cutting the RECLAIMED edges of a FINISHED tile — the mechanism that works. +# A tile whose final_cat is on disk needs nothing further from its +# exposures: it has already extracted everything it will ever read. So for +# such a tile the input list drops the manifests that are GONE, and the +# propagation has nowhere to go. An UNfinished tile keeps its full edge set +# and therefore still drags in — and rebuilds — every exposure it needs, +# which is the accepted price of a late append, unchanged. +# +# Only the missing ones are dropped, never a manifest that still exists: a +# campaign that has cleaned nothing then declares exactly the edges it always +# did, and the cut cannot perturb it. +# +# The marker is final_cat, not the tile's own vignets manifest: on a tile whose +# catalogue was lost, the vignets manifest still exists while the vignette store +# (temp()) does not, so keying on vignets would cut the edge on exactly the tile +# that has to rerun, and run it against a deleted exposure store. +# +# THE CUT REQUIRES THE `input` RERUN-TRIGGER TO BE OFF (profiles/nibi sets the +# trigger list). Dropping an input is itself a change in the set of input files, +# which that trigger reads as a reason to rerun — reinstating the very cascade, +# now as "Set of input files has changed", and running finished tiles against a +# store that is gone. Measured on fixture t4, one damaged tile of four: 82 jobs +# with neither fix, 70 with the cut but the trigger on, 28 with both (= exactly +# the damaged tile's own chain, its two exposures, and the clean jobs). +# +# The cost, stated plainly: `--forcerun` on a tile whose final_cat exists will +# NOT rebuild its reclaimed exposures, because those edges are not in the DAG. +# Delete that tile's final_cat first and the whole chain comes back. +# +# What none of this weakens: clean_exposure's own inputs are neither ancient nor +# cut, so a tile that really did rebuild its vignets still reschedules the cleans +# of the exposures it read, and a grown consumer set still travels through +# params.consumers. +def tile_finished(tile): + return Path(final_cat(tile)).exists() + + +def exp_manifests(wc, stage): + paths = [exp_manifest(e, stage) for e in tile_exp(wc)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] + +def tile_exp_split(wc): return exp_manifests(wc, "exp_split") +def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") +def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) + + +# Build the per-tile symlink forest. Declaring the exposure manifests as input +# makes this wait on its exposures; the forest itself is only the $SP_EXP view. +# Its output stays a directory() (it has no ShapePipe run dir and no manifest — +# it is not a shapepipe_run at all). +rule tile_exp_forest: + group: "tile_gather" + input: + tile_exp_all + output: + forest = directory(f"{TILE_DIR}/exp_forest") + params: + cmd = lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB}"), + # build_forest.py's own content hash rides here and nowhere else. + script_hash = FOREST_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 + shell: + # --forest {output} lives in the shell string: snakemake formats shell + # ONCE, so an {output} placeholder inside params.cmd would survive + # literally and every forest job would race one './{output}'. + "{params.cmd} --forest {output.forest}" + +# Merge single-exposure WCS headers into the tile-level sqlite +# (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). +# Reads headers-*.npy through the forest -> the split manifests are the edge. +rule tile_merge_headers: + group: "tile_gather" + input: + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + # config_tile_Mh_exp.ini reads run_sp_tile_Fe output, which the PREPARE + # phase produced. Declaring the Fe manifest gives the COMPUTE DAG a + # regeneration path for it instead of a silent dependency on a + # previous invocation (prepare.smk is included in every parse, so the + # rule exists here too). Normally a satisfied no-op. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" + log: + f"{TILE_DIR}/logs/tile_merge_headers.json" + params: + pre = lambda wc: unit_pre("tile_merge_headers", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") + +# SExtractor object detection on the tile. +rule tile_detect: + input: + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_detect.json" + log: + f"{TILE_DIR}/logs/tile_detect.json" + params: + pre = lambda wc: unit_pre("tile_detect", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 + shell: + sp_shell("tile_detect", "config_tile_Sx.ini") + +# PSFEx interpolation to galaxies + vignet postage stamps: the last stage that +# reads exposure products, and the bulk intra-tile intermediate. The store it +# writes is node-local (see TILE_LOCAL above). +rule tile_vignets: + group: TILE_GROUP + input: + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + psf = tile_exp_psf, + # config_tile_PiViVi.ini reads run_sp_tile_Fe output — same reason as + # tile_merge_headers above. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + # THE MANIFEST IS THE ONLY DECLARED OUTPUT. The vignette store used to + # ride along as a second temp(directory()) so native temp() would + # reclaim it; node-local, it cannot be declared at all — and need not + # be. It was never DAG currency, its readers are all inside this group + # job, and TILE_CLEAN's trap reclaims it. Lost affordance: `--notemp` + # can no longer keep it for debugging. + manifest = f"{TILE_DIR}/manifests/tile_vignets.json", + log: + f"{TILE_DIR}/logs/tile_vignets.json" + params: + pre = lambda wc: unit_pre("tile_vignets", wc.tile, + forest=forest_dir(wc.tile), + pre_run=[tile_local(wc.tile), TILE_VIGNET_FRESH, + TILE_NGMIX_RANGES]), + script_hash = SCRIPT_HASH, + # tile_vignets now PRODUCES the chunk partition (TILE_NGMIX_RANGES), so + # it carries the splitter's fingerprint too. Same hash, same constant — + # the Snakefile's NGMIX_RANGE_HASH argues what it guards, and + # tile_ngmix's copy below carries the mid-campaign-edit warning. + range_hash = NGMIX_RANGE_HASH + # 8, not 16, for the same reason tile_ngmix is 1: `-b {threads}` is SMP + # batch size over input FILE SETS, and a tile is one set -- this run's own + # log says "Batch size: 16 / Total number of processes: 1". 16 was the + # widest member and therefore set the whole GROUP's cpus_per_task; at 8 the + # group asks exactly what the eight-chunk ngmix wave needs. Billing is + # unchanged either way (nibi is MAX_TRES and the 112 GB memory term is + # 28 core-equivalents, well above both), but the group now packs onto a + # node in 8 cores instead of 16. + threads: 8 + resources: + mem_mb = lambda wc, attempt: 32000 * attempt, + # Measured median 3m29s, max 5:25 (this branch: 2m38s on 198.305). + # 20 min is p99 plus a wide margin, and it is a term in the GROUP's + # runtime sum, so the old defensive 240 is not free any more. + runtime = 20, + slurm_extra = TILE_SLURM_EXTRA + shell: + # The completeness check is pointed at the NODE-LOCAL run root; see + # sp_shell's check_args for what the two flags do. + sp_shell("tile_vignets", "config_tile_PiViVi.ini", + check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') + +# ngmix shape measurement — N chunks per tile (D4). Each chunk LOOKS UP its own +# CLOSED object-ID range in the file tile_vignets materialised at the top of this +# group job (TILE_NGMIX_RANGES); the ranges are knowable only at EXECUTION time, +# from this tile's own sexcat, which is why a params function cannot supply them. +# Closed, not open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour +# straggler's root cause (ngmix treats id_obj_max <= 0 as unbounded). +# +# Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and +# merge_sep_cats — DAG-serialised after all chunks — is the gather. +rule tile_ngmix: + group: TILE_GROUP + input: + # The manifest, and only the manifest. The vignette store is no longer a + # declared input because it is no longer a declared output: it is + # node-local, produced by tile_vignets earlier in THIS SAME group job + # (see TILE_LOCAL). The manifest was always the real edge. + vignets = rules.tile_vignets.output.manifest, + sx = rules.tile_detect.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", + # STAYS ON SCRATCH, unlike the vignette store. It is DAG currency: + # tile_merge_cats reads it, and merge_sep_cats derives chunks 2..N from + # chunk 1's path. It is also small (~300 KB/chunk), so it costs the + # scratch high-water nothing worth chasing. temp() still reclaims it + # once merge_cats has run. + chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), + log: + f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" + params: + pre = lambda wc: unit_pre("tile_ngmix", wc.tile, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing sexcat would fall + # through to shapepipe_run with an unset range and fail as something + # else. Capture, check, then eval — the range script fails as itself. + # tile_local FIRST, because it is what exports $SP_LOCAL — and the + # ranges file the lookup reads lives there, written once by + # tile_vignets (TILE_NGMIX_RANGES). This chunk only looks its row + # up; it never recomputes, and the script refuses to. + # + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing ranges file would + # fall through to shapepipe_run with an unset range and fail as + # something else. Capture, check, then eval — the lookup fails as + # itself. + pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, + f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py ' + f'--read "{NGMIX_RANGES}" --chunk {wc.chunk}) || exit 1', + 'eval "$ngmix_range_out"']), + script_hash = SCRIPT_HASH, + # `pre_run` already puts the INVOCATION in params, but the invocation is + # invariant to the script's body, and what that body decides is a + # PARTITION (the Snakefile's NGMIX_RANGE_HASH argues the corruption). + # Carried on tile_vignets too, since the split moved there. + # + # WHY THE PARTITION IS MATERIALISED ONCE, kept as history because the + # design only reads as over-careful until you have seen this. The eight + # chunks used to each run ngmix_range.py in their own shell and trust the + # others to have landed on the same boundaries. A fused group holds them + # open for hours (smk-g4: 6,236-7,762 s elapsed per chunk), and each chunk + # read the script only when its own shell started -- against the LIVE + # checkout, as every job did before the launch code snapshot (bin/sp) -- + # so an edit landed mid-flight was read by some and not others. + # Seen once, live: an invocation four seconds after a rewrite returned + # chunk 5 of 186.307 as 17649..22060 where the other seven had been split + # 17129..21229 — 520 objects orphaned, 831 measured twice, and + # merge_sep_cats concatenates whatever it is handed, so the tile would + # have completed green. (Twelve sequential and thirty-six concurrent runs + # against a stable checkout gave the identical correct partition; the + # race was purely the edit.) That is now STRUCTURALLY closed: tile_vignets + # writes the ranges once and the chunks only look their row up, so there + # is nothing left for them to disagree about. The hash below no longer + # buys sibling agreement — it buys REPRODUCIBILITY across a resume. + # + # WHAT A ONE-RULE FINGERPRINT DOES MID-CAMPAIGN: IT DELETES SCIENCE + # PRODUCTS. Observed when range_hash lived on tile_ngmix ALONE, which is + # no longer the case -- tile_vignets carries it too now that it is the + # rule that runs the splitter, so an edit drags tile_vignets back into + # the group, the store and the ranges are rebuilt, and nothing trips. + # Kept because the mechanism is general and the next single-rule param is + # one edit away. A new param entry replans the fused group with "Params have changed since last + # execution", scheduling the eight chunks, tile_merge_cats and + # tile_make_cat but NOT tile_vignets or tile_detect (their manifests + # exist, so missing_output never queues them). That is exactly the state + # TILE_VIGNET_REQUIRED catches: every chunk trips the guard, the group + # fails, and GroupJob.postprocess(error=True) removes every member's + # EXISTING outputs -- tile_make_cat's final_cat on the persistent root + # among them. + # + # OBSERVED end to end, not derived: SLURM job 20818649 against smk-g5, + # 2026-08-30, 32 seconds. The dry run scheduled exactly that member set + # (the group's SLURM label came back + # tile_shape_tile_make_cat_tile_merge_cats_tile_ngmix, no tile_vignets); + # every chunk tripped the guard; the group failed; and + # final_cat-186.307.fits was GONE from /project afterwards, the tile's + # directory on the persistent root empty. It had never been seen before + # because "Group jobs: inactive (local execution)" puts it out of reach + # of any login-node fixture. (Restored from a copy taken first; design + # and transcript in sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md.) + # + # WORTH SITTING WITH: the guard is what makes this loud rather than + # silent, and the loud failure is precisely what triggers the deletion. + # Still the right trade -- but it argues for never reaching this state, + # not for relaxing the guard. + # + # THE RULE THE INCIDENT LEFT BEHIND: a fingerprint that changes on the + # chunks but NOT on tile_vignets is the dangerous shape. tile_local() + # was always safe for exactly this reason -- it sits in three rules' + # params.pre, so an edit there pulls tile_vignets in and the store is + # rebuilt. range_hash is now the same shape by construction. job_head.sh's + # runbook sweep would not have helped either: it skips any tile with a + # final_cat, exactly the set this breaks. + # + # INVERTED BY clean_tiles, so read the default carefully. With + # reclamation ON the tombstoned tile has lost tile_detect.json and the + # structural "Input files updated by another job" propagation puts + # tile_vignets back in the group (measured at clean_tile below), so the + # store is rebuilt and nothing trips. The SHIPPED DEFAULT + # clean_tiles: false is the dangerous configuration -- the opposite of + # how reclamation reads everywhere else in this file. + # + # So: land this hash, and every later edit to ngmix_range.py, at a + # campaign boundary on a fresh root. Same rule and same escape hatch as + # tile_local()'s -- `--rerun-triggers mtime code software-env`. + range_hash = NGMIX_RANGE_HASH + # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE + # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over + # filehd.process_list, i.e. parallelism ACROSS INPUT FILE SETS. An ngmix + # chunk is one catalogue, so process_list has exactly one entry: every real + # log says "Batch size: 4 / Total number of processes: 1". There is no + # internal parallelism either (no multiprocessing/Pool/joblib/threading in + # ngmix_package/ngmix.py), and OMP/BLAS are pinned to 1 by both the prologue + # and the profile. The reserved cores 2-4 never had anything to run. + # + # Worth being precise about what this saves. nibi bills + # TRESBillingWeights=CPU=1000,Mem=250G under PriorityFlags=MAX_TRES, i.e. + # max(cores, mem_GB/4). At 14 GB the memory term alone is 3.5 core- + # equivalents, so 4 -> 1 core moves the reservation from 4.0 to 3.5, a 12% + # saving -- NOT 75%. The real core-hour win is the elapsed-time collapse + # from killing the NFS random reads, not this. + threads: 1 + retries: 2 + benchmark: + f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" + resources: + # 5000, down from 14000, and this is the campaign's largest single cost + # saving — but read what the number means before moving it again. + # + # sacct's MaxRSS here is NOT process memory. nibi runs + # JobAcctGatherType=jobacct_gather/cgroup, so it reports the cgroup's + # memory.current, which under cgroup v2 CHARGES PAGE CACHE to the job. + # Cache is reclaimable — the kernel evicts it before it kills anything — + # so the cgroup high-water is an upper bound on what the job NEEDS, not + # a floor. What a tight reservation can still do is squeeze the cache + # that keeps the node-local store resident, which is part of why the + # fused tile is fast. + # + # MEASURED ACROSS 31 TILES (campaign smk-g4, 2026-08-30): cgroup + # high-water 27.40 GiB max, 22.14 GiB mean, against the 109.4 GiB the + # group was reserving. The eight chunks are SIBLINGS in the group's + # toposort so snakemake SUMS their mem_mb; at 5000 the group asks + # 40000 MiB = 39.1 GiB, a 1.43x margin over the worst tile observed. + # + # The ANONYMOUS half of that, which is the part that can actually OOM, + # is the number to argue about, and this repo carries two estimates that + # disagree by 1.8x. Snakemake's psutil benchmark says ~1.25 GiB per + # chunk (~10 GiB for eight); config_tile_Ng_template.ini's own + # SAVE_BATCH note says ~2.3 GB per worker from an A/B test (job + # 17607877), i.e. ~18.4 GiB. Prefer the larger: snakemake samples RSS on + # a 30-second grid (BENCHMARK_INTERVAL), and a SAVE_BATCH = 250 flush + # cycle is exactly the sawtooth such a grid misses. So read the margin + # as ~2.1x over anonymous memory with ~20 GiB left for cache against an + # 8.1 GiB store — comfortable, but not the 4x a psutil-only reading + # would suggest. An OOM is also self-healing: mem_mb scales with + # attempt, so a retry asks 80000. + # + # WHY THIS IS THE BIG ONE. nibi bills max(cores, mem_GB/4), so at + # 112 GB the fused group billed 28 core-equivalents for 8 real cores — + # 3.4x, measured live at 905 billed against 266 allocated. At 40 GB it + # bills 10. That is not a saving so much as a schedule: the account's + # fairshare target is ~250 CE, which buys 9 tiles in flight at 28 and 25 + # at 10, and DR6's wall clock is (tiles / tiles-in-flight) x elapsed. + # + # 4000 is the floor and is NOT recommended yet: it would take the group + # to 32000 MiB, where tile_vignets' own 32000 becomes the binding term + # and the group finally bills its 8 real cores — but that is a 1.14x + # margin over the worst tile measured, and the first thing to give would + # be the page cache holding the store. Take it only with a measurement + # of cache behaviour under pressure, not on the arithmetic alone. + mem_mb = lambda wc, attempt: 5000 * attempt, + # 120 on the FIRST attempt, and ATTEMPT-SCALED after it. MEASURED + # two ways, and the second is why the margin is thinner than it looks: + # * alone (job 20795277, tile 198.305 chunk 1): ~76 min for 3547 + # objects = 1.29 s/object, against a 7h34m median on NFS -- the + # same ~50 min of TotalCPU either way, so the collapse is pure I/O. + # * EIGHT-WIDE on one node (job 20799387, tile 186.307, 4412 + # objects/chunk): 1.54 s/object steady-state, i.e. concurrency + # costs ~19%, and the chunk lands at ~113 min. The campaign's + # largest tile (198.306, 4678 objects/chunk) projects to ~120 -- + # exactly this number, with nothing left over. + # + # Inside a group SLURM enforces only the GROUP's wall (165 min), never + # a member's, so 120 is a budgeting term rather than a kill line and + # the worst tile still lands ~127 min inside 165. What is NOT safe is a + # flat retry: a group that TIMEOUTs re-queues against the identical + # wall and fails identically, burning three 165-minute allocations to + # learn nothing. Scaling with `attempt` keeps the happy path in + # cpubase_bycore_b1 (20+120+10+15 = 165 <= 180) and gives a retry real + # headroom (285 min, which is b2) instead of a rerun of the same + # failure. Attempt 1 is unchanged, so this does not perturb the + # benchmark -- it only makes the failure branch mean something. + runtime = lambda wc, attempt: 120 * attempt, + slurm_extra = TILE_SLURM_EXTRA + shell: + sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + + +def ngmix_manifests(wc): + return [f"{tile_dir(wc.tile)}/manifests/tile_ngmix_{k}.json" + for k in range(1, NGMIX_CHUNKS + 1)] + +def ngmix_chunkdirs(wc): + return [f"{tile_dir(wc.tile)}/output/run_sp_tile_ngmix_Ng{k}u" + for k in range(1, NGMIX_CHUNKS + 1)] + +# The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's +# own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). +rule tile_merge_cats: + group: TILE_GROUP + input: + manifests = ngmix_manifests, + chunkdirs = ngmix_chunkdirs, + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" + log: + f"{TILE_DIR}/logs/tile_merge_cats.json" + params: + pre = lambda wc: unit_pre("tile_merge_cats", wc.tile, + env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + # Measured median 0:15, max 0:42. A term in the group's runtime sum. + runtime = 10, + slurm_extra = TILE_SLURM_EXTRA + shell: + sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") + +# The run's science product. make_cat also reads the vignette store's +# psfex_interp output, so it — not ngmix — is the store's last reader. +# +# No protected(): the full default rerun-triggers govern, and protected() only +# ever forced people through a `--forcerun` detour. +rule tile_make_cat: + group: TILE_GROUP + input: + # No store input: it is node-local, written by tile_vignets in this same + # group job. make_cat reads its psfex_interp output through + # $NGMIX_VIGNET_DIR (config_tile_Mc.ini). + ms = rules.tile_merge_cats.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{PROD_TILE_DIR}/final_cat-{{tile}}.fits", + log: + f"{TILE_DIR}/logs/tile_make_cat.json" + params: + pre = lambda wc: unit_pre("tile_make_cat", wc.tile, + pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, + TILE_CLEAN]), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + # Measured median 1:33, max 1:50. A term in the group's runtime sum. + runtime = 15, + slurm_extra = TILE_SLURM_EXTRA + shell: + # The plain body plus the catalogue publish: final_cat is a real file, so + # it is a real declared output (and it persists — never temp()). The + # publish is guarded on rc, so a job whose shapepipe_run died never + # publishes a catalogue for a manifest snakemake is about to delete. + sp_shell("tile_make_cat", "config_tile_Mc.ini", + post="if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner' + '/output/final_cat*.fits | head -1)" {output.final_cat}\n' + "fi\n") + + +# --- tile reclamation (D5) -------------------------------------------------- +# The tile-side counterpart of clean_exposure, and the differences are the whole +# design. Read that rule's commentary in exposure.smk first; this one only says +# where the tile case departs from it. +# +# WHY IT EXISTS. clean_exposure reclaims exposure stores; nothing reclaimed +# tiles, so a FINISHED tile held 1.19 GiB across 137 inodes on scratch forever +# (measured on 186.307; the per-directory breakdown is in clean_tile.py). At +# DR6's 23,114 tiles that is 26.9 TiB against a 1 TiB quota and 3.17M inodes +# against a 1M one. BOTH BOUNDS BIND, and the byte one binds first: without this +# rule no batch may exceed ~859 tiles. +# +# ELIGIBILITY IS TRIVIAL, AND THAT IS THE POINT: a tile's store has no consumer +# outside that tile, so there is no consumer set, no eligibility test and no +# staleness to detect — only an ordering edge on the tile's own final_cat +# (clean_tile.py opens with the asymmetry; the survivor set is argued there too). +# +# THE INPUT IS final_cat, ON THE PERSISTENT ROOT, AND NOT ancient(). It is +# already the campaign's designated tile-finished marker (see final_cat()'s +# docstring and tile_finished() above), and an ordinary input is what orders the +# clean after the tile — which is what makes reclamation ROLLING: within one +# invocation a tile is reclaimed as soon as its own chain lands, so the scratch +# high-water tracks the tiles in flight instead of the tiles in the batch. That +# is the whole reason the rule is worth having; ancient() would keep the +# dependency but drop the ordering, and the batch would peak at its full size. +# +# SCOPE: TILES_READY only, never every tile in the index — see +# clean_tile_targets() in the Snakefile for what an out-of-scope tombstone drags +# into the DAG. +# +# THE SURVIVING tile_vignets.json DOES NOT TRIP TILE_VIGNET_REQUIRED, AND THE +# REASON IS WHAT GETS DELETED, NOT WHAT GETS KEPT. Read that guard above first. +# A cleaned tile is, on its face, exactly the state it exists to catch: a valid +# tile_vignets manifest over a node-local store that is not there (here because +# it never outlived its job, rather than because a fused group half-ran). +# smk-g4's job_head.sh carries a runbook sweep for that state, and it skips any +# tile with a final_cat — i.e. every tile this rule ever touches — so a cleaned +# tile is invisible to it either way. +# +# Force a cleaned tile back anyway (delete its final_cat) and tile_vignets IS +# rescheduled, so the store is rebuilt and no chunk trips the guard. Measured on +# the fixture: 19 jobs, the whole chain from tile_get_images, tile_vignets inside +# the tile_shape group. Snakemake names the mechanism itself -- +# reason: Input files updated by another job: .../exp_forest, +# .../manifests/tile_find_exposures.json, .../manifests/tile_detect.json +# -- and it is NOT the `mtime` trigger. It is the structural propagation that +# makes every dependent of a rerunning job rerun, the same edge the cascade +# commentary above says ancient() cannot suppress. Worth the distinction: that +# propagation is not in profiles/nibi's rerun-triggers list and so cannot be +# switched off there, where an mtime argument could be. +# +# So the guard stays silent only because tile_detect.json and exp_forest/ GO. +# Counterfactual on the same fixture, upstream manifests restored and mtimes +# controlled so nothing reruns on timestamp -- i.e. the "delete only output/" +# design -- schedules 8 x tile_ngmix + tile_merge_cats + tile_make_cat with +# tile_vignets ABSENT, and every chunk trips the guard. +# ADDING A MANIFEST TO THE SURVIVOR LIST IS THEREFORE NOT FREE: tile_vignets is +# safe because nothing upstream of it survives, and tile_detect.json in +# particular must never join the list. +# +# A LOCALRULE (declared in the Snakefile), same as clean_exposure. Unlike +# exp_star_cat it is a DAG LEAF, so being local can never make it both a +# dependency and a dependent of a group — no `group:` label here, and none +# possible. +# +# WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to +# tile_local() mid-campaign reruns finished tiles unsatisfiably because their +# EXPOSURE stores are reclaimed. With this rule on, the tile's own store is gone +# too, so the rerun has even less to stand on. The recommendation is unchanged — +# land those edits between campaigns — but the margin for being wrong is smaller. +rule clean_tile: + input: + # The tile-finished marker, on the persistent root. A lambda rather than + # the PROD_TILE_DIR pattern so there is exactly one definition of this + # path (final_cat() in the Snakefile), the same way clean_exposure keys + # off wildcards.exp alone. + lambda wc: final_cat(wc.tile) + output: + tombstone = f"{TILE_DIR}/cleaned.json" + params: + # clean_tile.py is external to the shell string, so the `code` + # rerun-trigger cannot see it — same reason SCRIPT_HASH exists. + script_hash = CLEAN_TILE_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_tile.py" + " --tile-dir $(dirname {output.tombstone}) --tile {wildcards.tile}" + " --tombstone {output.tombstone}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py new file mode 100644 index 000000000..f780531e1 --- /dev/null +++ b/workflow/scripts/build_forest.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the SP_EXP view). + +A plain script (not a run: block) so the tile chain stays group-compatible. +Reads the tile's exposures from run_index.sqlite and symlinks each exposure's +``exp///output`` into ``///output`` by exact +name (no glob). The 2-char ```` shard level is NOT cosmetic: ShapePipe's +``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its +$SP_EXP glob (``///output/run_sp_*/...``), so a flat +forest makes every tile gather stage fail "No split_exp_runner output found". +(The exposure STORE is sharded the same way, for the filesystem's sake.) +The forest is a convenience view; the DAG edge to the exposures is declared in +the rule's input (tile.smk), not here. +""" + +import argparse +import shutil +import sqlite3 +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + con = sqlite3.connect(args.index, timeout=60) + exps = [r[0] for r in con.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] + con.close() + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e[:2] / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape + dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree — unlink() + # raises IsADirectoryError on it and would kill the job. + if dst.is_symlink() or dst.exists(): + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py new file mode 100644 index 000000000..2dd191306 --- /dev/null +++ b/workflow/scripts/build_index.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Build the run index (``run_index.sqlite``) that drives the compute DAG. + +The index is *parse-time data*, never a rule input: the Snakefile loads it once +at parse time into plain dicts, so extending a run's tile list changes which +jobs exist without touching the mtime chain of completed work. + +It ACCUMULATES ACROSS INVOCATIONS, but is AUTHORITATIVE FOR THE CURRENT TILE +LIST (D1). Precisely: + + * a tile in the current list that has find_exposures output has its ``tiles`` + row replaced and its ``tile_exposures`` edges DELETED AND REBUILT — the Fe + output is the truth, so a tile whose exposure list shrank or changed must + not keep stale edges to exposures it no longer reads; + * a tile NOT in the current list is left completely untouched, which is what + makes the index span the campaign and lets a later ``clean_exposure`` see + every consuming tile; + * ``exposures`` rows only ever accumulate (``INSERT OR IGNORE``). An exposure + no tile references any more is a harmless orphan: nothing reads that table + except by joining through ``tile_exposures``. + +It records: + + tiles(tile_id, ra_dir, n_exp, status) + exposures(exp_id) -- deduplicated union over tiles + tile_exposures(tile_id, exp_id) -- the tile->exposure edges + +The tile->exposure edges are *data-derived*: they are read from each tile's +``find_exposures`` output (``exp_numbers--.txt``), which +``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. +So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME +by the Snakefile of the COMPUTE invocation ONLY — ``SP_PHASE == "compute"``, +which ``bin/sp`` sets — after the PREPARE invocation has produced the tiles' +find_exposures output. No other parse builds anything: a prepare parse or a +passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is +already on disk. (There is no ``sp index`` verb; the CLI below stays for +hand-inspection.) ``build`` below documents the missing-tile policy and the +write ordering. + +Exposure IDs are stored with their trailing single-char suffix stripped +(``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the +exposure-rule wildcard, matching the sharded ``exp///`` store. +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + + +def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: + """Return ``(exp_id, name)`` pairs from one tile's find_exposures output. + + Each line is an exposure *name* like ``2243881p``; the bare base ID (suffix + stripped) is the dedup key everywhere in the DAG, but the original name is + kept in the index — the fabricated per-unit ``exp_numbers`` list must carry + it verbatim (``get_images`` matches ``.fits.fz`` in the store; the + bare ID matches nothing). + """ + pairs = [] + for line in exp_numbers_file.read_text().splitlines(): + name = line.strip() + if not name: + continue + pairs.append((name[:-1] if name[-1].isalpha() else name, name)) + return pairs + + +def exp_list_path(run_dir: Path, tile_id: str) -> Path: + """This tile's find_exposures output, at its deterministic path. + + Sharded store (D2), fixed run dir (RUN_DATETIME=False) — an existence check, + never a glob (no ``ls`` at scale). + """ + idra, iddec = tile_id.split(".") + return (run_dir / "tiles" / tile_id[:2] / tile_id / "output" / + "run_sp_tile_Fe" / "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + + +def build(tile_ids: list[str], run_dir: Path, db_path: Path, + missing_threshold: float | None = 0.0) -> dict: + """Build the index over ``tile_ids``; return a summary dict. + + For each tile, check its ``exp_numbers--.txt`` at the tile's + deterministic ``find_exposures`` run dir (RUN_DATETIME=False, no glob). A + tile whose exposure list is missing is recorded in ``missing.json`` and the + index is built over the rest (a bad tile costs that tile, not the run). The + build is fatal only if the missing fraction exceeds ``missing_threshold``. + + ORDER MATTERS: the threshold is evaluated FIRST, from the missing set, and + the database + ``missing.json`` are written only if it passes. A build that + aborts must leave no trace — an aborted parse that had already mutated + durable state was the bug this ordering fixes. + + The write is idempotent, which is what makes it acceptable that the compute + parse runs it even under ``-n``: re-running over an unchanged tree produces + an identical database. + """ + missing = [t for t in tile_ids if not exp_list_path(run_dir, t).exists()] + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if missing_threshold is not None and frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + + db_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db_path, timeout=60) + # No DROP: the index accumulates across invocations (D1). + con.executescript( + """ + CREATE TABLE IF NOT EXISTS tiles( + tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE IF NOT EXISTS exposures( + exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS tile_exposures( + tile_id TEXT, exp_id TEXT, + PRIMARY KEY (tile_id, exp_id)); + """ + ) + + missing_set = set(missing) + all_exposures: set[tuple[str, str]] = set() + for tile_id in tile_ids: + if tile_id in missing_set: + continue + ra_dir = tile_id.split(".")[0] + exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) + con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", + (tile_id, ra_dir, len(exp_pairs))) + # Replace this tile's edge set wholesale. INSERT OR IGNORE alone only + # ever added, so a tile whose exposure list shrank kept edges to + # exposures it no longer reads — and those stale edges would block it in + # the report and pin those exposures against cleanup. + con.execute("DELETE FROM tile_exposures WHERE tile_id = ?", (tile_id,)) + con.executemany("INSERT INTO tile_exposures VALUES (?,?)", + [(tile_id, exp_id) for exp_id, _ in exp_pairs]) + all_exposures.update(exp_pairs) + + # Exposures accumulate: OR IGNORE, never REPLACE (the name never changes, + # and orphans left by a shrunken tile are harmless). + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + sorted(all_exposures)) + con.commit() + con.close() + + (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) + return {"n_tiles": len(tile_ids) - len(missing), + "n_exposures": len(all_exposures), + "n_missing": len(missing)} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-list", required=True, type=Path, + help="file of tile IDs, one per line") + p.add_argument("--run-dir", required=True, type=Path, + help="$SP_RUN: root of the tiles/ work-dir forest") + p.add_argument("--db", required=True, type=Path, + help="output run_index.sqlite path") + p.add_argument("--missing-threshold", type=float, default=0.0, + help="fatal if the missing-tile fraction exceeds this " + "(default 0.0: any missing tile is fatal)") + args = p.parse_args() + + tile_ids = [ln.strip() for ln in args.tile_list.read_text().splitlines() + if ln.strip()] + summary = build(tile_ids, args.run_dir, args.db, args.missing_threshold) + print(f"run_index: {summary['n_tiles']} tiles, " + f"{summary['n_exposures']} exposures, " + f"{summary['n_missing']} missing -> {args.db}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py new file mode 100644 index 000000000..728f6c527 --- /dev/null +++ b/workflow/scripts/clean_exposure.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Reclaim ONE exposure's store and leave a tombstone (PRD #848 D5, S5). + +Run as the shell of the in-DAG ``clean_exposure`` rule, never by hand: the rule's +``input:`` is every consuming tile's ``tile_vignets`` manifest, so by the time +this executes, every campaign tile that reads this exposure has already extracted +its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. + +What it deletes: the exposure's whole ``output/`` tree (the bulk store — +run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its +star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are +reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted +here like every other, so the exposure's chain must read as unbuilt, and 40 +symlinks left behind are a farm no rule now owns. The catalogue itself lives in +the run-independent cache, so rebuilding the farm costs a relink and no query. + +Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not +``rmtree``d. Legacy unit dirs carry ``star_cat_exp`` as a link into the old +shared pool, and an rmtree would recurse through it and delete the shared cache +for every other exposure in the campaign. + +Deleting the manifests is deliberate and load-bearing, not tidiness: + + * the manifests are the exposure rules' DECLARED outputs. If they survived, a + tile appended later would find the exposure chain "up to date" and run + tile_vignets against products that are no longer on disk. With them gone the + DAG sees the chain as unbuilt and regenerates it — the accepted cost of a + late append (D5), expressed as ordinary Snakemake bookkeeping rather than as + a special case. + * Snakemake only demands a missing intermediate when something downstream of it + needs to run, so tiles already finished are NOT rerun by their exposures' + manifests vanishing. + +``logs/`` goes with them, and for the same reason rather than for bytes. Each +log holds the completeness verdict of one stage, written on every run and kept by +snakemake through failures; a log left behind would attest "complete" for a store +that is no longer there, contradicting the unbuilt chain the DAG must now see. +Its content for a successful stage is byte-identical to the manifest beside it, +so absorbing the logs into the tombstone would duplicate what the manifests +already carry — they are deleted, not copied. + +Nothing is lost to the report: every ``manifests/*.json`` is copied verbatim into +the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned +exposure's record out of the tombstone — it reports the unit as ``cleaned``, +warn counts and shortfalls intact, instead of "not run". + +The absorption is by GLOB, so it takes whatever is in ``manifests/``, keyed by +file stem; the report re-keys on each manifest's own ``stage`` field. A legacy +``.failed.json`` from the pre-``log:`` convention is therefore carried +through unremarkably — it should never be there (an exposure with a failed stage +has no complete vignets consumer and so is not eligible for cleaning), but it +costs nothing to be right about. + +Order matters, and it is the reverse of the obvious one: the tombstone is +written FIRST, complete, and only then is anything deleted. A crash between the +two leaves a tombstone beside a store that is still there — the next invocation +treats the exposure as cleaned and only the disk is lost. Deleting first would +put the crash window where the manifests are already gone and the record that +replaces them was never written, and the report would be blind to that exposure +forever. + +The exp_psf benchmark tsv lives beside ``manifests/`` and ``logs/``, not inside +either, so it survives this job — it is the measured-memory feed for resource sizing (D4). + +The tombstone records the consumer set it was cleaned against. The rule carries +that same set as a ``params`` value, so when the index grows a new consumer the +tombstone goes stale under the default ``params`` rerun-trigger and the clean job +is rescheduled after the new tile's vignets — the exposure is cleaned once per +consumer set, not once per campaign. +""" + +import argparse +import json +import shutil +import time +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path) + p.add_argument("--exp", required=True) + p.add_argument("--tombstone", required=True, type=Path) + p.add_argument("--consumers", default="", + help="comma-separated tile ids this exposure was cleaned against") + args = p.parse_args() + + consumers = [t for t in args.consumers.split(",") if t] + + # Absorb the manifests before they go: the tombstone becomes the exposure's + # surviving record. + manifests = {} + mdir = args.exp_dir / "manifests" + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + manifests[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + manifests[f.stem] = {"unreadable": str(exc)} + + # is_symlink() first, and OR'd with exists(): exists() follows the link, so a + # dangling legacy star_cat_exp would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs", + args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") + targets = [t for t in candidates if t.is_symlink() or t.exists()] + + # Tombstone first, complete — then delete (see the module docstring). + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ + "exp": args.exp, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "consumers": consumers, + "removed": [str(t) for t in targets], + "manifests": manifests, + }, indent=2) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed = [] + for target in targets: + # NEVER rmtree a symlink (see the module docstring). + if target.is_symlink(): + target.unlink() + else: + shutil.rmtree(target) + removed.append(str(target)) + print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " + f"{len(consumers)} consuming tile(s)") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py new file mode 100644 index 000000000..85686df69 --- /dev/null +++ b/workflow/scripts/clean_tile.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Reclaim ONE finished tile's scratch store and leave a tombstone (PRD #848 D5). + +Run as the shell of the in-DAG ``clean_tile`` rule, never by hand: the rule's +``input:`` is the tile's ``final_cat`` on the PERSISTENT root, so by the time +this executes the tile has published the only thing the campaign wanted from it. +A tile's scratch store has no reader outside that tile — tiles read exposures, +nothing reads another tile's store — so unlike the exposure case there is no +consumer set to close over and no eligibility test to make. Writer, then +cleaner, and the DAG edge is the whole ordering argument. + +WHAT IT DELETES: the tile's whole ``/tiles///`` directory. +Measured on 186.307 (smk-g4, a finished 34-tile-campaign tile): 1,279,231,196 +bytes across 137 inodes (71 regular files, 9 symlinks, 57 directories) — +``output/run_sp_tile_Sx`` 745 MB, ``run_sp_tile_Uz`` 382 MB, ``run_sp_Mc`` +46 MB, ``run_sp_Ms`` 39 MB, ``run_sp_tile_Mh_exp`` 11 MB, and a non-``output/`` +remainder of 62 inodes totalling 15,726 bytes. Deleting only ``output/`` is NOT +enough: 62 x 23,114 DR6 tiles is 1.43M inodes against a 1M quota, so the inode +bound binds on its own and the directory has to go as a whole. + +THE FOUR SURVIVORS, AND WHY EACH ONE IS NOT RESIDUE +--------------------------------------------------- +Everything kept here is currency some OTHER mechanism owns and re-reads long +after this tile is done. Nothing is kept for tidiness, and the three that +already exist are a CONTRACT: ``require_survivors`` checks all of them BEFORE +anything is deleted, because a silent drift in one of these paths would not show +up as a broken clean — it would show up much later as a campaign that cannot +resume. The fourth, ``cleaned.json``, is this job's own output and is the one +thing here that cannot be pre-checked; it is written first instead (see ORDER +below). + + 1. ``cleaned.json`` — this script's own tombstone, and the tile's surviving + record. It absorbs every manifest verbatim so ``sp report`` can still + report a reclaimed tile as ``cleaned`` rather than as "not run"; see + ``run_report.absorb_tombstones``. + + 2. ``manifests/tile_vignets.json`` — CLEAN_EXPOSURE'S CURRENCY, not ours. + Exposure reclamation keys campaign-wide eligibility on exactly this path: + ``clean_targets()`` tests it for out-of-scope consumers and ``rule + clean_exposure`` declares it as an input for in-scope ones. Deleting it + would silently strand every exposure shared with an out-of-scope tile + (~7-10 tiles read each exposure), and would fail an in-scope + ``clean_exposure`` job outright on a missing input if the tile finished + mid-invocation. It is also not stale in any dangerous sense: it attests + that ``tile_vignets`` succeeded, which stays true forever, and its product + — the vignette store — was node-local and died with its job, so it never + lived on scratch to be contradicted. + + 3. ``manifests/tile_find_exposures.json`` — the PREPARE invocation's target. + ``rule prepare_all_tiles`` declares this manifest for EVERY tile in the + list, and the tile list accumulates across the campaign, so a cleaned tile + is still demanded by every later ``sp run``. Delete it and the whole + ``tile_prep`` group (get_images -> uncompress -> find_exposures) reruns per + cleaned tile per invocation, which does not merely cost jobs: uncompress + writes 382 MB back into the store this job just emptied. + + 4. ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-*.txt`` — + the campaign's tile->exposure edge data, and the reason this is a survivor + rather than a manifest is that NOTHING re-derives it. ``build_index.build()`` + runs at the parse of EVERY compute invocation, over the whole declared tile + list, and checks this exact path (``build_index.exp_list_path``); a tile + whose file is gone counts as missing, and the default + ``SP_MISSING_THRESHOLD`` of 0.0 makes ONE missing tile a fatal parse. So + deleting it does not degrade the campaign, it stops it: the first `sp run` + after the first reclaimed tile cannot build a DAG at all. + Only the one file is kept — ``run_sp_tile_Fe``'s own ``logs/``, ``tmp/`` + and the runner's process log are ordinary residue and go. + +The rest of ``run_sp_tile_Fe``'s parents come along because a file cannot +outlive its directories: 10 inodes per cleaned tile in total (the tile dir, +``cleaned.json``, ``manifests/`` + 2 manifests, and the 4-deep Fe path + its +file), i.e. ~231k inodes at DR6 against the 1M scratch quota — versus 3.2M if +nothing were reclaimed and 1.4M if only ``output/`` were. + +WHAT IS LOST, STATED PLAINLY. The per-tile audit trail, for BOTH tools that +read it — ``sp_tilecost.py`` and ``sp_costmodel.py``. They attribute the fused +``tile_shape`` group job's cost per tile by reading the tile's SExtractor +catalogue (NAXIS2 of the sexcat = the object count, the cost model's independent +variable, ~380 MB and unkeepable; sp_costmodel also reads its ``EPOCH_k`` +extensions for the geometric epoch count) and the eight +``tile_ngmix_.benchmark.tsv`` files. The sexcat is gone for good; the +benchmark ROWS are absorbed into the tombstone under ``benchmarks``, because +they are two lines each and they are the measured-memory feed D4 sizes +``mem_mb`` from — exposure.smk moves ``exp_psf``'s benchmark outside +``manifests/`` for exactly this reason. They are absorbed rather than left on +disk (8 more inodes per tile = 185k at DR6 buys nothing a JSON blob does not), +so ``sp_tilecost.py`` will not find them at their old paths: the record +survives, the tool's current reader does not. FOLLOW-UP, deliberately not done +here: teach ``sp_tilecost.py`` to fall back to ``cleaned.json`` for a tile whose +benchmark TSVs are gone. Until it does, per-chunk cost attribution stops at the +first reclaimed tile even though the numbers are still on disk. + +DELETION IS SYMLINK-SAFE (``clean_exposure`` gives the general reason), and here +that is not a nicety. A finished tile holds NINE symlinks in TWO classes, and +the second is the one that matters: + + * ``exp_forest///output`` — 7 links into the EXPOSURE stores, + each shared with 7-10 other tiles. Rebuildable, but only by re-running those + chains from VOS. + * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` + — 2 links into ``/project/def-mjhudson/unions-wl/tiles``, the staged survey + imaging: 621 GB across 2,536 files, on the BACKED-UP, GROUP-SHARED + ``/project``, and not this campaign's to lose. get_images RETRIEVE=symlink + is what puts them there, so every tile in every campaign carries a pair. + +Both classes are handed WHOLESALE to ``shutil.rmtree`` — ``exp_forest/`` as a +top-level entry, ``run_sp_tile_Git/`` as one inside ``output/``, which ``prune`` +descends only because the Fe survivor lives there. So the safety rests on +rmtree's own semantics (it unlinks a symlinked entry rather than recursing +through it), NOT on ``prune``'s ``is_symlink()`` test, which fires only for a +link that is itself a direct entry of a level prune walks. Both were verified on +the fixture: every link unlinked, no target followed. Whoever edits ``prune`` +next should know that the worst case is not a scratch store they could rebuild — +it is a rmtree walking into half a terabyte of shared, backed-up survey data. + +LOGS ARE DELETED, NOT ABSORBED, as in ``clean_exposure``, and here the +duplication is exact: on a finished tile every ``logs/.json`` is +BYTE-IDENTICAL to the ``manifests/.json`` beside it (verified across all +16 stage records of 186.307), because a tile with a failed stage has no +final_cat and so is never cleaned. + +ORDER: tombstone FIRST, then deletion — ``clean_exposure``'s docstring argues +the crash window. + +There is no ``consumers`` field and no consumer-set staleness to detect, because +a tile has no consumers. What reruns this job is the ordinary machinery +(``script_hash``, and ``mtime`` if final_cat is rewritten). A rerun over an +already-pruned tree deletes nothing and, because absorption is ADDITIVE, does +not blank the record either — see ``previous_record``. +""" + +import argparse +import csv +import json +import shutil +import time +from pathlib import Path + + +def survivor_paths(tile_dir: Path, tile: str) -> dict: + """``{what it is: path}`` for the three PRE-EXISTING survivors. + + Three, not four: ``cleaned.json`` is this job's own output and does not + exist yet when this is called. + + The Fe path is spelled out rather than globbed and MUST agree with + ``build_index.exp_list_path`` — that function is what re-reads it at every + compute parse. It is not imported, because this script runs inside the + container as a job shell and stays stdlib-only and import-free like + ``clean_exposure.py``; ``require_survivors`` below turns a drift between the + two into a loud failure on the first tile instead of a fatal parse later. + """ + idra, iddec = tile.split(".") + return { + "clean_exposure eligibility (clean_targets / rule clean_exposure input)": + tile_dir / "manifests" / "tile_vignets.json", + "prepare_all_tiles target (rule prepare_all_tiles input)": + tile_dir / "manifests" / "tile_find_exposures.json", + "index build input (build_index.exp_list_path)": + tile_dir / "output" / "run_sp_tile_Fe" / "find_exposures_runner" + / "output" / f"exp_numbers-{idra}-{iddec}.txt", + } + + +def require_survivors(survivors: dict, tile: str) -> None: + """Abort before deleting anything if the survivor contract is not met. + + Fatal, not a warning. Each of these is read by a mechanism OUTSIDE this + tile, and each failure mode is silent at deletion time and loud much later: + a missing vignets manifest strands shared exposures, a missing Fe manifest + reruns the prepare chain, a missing exposure list makes the next compute + parse exit on the missing-tile threshold. Failing here costs one red job in + a keep-going run (clean_tile is a leaf, so it poisons no cone) and leaves + the store intact for inspection. + """ + missing = {k: p for k, p in survivors.items() if not p.exists()} + if missing: + lines = [f"[clean_tile] {tile}: refusing to reclaim — " + f"{len(missing)} survivor(s) not on disk:"] + lines += [f" {p}\n owned by: {k}" for k, p in missing.items()] + lines.append(" Nothing was deleted. Either this tile's store is not in " + "the state a finished tile should be in, or one of these " + "paths has moved and clean_tile.py has not followed it.") + raise SystemExit("\n".join(lines)) + + +def previous_record(tombstone: Path) -> tuple: + """``(manifests, benchmarks)`` from an existing tombstone, or two empties. + + THE ABSORPTION IS ADDITIVE, and this is why. A second clean of an + already-cleaned tile finds only the two surviving manifests on disk, so a + fresh absorption would overwrite a complete record with a two-entry one and + the tile's history would be gone — silently, and for good. Found by running + the fixture clean twice; not hypothetical, since ``script_hash`` reruns this + job on any edit to this file. So the previous record is the BASE and what is + on disk is laid over it: a rebuilt stage's fresh manifest still wins, and a + reclaimed one keeps the only copy that exists. + """ + if not tombstone.exists(): + return {}, {} + try: + old = json.loads(tombstone.read_text()) + except (OSError, json.JSONDecodeError): + return {}, {} # unreadable: start over rather than refuse + return (dict(old.get("manifests") or {}), + dict(old.get("benchmarks") or {})) + + +def absorb_manifests(mdir: Path) -> dict: + """Every ``manifests/*.json`` verbatim, keyed by file stem. + + By GLOB, so it takes whatever is there; ``run_report.py`` re-keys on each + body's own ``stage`` field. Same shape as ``clean_exposure``'s absorption, + including its tolerance for a legacy ``.failed.json``. + """ + out = {} + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + out[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + out[f.stem] = {"unreadable": str(exc)} + return out + + +def absorb_benchmarks(mdir: Path) -> dict: + """The ngmix chunks' benchmark rows, keyed by file stem. + + Two lines each (header + values), so this is a few hundred bytes for the + campaign's only per-chunk record of runtime, RSS and mean load — the feed D4 + sizes ``tile_ngmix``'s ``mem_mb`` from. Numbers are kept as strings: this is + an archive of what the TSV said, not a re-measurement. + """ + out = {} + if mdir.is_dir(): + for f in sorted(mdir.glob("*.benchmark.tsv")): + try: + rows = list(csv.DictReader(f.read_text().splitlines(), + delimiter="\t")) + except OSError as exc: + out[f.name] = {"unreadable": str(exc)} + continue + if rows: + out[f.name] = dict(rows[0]) + return out + + +def prune(root: Path, keep: set, removed: list) -> None: + """Delete everything under ``root`` except ``keep`` and the dirs leading to it. + + A whitelist walk rather than an rmtree-with-exceptions, so adding a survivor + is one line and can never be half-implemented: a path is kept iff it is a + survivor, recursed into iff it is a real directory on the way to one, and + deleted otherwise. + + ``is_symlink()`` is tested BEFORE ``is_dir()`` and before the recursion — + see the module docstring on the two symlink classes. + """ + ancestors = {p for k in keep for p in k.parents} + for entry in sorted(root.iterdir()): + if entry in keep: + continue + if entry in ancestors and not entry.is_symlink(): + prune(entry, keep, removed) + continue + if entry.is_symlink(): + entry.unlink() + elif entry.is_dir(): + shutil.rmtree(entry) # unlinks nested symlinks, never follows + else: + entry.unlink() + removed.append(str(entry)) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-dir", required=True, type=Path) + p.add_argument("--tile", required=True) + p.add_argument("--tombstone", required=True, type=Path) + args = p.parse_args() + + survivors = survivor_paths(args.tile_dir, args.tile) + require_survivors(survivors, args.tile) + + mdir = args.tile_dir / "manifests" + manifests, benchmarks = previous_record(args.tombstone) + manifests.update(absorb_manifests(mdir)) + benchmarks.update(absorb_benchmarks(mdir)) + + # Tombstone first, complete — then delete (see the module docstring). + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ + "tile": args.tile, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "kept": sorted(str(p) for p in survivors.values()), + "manifests": manifests, + "benchmarks": benchmarks, + }, indent=2, sort_keys=True) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed: list = [] + prune(args.tile_dir, set(survivors.values()) | {args.tombstone}, removed) + print(f"[clean_tile] {args.tile}: removed {len(removed)} path(s); kept " + f"{len(survivors)} survivor(s) + the tombstone") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py new file mode 100644 index 000000000..cf67363db --- /dev/null +++ b/workflow/scripts/completeness.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +"""The count-floor completeness table — the single failure policy. + +This is the ported ``complete_check`` count table from the v2.0 bash layer +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). It is the *only* +failure policy in the design: there is no 3-class taxonomy and no error-signature +whitelist. A stage is a real failure iff a mandatory runner produced fewer than +its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) +sits between ``floor`` and ``expect`` and is tolerated. + +This file is also the ``check`` CLI — the second half of every rule's shell line +(PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing +onto it, so the check runs — and the verdict is recorded — even when +``shapepipe_run`` failed:: + + rc=0 + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? + completeness.py check exp_mask {output} --log {log} --job-rc "$rc" || rc=1 + exit $rc + +It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory +runner is below its floor OR ``--job-rc`` is nonzero — the verdict is COMPOSED of +the counts and shapepipe_run's own exit status, because a runner can raise after +the counted ones have written their files. + +WHERE it writes the verdict is the whole point, and it is two files with two +different jobs: + + * the LOG (``--log``, the rule's snakemake ``log:``) gets the full verdict on + EVERY run, success or failure — counts against floors, per-runner detail, + scraped failure reasons, ``job_rc`` when nonzero. Snakemake never deletes a + log file, so it survives the failed job that wrote it and is the post-mortem + evidence ``run_report.py`` reads. + * the MANIFEST (the rule's declared ``output:``) gets that same verdict ONLY + when it is a success. Snakemake deletes a failed job's declared output + natively, so nothing here has to unlink anything. + +``.json`` therefore means "this stage succeeded": a resume cannot schedule +a downstream stage on top of a failed one. The removal of a PREVIOUS success's +manifest is snakemake's to do, not this script's, and the one gap that leaves is +a head process SIGKILLed between the job's failure and that deletion — a +success-named manifest then outlives the failure it no longer describes. The +profile's ``rerun-incomplete`` covers the DAG side, and ``run_report.py`` takes +the WORST status across a stage's log and manifest, so the report is right even +in that window. + +Neither file carries wall-clock, and each is rewritten ONLY when its content +changes — identical on-disk state must leave a byte-identical manifest with an +UNMOVED mtime, or the mtime rerun-trigger churns the cone on every unrelated +``--forcerun``. + +Per-runner fields: + expect nominal file count for a fully complete unit (report yardstick) + floor the fail-loud minimum (below this the job exits nonzero) + warn if True the runner never fails the unit at all (bash ``:warn`` — + e.g. psfex_interp on tiles missing some epochs) + subpath count files in ``/output//`` instead of + ``/output/`` (bash ``:rand_split`` — setools split cats) + +Counts are file counts in the runner's output dir, matching the bash +``ls / | wc -l`` semantics (broken symlinks excluded by the caller). +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +# stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +COMPLETENESS = { + # --- tile prepare (phase A) --- + # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 + # for the canfar vos flavor; the nibi symlink configs produce one file per + # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), + # verified against the p3-batch1 baseline tree (100 files / 50 tiles). + "tile_get_images": {"get_images_runner": dict(expect=2, floor=2)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, + + # --- exposure chain --- + "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, + "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, + "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree + # AND the bash baseline (both 120/exposure). + "exp_psf": { + "sextractor_runner": dict(expect=120, floor=2), + "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), + "psfex_runner": dict(expect=80, floor=2), + "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + }, + + # --- tile post --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, + "tile_vignets": { + "psfex_interp_runner": dict(expect=1, floor=1), + "vignetmaker_runner_run_1": dict(expect=1, floor=1), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern + # (expect=floor: all-or-nothing, every vignette feeds ngmix). + "vignetmaker_runner_run_2": dict(expect=5, floor=5), + }, + "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1, floor=1)}, +} + + +def count_products(run_dir, runner, spec): + """Count files in ``run_dir//output[/]/`` (live links only). + + scandir, not iterdir: the dirent already says whether an entry is a symlink, + so only the symlinks need the follow-stat that drops dead links. A plain + ``p.exists()`` per entry stats every one of them, and at DR6 scale this runs + once per runner per job over directories of tens to hundreds of files on a + network filesystem. + """ + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + n = 0 + try: + with os.scandir(out) as entries: + for e in entries: + # A dead symlink is the one thing that must not count (the bash + # `ls | wc -l` semantics this ports counted live files only), and + # a symlink is the only entry that can be dead — so it is the + # only one worth a follow-stat. + if not e.is_symlink() or os.path.exists(e.path): + n += 1 + except OSError: + return 0 + return n + + +def check_floor(stage, run_dir): + """Return (ok, details). ok is False iff a mandatory runner is below floor. + + ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. + """ + table = COMPLETENESS[stage] + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + details.append((runner, n, spec["floor"], spec["expect"], + spec.get("warn", False))) + if not spec.get("warn", False) and n < spec["floor"]: + ok = False + return ok, details + + +# --- where a stage writes ------------------------------------------------- +# +# stage -> (level, run_sp_ dir under $SP_RUN/output/). These are the +# committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so +# the check never resolves a run-log. The ngmix entry interpolates the same env +# var its config does, so chunk K's check looks at chunk K's dir. +# +# EVERY ENTRY HERE (and in COMPLETENESS above) HAS A RULE. The table used to +# carry two stages that did not: `tile_mask` (run_sp_tile_Ma, mask_runner 1/1) +# and `tile_detect_uc` (run_sp_tile_Uc). The committed config chain is the +# "sx_nomask" tile_detect variant and no tile-mask config was committed, so both +# were unreachable — tile.smk's docstring is where the masked variant is argued, +# and it is a config plus a rule plus these two rows, added back together. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +# --- failure reasons ------------------------------------------------------ + +# Lines worth showing a human who asks "why is this runner short?". Deliberately +# crude: the point is a pointer into the logs, not a taxonomy (there is no error +# whitelist in this design — the count floor is the policy). +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 # logs are per-CCD; a handful is enough to characterise +MAX_TAIL_LINES = 120 # per file +MAX_REASONS = 3 # per runner + + +def _normalise(line: str) -> str: + """Collapse a log line to its shape, so 40 per-CCD copies dedupe to one.""" + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) # paths differ per CCD + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + """Best-effort, bounded: distinct error-looking lines from a runner's logs. + + Two sources, in order of usefulness: the runner's per-process worker logs + (``/logs/process-*.log`` — where the module's own exception lands), + and the stage's ``logs/log_sp.log`` (where ShapePipe records its error + tally). Sorted, truncated, deduped by shape — a manifest must stay + byte-stable for a given tree. + """ + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +# --- manifest ------------------------------------------------------------- + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + """Count, classify and (on shortfall) scrape. Returns (manifest, ok). + + Stages absent from the table fall back to the zero-output floor: any product + anywhere under the stage dir passes, nothing at all fails. + """ + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, + "level": level, + "unit": unit, + "run_dir": str(run_dir), + "stage_dir": str(stage_dir), + "runners": {}, + "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "floor": 1, + "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_floor(stage, stage_dir) + short = False + for runner, n, floor, expect, warn in details: + below = n < floor + if n < expect: + short = True + manifest["runners"][runner] = { + "found": n, "expect": expect, "floor": floor, "warn": warn, + "status": ("complete" if n >= expect else + "warn" if (warn or not below) else "below_floor"), + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "floor": floor, "expect": expect, + "warn": warn, "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def write_if_changed(path: Path, text: str) -> None: + """Write only when the bytes differ — see the module docstring on mtime.""" + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != text: + path.write_text(text) + + +def _unit_from_run_dir(run_dir): + """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). + + NOT ``SP_UNIT_NUM``, which carries ShapePipe's dashed numbering form + (``-210-282``) and would put ``210-282`` in the manifest — a key that joins + to nothing. ``run_report`` keys units on the store directory name, which is + exactly this basename, so the two now agree. + """ + return Path(str(run_dir)).name or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="ShapePipe per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--log", type=Path, required=True, + help="the rule's log: path — the verdict is written here every " + "run, success or failure") + c.add_argument("--run-dir", type=Path, default=None, + help="the unit's $SP_RUN (default: the env var)") + c.add_argument("--unit", default=None, + help="override the unit ID (default: basename of $SP_RUN)") + c.add_argument("--stage-dir", default=None, + help="override the run_sp_* subdir (default: the stage table)") + c.add_argument("--job-rc", type=int, default=0, + help="shapepipe_run's exit status, composed into the verdict") + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_run_dir(run_dir) + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + + # The verdict is COMPOSED of two independent statements: the count floors + # (above) and shapepipe_run's own exit status (here). Counts alone are not + # enough — a runner can raise AFTER the counted runners have written their + # files, so the floors are met while the job died. Without this, such a job + # publishes a SUCCESS manifest that snakemake then deletes as a failed job's + # output — and the log would claim "complete" for a stage with no manifest, + # which reads as a bookkeeping bug rather than as the failure it is. + # + # Recorded only when nonzero, which keeps every existing success manifest + # byte-identical (the mtime rerun-trigger reads those bytes). + if args.job_rc != 0: + ok = False + manifest["status"] = "failed" + manifest["job_rc"] = args.job_rc + manifest["failures"].append({ + "runner": "shapepipe_run", "found": 0, "floor": 1, "expect": 1, + "warn": False, + "reasons": [f"shapepipe_run exited {args.job_rc} " + f"(counts above floor)"], + }) + + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + + # The log ALWAYS gets the verdict; the manifest gets it only on success. No + # unlink of anything: snakemake deletes a failed job's declared output, so + # the presence of .json IS the statement "this stage succeeded" and + # the DAG can never build on top of a failure. A failure->success transition + # publishes the manifest; a success->failure has the manifest removed for us, + # and the log is overwritten with the new verdict either way. + write_if_changed(args.log, text) + if ok: + write_if_changed(args.manifest, text) + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} + print(f"[completeness] {runner}: {r['found']}/{r['expect']} " + f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) + print(f"[completeness] {args.stage} {unit}: {manifest['status']} " + f"-> {args.log}" + (f" + {args.manifest}" if ok else ""), file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py new file mode 100644 index 000000000..bbd7e0bb9 --- /dev/null +++ b/workflow/scripts/container.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Manage this user's copy of the ShapePipe container image. + +Two layers, and the second only exists when you ask for one: + +* the **SIF** (``~/.cache/shapepipe/shapepipe.sif``) -- a pristine, read-only + copy of the published image, pulled into your own cache. Per-user by + construction: one file, one owner, nobody else's refresh moves the ground + under a running job. +* an optional **sandbox** (``~/.cache/shapepipe/sandbox/``) -- the same image + unpacked into a writable directory, so a ``pip install`` inside it sticks. + The escape hatch for work that needs a package the image does not carry yet. + +Resolution order, shared by this CLI and by the workflow: **sandbox if it +exists, else the cached SIF if it exists, else the ``container:`` path in +workflow/config.yaml**. That last one is the current shared /project image, so +a checkout with an empty cache behaves exactly as it did before this verb +existed, and a package installed into your sandbox is there for your workflow +jobs too. + +Subcommands, exposed as ``sp container ``:: + + sp container pull # fetch the tag into the cache + sp container status # what is here, and how current it is + sp container sandbox # unpack the SIF into a writable dir + sp container exec # run something inside it + sp container exec --writable # ... with writes that persist + +``pull`` needs the network. Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an ``salloc`` allocation -- never +from a batch job. + +Deliberately **stdlib-only**: it runs on the bare host, outside the container, +where the science stack is not installed, and so must import without it. +""" + +import argparse +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + + +class ContainerError(Exception): + """A misconfiguration the caller must fix (bad override, no image at all). + + Raised rather than ``sys.exit``ed so the Snakefile, which imports this + module at parse time, can turn it into a WorkflowError instead of a + SystemExit. ``main`` below turns it back into a one-line CLI error. + """ + +# The published image. CI pushes one tag per branch, sanitized; `-runtime` is +# the slim variant the workflow runs. +CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" + +# The single source of truth for the fallback image: the workflow's own +# `container:` key, which is also what the Snakefile reads. Written down once, +# here, so the CLI and the workflow cannot disagree about the default. +CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" +CONFIG_KEY = "container" + +# The profile whose `apptainer-args:` every workflow job runs under. `exec` reads +# it at runtime rather than restating it, so a one-off `sp container exec` and a +# job see the same environment (the PYTHONPATH pin above all: a divergence there +# means the one-off imports a different src/ than the workflow does). +PROFILE_FILE = Path(__file__).resolve().parents[2] / "profiles" / "nibi" / "config.yaml" + +# ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto +# a filesystem with room), XDG_CACHE_HOME moves it with everything else. +CACHE_DIR = Path( + os.environ.get("SP_CACHE_DIR") + or Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "shapepipe" +).expanduser() + +# This user's pristine image. Override with ``SP_CONTAINER`` (absolute path). +DEFAULT_SIF = CACHE_DIR / "shapepipe.sif" + +# The optional writable unpacking of it. Override with ``SP_SANDBOX``. +DEFAULT_SANDBOX = CACHE_DIR / "sandbox" + +# Bind mounts for `exec`, matching the nibi profile's apptainer-args (the two +# cluster filesystems this workflow reads and writes, plus the home that holds +# ~/.ssl/cadcproxy.pem). Override wholesale with ``SP_APPTAINER_BINDS``. +DEFAULT_BINDS = "/project,/scratch,/home" + + +def configured_default(): + """Return the ``container:`` path from workflow/config.yaml, or ``None``. + + A deliberately minimal scalar read (the same one bin/sp does in sed): this + module is stdlib-only, so there is no yaml to import. + """ + try: + text = CONFIG_FILE.read_text() + except OSError: + return None + match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) + return match.group(1) if match else None + + +def profile_apptainer_args(): + """Return the profile's ``apptainer-args`` as a token list, or ``[]``. + + Minimal scalar read again (stdlib-only); a missing or unreadable profile + yields ``[]``, which callers replace with their own defaults. + """ + try: + text = PROFILE_FILE.read_text() + except OSError: + return [] + match = re.search(r'^apptainer-args:[ \t]*"(.*)"[ \t]*$', text, re.MULTILINE) + return shlex.split(match.group(1)) if match else [] + + +def local_sif(): + """Return this user's cached image path (may not exist yet).""" + override = os.environ.get("SP_CONTAINER") + return (Path(override) if override else DEFAULT_SIF).expanduser() + + +def local_sandbox(): + """Return this user's writable sandbox directory (may not exist).""" + override = os.environ.get("SP_SANDBOX") + return (Path(override) if override else DEFAULT_SANDBOX).expanduser() + + +def resolve_image(): + """Return ``(path, kind)`` for the image everything should run. + + ``kind`` is ``"sandbox"``, ``"sif"``, ``"configured"`` (the shared + /project image named in config.yaml -- the default when the cache is + empty) or ``"none"``. + """ + sandbox = local_sandbox() + if sandbox.is_dir(): + return str(sandbox), "sandbox" + sif = local_sif() + if sif.exists(): + return str(sif), "sif" + # An override that resolves to nothing is a typo, never an intention: falling + # through to the shared image would run the job against something other than + # what the user named. + if os.environ.get("SP_CONTAINER"): + raise ContainerError( + f"SP_CONTAINER={os.environ['SP_CONTAINER']} does not exist " + f"(resolved to {sif}). Unset it, or point it at an image that does." + ) + default = configured_default() + if default: + return default, "configured" + return "", "none" + + +def image_labels(image): + """Return the image's OCI labels as a dict, or ``{}`` if unreadable. + + Never raises: a missing file, a missing ``apptainer`` or a corrupt image + all mean "we don't know", which every caller treats as non-fatal. + """ + path = Path(image) + if not path.exists() or shutil.which("apptainer") is None: + return {} + try: + out = subprocess.run( + ["apptainer", "inspect", "--labels", str(path)], + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return {} + if out.returncode != 0: + return {} + labels = {} + for line in out.stdout.splitlines(): + key, sep, value = line.partition(":") + if sep: + labels[key.strip()] = value.strip() + return labels + + +def image_revision(image): + """Return the commit the image was built from, or ``None``.""" + return image_labels(image).get("org.opencontainers.image.revision") + + +def _require_apptainer(): + """Exit unless ``apptainer`` is on PATH (bin/sp loads the module).""" + if shutil.which("apptainer") is None: + sys.exit("apptainer is not on PATH (bin/sp loads apptainer/1.4.5)") + + +def _git(*args, cwd=None): + """Run git, returning stripped stdout or ``None`` on any failure.""" + try: + out = subprocess.run( + ["git", *args], capture_output=True, text=True, cwd=cwd, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + +def compare_revision(revision, repo=None): + """Place an image revision relative to this checkout's HEAD. + + One of ``"in-sync"``, ``"behind"`` (the image predates HEAD), ``"ahead"``, + ``"diverged"``, or ``"unknown"`` (no label, no git, or a commit this clone + has never fetched). + """ + if not revision: + return "unknown" + repo = repo or Path(__file__).resolve().parents[2] + head = _git("rev-parse", "HEAD", cwd=repo) + if head is None: + return "unknown" + if head == revision: + return "in-sync" + if _git("cat-file", "-e", f"{revision}^{{commit}}", cwd=repo) is None: + return "unknown" + if _git("merge-base", "--is-ancestor", revision, head, cwd=repo) is not None: + return "behind" + if _git("merge-base", "--is-ancestor", head, revision, cwd=repo) is not None: + return "ahead" + return "diverged" + + +def cmd_pull(args): + """Pull ``--tag`` into the cache, atomically.""" + _require_apptainer() + sif = local_sif() + sif.parent.mkdir(parents=True, exist_ok=True) + # Pull to a sibling temp name and rename: an atomic rename within one + # directory, so an in-flight job sees either the whole old image or the + # whole new one. Pulling in place leaves the file half-written for the many + # minutes the pull takes. Jobs already running hold the old inode open. + tmp = sif.with_name(sif.name + f".pull.{os.getpid()}") + print(f"pulling {args.tag}\n -> {sif}") + try: + subprocess.run( + ["apptainer", "pull", "--force", "--name", str(tmp), args.tag], check=True + ) + os.replace(tmp, sif) + except subprocess.CalledProcessError as exc: + tmp.unlink(missing_ok=True) + sys.exit(f"pull failed ({exc.returncode}); {sif} is unchanged") + except KeyboardInterrupt: + tmp.unlink(missing_ok=True) + raise + labels = image_labels(sif) + print(f"revision: {labels.get('org.opencontainers.image.revision', 'unknown')}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + return 0 + + +def cmd_sandbox(args): + """Unpack the image into a writable directory -- the opt-in escape hatch.""" + _require_apptainer() + sandbox = local_sandbox() + if sandbox.exists() and not args.force: + sys.exit( + f"sandbox already exists at {sandbox}\n" + "pass --force to discard it and rebuild from a clean image" + ) + source = args.source + if not source: + image, kind = resolve_image() + if kind == "sandbox": + # Rebuilding from the sandbox itself would just re-copy the drift. + image = str(local_sif()) if local_sif().exists() else ( + configured_default() or CONTAINER_URI + ) + source = image or CONTAINER_URI + sandbox.parent.mkdir(parents=True, exist_ok=True) + print(f"building sandbox from {source}\n -> {sandbox}") + # Build beside the target and swap it in, as `pull` does -- and for a + # sharper reason. A half-written .sif fails loudly, but a half-unpacked + # sandbox *directory* is still a directory, so resolve_image() would elect + # it and every job would silently run a broken tree. Staging also means a + # --force rebuild that fails leaves the sandbox you already had intact. + # + # `--fix-perms` so the tree can be deleted again later. No `--fakeroot`: an + # unprivileged build from an existing image goes through user namespaces, + # which is what the Alliance clusters provide. + staging = sandbox.with_name(f"{sandbox.name}.build.{os.getpid()}") + shutil.rmtree(staging, ignore_errors=True) + try: + subprocess.run( + ["apptainer", "build", "--sandbox", "--fix-perms", str(staging), source], + check=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"sandbox build failed ({exc.returncode}); {sandbox} is unchanged") + except (KeyboardInterrupt, OSError): + shutil.rmtree(staging, ignore_errors=True) + raise + + if sandbox.exists(): + print(f"replacing {sandbox}") + shutil.rmtree(sandbox, ignore_errors=True) + if sandbox.exists(): + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"could not remove {sandbox}; remove it by hand and retry") + os.replace(staging, sandbox) + print( + "\nthis sandbox now takes precedence over the SIF everywhere, including " + "workflow jobs.\ninstall into it with: sp container exec --writable pip " + "install \nreset to a clean image with: sp container pull && " + "sp container sandbox --force" + ) + return 0 + + +def cmd_status(args): + """Report which image layer is live, its revision, and how current it is.""" + sif = local_sif() + sandbox = local_sandbox() + # status is the verb you run WHEN something is wrong, so a broken override is + # reported here rather than raised. + try: + active, kind = resolve_image() + except ContainerError as exc: + print(f"active: NONE -- {exc}") + return 1 + + if sif.exists(): + print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") + else: + print(f"SIF: absent ({sif})") + if sandbox.is_dir(): + print(f"sandbox: {sandbox} (writable; may carry local modifications)") + else: + print("sandbox: none") + print(f"configured: {configured_default() or 'unset'} ({CONFIG_FILE})") + + if kind == "none": + print("\nactive: NONE -- no cached image and no container: in config.yaml") + print(f"run: sp container pull --tag {CONTAINER_URI}") + return 1 + + print(f"\nactive: {active} ({kind})") + if kind == "configured" and not Path(active).exists(): + print(" WARNING: that path does not exist on this host") + return 1 + + labels = image_labels(active) + revision = labels.get("org.opencontainers.image.revision") + source = "" + if revision is None and kind == "sandbox" and sif.exists(): + # Some sandbox trees do not carry the original labels through. The SIF + # beside it is the best remaining evidence of what it was built from -- + # a guess, so it is labelled as one rather than printed as fact. + revision = image_revision(sif) + if revision: + source = " (inferred from the SIF beside it, not read from the sandbox)" + print(f"revision: {revision or 'unknown'}{source}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + if kind == "sandbox": + print( + " (that revision is what the sandbox was built from; " + "anything\n installed into it since is in no label)" + ) + verdict = compare_revision(revision) + explain = { + "in-sync": "matches this checkout's HEAD", + "behind": ( + "older than this checkout's HEAD -- `sp container pull` fetches " + f"{CONTAINER_URI}, which is not built from this branch unless you " + "pass --tag" + ), + "ahead": "newer than this checkout's HEAD", + "diverged": "on a different branch from this checkout", + "unknown": "cannot compare (no label, or a commit this clone lacks)", + }[verdict] + print(f"checkout: {verdict} ({explain})") + return 0 + + +def cmd_exec(args): + """Run a command inside the resolved image -- the one-off path.""" + _require_apptainer() + command = [a for a in args.command if a != "--"] + if not command: + sys.exit("nothing to run; pass a command after `exec`") + + # Same environment the workflow's jobs get: the profile's apptainer-args + # verbatim (--cleanenv, the PYTHONPATH pin, --home, its binds). Explicit + # binds still win, and a profile that cannot be read falls back to the old + # standalone defaults. + env_args = profile_apptainer_args() + explicit_binds = args.bind or os.environ.get("SP_APPTAINER_BINDS") + if not env_args: + print( + f"warning: could not read apptainer-args from {PROFILE_FILE}; " + "falling back to --cleanenv and the default binds, which may not " + "match what jobs run under", + file=sys.stderr, + ) + env_args = ["--cleanenv"] + explicit_binds = explicit_binds or DEFAULT_BINDS + if explicit_binds: + env_args += ["--bind", explicit_binds] + + if args.writable: + # A SIF is a read-only filesystem, so `--writable` against one fails + # obscurely; only a sandbox takes writes. + sandbox = local_sandbox() + if not sandbox.is_dir(): + sys.exit( + f"--writable needs a sandbox, and there is none at {sandbox}\n" + "build one with: sp container sandbox" + ) + image, extra = str(sandbox), ["--writable"] + else: + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + extra = [] + + cmd = ["apptainer", "exec", *extra, *env_args, image, *command] + return subprocess.run(cmd).returncode + + +def cmd_resolve(args): + """Print just the resolved image path -- what the Snakefile consumes.""" + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + print(image) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="sp container", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="subcommand", required=True) + + p_pull = sub.add_parser( + "pull", + help="fetch the image into the cache (login node or salloc: compute " + "nodes have no network)", + ) + p_pull.add_argument( + "--tag", default=CONTAINER_URI, help=f"image to pull (default: {CONTAINER_URI})" + ) + p_pull.set_defaults(func=cmd_pull) + + p_status = sub.add_parser( + "status", help="report which image layer is live and how current it is" + ) + p_status.set_defaults(func=cmd_status) + + p_sandbox = sub.add_parser( + "sandbox", help="unpack the image into a writable directory (opt-in)" + ) + p_sandbox.add_argument( + "--source", help="image to unpack (default: the cached SIF, or the config path)" + ) + p_sandbox.add_argument( + "--force", + action="store_true", + help="discard an existing sandbox and rebuild from a clean image", + ) + p_sandbox.set_defaults(func=cmd_sandbox) + + p_exec = sub.add_parser("exec", help="run a command inside the resolved image") + p_exec.add_argument("--bind", help=f"bind mounts (default: {DEFAULT_BINDS})") + p_exec.add_argument( + "--writable", + action="store_true", + help="run against the sandbox so writes (e.g. pip install) persist", + ) + p_exec.add_argument("command", nargs=argparse.REMAINDER) + p_exec.set_defaults(func=cmd_exec) + + p_resolve = sub.add_parser( + "resolve", help="print the resolved image path (what the workflow runs)" + ) + p_resolve.set_defaults(func=cmd_resolve) + + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + try: + return args.func(args) + except ContainerError as exc: + sys.exit(str(exc)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/coverage_map.py b/workflow/scripts/coverage_map.py new file mode 100644 index 000000000..88039a434 --- /dev/null +++ b/workflow/scripts/coverage_map.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Build the campaign's HealSparse coverage (nexp) map from the exposure footprints. + +Run as the shell of the in-DAG ``coverage_map`` rule, never by hand. + +WHAT THE MAP IS. Per sky pixel, the number of exposures with a VALID PSF MODEL +covering it. The CCDs of one exposure do not overlap, so stamping value 1 per CCD +polygon and accumulating counts exposures. sp_validation applies it as a +structural mask (``notebooks/demo_apply_hsp_masks.py``). + +CAMPAIGN-CUMULATIVE, AND THAT IS THE POINT. This script GLOBS every +``/exp/*/*/manifests/exp_footprint.json`` — not just the ones the +rule declared as inputs, and INCLUDING exposures whose scratch stores have been +reclaimed. The declared inputs are the in-scope, non-tombstoned footprints, which +is what buys ordering and rerun semantics without dragging out-of-scope tiles +into the DAG; the records themselves live on the persistent root and stay valid +sky forever. So appending tiles GROWS the map rather than replacing it, which is +what a survey coverage mask should do. The rule's comment says the same thing +where a reader of the DAG will meet it. + +THE STAMPING IS NOT HERE. It is ``shapepipe.utilities.coverage_map_builder. +build_map``, which is where the RA-seam guard, the pole guard and the polygon +accumulation live. This script is the JSON-to-arrays half, and the records are +raw sky: unwrapping across RA=0 happens once, inside build_map, at the moment a +polygon is stamped. + +NSIDE IS NOT DEFAULTED ANYWHERE. Both values are required arguments, carried +from `coverage:` in config.yaml, and the difference they make is invisible in +the output: nside=131072 is ~0.1"/pixel, chosen to match the UNIONS bit-mask +resolution so coverage and mask align pixel-wise. A map built coarser would look +entirely reasonable and would not align, and the consumer would not notice. +""" + +import argparse +import filecmp +import json +import sys +from pathlib import Path + +import numpy as np + +from shapepipe.utilities.coverage_map_builder import build_map + +# Where exp_footprint writes, relative to the products root. The shard and the +# exposure id are both globbed: this map is the whole campaign's, so it is +# deliberately not built from a list of units. +FOOTPRINT_GLOB = "exp/*/*/manifests/exp_footprint.json" + + +def read_footprints(products_dir): + """Every exposure footprint on the persistent root, as flat arrays. + + Returns ``(ccd_ids, ra, dec, units)``: length-M id and ``(M, 4)`` corner + arrays over all CCDs of all exposures, plus the sorted exposure ids that + contributed. Records are read in sorted path order so the polygon order — + and hence the map — does not depend on readdir order. + """ + paths = sorted(Path(products_dir).glob(FOOTPRINT_GLOB)) + if not paths: + sys.exit(f"coverage_map: no {FOOTPRINT_GLOB} under {products_dir}; " + f"there is nothing to build a map from") + + ccd_ids, ra, dec, units = [], [], [], [] + for path in paths: + body = json.loads(path.read_text()) + units.append(body["unit"]) + for ccd in body["ccds"]: + ccd_ids.append(ccd["id"]) + ra.append(ccd["ra"]) + dec.append(ccd["dec"]) + + # Records but no CCDs is the other empty map, and it is the quieter one: it + # means every exposure on the root lost every CCD, which is a broken PSF + # stage rather than a survey with no coverage. Written out it would be a + # valid, plausible-looking .hsp full of nothing, and its consumer masks + # everything. + if not ccd_ids: + sys.exit(f"coverage_map: {len(paths)} footprint record(s) under " + f"{products_dir}, and not one names a CCD with a PSF model; " + f"there is nothing to stamp") + + return (np.array(ccd_ids, dtype=str), + np.array(ra, dtype=float).reshape(-1, 4), + np.array(dec, dtype=float).reshape(-1, 4), + sorted(units)) + + +def write_stable(path, body): + """Write JSON tmp-then-``cmp``-then-``mv``; an unchanged body keeps its mtime.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + try: + tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + if path.exists() and filecmp.cmp(tmp, path, shallow=False): + tmp.unlink() + else: + tmp.replace(path) + finally: + tmp.unlink(missing_ok=True) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--products-dir", required=True, type=Path, + help="the persistent root; every exposure footprint under " + "it goes into the map") + p.add_argument("--out", required=True, type=Path, + help="the HealSparse map, /coverage/coverage.hsp") + p.add_argument("--manifest", required=True, type=Path) + p.add_argument("--nside-coverage", required=True, type=int) + p.add_argument("--nside", required=True, type=int) + p.add_argument("--verbose", action="store_true") + args = p.parse_args() + + ccd_ids, ra, dec, units = read_footprints(args.products_dir) + print(f"[coverage_map] {len(units)} exposure(s), {len(ccd_ids)} CCD " + f"footprint(s) from {args.products_dir}") + + hsp_map = build_map(ccd_ids, ra, dec, args.nside_coverage, args.nside, + verbose=args.verbose) + + args.out.parent.mkdir(parents=True, exist_ok=True) + hsp_map.write(str(args.out), clobber=True) + + # The manifest is written AFTER the map, and it is the rule's record of + # which exposures the map contains — the question a mask's consumer asks + # months later, and one nothing else on disk can answer once the campaign + # has grown past it. + write_stable(args.manifest, { + "stage": "coverage_map", "level": "campaign", "status": "complete", + "map": str(args.out), + "nside_coverage": args.nside_coverage, + "nside": args.nside, + "n_exposures": len(units), + "n_ccds": len(ccd_ids), + "exposures": units, + }) + print(f"[coverage_map] -> {args.out}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/exp_footprint.py b/workflow/scripts/exp_footprint.py new file mode 100644 index 000000000..d4573620b --- /dev/null +++ b/workflow/scripts/exp_footprint.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Record ONE exposure's per-CCD sky footprint, for the CCDs that have a PSF model. + +Run as the shell of the in-DAG ``exp_footprint`` rule, never by hand. + +WHAT THIS REPLACES. The v1.x coverage chain answered "which CCDs have a valid +PSF, and where are they on the sky" by downloading ~25k exposure headers from +VOSpace (``header_downloader.py``), scraping a finished campaign's patch +directories for a ``missing_job_32_all.txt`` and SUBTRACTING it from all 40*N +candidates (``ccd_psf_handler.py``), then appending one row per CCD to a shared +``exp_ra_dec.txt``. Every one of those three moves exists because the pre- +Snakemake pipeline kept no per-unit record. The workflow keeps two, both already +on disk, so this rule is a local read of things it produced itself. + +THE TWO INPUTS, AND WHY EACH IS THE RIGHT ONE. + +1. WHICH CCDS HAVE A PSF -> ``exp_persist.json``'s ``files[].name``, the + ``validation_psf--.fits`` members of the persisted tar. This is + EXACT, not inferred: ``psfex_interp`` returns WITHOUT writing that file on + NOT_ENOUGH_STARS, BAD_CHI2 and FILE_NOT_FOUND, and writes it on success, so + the member list IS the valid-PSF set. It is also DURABLE (persistent root, + survives ``clean_exposure``) and a DECLARED RULE OUTPUT, so the DAG orders + this rule after it for free. The alternative considered and rejected was + ``exp_psf.json``'s psfex_interp entry: that is a COUNT, not a list of names, + and it lives inside the directory ``clean_exposure`` deletes wholesale. + +2. WCS -> ``headers-.npy``, written by ``split_exp`` beside the per-CCD + images. A length-N ``dtype=object`` array; element ``i`` is + ``{"WCS": WCS(h), "header": h.tostring()}``, loaded with ``allow_pickle=True`` + exactly as ``merge_headers.py`` does. + +THE LOAD-BEARING INVARIANT, PINNED BY tests/unit/test_exp_footprint.py: +array index ``i`` == the CCD index in ``image--.fits`` == ``-`` +== ``validation_psf--.fits``. ``split_exp`` writes both the image and the +array element from the same ``idx-1`` in one loop, so the alignment is +structural — but it is a cross-component contract (split_exp's numbering vs. +psfex_interp's filenames vs. this record's ids), and nothing else asserts it now +that the old ``summary.get_all_shdus`` test is gone. No index is ever re-derived +here: a CCD's id comes from its position in the array, never from parsing a name. + +WHY THE SHAPE COMES FROM THE STORED HEADER AND NOT FROM THE WCS. astropy hands +back the DECOMPRESSED header for a tile-compressed HDU, so this npy carries true +``NAXIS1/2`` and no ``ZIMAGE`` — while the old VOSpace text headers carried the +binary-table ``NAXIS`` and needed ``ZNAXIS1/2``. ``_image_shape`` handles both, +and is imported rather than reimplemented for exactly that reason. ``WCS`` drops +the ``Z*`` keywords, so it cannot answer the question either way. + +DATA AND MANIFEST IN ONE FILE. 40 rows of 8 floats is a few KB, and inodes are +what bind on /project (persist_exp.py argues the quota arithmetic). Per-exposure +JSON, never an appended shared text file: a single appended file is precisely +what cannot survive 20k parallel jobs, and it is what the v1.x chain used. +``ccds_no_psf`` is carried explicitly so the record is self-describing about +attrition — a reader can tell "this CCD is not in the map" from "this CCD was +never looked at". + +Written byte-stable, no timestamp, tmp-then-``cmp``-then-``mv``, exactly as +``persist_exp.py`` does and for the same reason: mtime is a rerun trigger, and an +unconditional rewrite would make ``clean_exposure`` look out of date once per +invocation. +""" + +import argparse +import filecmp +import json +import sys +from fnmatch import fnmatch +from pathlib import Path + +import numpy as np +from astropy.io.fits import Header + +from shapepipe.utilities.ccd_footprint import ( + _ccd_corners, + _image_shape, +) + +# The split stage's run dir (RUN_NAME in config_exp_Sp.ini) and its module +# output dir. Hardcoded rather than passed, for persist_exp.py's reason: this +# rule reads the split stage's headers and nothing else, and a knob here would +# be a knob for "read some other stage". +HEADERS_DIR = "output/run_sp_exp_Sp/split_exp_runner/output" + +# The name psfex_interp gives a CCD's PSF model at the validation positions. +# Only the PREFIX is ours to know; the rest of the name is the unit id and the +# CCD index, which is what makes the set a set of indices. +PSF_PREFIX = "validation_psf" + + +def valid_ccds(manifest, exp): + """The CCD indices with a PSF model, read off an ``exp_persist`` manifest. + + Raises if the manifest was written by a keep list that never asked for the + PSF files: an empty answer would then be indistinguishable from an exposure + that genuinely lost every CCD, and the coverage map would silently lose a + whole exposure. The Snakefile guards the same precondition at parse time; + this is the per-unit half, and it is the one that sees the manifest. + """ + body = json.loads(Path(manifest).read_text()) + patterns = body.get("patterns") or [] + # Does the keep list ask for these files AT ALL? Asked by matching a + # hypothetical member name rather than by string-matching the pattern, so + # any spelling that would have packed them counts. + probe = f"{PSF_PREFIX}-{exp}-0.fits" + if not any(fnmatch(probe, pat) for pat in patterns): + sys.exit( + f"exp_footprint: {exp}: {manifest} was written by a keep list " + f"({patterns}) that packs no {PSF_PREFIX} files, so it names no " + f"valid-PSF set. Add a pattern matching {probe} to `persist_exp:` " + f"in config.yaml and rerun exp_persist.") + + ccds = set() + for entry in body.get("files") or []: + name = entry.get("name", "") + if not name.startswith(f"{PSF_PREFIX}-{exp}-"): + continue + stem = name[len(f"{PSF_PREFIX}-{exp}-"):].removesuffix(".fits") + if not stem.isdigit(): + sys.exit(f"exp_footprint: {exp}: cannot read a CCD index out of " + f"tar member {name!r}") + ccds.add(int(stem)) + return ccds + + +def footprint(headers, exp, with_psf): + """One record's ``ccds`` and ``ccds_no_psf``, in array order. + + ``headers`` is the ``headers-.npy`` array; index ``i`` IS the CCD + index (see the module docstring). Corners are computed only for the CCDs in + ``with_psf`` — a CCD with no PSF model contributes nothing to a map that + counts exposures with a valid PSF, and computing its corners anyway would + invite a later reader to use them. + """ + ccds, no_psf = [], [] + for i, entry in enumerate(headers): + ccd_id = f"{exp}-{i}" + if i not in with_psf: + no_psf.append(ccd_id) + continue + # The WCS is the pickled object split_exp built; the SHAPE must come + # from the stored header text (module docstring). + shape = _image_shape(Header.fromstring(entry["header"])) + ra, dec = _ccd_corners(entry["WCS"], shape) + # float(), because _ccd_corners hands back numpy scalars and this record + # has to be byte-stable: a plain double's repr is, a numpy type's + # serialisation is json's business rather than ours. + ccds.append({"id": ccd_id, + "ra": [float(x) for x in ra], + "dec": [float(x) for x in dec]}) + return ccds, no_psf + + +def write_stable(path, body): + """Write JSON tmp-then-``cmp``-then-``mv``; an unchanged body keeps its mtime.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + try: + tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + if path.exists() and filecmp.cmp(tmp, path, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(path) + finally: + tmp.unlink(missing_ok=True) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path, + help="the exposure's scratch store") + p.add_argument("--exp", required=True) + p.add_argument("--persist-manifest", required=True, type=Path, + help="/exp///manifests/" + "exp_persist.json; names the valid-PSF CCDs") + p.add_argument("--manifest", required=True, type=Path) + args = p.parse_args() + + npy = args.exp_dir / HEADERS_DIR / f"headers-{args.exp}.npy" + if not npy.exists(): + sys.exit(f"exp_footprint: {args.exp}: no WCS array at {npy} — the " + f"split stage's store is gone or never wrote one " + f"(config_exp_Sp.ini's OUTPUT_SUFFIX must include `image`).") + headers = np.load(npy, allow_pickle=True) + + with_psf = valid_ccds(args.persist_manifest, args.exp) + # A PSF file for a CCD the split never produced means the two stages + # disagree about the focal plane — the one failure the id alignment cannot + # absorb, so it is loud rather than silently dropped. + stray = sorted(i for i in with_psf if i >= len(headers)) + if stray: + sys.exit(f"exp_footprint: {args.exp}: {args.persist_manifest} names " + f"CCD(s) {stray} but {npy} holds only {len(headers)}") + + ccds, no_psf = footprint(headers, args.exp, with_psf) + + write_stable(args.manifest, { + "stage": "exp_footprint", "level": "exp", "unit": args.exp, + "status": "complete", + "source_manifest": str(args.persist_manifest), + "n_ccd_headers": len(headers), + "n_valid_psf": len(ccds), + "ccds": ccds, + "ccds_no_psf": no_psf, + }) + + warn = f" ({len(no_psf)} without a PSF model)" if no_psf else "" + print(f"[exp_footprint] {args.exp}: {len(ccds)}/{len(headers)} CCD " + f"footprint(s){warn} -> {args.manifest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py new file mode 100644 index 000000000..a6879c504 --- /dev/null +++ b/workflow/scripts/ngmix_range.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Materialise the ngmix chunk partition once per tile, then serve it. + +TWO MODES, ONE WRITER. The partition is computed ONCE, by tile_vignets — the +first member of the fused tile_shape group — and written to the group's +node-local scratch; each of the n_chunks chunk shells then only LOOKS UP its +own row:: + + # tile_vignets, once (see tile.smk's TILE_NGMIX_RANGES) + ngmix_range.py --run-dir $SP_RUN --n-chunks 8 --write $SP_LOCAL/ngmix_ranges.json + + # each chunk, at its own start + eval "$(ngmix_range.py --read $SP_LOCAL/ngmix_ranges.json --chunk 3)" + # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 + +The file is group-internal plumbing, NOT DAG currency: it lives on $SP_LOCAL and +dies with the group job, exactly like the vignette store. It is deliberately not +a rule output — a chunk can only ever read the file its own group job wrote. + +Read mode does NOT fall back to recomputation when the file is absent; it exits +non-zero. The fallback would restore the very failure this design removes (see +DETERMINISM, below). + +The range is still only knowable at EXECUTION time (PRD D4) — a params function +cannot compute it, because params evaluate before the sexcat exists. + +SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering +[1, N] processes every object exactly once. The bounds are CLOSED, never +ID_OBJ_MAX = -1 — ngmix treats ``id_obj_max <= 0`` as unbounded +(``ngmix_package/ngmix.py:804-806``), so an open-ended last chunk silently +re-measures the whole tile instead of its share; rule tile_ngmix carries the +straggler that taught this. + +CHUNKS ARE BALANCED BY EPOCH-WEIGHTED COST, NOT BY OBJECT COUNT. ngmix fits an +object jointly across every exposure it lands on, so its cost scales with +(object, epoch) pairs, not objects: the campaign measured 0.2714 CPU-s per +pair with an intercept consistent with zero, plus ~0.05 CPU-s per-object of +setup. The old equal-count split therefore produced chunks whose cost varied +1.6x WITHIN a single tile, tracking their epoch counts at R^2 0.91-1.000. Those +chunks are siblings in the fused tile_shape group job, which ends when its +SLOWEST member does, so the spread was pure wall clock: 7.7 hours over the +34-tile smk-g4 campaign, and on the worst tile (196.307) the slowest chunk ran +26.3 min past the median and took the group to 97.4% of its wall limit. + +Re-splitting all 34 of that campaign's sexcats: the old boundaries leave the +slowest chunk at 1.131x-1.627x its tile's median predicted cost (worst +200.302), the new ones at 1.0000x-1.0002x, and the sum over tiles of +slowest-chunk cost falls 132,875 -> 113,710 predicted CPU-s, 14.4%. + +Moving the boundaries is scientifically free: ngmix's RNG is seeded per object +from its sky position, so which chunk an object falls in cannot change its +measurement (see ``ngmix_package.ngmix.position_seed``). + +DETERMINISM HERE IS CORRECTNESS, NOT TIDINESS. A tile's chunk ranges are a +PARTITION of its object IDs: if two chunks disagree about the boundaries, +objects are silently measured twice or silently dropped and nothing downstream +notices — merge_sep_cats concatenates whatever it is given. + +SINGLE-WRITER IS WHY THAT NO LONGER DEPENDS ON AGREEMENT. Every chunk reads the +same file, produced by one process from one read of the sexcat, so sibling +chunks cannot disagree even in principle — the boundaries are a fact of the +group job rather than a computation eight processes each have to land on. The +mechanism that closes is a real one: this script used to run independently in +each chunk's shell, and a group holds its chunks open for hours (smk-g4: +6,236-7,762 s of elapsed per chunk), reading — as every job did before the launch +code snapshot (bin/sp) — the LIVE checkout, so an edit landed mid-flight was read +by some chunks and not others. Observed once, live — the transcript is at +tile.smk's range_hash, which is the home for that history. + +The rest of the determinism discipline stays, because it is what makes the +written file trustworthy rather than merely shared: integer arithmetic end to +end (weights scale to milli-epochs so no float ever decides a boundary), and NO +FALLBACK — if the EPOCH extensions cannot be read, or the ranges file is absent +in read mode, this script exits non-zero. A fallback would be the worst +available behaviour precisely because it would apply only to the processes that +hit the failure, shredding the tile's coverage instead of failing it. + +What remains uncovered is the cross-TIME case, and NGMIX_RANGE_HASH (Snakefile) +is what covers it: a RESUME that reruns some chunks after an edit to this script +would otherwise mix old and new boundaries within one tile. +""" + +import argparse +import json +from pathlib import Path + +# Weights are integers in milli-epochs (one epoch = 1000) so every boundary is +# decided by exact integer comparison, identically on every run of the splitter +# — which is now what makes the written file REPRODUCIBLE across attempts and +# resumes, rather than what makes eight siblings agree. +MILLI_EPOCH = 1000 + +# The per-object setup cost, expressed in epochs so one integer weight carries +# both terms of the measured cost law: 0.05 CPU-s of setup / 0.2714 CPU-s per +# (object, epoch) = 0.184 epoch-equivalents. It is what keeps the zero-epoch +# objects (13 of 35,298 on tile 186.307) from weighing nothing — they still +# cost ~6% of a typical object, and a chunk handed thousands of them for free +# would be a straggler of a new kind. Only the RATIO of the two costs moves a +# boundary, which makes this robust: refitting on 186.307's eight measured +# chunk CPU times with the setup term held fixed gives 0.2619 per GEOMETRIC +# epoch (see object_epochs for why that is below 0.2714), i.e. ALPHA 0.191, +# and every boundary on that tile then moves by at most one object. Zeroing +# ALPHA entirely moves them by at most 44. +ALPHA_MILLI_EPOCHS = 184 + + +def id_ranges(epochs, n_chunks: int) -> list[tuple[int, int]]: + """Split object IDs ``1..len(epochs)`` into ``n_chunks`` closed ranges. + + ``epochs[i]`` is object ``i + 1``'s geometric epoch count. The ranges are + CONTIGUOUS — ``NGMIX_ID_MIN``/``NGMIX_ID_MAX`` is an interval, not a set — + and tile ``[1, n_obj]`` exactly, so every object is measured once. + + The objective is the SLOWEST chunk, not the average one, because the + group job waits for it. So this minimises the maximum chunk weight + exactly: binary-search the smallest feasible capacity, then fill left to + right under it. Ties in that maximum break toward the earlier chunks, + which is arbitrary but fixed, and fixed is the property that matters. + + With ``n_obj < n_chunks`` the first ``n_obj`` chunks take one object each + and the remainder are EMPTY, written ``(n_obj + 1, n_obj)``: ``lo = hi+1`` + is the canonical empty closed interval and preserves the chain + ``ranges[k][0] == ranges[k-1][1] + 1``. Deliberately not ``(1, 0)``, which + is what the old equal-count split emitted here — ``ID_OBJ_MAX = 0`` is + ngmix's unbounded sentinel, so each empty chunk would have re-measured the + ENTIRE tile. + """ + if n_chunks < 1: + raise ValueError(f"n_chunks must be >= 1, got {n_chunks}") + n_obj = len(epochs) + if n_obj < 1: + raise ValueError("cannot split a catalogue of zero objects") + + if n_obj < n_chunks: + return ([(k, k) for k in range(1, n_obj + 1)] + + [(n_obj + 1, n_obj)] * (n_chunks - n_obj)) + + weights = [MILLI_EPOCH * int(e) + ALPHA_MILLI_EPOCHS for e in epochs] + cap = _min_feasible_capacity(weights, n_chunks) + + ranges: list[tuple[int, int]] = [] + start, load = 0, 0 + for i, w in enumerate(weights): + # Close before object i when the current chunk holds something and + # either it cannot take i under `cap`, or the objects still to come + # (n_obj - i) no longer outnumber the chunks still to open — the second + # clause is what guarantees exactly n_chunks non-empty ranges when a + # heavy head would otherwise leave the tail with nothing to hold. + if start < i and (load + w > cap + or n_obj - i < n_chunks - len(ranges)): + ranges.append((start + 1, i)) + start, load = i, 0 + load += w + ranges.append((start + 1, n_obj)) + return ranges + + +def _chunks_needed(weights: list[int], cap: int) -> int: + """Chunks a left-to-right fill uses when none may exceed ``cap``.""" + used, load = 1, 0 + for w in weights: + if load + w > cap: + used, load = used + 1, w + else: + load += w + return used + + +def _min_feasible_capacity(weights: list[int], n_chunks: int) -> int: + """Smallest ``cap`` that fits ``weights`` into ``n_chunks`` chunks. + + ``_chunks_needed`` is monotone non-increasing in ``cap``, so bisection on + the integers ``[max(weights), sum(weights)]`` lands on the exact optimum + without a float ever entering the comparison. + """ + lo, hi = max(weights), sum(weights) + while lo < hi: + mid = (lo + hi) // 2 + if _chunks_needed(weights, mid) <= n_chunks: + hi = mid + else: + lo = mid + 1 + return lo + + +def object_epochs(run_dir: Path): + """Per-object geometric epoch count, from this tile's own sexcat. + + tile_detect's SExtractor post-process (``MAKE_POST_PROCESS`` in + ``config_tile_Sx.ini``) writes one ``EPOCH_`` extension per exposure + overlapping the tile — so the extension COUNT is tile-specific and is + discovered by name, never assumed — each with ``n_obj`` rows in NUMBER + order and ``CCD_N < 0`` where the object misses that exposure. Summing + ``CCD_N >= 0`` across them reproduces the final catalogue's ``N_EPOCH`` + column exactly — checked row by row against 186.307's + ``run_sp_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, + 116,727 pairs, mean 3.31. The post-process is upstream of the whole + tile_shape group, so the extensions always exist by the time ngmix runs; + their absence is a broken tile, not a case to accommodate. + + N_EPOCH is a GEOMETRIC count and mildly over-states the work, because ngmix + drops epochs it cannot fit. The same catalogue's NGMIX_N_EPOCH is lower for + 2,203 of the 35,298 objects and never higher: 113,947 pairs against + 116,727, 2.4%. That gap is the whole of the ~3.4% by which this cost model + over-predicts the tile's measured chunk times — refit on NGMIX_N_EPOCH the + law is 0.2681 CPU-s per pair at R^2 0.997, against 0.2619 at R^2 0.973 on + the geometric count. The better number is unavailable before ngmix runs and + is not worth wanting anyway: splitting on it moves boundaries by up to 177 + objects and improves the true slowest chunk by 1.07%. + + Only the EPOCH extensions are read. ``LDAC_OBJECTS`` carries a 10 kB + VIGNET per row (376 MB on 186.307) and pulling it in would cost more than + the straggle this split exists to remove; the EPOCH tables are ~530 kB + each, and its NAXIS2 comes from the header alone as the row-count check. + """ + import numpy as np + from astropy.io import fits + + cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not cats: + raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") + with fits.open(cats[0], memmap=True) as hdul: + epoch_hdus = [h for h in hdul if h.name.startswith("EPOCH")] + if not epoch_hdus: + raise SystemExit( + f"[ngmix_range] FATAL: no EPOCH extensions in {cats[0]}; " + "tile_detect's SExtractor post-process did not run. " + "Refusing to fall back to an equal-object split: it would " + "apply to only the chunks that saw this failure, and the " + "tile's objects would be double-measured or dropped." + ) + if "LDAC_OBJECTS" not in hdul: + raise SystemExit( + f"[ngmix_range] FATAL: no LDAC_OBJECTS in {cats[0]}" + ) + n_obj = int(hdul["LDAC_OBJECTS"].header["NAXIS2"]) + expected = np.arange(1, n_obj + 1, dtype=np.int64) + counts = np.zeros(n_obj, dtype=np.int64) + for hdu in epoch_hdus: + data = hdu.data + # NUMBER is asserted, not assumed: the whole scheme is an ID + # INTERVAL, so a permuted or gappy NUMBER column would make the + # weights describe different objects than the bounds select. + if len(data) != n_obj or not np.array_equal( + np.asarray(data["NUMBER"], dtype=np.int64), expected + ): + raise SystemExit( + f"[ngmix_range] FATAL: {cats[0]}[{hdu.name}] is not " + f"{n_obj} rows of NUMBER = 1..{n_obj} in order" + ) + counts += np.asarray(data["CCD_N"]) >= 0 + return counts + + +def partition(epochs, n_chunks: int) -> dict: + """The whole partition, as the dict that gets serialised to JSON. + + Minimal on purpose: the ranges themselves, plus the two numbers a reader + needs to check that the file is the one it expects (``n_obj`` so a mismatch + against the catalogue is visible, ``n_chunks`` so a chunk index is validated + against what was actually written, not against what the caller believes). + ``chunk`` is 1-based, matching SP_NGMIX_CHUNK and the run-directory suffix. + """ + ranges = id_ranges(epochs, n_chunks) + return { + "n_obj": len(epochs), + "n_chunks": n_chunks, + "chunks": [ + {"chunk": k, "id_min": lo, "id_max": hi} + for k, (lo, hi) in enumerate(ranges, start=1) + ], + } + + +def chunk_range(doc: dict, chunk: int) -> tuple[int, int]: + """Chunk ``chunk``'s closed range out of a ``partition()`` document. + + Validated rather than indexed: a negative ``chunk`` would index from the end + and hand this process some other chunk's range without any error. + """ + n_chunks = doc["n_chunks"] + if not 1 <= chunk <= n_chunks: + raise SystemExit( + f"[ngmix_range] FATAL: --chunk {chunk} outside 1..{n_chunks}" + ) + row = doc["chunks"][chunk - 1] + if row["chunk"] != chunk: + raise SystemExit( + f"[ngmix_range] FATAL: ranges file row {chunk - 1} is chunk " + f"{row['chunk']}, not {chunk}" + ) + return int(row["id_min"]), int(row["id_max"]) + + +def read_ranges(path: Path) -> dict: + """Load the ranges file, or die saying what should have written it.""" + try: + return json.loads(path.read_text()) + except FileNotFoundError: + raise SystemExit( + f"[ngmix_range] FATAL: no ranges file at {path}.\n" + " tile_vignets writes it once per tile, on the node-local scratch " + "this group job shares.\n" + " Its absence means tile_vignets did NOT run in this group job " + "(same cause as a missing vignette store).\n" + " FIX: rm the tile's tile_vignets.json and resume.\n" + " NOT recomputed here on purpose: a per-chunk fallback would let " + "chunks disagree about the partition." + ) + + +def main() -> None: + """Write the whole partition, or emit one chunk's range as bash exports.""" + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--write", type=Path, metavar="JSON", + help="compute the whole partition and write it here") + p.add_argument("--read", type=Path, metavar="JSON", + help="look one chunk's range up in a file --write made") + p.add_argument("--run-dir", type=Path, help="--write: the tile's run root") + p.add_argument("--n-chunks", type=int, help="--write: chunks to split into") + p.add_argument("--chunk", type=int, help="--read: 1-based chunk index") + a = p.parse_args() + + if bool(a.write) == bool(a.read): + raise SystemExit( + "[ngmix_range] FATAL: pass exactly one of --write / --read" + ) + + if a.write: + if a.run_dir is None or a.n_chunks is None: + raise SystemExit( + "[ngmix_range] FATAL: --write needs --run-dir and --n-chunks" + ) + doc = partition(object_epochs(a.run_dir), a.n_chunks) + # Temp name plus rename, the same all-or-nothing publish tile_local() + # uses for the WCS store: a reader must never see a half-written file. + tmp = a.write.with_name(f".{a.write.name}.tmp") + tmp.write_text(json.dumps(doc)) + tmp.replace(a.write) + return + + if a.chunk is None: + raise SystemExit("[ngmix_range] FATAL: --read needs --chunk") + lo, hi = chunk_range(read_ranges(a.read), a.chunk) + print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py new file mode 100644 index 000000000..c13fca604 --- /dev/null +++ b/workflow/scripts/persist_exp.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Pack ONE exposure's keepable PSF products into a tar off scratch, and record what went. + +Run as the shell of the in-DAG ``exp_persist`` rule, never by hand. + +WHY A COPY AND NOT AN EXEMPTION FROM CLEANUP. The obvious alternative — teach +``clean_exposure`` to spare these files — does not work, because reclamation is +not what threatens them. The exposure store lives on ``run_dir``, which is +/scratch: a 60-day purge takes everything there whether or not this workflow +ever cleaned it. ``products_dir`` is /project, backed up and not purged. So the +only way a per-exposure product outlives its campaign is to LEAVE THE +FILESYSTEM, and that is a copy. Reclamation ordering then falls out for free: +``clean_exposure`` takes this rule's manifest as an input, so the store is never +deleted before its keepers have been written elsewhere. + +WHY A SEPARATE RULE AND NOT A ``cp`` APPENDED TO ``exp_psf``. The list of what +to keep is a decision that will be revisited — rho statistics want one file +today, a residual study may want three tomorrow — and ``exp_psf`` is four hours +per exposure. The list rides on this rule's ``params``, so editing it makes +snakemake rerun THIS rule (seconds of cp) and leaves the PSF chain alone. Folded +into ``exp_psf``, the same edit would re-derive every PSF model in the campaign. + +WHAT IT SEARCHES. ``/output/run_sp_exp_SxSePsfPi/*/output/`` — the four +module output dirs of the PSF config (sextractor, setools, psfex, psfex_interp) +— RECURSIVELY. The recursion is not laziness: setools does not write flat, it +writes into ``mask/``, ``rand_split/``, ``new_cat/``, ``plot/`` and ``stat/`` +beneath its own output dir, so a caller who wrote ``star_split_ratio_80-*.fits`` +meaning "the training star sample" would match nothing under a non-recursive +glob. Patterns are therefore plain FILE names and the layout is ours to know, +not the config author's. + +ZERO MATCHES FOR ONE PATTERN IS A WARNING, NOT A FAILURE. setools rejects sparse +CCDs (~0.2% attrition, tolerated by exp_psf's own count floor), so per-CCD +counts are not fixed, and a pattern naming an optional diagnostic may legitimately +find nothing. ZERO FILES IN TOTAL IS A FAILURE: it means the store was not what +we think it is, and writing a green manifest over that would let +``clean_exposure`` delete an exposure whose products were never saved. + +The manifest lists every member (name, pattern, source path, bytes), so a reader +knows what the tar holds without opening it. + +ONE UNCOMPRESSED TAR PER EXPOSURE, ``/.tar``, NOT LOOSE COPIES. +Inodes, not bytes, are what bind on /project: the group quota is ~1 M files, +and loose per-CCD copies are ~200 per exposure with all candidates on — ~25k for +a 64-tile campaign, ~2 M at DR6 scale, against ~7 GB of bytes. A tar collapses +that to one inode per exposure and costs nothing to read: FITS members go +``tarfile.open(t).extractfile(m).read()`` -> ``fits.open(io.BytesIO(...))``, +which is why a tar rather than a multi-HDU FITS bundle (the keep list mixes +FITS, ``.psf`` and ``.txt``; a FITS container could not hold the last two). +Uncompressed because FITS barely compresses and a plain tar is seekable. + +Members are FLAT — file name only, no module subtree — because the module a +file came from is already in its name and the consumer globs member names. A +name collision between two modules is therefore a hard error rather than a +silent overwrite; nothing in the current config can produce one, and if a +future one can we want to hear about it. + +The tar is written DETERMINISTICALLY (ownership zeroed, members in sorted +order, source mtimes kept), tmp-then-``cmp``-then-``mv``: a rerun over an +unchanged store produces a byte-identical tar and leaves the existing one's +mtime alone. + +The manifest is the rule's ONLY declared output, and it lives on the persistent +root beside the tar (``/exp///manifests/``, beside the tar's ``psf/``), NOT in +the exposure's scratch ``manifests/`` dir which ``clean_exposure`` deletes +wholesale. It is deliberately NOT a ``directory()`` output: what was copied, and +how big each file was, is provenance we want written down, and a directory +output attests only that some directory exists. + +It carries no timestamp and is written tmp-then-``cmp``-then-``mv`` (the pattern +``exp_star_cat`` uses), so a rerun that packs the same files leaves the mtime +alone — mtime is a rerun trigger, and an unconditional rewrite would make every +downstream ``clean_exposure`` look out of date once per invocation. +""" + +import argparse +import filecmp +import json +import sys +import tarfile +from pathlib import Path + +# The PSF chain's run dir (RUN_NAME in config_exp_psfex.ini). Hardcoded rather +# than passed: this rule persists the PSF stage's products and nothing else, and +# a knob here would be a knob for "persist some other stage", which is a +# different rule. +RUN_NAME = "run_sp_exp_SxSePsfPi" + + +def collect(exp_dir: Path, patterns: list) -> tuple: + """Matched files per pattern, in a stable order, plus the empty patterns.""" + root = exp_dir / "output" / RUN_NAME + found, empty = {}, [] + for pat in patterns: + # One glob per module output dir, recursive beneath it (see the module + # docstring on setools' subdirectories). sorted() over the union keeps + # the manifest byte-stable across filesystem readdir order. + hits = sorted({p for mod in sorted(root.glob("*/output")) + for p in mod.rglob(pat) if p.is_file()}) + if hits: + found[pat] = hits + else: + empty.append(pat) + return found, empty + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path, + help="the exposure's scratch store") + p.add_argument("--exp", required=True) + p.add_argument("--dest", required=True, type=Path, + help="/exp///psf; the tar is " + "/.tar") + p.add_argument("--manifest", required=True, type=Path) + p.add_argument("--pattern", action="append", default=[], + help="repeatable; a plain file-name glob") + args = p.parse_args() + + if not args.pattern: + sys.exit("persist_exp: no --pattern given (config persist_exp is empty)") + + found, empty = collect(args.exp_dir, args.pattern) + if not found: + sys.exit(f"persist_exp: {args.exp}: no file matched any of " + f"{args.pattern} under {args.exp_dir}/output/{RUN_NAME}") + + args.dest.mkdir(parents=True, exist_ok=True) + tar_path = args.dest / f"{args.exp}.tar" + seen, files = {}, [] + for pat, hits in found.items(): + for src in hits: + if src.name in seen: + sys.exit(f"persist_exp: {args.exp}: two source files are both " + f"named {src.name} ({seen[src.name][0]} and {src}); tar " + f"members are flat, so this would silently overwrite") + seen[src.name] = (src, pat) + files.append({"name": src.name, "pattern": pat, + "src": str(src), "bytes": src.stat().st_size}) + files.sort(key=lambda f: f["name"]) + + def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: + # Ownership is the one thing that would differ between two writes of + # the same files from different accounts/nodes; drop it. mtime stays: + # it is the product's, and it is stable while the store is. + ti.uid = ti.gid = 0 + ti.uname = ti.gname = "" + return ti + + # tmp-then-cmp-then-mv, and the tmp NEVER outlives a failure: an orphaned + # .tmp on /project is an inode nothing revisits — the leak this whole tar + # design exists to avoid, one per failed attempt at DR6 scale. + tmp = tar_path.with_name(tar_path.name + ".tmp") + try: + with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: + for f in files: + tf.add(seen[f["name"]][0], arcname=f["name"], filter=anonymous) + if tar_path.exists() and filecmp.cmp(tmp, tar_path, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(tar_path) # atomic: no half-written archive + finally: + tmp.unlink(missing_ok=True) + + body = { + "stage": "exp_persist", "level": "exp", "unit": args.exp, + "status": "complete", + "tar": str(tar_path), + "patterns": list(args.pattern), + # The warning the docstring argues for: named patterns that matched + # nothing. Present as a key even when empty, so a reader never has to + # wonder whether an old manifest predates the field. + "patterns_unmatched": empty, + "n_files": len(files), + "bytes": sum(f["bytes"] for f in files), + "files": files, + } + args.manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = args.manifest.with_name(args.manifest.name + ".tmp") + try: + tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + if args.manifest.exists() and filecmp.cmp(tmp, args.manifest, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(args.manifest) + finally: + tmp.unlink(missing_ok=True) + + warn = f" ({len(empty)} pattern(s) matched nothing: {empty})" if empty else "" + print(f"[persist_exp] {args.exp}: {len(files)} file(s), " + f"{body['bytes'] / 1e6:.1f} MB -> {tar_path}{warn}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py new file mode 100644 index 000000000..8bf4908b5 --- /dev/null +++ b/workflow/scripts/run_report.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +"""``sp report`` — the run's success/failure tables, read from the manifests. + +NOT a DAG node. A report rule that declared all tiles' outputs as inputs would be +a descendant of every job, so one hard failure under --keep-going would poison +its cone and the report would never run — the exact scenario it exists for. So it +is a plain script, runnable at any time including mid-run; the Snakefile's +onsuccess/onerror hooks call it so every invocation ends with one. + +It reads two things and nothing else (PRD D3): + + * the **index** (``run_index.sqlite``) — the units the run declared, and the + tile->exposure edges that let an exposure failure be blamed on the tiles it + blocks; + * the **verdicts** written by ``completeness.py check`` — per-runner + found/expect/floor and log-scraped failure reasons — in the unit's + ``logs/`` (every run) and ``manifests/`` (successes only; completeness.py + argues the split). + +Both dirs are read here and the status comes from the file BODY, so a failed +stage speaks through its log while a successful one is corroborated by two +identical files. A unit with neither ran nothing. Also read, for continuity with +stores written before this convention: ``manifests/.failed.json``, which +is where failure evidence used to go. + +A reclaimed exposure has neither dir: ``clean_exposure`` deleted both, after +copying the manifests into ``/cleaned.json``. That tombstone is read as the +unit's record and the unit is reported as **cleaned** — not "not run", and it +blocks no tile. + +A reclaimed TILE is the same story with one asymmetry: ``clean_tile`` deletes +``logs/`` but cannot empty ``manifests/``, because two of those manifests are +currency other mechanisms read (SURVIVING_TILE_STAGES). So the "records on disk +mean a rebuilt chain" test takes that survivor set explicitly — see +``absorb_tombstones``. + +No disk scanning: counting products is the *check's* job, done once at the moment +the products were fresh. A unit with no manifest for a stage is "not run" — which +is a real and distinct answer from "ran and produced nothing". + +Records are discovered by glob (``tiles/*/*/manifests/*.json`` and +``tiles/*/*/logs/*.json``), not by constructed path: units are found rather than +named, so a store holding a unit the index never heard of still reports. The +depth is FIXED at two, because the sharded layout is exactly two levels +(``tiles///``) — a ``**`` walked the whole tree, including every +``output/`` a run has not reclaimed yet, to find records that can only ever be +at one depth. +""" + +import argparse +import json +import sqlite3 +import sys +from collections import defaultdict +from pathlib import Path + +# Stage order per level — the report's column order, and the definition of +# "expected" (a declared unit with no manifest for one of these is not run). +TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat"] +EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] + +# exp_persist and exp_footprint are DELIBERATELY NOT in that list. This report +# disk-scans the scratch run_dir, and those two are the exposure manifests that +# live on products_dir instead — that placement is what makes them survive +# clean_exposure. Listed here they would read as "not run" for every exposure +# in the campaign. Reporting on the persisted products means scanning the +# second root, which is a report this one does not yet do. + +# The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names +# the mechanism that owns each). Their presence is therefore NOT evidence that a +# tile's chain was rebuilt, which is the one thing absorb_tombstones has to know +# to read a reclaimed tile's record out of its tombstone. +SURVIVING_TILE_STAGES = frozenset({"tile_vignets", "tile_find_exposures"}) + +STATUSES = ("complete", "warn", "failed", "not_run") + + +def _rank(m: dict) -> int: + """How BAD a record is, as a position in STATUSES; unknown sorts worst.""" + return STATUSES.index(m["status"]) if m.get("status") in STATUSES else len(STATUSES) + + +def keep_worst(records: dict, stage: str, m: dict) -> None: + """Collapse the several records of one stage into the WORST of them. + + ONE rule, used by both readers in this module, and that is the point. + Several files map to one (unit, stage) either way: on disk, a stage's + manifest and its byte-identical log, plus the eight ngmix chunks, which all + carry ``stage: "tile_ngmix"`` under per-chunk filenames; inside a tombstone, + those same eight chunks again, keyed by file stem. + + absorb_tombstones used to resolve that collision by first-key-wins + (``setdefault`` over sorted stems), so only ``tile_ngmix_1`` survived and a + ``warn`` on any other chunk disappeared the moment the tile was reclaimed — + ``tile_ngmix ok 0 warn 1`` before the clean, ``ok 1 warn 0`` after, and the + tile silently left the "tiles not complete" table. The data was in the + tombstone the whole time; only the reader dropped it (found in review, + reproduced on 186.307 with chunk 5 flipped to warn). + + What this does NOT fix, because it is not a reclamation bug: a stage's + per-runner ``products`` aggregate is taken from the single surviving record, + so tile_ngmix attrition is counted over one chunk of eight. That is true + before and after a clean, identically — it is the price of collapsing the + chunks to one stage row, and it is the same on both paths by construction + now. + """ + prev = records.get(stage) + if prev is None or _rank(m) > _rank(prev): + records[stage] = m + + +def load_manifests(run_dir: Path, sub: str) -> dict: + """``{unit: {stage: verdict}}`` for one store (``tiles`` or ``exp``). + + Reads BOTH of a unit's record dirs: ``manifests/`` (the rules' declared + outputs, success-only) and ``logs/`` (the rules' ``log:``, written every run + and never deleted by snakemake, so this is where a failure survives). + + The unit key is the record dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the record's own + ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is + not the index's ``210.282``). The stage comes from the body, never the + filename: ngmix chunks share a stage under per-chunk filenames, and a log + names the same stage as the manifest beside it. + + Several files therefore map to one (unit, stage), and the WORST status wins. + That is what collapses the ngmix chunks to one entry, and it is why a + successful stage's two byte-identical records cost nothing while a failure + always speaks. A body with no ``stage`` field is skipped: it is not one of + ours, which is what keeps a stray JSON deeper in the tree inert. + """ + out: dict = defaultdict(dict) + paths = sorted((run_dir / sub).glob("*/*/manifests/*.json")) \ + + sorted((run_dir / sub).glob("*/*/logs/*.json")) + for path in paths: + try: + m = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable record {path}: {exc}", file=sys.stderr) + continue + if not isinstance(m, dict) or "stage" not in m: + continue + unit = path.parent.parent.name + keep_worst(out[unit], m["stage"], m) + return out + + +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, + survivors: frozenset = frozenset()) -> set: + """Fill in reclaimed units from their ``cleaned.json``; return their ids. + + A cleaned exposure has neither ``manifests/`` nor ``logs/``; every manifest + was copied verbatim into the tombstone first. Read them back, or the report + inverts the truth exactly when reclamation works: the exposure shows as "not + run" and blocks the very tiles whose completion authorised the deletion. + + Manifests on disk win if both exist — that is a re-built chain, and the + tombstone is then a stale record of the previous generation. + + ``survivors`` is what makes that test work for TILES. ``clean_tile`` cannot + empty ``manifests/`` the way ``clean_exposure`` does: two of the manifests + there are currency other mechanisms read (SURVIVING_TILE_STAGES below), so a + reclaimed tile ALWAYS has records on disk. Without this argument the + "manifests win" guard fires on every cleaned tile, the tombstone is ignored, + and a fully reclaimed campaign reports as one that ran two stages and + stopped. A unit counts as REBUILT — and keeps the guard — iff it has a record + for some stage that is not a survivor; the default empty set reproduces the + exposure test exactly. + + KNOWN AND DELIBERATE: the guard is all-or-nothing, so a PARTIALLY rebuilt + cleaned tile loses its whole tombstone record. Force a rerun that restores + tile_detect..tile_make_cat but not the prepare stages (which only the + prepare invocation produces) and tile_get_images / tile_uncompress read + "not run" though they did run and nothing invalidated them. + + Not fixed, because the obvious fix is wrong rather than long. Filling + per-stage from the tombstone would, on a unit whose chain is rebuilding, + report the PREVIOUS generation's "complete" for a stage whose manifest is + absent precisely because it is mid-rerun or failed — a stale complete is + invisibly wrong where "not run" is visibly incomplete, and the same hazard + reaches exposures, where the mixing would be silent and campaign-wide. The + all-or-nothing guard is a generation boundary and is worth more than the + cosmetics. A correct fix needs a per-stage notion of which generation a + record belongs to, which nothing here records today. + """ + cleaned = set() + for path in sorted((run_dir / sub).glob("*/*/cleaned.json")): + unit = path.parent.name + try: + tomb = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) + continue + if any(s not in survivors for s in manifests.get(unit, {})): + continue + # Collapse the tombstone's per-stem records to one per stage FIRST, + # by the same rule the on-disk path uses (keep_worst), and only then + # merge. Reversing those two steps is what lost a warning chunk. + absorbed: dict = {} + for key, m in (tomb.get("manifests") or {}).items(): + if not isinstance(m, dict): + continue + keep_worst(absorbed, m.get("stage", key), m) + for stage, m in absorbed.items(): + # setdefault, not assignment: the surviving on-disk records are the + # live ones and stay authoritative, even though the tombstone's + # copies of them are byte-identical today. + manifests[unit].setdefault(stage, m) + cleaned.add(unit) + return cleaned + + +def shortfalls(m: dict) -> dict: + """``{runner: (found, expect, floor)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"], d["floor"]) + for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} + + +def reasons(m: dict) -> list: + """Flattened failure reasons, runner-tagged, for the report's why column.""" + out = [] + for f in m.get("failures", []): + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')} (floor {f['floor']})" + out += [f"{head}: {r}" for r in f["reasons"]] or [head] + return out + + +def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: + """Per-stage counts + named unit lists, for one level. + + ``cleaned`` units are counted by the status their absorbed manifests carry + and additionally listed under ``cleaned``, so a reclaimed campaign reads as + reclaimed rather than as a campaign that never ran. The STATUS is preserved + exactly, including a warn on any one of the eight ngmix chunks (keep_worst + is what makes that true on the tombstone path as well as on disk). + + The per-runner ``products`` aggregate is NOT a per-chunk total: reading it + as a whole-tile figure over-states completeness by 8x (see ``keep_worst``). + """ + per_stage = {} + for stage in stages: + # All five status keys are unit-id LISTS, "complete" included: it used + # to be a bare int, which made it the one key a caller had to special- + # case. The emitted JSON gains the complete-unit list; the printed + # counts are len() of it. + t = {"complete": [], "warn": [], "failed": [], "not_run": [], "cleaned": []} + agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) + for u in units: + m = manifests.get(u, {}).get(stage) + if m is None: + t["not_run"].append(u) + continue + if u in cleaned: + t["cleaned"].append(u) + status = m.get("status", "failed") + status = status if status in ("complete", "warn") else "failed" + t[status].append(u) + if status == "failed": + # Failed units are named above, never folded into the attrition + # aggregate: a whole-unit failure is not per-CCD attrition, and + # mixing them hides real deletion bugs behind a big denominator. + continue + for runner, d in m.get("runners", {}).items(): + a = agg[runner] + a["found"] += d["found"] + a["expect"] += d["expect"] + if d["found"] < d["expect"]: + a["by_unit"][u] = d["expect"] - d["found"] + for a in agg.values(): + if not a["by_unit"]: + del a["by_unit"] + t["products"] = dict(agg) + per_stage[stage] = t + return per_stage + + +def unit_rows(units, stages, manifests) -> list: + """One row per non-clean unit: its first bad stage, shortfalls, why.""" + rows = [] + for u in units: + got = manifests.get(u, {}) + bad = [s for s in stages + if got.get(s) is None or got[s].get("status") != "complete"] + if not bad: + continue + stage = bad[0] + m = got.get(stage) + rows.append({ + "unit": u, + "stage": stage, + "status": "not_run" if m is None else m.get("status", "failed"), + "shortfalls": shortfalls(m) if m else {}, + "reasons": reasons(m) if m else [], + "n_bad_stages": len(bad), + }) + return rows + + +def print_table(title, rows, limit=25): + print(f"\n{title} ({len(rows)} affected)") + if not rows: + print(" — none") + return + print(f" {'unit':<14} {'stage':<20} {'status':<8} why") + for r in rows[:limit]: + short = ", ".join(f"{k} {v[0]}/{v[1]}" for k, v in r["shortfalls"].items()) + why = (r["reasons"][0] if r["reasons"] else short) or "-" + print(f" {r['unit']:<14} {r['stage']:<20} {r['status']:<8} {why[:90]}") + if len(rows) > limit: + print(f" … and {len(rows) - limit} more (see the JSON report)") + + +def print_stage_table(title, per_stage, n_units): + print(f"\n{title} ({n_units} units declared)") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} " + f"{'cleaned':>8} attrition") + for stage, t in per_stage.items(): + att = [f"{r} {a['found']}/{a['expect']}" + for r, a in t["products"].items() if a["found"] < a["expect"]] + print(f" {stage:<20} {len(t['complete']):>6} {len(t['warn']):>6} " + f"{len(t['failed']):>6} {len(t['not_run']):>8} " + f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--status", default="manual") + p.add_argument("--out", type=Path, default=None) + p.add_argument("--limit", type=int, default=25, + help="rows per stdout table (the JSON report is complete)") + args = p.parse_args() + + tiles, exps, tile_exp = [], [], defaultdict(list) + if args.index.exists(): + con = sqlite3.connect(args.index, timeout=60) + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] + for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + tile_exp[tile_id].append(exp_id) + con.close() + else: + print(f"[run_report] no index at {args.index} — reporting manifests only", + file=sys.stderr) + + tile_m = load_manifests(args.run_dir, "tiles") + exp_m = load_manifests(args.run_dir, "exp") + # Reclaimed units speak through their tombstones (D5, S5). Tiles pass the + # survivor set: clean_tile leaves two manifests on disk on purpose, and + # without that argument they would read as a rebuilt chain. + cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) + cleaned_tiles = absorb_tombstones(args.run_dir, "tiles", tile_m, + SURVIVING_TILE_STAGES) + tiles = tiles or sorted(tile_m) + exps = exps or sorted(exp_m) + + missing_json = args.index.parent / "missing.json" + missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] + + report = { + "status": args.status, + "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m, cleaned_tiles), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), + "cleaned_exposures": sorted(cleaned_exp), + "cleaned_tiles": sorted(cleaned_tiles), + "tiles": unit_rows(tiles, TILE_STAGES, tile_m), + "exposures": unit_rows(exps, EXP_STAGES, exp_m), + } + + # Blame propagation: a BLOCKING exposure blocks every tile that reads it. + # Without this, a tile stalled at tile_vignets looks like its own failure. + # + # Blocking means "failed" or "never ran" — NOT "warn". Warn is the expected + # per-CCD attrition (setools rejecting a sparse CCD, psfex_interp short an + # epoch); it is present in essentially every exposure at production scale, so + # counting it here made every exposure block every tile and the table said + # nothing. + # Judged over ALL the exposure's stages, not just the first bad one, so an + # exposure that warns early and fails late still blocks. + # A CLEANED exposure never blocks: its store is gone precisely because every + # consuming tile already had its vignets. Its absorbed manifests are read + # above, so a cleaned exposure that genuinely failed still shows in the + # tables — it just does not get to hold complete tiles hostage. + def _blocks(unit): + if unit in cleaned_exp: + return False + for stage in EXP_STAGES: + m = exp_m.get(unit, {}).get(stage) + if m is None or m.get("status", "failed") == "failed": + return True + return False + + bad_exp = {e for e in exps if _blocks(e)} + blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} + report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} + + done = len(report["tile_stages"]["tile_make_cat"]["complete"]) + report["final_cats"] = {"present": done, "of": len(tiles)} + + out = args.out or (args.index.parent / "run_report.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "") + + (f" ({len(cleaned_tiles)} tiles reclaimed)" if cleaned_tiles else "")) + print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) + print_stage_table("TILES", report["tile_stages"], len(tiles)) + print_table("exposures not complete", report["exposures"], args.limit) + print_table("tiles not complete", report["tiles"], args.limit) + nb = report["tiles_blocked_by_exposures"] + if nb: + print(f"\ntiles waiting on incomplete exposures ({len(nb)})") + for t, e in list(nb.items())[:args.limit]: + print(f" {t:<14} {', '.join(e[:6])}" + + (f" (+{len(e) - 6})" if len(e) > 6 else "")) + print(f"\n[run_report] -> {out}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py new file mode 100644 index 000000000..2407af71b --- /dev/null +++ b/workflow/scripts/star_cats.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""The campaign's GSC 2.3 star catalogue, as a HEALPix-chunked sky store. + +Masking needs, for every exposure, the bright stars over its focal plane. The +sky does not change between exposures, so the network cost of that is a property +of the campaign's SKY AREA, not of its exposure count: exposures overlap each +other ~7-10 deep, and a tile's exposures all look at the same square degree. + +So the store is chunked by sky, not by exposure. One GSC 2.3 cone query per +HEALPix pixel of NSIDE=32, written run-independently under the ``star_cats`` +config root and never fetched twice. A campaign that grows past the fetched +footprint queries only the chunks its new tiles add; one that grows within it +queries nothing. + +Two numbers set the scale. A full-UNIONS footprint is ~1.5k chunks against ~25k +exposures, so the QUERY COUNT drops ~16x. The queried AREA drops ~4x: the old +design covered the footprint ~8 times over (that is just the exposure overlap +depth), the new one ~2 times, the 2x being the price of bounding a HEALPix +quadrilateral by the cone Vizier speaks (see ``pixel_cone``) — 5.6-9 deg^2 for a +3.36 deg^2 pixel, ~40-60k rows and ~3-4 MB per chunk. + +Two subcommands, one module, deliberately: ``fetch`` and ``cut`` must agree +EXACTLY on which pixel holds which star, and a shared NSIDE constant in one file +is the only version of that agreement which cannot drift. + + fetch --tile-list ... --store ... --manifest ... + The campaign side. Turns the tile list into the set of pixels its + exposures can possibly need, fetches the missing ones, writes a manifest. + + cut --images ... --store ... --out ... + The per-exposure side, purely local: read the focal-plane footprint from + the exposure's image headers, load the chunks covering it, deduplicate, + and cut to the focal-plane disc. Reproduces byte-for-byte the same sky + selection the old one-query-per-exposure cone did. + +Geometry, and why the fetch pad is what it is. Chunk-need is computed from the +TILE list rather than from exposure pointings, because tile IDs are the one thing +known before any download: a pointing center means reading a FITS header of an +image get_images has not fetched yet, and the DAG needs the chunk set at parse +time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so +each tile is a disc of half-diagonal 0.354 deg; find_exposures gives a tile every +exposure whose footprint covers it, and the MegaCam focal plane is a disc of +radius 0.73 deg (measured on the cached catalogues). An exposure center is +therefore at most 0.354 + 0.73 deg from the tile center, and its stars 0.73 deg +beyond that: 1.81 deg, padded to ``PAD_DEG`` = 2.0. The pad is a perimeter cost +— negligible for a contiguous campaign, and paid once. + +The pad is a bound, not a promise: ``cut`` verifies that every chunk covering the +exposure it was handed is on disk, and fails loudly if one is not. A missing +chunk means the geometry above is wrong, and that must not degrade quietly into +an under-masked exposure. +""" + +import argparse +import json +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np +import healpy as hp +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.table import Table, vstack + +# The PYTHONPATH pin in profiles/nibi puts this checkout's src/ on the path (see +# exposure.smk's in_container), the same way the vizier helper is reached below. +from shapepipe.utilities.file_io import write_atomic +from shapepipe.utilities.focal_plane import focal_plane_disc + +# GSC 2.3. The same catalogue the mask module's own CDS path uses +# (mask.py: _CDS_cat_ID), so the store is a drop-in for it. +CAT_ID = "I/305/out" + +# NSIDE=32 -> 3.36 deg^2 per pixel, 12288 pixels over the sky. Chosen so one +# chunk is a couple of MegaCam focal planes: small enough that a Vizier query +# stays within a small multiple of the per-exposure queries this replaces, large +# enough that a full-UNIONS footprint is ~1.5k chunks rather than ~25k. The +# cone-vs-quadrilateral overhead is scale-free, so NSIDE trades query count +# against query size and nothing else. NESTED, so a chunk id is a hierarchical +# sky address and a future NSIDE change is a subdivision. +NSIDE = 32 +NEST = True + +# Angular padding on the disc used to select chunks (see the module docstring). +PAD_DEG = 2.0 + +# The MegaCam focal-plane disc, and the margin added to a pixel's own bounding +# cone. Both in degrees. +MARGIN_DEG = 0.02 + +# GSC 2.3's object id: the deduplication key where chunk cones overlap. +ID_COL = "GSC2.3" + + +# --- the chunk store -------------------------------------------------------- + + +def store_dir(store: Path) -> Path: + """Chunks live under the catalogue and resolution that produced them, so a + later NSIDE or catalogue change is a new directory beside the old one rather + than a silent reinterpretation of files already on disk.""" + return Path(store) / CAT_ID.replace("/", "_") / f"nside{NSIDE}" + + +def chunk_path(store: Path, ipix: int) -> Path: + return store_dir(store) / f"star_chunk-{ipix:06d}.fits" + + +def chunks_for_disc(ra_deg: float, dec_deg: float, radius_deg: float) -> list[int]: + """Every pixel that touches the disc, as sorted ids. + + ``inclusive=True`` makes this a conservative superset — the guarantee ``cut`` + relies on is that no star inside the disc lives in a pixel this omits. + """ + vec = hp.ang2vec(ra_deg, dec_deg, lonlat=True) + return sorted(int(i) for i in hp.query_disc( + NSIDE, vec, np.radians(radius_deg), inclusive=True, fact=4, nest=NEST)) + + +def pixel_cone(ipix: int) -> tuple[float, float, float]: + """(ra, dec, radius_arcmin) of a cone that CONTAINS pixel ``ipix``. + + Vizier speaks cones, HEALPix speaks quadrilaterals, so the query is the + pixel's bounding cone: its center, and the largest center-to-boundary + distance plus a margin. The cone spills over the pixel edges, which costs a + little duplication between neighbours and buys the containment ``cut`` + depends on. The duplicates are removed on read, by ``ID_COL``. + """ + ra_c, dec_c = hp.pix2ang(NSIDE, ipix, nest=NEST, lonlat=True) + ra_b, dec_b = hp.vec2ang(hp.boundaries(NSIDE, ipix, step=8, nest=NEST).T, + lonlat=True) + center = SkyCoord(ra_c * u.deg, dec_c * u.deg) + radius = center.separation(SkyCoord(ra_b * u.deg, dec_b * u.deg)).deg.max() + return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) + + +def read_chunks(store: Path, ipixels: list[int]) -> Table: + """Load and deduplicate the given chunks. + + A missing chunk is fatal (see the module docstring): it means the fetch + footprint did not cover this exposure, and an under-masked exposure is worse + than a failed job. + """ + missing = [i for i in ipixels if not chunk_path(store, i).exists()] + if missing: + raise SystemExit( + f"star chunk(s) {missing} not in {store_dir(store)}. The campaign's " + f"star_catalogue fetch did not cover this exposure — re-run it " + f"(and check that its tile list contains this exposure's tiles).") + + table = vstack([Table.read(chunk_path(store, i)) for i in ipixels], + metadata_conflicts="silent") + _, keep = np.unique(np.asarray(table[ID_COL]), return_index=True) + return table[np.sort(keep)] + + +# --- exposure footprint ----------------------------------------------------- +# The WCS construction and the focal-plane disc live in +# shapepipe.utilities.focal_plane, beside the vizier helper and imported the same +# way: create_star_cat.py needs exactly the same geometry, and the two must not +# be able to disagree about which sky an exposure covers. + + +def exposure_image(images_dir: Path) -> Path: + """The one multi-extension exposure image in a get_images output dir. + + That dir is a symlink farm holding ``image-.fitsfz`` plus its weight and + flag; only the image carries the 40 CCD WCSs. + """ + found = sorted(p for p in Path(images_dir).iterdir() if "image" in p.name) + if not found: + raise SystemExit(f"no image file in {images_dir}") + return found[0] + + +# --- fetch ------------------------------------------------------------------ + + +def campaign_chunks(tile_ids: list[str]) -> list[int]: + """Every chunk the campaign's exposures can need, from the tile list alone.""" + from shapepipe.utilities.cfis import get_tile_coord_from_nixy + + needed: set[int] = set() + for tile_id in tile_ids: + nix, niy = tile_id.split(".") + ra, dec = get_tile_coord_from_nixy(nix, niy) + needed.update(chunks_for_disc(ra.degree, dec.degree, PAD_DEG)) + return sorted(needed) + + +def fetch(args: argparse.Namespace) -> None: + from shapepipe.utilities.vizier import query_vizier + + tile_ids = [ln.strip() for ln in Path(args.tile_list).read_text().splitlines() + if ln.strip()] + needed = campaign_chunks(tile_ids) + out_dir = store_dir(args.store) + out_dir.mkdir(parents=True, exist_ok=True) + todo = [i for i in needed if not chunk_path(args.store, i).exists()] + print(f"star chunks: {len(needed)} needed for {len(tile_ids)} tiles, " + f"{len(todo)} to fetch -> {out_dir}", file=sys.stderr) + + def one(ipix: int) -> int: + ra, dec, radius_arcmin = pixel_cone(ipix) + table = query_vizier(ra, dec, radius_arcmin, CAT_ID) + write_atomic(table, chunk_path(args.store, ipix)) + print(f"chunk {ipix}: {len(table)} rows " + f"(ra={ra:.4f} dec={dec:.4f} r={radius_arcmin:.1f}')", + file=sys.stderr) + return len(table) + + # A handful of concurrent queries, never one per exposure: the same modest + # concurrency the per-exposure rule reached through --local-cores, now an + # explicit number instead of an accident of the head node's CPU count. + if todo: + with ThreadPoolExecutor(max_workers=args.workers) as pool: + list(pool.map(one, todo)) + + write_manifest(args, tile_ids, needed, len(todo)) + + +def write_manifest(args, tile_ids, needed, n_fetched) -> None: + """The rule's declared output. + + Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and + has no per-runner count floors, so there is nothing to compose and no + separate ``log:`` — under ``set -euo pipefail`` the job either completes or + aborts at the failing query, and snakemake's captured stderr is the evidence. + The manifest keeps the workflow's "one rule, one manifest" currency: written + last, and only when the content changed, so an unchanged campaign leaves the + mtime where it was rather than churning the `mtime` rerun-trigger. + """ + body = json.dumps({ + "stage": "star_catalogue", + "level": "campaign", + "status": "complete", + "catalogue": CAT_ID, + "nside": NSIDE, + "nest": NEST, + "pad_deg": PAD_DEG, + "store": str(store_dir(args.store)), + "n_tiles": len(tile_ids), + "n_chunks": len(needed), + "n_fetched": n_fetched, + "chunks": needed, + }, indent=2, sort_keys=True) + path = Path(args.manifest) + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != body: + path.write_text(body) + + +# --- cut -------------------------------------------------------------------- + + +def cut(args: argparse.Namespace) -> None: + image = exposure_image(args.images) + ra, dec, radius = focal_plane_disc(image) + ipixels = chunks_for_disc(ra, dec, radius) + table = read_chunks(args.store, ipixels) + + center = SkyCoord(ra * u.deg, dec * u.deg) + stars = SkyCoord(np.asarray(table["RAJ2000"]) * u.deg, + np.asarray(table["DEJ2000"]) * u.deg) + inside = table[center.separation(stars).deg <= radius] + + print(f"{image.name}: ra={ra:.4f} dec={dec:.4f} r={radius:.4f} deg, " + f"{len(ipixels)} chunks -> {len(inside)} stars", file=sys.stderr) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + write_atomic(inside, out) + + +# --- CLI -------------------------------------------------------------------- + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + + f = sub.add_parser("fetch", help="fetch the campaign footprint's chunks") + f.add_argument("--tile-list", required=True, type=Path) + f.add_argument("--store", required=True, type=Path) + f.add_argument("--manifest", required=True, type=Path) + f.add_argument("--workers", type=int, default=4) + f.set_defaults(func=fetch) + + c = sub.add_parser("cut", help="cut one exposure's catalogue from the store") + c.add_argument("--images", required=True, type=Path, + help="a get_images output dir holding image-.fitsfz") + c.add_argument("--store", required=True, type=Path) + c.add_argument("--out", required=True, type=Path) + c.set_defaults(func=cut) + + args = p.parse_args() + args.func(args) + + +if __name__ == "__main__": + main()