diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index f0ef0b35c..390a165cb 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -83,12 +83,11 @@ jobs: # Smoke-test the binaries baked into the runtime image. Catches the # class of regression where the image builds but a runtime tool - # (sextractor, weightwatcher) is missing or unrunnable. + # (sextractor, psfex) is missing or unrunnable. - name: Test runtime — binaries run: | IMAGE=$(echo "${{ steps.meta-runtime.outputs.tags }}" | head -n1) docker run --rm "$IMAGE" source-extractor --version - docker run --rm "$IMAGE" weightwatcher --version docker run --rm "$IMAGE" psfex --version - name: Test runtime — shapepipe entry point (read-only fs) diff --git a/.gitignore b/.gitignore index db20f3086..4c4b6d0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ code # .felt here is a machine-local symlink into it. Never track it in this repo. /.felt/ /.felt +.snakemake/ diff --git a/Dockerfile b/Dockerfile index b11ab7a6e..bd8a07d49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV SHELL=/bin/bash \ COVERAGE_FILE=/tmp/.coverage # System dependencies — three categories: -# - astromatic binaries (psfex, source-extractor, weightwatcher) ship as +# - astromatic binaries (psfex, source-extractor) ship as # Debian packages on bookworm; preferred over building from source. # - compilers and dev libs needed to build the heavier wheels (galsim, # mpi4py, python-pysap, fitsio). @@ -55,7 +55,7 @@ RUN apt-get update -y --quiet && \ libcfitsio-dev \ libproj-dev proj-bin \ libgl1-mesa-glx \ - psfex source-extractor weightwatcher && \ + psfex source-extractor && \ apt-get clean && rm -rf /var/lib/apt/lists/* # OpenMPI from source — required for hybrid Apptainer MPI on HPC clusters. diff --git a/docs/source/container.md b/docs/source/container.md index 174cde9dc..1a83b0f55 100644 --- a/docs/source/container.md +++ b/docs/source/container.md @@ -165,7 +165,7 @@ The Dockerfile does **not** duplicate Python deps — those come from The asymmetry is deliberate: Python deps go through pyproject + lockfile (reproducible, auditable), system deps go through Dockerfile (Debian's versioning). Don't `apt install` something that has a Python wheel; don't -`pip install` something Debian packages directly (e.g. `weightwatcher`). +`pip install` something Debian packages directly (e.g. `source-extractor`). ## Why this shape @@ -175,8 +175,8 @@ versioning). Don't `apt install` something that has a Python wheel; don't - **`uv sync --frozen`** at build time means the image is bit-exactly reproducible from a tagged commit, and impossible to ship with a stale lockfile. -- **Astromatic binaries from Debian** (`psfex`, `source-extractor`, - `weightwatcher`) instead of source builds — Debian carries the +- **Astromatic binaries from Debian** (`psfex`, `source-extractor`) + instead of source builds — Debian carries the GCC-compatibility patches that the previous Dockerfile had to apply inline with `sed`. - **Two targets** so canfar batch deployments stay slim while interactive diff --git a/docs/source/dependencies.md b/docs/source/dependencies.md index 9378bfe00..b907a452d 100644 --- a/docs/source/dependencies.md +++ b/docs/source/dependencies.md @@ -57,7 +57,6 @@ packages (no source builds), plus the MPI stack: |---------|------------| | [Source Extractor](https://www.astromatic.net/software/sextractor/) | {cite:p}`bertin:96` | | [PSFEx](https://www.astromatic.net/software/psfex/) | {cite:p}`bertin:11` | -| [WeightWatcher](https://www.astromatic.net/software/weightwatcher/) | {cite:p}`marmo:08` | | OpenMPI (5.0.x) | | Python dependencies themselves are managed with [uv](https://docs.astral.sh/uv/); diff --git a/docs/source/installation.md b/docs/source/installation.md index 83df0b6bb..ee3f8e182 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -38,8 +38,8 @@ docker pull ghcr.io/cosmostat/shapepipe:develop-runtime We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance. ``` -The image bundles the astromatic binaries (`source-extractor`, `psfex`, -`weightwatcher`), MPI (`mpi4py` + OpenMPI), and every Python dependency, so +The image bundles the astromatic binaries (`source-extractor`, `psfex`), +MPI (`mpi4py` + OpenMPI), and every Python dependency, so there is nothing else to install or build. To process data on a cluster with MPI, run the pipeline through Apptainer the same way you would any MPI job. diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index cfdd16507..41e2c1301 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -170,22 +170,16 @@ The downloaded tile weights are compressed. The following call uncompresses all. shapepipe_run -c cfis/config_tile_Uz.ini ``` -### Mask tiles +### Masks -This step is done globally for all tiles. There might be job failures or interruptions. The following -command to the `ShapePipe` job script can be run repeatedly; already created masks will be skipped. - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 4 -``` - -If masks were created in more than one run, i.e. situated in more than one output directory, these have to be -combined for subsequent pipeline module runs. This is done by creating a new output directory with symbolic -links, using the script - -```bash -combine_runs.bash -c flag_tile -``` +There is no masking step. `ShapePipe` generates no masks: the sky-fixed +healsparse maps are queried once per object, by `mask_query` on the exposure +catalogues (`MASK_EXT`, recorded but not cut on) and by `make_cat` on the tile +catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` +config entries at the maps and nothing else is needed — no star-catalogue +download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that +touches pixels is the instrument flag image shipped with each exposure, which +`split_exp` splits per CCD. ## Tile detection @@ -205,7 +199,7 @@ canfar_submit_job -j 16 -f tile_numbers.txt -P N_PAR -v -J JMAX ### Exposure Processing -#### Option 0: Global split and exp masks (deprecated; used for earlier v1.x patch runs) +#### Option 0: Global split (deprecated; used for earlier v1.x patch runs) For this option, set `sp_local=0`. @@ -217,21 +211,7 @@ For `sp_local=-` both `mh_local` (0, 1) are ok: export mh_local=0 ``` -#### Option 0: Mask exposures (deprecated) - -Run repeatedly if necessary: - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 8 -``` - -Combine all runs: - -```bash -combine_runs.bash -c flag_exp -``` - -### Option 1: Local split and mask exposures (recommended) +### Option 1: Local split exposures (recommended) Optional: Enable flags for local split processing and merge header runs as @@ -258,12 +238,6 @@ First, determine the number of maximum jobs with the option `-s` (see above). Th canfar_submit_job -j 2 -v -f exp_shdu.txt -v -P N_PAR -J JMAX ``` -### Mask exposures - -```bash -canfar_submit_job -j 8 -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - ### Exposure detection ```bash @@ -400,7 +374,7 @@ Rename to general PSF and star catalogue used for all ("a") sub-versions: ```bash -cp output/run_sp_Ms/merge_starcat_runner/output/full_starcat-0000000.fits \ +cp output/run_sp_tile_Ms/merge_starcat_runner/output/full_starcat-0000000.fits \ unions_shapepipe_psf_2024_v1.6.a.fits ``` diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 95a0aa94a..b7ae7985c 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -44,11 +44,11 @@ Naming and numbering of the input files can closely follow the original image na A stacked image is also called *tile*. These files are used on input by `ShapePipe`. The pixel data can contain the observed image, a weight map, or a flag map. Tile images and weights are created in the case of CFIS by Stephen Gwyn using a combination of `swarp` and his own software. Examples of file names are - `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tile flag files - are created the mask module of `ShapePipe` (see [Mask images](#mask-images)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for + `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tiles have no flag file + (see [Masks](#masks)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for clarity, we include the string `image` for a tile image type. Default convention: **-.fits** - Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz`, `pipeline_flag-239-293.fits` + Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz` - Database catalogue files For very large files that combine information from multiple tiles or single exposures, `ShapePipe` creates `sqlite` @@ -128,8 +128,6 @@ for all options. This script creates the subdirectory `$SP_RUN/output` to store all pipeline outputs (log files, diagnostics, statistics, output images, catalogues, single-exposure headers with WCS information). -Optionally, the subdir `output_star_cat` is created by the used to store the external star catalogues for masking. This is only necessary if the pipeline is run on a cluster without internet connection to access star catalogues. In that case, the star catalogues need to be retrieved outside the pipeline, for example on a login node, and copied to `output_star_cat`. - The job script automaticall performs a number of subsequent calls to the `ShapePipe` executable `shapepipe_run`, as ```bash shapepipe_run -c $SP_CONFIG/.ini @@ -189,32 +187,49 @@ Finally, the headers of all single-exposure single-CCD files are merged into a s Two output directories are created, `run_sp_Uz` for `uncompress_fits`, and `run_sp_exp_SpMh` for the output of the modules `split_exp` (`Sp`) and `merge_headers` (`Mh`). -## Mask images - -Run -```bash -job_sp TILE_ID -j 4 -``` -to mask tile and single-exposure single-CCD images. Both tasks are performed by two calls to the `mask` runner. - -Note that internet access is required for this step, since a reference star catalogue is downloaded. - -The output of both masking runs are stored in the output directory `run_sp_MaMa`, with run 1 (2) of -`mask` corresponding to tiles (exposures). - -**Diagnostics:** Open a single-exposure single-CCD image and the corresponding pipeline flag -in `ds9`, and display both frames next to each other. Example -```bash -ds9 image-2113737-10.fits pipeline_flag-2113737-10.fits -``` -Choose `zoom fit` for both frames, click `scale zscale` for the image, and `color aips0` for the flag, to display something like this: - - - -By eye the correspondence between the different flag types and the image can be -seen. Note that the two frames might not match perfectly, since (a) WCS -information is not available in the flag file FITS headers; (b) the image can -have a zero-padded pixel border, which is not accounted for by `ds9`. +## Masks + +`ShapePipe` does not generate masks. Sky-fixed masks — star halos, stars, +manual masks for large galaxies, per-band coverage, MaxiMask defects — are +supplied as [healsparse](https://healsparse.readthedocs.io) maps and are +consumed by *querying them at object positions*, never by rasterizing them onto +pixels. Two modules do the querying, from the same shared lookup +(`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and +`setools` on the single-exposure single-CCD catalogues and writes one integer +`MASK_EXT` column (0 = clean), recording the star-body map (bit 2) against every +detection; `make_cat` writes one +`MASK_` column per band onto the final tile catalogue, carrying the map +value verbatim so downstream selections choose their own cuts. Map paths and +bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and +`MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. + +The distinction that drives all of this is what a mask *means*. An **instrument +flag** marks a corrupted measurement — the pixels carry no usable signal — so +these are the only masks that reject anything inside the pipeline. The +**healsparse masks** are sky-fixed location flags: they say where an object +sits, not that its pixels are broken, so what to do about one is an analysis +decision and is made downstream. + +**Nothing in the pipeline cuts on the queried columns**, and on exposures the +query ships off entirely: `MASK_PATHS` is commented out, which makes +`mask_query` a strict no-op that passes the catalogue through with no +`MASK_EXT` column (the module stays in the chain, so enabling it is +uncommenting one line). `star_selection.setools` rejects on `IMAFLAGS_ISO == 0` +and nothing else, deliberately starting from outlier rejection alone, and the +final catalogue's `MASK_` columns are written unfiltered. `MASK_EXT` is +the configurable pickup if outlier rejection proves insufficient: add +`MASK_EXT == 0` beside each `IMAFLAGS_ISO == 0`, one line per mask block, as +that file's header documents. + +No internet access is needed at any point, and there is no reference star +catalogue to download. + +The one mask that still reaches pixels is the **instrument flag image** +(`p.flag.fits.fz`) delivered with each exposure, which records bad columns +and saturation. `split_exp` splits it per CCD beside the image and weight, +`sextractor` reads it as `IMAFLAGS_ISO`, and `ngmix` zero-weights flagged +pixels in its postage stamps. Tiles have no such image, so tile detection runs +with `FLAG_IMAGE = False`. ## Detect objects on tiles and process stars on single exposures @@ -328,8 +343,8 @@ Included are galaxy detection and basic measurement parameters, the PSF model at galaxy positions, the spread-model classification, and the shape measurement. Two output directories are created. -The first one is `run_sp_Ms` for the `merge_sep` run. -The second is `run_sp_Mc` for the `make_cat` task; the name is the same for both the `MCCD` and `PSFEx` PSF model. +The first one is `run_sp_tile_Ms` for the `merge_sep` run. +The second is `run_sp_tile_Mc` for the `make_cat` task; the name is the same for both the `MCCD` and `PSFEx` PSF model. ## Upload results diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md index e8a3768f4..f6af7f8e2 100644 --- a/docs/source/post_processing.md +++ b/docs/source/post_processing.md @@ -9,11 +9,6 @@ catalogue via _metacalibration_), a joint star catalogue, and PSF diagnostic plo ---- - -If main ShapePipe processing happened at the old canfar VM system (e.g. CFIS v0 and v1), go -[here](vos_retrieve.md) for details how to retrieve the ShapePipe output files. - --- ```{note} @@ -76,6 +71,6 @@ The following steps were used for pre-v1.4 runs performed on the canfar VM syste ``` Choose as input directory `input_dir` the `make_cat` output of the runs being combined. A default parameter file `` is - `/path/to/shapepipe/example/cfis/final_cat.param`. + `/path/to/shapepipe/workflow/config/cfis/final_cat.param`. On success, the file `./final_cat.npy` is created. Depending on the number of input tiles, this file can be several tens of Gb large. diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md deleted file mode 100644 index a930d40ed..000000000 --- a/docs/source/random_cat.md +++ /dev/null @@ -1,128 +0,0 @@ -# Create random catalogues and masks - -This section describes how to create tile-based random catalogues and healpix -masks, and combined randoms and masks for a selection of tiles. - -The masked regions are obtained on input from ShapePipe pixel mask ("pipeline flag") -files. - -```{note} -Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see -[VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The `random_cat` module itself is current; -the input-staging and joint-mask steps now overlap with -[`sp_validation`](https://github.com/CosmoStat/sp_validation). The steps are -retained for reference. -``` - -## Set up - -### ID file and shell variables - -First, if if does not exist already, create the file ``tile_numbers.txt`` containing a list of tile IDs, -one per line. This is the same format as the input file to ``get_images_runner``. -For example, link to a patch ID list, -```bash -ln -s tiles_PX.txt tile_numbers.txt -``` -Next, set the run and config paths, -```bash -export SP_RUN=. -export SP_CONFIG=/path/to/config-files -``` - -### Get images or image headers - -We need to footprint of the image tiles. If they have been downloaded for a ``ShapePipe`` run, -check that they are accessible as last run of the ``get_images_runner`` module. - -If not, we can just download the headers to gain significant download time. -```bash -shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini -``` - -### Check pixel mask files - -Make sure that all pixel mask files are present. If they have been downloaded from ``vos`` as ``.tgz`` files, -type -```bash -canfar_avail_results -i tile_numbers.txt --input_path . -v -m -o missing_mask.txt -``` -In case of missing mask files, check whether they are present in the ``vos`` remote directory, -```bash -canfar_avail_results -i tile_numbers.txt --input_path vos:cfis/vos-path/to/results -v -m -``` -If missing on ``vos``, process those tiles. If processing only up the the mask is necessary, -the following steps can be carried out, -```bash -job_sp -j 7 TILE_ID -job_sp -j 128 TILE_ID -``` -The first command processes the tile up to the mask; the second line uploads the mask files -to ``vos``. - -Now, download the missing masks with -```bash -canfar_download_results -i missing_mask.txt --input_vos vos-path/to/results -m -v -``` -Untar .tgz files if required, -```bash -while read p; do tar xvf pipeline_flag_$p.tgz; done `. + +The per-module pipeline configurations driven by the Snakemake workflow live +in [`workflow/config/cfis/`](../../workflow/config/cfis) instead; see +[`workflow/README.md`](../../workflow/README.md). + +The shared SExtractor / PSFEx / catalogue-parameter files (`default.sex` +variants, `default.param`, `default.psfex`, `default.conv`, `final_cat.param`, +`star_selection.setools`) also live in `workflow/config/cfis/`, which is what +`$SP_CONFIG` points at. None of the configs kept here reference them by path, +but if you add one that does, point it at `workflow/config/cfis/` rather than +copying the file back into this directory. diff --git a/example/cfis/config_Ms_psfex.ini b/example/cfis/config_Ms_psfex.ini index 5698ee6dd..a27b43920 100644 --- a/example/cfis/config_Ms_psfex.ini +++ b/example/cfis/config_Ms_psfex.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: False RUN_DATETIME = False diff --git a/example/cfis/config_Ms_psfex_conv.ini b/example/cfis/config_Ms_psfex_conv.ini index 3a68acc41..9d0b97207 100644 --- a/example/cfis/config_Ms_psfex_conv.ini +++ b/example/cfis/config_Ms_psfex_conv.ini @@ -9,7 +9,7 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: False RUN_DATETIME = False diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini deleted file mode 100644 index 2fec0a0fc..000000000 --- a/example/cfis/config_Rc.ini +++ /dev/null @@ -1,76 +0,0 @@ -# ShapePipe configuration file for: create random catalogue - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Rc - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = random_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 24 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[RANDOM_CAT_RUNNER] - -#INPUT_DIR = last:get_images_runner, last:mask_runner -INPUT_DIR = last:get_images_runner, $SP_RUN/output/run_sp_combined_flag:mask_runner - -FILE_PATTERN = CFIS_image, pipeline_flag - -NUMBERING_SCHEME = 000-000 - -# Number of random objects -N_RANDOM = 50000 - -# N_RANDOM is per square degrees if True -DENSITY = True - -# Output healpix mask if True -SAVE_MASK_AS_HEALPIX = True - -# Healpix mask file base name (used if SAVE_MASK_AS_HEALPIX is True) -HEALPIX_OUT_FILE_BASE = mask_hp - -# Healpix mask nside (used if SAVE_MASK_AS_HEALPIX is True) -HEALPIX_OUT_NSIDE = 1024 diff --git a/example/cfis/config_exp_Ma_onthefly.ini b/example/cfis/config_exp_Ma_onthefly.ini deleted file mode 100644 index 71cd45b54..000000000 --- a/example/cfis/config_exp_Ma_onthefly.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -# Path to check for existing output mask files -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/example/cfis/config_get_tiles_vos_headers.ini b/example/cfis/config_get_tiles_vos_headers.ini deleted file mode 100644 index 4b79a2493..000000000 --- a/example/cfis/config_get_tiles_vos_headers.ini +++ /dev/null @@ -1,95 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Git - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[GET_IMAGES_RUNNER] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/tiles_DR5 - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r - -# Input file extensions -INPUT_FILE_EXT = .fits - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image- -#, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = vos - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Copy command options, optional -RETRIEVE_OPTIONS = --head diff --git a/example/cfis/config_make_cat_mccd.ini b/example/cfis/config_make_cat_mccd.ini deleted file mode 100644 index 858341e1f..000000000 --- a/example/cfis/config_make_cat_mccd.ini +++ /dev/null @@ -1,74 +0,0 @@ -# ShapePipe post-run configuration file: create final catalogs - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = make_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = last:sextractor_runner_run_1, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner - -# Output directory -OUTPUT_DIR = ./output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MAKE_CAT_RUNNER] - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .sqlite, .fits - -# Numbering convention, string that exemplifies a numbering pattern. -# Matches input single exposures (with 'p' removed) -# Needs to be given in this section, will be updated in module -# sections below -NUMBERING_SCHEME = -000-000 - -SM_DO_CLASSIFICATION = True -SM_STAR_THRESH = 0.003 -SM_GAL_THRESH = 0.01 - -SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis/config_make_cat_psfex.ini b/example/cfis/config_make_cat_psfex.ini deleted file mode 100644 index a7407d990..000000000 --- a/example/cfis/config_make_cat_psfex.ini +++ /dev/null @@ -1,77 +0,0 @@ -# ShapePipe post-run configuration file: create final catalogs - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = make_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = ./output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[MAKE_CAT_RUNNER] - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:spread_model_runner, last:psfex_interp_runner, last:merge_sep_cats_runner - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .sqlite, .fits - -# Numbering convention, string that exemplifies a numbering pattern. -# Matches input single exposures (with 'p' removed) -# Needs to be given in this section, will be updated in module -# sections below -NUMBERING_SCHEME = -000-000 - -SM_DO_CLASSIFICATION = True -SM_STAR_THRESH = 0.003 -SM_GAL_THRESH = 0.01 - -SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis/config_onthefly.mask b/example/cfis/config_onthefly.mask deleted file mode 100644 index 7c185c602..000000000 --- a/example/cfis/config_onthefly.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_save.mask b/example/cfis/config_save.mask deleted file mode 100644 index 497dedda9..000000000 --- a/example/cfis/config_save.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_tile_Ma_onthefly.ini b/example/cfis/config_tile_Ma_onthefly.ini deleted file mode 100644 index 8f7ef4206..000000000 --- a/example/cfis/config_tile_Ma_onthefly.ini +++ /dev/null @@ -1,82 +0,0 @@ -# ShapePipe configuration file for masking of tiles - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/config_tile_Ng_batch_psfex_uc.ini b/example/cfis/config_tile_Ng_batch_psfex_uc.ini deleted file mode 100644 index f2696bd8f..000000000 --- a/example/cfis/config_tile_Ng_batch_psfex_uc.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ng - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 24 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner,last:psfex_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as -# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for -# every tile (missing file -> error, no per-tile fallback); omit the option -# entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -SAVE_BATCH = 1000 - -ID_OBJ_MIN = -1 -ID_OBJ_MAX = -1 diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini deleted file mode 100644 index bd2af3ea1..000000000 --- a/example/cfis/config_tile_Ng_template.ini +++ /dev/null @@ -1,121 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_NgXu - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as -# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for -# every tile (missing file -> error, no per-tile fallback); omit the option -# entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite - -# Magnitude zero-point -MAG_ZP = 30.0 - -# CENTROID_SOURCE (optional): how to place the galaxy Jacobian origin for the -# centroid prior. "wcs" (default) uses the catalog sky position projected -# through the WCS, trusting the astrometry — the recommended choice. "hsm" -# re-centers on the HSM adaptive-moment centroid (legacy, being phased out; -# noisy for stars and flagged as incorrect — see #767). -CENTROID_SOURCE = wcs - -# BLEND_HANDLING (optional): neighbour treatment. "noisefill" (default, -# historical) replaces a neighbour's pixels with a noise realisation; -# "uberseg" hard-masks (weight -> 0) every pixel closer to a neighbour's -# segmentation footprint than to the central object. "uberseg" REQUIRES -# SEG_VIGNET_PATH below. -# BLEND_HANDLING = uberseg - -# SEG_VIGNET_PATH (optional): coadd-frame SExtractor segmentation vignets -# (the CLASSIC-mode VIGNETMAKER_RUNNER_RUN_3 seg output), row-aligned to the -# tile catalogue and on the same 51x51 grid as the coadd VIGNET. Required for -# BLEND_HANDLING = uberseg; when set, the file must exist for every tile -# (missing file -> error). Omit for the noise-fill path. -# SEG_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_3/output/seg_vignet{file_number_string}.fits - -# DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the uberseg -# neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay offset. -# Ignored unless BLEND_HANDLING = uberseg. Default 1 (~one pixel); 0 recovers -# the pure Sheldon UberSeg mask. -# DILATE_NEIGHBOUR = 1 - -# SEED_FROM_POSITION (optional): FALSE (default) seeds one RNG per tile. -# TRUE seeds a per-object RNG from the object's sky position instead, so that -# metacal's fixnoise counter-noise and the fit guesses are identical for the -# same object across Pujol image-simulation shear branches and cancel in the -# branch difference — shrinking the m-bias error. This is FOR IMAGE -# SIMULATIONS ONLY (Pujol noise cancellation, ngmix#796); leave it off (or -# omit) for real data, where it has no benefit. -# SEED_FROM_POSITION = FALSE - -# METACAL_PSF (optional): the metacal reconvolution-kernel scheme -# (metacal_pars['psf']). "fitgauss" (default) fits a Gaussian to the PSF and -# rounds it; "gauss" reconvolves with a fixed round Gaussian sized from the -# PSF; "dilate" dilates the original PSF; "azgauss" (ngmix >= 2.4.1) is a -# noise-robust variant of "gauss". Sets the round PSF metacal reconvolves -# with after shearing, so it moves the metacal response. -METACAL_PSF = fitgauss - -ID_OBJ_MIN = X -ID_OBJ_MAX = X diff --git a/example/cfis/config_tile_Ng_template_batch.ini b/example/cfis/config_tile_Ng_template_batch.ini deleted file mode 100644 index 655397156..000000000 --- a/example/cfis/config_tile_Ng_template_batch.ini +++ /dev/null @@ -1,81 +0,0 @@ -# ShapePipe configuration file for tiles: ngmix + KSB - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_ngmix_NgXu - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = ngmix_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Model-fitting shapes with ngmix -[NGMIX_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers - -FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as -# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for -# every tile (missing file -> error, no per-tile fallback); omit the option -# entirely to fall back to the scalar sigma_mad noise estimate. -BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite - -# Number of objects to batch save during processing, optional. Omit or set -# to -1 for no batch saving -SAVE_BATCH = 1000 - -# Magnitude zero-point -MAG_ZP = 30.0 - -ID_OBJ_MIN = X -ID_OBJ_MAX = X diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini deleted file mode 100644 index 12f158508..000000000 --- a/example/cfis/config_tile_Sx.ini +++ /dev/null @@ -1,114 +0,0 @@ -# ShapePipe configuration file for tile detection - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner, run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag, log_exp_headers - -FILE_EXT = .fits, .fits, .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND, SEGMENTATION - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis/config_tile_onthefly.mask b/example/cfis/config_tile_onthefly.mask deleted file mode 100644 index 69ad20769..000000000 --- a/example/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/config_tile_save.mask b/example/cfis/config_tile_save.mask deleted file mode 100644 index 82c4b66af..000000000 --- a/example/cfis/config_tile_save.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/defunct/config_Gie_symlink.ini b/example/cfis/defunct/config_Gie_symlink.ini deleted file mode 100644 index e6fa49f4c..000000000 --- a/example/cfis/defunct/config_Gie_symlink.ini +++ /dev/null @@ -1,97 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Gie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get exposures -[GET_IMAGES_RUNNER] - -INPUT_DIR = last:find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem diff --git a/example/cfis/defunct/config_GitFeGie_symlink.ini b/example/cfis/defunct/config_GitFeGie_symlink.ini deleted file mode 100644 index 6f84d1482..000000000 --- a/example/cfis/defunct/config_GitFeGie_symlink.ini +++ /dev/null @@ -1,151 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_GitFeGie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner, find_exposures_runner, get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER_RUN_1] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink - -# Copy command options, optional -RETRIEVE_OPTIONS = -L - - -[FIND_EXPOSURES_RUNNER] - -INPUT_MODULE = get_images_runner_run_1 - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p - -# Get exposures -[GET_IMAGES_RUNNER_RUN_2] - -INPUT_MODULE = find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = -L diff --git a/example/cfis/defunct/config_GitFeGie_vos.ini b/example/cfis/defunct/config_GitFeGie_vos.ini deleted file mode 100644 index 75044bf44..000000000 --- a/example/cfis/defunct/config_GitFeGie_vos.ini +++ /dev/null @@ -1,155 +0,0 @@ -# ShapePipe configuration file for: get images - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_GitFeGie - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner, find_exposures_runner, get_images_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER_RUN_1] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/tiles_DR5, vos:cfis/tiles_DR5 - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = vos - -# Copy command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem - -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output - -[FIND_EXPOSURES_RUNNER] - -INPUT_MODULE = get_images_runner_run_1 - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p - - -# Get exposures -[GET_IMAGES_RUNNER_RUN_2] - -INPUT_DIR = last:find_exposures_runner - -FILE_PATTERN = exp_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - - -# Paths - -# Output path (optional, default is [FILE]:OUTPUT_DIR -# OUTPUT_PATH = input_images - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags -# LSB images: -#INPUT_PATH = vos:cfis/lsb_individual, vos:cfis/weights, vos:cfis/flags - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag -# LSB images -#INPUT_FILE_PATTERN = 000000s, 000000p.weight, 000000p.flag - -# Input file extensions -INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{6} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = image-, weight-, flag- - -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = vos - -# If RETRIEVE=vos, number of attempts to download -# Optional, default=3 -N_TRY = 3 - -# Retrieve command options, optional -RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem diff --git a/example/cfis/defunct/config_GitFe_symlink.ini b/example/cfis/defunct/config_GitFe_symlink.ini deleted file mode 100644 index 4e7ce75cb..000000000 --- a/example/cfis/defunct/config_GitFe_symlink.ini +++ /dev/null @@ -1,105 +0,0 @@ -# ShapePipe configuration file for: get images and find exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = False - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_GitFe - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = get_images_runner, find_exposures_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -# Get tiles -[GET_IMAGES_RUNNER] - -FILE_PATTERN = tile_numbers - -FILE_EXT = .txt - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = - -# Paths - -# Input path where original images are stored. Can be local path or vos url. -# Single string or list of strings -INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles - -# Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight - -# Input file extensions -INPUT_FILE_EXT = .fits, .fits.fz - -# Input numbering scheme, python regexp -INPUT_NUMBERING = \d{3}\.\d{3} - -# Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- - -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink - -# Copy command options, optional -RETRIEVE_OPTIONS = -L - - -[FIND_EXPOSURES_RUNNER] - -INPUT_MODULE = get_images_runner - -FILE_PATTERN = CFIS_image - -FILE_EXT = .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Column number of exposure name in FITS header -COLNUM = 3 - -# Prefix to remove from exposure name -EXP_PREFIX = p diff --git a/example/cfis/defunct/config_MaMa_onthefly.ini b/example/cfis/defunct/config_MaMa_onthefly.ini deleted file mode 100644 index 84f117e65..000000000 --- a/example/cfis/defunct/config_MaMa_onthefly.ini +++ /dev/null @@ -1,105 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_MaMa_save.ini b/example/cfis/defunct/config_MaMa_save.ini deleted file mode 100644 index 4bd1b00ef..000000000 --- a/example/cfis/defunct/config_MaMa_save.ini +++ /dev/null @@ -1,109 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, star_cat_tiles - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight, star_cat - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner, star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_exp_SpMh.ini b/example/cfis/defunct/config_exp_SpMh.ini deleted file mode 100644 index cfe51ccfc..000000000 --- a/example/cfis/defunct/config_exp_SpMh.ini +++ /dev/null @@ -1,85 +0,0 @@ -# ShapePipe configuration file for single-exposures, -# split images, merge headers - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SpMh - -# Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = True - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = split_exp_runner, merge_headers_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SPLIT_EXP_RUNNER] - -INPUT_DIR = last:get_images_runner - -FILE_PATTERN = image, weight, flag - -# Matches compressed single-exposure files -FILE_EXT = .fitsfz, .fitsfz, .fitsfz - -NUMBERING_SCHEME = -0000000 - -# OUTPUT_SUFFIX, actually file name prefixes. -# Expected keyword "flag" will lead to a behavior where the data are saved as int. -# The code also expects the image data to use the "image" suffix -# (default value in the pipeline). -OUTPUT_SUFFIX = image, weight, flag - -# Number of HDUs/CCDs of mosaic -N_HDU = 40 - - -[MERGE_HEADERS_RUNNER] - -FILE_PATTERN = headers - -FILE_EXT = .npy - -# Single-exposure numbering scheme -NUMBERING_SCHEME = -0000000 - diff --git a/example/cfis/defunct/config_tile_PiViSmVi.ini b/example/cfis/defunct/config_tile_PiViSmVi.ini deleted file mode 100644 index 03ebf5fbb..000000000 --- a/example/cfis/defunct/config_tile_PiViSmVi.ini +++ /dev/null @@ -1,184 +0,0 @@ -# ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_PsViSmVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -#MODULE = psfex_interp_runner, - -MODULE = psfex_interp_runner, vignetmaker_runner, spread_model_runner, - vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[PSFEX_INTERP_RUNNER] - -INPUT_DIR = last:sextractor_runner_run_1, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -# Number of stars threshold -STAR_THRESH = 20 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -ME_DOT_PSF_DIR = psfex_runner - -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - - -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = last:sextractor_runner_run_1, last:uncompress_fits_runner - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[SPREAD_MODEL_RUNNER] - -INPUT_DIR = last:sextractor_runner_run_1, last:psfex_interp_runner, last:vignetmaker_runner_run_1 - -FILE_PATTERN = sexcat, galaxy_psf, weight_vignet - -FILE_EXT = .fits, .sqlite, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Pixel scale in arcsec -PIXEL_SCALE = 0.186 - -# Output mode: -# new: create a new catalog with: [number, mag, sm, sm_err] -# add: create a copy of the input SExtractor with the column sm and sm_err -OUTPUT_MODE = new - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_DIR = last:sextractor_runner_run_1, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -PREFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs -ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2 -ME_IMAGE_PATTERN = flag, image, weight, background diff --git a/example/cfis/defunct/config_tile_PiViSmVi_canfar.ini b/example/cfis/defunct/config_tile_PiViSmVi_canfar.ini deleted file mode 100644 index ceb356855..000000000 --- a/example/cfis/defunct/config_tile_PiViSmVi_canfar.ini +++ /dev/null @@ -1,184 +0,0 @@ -# ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_PsViSmVi - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -#MODULE = psfex_interp_runner, - -MODULE = psfex_interp_runner, vignetmaker_runner, spread_model_runner, - vignetmaker_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[PSFEX_INTERP_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Column names of position parameters -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# If True, measure and store ellipticity of the PSF -GET_SHAPES = True - -# Number of stars threshold -STAR_THRESH = 20 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -ME_DOT_PSF_DIR = all:psfex_runner - -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - - -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[SPREAD_MODEL_RUNNER] - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:vignetmaker_runner_run_1 - -FILE_PATTERN = sexcat, galaxy_psf, weight_vignet - -FILE_EXT = .fits, .sqlite, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Pixel scale in arcsec -PIXEL_SCALE = 0.186 - -# Output mode: -# new: create a new catalog with: [number, mag, sm, sm_err] -# add: create a copy of the input SExtractor with the column sm and sm_err -OUTPUT_MODE = new - - -[VIGNETMAKER_RUNNER_RUN_2] - -# Create multi-epoch vignets for tiles corresponding to -# positions on single-exposures - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_exp_Mh:merge_headers_runner - -FILE_PATTERN = sexcat, log_exp_headers - -FILE_EXT = .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = MULTI-EPOCH - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = SPHE -POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is vignet.fits -PREFIX = - -# Additional parameters for path and file pattern corresponding to single-exposure -# run outputs -ME_IMAGE_DIR = all:split_exp_runner, all:split_exp_runner, all:split_exp_runner, all:sextractor_runner -ME_IMAGE_PATTERN = flag, image, weight, background diff --git a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini b/example/cfis/defunct/config_tile_Sx_exp_mccd.ini deleted file mode 100644 index fec79f177..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini +++ /dev/null @@ -1,274 +0,0 @@ -# ShapePipe configuration file for single-exposures, MCCD PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -## Detection on tile -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -## Detection on single exposures -[SEXTRACTOR_RUNNER_RUN_2] - -INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2 - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner_run_2 - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) if True -DETECTION_WEIGHT = False - -# Se to True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[MCCD_PREPROCESSING_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -INPUT_DIR = last:setools_runner - -# Input are individual CCDs, thus single-exposure single-HDU images -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - - -[MCCD_FIT_VAL_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -NUMBERING_SCHEME = -0000000 - - -[MERGE_STARCAT_RUNNER] - -INPUT_DIR = last:mccd_fit_val_runner - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -PSF_MODEL = mccd - -NUMBERING_SCHEME = -0000000 - - -[MCCD_PLOTS_RUNNER] - -# Now MCCD has created a focal-plane PSF model, including all CCDS per images, -# thus single-exposure files -NUMBERING_SCHEME = -0000000 - -PSF = mccd - -PLOT_MEANSHAPES = True - -# X_GRID, Y_GRID: correspond to the number of bins in each direction of each -# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. -X_GRID = 5 -Y_GRID = 10 - -PLOT_HISTOGRAMS = True - -# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape -# before drawing the plots. -REMOVE_OUTLIERS = False - diff --git a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini b/example/cfis/defunct/config_tile_Sx_exp_psfex.ini deleted file mode 100644 index ea86ea048..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini +++ /dev/null @@ -1,248 +0,0 @@ -# ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 40 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -[SEXTRACTOR_RUNNER_RUN_2] - -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -# True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[PSFEX_RUNNER] - -# Use 80% sample for PSF model -FILE_PATTERN = star_split_ratio_80 - -NUMBERING_SCHEME = -0000000-0 - -# Path to executable for the PSF model (optional) -EXEC_PATH = psfex - -# Default psfex configuration file -DOT_PSFEX_FILE = $SP_CONFIG/default.psfex - - -[PSFEX_INTERP_RUNNER] - -# Use 20% sample for PSF validation -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat - -FILE_EXT = .psf, .fits, .cat - -NUMBERING_SCHEME = -0000000-0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = VALIDATION - -# Column names of position parameters -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# If True, measure and store ellipticity of the PSF (using moments) -GET_SHAPES = True - -# Minimum number of stars per CCD for PSF model to be computed -STAR_THRESH = 22 - -# Maximum chi^2 for PSF model to be computed on CCD -CHI2_THRESH = 2 diff --git a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg deleted file mode 100644 index 4e4164aaf..000000000 --- a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg +++ /dev/null @@ -1,24 +0,0 @@ --11.5 68 --6 186.5 -7 188 -10 64.5 -31 55 -50 38.5 -56.5 11.5 -188 8 -192 -4 -59.5 -11.5 -45 -33 -13.5 -64 -5 -154 --6 -155 --11 -64.5 --40 -44.5 --51.5 -30.5 --62.5 -22.5 --68 -9.5 --177 -2 --176 3 --78 12.5 --67.5 14.5 --38.5 50 diff --git a/example/cfis/mask_default/Messier_catalog.npy b/example/cfis/mask_default/Messier_catalog.npy deleted file mode 100644 index ef07eb032..000000000 Binary files a/example/cfis/mask_default/Messier_catalog.npy and /dev/null differ diff --git a/example/cfis/mask_default/Messier_catalog_updated.fits b/example/cfis/mask_default/Messier_catalog_updated.fits deleted file mode 100644 index 6a9f00096..000000000 Binary files a/example/cfis/mask_default/Messier_catalog_updated.fits and /dev/null differ diff --git a/example/cfis/mask_default/default.ww b/example/cfis/mask_default/default.ww deleted file mode 100644 index c2797f904..000000000 --- a/example/cfis/mask_default/default.ww +++ /dev/null @@ -1,40 +0,0 @@ -#--------------------------------- Weights ------------------------------------ - -WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) - -WEIGHT_MIN 0. # Pixel below those thresholds will be flagged -WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged -WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels - -#---------------------------------- Flags ------------------------------------- - -FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) - -FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels -FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs -FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits - -#---------------------------------- Polygons ---------------------------------- - -POLY_NAMES "" # Filename(s) of input DS9 regions -POLY_OUTFLAGS # FLAG values for polygon masks -POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks -POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? - -#---------------------------------- Output ------------------------------------ - -OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename -OUTFLAG_NAME flag.fits # Output FLAG-map filename - -#----------------------------- Miscellaneous --------------------------------- - -GETAREA N # Compute area for flags and weights (Y/N)? -GETAREA_WEIGHT 0.0 # Weight threshold for area computation -GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area -MEMORY_BUFSIZE 256 # Buffer size in lines -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -WRITE_XML N # Write XML file (Y/N)? -XML_NAME ww.xml # Filename for XML output -XSL_URL file:///usr/local/share/weightwatcher/ww.xsl - # Filename for XSL style-sheet -NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/example/cfis/mask_default/halo_mask.reg b/example/cfis/mask_default/halo_mask.reg deleted file mode 100644 index c44f25167..000000000 --- a/example/cfis/mask_default/halo_mask.reg +++ /dev/null @@ -1,50 +0,0 @@ - 274.66813 -1.25966 - 272.54579 32.47406 - 266.21222 65.67579 - 255.76731 97.82190 - 241.37579 128.40544 - 223.26462 156.94408 - 201.71942 182.98775 - 177.07997 206.12573 - 149.73486 225.99312 - 120.11532 242.27660 - 88.68848 254.71937 - 55.94996 263.12519 - 22.41606 267.36151 - -11.38436 267.36151 - -44.91826 263.12519 - -77.65678 254.71937 --109.08362 242.27660 --138.70315 225.99312 --166.04827 206.12573 --190.68772 182.98775 --212.23292 156.94408 --230.34409 128.40544 --244.73561 97.82190 --255.18052 65.67579 --261.51409 32.47406 --263.63643 -1.25966 --261.51409 -34.99339 --255.18052 -68.19511 --244.73561 -100.34123 --230.34409 -130.92476 --212.23292 -159.46341 --190.68772 -185.50708 --166.04827 -208.64506 --138.70315 -228.51245 --109.08362 -244.79593 - -77.65678 -257.23870 - -44.91826 -265.64452 - -11.38436 -269.88084 - 22.41606 -269.88084 - 55.94996 -265.64452 - 88.68848 -257.23870 - 120.11532 -244.79593 - 149.73486 -228.51245 - 177.07997 -208.64506 - 201.71942 -185.50708 - 223.26462 -159.46341 - 241.37579 -130.92476 - 255.76731 -100.34123 - 266.21222 -68.19511 - 272.54579 -34.99339 diff --git a/example/cfis/mask_default/ngc_cat.fits b/example/cfis/mask_default/ngc_cat.fits deleted file mode 100644 index f51546da7..000000000 Binary files a/example/cfis/mask_default/ngc_cat.fits and /dev/null differ diff --git a/example/cfis_image_sims/README.md b/example/cfis_image_sims/README.md index b308cd6a8..fa0e00ac4 100644 --- a/example/cfis_image_sims/README.md +++ b/example/cfis_image_sims/README.md @@ -18,7 +18,10 @@ those cases are documented in the last column below and, at more length, under Every row is derived from the two bash scripts. "Module(s)" is the ShapePipe runner(s) the selected `.ini` names; "`.ini` selected" is what `job_sp_canfar_v2.0.bash` picks for that bit under sim settings -(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`, `star_cat_for_mask=onthefly`). +(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`). Bit 32 (mask exposures) is +gone: ShapePipe generates no masks (PR #847), so the bash scripts' +`star_cat_for_mask` setting and the `config_*_Ma_*.ini` configs it selected no +longer exist. | Bit | Stage | Module(s) | `.ini` selected (sim settings) | Sim special-casing | |----:|-------|-----------|--------------------------------|--------------------| @@ -27,10 +30,9 @@ runner(s) the selected `.ini` names; "`.ini` selected" is what | 4 | find exposures | `find_exposures_runner` | `config_tile_Fe.ini` | — | | 8 | retrieve exposure images | `get_images_runner` | `config_exp_Gie_symlink.ini` | symlink retrieval; completeness check expects 3 files vs. 6 for data | | 16 | split exposures, merge WCS headers | `split_exp_runner` | `config_exp_Sp.ini` | — | -| 32 | mask exposures | `mask_runner` | `config_exp_Ma_onthefly.ini` | — | | 64 | exposure PSF model | *(none — placeholder)* | *(none)* | **Placeholder.** For data this runs full exposure PSF modelling. For sims run_job writes a placeholder log and does nothing here; the sim PSF (`fake_psf_runner`) actually runs inside bit 512 | | 128 | merge exposure WCS headers → tile sqlite log | `merge_headers_runner` | `config_tile_Mh_exp.ini` | — | -| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx_nomask.ini` | `tile_det` is forced to `sx`, so the SExtractor-no-mask branch is always taken (the `uc` external-catalogue branch is never reached for sims) | +| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx.ini` | `tile_det` is forced to `sx`, so the SExtractor branch is always taken (the `uc` external-catalogue branch is never reached for sims). Tiles carry no flag image — ShapePipe generates no masks — so this runs with `FLAG_IMAGE = False` | | 512 | fake PSF + postage stamps | `fake_psf_runner`, then `vignetmaker_runner` ×2 | `config_exp_psfex.ini` (fake PSF), then `config_tile_PiViVi_canfar_sx.ini` (vignets) | **Two sub-runs.** run_job first calls the job script with `-j 64` → `config_exp_psfex.ini`, which despite its name runs `fake_psf_runner` (needs the sexcat from bit 256; run dir `run_sp_tile_fpsf`), then `-j 512` → `config_tile_PiViVi_canfar_sx.ini` for the two `vignetmaker_runner` runs. Data instead runs `psfex_interp_runner` + vignets here | | 1024 | multi-epoch shape measurement | `ngmix_runner` | `config_tile_Ng_batch_psfex_sx.ini` | — | | 2048 | create final catalogue | `make_cat_runner` | `config_tile_Mc_psfex.ini` | — | diff --git a/example/cfis_image_sims/config_exp_Ma_onthefly.ini b/example/cfis_image_sims/config_exp_Ma_onthefly.ini deleted file mode 100644 index df0307a19..000000000 --- a/example/cfis_image_sims/config_exp_Ma_onthefly.ini +++ /dev/null @@ -1,76 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask_simu - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis_image_sims/config_onthefly.mask_simu b/example/cfis_image_sims/config_onthefly.mask_simu deleted file mode 100644 index 1a63cc2e5..000000000 --- a/example/cfis_image_sims/config_onthefly.mask_simu +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = False - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = False - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = False - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = False - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis_image_sims/config_tile_Ma_onthefly.ini b/example/cfis_image_sims/config_tile_Ma_onthefly.ini deleted file mode 100644 index 0f49eccea..000000000 --- a/example/cfis_image_sims/config_tile_Ma_onthefly.ini +++ /dev/null @@ -1,82 +0,0 @@ -# ShapePipe configuration file for masking of tiles - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = run_sp_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask_simu - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini index 8ca298565..3479a972a 100644 --- a/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini +++ b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini @@ -72,11 +72,5 @@ BKG_SUB = False SAVE_BATCH = 1000 -# Position-seeded per-object fixnoise RNG (#796/#803): the SKiLLS shear -# branches share their sky-noise realization, so seeding the metacal fixnoise -# from sky position makes the added noise identical across branches too and -# the Pujol estimator (Pujol, Kilbinger, Sureau & Bobin 2018, 621, A2) cancels it. -SEED_FROM_POSITION = True - ID_OBJ_MIN = -1 ID_OBJ_MAX = -1 diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini index 28ed565af..26564e4fa 100644 --- a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -130,7 +130,7 @@ ME_IMAGE_PATTERN = flag, image, weight [VIGNETMAKER_RUNNER_RUN_3] # Cut per-object coadd-frame segmentation stamps from the tile SExtractor -# SEGMENTATION check image (config_tile_Sx_nomask.ini: CHECKIMAGE = BACKGROUND, +# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, # SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode # guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN # centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg diff --git a/example/cfis_image_sims/config_tile_Sx_nomask.ini b/example/cfis_image_sims/config_tile_Sx.ini similarity index 91% rename from example/cfis_image_sims/config_tile_Sx_nomask.ini rename to example/cfis_image_sims/config_tile_Sx.ini index a5c12771c..78bcfb646 100644 --- a/example/cfis_image_sims/config_tile_Sx_nomask.ini +++ b/example/cfis_image_sims/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options diff --git a/example/cfis_image_sims/config_tile_onthefly.mask_simu b/example/cfis_image_sims/config_tile_onthefly.mask_simu deleted file mode 100644 index 2e404ed96..000000000 --- a/example/cfis_image_sims/config_tile_onthefly.mask_simu +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 1 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = False - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = False - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = False - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = False - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml new file mode 100644 index 000000000..23d385bd3 --- /dev/null +++ b/profiles/nibi/config.yaml @@ -0,0 +1,157 @@ +# Snakemake profile for the Nibi cluster (Digital Research Alliance). +# +# SLURM-EXECUTOR mode (PRD #848 D-profile): one SLURM job per rule instance, +# each carrying its rule's own attempt-scaled resources (cpus_per_task, +# mem_mb, runtime). Snakemake feeds the queue as jobs finish, so DR6's ~170k +# total jobs never queue at once, and multi-node scaling is inherent — this +# supersedes the earlier one-allocation/local-scheduler mode. +# +# Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). +# software-deployment-method wraps every job's shell in `apptainer exec` — the +# user never types apptainer. WHICH image is resolved in the Snakefile through +# workflow/scripts/container.py, whose docstring documents the layers and the +# resolution order; `sp container status` prints which layer is live. + +executor: slurm + +# Per-user submit cap, queried 2026-07-30 on nibi: +# sacctmgr show assoc user=cdaley format=Account,MaxSubmitPU -P +# -> def-mjhudson_cpu 1000, def-mjhudson_gpu 1000 (MaxSubmitPU; no site-wide +# MaxSubmitJobs in `scontrol show config`, so the association limit governs) +# Set to ~80% of that (1000) so this workflow never starves other submissions +# under the same account. Re-query if the association limits change. +jobs: 800 + +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: def-mjhudson_cpu + +software-deployment-method: [apptainer] +# Explicit container environment (finding 2/3): the apptainer SDM otherwise drops +# the proven-recipe env and hardcodes --home , hiding ~/.ssl/cadcproxy.pem. +# --cleanenv strict host-env isolation (APPTAINERENV_*/SINGULARITYENV_* survive) +# OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) +# MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) +# --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# >>> EDIT THIS: THE ONE NON-PORTABLE LINE IN THIS FILE. <<< +# The apptainer-args string below hardcodes one user's home +# (--home /home/cdaley). A different user must edit it by hand. +# IT CANNOT BE FIXED HERE: YAML cannot splice an env var, and +# snakemake escapes a literal `$` before the string reaches +# any shell (see the $SLURM_TMPDIR post-mortem below, which +# cost a whole campaign). Nor can bin/sp inject it through +# --apptainer-args: that REPLACES the profile value rather +# than appending to it, so the launcher would have to restate +# this whole string -- and workflow/scripts/container.py reads +# THIS LINE to give `sp container exec` the same environment +# jobs get, so a second definition is exactly the divergence +# that mechanism exists to prevent. +# THE PYTHONPATH PIN IS THE ONE EXCEPTION, and it is not +# edited by hand: `sp run` copies this whole file into the +# campaign's code snapshot and rewrites that ONE path to the +# snapshot's src/ (bin/sp, "the launch code snapshot"). Still +# one source of truth for the flags -- the copy is generated, +# never edited -- and jobs stop reading a live checkout. +# --bind /local NODE-LOCAL NVME. $SLURM_TMPDIR on nibi is +# /local/scratch/..0, and the mount is /local +# (/dev/nvme0n1, ext4, 3.5 TB) -- binding /localscratch is +# NOT enough, the real path is under /local (verified, +# probe job 20794182). The fused `tile_shape` group job +# WRITES its 5.6 GB vignette store there and never puts it +# on scratch at all (workflow/rules/tile.smk); every other +# rule simply ignores the bind. Adding it does NOT +# invalidate anything: the `software-env` rerun trigger +# hashes only job.container_img_url (+ conda env, env +# modules) -- +# snakemake/persistence/__init__.py::_software_stack_hash, +# v9.23.1 -- and apptainer-args is not in that hash. +# +# HOW the in-container shell LEARNS its node-local dir: +# IT DOES NOT. It DERIVES the path from the tile wildcard +# (tile.smk::tile_local -> /local/scratch/sp-), and +# that is the only mechanism that survives contact with +# snakemake. Three alternatives were tried; all fail, and +# the third failed IN PRODUCTION, so read this before +# reaching for any of them again: +# * {resources.tmpdir} is NOT $SLURM_TMPDIR. Snakemake +# defers that resource to the job and evaluates it as +# tempfile.gettempdir(); nibi's Slurm does not export +# TMPDIR, so it lands on /tmp -- which on a compute +# node is a 378 GB RAM-backed tmpfs whose pages are +# charged to the job's memory cgroup. Staging 6 GB +# there would eat the job's RAM, not use the NVMe. +# (Measured, job 20794656.) +# * APPTAINERENV_SLURM_TMPDIR would have to be exported +# by something that already knows the JOB's value; +# bin/sp runs on the login node, where it is unset. +# * SPLICING "$SLURM_TMPDIR" INTO apptainer-args AND +# LETTING THE JOB'S SHELL EXPAND IT DOES NOT WORK. +# This was the shipped design for exactly one campaign +# and it killed every tile of run 20798193 in eight +# minutes: snakemake ESCAPES the `$` before the string +# reaches any shell, so the container receives the +# literal characters and never a path. NOTHING can be +# carried into the container this way -- not a tmpdir, +# not anything else. The bind below is static, and the +# VALUE is derived job-side from a wildcard instead. +# Every tile_shape member still REFUSES TO RUN if the +# derived directory is not real (tile.smk), which is the +# only reason that campaign cost minutes and not a night: +# a broken node-local path is a loud failure, never a +# silent slide back onto NFS, and never a 5.6 GB write +# into the RAM-backed /tmp tmpfs. +# PYTHONPATH SETTLED CALL 3 — pins THIS branch's src/, which is +# develop@97e16d50 plus the four commits that genuinely need +# module code: ngmix chunk fields + position-seeded RNG, +# merge_sep_cats chunk paths, the vizier star-cat helpers, +# and the cherry-picked #873 (seeded setools split). +# NOT shapepipe-prod (drifted to a PR branch mid-run, and the +# live p3-batch1 job reads it) and not the sif default +# (frozen pre-#843, and pre-#873). Production later rebuilds +# the sif at the validated commit and DROPS this --env line. +# The path written HERE is the checkout's, and is what +# `sp container exec` uses; the value JOBS see is the +# snapshot's src/, substituted per the paragraph above. So a +# running campaign holds its launch code even after this +# line (or the tree it names) changes. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch --bind /local" + +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true + +# No `keep-incomplete` here. The declared output IS the manifest, so letting +# snakemake delete a failed job's output is exactly the manifest-vs-log +# semantics this workflow wants (completeness.py's docstring argues it), and +# `show-failed-logs` above surfaces the surviving log on the console. + +# rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still +# propagate — completeness.py writes the manifest only on change, so mtimes move +# only when reality moves. +# +# `input` is dropped because reclamation needs the tile->exposure edge to be +# CONDITIONAL; with the trigger on, that conditional itself reruns every +# finished tile against a reclaimed exposure store. tile.smk's tile_finished +# commentary carries the mechanism and the fixture-t4 job counts. +# +# Nothing this workflow relied on is lost. The two genuinely data-derived input +# sets are covered another way: a changed exposure list arrives through the +# (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides +# in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute +# + this profile), so a bare snakemake invocation cannot quietly recombine +# reclamation with the `input` trigger. +rerun-triggers: [mtime, params, code, software-env] + +# NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a +# rule's own values (verified snakemake 9.23), which would kill the +# attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +# ngmix thread count, and would flatten the group resource composition. The +# RULES own threads and resources; this profile only sets defaults for rules +# that state nothing (default-resources above). +# +# `group:` fusion of the short rules (PRD D-profile) lives in the rule files, +# each label documented in its file's docstring: `tile_prep` (prepare.smk), +# `exp_short` (exposure.smk) and `tile_shape` (tile.smk). diff --git a/pyproject.toml b/pyproject.toml index 5333aa91c..74d0299fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ requires-python = ">=3.12" # the code actually requires the newer API. dependencies = [ "astropy>=7.0", # major 6 → 7 - "astroquery", "canfar", "cs_util>=0.2.1", "galsim>=2.8", @@ -38,7 +37,6 @@ dependencies = [ "python-pysap>=0.3", "PyQt5", "pyqtgraph", - "reproject>=0.19", "sf_tools>=2.0.4", "skaha>=1.7", "skyproj", @@ -150,4 +148,5 @@ testpaths = ["tests"] markers = [ "slow: heavy compute (minutes); excluded from the fast inner loop.", "candide: needs the candide cluster and/or its real data; auto-skipped elsewhere.", + "unions: uses UNIONS-specific survey layout or data.", ] diff --git a/scripts/README.rst b/scripts/README.rst index 1d3e265a6..2b8067cb0 100644 --- a/scripts/README.rst +++ b/scripts/README.rst @@ -9,7 +9,6 @@ Python scripts ============== 1. `create_log_exp_headers`_ -2. `create_star_cat`_ create_log_exp_headers ====================== @@ -18,11 +17,3 @@ This as to run after the module `split_exp_runner` it will create a master log file containing all the WCS information for each CCDs of each single exposures. To run the script : `python create_log_exp_headers.py path/to/split_exp_runner/output path/to/srcipt/output_dir` - -create_star_cat -=============== - -This script create all the star catalogs required to run the mask module for a -computational node without internet access. -To run the script : -`python create_star_cat.py path/to/image_dir path/to/script/output_dir` diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py deleted file mode 100755 index 62fbf0564..000000000 --- a/scripts/python/canfar_avail_results.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python - -"""Script canfar_avail_results.py - -Check whether results files are available on vos. - -:Author: Martin Kilbinger - -:Date: 07/2020 -""" - -import re -import os -import sys -import glob -import copy -import io -from contextlib import redirect_stdout - -from optparse import OptionParser - -from cs_util.canfar import dir_list - -from shapepipe.utilities import cfis - - -def params_default(): - """Set default parameter values. - - Parameters - ---------- - None - - Returns - ------- - p_def: class cfis.param - parameter values - """ - - p_def = cfis.param( - input_IDs=".", - input_path="vos:cfis/cosmostat/kilbinger/results", - psf="mccd", - extension="tgz", - ) - - return p_def - - -def parse_options(p_def): - """Parse command line options. - - Parameters - ---------- - p_def: class cfis.param - parameter values - - Returns - ------- - options: tuple - Command line options - args: string - Command line string - """ - - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - # I/O - parser.add_option( - "-i", - "--input_IDs", - dest="input_IDs", - type="string", - default=p_def.input_IDs, - help="input tile ID file(s) or directory path, default='{}'".format( - p_def.input_IDs - ), - ) - parser.add_option( - "", - "--input_path", - dest="input_path", - type="string", - default=p_def.input_path, - help="input path, local or vos url, default='{}'".format( - p_def.input_path - ), - ) - parser.add_option( - "-o", - "--output_not_avail", - dest="output_not_avail", - type="string", - help="output file for not-available IDs, default no output", - ) - - parser.add_option( - "-p", - "--psf", - dest="psf", - type="string", - default=p_def.psf, - help="PSF model, one in ['psfex'|'mccd'], default='{}'".format( - p_def.psf - ), - ) - - parser.add_option( - "-f", - "--final_only", - dest="final_only", - action="store_true", - help="only check final catalogues", - ) - parser.add_option( - "-m", - "--mask_only", - dest="mask_only", - action="store_true", - help="only check mask files (pipeline_flag)", - ) - parser.add_option( - "-x", - "--extension", - dest="extension", - type="string", - default=p_def.extension, - help=f"file extension, default='{p_def.extension}'", - ) - parser.add_option( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose output", - ) - - options, args = parser.parse_args() - - return options, args - - -def check_options(options): - """Check command line options. - - Parameters - ---------- - options: tuple - Command line options - - Returns - ------- - erg: bool - Result of option check. False if invalid option value. - """ - - if options.psf not in ["psfex", "mccd"]: - print("Invalid PSF model '{}'".format(options.psf)) - return False - - if options.final_only and options.mask_only: - print("One one of the options '-f' or '-m' can be given") - return False - - return True - - -def update_param(p_def, options): - """Return default parameter, updated and complemented according to options. - - Parameters - ---------- - p_def: class param - parameter values - optiosn: tuple - command line options - - Returns - ------- - param: class param - updated paramter values - """ - - param = copy.copy(p_def) - - # Update keys in param according to options values - for key in vars(param): - if key in vars(options): - setattr(param, key, getattr(options, key)) - - # Add remaining keys from options to param - for key in vars(options): - if not key in vars(param): - setattr(param, key, getattr(options, key)) - - # Do extra stuff if necessary - - return param - - -def read_input_files(input_path, verbose=False): - """Return list of ID files. - - Parameters - ---------- - input_path: string - list of files or directory name - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - ID_files: list of strings - file names with tile IDs - """ - - if os.path.isdir(input_path): - input_files = glob.glob("{}/*".format(input_path)) - else: - input_files = cfis.my_string_split(input_path, stop=True, sep=" ") - - ID_files = [] - for f in input_files: - if os.path.isdir(f): - if verbose: - print("Skipping directory '{}'".format(f)) - else: - ID_files.append(f) - - if verbose: - print("{} input files found".format(len(ID_files))) - - return ID_files - - -def check_results( - ID_files, - input_path, - result_base_names, - n_complete, - extension, - verbose=False, -): - """Count the number of result files uploaded to vos for each input ID file. - - Parameters - ---------- - ID_files: list of strings - file name with tile IDs - input_path: string - path input directory - result_base_names: list of strings - result file base names - n_complete: int - number of files for complete result set - verbose: bool, optional, default=False - verbose output if True - - Returns - ------- - n_found: dictionary - number of files found for each input file and each ID - n_IDs: dictionary - number of ID for each input file - IDs_not_avail: list - IDs that are not available on vos - """ - - m = re.match("vos:", input_path) - if m: - ls_tmp = dir_list(input_path) - else: - ls_tmp = glob.glob(f"{input_path}/*") - ls_out = [os.path.basename(path) for path in ls_tmp] - - n_found = {} - n_IDs = {} - IDs_not_avail = [] - - # Loop over all input files - for ID_list in ID_files: - with open(ID_list) as f: - if verbose: - print("Checking ID list file {}...".format(ID_list)) - n_found[ID_list] = {} - n_IDs[ID_list] = 0 - - # Loop over all lines = IDs in file - for line in f: - ID = line.rstrip() - n_found[ID_list][ID] = 0 - - if extension == "fits": - ID_fname = ID.replace(".", "-") - ID_fname = f"-{ID_fname}" - else: - ID_fname = f"_{ID}" - # Count how many result files are available - for base in result_base_names: - name = "{}{}.{}".format(base, ID_fname, extension) - if name in ls_out: - n_found[ID_list][ID] = n_found[ID_list][ID] + 1 - n_IDs[ID_list] = n_IDs[ID_list] + 1 - - # If not complete set found, add to not-avail list - if n_found[ID_list][ID] != n_complete: - IDs_not_avail.append(ID) - - return n_found, n_IDs, IDs_not_avail - - -def output_summary(n_found, n_IDs, n_complete): - """Create output with summary of result availability. - - Parameters - ---------- - n_found: dictionary - number of files found for each input file and each ID - n_IDs: dictionary - number of ID for each input file - n_complete: int - number of files of a complete set of results - """ - - for ID_list in n_found.keys(): - nf = sum(value == n_complete for value in n_found[ID_list].values()) - print( - "{}: {}/{} ({:.1f}%) complete".format( - os.path.basename(ID_list), - nf, - n_IDs[ID_list], - nf / n_IDs[ID_list] * 100, - ) - ) - - -def output_IDs(ID_list, output): - """Write IDs to file - - Parameters - ---------- - ID_list: list of string - IDs - output: string - output file name - """ - - f = open(output, "w") - for ID in ID_list: - print(ID, file=f) - f.close() - - -def main(argv=None): - - # Set default parameters - p_def = params_default() - - # Command line options - options, args = parse_options(p_def) - - if check_options(options) is False: - return 1 - - param = update_param(p_def, options) - - # Save calling command - cfis.log_command(argv) - if param.verbose: - cfis.log_command(argv, name="sys.stdout") - - ### Start main program ### - - if param.verbose: - print("Start of program {}".format(os.path.basename(argv[0]))) - - ID_files = read_input_files(param.input_IDs, verbose=param.verbose) - - if param.final_only: - result_base_names = ["final_cat"] - elif param.mask_only: - result_base_names = ["pipeline_flag"] - else: - result_base_names = [] - types = [ - "final_cat", - "pipeline_flag", - "logs", - "setools_mask", - "setools_stat", - "setools_plot", - ] - for t in types: - result_base_names.append(t) - - if param.psf == "psfex": - result_base_names.append("psfex_interp_exp") - elif param.psf == "mccd": - result_base_names.append("mccd_fit_val_runner") - - n_complete = len(result_base_names) - - n_found, n_IDs, IDs_not_avail = check_results( - ID_files, - param.input_path, - result_base_names, - n_complete, - param.extension, - verbose=param.verbose, - ) - - output_summary(n_found, n_IDs, n_complete) - - if param.output_not_avail: - output_IDs(IDs_not_avail, param.output_not_avail) - - ### End main program - - if param.verbose: - print("End of program {}".format(os.path.basename(argv[0]))) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 75e3a67c0..2b583b857 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -6,7 +6,7 @@ ShapePipe module ``make_catalogue_runner``. Supercedes `merge_final_cat.py`. Usage: in parent dir of patches: -create_final_cat.py -p ~/shapepipe/example/cfis/final_cat.param -i . -P 7 -v -m final_cat_P7.hdf5 +create_final_cat.py -p ~/shapepipe/workflow/config/cfis/final_cat.param -i . -P 7 -v -m final_cat_P7.hdf5 :Author: Martin Kilbinger @@ -394,7 +394,7 @@ def process(params): run_prefix = "run_sp_tile_Mc_*" else: patch_name = rf"P{params['patch']}" - run_prefix = "run_sp_Mc_*" + run_prefix = "run_sp_tile_Mc_*" patch_pattern = re.compile(patch_name) diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py deleted file mode 100755 index 0e3586f88..000000000 --- a/scripts/python/create_star_cat.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script create_star_cat.py - -:Description: Create reference star catalogue for masking of -bright star halos and diffraction spikes - -:Authors: Axel Guinot, Martin Kilbinger - -""" - - -import os -import re -import sys - -from cs_util import args as cs_args -from cs_util import logging as cs_logging - -import numpy as np - -from astropy.coordinates import SkyCoord -from astropy.wcs import WCS -from astropy.io import fits -from astropy import units as u -from astropy.table import Table - -from shapepipe.utilities.vizier import query_vizier as _query_vizier - - -# GSC 2.3 catalog ID -CDS_CAT_ID = "I/305/out" - - -def _get_wcs(header): - """Get WCS. - - Compute the astropy WCS from header manually. - (The purpose of this is to avoid possible incompatibility on distortion - convention) - - Parameters - ---------- - header : astropy.header - Image header - - Returns - ------- - astropy.wcs.WCS - WCS object - - """ - final_wcs = WCS(naxis=2) - final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except: - final_wcs.wcs.cunit = ["deg", "deg"] - final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - final_wcs.wcs.cd = [ - [header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]], - ] - - return final_wcs - - -def _sphere_dist_arcmin(ra1, dec1, ra2, dec2): - """Compute angular distance between two sky positions in arcmin.""" - c1 = SkyCoord(ra=ra1 * u.deg, dec=dec1 * u.deg) - c2 = SkyCoord(ra=ra2 * u.deg, dec=dec2 * u.deg) - return c1.separation(c2).arcmin - - -def _ccd_center_and_radius(header): - """Return (ra, dec, radius_arcmin) for a single CCD.""" - w = _get_wcs(header) - nx, ny = header["NAXIS1"], header["NAXIS2"] - cx, cy = nx / 2.0, ny / 2.0 - ra_c, dec_c = w.all_pix2world([[cx, cy]], 1)[0] - # radius = half-diagonal to CCD corner - ra_corner, dec_corner = w.all_pix2world([[0, 0]], 1)[0] - radius = _sphere_dist_arcmin(ra_c, dec_c, ra_corner, dec_corner) - return ra_c, dec_c, radius - - -def _focal_plane_center_and_radius(f, n_ccd=40): - """Return (ra, dec, radius_arcmin) covering all CCDs of an exposure.""" - ras, decs, radii = [], [], [] - for ind in range(1, n_ccd + 1): - h = fits.getheader(f, ind) - ra, dec, r = _ccd_center_and_radius(h) - ras.append(ra) - decs.append(dec) - radii.append(r) - - ras = np.array(ras) - decs = np.array(decs) - radii = np.array(radii) - - ra_center = np.mean(ras) - dec_center = np.mean(decs) - - # Radius = max distance from focal plane center to any CCD center + that CCD's half-diagonal - dists = np.array([ - _sphere_dist_arcmin(ra_center, dec_center, ras[i], decs[i]) - for i in range(len(ras)) - ]) - radius = np.max(dists + radii) - - return ra_center, dec_center, radius - - -def query_vizier(ra, dec, radius_arcmin): - return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) - - -def main(input_dir, output_dir, kind): - - file_list = os.listdir(input_dir) - - for f in file_list: - if "image" not in f: - continue - - img_number = re.split("image", os.path.splitext(f)[0])[1] - fpath = os.path.join(input_dir, f) - - if kind == "exp": - # One query covering the full MegaCam focal plane - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - ra, dec, radius = _focal_plane_center_and_radius(fpath) - print( - f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" - ) - table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) - - else: - h = fits.getheader(fpath, 0) - w = _get_wcs(h) - nx, ny = h["NAXIS1"], h["NAXIS2"] - cx, cy = nx / 2.0, ny / 2.0 - ra, dec = w.all_pix2world([[cx, cy]], 1)[0] - ra_corner, dec_corner = w.all_pix2world([[0, 0]], 1)[0] - radius = _sphere_dist_arcmin(ra, dec, ra_corner, dec_corner) - - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) - - return 0 - - -def params_default(): - """Return default parameters, short options, types, and help strings.""" - _params = { - "input_dir": ".", - "output_dir": ".", - "kind": "exp", - } - _short_options = { - "input_dir": "-i", - "output_dir": "-o", - "kind": "-k", - } - _types = {} - _help_strings = { - "input_dir": "input directory containing image files; default is {}", - "output_dir": "output directory for star catalogues; default is {}", - "kind": "processing kind, 'exp' for full MegaCam focal plane, 'tile' for single image; default is {}", - } - return _params, _short_options, _types, _help_strings - - -if __name__ == "__main__": - - _params, _short_options, _types, _help_strings = params_default() - - options = cs_args.parse_options( - _params, - _short_options, - _types, - _help_strings, - ) - - cs_logging.log_command(sys.argv) - - main(options["input_dir"], options["output_dir"], options["kind"]) diff --git a/scripts/python/stats_headless_canfar.py b/scripts/python/stats_headless_canfar.py deleted file mode 100755 index 1f728949d..000000000 --- a/scripts/python/stats_headless_canfar.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -# Name: stats_headless_canfar.py - -# Caution: Does not show all running or pending -# headless jobs, for some reason. - -import sys -from skaha.session import Session - - -def main(argv=None): - - print( - "# Depreciated, does not show pending jobs; use stats_jobs_canfar.sh", - file=sys.stderr, - ) - - session = Session() - - n_headless = session.stats()["instances"]["headless"] - - print(n_headless) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/sh/canfar_download_results.bash b/scripts/sh/canfar_download_results.bash deleted file mode 100755 index 991af515a..000000000 --- a/scripts/sh/canfar_download_results.bash +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_download_results.bash -# Description: Download ShapePipe results (.tgz files) -# from canfar with vos -# Author: Martin Kilbinger -# Date: v1.0 05/2020 -# v1.1 01/2021 - -# Command line - -## Default parameters -INPUT_VOS="cosmostat/kilbinger/results" -VERBOSE=0 -psf="mccd" -only_mask=0 - - -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -i, --input_IDs ID_FILE\n - \tASCII file with tile IDs to download, default:\n - \tdownload all available IDs\n - --input_vos PATH\n - \tinput path on vos:cfis, default='$INPUT_VOS'\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m\tonly mask files\n - -v\tverbose output\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -i|--input_IDs) - IDs=(`cat $2`) - echo "Downloading ${#IDs[@]} ID(s)" - shift - ;; - --input_vos) - INPUT_VOS="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m) - only_mask=1 - ;; - -v) - VERBOSE=1 - ;; - *) - echo "Invalid command line argument '$1'" - echo -ne $usage - exit 1 - ;; - esac - shift -done - -## Check options -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psf' or 'mccd'" - exit 2 -fi - -## Paths -remote="vos:cfis/$INPUT_VOS" -local="." - -if [ $only_mask == 1 ]; then - NAMES=("pipeline_flag") -else - NAMES=( - "final_cat" - "logs" - "setools_mask" - "setools_stat" - "setools_plot" - "pipeline_flag" - ) - - if [ $psf == "psfex" ]; then - NAMES+=( - "psfex_interp_exp" - ) - else - NAMES+=( - "mccd_fit_val_runner" - ) - fi -fi - -if [ $VERBOSE == 1 ]; then - vflag="-v" -else - vflag="" -fi -export VCP="vcp $vflag" - - -### Start ### - -# Download files -for name in ${NAMES[@]}; do - if [ ${#IDs[@]} == 0 ]; then - cmd="$VCP $remote/$name*.tgz $local" - $cmd - else - for ID in ${IDs[@]}; do - cmd="$VCP $remote/${name}_$ID.tgz $local" - $cmd - done - fi -done - -# Check number of files -for name in ${NAMES[@]}; do - n_downl=(`ls -l $local/${name}_*.tgz | wc`) - echo "$n_downl '$name' result files downloaded from $remote" -done diff --git a/scripts/sh/canfar_submit_selection.sh b/scripts/sh/canfar_submit_selection.sh deleted file mode 100755 index 3698f0722..000000000 --- a/scripts/sh/canfar_submit_selection.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# Name: canfar_submit_selection.sh -# Author: Martin Kilbinger, martin.kilbinger@cea.fr -# Date: 2020 -# Description: Submits jobs to canfar, each jobs processes -# one tile - - -# Variables - -# Job file for one tile -SP_ROOT=$HOME/shapepipe -sp_job="$SP_ROOT/scripts/sh/job_sp.bash" - - -# Functions - -function add_queue() { - local job_file=$1 - - echo "queue" >> $job_file - echo >> $job_file -} - -function create_job_file() { - local tile_ID=$1 - local job_file=$2 - - echo "executable = $sp_job" > $job_file - echo >> $job_file - - add_to_job_file $tile_ID $job_file - - finalize_job_file $job_file - add_queue $job_file -} - -function add_to_job_file () { - local tile_ID=$1 - local job_file=$2 - - echo "arguments = $tile_ID" >> $job_file - echo "output = log_canfar_sp_$tile_ID.out" >> $job_file - echo "error = log_canfar_sp_$tile_ID.err" >> $job_file - echo "log = log_canfar_sp_$tile_ID.log" >> $job_file -} - -function finalize_job_file() { - local job_file=$1 - - echo "request_cpus = 8" >> $job_file - - # Cannot be larger than VM available RAM - echo "request_disk = 100G" >> $job_file - - echo >> $job_file -} - -function usage () { - ex=$1 - - echo "Usage: canfar_submit_selection.sh ID_path image [-h] [-n]" - echo "Options:" - echo " ID_path ascii file with tile IDs" - echo " image VM image name, newest is ShapePipe2-mk-20200820" - echo " -n dry run" - echo " -h this message" - - exit $ex -} - -if [ "${!#}" == "-h" ]; then - usage 0 -fi -if [ "$#" -lt 2 ] ; then - usage 1 -fi - -if [ "${!#}" == "-n" ]; then - dry_run=1 - dry_str=" (dry run)" -else - dry_run=0 - dry_str="" -fi - - -## Start - -tile_ID_list=$1 -image=$2 - -# Create job file -job_file="job_tile.sh" -echo "executable = $sp_job" > $job_file -echo >> $job_file - -echo "output = log_canfar_sp_\$(arguments).out" >> $job_file -echo "error = log_canfar_sp_\$(arguments).err" >> $job_file -echo "log = log_canfar_sp_\$(arguments).log" >> $job_file -echo >> $job_file -finalize_job_file $job_file -echo >> $job_file -echo "queue arguments from (" >> $job_file - -# Go through tile IDs and add job -while read -r tile_ID; do - echo "$tile_ID" >> $job_file -done < $tile_ID_list - echo ")" >> $job_file - -# Submit -cmd="canfar_submit $job_file $image c8-30gb-186" -echo "Running $cmd$dry_str" -if [ $dry_run == 0 ]; then - $cmd - condor_q -fi diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash index 8cbf9bbdf..91b2ca42a 100755 --- a/scripts/sh/combine_runs.bash +++ b/scripts/sh/combine_runs.bash @@ -104,9 +104,9 @@ run_out="run_sp_combined_$cat" if [ "$cat" == "final" ]; then # v1 - #run_in="$pwd/$out_base/run_sp_Mc_*" + #run_in="$pwd/$out_base/run_sp_tile_Mc_*" # v2 - run_in="$pwd/tile_runs/*/$out_base/run_sp_Mc_*" + run_in="$pwd/tile_runs/*/$out_base/run_sp_tile_Mc_*" module="make_catalog_runner" pattern="final_cat-*" diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh deleted file mode 100755 index ada5dc4f0..000000000 --- a/scripts/sh/curl_canfar_local.sh +++ /dev/null @@ -1,303 +0,0 @@ -# Global variables -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session -IMAGE=images.canfar.net/unions/shapepipe -NAME=shapepipe - -# :TODO: Not working -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source $HOME/shapepipe/scripts/sh/functions.sh - -# Command line arguments - -## Default values -job=-1 -psf="psfex" -ID=-1 -file_IDs=-1 -N_SMP=1 -fix=0 -version="1.1" -cmd_remote="$HOME/shapepipe/scripts/sh/init_run_exclusive_canfar.sh" -batch=30 -batch_max=200 -dry_run=0 -mh_local=0 -sp_local=0 -test_only=0 -debug_out="-1" -scratch="-1" -sm=1 - -pat="- " - -## Help string -usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRunning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -f, --file_IDs path - \tfile containing IDs\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n - -s, --sp_local SP\n - \tsplit local run local (SP=1) or global (SP=0); default is SP=$sp_local\n - --sm SM\n - \tWith (SM=1; default) or without (SM=0) spread model input\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default=$N_SMP\n - -F, --fix FIX\n - \tfix missing data (re-download tile, unzip) for FIX=1; default is $fix\n - -V, --version\n - \tversion of docker image, default='$version'\n - -C, --command_remote\n - \tremote command to run on canfar, default='$cmd_remote'\n - -S, --scratch\n - \tprocessing scratch directory, default is None ($scratch)\n - -b, --batch_max\n - \tmaximum batch size = number of jobs run simultaneously, default=$batch_max\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n - -n, --dry_run LEVEL\n - \tdry run, from LEVEL=2 (no processing) to 0 (full run; default)\n - --test\n - \ttest mode, no processing\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -s|--sp_local) - sp_local="$2" - shift - ;; - --sm) - sm="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -f|--file_IDs) - file_IDs="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -F|--fix) - fix="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -V|--version) - version="$2" - shift - ;; - -B|--batch) - batch="$2" - shift - ;; - -b|--batch_max) - batch_max="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - --test) - test_only=1 - ;; - esac - shift -done - - -## Check options - -if [ "$test_only" == "1" ]; then - test_arg="--test" -else - test_arg="" -fi - -if [ "$job" == "-1" ]; then - echo "No job indicated, use option -j" - exit 2 -fi - -if [ "$ID" == "-1" ] && [ "$file_IDs" == "-1" ]; then - echo "No image ID(s) indicated, use option -e ID or -f file_IDs" - exit 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 4 -fi - -if [ "$dry_run" != 0 ] && [ "$dry_run" != 1 ] && [ "$dry_run" != 2 ]; then - echo "Invalid dry_run option, allowed are 0, 1, and 2" - exit 5 -fi - -if [ "$debug_out" != "-1" ]; then - echo "${pat}Starting $(basename "$0") $test_arg" >> $debug_out - echo "${pat}curl ID=$ID" >> $debug_out - echo ${pat}`date` >> $debug_out -fi - -source activate shapepipe -if [ "$debug_out" != "-1" ]; then - echo "${pat}conda prefix = ${CONDA_PREFIX}" >> $debug_out - echo "${pat}script version = ${script_version}" >> $debug_out -fi - -# command line arguments for remote script: -# collect into string - - -RESOURCES="ram=4&cores=$N_SMP" -dir=`pwd` - - -function submit_batch() { - path=$1 - - for ID in `cat $path`; do - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - done -} - -batch=50 -if [ "$batch" -ge "$batch_max" ]; then - ((batch=batch_max/2)) - echo "Reducing batch size to $batch" -fi -sleep=75 - -((n_thresh=batch_max-batch)) - - -if [ "$dry_run" == 2 ]; then - - # Do not call curl (dry run = 2) - echo "Running command dry run:" - - if [ "$ID" == "-1" ]; then - - - # Submit file (dry run = 2) - for ID in `cat $file_IDs`; do - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - done - - else - - # Submit image (dry run = 2) - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - - fi - -else - - # Call curl - rm -rf session_IDs.txt session_image_IDs.txt - - if [ "$ID" == "-1" ]; then - - # Submit file - n_jobs=`cat $file_IDs | wc -l` - if [ "$n_jobs" -gt "$batch_max" ]; then - - # Split into batches - prefix="${file_IDs}_split_" - split -d -l $batch $file_IDs $prefix - n_split=`ls -l $prefix* | wc -l` - echo "Split '$file_IDs' into $n_split batches of size $batch" - - count=1 - n_queued=`stats_jobs_canfar.sh -w all` - for batch in $prefix*; do - echo "Number of queued jobs = $n_queued" - echo "Submitting batch $batch ($count/$n_split)" - echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007" - submit_batch $batch - ((count=count+1)) - - n_queued=`stats_jobs_canfar.sh -w all` - - while [ "$n_queued" -gt "$n_thresh" ]; do - echo "Wait for #jobs = $n_queued jobs to go < $n_thresh ..." - sleep $sleep - n_queued=`stats_jobs_canfar.sh -w all` - done - - done - - else - - # Submit entire file (single batch) - echo "Submit '$file_IDs' in single batch" - submit_batch $file_IDs - - fi - - else - - # Submit image - IDt=`echo $ID | tr "." "-"` - my_name="SP-${patch}-J${job}-${IDt}" - call_curl $my_name $job $psf $ID $N_SMP $dry_run $dir $mh_local $sp_local $sm $debug_out $fix $scratch $test_arg - - fi - -fi - -echo "Done $(basename "$0")" - -if [ "$debug_out" != "-1" ]; then - echo "${pat}End $(basename "$0") $test_arg" >> $debug_out -fi diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh index 27efadc11..556153b0d 100755 --- a/scripts/sh/init_run_exclusive_canfar.sh +++ b/scripts/sh/init_run_exclusive_canfar.sh @@ -574,7 +574,7 @@ fi if [[ $do_job != 0 ]]; then # Remove previous runs of this job - rm -rf run_sp_Ms_20??-* + rm -rf run_sp_tile_Ms_20??-* fi @@ -582,7 +582,7 @@ fi if [[ $do_job != 0 ]]; then # Remove previous runs of this job - rm -rf run_sp_Mc_20??-* + rm -rf run_sp_tile_Mc_20??-* fi diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash index c99435823..e64ed1e5f 100755 --- a/scripts/sh/job_sp_canfar.bash +++ b/scripts/sh/job_sp_canfar.bash @@ -564,11 +564,11 @@ if [[ $do_job != 0 ]]; then cat $SP_CONFIG/config_merge_sep_cats_template.ini | \ perl -ane \ 's/(N_SPLIT_MAX =) X/$1 '$nsh_jobs'/; print' \ - > $SP_CONFIG_MOD/config_merge_sep_cats.ini + > $SP_CONFIG_MOD/config_tile_Ms.ini ### Merge separated shapes catalogues command_sp \ - "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" \ + "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ms.ini" \ "Run shapepipe (tile: merge sep cats)" \ "$VERBOSE" \ "$ID" diff --git a/scripts/sh/run_scratch_local.sh b/scripts/sh/run_scratch_local.sh deleted file mode 100755 index 1223e6683..000000000 --- a/scripts/sh/run_scratch_local.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash - -# Command line arguments -## Default values -job=-1 -ID=-1 -N_SMP=1 -dry_run=0 -dir=`pwd` -debug_out=-1 -scratch=/n17data/`whoami`/scratch -exec_path=$HOME/shapepipe/scripts/sh -slurm=1 - -# mh_local is 0 (1) if merge_header_runner is run on all exposures, -# which is standard so far (run on exposures of given tile only; new) -mh_local=0 - -# sp_local is 0 (1) is split_headers_runner and mask_runner is run -# on all exposures (locally). Not 100% automatic yet. -sp_local=1 -VERBOSE=1 - -pat="-- " - -# Help string -usage="Usage: $(basename "$0") -j JOB -e ID -k KIND [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -j, --job JOB\tRUnning JOB, bit-coded\n - -e, --exclusive ID - \timage ID\n - -p, --psf MODEL\n - \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n - -m, --mh_local MH\n - \tmerged header file local (MH=0) or global (MH=1); default is $mh_local\n - -N, --N_SMP N_SMOp\n - \tnumber of jobs (SMP mode only), default from original config files\n - -d, --directory\n - \trun directory, default is pwd ($dir)\n - -S, --scratch\n - \tprocessing scratch directory, default is $scratch\n - -n, --dry_run LEVEL\n - \tdry run, no actuall processing\n - --debug_out PATH\n - \tdebug output file PATH, default not used\n -" - -## Help if no arguments -if [ -z $1 ]; then - echo -ne $usage - exit 1 -fi - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -j|--job) - job="$2" - shift - ;; - -e|--exclusive) - ID="$2" - shift - ;; - -p|--psf) - psf="$2" - shift - ;; - -m|--mh_local) - mh_local="$2" - shift - ;; - -N|--N_SMP) - N_SMP="$2" - shift - ;; - -d|--directory) - dir="$2" - shift - ;; - -S|--scratch) - scratch="$2" - shift - ;; - -n|--dry_run) - dry_run="$2" - shift - ;; - --debug_out) - debug_out="$2" - shift - ;; - esac - shift -done - -## Check options -if [ "$job" == "-1" ]; then - echo "No job indicated, use option -j" - exit 2 -fi - -if [ "$exclusive" == "-1" ]; then - echo "No image ID indicated, use option -e" - exit 3 -fi - -if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - echo "PSF (option -p) needs to be 'psfex' or 'mccd'" - exit 4 -fi - - -source $HOME/shapepipe/scripts/sh/functions.sh - - -kind=$(get_kind_from_job $job) - - -# Load common functions -source $HOME/shapepipe/scripts/sh/functions.sh - - -# Start script - -if [ "$scratch" != "-1" ]; then - - command "mkdir -p $scratch/${kind}_runs" $dry_run - command "cp -R ${kind}_runs/$ID $scratch/${kind}_runs" $dry_run - command "cd $scratch" $dry_run - -fi - - if [ "$slurm" == "0" ]; then - command "init_run_exclusive_canfar.sh -j $job -p $psf -m $mh_local -N $N_SMP -e $ID" $dry_run - else - STATUS=$(sbatch --output=./sbatch-$ID.out --partition=comp --job-name="j${job}_${ID}" --ntasks-per-node=$N_SMP --time=32:00:00 --mem=64G $exec_path/init_run_exclusive_canfar.sh -j $job -p $psf -m $mh_local -N $N_SMP -e $ID) - - JOB_ID=$(echo $STATUS | cut -d ' ' -f 4) - echo "JOB_ID=$JOB_ID" - - # Wait for the job to finish - while true; do - STATUS=$(squeue -j "$JOB_ID" -h -o "%T") - if [[ -z "$STATUS" ]]; then - echo "job $JOB_ID no longer in the queue" - break - fi - - echo "Waiting for job $JOB_ID in state '$STATUS' to complete..." - sleep 10 - done - - echo "Job $JOB_ID has completed. Proceeding with the script..." - fi - -if [ "$scratch" != "-1" ]; then - - if [ "$job" == "32" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_exp_SxSe* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "64" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_PsViSm* $dir/${kind}_runs/$ID/output" $dry_run - elif [ "$job" == "128" ]; then - command "mv ${kind}_runs/$ID/output/run_sp_tile_ngmix_* $dir/${kind}_runs/$ID/output" $dry_run - fi - - command "rm -rf ${kind}_runs/$ID" $dry_run - command "cd $dir/${kind}_runs/$ID" $dry_run - # Gave Input/Output python error - #command "update_runs_log_file.py" $dry_run - command "cd $dir" $dry_run - -fi diff --git a/scripts/sh/stats_jobs_canfar.sh b/scripts/sh/stats_jobs_canfar.sh deleted file mode 100755 index f9c51101a..000000000 --- a/scripts/sh/stats_jobs_canfar.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# Name: stats_jobs_canfar.sh -# Author: Martin Kilbinger -# Description: Handles headless jobs on canfar - - -# Global variables - -## Temporary files -tmpfile_jobs="jobinfo.txt" -tmpfile_ids="ids.txt" -tmpfile_running="jobs_running.txt" - -## curl options -SSL=~/.ssl/cadcproxy.pem -SESSION=https://ws-uv.canfar.net/skaha/v0/session - - -# Command line arguments - -## Default values -mode="count" -debug=0 - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] -\n\nOptions:\n - -h\tthis message\n - -m, --mode MODE\n - \tmode, allowed are 'count' (default), 'delete'\n - -d, --debug\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -m|--mode) - mode="$2" - shift - ;; - -d|--debug) - debug=1 - ;; - esac - shift -done - -## Check options -case $mode in - "count"|"delete") - # valid option - ;; - *) - echo "Invalid mode $mode" - exit 1 - ;; -esac - - -# Main program - -# Get all instances -if [ "$debug" == "1" ]; then - curl -E $SSL $SESSION - echo curl -E $SSL $SESSION - exit 0 -else - curl -E $SSL $SESSION &> /dev/null > $tmpfile_jobs -fi -res=$? - -if [ "$res" == "0" ]; then - - # Get headless job IDs - cat $tmpfile_jobs | grep headless -B 4 -A 2 | grep Running -A 1 > $tmpfile_ids - - # Number of jobs - n_headless=`cat $tmpfile_ids | grep Running | wc -l` - - # Get running job info - cat $tmpfile_ids | grep name | perl -F\- -ane 'chomp; $F[4] =~ s/[",]//g; print "$F[3].$F[4]"' > $tmpfile_running - -else - - # Failure: set to very high number - n_headless=10000 - -fi - - -if [ "$mode" == "count" ]; then - - echo $n_headless - -elif [ "$mode" == "delete" ]; then - - echo -n "Delete $n_headless jobs? [y|n] " - read answer - if [ "$answer" == "y" ]; then - cat $tmpfile_jobs | grep headless -B 34 -A 6 | grep Running -A 34 | grep id | grep -v user | perl -F\" -ane 'print "$F[3]\n"' > $tmpfile_ids - for ID in `cat $tmpfile_ids`; do - echo $ID - # Delete headless jobs - echo "curl -X DELETE -E $SSL $SESSION/$ID" - curl -X DELETE -E $SSL $SESSION/$ID - echo $? - done - fi - -fi - - -# Remove temporary files -#rm -f $tmpfile_jobs $tmpfile_ids diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 24e72693a..946df5a8e 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -16,6 +16,7 @@ from sqlitedict import SqliteDict from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query def get_output_name(output_dir, file_number_string): @@ -209,6 +210,68 @@ def save_sm_data( return n_obj +def parse_mask_ext_paths(paths_str): + """Parse Mask Ext Paths. + + Parse the ``MASK_EXT_PATHS`` config value into a ``band -> path`` mapping. + + Parameters + ---------- + paths_str : str + Comma-separated ``band:path`` pairs, e.g. + ``u:/path/mask_u.hsp, g:/path/mask_g.hsp`` + + Returns + ------- + dict + Mapping from band name to healsparse map path + + """ + band_paths = {} + for pair in paths_str.split(","): + band, path = pair.split(":", 1) + band_paths[band.strip()] = path.strip() + + return band_paths + + +def save_mask_ext_data(final_cat_file, band_paths, w_log): + """Save External Mask Data. + + Query per-band external healsparse masks at each object's world position + and write one ``MASK_`` column per band into the final catalogue. + Object positions are read from the SExtractor windowed world coordinates + (``XWIN_WORLD`` = RA, ``YWIN_WORLD`` = Dec, both in degrees) carried in the + ``RESULTS`` extension. Objects falling outside a map's coverage receive + that map's sentinel value (``healsparse.HealSparseMap.get_values_pos`` + returns the map's sentinel — ``-1`` for integer maps — verbatim), which is + the documented off-map flag. + + The lookup itself is ``shapepipe.utilities.mask_query.query_map``, + shared with the ``mask_query`` module: one primitive, two consumers. + + Parameters + ---------- + final_cat_file : file_io.FITSCatalogue + Final catalogue + band_paths : dict + Mapping from band name to healsparse map path + w_log : logging.Logger + Logging instance + + """ + final_cat_file.open() + ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"]) + dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"]) + + for band, path in band_paths.items(): + w_log.info(f"Query external mask for band {band}: {path}") + values = mask_query.query_map(path, ra, dec) + final_cat_file.add_col(f"MASK_{band}", values) + + final_cat_file.close() + + class SaveCatalogue: """Save Catalogue. diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 307dc2ffe..176341757 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -137,4 +137,14 @@ def make_cat_runner( if save_psf: err_msg = sc_inst.process("psf", galaxy_psf_path) + # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38): + # add one MASK_ column per band, queried at each object's world + # position. Absent config is a strict no-op. + if config.has_option(module_config_sec, "MASK_EXT_PATHS"): + band_paths = make_cat.parse_mask_ext_paths( + config.getexpanded(module_config_sec, "MASK_EXT_PATHS") + ) + w_log.info("Save external mask data") + make_cat.save_mask_ext_data(final_cat_file, band_paths, w_log) + return None, None diff --git a/src/shapepipe/modules/mask_package/__init__.py b/src/shapepipe/modules/mask_package/__init__.py deleted file mode 100644 index bdfece65b..000000000 --- a/src/shapepipe/modules/mask_package/__init__.py +++ /dev/null @@ -1,185 +0,0 @@ -"""MASK MODULE. - -This package contains the module for ``mask``. - -:Author: Axel Guinot - -:Parent module: ``split_exp_runner`` or None - -:Input: Single-exposure single-CCD image, weight file, flag file (optional), - and star catalogue (optional) - -:Output: Single-exposure single-CCD flag files - -Description -=========== - -This module creates masks for bright stars, diffraction spikes, deep sky -objects (from the Messier and NGC catalogues), borders, and other artifacts. If -a flag file is given as input, for example from pre-processing, the mask that -is created by this module is joined with the mask from this external flag file. -In this case the config flag ``USE_EXT_FLAG`` needs to be set to ``True``. To -distinguish the newly created output flag file from the input ones, a prefix -can added as specificed by the config entry ``PREFIX``. - -An NGC catalogue with positions, sizes, and types is provided with -``shapepipe``, -`source `_. - -Masked pixels of different mask types are indicated by integers, which -conveniently are powers of two such that they can be combined bit-wise. - -To mask bright stars, this module either creates a star catalogue from the -online -`guide star catalogue `_ -database relevant to the the footprint. This is done by calling a CDs -(Centre de Données astronomique de Strasbourg) -`client program `_. -Note that this requires online access, -which in some cases is not granted on compute nodes of a cluster. In this case, -set the config flag ``USE_EXT_STAR = False``. Alternatively, a star -catalogue can be created before running this module via the script -``create_star_cat``. During the processing of this module, this star catalogue -is read from disk, with ``USE_SET_STAR = True``. - -The masking is done with the software ``WeightWatcher`` :cite:`marmo:08`, -which is installed by ``ShapePipe`` by default. - -Module-specific config file entries -=================================== - -USE_EXT_FLAG : bool - Use external flag file to join with the mask created here; - if ``True`` flag file needs to be given on input -USE_EXT_STAR : bool - Read external star catalogue instead of creating one during the - call of this module; - if ``True`` star catalogue file needs to be given on input -MASK_CONFIG_PATH : str - Path to mask config file -HDU : int, optional - HDU of external flag FITS file; the default value is ``0`` -PREFIX : str, optional - Prefix to be appended to output file name ``flag``; - helps to distinguish the file patterns of newly created and external - mask files -CHECK_EXISTING_DIR : str, optional - If given, search this directory for existing mask files; the - corresponding images will then not be processed - -Mask config file -================ - -An additional configuration file is used by the mask module, its path is -``MASK_CONFIG_PATH`` in the module config section, see above. The following -describes the config file sections and their entries. - -[PROGRAM_PATH] --------------- - -WW_PATH : str, optional - Full path to the WeightWatcher executable (``ww``) on the system ; if not - set the version controlled WeightWatcher installation in the ShapePipe - environment will be used -WW_CONFIG_FILE : str - Path to the WeightWatcher configuration file -CDSCLIENT_PATH : str, optional - Path to CDS client executable; required if ``USE_EXT_STAR = False`` - -[BORDER_PARAMETERS] -------------------- - -BORDER_MAKE : bool - Create mask around borders if ``True`` -BORDER_WIDTH : int - Width of border mask in pixels -BORDER_FLAG_VALUE : int - Border mask pixel value, power of 2 - -[HALO_PARAMETERS] ------------------ - -HALO_MAKE : bool - Create mask for halos of bright stars if ``True`` -HALO_MASKMODEL_PATH : str - Path to halo mask geometry (``.reg`` file) -HALO_MAG_LIM : float - Faint stellar magnitude limit for halo mask -HALO_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and halo mask size -HALO_MAG_PIVOT : float - Pivot stellar magnitude -HALO_FLAG_VALUE : int - Halo mask pixel value, power of 2 -HALO_REG_FILE : str - Output halo mask ``.reg`` file - -[SPIKE_PARAMETERS] ------------------- - -SPIKE_MAKE : bool - Create mask for diffraction spikes of bright stars if ``True`` -SPIKE_MASKMODEL_PATH : str - Path to diffraction spike geometry (``.reg`` file) -SPIKE_MAG_LIM : - Faint stellar magnitude limit for spike mask -SPIKE_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and spike mask size -SPIKE_MAG_PIVOT : float - Pivot stellar magnitude -SPIKE_FLAG_VALUE : int - Diffraction spike pixel value, power of two -SPIKE_REG_FILE : str - Output spike mask ``.reg`` file - -[MESSIER_PARAMETERS] --------------------- - -MESSIER_MAKE : bool - Create mask around Messier objects if ``True`` -MESSIER_CAT_PATH : str - Path to Messier catalogue -MESSIER_SIZE_PLUS : float - Fraction to increase Messier mask -MESSIER_FLAG_VALUE : int - Messier mask pixel value, power of 2 - -[NGC_PARAMETERS] --------------------- - -NGC_MAKE : bool - Create mask around NGC objects if ``True`` -NGC_CAT_PATH : str - Path to NGC catalogue -NGC_SIZE_PLUS : float - Fraction to increase NGC mask -NGC_FLAG_VALUE : int - NGC mask pixel value, power of 2 - -[MD_PARAMETERS] ---------------- - -MD_MAKE : bool - Account for missing data (zero-valued pixels) if ``True`` -MD_THRESH_FLAG : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked as flagged -MD_THRESH_REMOVE : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked for removal -MD_REMOVE : bool - Image is removed if marked for removal - -[OTHER] -------- - -TEMP_DIRECTORY : str - Path to temporary dictionary -KEEP_INDIVIDUAL_MASK : bool - Keep individual masks in addition to merged mask file -KEEP_REG_FILE : bool - Keep ``.reg`` mask file - -""" - -__all__ = ["mask"] diff --git a/src/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py deleted file mode 100644 index ba6d586ad..000000000 --- a/src/shapepipe/modules/mask_package/mask.py +++ /dev/null @@ -1,1271 +0,0 @@ -"""MASK. - -This module contains a class to create star mask for an image. - -:Authors: Axel Guinot, Martin Kilbinger - -""" - -import os -import re - -import numpy as np -from astropy import units, wcs -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.table import Table - -from shapepipe.pipeline import file_io -from shapepipe.pipeline.config import CustomParser -from shapepipe.pipeline.execute import execute -from shapepipe.utilities.file_system import mkdir -from shapepipe.utilities.vizier import query_vizier - - -class Mask(object): - """Mask. - - Class to create mask based on a star catalogue. - - Parameters - ---------- - image_path : str - Path to image (FITS format) - weight_path : str - Path to the weight image (FITS format) - image_prefix : str - Prefix to input image name, specify as ``'none'`` for no prefix - image_num : str - File number identified - config_filepath : str - Path to the ``.mask`` config file - output_dir : str - Path to the output directory - w_log : logging.Logger - Log file - path_external_flag : str, optional - Path to external flag file, default is ``None`` (not used) - outname_base : str, optional - Output file name base, default is ``flag`` - check_existing_dir : str, optional - If not ``None`` (default), search path for existing mask files - star_cat_path : str, optional - Path to external star catalogue, default is ``None`` (not used; - instead the star catalogue is produced on the fly at run time) - hdu : int, optional - HDU number, default is ``0`` - - """ - - def __init__( - self, - image_path, - weight_path, - image_prefix, - image_num, - config_filepath, - output_dir, - w_log, - path_external_flag=None, - outname_base="flag", - check_existing_dir=None, - star_cat_path=None, - hdu=0, - ): - - # Path to the image to mask - self._image_fullpath = image_path - - # Path to the weight associated to the image - self._weight_fullpath = weight_path - - # Input image prefix - if (image_prefix.lower() != "none") and (image_prefix != ""): - self._img_prefix = f"{image_prefix}_" - else: - self._img_prefix = "" - - # File number identified - self._img_number = image_num - - # Path to mask config file - self._config_filepath = config_filepath - - # Path to the output directory - self._output_dir = output_dir - - # Log file - self._w_log = w_log - - # Path to an external flag file - self._path_external_flag = path_external_flag - - # Output file base name - self._outname_base = outname_base - - # Search path for existing mask files - self._check_existing_dir = check_existing_dir - - # Set external star catalogue path if given - if star_cat_path is not None: - self._star_cat_path = star_cat_path - - self._hdu = hdu - - # Read mask config file - self._get_config() - - # Set parameters needed for the star detection - self._set_image_coordinates() - - # Set error flag - self._err = False - - # Guide Star Catalogue parameters - #self._CDS_cat_ID = "I/271/out" # GSC 2.2, does not have Fmag - self._CDS_cat_ID = "I/305/out" # GSC 2.3 - - # Keys in CDS astroquery result - self._cds_keys = ["GSC2.3", "RAJ2000", "DEJ2000", "Fmag", "jmag", "Vmag", "Nmag", "Class"] - - # Minimal scaling for halo and spike polygon templates - self._scaling_min = 0.1 - - def _get_config(self): - """Get Config. - - Read the config file and set parameters. - - Raises - ------ - ValueError - If config file name is ``None`` - IOError - If config file not found - - """ - if self._config_filepath is None: - raise ValueError("No path to config file given") - - if not os.path.exists(self._config_filepath): - raise IOError(f'Config file "{self._config_filepath}" not found') - - conf = CustomParser() - conf.read(self._config_filepath) - - self._config = { - "PATH": {}, - "BORDER": {}, - "HALO": {}, - "SPIKE": {}, - "MESSIER": {}, - "NGC": {}, - "MD": {}, - } - - if conf.has_option("PROGRAM_PATH", "WW_PATH"): - self._config["PATH"]["WW"] = conf.getexpanded( - "PROGRAM_PATH", "WW_PATH" - ) - else: - self._config["PATH"]["WW"] = "weightwatcher" - self._config["PATH"]["WW_configfile"] = conf.getexpanded( - "PROGRAM_PATH", "WW_CONFIG_FILE" - ) - if conf.has_option("PROGRAM_PATH", "CDSCLIENT_PATH"): - self._config["PATH"]["CDSclient"] = conf.getexpanded( - "PROGRAM_PATH", "CDSCLIENT_PATH" - ) - elif self._star_cat_path is not None: - self._config["PATH"]["star_cat"] = self._star_cat_path - else: - raise ValueError( - "Either [PROGRAM_PATH]:CDSCLIENT_PATH in the mask config file " - + " or a star catalogue as module input needs to be present" - ) - - self._config["PATH"]["temp_dir"] = self._get_temp_dir_path( - conf.getexpanded("OTHER", "TEMP_DIRECTORY") - ) - self._config["BORDER"]["make"] = conf.getboolean( - "BORDER_PARAMETERS", "BORDER_MAKE" - ) - if self._config["BORDER"]["make"]: - self._config["BORDER"]["width"] = conf.getint( - "BORDER_PARAMETERS", "BORDER_WIDTH" - ) - self._config["BORDER"]["flag"] = conf.get( - "BORDER_PARAMETERS", "BORDER_FLAG_VALUE" - ) - - for mask_shape in ["HALO", "SPIKE"]: - - self._config[mask_shape]["make"] = conf.getboolean( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAKE", - ) - self._config[mask_shape]["individual"] = conf.getboolean( - "OTHER", "KEEP_INDIVIDUAL_MASK" - ) - - if self._config[mask_shape]["make"]: - - self._config[mask_shape]["maskmodel_path"] = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MASKMODEL_PATH", - ) - self._config[mask_shape]["mag_lim"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_LIM", - ) - self._config[mask_shape]["scale_factor"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_SCALE_FACTOR", - ) - self._config[mask_shape]["mag_pivot"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_PIVOT", - ) - self._config[mask_shape]["flag"] = conf.getint( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_FLAG_VALUE", - ) - - if conf.getboolean("OTHER", "KEEP_REG_FILE"): - reg_file = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_REG_FILE", - ) - self._config[mask_shape]["reg_file"] = ( - f'{self._config["PATH"]["temp_dir"]}/' - + f'{re.split(".reg", reg_file)[0]}' - + f"{self._img_number}.reg" - ) - else: - self._config[mask_shape]["reg_file"] = None - - for mask_type in ["MESSIER", "NGC"]: - - self._config[mask_type]["make"] = conf.getboolean( - f"{mask_type}_PARAMETERS", f"{mask_type}_MAKE" - ) - - if self._config[mask_type]["make"]: - self._config[mask_type]["cat_path"] = conf.getexpanded( - f"{mask_type}_PARAMETERS", - f"{mask_type}_CAT_PATH", - ) - self._config[mask_type]["size_plus"] = conf.getfloat( - f"{mask_type}_PARAMETERS", - f"{mask_type}_SIZE_PLUS", - ) - self._config[mask_type]["flag"] = conf.getint( - f"{mask_type}_PARAMETERS", - f"{mask_type}_FLAG_VALUE", - ) - - self._config["MD"]["make"] = conf.getboolean("MD_PARAMETERS", "MD_MAKE") - - if self._config["MD"]["make"]: - self._config["MD"]["thresh_flag"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_FLAG" - ) - self._config["MD"]["thresh_remove"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean( - "MD_PARAMETERS", "MD_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean("MD_PARAMETERS", "MD_REMOVE") - - def _set_image_coordinates(self): - """Set Image Coordinates. - - Compute the image coordinates for matching with the star catalogue - and star mask. - - """ - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - self._header = img.get_header() - img_shape = img.get_data().shape - img.close() - del img - - self._wcs = wcs.WCS(self._header) - - # Compute field center - - # Note: get_data().shape corresponds to (n_y, n_x) - pix_center = [img_shape[1] / 2.0, img_shape[0] / 2.0] - wcs_center = self._wcs.all_pix2world([pix_center], 1)[0] - self._fieldcenter = {} - self._fieldcenter["pix"] = np.array(pix_center) - self._fieldcenter["wcs"] = SkyCoord( - ra=wcs_center[0], dec=wcs_center[1], unit="deg" - ) - - # Get the four corners of the image - corners = self._wcs.calc_footprint() - self._corners_sc = SkyCoord( - ra=corners[:, 0] * units.degree, - dec=corners[:, 1] * units.degree, - ) - - # Compute image radius = image diagonal - self._img_radius = self._get_image_radius() - - def make_mask(self): - """Make Mask. - - Main function to create the mask. - - """ - output_file_name = ( - f"{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - if os.path.exists(f"{self._check_existing_dir}//{output_file_name}"): - return None, None - - if self._config["MD"]["make"]: - self.missing_data() - - if self._config["HALO"]["make"] or self._config["SPIKE"]["make"]: - stars = self.find_stars( - np.array( - [ - self._fieldcenter["wcs"].ra.value, - self._fieldcenter["wcs"].dec.value, - ] - ), - radius=self._img_radius, - ) - - if not self._err: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._create_mask( - stars=stars, - types=_type, - mag_limit=self._config[_type]["mag_lim"], - scale_factor=self._config[_type]["scale_factor"], - mag_pivot=self._config[_type]["mag_pivot"], - ) - - if not self._err: - mask_name = [] - if self._config["HALO"]["make"] and self._config["SPIKE"]["make"]: - self._exec_WW(types="ALL") - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}halo_spike_flag' - + f"{self._img_number}.fits" - ) - mask_name.append(None) - else: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._exec_WW(types=_type) - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}' - + f"{_type.lower()}_flag{self._img_number}.fits" - ) - else: - mask_name.append(None) - - masks_internal = {} - if not self._err: - if self._config["BORDER"]["make"]: - masks_internal["BORDER"] = self.mask_border( - width=self._config["BORDER"]["width"] - ) - - if not self._err: - for _type in ("MESSIER", "NGC"): - if self._config[_type]["make"]: - masks_internal[_type] = self.mask_dso( - self._config[_type]["cat_path"], - size_plus=self._config[_type]["size_plus"], - flag_value=self._config[_type]["flag"], - obj_type=_type, - ) - - if not self._err: - try: - im_pass = self._config["MD"]["im_remove"] - except Exception: - im_pass = True - - if not self._err: - path_external_flag = self._path_external_flag - - if not self._err: - if im_pass: - final_mask = self._build_final_mask( - path_mask1=mask_name[0], - path_mask2=mask_name[1], - masks_internal=masks_internal, - path_external_flag=path_external_flag, - ) - - if not self._config["HALO"]["individual"]: - if mask_name[0] is not None: - self._rm_fits1_stdout, self._rm_fits1_stderr = execute( - f"rm {mask_name[0]}" - ) - if mask_name[1] is not None: - self._rm_fits2_stdout, self._rm_fits2_stderr = execute( - f"rm {mask_name[1]}" - ) - - output_file_name = ( - f"{self._output_dir}/{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - - self._mask_to_file( - input_mask=final_mask, - output_fullpath=output_file_name, - ) - - # Handle stdout / stderr - # _CDS_stdout/_CDS_stderr are only set when find_stars ran, i.e. - # when HALO_MAKE or SPIKE_MAKE is True (False for image sims) - general_stdout = "" - general_stderr = "" - if hasattr(self, "_CDS_stdout"): - general_stdout += f"\nCDSClient\n{self._CDS_stdout}" - if self._CDS_stderr != "": - general_stderr += f"\nCDSClient\n{self._CDS_stderr}" - if hasattr(self, "_WW_stdout") or hasattr(self, "_WW_stdout"): - general_stdout += f"\n\nWeightWatcher\n{self._WW_stdout}" - if self._WW_stderr != "": - general_stderr += f"\n\nWeightWatcher\n{self._WW_stderr}" - if hasattr(self, "_rm_reg_stderr") or hasattr(self, "_rm_reg_stdout"): - general_stdout += f"\n\nrm reg file\n{self._rm_reg_stdout}" - if self._rm_reg_stderr != "": - general_stderr += f"\n\nrm reg file\n{self._rm_reg_stderr}" - if hasattr(self, "_rm_fits1_stderr") or hasattr( - self, "_rm_fits1_stdout" - ): - general_stdout += f"\n\nrm fits1 file\n{self._rm_fits1_stdout}" - if self._rm_fits1_stderr != "": - general_stderr += f"\n\nrm fits1 file\n{self._rm_fits1_stderr}" - if hasattr(self, "_rm_fits2_stderr") or hasattr( - self, "_rm_fits2_stdout" - ): - general_stdout += f"\n\nrm fits2 file\n{self._rm_fits2_stdout}" - if self._rm_fits2_stderr != "": - general_stderr += f"\n\nrm fits2 file\n{self._rm_fits2_stderr}" - - return general_stdout, general_stderr - - def find_stars(self, position, radius): - """Find Stars. - - Return GSC (Guide Star Catalog) objects for a field with center - (RA, Dec) and radius :math:`r`. - - Parameters - ---------- - position : numpy.ndarray - Position of the center of the field - radius : float - Radius in which the query is done (in arcmin) - - Returns - ------- - dict - Star dictionnary for GSC objects in the field - - Raises - ------ - ValueError - For invalid configuration options - - """ - if "star_cat" in self._config["PATH"]: - self._CDS_stdout = Table.read(self._config["PATH"]["star_cat"]) - else: - # For some exposures, Vizier returned empty star list if input position - # is not single (? or double) precision - p = np.array(position, dtype='double') - - coord = SkyCoord(ra=p[0] * units.deg, dec=p[1] * units.deg, frame="icrs") - - self._CDS_stdout = query_vizier(p[0], p[1], radius, self._CDS_cat_ID) - - self._CDS_stderr = "" - - return self._make_star_cat(self._CDS_stdout) - - def mask_border(self, width=100, flag_value=4): - """Create Mask Border. - - Mask ``width`` pixels around the image. - - Parameters - ---------- - width : int - Width of the mask mask border - flag_value : int - Value of the flag for the border (power of 2) - - Returns - ------- - numpy.ndarray - Array containing the mask - - Raises - ------ - ValueError - If ``width`` is ``None`` - - """ - if width is None: - raise ValueError("Width for border mask not provided") - - # Note that python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - flag[0:width, :] = flag_value - flag[-width:, :] = flag_value - flag[:, 0:width] = flag_value - flag[:, -width:] = flag_value - - return flag - - def mask_dso( - self, - cat_path, - size_plus=0.1, - flag_value=8, - obj_type="Messier", - ): - """Mask DSO. - - Create a circular patch for deep-sky objects (DSOs), e.g. - Messier or NGC objects. - - Parameters - ---------- - cat_path : str - Path to the deep-sky catalogue - size_plus : float - Increase the size of the mask by this factor - (e.g. ``0.1`` means 10%) - flag_value : int - Value of the flag, some power of 2 - obj_type : {'Messier', 'NGO'}, optional - Object type - - Returns - ------- - numpy.ndarray or ``None`` - If no deep-sky objects are found in the field return ``None`` and - the flag map - - Raises - ------ - ValueError - If ``size_plus`` is negative - ValueError - If ``cat_path`` is ``None`` - - """ - if size_plus < 0: - raise ValueError( - "deep-sky mask size increase variable cannot be negative" - ) - - if cat_path is None: - raise ValueError("Path to deep-sky object catalogue not provided") - - m_cat, header = fits.getdata(cat_path, header=True) - - unit_ra = file_io.get_unit_from_fits_header(header, "ra") - unit_dec = file_io.get_unit_from_fits_header(header, "dec") - m_sc = SkyCoord( - ra=m_cat["ra"] * unit_ra, - dec=m_cat["dec"] * unit_dec, - ) - - unit_size_X = file_io.get_unit_from_fits_header(header, "size_X") - unit_size_Y = file_io.get_unit_from_fits_header(header, "size_Y") - - # Loop through all deep-sky objects and check whether the object's - # disc overlaps the image footprint - indices = [] - size_max_deg = [] - for idx, m_obj in enumerate(m_cat): - - # DSO size - # r = max(m_obj['size']) * units.arcmin - r = max( - m_obj["size_X"] * unit_size_X, - m_obj["size_Y"] * unit_size_Y, - ) - r_deg = r.to(units.degree) - size_max_deg.append(r_deg) - - # Add index to list if the DSO disc overlaps the image: - # distance between DSO centre and image centre smaller than - # DSO radius plus image half-diagonal. (Testing only the image - # corners against the DSO radius, as done previously, misses - # objects that are smaller than the image and lie away from - # the corners.) - dist = self._fieldcenter["wcs"].separation(m_sc[idx]) - if dist < r_deg + self._img_radius * units.arcmin: - indices.append(idx) - - self._w_log.info( - f"Found {len(indices)} {obj_type} objects overlapping with" " image" - ) - - if len(indices) == 0: - # No closeby deep-sky object found - return None - - # Compute number of DSO center coordinates in footprint, for logging - # purpose only - n_dso_center_in_footprint = 0 - for idx in indices: - in_img = self._wcs.footprint_contains(m_sc[idx]) - self._w_log.info( - "(obj_type, ra, dec, in_img) = " - + f"({obj_type}, " - + f'{m_cat["ra"][idx]}, ' - + f'{m_cat["dec"][idx]}, ' - + f"{in_img})" - ) - - # Note: python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - nx = self._fieldcenter["pix"][0] * 2 - ny = self._fieldcenter["pix"][1] * 2 - for idx in indices: - m_center = np.hstack( - self._wcs.all_world2pix( - m_cat["ra"][idx], - m_cat["dec"][idx], - 0, - ) - ) - r_pix = ( - size_max_deg[idx].to(units.deg).value - * (1 + size_plus) - / np.abs(self._wcs.pixel_scale_matrix[0][0]) - ) - - # The following accounts for deep-sky centers outside of image, - # without creating masks for coordinates out of range - y_c, x_c = np.ogrid[0:ny, 0:nx] - mask_tmp = (x_c - m_center[0]) ** 2 + ( - y_c - m_center[1] - ) ** 2 <= r_pix**2 - - flag[mask_tmp] = flag_value - - return flag - - def missing_data(self): - """Find Missing Data. - - Look for zero-valued pixels in image. Flag if their relative number - is larger than a threshold. - """ - # Open image - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - - # Get total number of pixels - im_shape = img.get_data().shape - tot = float(im_shape[0] * im_shape[1]) - - # Compute number and ratio of missing data (zero-valued pixels) - missing = float(len(np.where(img.get_data() == 0.0)[0])) - self._ratio = missing / tot - - # Mark image as to be flagged if ratio larger than 'flag' threshold - if self._ratio >= self._config["MD"]["thresh_flag"]: - self._config["MD"]["im_flagged"] = True - else: - self._config["MD"]["im_flagged"] = False - - # Mark image as to be removed if flag is True and - # ratio large than 'remove' threshold. - # Reset all other mask 'make' flags to False (no other mask needs - # to be created) - if self._config["MD"]["remove"]: - if self._ratio >= self._config["MD"]["thresh_remove"]: - self._config["MD"]["im_remove"] = True - for idx in ["HALO", "SPIKE", "MESSIER", "BORDER"]: - self._config[idx]["make"] = False - else: - self._config["MD"]["im_remove"] = False - - img.close() - - def sphere_dist(self, position1, position2): - """Compute Spherical Distance. - - Compute spherical distance between 2 points. - - Parameters - ---------- - position1 : numpy.ndarray - [x,y] first point (in pixels) - position2 : numpy.ndarray - [x,y] second point (in pixels) - - Returns - ------- - float - The distance in degrees. - - Raises - ------ - ValueError - If input positions are not Numpy arrays - - """ - if ( - type(position1) is not np.ndarray - or type(position2) is not np.ndarray - ): - raise ValueError("Object coordinates need to be a numpy.ndarray") - - p1 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position1[0], position1[1], 1) - ) - p2 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position2[0], position2[1], 1) - ) - - dTheta = p1 - p2 - dLong = dTheta[0] - dLat = dTheta[1] - - dist = 2 * np.arcsin( - np.sqrt( - np.sin(dLat / 2.0) ** 2.0 - + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(dLong / 2.0) ** 2.0 - ) - ) - - return dist * (180.0 / np.pi) * 3600.0 - - def _get_image_radius(self, center=None): - """Get Image Radius. - - Compute the diagonal distance of the image in arcmin. - - Parameters - ---------- - center : numpy.ndarray, optional - Coordinates of the center of the image (in pixels) - - Returns - ------- - float - The diagonal distance of the image in arcmin - - Raises - ------ - TypeError - If centre is not a Numpy array - - """ - if center is None: - return ( - self.sphere_dist(self._fieldcenter["pix"], np.zeros(2)) / 60.0 - ) - - else: - if isinstance(center, np.ndarray): - return self.sphere_dist(center, np.zeros(2)) / 60.0 - else: - raise TypeError( - "Image center coordinates has to be a numpy.ndarray" - ) - - def _make_star_cat(self, CDSclient_output): - """Make Star Catalogue. - - Create a dictionary from an astroquery request. - - Parameters - ---------- - CDSclient_output : str - Output astroquery - - Returns - ------- - dict - Star dictionary containing all information - - """ - header = [] - stars = {} - - for key in self._cds_keys: - stars[key] = CDSclient_output[key] - - return stars - - def _create_mask( - self, - stars, - types="HALO", - mag_limit=18.0, - mag_pivot=13.8, - scale_factor=0.3, - ): - """Create Mask. - - Apply mask from model to stars and save into DS9 region file. - - Parameters - ---------- - stars : dict - Stars dictionary (output of ``find_stars``) - types : {'HALO', 'SPIKE'}, optional - Type of mask, options are ``HALO`` or ``SPIKE`` - mag_limit : float, optional - Faint magnitude limit for mask, default is ``18.0`` - mag_pivot : float, optional - Pivot magnitude for the model, default is ``13.8`` - scale_factor : float, optional - Scaling for the model, default is ``0.3`` - - Raises - ------ - ValueError - If no star catalogue is provided - ValueError - If an invalid option is provided for type - - """ - if stars is None: - raise ValueError("Star catalogue dictionary not provided") - - if types not in ("HALO", "SPIKE"): - raise ValueError('Mask types need to be in ["HALO", "SPIKE"]') - - if self._config[types]["reg_file"] is None: - reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - else: - reg = self._config[types]["reg_file"] - - mask_model = np.loadtxt( - self._config[types]["maskmodel_path"] - ).transpose() - mask_reg = open(reg, "w") - - stars_used = [[], [], []] - - """ - star_zip = zip( - stars["RA(J2000)"], - stars["Dec(J2000)"], - stars["Fmag"], - stars["Jmag"], - stars["Vmag"], - stars["Nmag"], - stars["Clas"], - ) - """ - - # Get keys without object name - keys_to_use = self._cds_keys[1:] - star_zip = zip(*(stars[k] for k in keys_to_use)) - - for ra, dec, Fmag, Jmag, Vmag, Nmag, clas in star_zip: - # Compute mean magnitude over the available (finite) bands. - # Missing GSC bands are NaN and must be excluded: a single NaN - # would make the mean NaN and silently fail the - # ``mag < mag_limit`` test below, leaving bright stars with - # incomplete photometry (the ones that most need masking) - # unmasked. - mags = [ - band - for band in (Fmag, Jmag, Vmag, Nmag) - if band is not None and np.isfinite(band) - ] - if len(mags) > 0: - mag = sum(mags) / len(mags) - else: - mag = None - self._w_log.info( - f"No finite {types} magnitude for star at ra={ra} " - + f"dec={dec}; object not masked" - ) - - if ( - ra is not None - and dec is not None - and mag is not None - and clas is not None - ): - if (mag < mag_limit) and (clas == 0): - scaling = 1.0 - scale_factor * (mag - mag_pivot) - if scaling < self._scaling_min: - scaling = self._scaling_min - pos = self._wcs.all_world2pix(ra, dec, 0) - stars_used[0].append(pos[0]) - stars_used[1].append(pos[1]) - stars_used[2].append(scaling) - - for idx in range(len(stars_used[0])): - poly = "polygon(" - for x, y in zip(mask_model[0], mask_model[1]): - angle = np.arctan2(y, x) - ll = stars_used[2][idx] * np.sqrt(x**2 + y**2) - xnew = ll * np.cos(angle) - ynew = ll * np.sin(angle) - poly = ( - f"{poly}{str(stars_used[0][idx] + xnew + 0.5)} " - + f"{str(stars_used[1][idx] + ynew + 0.5)} " - ) - poly = f"{poly})\n" - mask_reg.write(poly) - - mask_reg.close() - - def _exec_WW(self, types="HALO"): - """Execute WeightWatcher. - - Execute WeightWatcher to transform ``.reg`` to ``.fits`` flag map. - - Parameters - ---------- - types : {'HALO', 'SPIKE', 'ALL'}, optional - Type of WeightWatcher execution, options are ``HALO``, - ``SPIKE`` or ``ALL`` - - Raises - ------ - BaseCatalogue.CatalogFileNotFound - If catalogue file not found - - """ - if types in ("HALO", "SPIKE"): - - default_reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - default_out = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}_flag' - + f"{self._img_number}.fits" - ) - - if self._config[types]["reg_file"] is None: - reg = default_reg - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute(f"rm {reg}") - - - else: - reg = self._config[types]["reg_file"] - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - - elif types == "ALL": - - default_reg = [ - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo{self._img_number}.reg" - ), - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"spike{self._img_number}.reg" - ), - ] - default_out = ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo_spike_flag{self._img_number}.fits" - ) - - if self._config["HALO"]["reg_file"] is None: - reg = default_reg - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute( - f"rm {reg[0]} {reg[1]}" - ) - else: - reg = [ - self._config["HALO"]["reg_file"], - self._config["SPIKE"]["reg_file"], - ] - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - else: - raise ValueError("Types must be in ['HALO','SPIKE','ALL']") - - if (self._WW_stderr != "") or (self._rm_reg_stderr != ""): - self._err = True - - def _build_final_mask( - self, - path_mask1, - path_mask2=None, - masks_internal=None, - path_external_flag=None, - ): - """Create Final Mask. - - Create the final mask by combining the individual masks. - - Parameters - ---------- - path_mask1 : str - Path to a mask (FITS format) - path_mask2 : str, optional - Path to a mask (FITS format) - masks_internal : dict, optional - Internally created masks - path_external_flag : str, optional - Path to an external flag file - - Returns - ------- - numpy.ndarray - Array containing the final mask - - Raises - ------ - ValueError - If all masks are of type ``None`` - TypeError - If border is not a Numpy array - TypeError - If Messier mask is not a Numpy array - - """ - final_mask = None - - if path_mask1 is None and path_mask2 is None and not masks_internal: - raise ValueError( - "No paths to mask files containing halos and/or spikes," - + " borders, or deep-sky objects provided" - ) - - if path_mask1 is not None: - mask1 = file_io.FITSCatalogue(path_mask1, hdu_no=self._hdu) - mask1.open() - dat = mask1.get_data() - final_mask = dat[:, :] - - if path_mask2 is not None: - mask2 = file_io.FITSCatalogue(path_mask2, hdu_no=self._hdu) - mask2.open() - if final_mask is not None: - final_mask += mask2.get_data()[:, :] - else: - final_mask = mask2.get_data()[:, :] - - for typ in masks_internal: - if masks_internal[typ] is not None: - if type(masks_internal[typ]) is np.ndarray: - if final_mask is not None: - final_mask += masks_internal[typ] - else: - final_mask = masks_internal[typ] - else: - raise TypeError( - f"internally created mask of type {typ} " - + "has to be numpy.ndarray" - ) - - if path_external_flag is not None: - external_flag = file_io.FITSCatalogue( - path_external_flag, - hdu_no=self._hdu, - ) - external_flag.open() - if final_mask is not None: - final_mask = final_mask.astype(np.int16, copy=False) - try: - ext_flag = external_flag.get_data() - except: - self._w_log.info( - "Problem while getting external flag data. Check" - + f" whether file {path_external_flag} is not corrupt" - ) - raise - final_mask += ext_flag[:, :] - else: - final_mask = external_flag.get_data()[:, :] - external_flag.close() - - return final_mask.astype(np.int16, copy=False) - - def _mask_to_file(self, input_mask, output_fullpath): - """Mask to File. - - Save the mask to a fits file. - - Parameters - ---------- - input_mask : numpy.ndarray - Mask to save - output_fullpath : str - Path of the output file - - Raises - ------ - ValueError - If input_mask is type ``None`` - ValueError - If output_fullpath is type ``None`` - - """ - if input_mask is None: - raise ValueError("input mask file path not provided") - if output_fullpath is None: - raise ValueError("output mask file path not provided") - - out = file_io.FITSCatalogue( - output_fullpath, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - hdu_no=0, - ) - out.save_as_fits(data=input_mask, image=True) - - if self._config["MD"]["make"]: - out.open() - out.add_header_card( - "MRATIO", - self._ratio, - "ratio missing_pixels/all_pixels", - ) - out.add_header_card( - "MFLAG", - self._config["MD"]["im_flagged"], - f'threshold value {self._config["MD"]["thresh_flag"]:.3}', - ) - - # Write WCS information to header - if self._wcs: - header_wcs = self._wcs.to_header() - for card in header_wcs: - out.add_header_card( - card, - header_wcs[card], - header_wcs.comments[card], - ) - out.close() - - def _get_temp_dir_path(self, temp_dir_path): - """Get Temporary Directory Path. - - Create the path and the directory for temporary files. - - Parameters - ---------- - temp_dir_path : str - Path to the temporary directory, a value of ``OUTPUT`` will include - the temporary files in the run directory - - Returns - ------- - str - Path to the temporary directory - - Raises - ------ - ValueError - If ``temp_dir_path`` is of type ``None`` - - """ - if temp_dir_path is None: - raise ValueError("Temporary directory path not provided") - - path = temp_dir_path.replace(" ", "") - - if path == "OUTPUT": - path = f"{self._output_dir}/temp" - - path += "/" - if not os.path.isdir(path): - mkdir(path) - - return path diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py new file mode 100644 index 000000000..fed9dcd92 --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -0,0 +1,103 @@ +"""MASK QUERY PACKAGE. + +This package contains the module for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +:Parent module: ``sextractor_runner`` + +:Input: Single-exposure single-CCD SExtractor catalogue + +:Output: The same catalogue with an added ``MASK_EXT`` column + +Two classes of mask +=================== + +ShapePipe distinguishes them, and they are not interchangeable: + +**Instrument flags** mark a CORRUPTED MEASUREMENT — bad columns, saturation — +in the flag image delivered with each exposure. A flagged pixel carries no +usable signal, so these are the only masks that reject anything inside the +pipeline: ``setools`` drops flagged stars via ``IMAFLAGS_ISO``, and ``ngmix`` +zero-weights flagged pixels and drops epochs that lose too many. + +**Healsparse masks** are sky-fixed LOCATION flags — a star halo, a manual +region, a band with no data. They say where an object is, not that its pixels +are broken, so what to do about one is an analysis decision. They are queried +into columns and every rejection happens downstream: ``MASK_EXT`` here, +``MASK_`` in ``make_cat``. Nothing in the pipeline cuts on either. + +Description +=========== + +This module sits between SExtractor and ``setools`` on the exposure chain: it +reads each detection's windowed world position (``XWIN_WORLD``, ``YWIN_WORLD`` +in the ``LDAC_OBJECTS`` extension) and looks it up in the configured healsparse +maps, writing one integer column: + +``MASK_EXT`` + ``0`` for an object no configured map flags, nonzero otherwise. What the + nonzero value *is* depends on the maps: a boolean map — which is what the + UNIONS per-bit products are, and what the shipped config names — can only + contribute ``1``, so with those the column is 0/1 and says nothing about + which map fired. An integer map contributes its own value (optionally + ``& MASK_BITS``), and contributions are OR-ed, so a bit-packed map does + carry its bits through. Nothing downstream reads more than ``== 0``. + +The single column exists because ``setools`` expressions support only +``< > <= >= == !=`` — no bitwise operators — so any bit selection has to happen +here, leaving the config a plain test for zero. + +Off by default, and a no-op when off +==================================== + +``MASK_PATHS`` ships COMMENTED OUT. With no maps configured the module is a +strict no-op: the catalogue is copied through unchanged, with no ``MASK_EXT`` +column at all — the same gating ``make_cat`` gives ``MASK_EXT_PATHS``. It stays +in the ``MODULE`` chain either way, so enabling the query is uncommenting one +line and never editing the chain. + +Even with maps configured, nothing cuts on the result. The PSF star selection +deliberately starts from outlier rejection alone, rejecting only on the +instrument flags, and ``MASK_EXT`` is the configurable pickup if that proves +insufficient. Writing the column without cutting on it is what lets the effect +of a mask on the star sample be *measured* before it is imposed. + +That pickup is one line: add ``MASK_EXT == 0`` beside each +``IMAFLAGS_ISO == 0`` in ``star_selection.setools``. The escape hatch is +deliberate, and that file's header says so. + +The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with +``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is +written once. That module's docstring documents the off-coverage convention. + +What gets queried is deliberately narrow +======================================== + +``MASK_PATHS`` is a *list of maps to record against each PSF-star candidate*, +not a list of every mask that exists. The committed configs name exactly one +map — the UNIONS star-body product (bit 2). Halo bits 0 and 1 are excluded on +purpose: halos flag objects for the final catalogue, they say nothing about +whether a star is a good PSF sample (mask-force telecon, 2026-07-21). MaxiMask +is not queried here either. + +Widening it costs a config edit and no code — add a path. That is why the +contract is a path list rather than a bit mask: the UNIONS products are one +boolean map per bit, so choosing bits *is* choosing files, and ``MASK_BITS`` +exists only for integer maps that pack several bits into one file. + +Module-specific config file entries +=================================== + +MASK_PATHS : str + Comma-separated healsparse map paths to query +MASK_BITS : int, optional + Bit mask applied to integer maps (``value & MASK_BITS`` flags); default is + to flag on any nonzero value. Ignored for boolean maps, which flag on + ``True`` +PREFIX : str, optional + Output file prefix + +""" + +__all__ = ["mask_query"] diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py new file mode 100644 index 000000000..aff212b5a --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -0,0 +1,133 @@ +"""MASK QUERY. + +Class to flag SExtractor detections against external healsparse masks. + +:Author: Claude Fable 5, for PR #847 + +""" + +import shutil + +import numpy as np + +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + + +class MaskQuery(object): + """Mask Query. + + Query external healsparse masks at every detection of a SExtractor + catalogue and write the result as a single ``MASK_EXT`` column into a copy + of that catalogue. + + With no maps configured this is a strict no-op: the input catalogue is + copied through unchanged, with no ``MASK_EXT`` column, matching + ``make_cat``'s ``MASK_EXT_PATHS`` contract (absent key, nothing happens). + The copy is what keeps the module in the chain — ``setools`` reads this + module's output, so producing no file would break the chain rather than + disable the query. + + Parameters + ---------- + sexcat_path : str + Path to the input SExtractor catalogue + output_path : str + Path to the output catalogue + mask_paths : list + Paths to the healsparse maps to query; empty means no-op + bits : int, optional + Bit mask applied to integer maps; default ``None`` flags any nonzero + value + w_log : logging.Logger, optional + Logging instance + + """ + + def __init__( + self, + sexcat_path, + output_path, + mask_paths, + bits=None, + w_log=None, + ): + + self._sexcat_path = sexcat_path + self._output_path = output_path + self._mask_paths = mask_paths + self._bits = bits + self._w_log = w_log + + def process(self): + """Process. + + Query the masks and write the flagged catalogue. + + Returns + ------- + int + Number of flagged objects + + """ + if not self._mask_paths: + # No maps configured: copy the catalogue through byte-for-byte. + # A rewrite through FITSCatalogue would round-trip the LDAC HDUs + # for no reason; this way an unconfigured run is provably + # identical to its input. + shutil.copyfile(self._sexcat_path, self._output_path) + if self._w_log is not None: + self._w_log.info( + "No MASK_PATHS configured; passing " + + f"{self._sexcat_path} through unchanged, no MASK_EXT" + + " column written" + ) + return 0 + + ori_cat = file_io.FITSCatalogue( + self._sexcat_path, + SEx_catalogue=True, + ) + ori_cat.open() + data = ori_cat.get_data() + + # A CCD SExtractor found nothing on is tolerated all along this chain + # (setools' ~0.2% attrition, psfex_interp's floor=0 warn), so it must + # not be an error here either. An empty LDAC table has no columns to + # index, so read the positions only when there are rows, and still + # publish an output file — a missing sexcat_ext would look to the file + # handler like a crash rather than like an empty CCD. + if len(data) == 0: + ra = np.zeros(0) + dec = np.zeros(0) + if self._w_log is not None: + self._w_log.info( + "No detections in " + + f"{self._sexcat_path}; writing an empty MASK_EXT column" + ) + else: + ra = np.copy(data["XWIN_WORLD"]) + dec = np.copy(data["YWIN_WORLD"]) + + flag = mask_query_util.flag_positions( + self._mask_paths, + ra, + dec, + bits=self._bits, + w_log=self._w_log, + ) + # int32 is what the catalogue carries; the UNIONS bit table needs 12 + # bits, and no OR of it can overflow. + flag = flag.astype(np.int32) + + new_cat = file_io.FITSCatalogue( + self._output_path, + SEx_catalogue=True, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + ori_cat.add_col( + "MASK_EXT", flag, new_cat=True, new_cat_inst=new_cat + ) + ori_cat.close() + + return int(np.count_nonzero(flag)) diff --git a/src/shapepipe/modules/mask_query_runner.py b/src/shapepipe/modules/mask_query_runner.py new file mode 100644 index 000000000..a636560b2 --- /dev/null +++ b/src/shapepipe/modules/mask_query_runner.py @@ -0,0 +1,76 @@ +"""MASK_QUERY RUNNER. + +Module runner for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +""" + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.modules.module_decorator import module_runner +from shapepipe.utilities import mask_query as mask_query_util + + +@module_runner( + version="1.0", + input_module="sextractor_runner", + file_pattern=["sexcat"], + file_ext=[".fits"], + depends=["numpy", "healsparse"], +) +def mask_query_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Mask Query Runner.""" + sexcat_path = input_file_list[0] + + # Get file prefix (optional) + if config.has_option(module_config_sec, "PREFIX"): + prefix = config.get(module_config_sec, "PREFIX") + if (prefix.lower() != "none") & (prefix != ""): + prefix = prefix + "_" + else: + prefix = "" + else: + prefix = "" + + # Absent (or empty) MASK_PATHS is a no-op, not an error — the same gating + # make_cat gives MASK_EXT_PATHS. The module stays in the MODULE chain and + # passes the catalogue through, so enabling the query is a config edit and + # disabling it never means editing the chain. + if config.has_option(module_config_sec, "MASK_PATHS"): + mask_paths = mask_query_util.parse_map_paths( + config.getexpanded(module_config_sec, "MASK_PATHS") + ) + else: + mask_paths = [] + + # Any nonzero map value flags unless a bit selection is given + if config.has_option(module_config_sec, "MASK_BITS"): + bits = config.getint(module_config_sec, "MASK_BITS") + else: + bits = None + + output_path = ( + f'{run_dirs["output"]}/{prefix}sexcat_ext{file_number_string}.fits' + ) + + mq_inst = MaskQuery( + sexcat_path, + output_path, + mask_paths, + bits=bits, + w_log=w_log, + ) + n_flagged = mq_inst.process() + + if mask_paths: + w_log.info(f"MASK_EXT nonzero for {n_flagged} objects") + + # No return objects + return None, None diff --git a/src/shapepipe/modules/mask_runner.py b/src/shapepipe/modules/mask_runner.py deleted file mode 100644 index 13fbf9b6c..000000000 --- a/src/shapepipe/modules/mask_runner.py +++ /dev/null @@ -1,120 +0,0 @@ -"""MASK RUNNER. - -Module runner for ``mask``. - -:Author: Axel Guinot, Martin Kilbinger - -""" - -from shapepipe.modules.mask_package.mask import Mask -from shapepipe.modules.module_decorator import module_runner - - -@module_runner( - version="1.0", - file_pattern=["image", "weight", "flag"], - file_ext=[".fits", ".fits", ".fits"], - depends=["numpy", "astropy"], - executes=["weightwatcher"], - numbering_scheme="_0", -) -def mask_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Mask Runner.""" - # Get number of input files - n_inputs = len(input_file_list) - - # Set options for 2 inputs - if n_inputs == 2: - ext_flag_name = None - ext_star_cat = None - - # Set options for 3 inputs - elif n_inputs == 3: - if config.getboolean(module_config_sec, "USE_EXT_FLAG"): - ext_flag_name = input_file_list[2] - ext_star_cat = None - elif config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = None - ext_star_cat = input_file_list[2] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag or " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Set options for 4 inputs - elif n_inputs == 4: - if config.getboolean( - module_config_sec, "USE_EXT_FLAG" - ) and config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = input_file_list[2] - ext_star_cat = input_file_list[3] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag and " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Raise error for invalid settings - else: - raise ValueError( - f'Found {n_inputs} inputs and these must be "image", "weight" and ' - + '"ext_flags", "ext_star_cat" (optional). Check the MASK_RUNNER ' - + "section of the config file to make sure you have the " - + "appropriate settings." - ) - - # Get path to mask configuration options - config_file = config.getexpanded(module_config_sec, "MASK_CONFIG_PATH") - - # Get mask HDU number - if config.has_option(module_config_sec, "HDU"): - hdu = config.getint(module_config_sec, "HDU") - else: - hdu = 0 - - # Get mask mask file name prefix - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - else: - prefix = "" - - outname_base = "flag" - - # Path to check for already created mask files - if config.has_option(module_config_sec, "CHECK_EXISTING_DIR"): - check_existing_dir = config.getexpanded( - module_config_sec, "CHECK_EXISTING_DIR" - ) - else: - check_existing_dir = None - - # Create instance of Mask - mask_inst = Mask( - *input_file_list[:2], - image_prefix=prefix.replace(" ", ""), - image_num=file_number_string, - config_filepath=config_file, - output_dir=run_dirs["output"], - path_external_flag=ext_flag_name, - outname_base=outname_base, - star_cat_path=ext_star_cat, - check_existing_dir=check_existing_dir, - hdu=hdu, - w_log=w_log, - ) - - # Process module - stdout, stderr = mask_inst.make_mask() - - # Return stdout and stderr - return stdout, stderr diff --git a/src/shapepipe/modules/mccd_interp_runner.py b/src/shapepipe/modules/mccd_interp_runner.py index 82ec7b65f..e61001fbf 100644 --- a/src/shapepipe/modules/mccd_interp_runner.py +++ b/src/shapepipe/modules/mccd_interp_runner.py @@ -8,6 +8,7 @@ import os +from shapepipe.pipeline.exp_utils import get_exp_output_dirs from shapepipe.pipeline.run_log import get_last_dir from shapepipe.modules.mccd_package import ( @@ -87,9 +88,34 @@ def mccd_interp_runner( ) elif mode == "MULTI-EPOCH": - module = config.getexpanded(module_config_sec, "PSF_MODEL_DIR") - psf_model_dir = get_last_dir(run_dirs["run_log"], module) - psf_model_pattern = config.get(module_config_sec, "PSF_MODEL_PATTERN") + if config.has_option(module_config_sec, "ME_DOT_PSF_EXP_DIR"): + exp_base_dir = config.getexpanded( + module_config_sec, "ME_DOT_PSF_EXP_DIR" + ) + if len(input_file_list) < 3: + raise ValueError( + "ME_DOT_PSF_EXP_DIR requires the exposure-numbers file" + " as a third input; add 'exp_numbers' to FILE_PATTERN" + f" and FILE_EXT in the [{module_config_sec}] config section." + ) + exp_numbers_file = input_file_list[2] + model_runner = config.get( + module_config_sec, + "ME_DOT_PSF_RUNNER", + fallback="mccd_fit_val_runner", + ) + psf_model_dir = get_exp_output_dirs( + exp_base_dir, exp_numbers_file, model_runner, w_log + ) + psf_model_pattern = config.get( + module_config_sec, "ME_DOT_PSF_PATTERN" + ) + else: + module = config.getexpanded(module_config_sec, "PSF_MODEL_DIR") + psf_model_dir = get_last_dir(run_dirs["run_log"], module) + psf_model_pattern = config.get( + module_config_sec, "PSF_MODEL_PATTERN" + ) galcat_path = input_file_list[0] # The WCS log is supplied as a positional input (via FILE_PATTERN) # in MULTI-EPOCH mode, not by the decorator default. diff --git a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py index a01ca095d..b6183f4e1 100644 --- a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py +++ b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py @@ -407,7 +407,9 @@ def process_me(self, dot_psf_dir, dot_psf_pattern, f_wcs_path): Path to the log file containing the WCS for each CCD """ - self._dot_psf_dir = dot_psf_dir + self._dot_psf_dir = ( + [dot_psf_dir] if isinstance(dot_psf_dir, str) else dot_psf_dir + ) self._dot_psf_pattern = dot_psf_pattern self._f_wcs_file = SqliteDict(f_wcs_path) @@ -467,14 +469,19 @@ def _interpolate_me(self): # dot_psf_path = self._dot_psf_dir + '/' +\ # self._dot_psf_pattern + '-' + exp_name + '-' + str(ccd) +\ # '.psf' - mccd_model_path = ( - self._dot_psf_dir - + "/" - + self._dot_psf_pattern - + "-" - + exp_name - + ".npy" - ) + found = False + for dot_psf_dir in self._dot_psf_dir: + mccd_model_path = ( + f"{dot_psf_dir}/{self._dot_psf_pattern}-{exp_name}.npy" + ) + if os.path.exists(mccd_model_path): + found = True + break + if not found: + self._w_log.info( + f"No .npy file found for exposure {exp_name}" + ) + continue ind_obj = np.where(cat.get_data(hdu_index)["CCD_N"] == ccd)[0] obj_id = all_id[ind_obj] diff --git a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py index 5ecdb6fb0..b0be88944 100644 --- a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py +++ b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py @@ -7,7 +7,6 @@ """ import os -import re import warnings import numpy as np @@ -16,6 +15,48 @@ from shapepipe.pipeline import file_io +def chunk_path(input_file, n): + """Chunk Path. + + Derive chunk ``n``'s input path from chunk 1's. + + The separate catalogues live in ShapePipe run directories whose names differ + only in the chunk number (``run_sp_tile_ngmix_Ng1u`` -> + ``run_sp_tile_ngmix_Ng2u``). The substitution is confined to that + run-directory component: replacing the first "1" found anywhere in the path + breaks for absolute paths whose parent directories carry digits, e.g. a + sharded store ``.../tiles/21/210.282/output/run_..._Ng1u/...``. + + Parameters + ---------- + input_file : str + Path to chunk 1's catalogue + n : int + Chunk number + + Returns + ------- + str + Path to chunk ``n``'s catalogue + + Raises + ------ + ValueError + If no run-directory component of the path carries a chunk number + + """ + parts = input_file.split(os.sep) + for idx in reversed(range(len(parts))): + if parts[idx].startswith("run_") and "1" in parts[idx]: + parts[idx] = parts[idx].replace("1", str(n), 1) + return os.sep.join(parts) + + raise ValueError( + f"Cannot derive chunk {n}'s path from '{input_file}': no 'run_*' " + + "directory component contains a chunk number '1'" + ) + + class MergeSep(object): """Merge Sep. @@ -79,8 +120,7 @@ def process(self): input_path_n = [] input_path_n.append(input_file) for n in range(2, self._n_split_max + 1): - res = re.sub("1", str(n), input_file, 1) - input_path_n.append(res) + input_path_n.append(chunk_path(input_file, n)) # Open first catalogue, read number of extensions and columns cat0 = file_io.FITSCatalogue(input_file, SEx_catalogue=True) diff --git a/src/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py index 927c79b76..6e99cf3ec 100644 --- a/src/shapepipe/modules/merge_sep_cats_runner.py +++ b/src/shapepipe/modules/merge_sep_cats_runner.py @@ -28,7 +28,7 @@ def merge_sep_cats_runner( ): """Define The Merge SEP Catalogues Runner.""" # Get config entries - n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") + n_split_max = int(config.getexpanded(module_config_sec, "N_SPLIT_MAX")) file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") file_ext = config.getlist(module_config_sec, "FILE_EXT") diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 89727caff..66323822c 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -43,15 +43,23 @@ (no batch saving) ID_OBJ_MIN : int ID of first galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, so an orchestrator can + set the object range per chunk, for example + ``ID_OBJ_MIN = $SP_NGMIX_ID_OBJ_MIN``. ID_OBJ_MAX : int ID of last galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, as for ``ID_OBJ_MIN``. BKG_RMS_VIGNET_PATH : str, optional Path to a ``background_rms_vignet*.sqlite`` file produced by ``vignetmaker_runner``. The string may contain ``{file_number_string}``, which is replaced by the current tile ID. +Random number generation +======================== + +Each object gets its own random number stream, seeded from its sky position +and CCD (``ngmix.position_seed``). + """ __all__ = ["ngmix"] diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index e07e0dd7b..1167f3199 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -128,16 +128,10 @@ def get_prior(pixel_scale, rng, T_range=None, F_range=None): def position_seed(ra, dec, ccd): """Deterministic RNG seed from an object's sky position (ngmix#796). - For image-simulation m-bias with the Pujol estimator, the same scene is - simulated under different applied shears ("image branches") and the shear - response is read from the branch difference of the SAME objects. Metacal's - ``fixnoise`` adds a counter-noise realisation drawn from an RNG; if that RNG - is seeded per tile, an object gets *different* added noise in different - branches (detection order differs), and the noise fails to cancel in the - difference, inflating ``sigma_m``. Seeding the per-object RNG from sky - position instead makes the same object draw the same added noise (and the - same fit guesses) in every branch, so both cancel and the m-bias error - shrinks. Off in production; a knob for the sim path only. + Position seeding gives the same object the same RNG stream in each image + branch, provided its sky position falls in the same seed box. It also makes + the result independent of how the tile is split into + ``ID_OBJ_MIN``/``ID_OBJ_MAX`` chunks, which is why it is now the only mode. Box math (kept exactly as Fabian's issue #796):: @@ -299,8 +293,7 @@ def __init__( self.ra = [] self.dec = [] # CCD number of the first epoch, used only to build the per-object - # position seed (see :func:`position_seed`, ngmix#796). ``None`` when - # position seeding is off, so the field costs nothing on the hot path. + # position seed (see :func:`position_seed`). self.ccd = None self.bkg_sub = bkg_sub self.megacam_flip = megacam_flip @@ -439,13 +432,11 @@ class Ngmix(object): dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. - seed_from_position : bool, optional - If ``True``, replace the tile-level RNG with a per-object RNG seeded - from the object's sky position (:func:`position_seed`) inside the - object loop, so metacal's ``fixnoise`` counter-noise and the fit - guesses cancel across Pujol image-simulation branches (ngmix#796). The - default ``False`` leaves the production path byte-identical. See - :func:`position_seed` for the physics and the seed construction. + + Notes + ----- + The RNG is always per object and seeded from that object's sky position; + :func:`position_seed` says what that buys. Raises ------ @@ -474,7 +465,6 @@ def __init__( blend_handling="noisefill", seg_cat_path=None, dilate_neighbour=1, - seed_from_position=False, metacal_psf="fitgauss", ): @@ -544,15 +534,14 @@ def __init__( self._blend_handling = blend_handling self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour - self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf self._w_log = w_log - # Initiatlise random generator - seed = int(''.join(re.findall(r'\d+', self._file_number_string))) - self._rng = np.random.RandomState(seed) - self._w_log.info(f'Random generator initialisation seed = {seed}') + self._w_log.info( + 'Per-object RNG seeded from sky position (ngmix#796): results are' + ' invariant to how the tile is split into object chunks' + ) # Pixel scale: an explicit positive PIXEL_SCALE overrides; otherwise # derive it from the image WCS so it can never drift from the pixels @@ -573,12 +562,6 @@ def __init__( f'PIXEL_SCALE from config = {self._pixel_scale:.6f} arcsec' ) - if self._seed_from_position: - self._w_log.info( - 'SEED_FROM_POSITION on: per-object RNG seeded from sky position' - ' for Pujol noise cancellation (image sims, ngmix#796)' - ) - @classmethod def MegaCamFlip(self, vign, ccd_nb): """Flip for MegaCam. @@ -607,18 +590,6 @@ def MegaCamFlip(self, vign, ccd_nb): # swap y axis so origin is on bottom-left return vign - def get_prior(self, T_range=None, F_range=None): - """Get Prior. - - Returns - ------- - ngmix.joint_prior.PriorSimpleSep - """ - return get_prior( - self._pixel_scale, self._rng, - T_range=T_range, F_range=F_range, - ) - def compile_results(self, results): """Compile Results. @@ -1001,7 +972,6 @@ def process(self): vignet_cat = self._vignet_cat final_res = [] - prior = self.get_prior() count = 0 n_empty_cat = 0 @@ -1023,36 +993,39 @@ def process(self): id_last = obj_id count += 1 - # Skip objects with no multi-epoch PSF or vignet data - if (vignet_cat.psf_vign_cat[str(obj_id)] == 'empty' - or vignet_cat.gal_vign_cat[str(obj_id)] == 'empty'): + # Skip objects with no multi-epoch PSF or vignet data. + # Read each store once here and pass the dicts down: every + # sqlitedict access unpickles the object's whole all-epoch dict. + psf_obj = vignet_cat.psf_vign_cat[str(obj_id)] + gal_obj = vignet_cat.gal_vign_cat[str(obj_id)] + if psf_obj == 'empty' or gal_obj == 'empty': n_empty_cat += 1 continue - stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat, self._bkg_sub) + stamp = prepare_postage_stamps( + vignet_cat, + obj_id, + i_tile, + tile_cat, + self._bkg_sub, + psf_obj, + gal_obj, + ) if len(stamp.gals) == 0: n_no_epoch += 1 continue - # Position-seeded per-object RNG for Pujol noise cancellation in - # image sims (ngmix#796): the same object gets the same fixnoise - # counter-noise and fit guesses in every shear branch, so both - # cancel in the branch difference. The prior is rebuilt from the - # same per-object RNG because the guesser draws its initial guess - # via prior.sample() (ngmix guessers.py), which consumes the RNG the - # prior was CONSTRUCTED with — so a per-object rng alone would leave - # the guess drawing from the shared tile stream and break - # cancellation. Off in production, where the single tile-level - # self._rng and the tile-level prior carry the whole loop. - if self._seed_from_position: - obj_rng = np.random.RandomState( - position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) - ) - obj_prior = get_prior(self._pixel_scale, obj_rng) - else: - obj_rng = self._rng - obj_prior = prior + # Per-object RNG, seeded from (ra, dec, ccd) — see + # :func:`position_seed`. The prior is rebuilt from that same RNG + # because the guesser draws its initial guess via prior.sample() + # (ngmix guessers.py), which consumes the RNG the prior was + # CONSTRUCTED with: a per-object rng alone would leave the guess + # drawing from a shared stream and break the invariance. + obj_rng = np.random.RandomState( + position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) + ) + obj_prior = get_prior(self._pixel_scale, obj_rng) try: flux_guess = ( @@ -1165,25 +1138,49 @@ def process(self): # Log mean ellipticity statistics self.log_mean_ellipticity() -def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): +def prepare_postage_stamps( + vignet, + obj_id, + i_tile, + tile_cat, + bkg_sub=True, + psf_obj=None, + gal_obj=None, +): # define per-object lists of individual exposures to go into ngmix stamp = Postage_stamp(bkg_sub=bkg_sub) + # Read each store's per-object dict ONCE: every sqlitedict access + # unpickles the object's whole all-epoch dict, so keeping these out of + # the epoch loop below saves O(n_epoch) full unpickles per store. + if psf_obj is None: + psf_obj = vignet.psf_vign_cat[str(obj_id)] + if gal_obj is None: + gal_obj = vignet.gal_vign_cat[str(obj_id)] + bkg_obj = ( + vignet.bkg_vign_cat[str(obj_id)] + if stamp.bkg_sub and vignet.bkg_vign_cat is not None + else None + ) + flag_obj = vignet.flag_vign_cat[str(obj_id)] + weight_obj = vignet.weight_vign_cat[str(obj_id)] + bkg_rms_obj = ( + vignet.bkg_rms_vign_cat[str(obj_id)] + if vignet.bkg_rms_vign_cat is not None + else None + ) + wcs_cache = {} #identify exposure and ccd number from psf catalog - psf_expccd_names = list(vignet.psf_vign_cat[str(obj_id)].keys()) + psf_expccd_names = list(psf_obj.keys()) for expccd_name in psf_expccd_names: exp_name, ccd_n = re.split('-', expccd_name) - gal_vign = ( - vignet.gal_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + gal_vign = gal_obj[expccd_name]['VIGNET'] if np.all(gal_vign == 0): continue if stamp.bkg_sub: - bkg_vign = ( - vignet.bkg_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + bkg_vign = bkg_obj[expccd_name]['VIGNET'] gal_vign_sub_bkg = background_subtract( gal_vign, bkg_vign @@ -1216,9 +1213,7 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): if stamp.megacam_flip and tile_seg is not None: tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) - flag_vign = ( - vignet.flag_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + flag_vign = flag_obj[expccd_name]['VIGNET'] if tile_vign is not None: flag_vign[np.where(tile_vign == -1e30)] = 2**10 v_flag_tmp = flag_vign.ravel() @@ -1226,23 +1221,27 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): if len(np.where(v_flag_tmp != 0)[0]) / v_flag_tmp.size > 1 / 3.0: continue - weight_vign = vignet.weight_vign_cat[str(obj_id)][expccd_name]['VIGNET'] + weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( - vignet.bkg_rms_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - if vignet.bkg_rms_vign_cat is not None + bkg_rms_obj[expccd_name]['VIGNET'] + if bkg_rms_obj is not None else None ) - epoch_wcs = vignet.f_wcs_file[exp_name][int(ccd_n)]['WCS'] + # One unpickle per exposure (all CCDs), reused across this object's + # epochs; the cache is per-call, so bounded by the object's exposures + if exp_name not in wcs_cache: + wcs_cache[exp_name] = vignet.f_wcs_file[exp_name] + ccd_wcs = wcs_cache[exp_name][int(ccd_n)] + + epoch_wcs = ccd_wcs['WCS'] jacob = get_galsim_jacobian( epoch_wcs, tile_cat.ra[i_tile], tile_cat.dec[i_tile] ) - header = fits.Header.fromstring( - vignet.f_wcs_file[exp_name][int(ccd_n)]['header'] - ) + header = fits.Header.fromstring(ccd_wcs['header']) # rescale by relative zero-points ( @@ -1258,9 +1257,7 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): # gather postage stamps in all of the epochs stamp.gals.append(gal_vign_scaled) - stamp.psfs.append( - vignet.psf_vign_cat[str(obj_id)][expccd_name]['VIGNET'] - ) + stamp.psfs.append(psf_obj[expccd_name]['VIGNET']) stamp.weights.append(weight_vign_scaled) stamp.flags.append(flag_vign) stamp.bkg_rms.append(bkg_rms_vign_scaled) @@ -1272,8 +1269,8 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): stamp.ra.append(tile_cat.ra[i_tile]) stamp.dec.append(tile_cat.dec[i_tile]) # CCD of the first surviving epoch — Fabian's coord_list[0] convention - # for the position seed (ngmix#796). All epochs of one object share the - # ra/dec above, so first-epoch CCD pins one deterministic seed stream. + # for the position seed. All epochs of one object share the ra/dec + # above, so first-epoch CCD pins one deterministic seed stream. if stamp.ccd is None: stamp.ccd = int(ccd_n) @@ -1574,8 +1571,8 @@ def prepare_ngmix_weights( weight : numpy.ndarray flag : numpy.ndarray rng : numpy.random.RandomState - Random state for the noise realisations (seeded per tile for - reproducibility). + Random state for the noise realisations (seeded per object; see + :func:`position_seed`). bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. If supplied, unmasked pixels use ``1 / bkg_rms**2`` as the ngmix inverse variance. @@ -1703,8 +1700,8 @@ def make_ngmix_observation( wcs : galsim.BaseWCS Local WCS Jacobian at the object position. rng : numpy.random.RandomState - Random state for the noise realisations (seeded per tile for - reproducibility). + Random state for the noise realisations (seeded per object; see + :func:`position_seed`). bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. centroid_source : {"hsm", "wcs"}, optional diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 16403a543..345a5a43a 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -109,9 +109,12 @@ def ngmix_runner( # No batch saving save_batch = -1 - # First and last galaxy ID to process - id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") - id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") + # First and last galaxy ID to process. Read via ``getexpanded`` so an + # orchestrator can drive the chunk bounds from environment variables + # (``$SP_NGMIX_ID_OBJ_MIN`` and friends); ``getexpanded`` is the only + # accessor in ShapePipe's config that expands ``$VAR``. + id_obj_min = int(config.getexpanded(module_config_sec, "ID_OBJ_MIN")) + id_obj_max = int(config.getexpanded(module_config_sec, "ID_OBJ_MAX")) # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) @@ -139,17 +142,6 @@ def ngmix_runner( else: dilate_neighbour = 1 - # Seed the per-object RNG from sky position instead of per tile, so - # metacal's fixnoise counter-noise (and the fit guesses) cancel across - # Pujol image-simulation shear branches (ngmix#796). Default False leaves - # the production path byte-identical. - if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - seed_from_position = config.getboolean( - module_config_sec, "SEED_FROM_POSITION" - ) - else: - seed_from_position = False - # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -204,7 +196,6 @@ def ngmix_runner( blend_handling=blend_handling, seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, - seed_from_position=seed_from_position, metacal_psf=metacal_psf, ) diff --git a/src/shapepipe/modules/random_cat_package/__init__.py b/src/shapepipe/modules/random_cat_package/__init__.py deleted file mode 100644 index 32c6f64b2..000000000 --- a/src/shapepipe/modules/random_cat_package/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""RANDOM CATALOGUE PACKAGE. - -This package contains the module for ``random_cat``. - -:Author: Martin Kilbinger - -:Parent module: None - -:Input: Images and masks - -:Output: Random catalogue FITS file - -Description -=========== - -This module creates a random catalogue, and computes the tile area accounting -for overlapping and masked regions. - -Module-specific config file entries -=================================== - -N_RANDOM : float - The number of random objects requested on output -DENSITY : bool, optional - Option to interpret the number of random objects per square degree; the - default is ``False`` -SAVE_MASK_AS_HEALPIX : bool - Output healpix mask if ``True`` -HEALPIX_OUT_FILE_BASE : str, optional - Output halpix mask file base name; used only if SAVE_MASK_AS_HEALPIX is - ``True`` -HEALPIX_OUT_NSIDE : int, optional - Output healpix mask nside; used only if SAVE_MASK_AS_HEALPIX is ``True`` - -""" - -__all__ = ["random_cat.py"] diff --git a/src/shapepipe/modules/random_cat_package/random_cat.py b/src/shapepipe/modules/random_cat_package/random_cat.py deleted file mode 100644 index cabbd18e0..000000000 --- a/src/shapepipe/modules/random_cat_package/random_cat.py +++ /dev/null @@ -1,235 +0,0 @@ -"""RANDOM CATALOGUE. - -This module contains a class to create a random catalogue, and to compute -the tile area accounting for overlapping and masked regions. - -:Author: Martin Kilbinger - -""" - -import os -import re - -import numpy as np - -import astropy.io.fits as fits -from astropy import wcs -from astropy.table import Table - -from reproject import reproject_to_healpix - -from shapepipe.pipeline import file_io -from shapepipe.utilities import cfis - - -class RandomCat: - """Random Catalogue. - - This class creates a random catalogue given a mask FITS file. - - Parameters - ---------- - input_image_path : str - Path to input image file - input_mask_path : str - Path to input mask file - output_dir : str - Output directory - file_number_pattern : str - ShapePipe image ID string - output_file_pattern : str - Output file pattern (base name) for random catalogue - n_rand : float - Number of random objects on output - density : bool - ``n_rand`` is interpreted per square degrees if ``True`` - w_log : logging.Logger - Logging instance - healpix_options : dict - Parameters for HEALPix output mask file - """ - - def __init__( - self, - input_image_path, - input_mask_path, - output_dir, - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ): - - self._input_image_path = input_image_path - self._input_mask_path = input_mask_path - self._output_dir = output_dir - self._file_number_string = file_number_string - self._output_file_pattern = output_file_pattern - self._n_rand = n_rand - self._density = density - self._w_log = w_log - self._healpix_options = healpix_options - - def save_as_healpix(self, hdu_mask, header): - """Save As Healpix. - - Save mask as healpix FITS file. - - Parameters - ---------- - hdu_mask : class HDUList - HDU with 2D pixel mask image - header : class Header - Image header with WCS information - - """ - if not self._healpix_options: - return - - mask_1d, footprint = reproject_to_healpix( - (hdu_mask, header), - 'galactic', - nside=self._healpix_options['OUT_NSIDE'] - ) - - t = Table() - t['flux'] = mask_1d - t.meta['ORDERING'] = 'RING' - t.meta['COORDSYS'] = 'G' - t.meta['NSIDE'] = self._healpix_options['OUT_NSIDE'] - t.meta['INDXSCHM'] = 'IMPLICIT' - - output_path = ( - f'{output_dir}/{self._healpix_options["FILE_BASE"]}-' - + f'{file_number_string}.fits' - ) - t.write(output_path) - - def process(self): - """Process. - - Main function to identify exposures. - - """ - # Read image FITS file header - try: - img = fits.open(self._input_image_path) - header = img[0].header - except (OSError, IOError) as error: - # FITS file might contain only header. - # Try as ascii file - try: - fin = open(self._input_image_path) - header = fits.Header.fromtextfile(fin) - fin.close() - except Exception: - raise - - # Get WCS - WCS = wcs.WCS(header) - - # Read mask FITS file - hdu_mask = fits.open(self._input_mask_path) - mask = hdu_mask[0].data - - # Save mask in healpix format (if option is set) - self._save_as_healpix(hdu_mask, header) - - # Number of pixels - n_pix_x = mask.data.shape[0] - n_pix_y = mask.data.shape[1] - n_pix = n_pix_x * n_pix_y - - # Number of non-masked pixels - n_unmasked = len(np.where(mask == 0)[0]) - - # Compute various areas - - # Pixel area in deg^2 - area_pix = wcs.utils.proj_plane_pixel_area(WCS) - - # Tile area - area_deg2 = area_pix * n_pix - - # Area of unmasked region - area_deg2_eff = area_pix * n_unmasked - - # Compute number of requested objects - if n_unmasked > 0: - if not self._density: - # Use value from config file - n_obj = self._n_rand - else: - # Compute number of objects from density - n_obj = int( - self._n_rand / area_deg2 * area_deg2_eff / area_deg2 - ) - - # Check that a reasonably large number of pixels is not masked - if n_unmasked < n_obj: - raise ValueError( - f"Number of un-masked pixels {n_unmasked} is smaller " - + f"than number of random objects requested {n_obj}" - ) - - else: - n_obj = 0 - - self._w_log.info(f"Creating {n_obj} random objects") - - # Draw points until n are in mask - n_found = 0 - xy_rand = [] - while n_found < n_obj: - idx_x = np.random.randint(n_pix_x) - idx_y = np.random.randint(n_pix_y) - - # Add points with additional random sub-pixel value - if mask[idx_x, idx_y] == 0: - d = np.random.random(2) - # MKDEBUG: the following seems to work, x and y interchanged - xy_rand.append([idx_y + d[1], idx_x + d[0]]) - n_found = n_found + 1 - xy_rand = np.array(xy_rand) - - # Transform to WCS - res = WCS.all_pix2world(xy_rand, 1) - if n_unmasked > 0: - ra_rand = res[:, 0] - dec_rand = res[:, 1] - x_rand = xy_rand[:, 0] - y_rand = xy_rand[:, 1] - else: - ra_rand = [] - dec_rand = [] - x_rand = [] - y_rand = [] - - # Tile ID - output_path = ( - f"{self._output_dir}/{self._output_file_pattern}-" - + f"{self._file_number_string}.fits" - ) - file_base = os.path.splitext(file_name)[0] - tile_ID_str = re.split("-", file_base)[1:] - tile_id = float(".".join(tile_ID_str)) - tile_id_array = np.ones(n_obj) * tile_id - - # Write to output - cat_out = [ra_rand, dec_rand, x_rand, y_rand, tile_id_array] - column_names = ["RA", "DEC", "x", "y", "TILE_ID"] - - # TODO: Add units to header - output = file_io.FITSCatalogue( - output_path, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite - ) - output.save_as_fits(cat_out, names=column_names) - - # Write area information to log file - self._w_log.info(f"Total area = {area_deg2:.4f} deg^2") - self._w_log.info(f"Unmasked area = {area_deg2_eff:.4f} deg^2") - self._w_log.info( - f"Ratio masked to total pixels = {n_unmasked / n_pix:.3f}" - ) diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py deleted file mode 100644 index c85854cb5..000000000 --- a/src/shapepipe/modules/random_cat_runner.py +++ /dev/null @@ -1,79 +0,0 @@ -"""RANDOM CAT RUNNER. - -Module runner for ``random_cat``. - -:Author: Martin Kilbinger - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.random_cat_package.random_cat import RandomCat - - -@module_runner( - version="1.1", - file_pattern=["image", "pipeline_flag"], - file_ext=[".fits", "fits"], - depends=["astropy"], - numbering_scheme="_0", -) -def random_cat_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Random Catalogue Runner.""" - # Get input file names of image and mask - input_image_name = input_file_list[0] - input_mask_name = input_file_list[1] - - # Set output file name - if config.has_option(module_config_sec, "OUTPUT_FILE_PATTERN"): - output_file_pattern = config.get( - module_config_sec, "OUTPUT_FILE_PATTERN" - ) - else: - output_file_pattern = "random_cat" - - # Get number of random objects requested on output - n_rand = config.getfloat(module_config_sec, "N_RANDOM") - - # Flag whether n_rand is total (DENSITY=False, default) - # or per square degree (DENSITY=True) - if config.has_option(module_config_sec, "DENSITY"): - density = config.getboolean(module_config_sec, "DENSITY") - else: - density = False - - # Get healpix output options - save_mask_as_healpix = config.getboolean( - module_config_sec, "SAVE_MASK_AS_HEALPIX" - ) - if save_mask_as_healpix: - healpix_options = {} - for option_trunc in ['FILE_BASE', 'OUT_NSIDE']: - option = f'HEALPIX_OUT_{option_trunc}' - healpix_options[option_trunc] = config.get( - module_config_sec, option - ) - # Create rand cat class instance - rand_cat_inst = RandomCat( - input_image_name, - input_mask_name, - run_dirs["output"], - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ) - - # Run processing - rand_cat_inst.process() - - # No return objects - return None, None diff --git a/src/shapepipe/modules/setools_package/setools.py b/src/shapepipe/modules/setools_package/setools.py index c430b52b1..33d8efa8b 100644 --- a/src/shapepipe/modules/setools_package/setools.py +++ b/src/shapepipe/modules/setools_package/setools.py @@ -658,13 +658,18 @@ def _make_rand_split(self): cat_size = len(np.where(mask)[0]) n_keep = int(np.ceil(cat_size * ratio)) - mask_ratio = [] - mask_left = list(range(0, cat_size)) - while len(mask_ratio) != n_keep: - idx = np.random.randint(0, len(mask_left)) - mask_ratio.append(mask_left.pop(idx)) - mask_ratio = np.array(mask_ratio) - mask_left = np.array(mask_left) + # Deterministic split, seeded from the unit's file number: the + # train/validation assignment is a pure function of the input + # catalogue, so the PSF star sample (and everything downstream + # of the PSF model) is reproducible run-to-run. An unseeded + # np.random here made the shear catalogue non-reproducible + # upstream of ngmix's own position seeding. + seed = int( + re.sub(r"\D", "", self._file_number_string) or 0 + ) % (2 ** 32) + perm = np.random.RandomState(seed).permutation(cat_size) + mask_ratio = perm[:n_keep] + mask_left = np.sort(perm[n_keep:]) self.rand_split[key]["mask"] = mask self.rand_split[key][f"ratio_{int(ratio * 100)}"] = mask_ratio self.rand_split[key][f"ratio_{100 - int(ratio * 100)}"] = mask_left diff --git a/src/shapepipe/modules/sextractor_package/__init__.py b/src/shapepipe/modules/sextractor_package/__init__.py index abb0b4572..40f40ca91 100644 --- a/src/shapepipe/modules/sextractor_package/__init__.py +++ b/src/shapepipe/modules/sextractor_package/__init__.py @@ -4,10 +4,10 @@ :Author: Axel Guinot -:Parent modules: ``mask_runner``, ``merge_headers_runner`` (the latter only +:Parent modules: ``split_exp_runner``, ``merge_headers_runner`` (the latter only when ``MAKE_POST_PROCESS`` is ``True``) -:Input: Single-exposure single-CCD image, weight and flag files +:Input: Single-exposure single-CCD image, weight and instrument flag files :Output: SExtractor output catalogue diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index c36a8596d..e45885b06 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -19,7 +19,7 @@ # first three entries only. @module_runner( version="1.0.1", - input_module=["mask_runner", "merge_headers_runner"], + input_module=["split_exp_runner", "merge_headers_runner"], file_pattern=["image", "weight", "flag", "log_exp_headers"], file_ext=[".fits", ".fits", ".fits", ".sqlite"], executes=["source-extractor"], diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 962953ede..fee0dad3e 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -84,7 +84,7 @@ def _set_run_name(self): Set the name of the current pipeline run. """ - self._run_name = self.config.get("DEFAULT", "RUN_NAME") + self._run_name = self.config.getexpanded("DEFAULT", "RUN_NAME") if self.config.getboolean("DEFAULT", "RUN_DATETIME"): self._run_name += datetime.now().strftime("_%Y-%m-%d_%H-%M-%S") diff --git a/src/shapepipe/utilities/__init__.py b/src/shapepipe/utilities/__init__.py index 5f0eab2d6..42e234003 100644 --- a/src/shapepipe/utilities/__init__.py +++ b/src/shapepipe/utilities/__init__.py @@ -7,4 +7,4 @@ """ -__all__ = ["file_system", "cfis", "galaxy", "summary"] +__all__ = ["file_system", "cfis", "galaxy", "mask_query", "summary"] diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py new file mode 100644 index 000000000..982bfd3d3 --- /dev/null +++ b/src/shapepipe/utilities/mask_query.py @@ -0,0 +1,294 @@ +"""MASK QUERY. + +The one healsparse lookup in ShapePipe. + +ShapePipe does not generate or rasterize masks. Sky-fixed masks are supplied +as healsparse maps and are consumed by *querying them at object positions*: +every object gets its mask value(s) as catalogue columns, and rejection happens +at the catalogue level, never at the pixel level. The only mask that still +reaches pixels is the per-exposure instrument flag image delivered with it. + +Two callers share the primitive defined here: + +* ``make_cat`` writes one ``MASK_`` column per configured map, carrying + the map value verbatim (no interpretation, no filtering); +* ``mask_query`` writes a single integer ``MASK_EXT`` column onto the exposure + SExtractor catalogue, combining the configured maps into "clean (0) or + flagged (nonzero)" so that ``setools`` — whose expression language has no + bitwise operators — *could* cut on ``MASK_EXT == 0``. The shipped selection + does not; the column is carried for measurement (see that module). + +Partial reads +------------- +Never read a whole map. The UNIONS products are ~550 MB each and ``mask_query`` +runs PER CCD, so a full read would cost ~22 GB of I/O per 40-CCD exposure to +answer questions about a 0.06 deg² footprint. Instead the coverage index is +read first (``HealSparseCoverage``, a few hundred kB), the coverage pixels the +catalogue actually touches are computed with ``hpgeom.angle_to_pixel`` at the +map's ``nside_coverage``, and only those get loaded. + +Every queried position falls inside exactly one coverage pixel and all of them +are requested, so the padding by ``hpgeom.neighbors`` is insurance rather +than necessity — at ``nside_coverage=128`` a coverage pixel of the star map +is ~23 kB, so the padding costs under a megabyte and buys immunity to any +edge convention we did not think of. ``test_partial_read_matches_full`` is +what actually holds the two paths equal. + +Measured on the DR6 star-body map — the bit-2 rung of the ladder the configs +name ``mask_ugriz_nside131072_n4.hsp``, run against the staged single-band copy +``mask_r_nside131072_n4.hsp``, 583 MB, ``nside_coverage=128`` — for 2000 +positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB +peak RSS, full 12.8 s / 3364 MiB, identical values. Per 40-CCD exposure that +is ~4 s against ~8.5 min of map reading, and the query adds ~0.15 GB to a rule +already asking for 16 GB. + +Coverage +-------- +``healsparse.HealSparseMap.get_values_pos`` returns a map's *sentinel* for +positions outside its coverage: ``False`` for boolean maps, and typically +``-1`` for integer maps. ``make_cat`` passes that sentinel through verbatim, +which is the documented off-map flag for the final catalogue. + +Coverage is reported from the COVERAGE MASK, not ``valid_mask=True``. For a +boolean map — which is what the UNIONS per-bit products are — healsparse +stores only the ``True`` pixels, so ``valid_mask`` returns the value itself +and cannot tell "inside the footprint and clean" from "outside it entirely". +The coverage mask can, at ``nside_coverage`` resolution, which is the scale the +question is asked at anyway: does this map reach this exposure at all? + +``flag_positions`` treats off-coverage as **not flagged**, for both map kinds. +This makes the integer case agree with the boolean case (whose sentinel is +literally ``False``) rather than diverge from it, and it keeps a map whose +coverage does not reach an exposure from silently rejecting every star on it. +That is also why the all-off-coverage case is logged as a WARNING: it is +indistinguishable from "nothing is masked here" in the output column, so it has +to be distinguishable in the log. + +:Author: Claude Fable 5, for PR #847 + +""" + +import numpy as np + + +def parse_map_paths(paths_str): + """Parse Map Paths. + + Parse a comma-separated list of healsparse map paths. + + Parameters + ---------- + paths_str : str + Comma-separated map paths, e.g. ``/a/star.hsp, /b/maximask.hsp`` + + Returns + ------- + list + Map paths, stripped of surrounding whitespace, empty entries dropped + + """ + return [path.strip() for path in paths_str.split(",") if path.strip()] + + +def _covering_pixels(coverage, ra, dec): + """Covering Pixels. + + The map's coverage pixels touched by these positions, padded by their + neighbours and intersected with what the map actually holds. + + Parameters + ---------- + coverage : healsparse.HealSparseCoverage + Coverage index of the map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Coverage pixel indices to load, possibly empty + + """ + import hpgeom + + nside_coverage = coverage.nside_coverage + touched = np.unique( + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ) + padded = np.unique( + np.concatenate( + [touched, hpgeom.neighbors(nside_coverage, touched).ravel()] + ) + ) + # neighbors() returns -1 for a non-existent neighbour. + padded = padded[padded >= 0] + + return padded[coverage.coverage_mask[padded]] + + +def query_map_coverage(path, ra, dec): + """Query Map With Coverage. + + Read only the part of a healsparse map these positions need, and return + both its value at each position and whether each position is inside the + map's coverage. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + tuple + ``(values, in_coverage)`` — the map value at each position (the map's + sentinel outside coverage) and a boolean array, both of length + ``len(ra)`` + + """ + import healsparse + import hpgeom + + ra = np.asarray(ra) + dec = np.asarray(dec) + + coverage = healsparse.HealSparseCoverage.read(path) + nside_coverage = coverage.nside_coverage + + in_coverage = coverage.coverage_mask[ + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ] + + pixels = _covering_pixels(coverage, ra, dec) + + if pixels.size == 0: + # healsparse raises when no requested pixel is in the coverage map, so + # the empty case is answered without asking it: load one arbitrary + # coverage pixel purely to learn the dtype and sentinel, and return + # that sentinel everywhere. Same answer, one small read. + covered = np.flatnonzero(coverage.coverage_mask) + if covered.size == 0: + raise ValueError(f"healsparse map {path} has empty coverage") + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(covered[0])] + ) + values = np.full(ra.size, mask_map.sentinel, dtype=mask_map.dtype) + return values, in_coverage + + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(pixel) for pixel in pixels] + ) + values = np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + + return values, in_coverage + + +def query_map(path, ra, dec): + """Query Map. + + Return a healsparse map's value at each world position, reading only the + coverage pixels those positions touch. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Map value at each position; positions outside the map's coverage carry + the map's sentinel value + + """ + values, _ = query_map_coverage(path, ra, dec) + + return values + + +def flag_positions(paths, ra, dec, bits=None, w_log=None): + """Flag Positions. + + Combine one or more healsparse masks into a single per-object integer flag. + + Each map contributes at each position: + + * boolean map: ``1`` where the map is ``True``, ``0`` elsewhere; + * integer map: the map value, optionally restricted to ``bits`` + (``value & bits``); ``0`` where the value is zero or the position is + outside coverage. + + Contributions are combined with a bitwise OR, so the returned flag is zero + for a clean object and carries the union of the bits that fired otherwise. + + Parameters + ---------- + paths : list + Paths to the healsparse maps to query + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + bits : int, optional + Bit mask applied to integer maps; default ``None`` means any nonzero + value flags + w_log : logging.Logger, optional + Logging instance + + Returns + ------- + numpy.ndarray + Integer flag per object, ``0`` for a clean object + + """ + ra = np.asarray(ra) + dec = np.asarray(dec) + flag = np.zeros(ra.size, dtype=np.int64) + + if ra.size == 0: + return flag + + for path in paths: + values, in_coverage = query_map_coverage(path, ra, dec) + + if values.dtype == bool: + contribution = values.astype(np.int64) + else: + integer = values.astype(np.int64) + # Off-coverage: the sentinel, negative by healsparse convention. + # Zeroed rather than OR-ed in, see this module's docstring. + integer = np.where(integer < 0, 0, integer) + if bits is not None: + integer &= bits + contribution = integer + + flag |= contribution + + n_off = int(np.count_nonzero(~in_coverage)) + if w_log is not None: + w_log.info( + f"Mask query {path}: " + f"{int(np.count_nonzero(contribution))}/{ra.size} objects " + f"flagged, {n_off} outside coverage" + ) + if n_off == ra.size: + # Every object reads the sentinel, so the column is all-zero + # and looks exactly like "nothing is masked here". Say it. + w_log.warning( + f"Mask query {path}: NO object is inside this map's" + + " coverage — the resulting MASK_EXT contribution is" + + " zero everywhere because the map does not reach these" + + " positions, not because they are clean." + ) + + return flag diff --git a/src/shapepipe/utilities/summary_params_pre_v2.py b/src/shapepipe/utilities/summary_params_pre_v2.py index 2def4d33b..ea67442e3 100644 --- a/src/shapepipe/utilities/summary_params_pre_v2.py +++ b/src/shapepipe/utilities/summary_params_pre_v2.py @@ -228,7 +228,7 @@ def set_jobs_v2_pre_v2(patch, verbose): jobs["256"] = summary.job_data( "256", - "run_sp_Ms", + "run_sp_tile_Ms", ["merge_sep_cats_runner"] * 2, "tile_IDs", path_main=path_main, @@ -241,7 +241,7 @@ def set_jobs_v2_pre_v2(patch, verbose): jobs["512"] = summary.job_data( "512", - ["run_sp_Mc"], + ["run_sp_tile_Mc"], ["make_cat_runner"], "tile_IDs", path_main=path_main, diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py deleted file mode 100644 index 1f4fd3472..000000000 --- a/src/shapepipe/utilities/vizier.py +++ /dev/null @@ -1,93 +0,0 @@ -"""VIZIER QUERY UTILITY. - -Consolidated Vizier query helper used by the mask module (to fetch the -reference star catalogue) and by ``scripts/python/create_star_cat.py``. -Retries over a list of mirror servers at progressively longer timeouts. - -:Author: Martin Kilbinger - -""" - -import random -import time - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astroquery.vizier import Vizier - - -VIZIER_SERVERS = [ - "vizier.cds.unistra.fr", - "vizier.cfa.harvard.edu", - "vizier.iucaa.in", -] - -VIZIER_TIMEOUTS = [10, 20, 40] - - -def query_vizier(ra, dec, radius_arcmin, cat_id): - """Query a Vizier catalogue with retries over timeouts and mirror servers. - - Parameters - ---------- - ra : float - Right ascension in degrees. - dec : float - Declination in degrees. - radius_arcmin : float - Cone-search radius in arcminutes. - cat_id : str - Vizier catalogue identifier (e.g. ``"I/305/out"`` for GSC 2.3). - - Returns - ------- - astropy.table.Table - First result table returned by Vizier. - - Raises - ------ - IndexError - If all server/timeout combinations return an empty result. - - """ - # Empirically, single-precision input positions can cause Vizier to return - # empty lists for some exposures; force double precision. - p = np.array([ra, dec], dtype="double") - coord = SkyCoord(ra=p[0] * u.deg, dec=p[1] * u.deg, frame="icrs") - - # Stagger concurrent queries to avoid hammering a single mirror. - time.sleep(random.uniform(0, 5)) - - for attempt, timeout in enumerate(VIZIER_TIMEOUTS): - for server in VIZIER_SERVERS: - v = Vizier( - row_limit=-1, timeout=timeout, vizier_server=server - ) - result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id - ) - if len(result) > 0: - print( - f"Vizier query successful " - f"(server={server}, timeout={timeout}s)" - ) - return result[0] - print( - f"Vizier returned empty list at {coord}, " - f"{radius_arcmin:.2f} arcmin, " - f"server={server}, timeout={timeout}s" - ) - if attempt < len(VIZIER_TIMEOUTS) - 1: - wait = 10 * 2**attempt - print( - f"All servers failed, retrying in {wait}s " - f"(attempt {attempt + 1}/{len(VIZIER_TIMEOUTS)}, " - f"next timeout={VIZIER_TIMEOUTS[attempt + 1]}s)" - ) - time.sleep(wait) - - raise IndexError( - f"Vizier astroquery returned empty list at {coord}, " - f"radius={radius_arcmin} arcmin, catalog={cat_id}" - ) diff --git a/tests/README.md b/tests/README.md index 82bd066d7..0b0df1524 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,6 +30,7 @@ candide needs the candide cluster and/or its real data; auto-skipped elsewhere ``` `--strict-markers` is on, so a typo'd marker is an error, not a silent no-op. +Use `pytest -m "not unions"` to run the survey-generic tests. A `candide`-marked test is **collected everywhere** (so `--collect-only` shows it exists) but **skipped off-cluster** with a clear reason. Candide is detected diff --git a/tests/helpers/artifacts.py b/tests/helpers/artifacts.py index 25cbbc8ff..d96d02242 100644 --- a/tests/helpers/artifacts.py +++ b/tests/helpers/artifacts.py @@ -297,7 +297,7 @@ def _m_cell(r): f"{rows}\n" f"As the galaxy shrinks toward the PSF, |m| grows negative (shear washes " f"out) and R11 dips through a ~0.92 minimum then rises toward the " - f"point-source floor. Only resolved rungs (ratio >= " + f"point-source limit. Only resolved rungs (ratio >= " f"{summary['resolved_ratio_threshold']}) are asserted.\n" ) diff --git a/tests/module/test_make_cat_mask_ext.py b/tests/module/test_make_cat_mask_ext.py new file mode 100644 index 000000000..91d0f5154 --- /dev/null +++ b/tests/module/test_make_cat_mask_ext.py @@ -0,0 +1,129 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MAKE_CAT external-mask columns. + +Exercises the optional per-band external healsparse mask lookup added to +``make_cat`` (PR #847 §4, the ShapePipe end of UNIONS-WL/spherex#38). A small +synthetic ``final_cat`` FITS carrying known ``XWIN_WORLD`` / ``YWIN_WORLD`` +object positions is queried against synthetic healsparse maps of known value, +locking in: (1) the ``MASK_`` column name and per-object values, (2) the +off-map sentinel (``-1`` for integer maps) written verbatim for objects outside +coverage, (3) multi-band handling, and (4) that absent config leaves the +catalogue untouched. +""" + +import numpy as np +import numpy.testing as npt +import pytest + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.make_cat_package import make_cat +from shapepipe.pipeline import file_io + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Object world positions (RA, Dec in degrees). The last object sits far from +# the map coverage so it exercises the off-map sentinel path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +def _make_map(value, dtype=np.int16, sentinel=-1): + """Build a healsparse map covering the first three RA/Dec positions. + + All covered pixels carry ``value``; everything else reads the sentinel. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=sentinel + ) + smap.update_values_pos( + RA[:3], DEC[:3], np.full(3, value, dtype=dtype), lonlat=True + ) + return smap + + +def _write_final_cat(path): + """Write a synthetic final_cat FITS with a RESULTS ext of known positions.""" + data = np.empty( + len(RA), + dtype=[ + ("NUMBER", "i4"), + ("XWIN_WORLD", "f8"), + ("YWIN_WORLD", "f8"), + ], + ) + data["NUMBER"] = np.arange(len(RA)) + data["XWIN_WORLD"] = RA + data["YWIN_WORLD"] = DEC + + cat = file_io.FITSCatalogue( + str(path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + cat.save_as_fits(data, ext_name="RESULTS") + return cat + + +def test_parse_mask_ext_paths(): + """band:path pairs parse into a stripped mapping, whitespace-tolerant.""" + parsed = make_cat.parse_mask_ext_paths( + "u:/a/mask_u.hsp, g:/b/mask_g.hsp,r:/c/mask_r.hsp" + ) + assert parsed == { + "u": "/a/mask_u.hsp", + "g": "/b/mask_g.hsp", + "r": "/c/mask_r.hsp", + } + + +def test_mask_ext_columns(tmp_path): + """Per-band columns carry the map value on-map and the sentinel off-map.""" + u_map = _make_map(16) + g_map = _make_map(32) + u_path = tmp_path / "mask_u.hsp" + g_path = tmp_path / "mask_g.hsp" + u_map.write(str(u_path)) + g_map.write(str(g_path)) + + cat_path = tmp_path / "final_cat-000.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue( + str(cat_path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + make_cat.save_mask_ext_data( + cat, + {"u": str(u_path), "g": str(g_path)}, + _NullLogger(), + ) + + cat.open() + data = cat.get_data() + # On-map objects (first three) carry the map value; the off-map object + # (last) carries the map's -1 sentinel. + npt.assert_array_equal(data["MASK_u"], [16, 16, 16, -1]) + npt.assert_array_equal(data["MASK_g"], [32, 32, 32, -1]) + # Integer dtype preserved from the map. + assert np.issubdtype(data["MASK_u"].dtype, np.integer) + cat.close() + + +def test_mask_ext_absent_is_noop(tmp_path): + """Not calling the lookup leaves the catalogue columns unchanged.""" + cat_path = tmp_path / "final_cat-001.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue(str(cat_path)) + cat.open() + cols = set(cat.get_data().dtype.names) + cat.close() + + assert cols == {"NUMBER", "XWIN_WORLD", "YWIN_WORLD"} + assert not any(name.startswith("MASK_") for name in cols) diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py new file mode 100644 index 000000000..f52d2e669 --- /dev/null +++ b/tests/module/test_mask_query.py @@ -0,0 +1,348 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MASK_QUERY. + +Exercises the exposure-side half of the query-everything mask design (PR #847): +``mask_query`` reads each SExtractor detection's windowed world position out of +the ``LDAC_OBJECTS`` extension, looks it up in the configured healsparse maps, +and writes a single integer ``MASK_EXT`` column into a NEW catalogue beside the +input. + +What is locked in here: (1) boolean maps flag with ``1``, (2) integer maps +contribute their value and ``MASK_BITS`` restricts which bits do, (3) the +off-coverage sentinel (``-1``) never flags — the one place this differs from +``make_cat``'s verbatim pass-through, argued in +:mod:`shapepipe.utilities.mask_query` — (4) several maps OR together, and +(5) the input catalogue is left untouched while the LDAC structure survives. +""" + +import pathlib + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Detection world positions (RA, Dec in degrees). The last one sits far from +# every map's coverage, so it exercises the off-coverage path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +class _NullLogger: + """Captures what the module logs, so coverage warnings can be asserted.""" + + def __init__(self): + self.info_msgs = [] + self.warning_msgs = [] + + def info(self, msg, *_a, **_kw): + self.info_msgs.append(str(msg)) + + def warning(self, msg, *_a, **_kw): + self.warning_msgs.append(str(msg)) + + +def _write_map(path, value, dtype=np.int16, n_covered=2): + """Build an integer map carrying ``value`` at the first ``n_covered``. + + Everything else reads the ``-1`` sentinel, i.e. off coverage. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=-1 + ) + if n_covered: + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.full(n_covered, value, dtype=dtype), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_bool_map(path, n_covered=2): + """Build a boolean map, ``True`` at the first ``n_covered``.""" + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, np.bool_ + ) + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.ones(n_covered, dtype=np.bool_), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_sexcat(path, n=None): + """Write a synthetic LDAC SExtractor catalogue of known positions. + + Written with astropy rather than ``FITSCatalogue.save_as_fits``, because + creating an LDAC catalogue through that API requires an existing LDAC file + to copy the ``LDAC_IMHEAD`` HDU from. The three-HDU layout below is what + SExtractor writes and what ``SEx_catalogue=True`` (``hdu_no=2``) indexes. + """ + n = len(RA) if n is None else n + imhead = fits.BinTableHDU.from_columns( + [ + fits.Column( + name="Field Header Card", format="1A", array=np.array(["x"]) + ) + ], + name="LDAC_IMHEAD", + ) + objects = fits.BinTableHDU.from_columns( + [ + fits.Column(name="NUMBER", format="J", array=np.arange(n)), + fits.Column(name="XWIN_WORLD", format="D", array=RA[:n]), + fits.Column(name="YWIN_WORLD", format="D", array=DEC[:n]), + fits.Column( + name="IMAFLAGS_ISO", format="J", array=np.zeros(n, dtype="i4") + ), + ], + name="LDAC_OBJECTS", + ) + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto( + str(path), overwrite=True + ) + return str(path) + + +def _read(path): + cat = file_io.FITSCatalogue(str(path), SEx_catalogue=True) + cat.open() + data = cat.get_data() + flag = np.copy(data["MASK_EXT"]) + names = set(data.dtype.names) + cat.close() + return flag, names + + +def test_parse_map_paths(): + """Comma-separated paths parse, whitespace-tolerant, empties dropped.""" + assert mask_query_util.parse_map_paths( + " /a/star.hsp, /b/maximask.hsp ,, " + ) == ["/a/star.hsp", "/b/maximask.hsp"] + + +def test_flag_positions_integer_map(tmp_path): + """An integer map contributes its value; off-coverage stays clean.""" + path = _write_map(tmp_path / "m.hsp", 4, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [4, 4, 4, 0] + ) + + +def test_flag_positions_bits_restrict(tmp_path): + """MASK_BITS selects which bits of an integer map flag.""" + path = _write_map(tmp_path / "m.hsp", 1028, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=4), [4, 4, 4, 0] + ) + # A bit the map does not carry leaves everything clean. + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=2), [0, 0, 0, 0] + ) + + +def test_flag_positions_ors_maps(tmp_path): + """Several maps combine with a bitwise OR.""" + a = _write_map(tmp_path / "a.hsp", 4, n_covered=1) + b = _write_map(tmp_path / "b.hsp", 1024, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([a, b], RA, DEC), [1028, 1024, 1024, 0] + ) + + +def test_flag_positions_boolean_map(tmp_path): + """A boolean map flags with 1; its False sentinel stays clean.""" + path = _write_bool_map(tmp_path / "bool.hsp", n_covered=2) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [1, 1, 0, 0] + ) + + +def test_mask_query_writes_flag_ext(tmp_path): + """The module writes MASK_EXT into a new catalogue and counts the hits.""" + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-0.fits") + out_path = tmp_path / "sexcat_ext-000-0.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=_NullLogger() + ).process() + + assert n_flagged == 2 + flag, names = _read(out_path) + npt.assert_array_equal(flag, [4, 4, 0, 0]) + assert np.issubdtype(flag.dtype, np.integer) + # The columns SExtractor wrote survive alongside the new one. + assert {"NUMBER", "XWIN_WORLD", "YWIN_WORLD", "IMAFLAGS_ISO"} <= names + # The LDAC structure survives: setools and psfex read HDU 2 by index. + with fits.open(str(out_path)) as hdus: + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + + # The input is not mutated: this module publishes a new file. + with fits.open(in_path) as hdus: + assert "MASK_EXT" not in hdus[2].data.dtype.names + + +def test_mask_query_bits_and_all_clean(tmp_path): + """MASK_BITS reaches the module, and a miss leaves every object clean.""" + map_path = _write_map(tmp_path / "m.hsp", 1024, n_covered=3) + in_path = _write_sexcat(tmp_path / "sexcat-000-1.fits") + out_path = tmp_path / "sexcat_ext-000-1.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], bits=4, w_log=_NullLogger() + ).process() + + assert n_flagged == 0 + flag, _ = _read(out_path) + npt.assert_array_equal(flag, [0, 0, 0, 0]) + + +def test_partial_read_matches_full(tmp_path): + """The partial read returns exactly what a full read of the map returns. + + This is the guarantee that makes the optimisation safe: query_map loads + only the coverage pixels the positions touch, and must be indistinguishable + from HealSparseMap.read(path) at every queried position. + """ + for dtype, writer in ((np.int16, _write_map), (np.bool_, None)): + path = ( + _write_map(tmp_path / f"full_{dtype.__name__}.hsp", 7, n_covered=3) + if writer is not None + else _write_bool_map(tmp_path / "full_bool.hsp", n_covered=3) + ) + full = healsparse.HealSparseMap.read(path) + expected = np.asarray(full.get_values_pos(RA, DEC, lonlat=True)) + npt.assert_array_equal( + mask_query_util.query_map(path, RA, DEC), expected + ) + + +def test_coverage_reported_for_both_map_kinds(tmp_path): + """in_coverage is the coverage mask, not valid_mask. + + For a boolean map healsparse stores only the True pixels, so valid_mask + would equal the value and could not distinguish an unmasked object from one + the map does not reach. The distant object must read as off-coverage while + the near, unflagged ones read as covered. + """ + for path in ( + _write_map(tmp_path / "int.hsp", 4, n_covered=2), + _write_bool_map(tmp_path / "bool.hsp", n_covered=2), + ): + _, in_coverage = mask_query_util.query_map_coverage(path, RA, DEC) + # The 4th position is 190 deg away; the first two are on the map. + assert in_coverage[0] and in_coverage[1] + assert not in_coverage[3] + + +def test_all_off_coverage_warns(tmp_path): + """A map that reaches nothing warns, instead of logging a silent zero.""" + path = _write_map(tmp_path / "elsewhere.hsp", 4, n_covered=2) + far_ra = np.array([200.0, 201.0]) + far_dec = np.array([-40.0, -41.0]) + log = _NullLogger() + + flag = mask_query_util.flag_positions([path], far_ra, far_dec, w_log=log) + + npt.assert_array_equal(flag, [0, 0]) + assert any("NO object is inside" in m for m in log.warning_msgs) + assert any("2 outside coverage" in m for m in log.info_msgs) + # A map that DOES reach the objects must not warn. + log2 = _NullLogger() + mask_query_util.flag_positions([path], RA, DEC, w_log=log2) + assert not log2.warning_msgs + + +def test_flag_positions_empty_input(tmp_path): + """Zero positions is not an error and reads no map.""" + flag = mask_query_util.flag_positions( + [str(tmp_path / "does-not-exist.hsp")], np.zeros(0), np.zeros(0) + ) + assert flag.shape == (0,) + + +def test_mask_query_empty_ccd(tmp_path): + """A CCD with no detections still publishes a catalogue, not an error. + + setools tolerates sparse-CCD attrition and psfex_interp's completeness + floor is 0/warn, so an empty sexcat must flow through rather than raise. + """ + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-2.fits", n=0) + out_path = tmp_path / "sexcat_ext-000-2.fits" + log = _NullLogger() + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=log + ).process() + + assert n_flagged == 0 + assert out_path.exists() + flag, names = _read(out_path) + assert flag.shape == (0,) + assert "MASK_EXT" in names + assert any("No detections" in m for m in log.info_msgs) + + +def test_empty_coverage_raises_clearly(tmp_path): + """A map with no coverage at all fails with a message naming the file. + + The probe read in query_map_coverage indexes the first covered pixel; with + nothing covered that would be an IndexError from deep inside the utility. + """ + path = _write_map(tmp_path / "empty.hsp", 4, n_covered=0) + with pytest.raises(ValueError): + mask_query_util.query_map_coverage(path, RA, DEC) + + +def test_mask_query_no_maps_is_noop(tmp_path): + """No MASK_PATHS means a strict pass-through, not a zero column. + + The shipped configs comment MASK_PATHS out, so this is the DEFAULT path. + mask_query stays in the MODULE chain either way — setools reads its output + — so the no-op has to publish a file, and that file must be identical to + its input with no MASK_EXT column at all (the gating make_cat gives + MASK_EXT_PATHS). + """ + in_path = _write_sexcat(tmp_path / "sexcat-000-3.fits") + out_path = tmp_path / "sexcat_ext-000-3.fits" + log = _NullLogger() + + n_flagged = MaskQuery(in_path, str(out_path), [], w_log=log).process() + + assert n_flagged == 0 + assert out_path.exists() + assert out_path.read_bytes() == pathlib.Path(in_path).read_bytes() + with fits.open(str(out_path)) as hdus: + assert "MASK_EXT" not in hdus[2].data.dtype.names + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + assert any("No MASK_PATHS configured" in m for m in log.info_msgs) + + +def test_flag_positions_no_maps(tmp_path): + """The utility with no paths returns an all-zero flag and reads nothing.""" + flag = mask_query_util.flag_positions([], RA, DEC) + npt.assert_array_equal(flag, [0, 0, 0, 0]) diff --git a/tests/module/test_psf_grammar_properties.py b/tests/module/test_psf_grammar_properties.py index b0ac1ee47..6bf9e4713 100644 --- a/tests/module/test_psf_grammar_properties.py +++ b/tests/module/test_psf_grammar_properties.py @@ -19,7 +19,7 @@ group is optional, so a regressed ``GAL`` token is rejected, not absorbed); (c) every ``NGMIX_*`` token the shipped param file - (``example/cfis/final_cat.param``) names is a column the writer can + (``workflow/config/cfis/final_cat.param``) names is a column the writer can produce — writer/param-file consistency; (d) the FULL frozen grammar (shapepipe#761) — ``ESTIMATOR_COMPONENT[_ERR]_ OBJECT[_metacaltype]`` — holds across both estimator families: both @@ -183,10 +183,10 @@ def _run_save_ngmix(ngmix_path, obj_ids, cat_size_target=None): # The shipped final-catalogue param files, two levels up from tests/module/. # Both are consumer contracts updated to the new grammar, so both are checked. -_EXAMPLE = Path(__file__).resolve().parents[2] / "example" +_ROOT = Path(__file__).resolve().parents[2] PARAM_PATHS = [ - _EXAMPLE / "cfis" / "final_cat.param", - _EXAMPLE / "unions_800" / "cat_matched.param", + _ROOT / "workflow" / "config" / "cfis" / "final_cat.param", + _ROOT / "example" / "unions_800" / "cat_matched.param", ] # The one param-file NGMIX token outside the _save_ngmix_data grammar: the @@ -357,7 +357,7 @@ def test_emitted_column_names_match_grammar(obj_ids, tmp_path_factory): def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): """Every NGMIX_* token the param file names is a column the writer produces. - Each shipped final-catalogue param file (``example/cfis/final_cat.param`` + Each shipped final-catalogue param file (``workflow/config/cfis/final_cat.param`` and ``example/unions_800/cat_matched.param``) is a consumer contract for the final catalogue; ``create_final_cat`` keeps only the listed columns, so a token it names that the writer cannot emit is a silent, empty column diff --git a/tests/science/test_additive_null.py b/tests/science/test_additive_null.py index 552738b55..c8440071b 100644 --- a/tests/science/test_additive_null.py +++ b/tests/science/test_additive_null.py @@ -14,7 +14,7 @@ m-bias path — the galaxy is round (``shear=(0, 0)``) and the PSF is sheared (``psf_shear=(0.05, 0)``). With the deconvolution wired correctly the true PSF is fed as the model, so the deconvolution is unbiased and the recovered ``c`` sits -at the ~few x 1e-5 noise floor per seed. A regression that breaks the +at the ~few x 1e-5 noise level per seed. A regression that breaks the deconvolution (deconvolves by a fitted model instead of the PSF image, drops it, or shears the galaxy rather than the PSF) leaks the 0.05 PSF ellipticity straight into ``c``, blowing past the tolerance in seconds. @@ -49,7 +49,7 @@ def test_additive_bias_consistent_with_zero(): A round galaxy through an ``e1 = 0.05`` PSF should yield ``c ~ 0`` once the deconvolution removes the PSF shape (the true PSF is fed as the model, so - the deconvolution is unbiased and only the noise floor remains). Asserts the + the deconvolution is unbiased and only the noise level remains). Asserts the ensemble-mean ``|c|`` below the twin's published ``C_TOL``; a deconvolution regression that leaks PSF ellipticity pushes ``c`` toward 0.05 and trips it. """ diff --git a/tests/science/test_resolution_ladder.py b/tests/science/test_resolution_ladder.py index 0e04f1d46..c1e0a6c2b 100644 --- a/tests/science/test_resolution_ladder.py +++ b/tests/science/test_resolution_ladder.py @@ -27,7 +27,7 @@ ``~+0.0004`` at ratio 1.2 through zero to ``~-0.012`` at ratio 0.15. The response ``R11`` does something less obvious: it does **not** slide monotonically. It dips through a shallow ~0.92 minimum at mid-resolution and then **rises** back -toward 1 as the object approaches the point-source floor (a true point source is +toward 1 as the object approaches the point-source limit (a true point source is pathological — the round Gaussian fit collapses and R stiffens). So the JSON carries the whole shape; the assertions pin only what a *resolved* survey galaxy must satisfy. @@ -35,7 +35,7 @@ The guardrail: on the **resolved** rungs (``ratio >= 0.5``) the response correction must leave ``|m| < 5e-3`` — the same few-x-1e-3 scale as ``test_mbias`` (observed max ``0.0036``). The unresolved rungs are recorded but -**not** asserted: they document where the estimator's floor lives, not a +**not** asserted: they document where the estimator's limit lives, not a requirement. A response mis-scale (wrong metacal step, R not applied, a constant factor on R) shifts ``m`` by a roughly constant offset across every rung — a +5% R error moves each resolved ``m`` by ~0.05, tripping the tripwire loudly (the diff --git a/tests/unit/test_clean_tile_prune.py b/tests/unit/test_clean_tile_prune.py new file mode 100644 index 000000000..d7f3e092d --- /dev/null +++ b/tests/unit/test_clean_tile_prune.py @@ -0,0 +1,170 @@ +"""``clean_tile`` deletes a tile's store without ever following a symlink out of it. + +``workflow/scripts/clean_tile.py`` reclaims a finished tile by deleting its whole +scratch directory bar four survivors. A finished tile holds NINE symlinks in two +classes, and the second is why this module exists: + + * ``exp_forest///output`` — 7 links into the exposure stores, each + shared with 7-10 other tiles; + * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` — + 2 links into ``/project/def-mjhudson/unions-wl/tiles``, 621 GB of staged + survey imaging across 2,536 files on the BACKED-UP, GROUP-SHARED filesystem. + +Both are handed wholesale to ``shutil.rmtree`` rather than caught by ``prune``'s +own ``is_symlink()`` test, so the safety argument rests on rmtree's semantics at +a depth ``prune`` never inspects. That is a fine thing to rely on and a bad thing +to leave unpinned: the failure mode is silent, immediate and unrecoverable, and +it would be introduced by an innocent-looking edit to ``prune``. + +Deliberately container-free — clean_tile.py is stdlib-only. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +pytestmark = pytest.mark.unions + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "clean_tile.py" + +TILE = "186.307" +SURVIVORS = ( + "cleaned.json", + "manifests/tile_vignets.json", + "manifests/tile_find_exposures.json", + "output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-186-307.txt", +) + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_clean_tile", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +clean_tile = _load() + + +@pytest.fixture +def store(tmp_path): + """A finished tile's store, plus the two off-store trees it links into. + + Mirrors the real layout of 186.307 (smk-g4) at the depths that matter: the + forest links sit three levels down, the Git links four, and ``prune`` + descends into ``output/`` only because the Fe survivor lives there. + """ + precious = {} + for name, rel in (("exposures", "exp/21/2114045/output"), + ("project_tiles", "unions-wl/tiles")): + d = tmp_path / name / rel + d.mkdir(parents=True) + (d / "DO_NOT_DELETE").write_text("survey data\n") + precious[name] = d + + tile_dir = tmp_path / "run" / "tiles" / TILE[:2] / TILE + for rel in SURVIVORS[1:]: # the tombstone is written by the job + p = tile_dir / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("{}" if rel.endswith(".json") else "2114045p\n") + + # bulk that must go + for rel in ("output/run_sp_tile_Sx/sextractor_runner/output/sexcat.fits", + "output/run_sp_tile_Uz/uncompress_fits_runner/output/image.fits", + "logs/tile_detect.json", "manifests/tile_detect.json", + "tile_numbers.txt"): + p = tile_dir / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("bulk") + + # class 1: the exposure forest (rmtree'd as a top-level entry) + forest = tile_dir / "exp_forest" / "21" / "2114045" + forest.mkdir(parents=True) + (forest / "output").symlink_to(precious["exposures"]) + + # class 2: the get_images links into /project (rmtree'd inside output/) + git = tile_dir / "output" / "run_sp_tile_Git" / "get_images_runner" / "output" + git.mkdir(parents=True) + (git / "CFIS_image-186-307.fits").symlink_to( + precious["project_tiles"] / "CFIS.186.307.r.fits") + (git / "CFIS_weight-186-307.fitsfz").symlink_to( + precious["project_tiles"] / "CFIS.186.307.r.weight.fits.fz") + + return tile_dir, precious + + +def test_prune_never_follows_a_symlink_out_of_the_store(store): + """The whole point: reclaiming one tile touches nothing outside that tile.""" + tile_dir, precious = store + before = {k: sorted(p.rglob("*")) for k, p in precious.items()} + + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + + for name, d in precious.items(): + assert d.is_dir(), f"{name} tree was removed" + assert sorted(d.rglob("*")) == before[name], f"{name} tree was modified" + assert (d / "DO_NOT_DELETE").read_text() == "survey data\n" + + +def test_prune_keeps_exactly_the_survivors(store): + """Ten inodes: the four survivors and the directories that carry them.""" + tile_dir, _ = store + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + left = sorted(str(p.relative_to(tile_dir)) for p in tile_dir.rglob("*")) + assert left == sorted([ + "manifests", "manifests/tile_vignets.json", + "manifests/tile_find_exposures.json", + "output", "output/run_sp_tile_Fe", + "output/run_sp_tile_Fe/find_exposures_runner", + "output/run_sp_tile_Fe/find_exposures_runner/output", + "output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-186-307.txt", + ]) + + +def test_prune_unlinks_a_dangling_link(store): + """A forest link whose exposure clean_exposure already reclaimed. + + ``exists()`` follows the link and is False for a dangling one, so an + exists()-first test would skip it and leave the link behind. + """ + tile_dir, precious = store + import shutil + shutil.rmtree(precious["exposures"]) + clean_tile.prune(tile_dir, {tile_dir / r for r in SURVIVORS}, []) + assert not (tile_dir / "exp_forest").exists() + + +def test_survivor_contract_refuses_before_deleting_anything(store): + """A missing survivor aborts with the store intact — it is a contract.""" + tile_dir, _ = store + (tile_dir / "manifests" / "tile_vignets.json").unlink() + survivors = clean_tile.survivor_paths(tile_dir, TILE) + with pytest.raises(SystemExit) as exc: + clean_tile.require_survivors(survivors, TILE) + assert "tile_vignets.json" in str(exc.value) + assert (tile_dir / "output" / "run_sp_tile_Sx").is_dir() + + +def test_absorption_is_additive_over_an_existing_tombstone(tmp_path): + """A second clean must not blank the record the first one saved. + + ``script_hash`` reruns this job on any edit to clean_tile.py, so a re-clean + over an already-pruned store is routine — and it finds only the surviving + manifests on disk. + """ + tomb = tmp_path / "cleaned.json" + tomb.write_text(json.dumps({ + "tile": TILE, + "manifests": {f"tile_ngmix_{k}": {"stage": "tile_ngmix"} for k in range(1, 9)}, + "benchmarks": {"tile_ngmix_1.benchmark.tsv": {"s": "6852.09"}}, + })) + manifests, benchmarks = clean_tile.previous_record(tomb) + assert len(manifests) == 8 + assert benchmarks["tile_ngmix_1.benchmark.tsv"]["s"] == "6852.09" diff --git a/tests/unit/test_merge_sep_cats_paths.py b/tests/unit/test_merge_sep_cats_paths.py new file mode 100644 index 000000000..754bc76dd --- /dev/null +++ b/tests/unit/test_merge_sep_cats_paths.py @@ -0,0 +1,40 @@ +"""Chunk-path derivation in the merge_sep_cats module.""" + +import pytest + +from shapepipe.modules.merge_sep_cats_package.merge_sep_cats import chunk_path + + +REL = "./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-210-282.fits" +ABS = ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng1u" + "/ngmix_runner/output/ngmix-210-282.fits" +) + + +def test_relative_path_unchanged_behaviour(): + """The bash pipeline's relative INPUT_DIR still resolves as before.""" + assert chunk_path(REL, 3) == ( + "./output/run_sp_tile_ngmix_Ng3u/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_sharded_absolute_path(): + """Digits in the sharded parent dirs and in the file number are untouched.""" + assert chunk_path(ABS, 2) == ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng2u" + "/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_double_digit_chunk(): + assert "Ng12u" in chunk_path(ABS, 12) + + +def test_chunk_one_is_identity(): + assert chunk_path(ABS, 1) == ABS + + +def test_no_run_directory_raises(): + with pytest.raises(ValueError, match="run_"): + chunk_path("/scratch/run/tiles/21/210.282/ngmix-210-282.fits", 2) diff --git a/tests/unit/test_ngmix_range.py b/tests/unit/test_ngmix_range.py new file mode 100644 index 000000000..30027cd29 --- /dev/null +++ b/tests/unit/test_ngmix_range.py @@ -0,0 +1,273 @@ +"""The ngmix chunk split tiles ``[1, n_obj]`` exactly. + +Nothing downstream cross-checks the chunk ranges: merge_sep_cats concatenates +the chunk catalogues, so an overlap silently duplicates objects and a gap +silently drops them. The one invariant that catches both is that the ranges +partition ``1..n_obj`` — that is what this module asserts, over the epoch-weight +distributions the splitter actually has to cope with. + +``workflow/scripts/ngmix_range.py`` now runs ONCE per tile (tile_vignets writes +the whole partition to the group's node-local scratch) and is then only read +from, once per chunk. The second thing this module pins is that the trip through +JSON changes nothing: row ``k`` of the written file is exactly what asking the +splitter for chunk ``k`` used to return. + +Deliberately container-free: the range function takes a plain sequence of epoch +counts, so nothing here imports astropy, numpy, or shapepipe. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "ngmix_range.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + spec = importlib.util.spec_from_file_location("_ngmix_range", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +ngmix_range = _load() + + +def assert_tiles(ranges, n_obj, n_chunks): + """Assert the partition invariant, in the form each failure mode takes.""" + assert len(ranges) == n_chunks + assert ranges[0][0] == 1 + assert ranges[-1][1] == n_obj + for (lo, hi), (prev_lo, prev_hi) in zip(ranges[1:], ranges): + assert lo == prev_hi + 1 + assert hi >= lo - 1, "an empty range is (hi + 1, hi), never narrower" + assert prev_hi >= prev_lo - 1 + covered = [i for lo, hi in ranges for i in range(lo, hi + 1)] + assert covered == list(range(1, n_obj + 1)) + # ID_OBJ_MAX <= 0 is ngmix's "unbounded" sentinel: a chunk emitting it + # re-measures the whole tile instead of nothing. + assert all(hi >= 1 for _, hi in ranges) + + +# --------------------------------------------------------------------------- # +# The property +# --------------------------------------------------------------------------- # + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=400), + n_chunks=st.integers(min_value=1, max_value=16), + data=st.data(), +) +def test_ranges_tile_the_catalogue(n_obj, n_chunks, data): + """Any epoch distribution, any shape: the ranges still partition 1..N.""" + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=40), + min_size=n_obj, + max_size=n_obj, + ) + ) + assert_tiles(ngmix_range.id_ranges(epochs, n_chunks), n_obj, n_chunks) + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=60), + n_chunks=st.integers(min_value=1, max_value=8), + data=st.data(), +) +def test_ranges_are_deterministic(n_obj, n_chunks, data): + """Same input, same ranges — the eight processes share nothing else. + + Integer weights make the split a pure function of the epoch list, with no + float accumulation whose order could matter; this pins that. + """ + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=99), + min_size=n_obj, + max_size=n_obj, + ) + ) + first = ngmix_range.id_ranges(epochs, n_chunks) + assert first == ngmix_range.id_ranges(list(epochs), n_chunks) + assert all(isinstance(b, int) for lo, hi in first for b in (lo, hi)) + + +# --------------------------------------------------------------------------- # +# Write-once / read-per-chunk is the same partition +# --------------------------------------------------------------------------- # + + +def _roundtrip(epochs, n_chunks): + """What a chunk shell sees: the doc as it comes back off disk.""" + return json.loads(json.dumps(ngmix_range.partition(epochs, n_chunks))) + + +@settings(deadline=None) +@given( + n_obj=st.integers(min_value=1, max_value=200), + n_chunks=st.integers(min_value=1, max_value=12), + data=st.data(), +) +def test_written_rows_reproduce_the_per_chunk_computation(n_obj, n_chunks, data): + """THE INVARIANCE TEST for materialising the split. + + Writing the whole partition and reading chunk ``k``'s row back must give + byte-for-byte what the old per-chunk ``id_ranges(...)[k - 1]`` returned. If + this ever fails, the two mechanisms have drifted and a tile's coverage is + what pays. + """ + epochs = data.draw( + st.lists( + st.integers(min_value=0, max_value=40), + min_size=n_obj, + max_size=n_obj, + ) + ) + doc = _roundtrip(epochs, n_chunks) + assert doc["n_obj"] == n_obj and doc["n_chunks"] == n_chunks + expected = ngmix_range.id_ranges(epochs, n_chunks) + got = [ngmix_range.chunk_range(doc, k) for k in range(1, n_chunks + 1)] + assert got == expected + assert_tiles(got, n_obj, n_chunks) + + +def test_chunk_index_outside_the_written_partition_is_fatal(): + """A chunk index the file does not hold must not index from the end.""" + doc = _roundtrip([3] * 40, 4) + for bad in (0, -1, 5): + with pytest.raises(SystemExit, match="outside 1..4"): + ngmix_range.chunk_range(doc, bad) + + +def test_reading_an_absent_ranges_file_is_fatal(tmp_path): + """NO FALLBACK: a missing file fails, it does not recompute. + + Recomputing per chunk is precisely the mechanism this design removed. + """ + with pytest.raises(SystemExit, match="no ranges file"): + ngmix_range.read_ranges(tmp_path / "ngmix_ranges.json") + + +# --------------------------------------------------------------------------- # +# Degenerate shapes — each pins a decision, not just an absence of crash +# --------------------------------------------------------------------------- # + + +def test_single_chunk_takes_everything(): + """n_chunks == 1: one range over the whole catalogue.""" + assert ngmix_range.id_ranges([3] * 17, 1) == [(1, 17)] + + +def test_one_object_per_chunk(): + """n_obj == n_chunks: one object each, however lopsided the weights.""" + assert ngmix_range.id_ranges([0, 9, 1, 40], 4) == [ + (1, 1), (2, 2), (3, 3), (4, 4) + ] + + +def test_fewer_objects_than_chunks_pads_with_empty_ranges(): + """DOCUMENTED: the surplus chunks get ``(n_obj + 1, n_obj)``, not (1, 0). + + The old equal-count split gave every surplus chunk ``(1, 0)``, and ngmix + reads ``ID_OBJ_MAX = 0`` as unbounded — so each of them would have measured + the entire tile rather than nothing. + """ + assert ngmix_range.id_ranges([2, 5, 1], 6) == [ + (1, 1), (2, 2), (3, 3), (4, 3), (4, 3), (4, 3) + ] + assert_tiles(ngmix_range.id_ranges([2, 5, 1], 6), 3, 6) + + +def test_zero_objects_is_fatal(): + """No split is meaningful, and (1, 0) is ngmix's unbounded sentinel.""" + with pytest.raises(ValueError, match="zero objects"): + ngmix_range.id_ranges([], 8) + + +def test_zero_chunks_is_fatal(): + """There is no zeroth chunk to hand a range to.""" + with pytest.raises(ValueError, match="n_chunks"): + ngmix_range.id_ranges([1, 2, 3], 0) + + +def test_equal_weights_reproduce_an_equal_count_split(): + """Uniform epochs: chunk sizes differ by at most one object.""" + ranges = ngmix_range.id_ranges([3] * 1000, 8) + assert_tiles(ranges, 1000, 8) + sizes = [hi - lo + 1 for lo, hi in ranges] + assert max(sizes) - min(sizes) <= 1 + assert sum(sizes) == 1000 + + +def test_zero_epoch_objects_still_weigh_something(): + """ALPHA is why: without it a 0-epoch tail is free and lands in one chunk. + + Ninety-six zero-epoch objects and four 1-epoch ones. Weighing only epochs + would let a single chunk swallow all ninety-six. + """ + ranges = ngmix_range.id_ranges([0] * 96 + [1] * 4, 4) + assert_tiles(ranges, 100, 4) + assert max(hi - lo + 1 for lo, hi in ranges) < 96 + + +def test_one_enormously_heavy_object_is_isolated(): + """The heavy object gets a chunk to itself; the tail still tiles.""" + epochs = [1] * 20 + [10_000] + [1] * 20 + ranges = ngmix_range.id_ranges(epochs, 4) + assert_tiles(ranges, 41, 4) + assert (21, 21) in ranges + + +# --------------------------------------------------------------------------- # +# The split is the optimal one, not merely a legal one +# --------------------------------------------------------------------------- # + + +def _brute_force_min_max(weights, n_chunks): + """Smallest achievable maximum chunk weight, by exhaustive enumeration.""" + n = len(weights) + best = {} + + def solve(start, chunks): + if chunks == 1: + return sum(weights[start:]) + if (start, chunks) not in best: + best[(start, chunks)] = min( + max(sum(weights[start:cut]), solve(cut, chunks - 1)) + for cut in range(start + 1, n - chunks + 2) + ) + return best[(start, chunks)] + + return solve(0, n_chunks) + + +@settings(deadline=None, max_examples=40) +@given( + epochs=st.lists( + st.integers(min_value=0, max_value=12), min_size=4, max_size=14 + ), + n_chunks=st.integers(min_value=2, max_value=4), +) +def test_slowest_chunk_is_minimal(epochs, n_chunks): + """The objective is the slowest chunk — check it against brute force.""" + if len(epochs) < n_chunks: + return + weights = [ + ngmix_range.MILLI_EPOCH * e + ngmix_range.ALPHA_MILLI_EPOCHS + for e in epochs + ] + ranges = ngmix_range.id_ranges(epochs, n_chunks) + loads = [sum(weights[lo - 1:hi]) for lo, hi in ranges] + assert max(loads) == _brute_force_min_max(weights, n_chunks) diff --git a/tests/unit/test_run_report_tombstones.py b/tests/unit/test_run_report_tombstones.py new file mode 100644 index 000000000..ccb791d17 --- /dev/null +++ b/tests/unit/test_run_report_tombstones.py @@ -0,0 +1,171 @@ +"""Reclamation must not change what ``sp report`` says about a unit. + +``workflow/scripts/run_report.py`` reads a unit's verdict from two places with +the same shape and different lifetimes: the records on disk +(``manifests/*.json`` + ``logs/*.json``), and — once ``clean_tile`` or +``clean_exposure`` has deleted those — the copies absorbed into the unit's +``cleaned.json``. Both readers face the same collision, because several files +map to one ``(unit, stage)``: a stage's manifest and its byte-identical log, and +the eight ngmix chunks, which all carry ``stage: "tile_ngmix"``. + +If the two readers resolve that collision differently, RECLAMATION SILENTLY +EDITS THE REPORT. It did: absorption used first-key-wins, so only +``tile_ngmix_1`` survived and a ``warn`` on any other chunk became ``complete`` +the moment the tile was cleaned — and the tile dropped out of the "tiles not +complete" table. This module pins the invariant that catches that whole class: +**the tombstone path and the on-disk path agree, stage for stage.** + +Deliberately container-free — run_report.py is stdlib-only, so nothing here +imports astropy, numpy, or shapepipe. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "run_report.py" + + +def _load(): + """Import the script by path — ``workflow/scripts`` is not a package.""" + assert SCRIPT.exists(), f"{SCRIPT} not found; bin/sp calls it by path" + spec = importlib.util.spec_from_file_location("_run_report", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +run_report = _load() + +TILE = "186.307" +NGMIX_CHUNKS = 8 + + +def _manifest(stage, status="complete", found=1, expect=1): + """One completeness verdict, in the shape completeness.py writes.""" + return { + "stage": stage, "level": "tile", "unit": TILE, "status": status, + "runners": {f"{stage}_runner": { + "found": found, "expect": expect, + "status": status, "warn": status == "warn"}}, + "failures": [], + } + + +def _records(ngmix_status=None): + """A finished tile's manifests, keyed by file stem as the store keys them. + + ``ngmix_status`` is ``{chunk_number: status}``; unlisted chunks are complete. + """ + out = {s: _manifest(s) for s in run_report.TILE_STAGES if s != "tile_ngmix"} + for k in range(1, NGMIX_CHUNKS + 1): + out[f"tile_ngmix_{k}"] = _manifest( + "tile_ngmix", (ngmix_status or {}).get(k, "complete")) + return out + + +def _cleaned_store(root, records, survivors=("tile_vignets", "tile_find_exposures")): + """A reclaimed tile store on disk: the survivors, and the tombstone. + + Exactly what ``clean_tile`` leaves behind — ``logs/`` gone, ``manifests/`` + holding only the manifests other mechanisms own, every record absorbed into + ``cleaned.json``. + """ + tile_dir = root / "tiles" / TILE[:2] / TILE + (tile_dir / "manifests").mkdir(parents=True) + for stem in survivors: + (tile_dir / "manifests" / f"{stem}.json").write_text( + json.dumps(records[stem])) + (tile_dir / "cleaned.json").write_text(json.dumps({ + "tile": TILE, "manifests": records, "benchmarks": {}})) + return tile_dir + + +def _live_store(root, records): + """The same tile BEFORE reclamation: every manifest, and its identical log.""" + tile_dir = root / "tiles" / TILE[:2] / TILE + for sub in ("manifests", "logs"): + (tile_dir / sub).mkdir(parents=True) + for stem, m in records.items(): + (tile_dir / sub / f"{stem}.json").write_text(json.dumps(m)) + return tile_dir + + +def _stage_status(run_dir, cleaned_survivors=None): + """``{stage: status}`` as the report would tally it, from a store on disk.""" + manifests = run_report.load_manifests(run_dir, "tiles") + if cleaned_survivors is not None: + run_report.absorb_tombstones(run_dir, "tiles", manifests, + cleaned_survivors) + return {s: m.get("status") for s, m in manifests[TILE].items()} + + +@pytest.mark.parametrize("chunk", range(1, NGMIX_CHUNKS + 1)) +def test_a_warning_chunk_survives_reclamation(tmp_path, chunk): + """A warn on ANY chunk reads the same before and after the tile is cleaned. + + Parametrised over all eight because the bug was invisible on chunk 1: that + is the stem first-key-wins happened to keep, so the one chunk anybody would + test by hand was the one chunk that worked. + """ + records = _records({chunk: "warn"}) + + live = tmp_path / "live" + _live_store(live, records) + before = _stage_status(live) + + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, records) + after = _stage_status(cleaned, run_report.SURVIVING_TILE_STAGES) + + assert before["tile_ngmix"] == "warn" + assert after == before + + +def test_a_failed_chunk_survives_reclamation(tmp_path): + """Worst-status-wins, not merely warn: failed beats warn beats complete.""" + records = _records({3: "warn", 6: "failed"}) + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, records) + after = _stage_status(cleaned, run_report.SURVIVING_TILE_STAGES) + assert after["tile_ngmix"] == "failed" + + +def test_reclaimed_tile_reports_every_stage(tmp_path): + """The survivors must not read as a rebuilt chain. + + ``clean_tile`` cannot empty ``manifests/`` — two manifests are currency + other mechanisms own — so without the survivor set the "records on disk mean + a rebuilt chain" guard fires and a reclaimed tile reports as one that ran + two stages and stopped. + """ + cleaned = tmp_path / "cleaned" + _cleaned_store(cleaned, _records()) + manifests = run_report.load_manifests(cleaned, "tiles") + got = run_report.absorb_tombstones(cleaned, "tiles", manifests, + run_report.SURVIVING_TILE_STAGES) + assert got == {TILE} + assert sorted(manifests[TILE]) == sorted(run_report.TILE_STAGES) + + +def test_rebuilt_chain_keeps_the_guard(tmp_path): + """A unit with a NON-survivor record on disk is a rebuilt chain: skip it. + + This is the default (empty survivor set) that the exposure side uses, and it + must keep behaving exactly as it did before tiles needed an argument here. + """ + cleaned = tmp_path / "cleaned" + tile_dir = _cleaned_store(cleaned, _records()) + (tile_dir / "manifests" / "tile_detect.json").write_text( + json.dumps(_manifest("tile_detect", "failed"))) + + manifests = run_report.load_manifests(cleaned, "tiles") + got = run_report.absorb_tombstones(cleaned, "tiles", manifests, + run_report.SURVIVING_TILE_STAGES) + assert got == set() + assert manifests[TILE]["tile_detect"]["status"] == "failed" + assert "tile_ngmix" not in manifests[TILE] diff --git a/uv.lock b/uv.lock index e7277a973..eecd374c7 100644 --- a/uv.lock +++ b/uv.lock @@ -153,24 +153,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/ef/a5cf36a402c4511a776405f12d0b35196f55f4312ce407012a2fbcf1e4e0/astropy-8.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b482bc6c57c966e6c6234410a1d9afbcf92bcac858cf287812f8c99ddc3fafc", size = 10407732, upload-time = "2026-07-05T07:24:40.774Z" }, ] -[[package]] -name = "astropy-healpix" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/c1/aeb3fe3be2ee863708d625014267c71abfe20ddaa293b3d4ddb72ee1d6e9/astropy_healpix-2.0.1.tar.gz", hash = "sha256:0e3f1c94064c45da779900cb90c938df7aef99a924abb23eeb893b16540e77e6", size = 112256, upload-time = "2026-07-20T21:07:30.004Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/3d/0cde0db89ac8dd4e5347322470530298915739f7e9b356b01c1939de8c4f/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:179119d5a69e7b9245919cbe04c3e6bf0a485516b36c29f3402951aad5452251", size = 191178, upload-time = "2026-07-20T21:07:16.512Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c4/71cf2bd4374cc17e015413462be8051fe08bc49e077b7d48072fff4e465d/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c092c54124c48f8d98e04fb22f4b2aa4c8675e65d81c351523f41377f9a6df22", size = 193429, upload-time = "2026-07-20T21:07:18.015Z" }, - { url = "https://files.pythonhosted.org/packages/fa/5c/3a50f68225b836b395da4fb8dfd3d702ded1916042490b16a613caa0e4a6/astropy_healpix-2.0.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce875a29c598c1a99f8f68351daeb4173463044dce4f1c7ebfe8c233ec5e9a49", size = 188694, upload-time = "2026-07-20T21:07:19.244Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a7/c369703ca3fc6f5bee31b3d1f6d0f0b38c15ec84e7fbb6f552c0c7cedfb2/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78f76785852bcc748f5efab8bb4f2ab1fe959d7a998b48e7ed1e59a46cbf0e51", size = 198717, upload-time = "2026-07-20T21:07:24.657Z" }, - { url = "https://files.pythonhosted.org/packages/32/d5/0c4183611b8f36877112882e25cc8a91655a2d11e120ea1f5c153cb7d3a0/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02cefde735da1fe74654e786e02286261b04cc43f5c908a86a8457b2989ac2aa", size = 201205, upload-time = "2026-07-20T21:07:26.128Z" }, - { url = "https://files.pythonhosted.org/packages/e2/54/42bcd02b7c604d3a1132dfa93195bdc2677bb7844172cd098e149cbcb4e9/astropy_healpix-2.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65bee7b70f35ddb81d6b6c849c5bf46768afbce6869395e4f9e0a5c27cb0ce17", size = 196134, upload-time = "2026-07-20T21:07:27.55Z" }, -] - [[package]] name = "astropy-iers-data" version = "0.2026.9.7.0.56.14" @@ -180,24 +162,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/5f/7872fc61dc9cbf27abb5303967dcdbc016fa80bc5cd483064fe74b59ab02/astropy_iers_data-0.2026.9.7.0.56.14-py3-none-any.whl", hash = "sha256:b24396f527feb4ec4b4ef0dcba820246ac7ecb1eed91ebb9e767d64774349659", size = 2001670, upload-time = "2026-09-07T00:56:54.114Z" }, ] -[[package]] -name = "astroquery" -version = "0.4.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "beautifulsoup4" }, - { name = "html5lib" }, - { name = "keyring" }, - { name = "numpy" }, - { name = "pyvo" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/48/273dbde090e071f9d264d084bc49193d126498d2906172b78febd9d62e28/astroquery-0.4.11.tar.gz", hash = "sha256:5537529bddc7fa07e773d5cd9baca593e3f5d93474edd1914f68e89506042b33", size = 12561055, upload-time = "2025-09-20T04:26:36.744Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/ca/944b328f2c60b83896a9223312e21e46cdc1fef57565e853da4544ff6a8e/astroquery-0.4.11-py3-none-any.whl", hash = "sha256:e34f114b285dd07a10ddb2065ebce829b01b0e740fd89dbc81a3077808e24b2d", size = 11139417, upload-time = "2025-09-20T04:26:31.881Z" }, -] - [[package]] name = "asttokens" version = "3.0.2" @@ -742,45 +706,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] -[[package]] -name = "dask" -version = "2026.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "cloudpickle" }, - { name = "fsspec" }, - { name = "packaging" }, - { name = "partd" }, - { name = "pyyaml" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/a7/6b3c7ac32b642fbbe0821111654e0bd8cfbe88f68560bcf23cc78ab35c71/dask-2026.8.0.tar.gz", hash = "sha256:8a94c37b5de6d869343340dc26c3c3acca7ec48a3abdabe00ea3abb1125884d5", size = 11561752, upload-time = "2026-08-24T19:21:25.906Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/3a/4fc99e788bcfa1b3b3f21abf57da45898d807d007e7f6fd1c7300904eb70/dask-2026.8.0-py3-none-any.whl", hash = "sha256:ccc0c83a189b0398602435189771d28dad7b5773b6089bb8dce14ae732dd782c", size = 1492182, upload-time = "2026-08-24T19:21:23.997Z" }, -] - -[package.optional-dependencies] -array = [ - { name = "numpy" }, -] - -[[package]] -name = "dask-image" -version = "2026.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dask", extra = ["array"] }, - { name = "numpy" }, - { name = "pims" }, - { name = "scipy" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/5b/15d6d6ff8697b188787609be059fe4f07f99fc00f43f68e9e1540fa8733e/dask_image-2026.5.0-py3-none-any.whl", hash = "sha256:acf86cd7f0f1e97804198d30b7cc931efd29f9dec86c65ec004b50405c3f5227", size = 43814, upload-time = "2026-05-27T14:05:56.237Z" }, -] - [[package]] name = "datetime" version = "6.0" @@ -833,18 +758,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - [[package]] name = "dpath" version = "2.2.0" @@ -933,15 +846,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, ] -[[package]] -name = "fsspec" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, -] - [[package]] name = "future" version = "1.0.0" @@ -998,20 +902,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d6/0b/29d7965215f8ef830a7ca1f42997fe13e5693d85e9edb18f938d063ef5f2/gitpython-3.1.62-py3-none-any.whl", hash = "sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e", size = 222753, upload-time = "2026-09-07T02:57:19.762Z" }, ] -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, -] - [[package]] name = "greenlet" version = "3.5.5" @@ -1171,19 +1061,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, ] -[[package]] -name = "html5lib" -version = "1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173, upload-time = "2020-06-22T23:32:36.781Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" @@ -1882,15 +1759,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/f9/7b7b50f80b4585bcd78675ff3110c256877b11df32a8cde284f851762f57/llvmlite-0.49.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32adb84fdaae28aeb86fdb6253084ee707ee157289a2e98fe3caf48a62bee82", size = 58344482, upload-time = "2026-08-11T16:25:51.527Z" }, ] -[[package]] -name = "locket" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, -] - [[package]] name = "lsstdesc-coord" version = "1.3.1" @@ -2355,24 +2223,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/f9/3a7b6dbf81e01a48958b45ad2239edbc64707522ab17f11f9f18c44bf6d1/numba-0.67.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83ab968b0e0fa744eba03351282dd8000796e6ec8e4518f47bd3ed86c0a20c7b", size = 3614644, upload-time = "2026-08-11T23:03:55.794Z" }, ] -[[package]] -name = "numcodecs" -version = "0.16.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, - { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, - { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, - { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, - { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, -] - [[package]] name = "numpy" version = "2.5.3" @@ -2473,19 +2323,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] -[[package]] -name = "partd" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "locket" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, -] - [[package]] name = "pexpect" version = "4.9.0" @@ -2539,19 +2376,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, ] -[[package]] -name = "pims" -version = "0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imageio" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "slicerator" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } - [[package]] name = "platformdirs" version = "4.11.7" @@ -2646,15 +2470,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "pyavm" -version = "0.9.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/ac/a925d36dd37fc37f89afccc1f62ffa03b7344c8e4ff7850be7879b4497e6/pyavm-0.9.9.tar.gz", hash = "sha256:bc0f605d957c1fd6d7765523fcba8b9a72377ac6c51461c2a838fe44600bcd9a", size = 220572, upload-time = "2026-03-12T09:54:50.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/ab/ba8d2b40aee05cd986807d58ee324369eb16248b717a08b51eee977f8d33/pyavm-0.9.9-py3-none-any.whl", hash = "sha256:8bba0ee9645a8a9f215af9ceea67b494a6ee1fe380cebdc00ba99d781539ad76", size = 379786, upload-time = "2026-03-12T09:54:49.455Z" }, -] - [[package]] name = "pybind11" version = "3.1.0" @@ -3013,19 +2828,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, ] -[[package]] -name = "pyvo" -version = "1.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/35/12c0f4fa0879316837ac56f275942f006f0735e5aedba529b4449dddc36f/pyvo-1.9.1.tar.gz", hash = "sha256:2f26c99af7c32f3c34b919e2d14eaf1a95914176d693fb7769773f3ab0b7999d", size = 2167800, upload-time = "2026-06-11T14:44:01.349Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/09/04ff6e8beaa6cd60a960f925d837c2afa3fffba7860042ab1b52a8358297/pyvo-1.9.1-py3-none-any.whl", hash = "sha256:098648d00943440f56c1d00ac76330433578a0725ea6187afaa6bae08545bb53", size = 1152763, upload-time = "2026-06-11T14:43:59.016Z" }, -] - [[package]] name = "pywavelets" version = "1.9.0" @@ -3155,28 +2957,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] -[[package]] -name = "reproject" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "astropy-healpix" }, - { name = "dask", extra = ["array"] }, - { name = "dask-image" }, - { name = "fsspec" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pyavm" }, - { name = "scipy" }, - { name = "zarr" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/5b/8d9b51c754ab014194d374cb4873d0729b397f67997e67721538b643e682/reproject-0.21.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2132fc5d2fa3fbbd57337099f9d47582136e653ac82402783c7ceffa26804816", size = 1776590, upload-time = "2026-06-25T15:11:30.358Z" }, - { url = "https://files.pythonhosted.org/packages/da/2d/f9d76e8e308813978227e1e43a1f25f64cd0f4b030cecaeff8baeec9eeac/reproject-0.21.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8525baaca84e949a69532c02ff491c23993e3bc22a59f694af576e473f2a9d2f", size = 1791889, upload-time = "2026-06-25T15:11:31.741Z" }, -] - [[package]] name = "requests" version = "2.34.2" @@ -3505,7 +3285,6 @@ version = "1.1.0" source = { editable = "." } dependencies = [ { name = "astropy" }, - { name = "astroquery" }, { name = "canfar" }, { name = "cs-util" }, { name = "galsim" }, @@ -3525,7 +3304,6 @@ dependencies = [ { name = "pyqtgraph" }, { name = "python-dateutil" }, { name = "python-pysap" }, - { name = "reproject" }, { name = "sf-tools" }, { name = "skaha" }, { name = "skyproj" }, @@ -3590,7 +3368,6 @@ test = [ [package.metadata] requires-dist = [ { name = "astropy", specifier = ">=7.0" }, - { name = "astroquery" }, { name = "build", marker = "extra == 'release'" }, { name = "canfar" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util?branch=develop" }, @@ -3620,7 +3397,6 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0" }, { name = "python-dateutil" }, { name = "python-pysap", specifier = ">=0.3" }, - { name = "reproject", specifier = ">=0.19" }, { name = "ruff", marker = "extra == 'lint'" }, { name = "sf-tools", specifier = ">=2.0.4" }, { name = "shapepipe", extras = ["doc", "jupyter", "lint", "plot", "release", "test", "fitsio"], marker = "extra == 'dev'" }, @@ -3693,15 +3469,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/55/23d3f7a48a2c75fe20100022a5b74fbd29bdc927900a169e9b9de8430fe9/skyproj-2.6.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9629114a768abe4a40f497eade86417c3e55a94a2e47010da0e81e54c229826a", size = 4343668, upload-time = "2026-08-24T23:40:32.691Z" }, ] -[[package]] -name = "slicerator" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/52/f38586b82b2935f8b59a09b0a79c545a22ed062e728c9418bafeb51f61e0/slicerator-1.1.0.tar.gz", hash = "sha256:44010a7f5cd87680c07213b5cabe81d1fb71252962943e5373ee7d14605d6046", size = 38283, upload-time = "2022-04-07T18:54:08.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, -] - [[package]] name = "smart-open" version = "7.7.1" @@ -4164,15 +3931,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, ] -[[package]] -name = "toolz" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, -] - [[package]] name = "tornado" version = "6.5.8" @@ -4394,23 +4152,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, ] -[[package]] -name = "zarr" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "donfig" }, - { name = "google-crc32c" }, - { name = "numcodecs" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/15/436cb1d3bbe86173bd44ce7a34ecb210d0c0416946e337858149a905ef5a/zarr-3.3.0.tar.gz", hash = "sha256:cd0c8cf738b4bb4807815bc1255acad5bdf1a7b7264b606c5a1bc0d0392a306b", size = 943626, upload-time = "2026-07-30T16:35:10.491Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/c6/6b726ddf4c3ac5a123f285c3650fa8268902c28ea541a0282867f1336e65/zarr-3.3.0-py3-none-any.whl", hash = "sha256:323bf5366d4f909052ef6e2e03e7481a7434c3ee75d3a981eb3a71fc1ae22cef", size = 363685, upload-time = "2026-07-30T16:35:08.794Z" }, -] - [[package]] name = "zipp" version = "4.1.0" diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 000000000..55611cedc --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,261 @@ +# ShapePipe Snakemake orchestration + +Snakemake workflow that orchestrates real-data ShapePipe runs. It replaces the +`curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` +bash layers and the per-site sbatch reimplementations. **Module code is +untouched**: rules call `shapepipe_run -c ` on the existing config +chains. Design and rationale: +[CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +(the design document). + +Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is +unsupported: `sp` sets the state directory, the SLURM profile, and +`SP_PHASE`, which the Snakefile needs to build the tile/exposure index at +parse time. Running snakemake directly skips all of that. + +## Quick start (nibi) + +```bash +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which +# is node-local; the SLURM executor re-invokes this python inside every job). +uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 +source /project/def-mjhudson/cdaley/snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' + +# Edit workflow/config.yaml: tile_list, inputs.tiles/exposures, outputs.run_dir, +# outputs.products_dir/index_db, and container. + +# `psf_model` is `psfex` or `mccd`; mccd is wired but unvalidated here, while psfex is exercised by smk-g4 through smk-g6. + +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a +# fresh shell always has the right state. +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp cancel # scancel this workflow's jobs +workflow/bin/sp container status # which image the jobs will run +``` + +Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, +queried 2026-07-30): `snakemake==9.23.1`, `snakemake-executor-plugin-slurm==2.7.1`. +Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The +v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became +plugins, and full `rerun-triggers` became the default. + +Anything other than `run`, `report`, `container`, `cancel` passes straight through to +snakemake with the workflow's profile and state dir — the direct command path for +`sp --unlock`, `sp --dag`, `sp exp_psf ...`. + +## The container image + +`sp container` owns which image the jobs run inside. Two layers, and the second +only exists if you ask for one: + +* your **cached SIF** (`~/.cache/shapepipe/shapepipe.sif`, `SP_CACHE_DIR` or + `SP_CONTAINER` to move it) — a read-only pull of the published image, private + to you, so nobody else's refresh moves the ground under your running jobs; +* an optional **sandbox** (`~/.cache/shapepipe/sandbox/`, `SP_SANDBOX`) — the + same image unpacked writable, so a `pip install` into it sticks. The escape + hatch for work needing a package the image does not carry yet. + +The Snakefile's `container:` is that resolution, in one order shared by the CLI +and the workflow: **sandbox → cached SIF → the `container:` path in +`config.yaml`**. With an empty cache — the normal case — that lands on the +shared `/project` `.sif` the workflow has always used, so this changes nothing +until you opt in. + +```bash +sp container status # layers present, active one, revision vs HEAD +sp container pull # ghcr.io/cosmostat/shapepipe:develop-runtime +sp container pull --tag docker://... # some other image +sp container sandbox # unpack the SIF writable (opt-in) +sp container exec --writable pip install +sp container exec python -c 'import shapepipe' +sp container resolve # just the path the workflow will run +``` + +`status` reads the image's OCI labels and places its +`org.opencontainers.image.revision` against this checkout's HEAD: +in-sync / behind / ahead / diverged, or unknown when the image carries no label +or the commit was never fetched here. + +**`pull` needs the network.** Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an `salloc` allocation — never from a +batch job. `pull` and `sandbox` both stage to a sibling path and swap it in, so +an in-flight job never sees a half-written image and a failed rebuild leaves the +one you had intact. + +## Execution: two static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, +so it cannot live in the same static DAG that produces it. `sp run` is +therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** — `snakemake prepare_all_tiles`: per-tile static DAG + (`Git_vos → Uz → Fe`), `keep-going` so tile failures are independent. A + nonzero exit here is not fatal to the run — tiles that lost their + exposure list are dropped at the compute parse — but it is a warning: + `SP_MISSING_THRESHOLD` (default 0.0) is the real gate. +2. **COMPUTE** — `snakemake all`: this invocation's *parse* builds the + tile↔exposure index (`build_index.py`, imported at parse time, not a DAG + node) and runs the full tile/exposure compute chain. The index + accumulates across invocations, so appending tiles later changes which + jobs exist without invalidating completed work. + +`sp run` chains both so the UX is one command; both exit codes are checked +and the run fails if either phase failed. + +## The launch code snapshot + +`sp run` copies the code it is about to launch — `workflow/` (config symlinks +dereferenced), `src/` and the profile — into `/code`, records HEAD +plus a dirty flag in `/code/snapshot.json`, and runs the campaign +entirely out of that copy. It matters because a campaign is not one process: the +SLURM executor re-invokes snakemake on every job's node, so jobs re-parse the +Snakefile and read `workflow/scripts/*` and the ini chain hours after launch. +**Editing the checkout while a campaign runs is therefore harmless; a change +takes effect on the next `sp run`.** + +Everything workflow-internal hangs off `workflow.basedir`, which *is* the +snapshot, including the profile's `PYTHONPATH` +pin, which YAML cannot interpolate: `sp run` rewrites that single path in the +snapshot's copy of the profile and launches `--profile` at the copy. The snapshot +is refreshed wholesale on every `sp run` — a new `sp run` *is* the relaunch — and +the other verbs run out of the existing snapshot, `sp container` excepted (it is +about the image you are working with now, not about a campaign). The mechanism +and its rationale live in one place: `bin/sp`. + +## Execution mode: one SLURM job per rule + +The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule +instance becomes its own SLURM job carrying that rule's own attempt-scaled +resources (`cpus_per_task = threads`, `mem_mb`, `runtime`) and runs inside +`apptainer exec` via the profile's software-deployment method — the workflow +never calls apptainer directly. Snakemake feeds the queue as jobs finish, so +the full campaign's job count never needs to queue at once, and multi-node +scaling is inherent. `jobs:` in the profile caps concurrent submissions at a +fraction of the cluster's per-user submit limit (queried, not invented — see +the comment in the profile file for the query and date). + +The profile intentionally sets no `set-resources` / `set-threads` overrides: +those replace a rule's own values wholesale, which would kill the +attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +ngmix thread count. Rules own their own resources; the profile only supplies +defaults for rules that state nothing. + +`group:` labels that fuse short rules (uncompress, merges) into their chunky +neighbours (queue-latency amortization, per the PRD) are **not yet wired**: +they require labels in `workflow/rules/*.smk`, out of scope for this +profile-only pass. + +## Layout + +``` +workflow/ + Snakefile parse-time index load; global container:; onsuccess/onerror report hooks + config.yaml the run: tile list, input/output paths, container, chunk count + bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) + rules/ + prepare.smk tile get_images/uncompress/find_exposures + exposure.smk per-exposure: get_images, split, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat + scripts/ + sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count check) + build_index.py prepare-phase run_index.sqlite builder (plain script) + build_forest.py per-tile exposure symlink forest (group-compatible shell) + completeness.py the ported count table (shared by sp_rule + run_report) + run_report.py standalone report (NOT a DAG node; run_report hooks call it) + container.py image layers + the resolution order behind `sp container` (stdlib-only) + clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) +profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going +``` + +## How it works + +- **The atom is one rule == one `shapepipe_run` on one unit.** Its single + declared output is that unit's manifest + (`/manifests/.json`), not its product files — a missing + CCD is often legitimate, and at DR6 scale per-CCD declaration means + millions of paths. +- **Manifests are the DAG's currency, and they are success-only.** + `completeness.py check` writes its full verdict — per-runner counts against + expected counts, scraped failure reasons, the `shapepipe_run` exit status when nonzero + — to the rule's `log:` (`/logs/.json`) on *every* run, and + additionally to the declared `.json` manifest only when that verdict is + a success. So `.json` on disk means "this stage succeeded", and a + resume after an unclean death cannot schedule downstream work on top of a + failure. Nothing unlinks anything: Snakemake deletes a failed job's declared + output natively and never touches its log, which is why the profile runs + *without* `keep-incomplete`. `sp report` reads both dirs — the manifest for + success, the log for failure — and a unit with neither ran nothing. +- **Completeness is an exact-count check, not a taxonomy.** After a run, + `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s expected count and exits nonzero below it. A shortfall + below `expect` fails unless `warn` is set. No 3-class taxonomy, no + error-signature whitelist. `--keep-going` isolates a failure to its own + DAG cone. +- **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` + in `tiles/<2-char prefix>//` or `exp///`. Configs are + committed under `workflow/config/cfis/` and version with the rules that set + the env vars they interpolate — there is no `config_src` knob, and no per-unit + config symlink; `$SP_CONFIG` points straight at the committed directory. +- **There is no masking stage, on either side.** ShapePipe generates no masks + (PR #847). The one mask that reaches pixels is the instrument flag image + delivered with each exposure, which `exp_split` splits per CCD beside image + and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star + halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed + healsparse maps and QUERIED once per object: the `mask_query` module writes a + `MASK_EXT` column onto each CCD's detection catalogue (inside `exp_psf`), and + `make_cat` writes one `MASK_` column per band onto the final catalogue + (inside `tile_make_cat`). Neither is cut on in the pipeline: instrument flags + mark corrupted measurements and are the only masks that reject anything here, + while the healsparse masks are location flags and every decision about them is + downstream. On exposures the query ships off — `MASK_PATHS` is commented out, + making `mask_query` a no-op pass-through — so turning it on is a config edit, + not a chain edit. Map paths are config, not + code, so regenerated products cost a config edit. Nothing is fetched from a + catalogue server, staged, or rasterized, which is why the old + `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are + gone. +- **The index is parse-time data, never a rule input.** Appending tiles + changes which jobs exist without invalidating completed work. +- **Exposure products are not `temp()`.** Exposures overlap tiles, so + `temp()` would cascade destructive reruns when a tile is appended later. + Reclamation is the in-DAG `clean_exposure` rule instead: one job per + exposure, taking every consuming tile's `tile_vignets` manifest as input + (the campaign-wide consumer set comes from the accumulating index), which + deletes the store *and* the exposure's `manifests/` and `logs/`, and leaves a + `cleaned.json` tombstone. Deleting the manifests is what makes a late append + correct: the + appended tile finds an unbuilt chain and regenerates it. The `clean:` flag + in `config.yaml` gates it; flipping it on later reclaims retroactively, + since the missing tombstones schedule exactly the outstanding clean jobs. + The tombstone is written *before* anything is deleted, so a crash can cost + disk but never the record. +- **A finished tile declares no reclaimed exposures.** Deleting an exposure's + manifests would otherwise rerun every other tile that reads it, and those + reruns spread across the exposure-overlap component. So a tile whose + `final_cat` exists drops the exposure manifests that are gone from its input + list, and holds the rest through `ancient()`. This is why the profile runs + with `rerun-triggers: [mtime, params, code, software-env]`: the `input` + trigger reads that cut as a reason to rerun the very tiles it protects. + Know the consequence — `--forcerun` on a tile whose `final_cat` exists will + not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **A dead tile can be told to stop pinning exposures.** An exposure is + cleanable only once every consuming tile has its vignets, so one + permanently-failed tile holds its ~80 exposures for the life of the + campaign. List it under `clean_ignore_tiles:` in `config.yaml` and it leaves + the consumer sets. Retrying an ignored tile later is legal and expensive: + its exposure chains are gone and rebuild from scratch. +- **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the + absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its + per-runner counts and blocks no tile. The logs go with the manifests — a log + claiming `complete` for a store that is gone would contradict the unbuilt + chain the DAG must now see, and its content duplicates the manifest anyway. + The `exp_psf` benchmark tsv lives beside both dirs, not inside either, so + reclamation does not eat the memory-sizing data. +- **Failure is a report, not a gate.** `run_report.py` disk-scans the trees + against the count table and enumerates shortfalls (whole-unit absence vs + per-CCD attrition). It runs standalone — a DAG report node would itself be + poisoned by the failures it must enumerate — and fires automatically from + the COMPUTE invocation's `onsuccess`/`onerror` hooks, or on demand via + `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 000000000..bbdfeefb6 --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,634 @@ +"""ShapePipe real-data orchestration — Snakemake workflow. + +Design / rationale: CosmoStat/shapepipe#848, D1-D5. + +`sp run` is TWO snakemake invocations over this one Snakefile: + + SP_PHASE=prepare snakemake prepare_all_tiles # Git -> Uz -> Fe, per tile + SP_PHASE=compute snakemake all # everything else + +They are two because the exposure job set is *data-derived*: it comes from the +tiles' find_exposures output, and a Snakemake DAG is fixed at parse time. The +join between them is the tile<->exposure index, built HERE at parse time of +invocation 2 (build_index.build(), imported — there is no `sp index` verb) and +loaded into plain dicts. The index is never a rule input, so appending tiles and +rebuilding it changes which jobs exist without invalidating completed work; it +ACCUMULATES across invocations, so a later clean_exposure (S5) sees every +consuming tile of the whole campaign, not just this tile list. + +Neither invocation reads the checkout: `sp run` snapshots the code into the +campaign's state dir and points snakemake at THAT tree, so `workflow.basedir` +below is the snapshot and mid-campaign edits are inert. The mechanism and its +rationale live in bin/sp. + +The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared +output is its MANIFEST (`/manifests/.json`), written by +`completeness.py check`. Product files are not declared — a missing CCD is often +legitimate, and at DR6 scale per-CCD declaration means millions of paths. + +Failure evidence rides on the rule's `log:` (`/logs/.json`): +the manifest says "this stage succeeded", the log says "here is what happened" +(the contract is argued in completeness.py's docstring). +""" + +import functools +import hashlib +import json +import os +import sqlite3 +import sys +from pathlib import Path + +from snakemake.exceptions import WorkflowError + +# Resolved relative to THIS file, not the working directory: snakemake runs with +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project +# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +configfile: str(Path(workflow.snakefile).parent / "config.yaml") + +# Every job's shell runs inside this container (apptainer software-deployment in +# the profile); the user never types apptainer. WHICH image is the one resolution +# order `sp container` exposes — this user's writable sandbox if they built one, +# else their cached SIF if they pulled one, else `container:` above. An empty +# cache therefore lands on exactly the shared /project image the workflow has +# always run, and a package installed into a sandbox reaches the jobs too. +sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import container as _container # noqa: E402 + +# Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to +# resolve at all (empty cache AND no `container:` key). The empty string the +# resolver returns for "none" is a valid-looking container directive that passes +# dry-run and only fails once jobs reach a node. +try: + _image, _kind = _container.resolve_image() +except _container.ContainerError as _exc: + raise WorkflowError(str(_exc)) +if _kind == "none": + raise WorkflowError( + f"No container image resolved: no sandbox, no cached SIF, and no " + f"'{_container.CONFIG_KEY}:' key in {_container.CONFIG_FILE}. " + f"Run `sp container pull`, or restore the key.") + +container: _image + +PSF_MODELS = {"psfex", "mccd"} +PSF_MODEL = config.get("psf_model", "psfex") +if PSF_MODEL not in PSF_MODELS: + raise WorkflowError( + f"Invalid psf_model={PSF_MODEL!r}; expected one of " + f"{sorted(PSF_MODELS)}.") + +# --- paths ----------------------------------------------------------------- +# RUN_DIR is the scratch root: bulk intermediates, sized so a batch finishes +# inside the purge window. PRODUCTS_DIR is the persistent root +# for the durable, low-volume products — the final catalogues, the index, the +# report. Snakemake's own state is the one durable-looking thing that stays on +# scratch (bin/sp explains why: tens of thousands of small, hot metadata writes +# against a backed-up, file-count-limited filesystem). +INPUTS = config["inputs"] +OUTPUTS = config["outputs"] +RUN_DIR = Path(OUTPUTS["run_dir"]) +# Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no +# second path: one root, exactly the pre-D5 layout. +PRODUCTS_DIR = Path(OUTPUTS.get("products_dir") or RUN_DIR) +INDEX_DB = Path(OUTPUTS["index_db"]) +SCRIPTS = Path(workflow.basedir) / "scripts" +# The config chain is the repo's committed directory (D2). The configs and +# rules that set their environment variables must be versioned together. There +# is no `config_src` knob. +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" + +sys.path.insert(0, str(SCRIPTS)) +import build_index # noqa: E402 +from completeness import STAGE_DIR # noqa: E402 + +# SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two +# invocations of `sp run`, `passthrough` on direct commands (`sp --unlock`, `sp +# --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index +# build and the report hooks — so that a passthrough parse never mutates durable +# state or dies on a threshold it was not asked about. +# +# Requiring it to be SET is the point of the check below: a bare `snakemake` +# would otherwise parse as a passthrough, find `rule all` gated on an index it +# never built, and exit 0 on an empty DAG. Jobs re-parse this file under the +# slurm executor and inherit the head process's environment, so a submitted job +# always carries the launching phase. +PHASE = os.environ.get("SP_PHASE", "") +if not PHASE: + raise WorkflowError( + "SP_PHASE is not set — run the workflow through workflow/bin/sp " + "(`sp run`), which exports it. A bare snakemake invocation would " + "silently build an empty DAG.") +if PHASE not in ("prepare", "compute", "passthrough"): + raise WorkflowError( + f"SP_PHASE={PHASE!r} is not one of prepare, compute, passthrough.") + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- ngmix scatter (D4) ---------------------------------------------------- +# Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one +# ngmix job per tile. We take the count and drive our own integer `chunk` +# wildcard rather than snakemake's `{scatteritem}` ("3-of-8") token, because the +# chunk number is not ours alone: it names the run dir +# (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and +# merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the +# chunk number in that run-directory name — which only works for bare integers. +scattergather: + ngmix=int(config.get("ngmix_chunks", 8)) + +NGMIX_CHUNKS = workflow._scatter["ngmix"] + +# --- parse-time index build + load (D1) ------------------------------------ +# The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY +# there — no "some Fe output exists" guard. That guard used to make a +# totally-failed prepare produce an empty index, an empty DAG and a green exit 0; +# with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the +# SP_MISSING_THRESHOLD gate. +# +# In every other phase (prepare, or unset for a passthrough invocation) the parse +# builds NOTHING and only loads whatever index is already on disk. +# +# is_main_process matters as much as PHASE: the SLURM executor re-invokes +# snakemake inside every job, that re-invocation parses this file again, and it +# inherits SP_PHASE=compute from the submitting environment. Without the guard, +# every one of the run's jobs re-runs the build — hundreds of concurrent sqlite +# writers on Lustre, which previously caused "database is locked" errors +# (2026-07-31). Job parses only LOAD the index below. +if PHASE == "compute" and workflow.is_main_process: + build_index.build( + TILES, RUN_DIR, INDEX_DB, + missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) + +# EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes +# verbatim into the fabricated per-unit exp_numbers list so get_images matches +# .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. +# EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure +# is keyed on (D5). +# +# SLURPED ONLY IN THE HEAD PROCESS. The executor re-parses this file inside every +# job (~800 per run), and each of those parses needs at most its own unit's edges +# — reading the whole campaign's tile_exposures table there is pure waste that +# grows with the campaign rather than with the job. Job parses get the same three +# lookups backed by memoised single-row queries instead (the table's primary key +# indexes both), so every input function returns exactly what it would have. +EXP, TILE_EXP, EXP_TILES = {}, {}, {} +if INDEX_DB.exists() and workflow.is_main_process: + # timeout=60: Lustre lock handoffs are slow; the default 5 s trips on + # nothing more sinister than a reader in another job's parse. + _con = sqlite3.connect(INDEX_DB, timeout=60) + EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) + for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(_tile, []).append(_exp) + EXP_TILES.setdefault(_exp, []).append(_tile) + _con.close() + + +if workflow.is_main_process: + + def exp_name(exp): + """The exposure's original name (get_images matches .fits.fz).""" + return EXP[exp] + + def tile_exposures(tile): + """This tile's exposure base-ids, sorted as the index stores them.""" + return TILE_EXP.get(tile, []) + + def exp_consumers(exp): + """Every tile in the campaign that reads this exposure (D5).""" + return EXP_TILES.get(exp, []) + +else: + # One connection for the whole parse, and lru_cache over it: a job parse asks + # about a handful of units, each of them several times (params, input, log). + _JOB_CON = sqlite3.connect(INDEX_DB, timeout=60) if INDEX_DB.exists() else None + + @functools.lru_cache(maxsize=None) + def exp_name(exp): + row = _JOB_CON.execute( + "SELECT name FROM exposures WHERE exp_id = ?", (exp,)).fetchone() + if row is None: + raise KeyError(exp) # same failure as EXP[exp] would give + return row[0] + + @functools.lru_cache(maxsize=None) + def tile_exposures(tile): + if _JOB_CON is None: + return [] + # ORDER BY rowid, not by exp_id: the eager slurp above is a table scan, + # so its lists are in INSERTION order, and the two paths must hand the + # DAG the same list and not merely the same set. + return [r[0] for r in _JOB_CON.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id = ? ORDER BY rowid", + (tile,))] + + @functools.lru_cache(maxsize=None) + def exp_consumers(exp): + if _JOB_CON is None: + return [] + # Unindexed on exp_id (the PK leads with tile_id), so this is a scan — + # harmless because the only caller, clean_consumers, is reachable from + # the head process alone (clean_exposure is a localrule). + return [r[0] for r in _JOB_CON.execute( + "SELECT tile_id FROM tile_exposures WHERE exp_id = ? ORDER BY rowid", + (exp,))] + +# Tiles this run can actually compute: declared AND indexed. The index spans the +# campaign, so it is intersected with the declared list, not used as it. +TILES_READY = [t for t in TILES if tile_exposures(t)] +READY_SET = set(TILES_READY) + +# A compute invocation with nothing to compute is never a success. Without this, +# an empty intersection yields `rule all` with no inputs, an empty DAG and exit +# 0 — the silent green run the threshold gate exists to prevent. +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} tiles " + f"are ready to compute (index {INDEX_DB}). Run the prepare phase first " + f"(`sp run`), or check {INDEX_DB.parent / 'missing.json'}.") + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", + chunk = r"\d+", + +# --- the sharded stores (D2) ---------------------------------------------- +# tiles/<2-char prefix>// and exp/<2-char prefix>// — no directory +# exceeds ~1k entries at full-UNIONS scale. Rules carry the shard as its own +# wildcard because an output pattern cannot compute it; every path the DAG uses +# is built by these helpers, so a mismatched (shard, id) pair is never requested. +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") +# The persistent root mirrors the scratch one, shard for shard, so the two trees +# read as the same campaign seen from two filesystems. +PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") + +def tile_dir(tile): + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" + +def exp_dir(exp): + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def final_cat(tile): + """The campaign's science product, and its tile-finished marker. + + It lives on the PERSISTENT root, not the scratch one, and both halves of + that sentence are load-bearing. As a product: it is what the campaign is + for, and a 60-day purge must not eat it. As a marker: tile_finished() keys + on this path to cut a finished tile's exposure edges (D5), so if a purge + could remove it, finished tiles would re-declare inputs against exposure + stores that reclamation deleted — the rerun avalanche the cut exists to + prevent, arriving by way of the purge instead. + """ + return f"{PRODUCTS_DIR}/tiles/{tile[:2]}/{tile}/final_cat-{tile}.fits" + +def unit_num(unit): + """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading + dash (tile ``210.282`` -> ``-210-282``; exposure ``2605805`` -> ``-2605805``). + The RULES do this transform; the configs just interpolate $SP_UNIT_NUM into + NUMBER_LIST (the `set_config_number_list` mechanism that replaced the retired + -e/--exclusive flag, #746).""" + return "-" + unit.replace(".", "-") + +# Content hash of completeness.py, computed once at parse time and carried as a +# param on every rule: the default rerun-triggers' `code` trigger hashes only the +# rule's own shell string, NOT external scripts it calls — without this, a fix to +# the count table silently leaves stale manifests in place (bitten live). Scoped +# to completeness.py alone, the one script every shell line runs; build_forest.py +# gets its own hash, on the forest rule only. +def script_hash(name): + """The 12-hex fingerprint of one script under workflow/scripts/.""" + return hashlib.md5((SCRIPTS / name).read_bytes()).hexdigest()[:12] + +SCRIPT_HASH = script_hash("completeness.py") +FOREST_HASH = script_hash("build_forest.py") +CLEAN_HASH = script_hash("clean_exposure.py") +CLEAN_TILE_HASH = script_hash("clean_tile.py") +# ngmix_range.py earns a hash for a stronger reason than the others. What it +# emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are +# a PARTITION of its object IDs: resume a tile across an edit to the split and +# the chunks that already succeeded keep the old ranges while the reruns take the +# new ones, so within one tile some objects are measured twice and others by +# nobody. merge_sep_cats concatenates whatever it is handed, so the tile +# completes green with a corrupt catalogue and no error anywhere. +# +# ACROSS TIME IS ALL THIS GUARDS NOW. The within-group version of the same +# corruption — eight sibling chunks each computing the split and disagreeing — is +# structurally closed: the partition is materialised ONCE per tile, by +# tile_vignets, and the chunks only look their row up (tile.smk's +# TILE_NGMIX_RANGES, which also keeps the incident transcript). So this hash buys +# REPRODUCIBILITY of the file across attempts and resumes, not agreement between +# siblings. It therefore fingerprints the script that PRODUCES the file, and +# is used by both tile_vignets and tile_ngmix because either rule can produce +# the file. +# +# Land this hash, and any later edit to the split, at a campaign boundary on a +# fresh root: a resume across it re-measures the tile, and the failure modes a +# mid-campaign params change can reach are catalogued at tile.smk's range_hash. +NGMIX_RANGE_HASH = script_hash("ngmix_range.py") + +# --- exposure reclamation (D5, S5) ----------------------------------------- + + +def flag(value, default=False): + """Truthiness for a config value that may arrive as a STRING. + + `--config clean=false` delivers the string "false", and every non-empty + string is truthy in Python — a plain bool() read that as ON and scheduled + the deletions the user had just switched off. YAML booleans pass through + unchanged; only strings are parsed, and an unparseable one is an error, not + a guess. + """ + if value is None: + return default + if isinstance(value, str): + v = value.strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off", ""): + return False + raise WorkflowError(f"Cannot read {value!r} as a boolean (use true/false).") + return bool(value) + + +# `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no +# jobs at all (nothing requests a tombstone); flipping it on later reclaims +# RETROACTIVELY, because the exposures already cleaned are exactly the ones with +# a tombstone, so the missing tombstones schedule exactly the clean jobs. +# Only ever active under SP_PHASE=compute: the prepare and passthrough parses +# read no index of their own and must schedule no deletions. +CLEAN = flag(config.get("clean", False)) and PHASE == "compute" + +# Reclamation REQUIRES the `input` rerun-trigger to be off (see tile.smk's +# tile_finished cascade note). The gate is structural rather than asserted: CLEAN +# needs SP_PHASE=compute, only bin/sp sets it, and bin/sp always launches with +# profiles/nibi — so the incompatible combination is unreachable. A runtime +# assertion is impossible (workflow.dag_settings is None until the DAG is built). + +# Tiles that must not pin an exposure's store. See config.yaml: a permanently +# failed tile otherwise holds every exposure it touches (~80) forever, because +# its vignets manifest will never exist and its exposures are therefore never +# eligible. Listing it here drops it from the consumer sets. +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) + + +def tombstone(exp): + """The clean_exposure output. Lives BESIDE manifests/ and logs/, not inside + either: the clean job deletes both wholesale, and `sp report` scans it.""" + return f"{exp_dir(exp)}/cleaned.json" + + +def clean_consumers(exp): + """Every tile in the campaign that reads this exposure — the set whose + vignets must all exist before the store may go — minus the ignored tiles.""" + return sorted(t for t in exp_consumers(exp) + if t not in CLEAN_IGNORE_TILES) + + +def clean_targets(): + """Which exposures this invocation may clean. + + An exposure is eligible only when every consuming tile is either in this + run's scope or has already produced its vignets on disk. Without that test, + requesting a tombstone for an exposure shared with a LATER batch would drag + that batch's whole tile chain into this DAG through the clean rule's input — + cleanup would otherwise expand the scope. Ineligible + exposures are simply skipped; the invocation that finishes their last + consumer picks them up. Deferral, never loss. + + Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in + `clean_ignore_tiles` neither gates eligibility nor appears in the job's + input — which is the purpose of that list. + + HEAD PROCESS ONLY. This feeds `rule all` at module level, so it runs on every + parse — including the ~800 per-job re-parses under the slurm executor, none of + which can ever schedule `all`. Each of those was stat()ing the whole + campaign's tile_vignets manifests for nothing. The empty list a job parse + returns costs it exactly the target it could not have used anyway. + """ + if not CLEAN or not workflow.is_main_process: + return [] + out = [] + for exp, raw in EXP_TILES.items(): + if not raw: + continue + # An empty set AFTER filtering means every consumer is ignored: nothing + # is left that could ever read this exposure, so it is eligible now. + # all() of an empty set is True, so this case is eligible. + tiles = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() + for t in tiles): + out.append(tombstone(exp)) + return sorted(out) + +# --- tile reclamation (D5) -------------------------------------------------- +# A separate flag from `clean:` (config.yaml carries the full +# argument): exposure reclamation costs nothing but a rebuild if a tile is +# appended later, while tile reclamation destroys the per-tile AUDIT TRAIL that +# the campaign's cost model was derived from. They are not the same decision and +# must not be made by the same switch. Gated on SP_PHASE=compute exactly as +# CLEAN is, and read through flag() for the same reason (`--config +# clean_tiles=false` arrives as the string "false"). +CLEAN_TILES = flag(config.get("clean_tiles", False)) and PHASE == "compute" + + +def tile_tombstone(tile): + """The clean_tile output. Beside manifests/ and logs/, not inside either — + same placement and same reason as the exposure tombstone() above.""" + return f"{tile_dir(tile)}/cleaned.json" + + +def clean_tile_targets(): + """Which tiles this invocation may clean: the in-scope ones, and no test. + + No eligibility test, because a tile's store has no consumer outside that + tile (clean_tile.py argues the asymmetry with clean_targets()). Every ready + tile is eligible the moment its own final_cat exists, and the rule's input + is exactly that. + + The restriction to TILES_READY is therefore scope containment, not + eligibility, and it runs the opposite way from the exposure case: an + OUT-OF-SCOPE tile's tombstone would drag that tile's final_cat, its + tile_make_cat job and the whole fused tile_shape group into this DAG, where + the `params` rerun-trigger can find a finished tile out of date and rerun it + against reclaimed exposure stores (see the TILE_LOCAL warning in tile.smk). + In scope the edge is free: `rule all` already requests the same final_cat. + Nothing is lost by skipping an out-of-scope tile — the tile list only ever + grows, so a later invocation has it in scope. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. + """ + if not CLEAN_TILES or not workflow.is_main_process: + return [] + return sorted(tile_tombstone(t) for t in TILES_READY) + +# --- the shell every rule runs (D2) ---------------------------------------- + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1), + ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) + + +def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, + pre_run=()): + """The unit-furniture + environment prologue, as bash. + + Returned as a rule ``params`` value, NEVER inlined into the ``shell:`` + string: snakemake formats a shell string ONCE, so a ``{output}``/``{threads}`` + placeholder inside a params value would survive literally — and, conversely, + the literal ``${SP_NGMIX_CHUNK}`` braces this prologue needs would blow up + that formatting if they lived in the shell string. Params values are + substituted after formatting, so both hazards go away together. + + What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT + -e/--exclusive): + * ``output/``, ``manifests/`` and ``logs/`` — the last two are a SIBLING + pair, not one dir with two naming conventions, because clean_exposure + deletes both wholesale (clean_exposure.py says why). The benchmark tsv + is deliberately outside both: it measures the job, not the store, so + reclamation must not eat it. + * tile: ``tile_numbers.txt`` (dot format — what get_images reads); + * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the + ORIGINAL exposure name from the index (``2605805p``), so get_images + matches ``.fits.fz`` in the store — the bare base id matches + nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale + pre-fix file with the bare id. + There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at + the committed config dir. + + It also exports the configured input roots as ``SP_INPUT_TILES`` and + ``SP_INPUT_EXPOSURES`` and the PSF choice as ``SP_PSF`` for the committed + ini chain. + + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's + FileHandler raises on an existing run dir, and it is how a rerun never sees + stale products (D2: the job clears its run dir at start). + """ + # The stage table already knows both halves of "where this stage writes": + # its level and its run dir. Taking them from there rather than from an + # argument means a caller cannot disagree with completeness.py about which + # store a stage belongs to. + level, subdir = STAGE_DIR[stage] + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_UNIT_NUM='{unit_num(unit)}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + f"export SP_INPUT_TILES='{INPUTS['tiles']}'", + f"export SP_INPUT_EXPOSURES='{INPUTS['exposures']}'", + f"export SP_PSF='{PSF_MODEL}'", + # Also set via apptainer-args in the profile; kept here so a hand-run of + # this same line outside snakemake behaves identically. + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + for k, v in (env or {}).items(): + lines.append(f"export {k}='{v}'") + + if level == "tile": + lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + else: + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + + lines += list(pre_run) + lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] + return "\n".join(lines) + + +def sp_shell(stage, config_name, *, check_args="", post=""): + """The rule's shell string: prologue, one shapepipe_run, one completeness check. + + EVERY rule that runs shapepipe_run goes through here, including the two that + need a little more than the plain body — they pass it rather than restating + it, so the rc composition below has exactly one definition: + * ``check_args`` — extra flags for the completeness check. tile_vignets + points it at the NODE-LOCAL run root (``--run-dir "$SP_LOCAL"``) and keeps + the manifest's unit field the tile ID rather than the local root's + basename (``--unit``), which also keeps the manifest content independent + of an ephemeral path the mtime rerun-trigger reads. + * ``post`` — bash between the check and the ``exit``. tile_make_cat + publishes its final_cat there, guarded on ``rc``. + + ``{threads}``, ``{output}`` and ``{log}`` are placeholders HERE and nowhere + else (see unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task + one number by construction (D4). + + The rc is CAPTURED rather than ``&&``-ed onto, so the check still runs when + shapepipe_run failed, and it is passed back as ``--job-rc "$rc"`` so the + verdict composes the count checks with shapepipe_run's own exit status. Both + halves are argued in completeness.py's docstring. + """ + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" + f"{check_args}" + " --log {log} --job-rc \"$rc\" || rc=1\n" + f"{post}" + "exit $rc\n" + ) + + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" + +# --- top-level targets ------------------------------------------------------ +# The aggregation targets, clean_exposure and clean_tile run in the head +# process. Both clean rules are seconds of rmtree and hang off `all`; submitted +# they would be ~20k (clean_exposure) or ~23k (clean_tile) sbatch submissions at +# DR6 scale for work shorter than the scheduling latency. +# +# Both are DAG LEAVES, so neither constrains a `group:` label. (A mid-chain +# localrule would: a local job cannot be fused into a submitted group. The old +# star-catalogue rules were exactly that, and they are gone with the internal +# mask generation.) +localrules: all, prepare_all_tiles, clean_exposure, clean_tile + +rule all: + input: + [final_cat(t) for t in TILES_READY], + clean_targets(), + clean_tile_targets(), + +# Invocation 1 — the static per-tile DAG, known from the tile list alone. +# keep-going makes tile failures independent; the ones that lose their exposure +# list are dropped by the index build at invocation 2's parse. +rule prepare_all_tiles: + input: + [tile_manifest(t, "tile_find_exposures") for t in TILES] + +# --- report hooks ----------------------------------------------------------- +# run_report is NOT a DAG node (a descendant of every job would be poisoned by +# any hard failure — the exact case it exists for). It is a standalone script, +# emitted automatically at the end of the COMPUTE invocation, runnable any time +# via `sp report`. +def _report(status): + shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " + f"--index {INDEX_DB} --status {status} || true") + +if PHASE == "compute": + + onsuccess: + _report("success") + + onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp new file mode 100755 index 000000000..fdc3d4f59 --- /dev/null +++ b/workflow/bin/sp @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). +# +# Three verbs, nothing else: +# +# sp run [ARGS...] bring the products on disk up to date with the tile list. +# Snapshots the code into the state dir first (see "the +# launch code snapshot" below) and runs out of the copy, so +# editing the checkout mid-campaign cannot reach the jobs. +# Two snakemake invocations over one Snakefile: +# 1. PREPARE snakemake prepare_all_tiles +# 2. COMPUTE snakemake all <- its PARSE builds the index +# ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# sp report [ARGS...] emit run_report.json now (mid-run is fine). +# sp container VERB manage the image every job runs inside: pull, status, +# sandbox, exec, resolve. `sp container --help` documents +# the layers and the resolution order. +# +# Anything else is passed straight through to snakemake with the same profile and +# state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). +# +# It also loads the apptainer module (snakemake resolves `apptainer` via PATH at +# job runtime) and activates the snakemake venv on the shared /project FS (the +# executor re-invokes it inside jobs, so it cannot live on a node-local path). +# One entry point, so a fresh tmux or a restart after a crash always launches +# with the right state. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ +REPO="$(dirname "$HERE")" +VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" +SCRIPTS="$HERE/scripts" +CONFIG="$HERE/config.yaml" + +module load apptainer/1.4.5 2>/dev/null || true +# shellcheck disable=SC1091 +source "$VENV/bin/activate" + +# Scalar reader for workflow/config.yaml. The venv is active by this point and +# snakemake depends on PyYAML, so this parses the file rather than pattern-matching +# it -- quotes, inline comments and nesting are the parser's problem, not ours. +cfg() { python -c 'import sys, yaml +value = yaml.safe_load(open(sys.argv[1])) +for key in sys.argv[2].split("."): + value = value[key] +print(value or "")' "$CONFIG" "$1"; } +RUN_DIR="$(cfg outputs.run_dir)"; INDEX_DB="$(cfg outputs.index_db)" + +# Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO +# THE RUN on /scratch, never on the persistent root — the one exception to D5's +# "durable, low-volume products go on /project". It is neither: one small, hot +# metadata file per output, rewritten on every job, which at DR6 is >170k files +# of churn against a backed-up filesystem with a 1M-inode group quota. It is also +# reconstructible — losing it costs a re-parse, not a re-run. (A hard 27/27 TiB +# group quota killed a metadata write mid-run, live. The quota has since eased; +# the placement is right on its own merits.) +# --directory only moves state: all data paths are absolute, and the Snakefile +# resolves its own configfile. +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" + +# --- the launch code snapshot ---------------------------------------------- +# THE ONE HOME for this concept; everything else points here. +# +# WHY. A campaign is not one process. The SLURM executor re-invokes snakemake on +# every job's node, so each job RE-PARSES the Snakefile, and each shell reads +# workflow/scripts/*.py and workflow/config/cfis/* when it STARTS -- hours after +# launch, out of whatever the checkout contains THEN. Editing the checkout +# mid-campaign therefore fed different code to different jobs of the same run +# (the 186.307 partition incident: an edit landed four seconds into a tile split +# it into two disagreeing halves). +# +# WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and +# runs the campaign entirely out of that copy: the Snakefile, the rules, the +# scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points +# into example/, and the copy must be self-contained), src/, and the profile. +# Every workflow-internal path hangs off `workflow.basedir`, which IS the +# snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the +# one that cannot (YAML splices nothing) and is rewritten below. +# +# CONSEQUENCE, and the reason the old "never edit workflow/ while a campaign +# runs" warnings are gone: mid-campaign edits to the checkout are inert. A change +# takes effect on the next `sp run`, and not before. +# +# LIFECYCLE: one snapshot per campaign state dir, REFRESHED WHOLESALE on every +# `sp run` -- a new `sp run` IS the relaunch, and snakemake's params/code rerun +# triggers decide what the new code invalidates. Nothing accumulates, and the +# path is stable, so a later `sp --unlock` reaches the same workflow that took +# the lock. +SNAPSHOT="$STATE_DIR/code" +snapshot_code() { + mkdir -p "$SNAPSHOT" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ + "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + else + rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" + cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + + fi + + # The profile's `apptainer-args:` hardcodes one checkout's src/ in its PYTHONPATH + # pin and CANNOT interpolate anything (snakemake escapes a literal `$` before the + # string reaches any shell -- the post-mortem is on that line). So the pin is + # rewritten HERE, in the snapshot's copy, and `sm` points --profile at that copy. + # The checked-in profile keeps its literal checkout path and stays the source of + # truth for the FLAGS; exactly one path is substituted, by one line of sed-work. + python - "$SNAPSHOT/profiles/nibi/config.yaml" "$SNAPSHOT/src" <<'PY' +import pathlib, re, sys +f, src = pathlib.Path(sys.argv[1]), sys.argv[2] +text, n = re.subn(r'(--env PYTHONPATH=)\S+', lambda m: m.group(1) + src, + f.read_text(), count=1) +if not n: + sys.exit("sp: no --env PYTHONPATH= found in the profile's apptainer-args; " + "the snapshot would run against the live checkout's src/") +f.write_text(text) +PY + + python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' +import json, pathlib, subprocess, sys, time +out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] +def git(*a): + try: + p = subprocess.run(["git", "-C", repo, *a], capture_output=True, + text=True, timeout=30) + except OSError: + return None + return p.stdout.strip() if p.returncode == 0 else None +status = git("status", "--porcelain") +out.write_text(json.dumps({ + "_comment": "Code snapshot taken by `sp run`; the campaign runs out of THIS " + "tree, never the checkout. Refreshed wholesale on every `sp run`.", + "source": repo, + "taken_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "head": git("rev-parse", "HEAD"), + "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(status), + "dirty_files": status.splitlines() if status else [], +}, indent=2) + "\n") +PY + echo "sp: code snapshot refreshed at $SNAPSHOT" >&2 +} + +# SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of +# declared tiles allowed to be missing their exposure list (default 0.0). +export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" + +# WHICH code a verb runs. `sp run` refreshes the snapshot and runs out of it; +# every other verb and the escape hatch run out of the EXISTING snapshot when the +# state dir has one -- `sp --unlock` and `sp report` must speak for the campaign +# that is actually on disk, not for a checkout that has moved since. Before the +# first `sp run` there is no snapshot and the live checkout is all there is. +# (`sp container` is the exception: it is interactive tooling about the image, +# not about a campaign, and stays on the checkout.) +code_root() { [ -f "$SNAPSHOT/workflow/Snakefile" ] && echo "$SNAPSHOT" || echo "$REPO"; } + +# --snakefile pins the workflow to one tree: sp must work from any cwd (an sbatch +# head job starts in the submission directory, not the repo), and the SLURM +# executor re-invokes snakemake with this same path inside every job -- which is +# exactly what makes the snapshot stick. +# +# SP_PHASE is REQUIRED by the Snakefile (a bare `snakemake` would build an empty +# DAG and exit 0). `sp run` sets prepare/compute on its two invocations; every +# other verb and the escape hatch fall through to `passthrough`, which parses the +# index without building it and schedules no side effects. +sm() { + local root; root="$(code_root)" + SP_PHASE="${SP_PHASE:-passthrough}" \ + snakemake --snakefile "$root/workflow/Snakefile" \ + --profile "$root/profiles/nibi" \ + --directory "$STATE_DIR" "$@" +} + +cmd="${1:-}" +case "$cmd" in + run) + shift + snapshot_code + # PREPARE failing is NOT fatal to the run: keep-going means a failed tile + # poisons only its own cone, and the tiles that lost their exposure list are + # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which + # the compute parse's index build enforces over the WHOLE tile list — so we + # record the failure and go on rather than letting `set -e` abort here. + prep_rc=0 + SP_PHASE=prepare sm prepare_all_tiles "$@" || prep_rc=$? + if [ "$prep_rc" -ne 0 ]; then + echo "" >&2 + echo "############################################################" >&2 + echo "## WARNING: the PREPARE phase exited $prep_rc." >&2 + echo "## Some tiles may be missing their exposure list and will be" >&2 + echo "## dropped from the compute DAG. Continuing to COMPUTE; the" >&2 + echo "## SP_MISSING_THRESHOLD gate (now $SP_MISSING_THRESHOLD) decides" >&2 + echo "## whether that is tolerable." >&2 + echo "############################################################" >&2 + echo "" >&2 + fi + + comp_rc=0 + SP_PHASE=compute sm all "$@" || comp_rc=$? + if [ "$prep_rc" -ne 0 ] || [ "$comp_rc" -ne 0 ]; then + exit "$([ "$comp_rc" -ne 0 ] && echo "$comp_rc" || echo "$prep_rc")" + fi + ;; + report) + # Out of the snapshot when there is one, so a manual report reads the trees + # with the same script the campaign's own onsuccess/onerror hooks use (they + # call it through the Snakefile's SCRIPTS, which is the snapshot). Read-only + # either way -- this is consistency, not safety. + shift + python "$(code_root)/workflow/scripts/run_report.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + --status manual "$@" + ;; + container) + # The image layer, outside snakemake entirely: the script is stdlib-only so + # it runs on the bare host, and the apptainer module is already loaded above. + # Deliberately the CHECKOUT's copy, not the snapshot's: `sp container` is + # about the image you are working with now (status compares against HEAD, and + # the snapshot has no .git), and `exec` should hand you the environment your + # NEXT run will have. It reads the checked-in profile's apptainer-args, so a + # running campaign whose snapshot predates an edit to that line gets the old + # value while `exec` shows the new one -- the only place the two can differ, + # and it differs in the direction that matters (jobs stay pinned). + shift + python "$SCRIPTS/container.py" "$@" + ;; + cancel) + # Kept only because it is two lines: scancel this workflow's jobs by name + # before an --unlock. Not part of the design surface. + run="${2:?usage: sp cancel }" + squeue --me --noheader --format='%i %j' \ + | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" + ;; + *) + sm "$@" + ;; +esac diff --git a/workflow/config.yaml b/workflow/config.yaml new file mode 100644 index 000000000..63b29413c --- /dev/null +++ b/workflow/config.yaml @@ -0,0 +1,142 @@ +# Run configuration for the ShapePipe Snakemake workflow. +# +# A "run" is declared by a tile list plus the paths below. Everything here is +# read at parse time; none of it is a rule input, so editing it (e.g. appending +# tiles) never invalidates completed work — it only changes which jobs exist. + +# The tile list that scopes this run (one "IDra.IDdec" per line). +# The campaign grows by appending to this file — parse-time config, so +# completed work is never invalidated. Current contents: smk-g5, ONE tile +# (186.307), the paired control for the two fixes landed after smk-g4 — the +# node-local WCS sqlite and tile_ngmix's mem_mb 14000 -> 5000. That tile ran +# ALONE as smk-g4 job 20799387 (elapsed 8063 s, chunk median 6813 s, chunk mean +# load 59%, MaxRSS 26.6 GiB), so the baseline carries no concurrency confound +# and the contrast is same tile, same solitude, two commits apart. +# The 34-tile smk-g4 set is preserved at smk-g4/tiles34.txt. +# (The 210/211 quad used earlier is unusable: its tile images are symlinks into +# anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are +# broken.) +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt + +# Inputs are the only site-specific data paths; all committed configs consume these +# through SP_INPUT_TILES and SP_INPUT_EXPOSURES. +inputs: + # Pre-staged P3 data (get_images RETRIEVE=symlink). + tiles: /project/def-mjhudson/unions-wl/tiles + exposures: /project/def-mjhudson/unions-wl/exposures + +# The container every job runs inside (apptainer software-deployment in the profile). +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif + +# PSF model used by the exposure and tile interpolation stages. +psf_model: psfex + +# THE TWO ROOTS (D5). +# +# run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk +# intermediates, sized so a batch finishes inside the 60-day purge window. The +# sharded per-unit stores live under it: +# /tiles/<2-char prefix>// and /exp/// +outputs: + run_dir: /scratch/cdaley/shapepipe-output/smk-g6 + +# products_dir is the PERSISTENT root: the durable, low-volume products — the +# final catalogues (/tiles///final_cat-.fits, +# mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB +# per tile, so a full DR6 campaign is a few hundred GB. +# +# The final catalogue is also the tile-finished MARKER that cuts a finished +# tile's exposure edges (D5), which is the second reason it cannot sit on +# scratch: a purge would not merely lose a product, it would make every finished +# tile re-declare inputs against exposure stores reclamation already deleted. +# +# Unset means "one root": products land under run_dir, exactly the pre-D5 +# layout, which is what a fixture or smoke test wants. +# +# Snakemake's own state is the one durable-looking thing that stays on scratch +# (-state; bin/sp explains why). + products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 + +# There is no config_src knob: the config chain is workflow/config/cfis, resolved +# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / +# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules +# are one artefact and must version together, so the dir is fixed by construction. + +# The run index, and — sharing its directory — missing.json and run_report.json. +# On the persistent root with the catalogues (D5): the index is the record of +# which tile reads which exposure, so it is what a post-purge reconstruction +# would otherwise have to rebuild from tile headers. + index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite + +# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_exposure` job per exposure. It fires once every campaign tile that reads +# that exposure has its vignets, deletes the exposure's store AND its manifests, +# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them +# back out of the tombstone and reports the exposure as `cleaned`. OFF for smk-g5: +# this is a one-tile control, its ~8 exposure stores cost ~54 GiB against 979 GiB +# free, reclamation is already measured exactly by smk-g4, and keeping the stores +# means a re-measurement re-runs only the shape chain instead of the whole tile. +clean: true + +# Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_tile` job per in-scope tile, ordered after that tile's own final_cat, so +# a tile's scratch store goes as soon as its chain lands rather than at the end +# of the batch. It deletes the tile's whole /tiles/// tree +# except four things another mechanism owns — `cleaned.json`, +# `manifests/tile_vignets.json` (clean_exposure's eligibility currency), +# `manifests/tile_find_exposures.json` (prepare_all_tiles' target) and the Fe +# exposure list under output/ (build_index re-reads it at every compute parse). +# clean_tile.py argues each one. +# +# A SEPARATE SWITCH FROM `clean:` ABOVE, AND NOT AN OVERSIGHT. Exposure +# reclamation is reversible in the only sense that matters: a tile appended later +# rebuilds the exposure chain from VOS, expensively but completely. TILE +# reclamation DESTROYS THE PER-TILE AUDIT TRAIL and nothing rebuilds it. +# TWO tools read it: sp-products/tools/sp_tilecost.py and sp_costmodel.py. Both +# attribute the fused tile_shape group job's cost per tile from that tile's +# scratch store — the SExtractor catalogue (NAXIS2 of the sexcat is the object +# count, the cost model's independent variable; sp_costmodel also reads its +# EPOCH_k extensions for the geometric epoch count) and the eight tile_ngmix +# benchmark TSVs — and that model is how every mem_mb and runtime number in +# tile.smk was derived. The sexcat is ~380 MB per tile and cannot be kept; the +# benchmark ROWS are absorbed into the tombstone, so the measurements survive +# but not at the paths the tools read. +# +# (The third artifact they read, run_sp_tile_ngmix_Ngu/, is NOT lost to this +# flag: tile_ngmix declares its chunk dir temp(), so snakemake reclaims it as +# soon as tile_merge_cats runs. It is already absent from every finished tile, +# clean_tiles or not.) +# +# The honest one-line summary: REQUIRED FOR ANY BATCH OVER ~850 TILES, AND IT +# TURNS OFF PER-TILE COST ATTRIBUTION. The arithmetic: a finished tile leaves +# 1.19 GiB across 137 inodes (measured on 186.307), so a 1 TiB scratch quota is +# full at 859 tiles, and DR6's 23,114 tiles would want 26.9 TiB and 3.17M inodes +# against a 1M quota. Reclaimed, a tile costs 10 inodes and 12.9 KB — 231k inodes +# and ~300 MB for the whole of DR6. +# +# OFF here for the same reason `clean:` is: smk-g5 is a one-tile control, and its +# store is what a re-measurement would read. +clean_tiles: true + +# Tiles that may NOT pin an exposure store (default: empty). +# +# An exposure is eligible for cleaning only once EVERY consuming tile has its +# vignets. One permanently-failed tile therefore holds all its exposures +# (~8 measured on P3: 7.9 exposures/tile) for the life of the campaign. A tile listed here is dropped from the +# consumer sets, and its exposures become eligible. +# +# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its +# exposures' stores. If you later retry that tile, those exposure chains are +# gone and will be REBUILT from scratch — get_images, split, psf, per +# exposure. That is correct, and expensive. Ignore a tile when you have decided +# it is dead, not while you are still debugging it. +clean_ignore_tiles: [] + +# ngmix within-tile chunking: static N chunks (closed ID ranges computed +# per-tile, in-job, from the tile's own sexcat). +ngmix_chunks: 8 + +# The container's installed shapepipe is overridden by the prod worktree via +# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# build_index.py's missing-tile fraction threshold is passed by workflow/bin/sp +# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/example/cfis/config_MCCD.ini b/workflow/config/cfis/config_MCCD.ini similarity index 100% rename from example/cfis/config_MCCD.ini rename to workflow/config/cfis/config_MCCD.ini diff --git a/example/cfis/config_exp_Gie_vos.ini b/workflow/config/cfis/config_exp_Gie.ini similarity index 92% rename from example/cfis/config_exp_Gie_vos.ini rename to workflow/config/cfis/config_exp_Gie.ini index 26f608b53..b0cce78e8 100644 --- a/example/cfis/config_exp_Gie_vos.ini +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -11,7 +11,7 @@ VERBOSE = False RUN_NAME = run_sp_exp_Gie # Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True +RUN_DATETIME = False ## ShapePipe execution options @@ -55,7 +55,7 @@ TIMEOUT = 96:00:00 # Get exposures [GET_IMAGES_RUNNER] -INPUT_DIR = last:find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output FILE_PATTERN = exp_numbers @@ -72,7 +72,7 @@ NUMBERING_SCHEME = -000-000 # Input path where original images are stored. Can be local path or vos url. # Single string or list of strings -INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags +INPUT_PATH = $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES # Input file pattern including tile number as dummy template INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag @@ -87,7 +87,7 @@ INPUT_NUMBERING = \d{6} OUTPUT_FILE_PATTERN = image-, weight-, flag- # Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = vos +RETRIEVE = symlink # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/example/cfis/config_exp_Sp.ini b/workflow/config/cfis/config_exp_Sp.ini similarity index 86% rename from example/cfis/config_exp_Sp.ini rename to workflow/config/cfis/config_exp_Sp.ini index 066c21e54..dd27d6ccd 100644 --- a/example/cfis/config_exp_Sp.ini +++ b/workflow/config/cfis/config_exp_Sp.ini @@ -12,7 +12,7 @@ VERBOSE = True RUN_NAME = run_sp_exp_Sp # Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = True +RUN_DATETIME = False ## ShapePipe execution options @@ -34,6 +34,10 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN INPUT_DIR = . @@ -55,7 +59,7 @@ TIMEOUT = 96:00:00 [SPLIT_EXP_RUNNER] -INPUT_DIR = last:get_images_runner +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output FILE_PATTERN = image, weight, flag diff --git a/example/cfis/config_exp_mccd.ini b/workflow/config/cfis/config_exp_mccd.ini similarity index 97% rename from example/cfis/config_exp_mccd.ini rename to workflow/config/cfis/config_exp_mccd.ini index 092df0417..635952ca7 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/workflow/config/cfis/config_exp_mccd.ini @@ -37,7 +37,7 @@ LOG_NAME = log_sp RUN_LOG_NAME = log_run_sp # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . +INPUT_DIR = $SP_RUN/output # Output directory OUTPUT_DIR = $SP_RUN/output @@ -114,7 +114,7 @@ BKG_FROM_HEADER = False #BKG_KEY = IMMODE # Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES CHECKIMAGE = BACKGROUND, BACKGROUND_RMS @@ -228,7 +228,7 @@ POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE GET_SHAPES = True # Directory with PSF models -PSF_MODEL_DIR = /Users/tliaudat/Documents/PhD/codes/venv_p3/MCCD_pipeline_integration/test_val_data/fitted_model/ +PSF_MODEL_DIR = $SP_RUN/output # PSF model patterns PSF_MODEL_PATTERN = fitted_model diff --git a/example/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini similarity index 63% rename from example/cfis/config_exp_psfex.ini rename to workflow/config/cfis/config_exp_psfex.ini index 199050927..d81080a11 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -1,5 +1,8 @@ # ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -9,19 +12,17 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_SxSePsfPi -#RUN_NAME = run_sp_exp_SxSePsf +RUN_NAME = run_sp_exp_SxSePsf # Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - +MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI MODE = SMP @@ -57,12 +58,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Input from two modules -#INPUT_DIR = last:split_exp_runner, run_sp_exp_Ma:mask_runner -INPUT_DIR = last:split_exp_runner, last:mask_runner +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag # Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on # the decorator's 4-entry FILE_EXT default (length check fails at startup) @@ -127,16 +127,50 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] -INPUT_DIR = last:sextractor_runner +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsf/sextractor_runner/output -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets MASK_EXT. Comma-separated. +# +# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue +# is passed through with no MASK_EXT column, the same gating make_cat gives +# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning +# the query on is uncommenting one line and never editing the chain. +# +# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive +# (instrument flags only) and the column is carried for transparency and +# measurement — see star_selection.setools' header for the one-line change +# that would impose it. +; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/mask_query_runner/output + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/example/cfis/config_tile_Fe.ini b/workflow/config/cfis/config_tile_Fe.ini similarity index 86% rename from example/cfis/config_tile_Fe.ini rename to workflow/config/cfis/config_tile_Fe.ini index 2661259d7..9546f062e 100644 --- a/example/cfis/config_tile_Fe.ini +++ b/workflow/config/cfis/config_tile_Fe.ini @@ -11,7 +11,7 @@ VERBOSE = False RUN_NAME = run_sp_tile_Fe # Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True +RUN_DATETIME = False ## ShapePipe execution options @@ -33,6 +33,10 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names INPUT_DIR = $SP_RUN @@ -55,7 +59,7 @@ TIMEOUT = 96:00:00 # Get tiles [FIND_EXPOSURES_RUNNER] -INPUT_DIR = last:get_images_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output FILE_PATTERN = CFIS_image diff --git a/example/cfis/config_tile_Git_vos.ini b/workflow/config/cfis/config_tile_Git.ini similarity index 95% rename from example/cfis/config_tile_Git_vos.ini rename to workflow/config/cfis/config_tile_Git.ini index 52fbdc15b..cccf0a8b9 100644 --- a/example/cfis/config_tile_Git_vos.ini +++ b/workflow/config/cfis/config_tile_Git.ini @@ -11,7 +11,7 @@ VERBOSE = False RUN_NAME = run_sp_tile_Git # Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True +RUN_DATETIME = False ## ShapePipe execution options @@ -66,7 +66,7 @@ NUMBERING_SCHEME = # Input path where original images are stored. Can be local path or vos url. # Single string or list of strings -INPUT_PATH = vos:cfis/tiles_DR6, vos:cfis/tiles_DR6 +INPUT_PATH = $SP_INPUT_TILES, $SP_INPUT_TILES # Input file pattern including tile number as dummy template INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight @@ -81,7 +81,7 @@ INPUT_NUMBERING = \d{3}\.\d{3} OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- # Copy/download method, one in 'vos', 'symlink' -RETRIEVE = vos +RETRIEVE = symlink # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/example/cfis/config_make_cat_psfex_nosm.ini b/workflow/config/cfis/config_tile_Mc.ini similarity index 74% rename from example/cfis/config_make_cat_psfex_nosm.ini rename to workflow/config/cfis/config_tile_Mc.ini index 1983f91c7..5920d0a91 100644 --- a/example/cfis/config_make_cat_psfex_nosm.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -9,10 +9,10 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Mc +RUN_NAME = run_sp_tile_Mc # Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options @@ -56,12 +56,20 @@ TIMEOUT = 96:00:00 [MAKE_CAT_RUNNER] # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:merge_sep_cats_runner +# The psfex_interp store is NODE-LOCAL: it is the vignette-store run this +# tile's fused group job wrote to node-local storage (see config_tile_PiViVi_.ini +# and workflow/rules/tile.smk). $NGMIX_VIGNET_DIR names it; the other two +# inputs and this run's own output stay on $SP_RUN. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $SP_RUN/output/run_sp_tile_Ms/merge_sep_cats_runner/output # Input file pattern(s), list of strings with length matching number of expected input file types # Cannot contain wild cards FILE_PATTERN = sexcat, galaxy_psf, ngmix +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282) +NUMBER_LIST = $SP_UNIT_NUM + # FILE_EXT (optional) list of string extensions to identify input files FILE_EXT = .fits, .sqlite, .fits diff --git a/example/cfis/config_tile_Mh_exp.ini b/workflow/config/cfis/config_tile_Mh_exp.ini similarity index 89% rename from example/cfis/config_tile_Mh_exp.ini rename to workflow/config/cfis/config_tile_Mh_exp.ini index 0d3f9f8b3..96512df1a 100644 --- a/example/cfis/config_tile_Mh_exp.ini +++ b/workflow/config/cfis/config_tile_Mh_exp.ini @@ -36,6 +36,10 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN INPUT_DIR = . @@ -58,7 +62,7 @@ TIMEOUT = 96:00:00 [MERGE_HEADERS_RUNNER] # Input: exp_numbers txt file from find_exposures_runner -INPUT_DIR = last:find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output FILE_PATTERN = exp_numbers diff --git a/example/cfis/config_merge_sep_cats_template.ini b/workflow/config/cfis/config_tile_Ms.ini similarity index 62% rename from example/cfis/config_merge_sep_cats_template.ini rename to workflow/config/cfis/config_tile_Ms.ini index 36b99e8da..08397399a 100644 --- a/example/cfis/config_merge_sep_cats_template.ini +++ b/workflow/config/cfis/config_tile_Ms.ini @@ -8,10 +8,10 @@ VERBOSE = True # Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Ms +RUN_NAME = run_sp_tile_Ms # Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options @@ -33,11 +33,19 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +# NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner +# derives the other chunks' paths itself from N_SPLIT_MAX and this pattern +# (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the +# same fixed output dir. +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output # Output directory -OUTPUT_DIR = ./output +OUTPUT_DIR = $SP_RUN/output ## ShapePipe job handling options @@ -68,5 +76,10 @@ NUMBERING_SCHEME = -000-000 # display/ignore warnings, and not raise error WARNING = always -# Maximum number of separated catalogues per input -N_SPLIT_MAX = X +# Maximum number of separated catalogues per input. +# merge_sep_cats_runner.py reads this with getexpanded (runner .py line ~31), so +# $NGMIX_N_CHUNKS DOES expand here, exactly like ID_OBJ_MIN/MAX in the ngmix +# module. The tile_merge_cats rule exports it from the workflow's scattergather +# chunk count, which is the single source of truth; this value must equal that +# count, so do not replace it with a literal. +N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini new file mode 100644 index 000000000..04606636d --- /dev/null +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -0,0 +1,125 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +# Per-chunk run dir: the workflow exports SP_NGMIX_CHUNK= for each chunk. +# RUN_NAME is env-expanded (run.py getexpanded); braces keep the trailing "u" +# out of the variable name. +RUN_NAME = run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +# INPUT and OUTPUT are independent here, and that is what makes node-local +# staging possible: [FILE] OUTPUT_DIR stays $SP_RUN/output (scratch, durable +# for the life of the job's tile), while the two directories ngmix actually +# READS AT RANDOM -- the psfex_interp galaxy_psf store and the vignetmaker +# run_2 image/background/weight/flag vignet stores, together the ~5.6 GB +# per-tile vignette store that all 8 chunks re-read -- come from +# $NGMIX_VIGNET_DIR. +# +# $NGMIX_VIGNET_DIR is the tile's run_sp_tile_PiViVi directory. tile_ngmix +# (workflow/rules/tile.smk) copies that directory to the job's node-local +# NVMe ($SLURM_TMPDIR) and points this variable at the copy; the outputs +# below are unaffected and still land on scratch. Set it to +# $SP_RUN/output/run_sp_tile_PiViVi to read the store in place. +# +# It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset $NGMIX_VIGNET_DIR is +# a loud error rather than a silent read of the wrong tree. +# +# $SP_WCS_DIR is the fourth input and the same idea applied to the file the +# vignette store left behind. log_exp_headers-.sqlite is 11.3 MB and +# ngmix reads it once per object PER EPOCH for the WCS; on NFS that is a +# network round trip per read, and thread-state sampling of the fused job +# measured every chunk 44% blocked in `rpc_wait_bit_killable` on this one file +# with everything else already node-local. tile_local() copies it beside the +# store and points this variable at the copy. Set it to +# $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output to read in place. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as +# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for +# every tile (missing file -> error, no per-tile fallback); omit the option +# entirely to fall back to the scalar sigma_mad noise estimate. +BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite + +# Number of objects to batch save during processing, optional. Omit or set +# to -1 for no batch saving. +# 250 (folded from runs/p3-batch1/cfis): worker RSS grows ~4.8 MB/object +# until the flush recycles it -> peak ~1.1 GB + 250 x 4.8 MB ~= 2.3 GB/worker +# (A/B test, job 17607877). Not superseded by -b {threads} (that sets fork +# width, this bounds per-worker memory). +SAVE_BATCH = 250 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, +# computed at execution time from the tile's own object count and expanded +# via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not +# plain getint). +ID_OBJ_MIN = $NGMIX_ID_MIN +ID_OBJ_MAX = $NGMIX_ID_MAX diff --git a/example/cfis/config_tile_PiViVi_canfar_uc.ini b/workflow/config/cfis/config_tile_PiViVi_mccd.ini similarity index 65% rename from example/cfis/config_tile_PiViVi_canfar_uc.ini rename to workflow/config/cfis/config_tile_PiViVi_mccd.ini index d59af1c4a..d312ca494 100644 --- a/example/cfis/config_tile_PiViVi_canfar_uc.ini +++ b/workflow/config/cfis/config_tile_PiViVi_mccd.ini @@ -1,5 +1,5 @@ # ShapePipe configuration file for tile, from detection up to shape measurement. -# PSFEx PSF model. +# MCCD PSF model. ## Default ShapePipe options @@ -12,14 +12,16 @@ VERBOSE = True RUN_NAME = run_sp_tile_PiViVi # Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner +#MODULE = mccd_interp_runner, + +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -34,11 +36,31 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names INPUT_DIR = . -# Output directory -OUTPUT_DIR = $SP_RUN/output +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT ## ShapePipe job handling options @@ -53,9 +75,9 @@ TIMEOUT = 96:00:00 ## Module options -[PSFEX_INTERP_RUNNER] +[MCCD_INTERP_RUNNER] -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output FILE_PATTERN = sexcat, log_exp_headers, exp_numbers @@ -76,27 +98,17 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD # If True, measure and store ellipticity of the PSF GET_SHAPES = True -# Number of stars threshold -STAR_THRESH = 20 - -# chi^2 threshold -CHI2_THRESH = 2 - -# Multi-epoch mode parameters - -# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR -# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered -# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +# MCCD models are discovered in the per-exposure stores listed by SP_EXP. ME_DOT_PSF_EXP_DIR = $SP_EXP +ME_DOT_PSF_RUNNER = mccd_fit_val_runner +ME_DOT_PSF_PATTERN = fitted_model -# Input psf file pattern -ME_DOT_PSF_PATTERN = star_split_ratio_80 - +# Multi-epoch mode parameters # Create vignets for tiles weights [VIGNETMAKER_RUNNER_RUN_1] -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner, last:uncompress_fits_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output FILE_PATTERN = sexcat, CFIS_weight @@ -130,7 +142,7 @@ PREFIX = weight # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures -INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output FILE_PATTERN = sexcat, log_exp_headers, exp_numbers diff --git a/example/cfis/config_tile_PiViVi_canfar_sx.ini b/workflow/config/cfis/config_tile_PiViVi_psfex.ini similarity index 70% rename from example/cfis/config_tile_PiViVi_canfar_sx.ini rename to workflow/config/cfis/config_tile_PiViVi_psfex.ini index cb72e1c11..27a759528 100644 --- a/example/cfis/config_tile_PiViVi_canfar_sx.ini +++ b/workflow/config/cfis/config_tile_PiViVi_psfex.ini @@ -12,7 +12,7 @@ VERBOSE = True RUN_NAME = run_sp_tile_PiViVi # Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options @@ -21,7 +21,7 @@ RUN_NAME = run_sp_tile_PiViVi # Module name, single string or comma-separated list of valid module runner names #MODULE = psfex_interp_runner, -MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner, vignetmaker_runner +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -36,11 +36,31 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names INPUT_DIR = . -# Output directory -OUTPUT_DIR = $SP_RUN/output +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT ## ShapePipe job handling options @@ -57,7 +77,7 @@ TIMEOUT = 96:00:00 [PSFEX_INTERP_RUNNER] -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output FILE_PATTERN = sexcat, log_exp_headers, exp_numbers @@ -79,7 +99,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 @@ -98,7 +118,7 @@ ME_DOT_PSF_PATTERN = star_split_ratio_80 # Create vignets for tiles weights [VIGNETMAKER_RUNNER_RUN_1] -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output FILE_PATTERN = sexcat, CFIS_weight @@ -132,7 +152,7 @@ PREFIX = weight # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures -INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output FILE_PATTERN = sexcat, log_exp_headers, exp_numbers @@ -166,39 +186,3 @@ PREFIX = ME_IMAGE_EXP_DIR = $SP_EXP ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner ME_IMAGE_PATTERN = flag, image, weight, background, background_rms - - -[VIGNETMAKER_RUNNER_RUN_3] - -# Cut per-object coadd-frame segmentation stamps from the tile SExtractor -# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, -# SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode -# guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN -# centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg -# stamp directly for uberseg neighbour masking (shapepipe#776). - -INPUT_DIR = run_sp_tile_Sx:sextractor_runner - -FILE_PATTERN = sexcat, segmentation - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# CLASSIC: cut stamps at object positions in the coadd (tile) frame. -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels — MUST equal the coadd VIGNET size (51) so the seg -# stamp overlays the galaxy stamp on an identical grid. -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = seg diff --git a/example/cfis/config_tile_Sx_nomask.ini b/workflow/config/cfis/config_tile_Sx.ini similarity index 87% rename from example/cfis/config_tile_Sx_nomask.ini rename to workflow/config/cfis/config_tile_Sx.ini index 731a8d338..5487f547c 100644 --- a/example/cfis/config_tile_Sx_nomask.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -11,7 +11,7 @@ VERBOSE = True RUN_NAME = run_sp_tile_Sx # Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options @@ -34,6 +34,10 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN INPUT_DIR = $SP_RUN/output @@ -55,7 +59,7 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers @@ -97,7 +101,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/example/cfis/config_tile_Uz.ini b/workflow/config/cfis/config_tile_Uz.ini similarity index 86% rename from example/cfis/config_tile_Uz.ini rename to workflow/config/cfis/config_tile_Uz.ini index 3521f0aca..fc8550af4 100644 --- a/example/cfis/config_tile_Uz.ini +++ b/workflow/config/cfis/config_tile_Uz.ini @@ -11,7 +11,7 @@ VERBOSE = True RUN_NAME = run_sp_tile_Uz # Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = True +RUN_DATETIME = False ## ShapePipe execution options @@ -33,6 +33,10 @@ LOG_NAME = log_sp # Runner log file name, optional, default: shapepipe_runs RUN_LOG_NAME = log_run_sp +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + # Input directory, containing input files, single string or list of names INPUT_DIR = . @@ -53,7 +57,7 @@ TIMEOUT = 96:00:00 ## Module options [UNCOMPRESS_FITS_RUNNER] -INPUT_DIR = last:get_images_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output FILE_PATTERN = CFIS_weight diff --git a/example/cfis/default.conv b/workflow/config/cfis/default.conv similarity index 100% rename from example/cfis/default.conv rename to workflow/config/cfis/default.conv diff --git a/example/cfis/default.param b/workflow/config/cfis/default.param similarity index 100% rename from example/cfis/default.param rename to workflow/config/cfis/default.param diff --git a/example/cfis/default.psfex b/workflow/config/cfis/default.psfex similarity index 100% rename from example/cfis/default.psfex rename to workflow/config/cfis/default.psfex diff --git a/example/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex similarity index 100% rename from example/cfis/default_exp.sex rename to workflow/config/cfis/default_exp.sex diff --git a/example/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param similarity index 100% rename from example/cfis/default_noimaflags.param rename to workflow/config/cfis/default_noimaflags.param diff --git a/example/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex similarity index 100% rename from example/cfis/default_tile.sex rename to workflow/config/cfis/default_tile.sex diff --git a/example/cfis/final_cat.param b/workflow/config/cfis/final_cat.param similarity index 100% rename from example/cfis/final_cat.param rename to workflow/config/cfis/final_cat.param diff --git a/example/cfis/star_selection.setools b/workflow/config/cfis/star_selection.setools similarity index 61% rename from example/cfis/star_selection.setools rename to workflow/config/cfis/star_selection.setools index 8330a1eff..32197b466 100644 --- a/example/cfis/star_selection.setools +++ b/workflow/config/cfis/star_selection.setools @@ -1,4 +1,32 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties +## +## ONE mask cut, and it is the instrument flags: +## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation) +## delivered with the exposure and read by SExtractor. +## +## That is deliberate, and it follows from what the two kinds of mask MEAN. +## An instrument flag marks a CORRUPTED MEASUREMENT — the pixels carry no +## usable signal — so a flagged star is not a star we could model badly, it is +## one we cannot model at all. The healsparse masks are sky-fixed LOCATION +## flags (a star halo, a manual region, a band with no data): they say where an +## object sits, not that its pixels are broken, so whether one disqualifies a +## PSF star is a judgement, not a fact about the data. +## +## So the star selection deliberately starts from OUTLIER REJECTION ALONE for +## that judgement. mask_query still queries the external masks per detection +## and writes MASK_EXT into this catalogue — carried for transparency and +## measurement, so their effect on the star sample can be measured before it is +## imposed. Flag transparently, cut downstream. +## +## MASK_EXT is the configurable pickup if outlier rejection proves +## insufficient. To impose it, add +## +## MASK_EXT == 0 +## +## beside each IMAFLAGS_ISO line below — one line per mask block, and that is +## the whole change. Which maps reach MASK_EXT is mask_query's MASK_PATHS +## config, which ships commented out; SETools has no bitwise operators, so the +## bit selection happens there and this file only ever tests for zero. [MASK:preselect] MAG_AUTO > 0 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk new file mode 100644 index 000000000..4e1ef45a4 --- /dev/null +++ b/workflow/rules/exposure.smk @@ -0,0 +1,153 @@ +"""Exposure chain — per exposure, keyed by exp base id (dedup is structural). + + exp_get_images -> exp_split -> exp_psf + +Each in the exposure's own sharded work dir, chained by manifests; every config +reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a +run log. There is no `prepare_exposures` aggregation target: these chains hang +off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). + +NO MASK RULE, and that is the design (PR #847). ShapePipe generates no masks. +The only mask that reaches pixels is the instrument flag image delivered with +the exposure, which ``exp_split`` splits per CCD alongside image and weight and +SExtractor reads directly. Sky-fixed masks are healsparse maps, queried once per +object: ``mask_query`` (inside exp_psf's config chain) writes ``FLAG_EXT`` onto +each CCD's SExtractor catalogue for setools' star cut, and ``make_cat`` writes +the per-band ``MASK_`` columns on the tile side. Neither needs a rule, a +star catalogue, or a network fetch — hence no ``star_catalogue`` / ``exp_star_cat`` +here, and no ``exp_mask``. + +NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by +construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, +not over one invocation — reclamation here is clean_exposure's job (S5), driven +by the accumulating index. A temp() here would delete an exposure the moment +this invocation's readers finished and cascade destructive reruns across spatial +neighbours the next time a tile is appended. + +NO GROUPING. The ``exp_short`` group existed to fuse exp_split and exp_mask — +two rules whose medians were 1:28 and 1:54, both well under the 15-minute floor +Alliance policy asks us to bundle away — into one sbatch per exposure. With +exp_mask gone there is nothing to fuse: a group of one rule submits exactly the +job the ungrouped rule submits, and the label would only obscure that. The +composition rules, should a second short rule ever appear here, are in +prepare.smk's docstring. exp_get_images stays separate for the same reason it +always did (a download, retried on its own), and exp_psf is heavy (16 GB, 4 h) +and never fuses with a short rule. + +NUMBER_LIST ($SP_UNIT_NUM, see unit_num in the Snakefile) is set only for +exp_split, whose numbering scheme IS the exposure id; never for get_images / +exp_psf, whose per-CCD or download numbering would turn tolerated per-CCD +attrition into a whole-exposure hard failure. It is a property of the committed +configs (config_exp_Sp.ini alone carries the entry). +""" + +rule exp_get_images: + output: + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" + log: + f"{EXP_DIR}/logs/exp_get_images.json" + params: + pre = lambda wc: unit_pre("exp_get_images", wc.exp, + exp_name=exp_name(wc.exp)), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("exp_get_images", "config_exp_Gie.ini") + +# Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the +# tiles' merge_headers reads). +rule exp_split: + input: + rules.exp_get_images.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_split.json" + log: + f"{EXP_DIR}/logs/exp_split.json" + params: + pre = lambda wc: unit_pre("exp_split", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("exp_split", "config_exp_Sp.ini") + +# SExtractor -> mask_query (FLAG_EXT) -> setools star selection -> PSFEx model +# -> psfex_interp, per CCD. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's +# :warn on psfex_interp_runner. +rule exp_psf: + input: + rules.exp_split.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_psf.json" + log: + f"{EXP_DIR}/logs/exp_psf.json" + params: + pre = lambda wc: unit_pre("exp_psf", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + retries: 2 + benchmark: + # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ + # wholesale, and this tsv is the measured-memory feed for mem_mb sizing + # (D4). Inside manifests/ it died with the first reclamation and took + # the campaign's only record of exp_psf's real footprint with it. + f"{EXP_DIR}/exp_psf.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 240 + shell: + sp_shell("exp_psf", f"config_exp_{PSF_MODEL}.ini") + + +# --- reclamation (D5) ------------------------------------------------------- +# The one exception to "no reclamation in this file": clean_exposure OWNS +# exposure-level deletion, and it is a real job, not temp() bookkeeping, because +# an exposure's consumer set closes over the CAMPAIGN. The index supplies that +# set (EXP_TILES, accumulated across invocations); the input is every consuming +# tile's tile_vignets manifest — vignets is the last stage that reads exposure +# products, everything after it reads tile-level files. +# +# What the job deletes, and why a late append still behaves, is argued in +# clean_exposure.py's docstring; params.consumers is what makes a grown consumer +# set stale (same file). +# +# The tile side reads the exposure manifests through ancient() and cuts the +# reclaimed edges of finished tiles (see tile.smk), which is what keeps this +# deletion from rebuilding every neighbouring tile. This rule's OWN inputs are +# deliberately not ancient: a tile that really did rebuild its vignets must +# reschedule the cleans of the exposures it read. +# +# A localrule (declared in the Snakefile). Local execution serialises the cleans +# under local-cores, which costs nothing at rmtree speed and never blocks the +# compute chains (this rule is in none of them). +rule clean_exposure: + input: + # ONLY the consumers this invocation may actually build. A consumer that + # is out of scope had its vignets manifest checked for existence at parse + # time (clean_targets' eligibility test) — declaring it here as well would + # pull that finished tile's whole chain into the DAG, where a rebuilt + # shared exposure then reruns it. That is how one damaged tile reached its + # spatial neighbours. In-scope consumers keep their edge: they may run in + # this DAG, so the clean must be ordered after them. + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp) if t in READY_SET] + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)), + script_hash = CLEAN_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk new file mode 100644 index 000000000..7e99fe2ae --- /dev/null +++ b/workflow/rules/prepare.smk @@ -0,0 +1,93 @@ +"""Invocation 1 — PREPARE: one static chain per tile. + + tile_get_images -> tile_uncompress -> tile_find_exposures + +Known from the tile list alone, cheap, wide, idempotent. find_exposures parses +the tile FITS HISTORY header into ``exp_numbers--.txt`` — the +data-derived tile->exposure edge that invocation 2's parse aggregates into the +index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node +tier). + +All three rules carry ``group: "tile_prep"``, so one tile's whole chain is ONE +sbatch instead of three (medians 0:41 / ~0:40 / 0:15 — all far under the +15-minute runtime limit Alliance policy asks us to bundle away, and at DR6 scale three +submissions per tile is a scheduler load out of all proportion to the work). +Group membership is per connected DAG component and distinct tiles share no +edge, so this is exactly one group job per tile, never a cross-tile bundle. +Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. + +Group resource composition (snakemake 9.23, ``GroupResources.basic_layered`` in +snakemake/resources.py): jobs are laid out per toposort level; within a level +non-additive resources (mem_mb, cpus) SUM — split into layers when a global +constraint is exceeded, the group's width being the widest layer — while the +additive resource ``runtime`` is maxed within a layer and SUMMED across layers. +This chain is strictly linear, one job per level, so the group asks for +max(mem_mb) = 8000*attempt, max(threads) = 4 and sum(runtime) = 150 min. +Attempt scaling survives grouping: ``GroupJob.attempt``'s setter clears the +cached group resources and re-sets ``attempt`` on every member (jobs.py), and +``GroupJob.restart_times`` is the max over members — so tile_get_images' +``retries: 2`` still governs. A retry re-runs the whole group, which is safe +because every rule ``rm -rf``s its own run dir at start. + +There is no masking node in this phase, or in any other: ShapePipe generates no +masks (PR #847). The instrument flag image ships with the exposure and is split +per CCD by ``exp_split``; the sky-fixed healsparse masks are queried per object +inside the ShapePipe configs (``mask_query`` on exposures, ``make_cat`` on +tiles). Nothing is fetched, staged or rasterized, so there is nothing to +prepare. +""" + +# No NUMBER_LIST for get_images — a download stage has nothing on disk to +# validate against (see exposure.smk's docstring for the convention). +rule tile_get_images: + group: "tile_prep" + output: + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" + log: + f"{TILE_DIR}/logs/tile_get_images.json" + params: + pre = lambda wc: unit_pre("tile_get_images", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("tile_get_images", "config_tile_Git.ini") + +rule tile_uncompress: + group: "tile_prep" + input: + rules.tile_get_images.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" + log: + f"{TILE_DIR}/logs/tile_uncompress.json" + params: + pre = lambda wc: unit_pre("tile_uncompress", wc.tile), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 + shell: + sp_shell("tile_uncompress", "config_tile_Uz.ini") + +rule tile_find_exposures: + group: "tile_prep" + input: + rules.tile_uncompress.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" + log: + f"{TILE_DIR}/logs/tile_find_exposures.json" + params: + pre = lambda wc: unit_pre("tile_find_exposures", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 + shell: + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk new file mode 100644 index 000000000..68d6da9ba --- /dev/null +++ b/workflow/rules/tile.smk @@ -0,0 +1,900 @@ +"""Tile post-chain — per tile: gather exposures, then detect / PSF / shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> tile_vignets -> tile_ngmix x N + -> tile_merge_cats -> tile_make_cat + +The DAG edge to the exposures is always the exposures' MANIFESTS, looked up +through the index (TILE_EXP). The per-tile exposure "forest" — a symlink view of +exactly this tile's exposures' products — exists only so the ShapePipe configs +have one deterministic ``$SP_EXP`` to glob; it is NEVER the edge. Its 2-char +shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires +``///output/run_sp_*`` into its glob, so a flat forest +makes every tile gather stage fail "No split_exp_runner output found". + +All rules are group-compatible (shell only, no mid-chain localrules), and the two +short regions are grouped, per the composition rules in prepare.smk's docstring. +Distinct tiles share no edge, so each is one group job per tile: + +* ``group: "tile_gather"`` — tile_exp_forest (2 GB, 20 min) and + tile_merge_headers (median 0:38; 8 GB, 4 threads, 120 min). The group asks max + mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also + consumes the forest, but a consumer OUTSIDE the group is just an ordinary DAG + edge on the group job — it does not pull tile_detect (16 GB) in. +* ``group: TILE_GROUP`` ("tile_shape") — tile_vignets -> 8 x tile_ngmix -> + tile_merge_cats -> tile_make_cat, THE WHOLE SHAPE CHAIN AS ONE JOB. This is + not a latency optimisation; it is what lets the 5.6 GB vignette store live on + node-local NVMe and never touch /scratch (see tile_local() below). + Composition, verified against snakemake 9.23.1 and a real sbatch: + cpus = max over levels of summed siblings = max(8, 8x1, 8, 8) = 8; + mem_mb = max(32000, 8x5000, 16000, 16000) = 40000, so nibi's + max(cores, mem_GB/4) bills the group 10 core-equivalents rather than the 28 + it billed at 8x14000 (see the mem_mb note on tile_ngmix for the 31-tile + measurement that sized it); tile_vignets' own 32000 is now the second term + and becomes binding if the chunks ever go below 4000; + runtime = sum along the chain of each level's MAX = 20 + 120 + 10 + 15 = 165. + 165 min is under the 180 min ceiling of ``cpubase_bycore_b1``, so the fused + job reaches the widest partition set (plus cpubackfill) — which is why each + member's runtime is measured p99 plus margin and not the old ceiling. + +The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor +run that the shape chain does not need co-scheduled, and folding it in would add +its runtime to a sum that has no room. + +There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe +generates no masks: tiles have no instrument flag image of their own, so +tile_detect runs SExtractor with FLAG_IMAGE = False against +default_noimaflags.param (config_tile_Sx.ini — what used to be the "sx_nomask" +variant, now the only one). Sky-fixed masks reach the tile as CATALOGUE columns +instead: ``tile_make_cat``'s make_cat queries the configured healsparse maps at +every object's (RA, Dec) and writes one ``MASK_`` column per band, which +is what downstream selections cut on. +""" + +# --- the node-local tile root (the I/O + scratch fix) ---------------------- +# +# MEASURED PROBLEM. tile_ngmix was I/O-bound, not CPU-bound: median 7h34m +# elapsed against 52 min of CPU (3% efficiency, 0.12 of 4 reserved cores). +# Each chunk read ~20 GB at 0.78 MB/s and all 8 chunks of a tile re-read the +# SAME 5.6 GB vignette store -- ~163 GB of small random sqlite preads per tile +# against /scratch, which on nibi is NFS (VAST, 4.5 PB, 95% full). What hurts +# is per-read LATENCY, not bandwidth: the sequential local-vs-scratch gap is +# only ~3x and is NOT the argument for this change. +# +# THE FIX IS THE FUSE. tile_vignets -> 8 x tile_ngmix -> tile_merge_cats -> +# tile_make_cat run as ONE group job on ONE node, and the vignette store is +# WRITTEN to that node's NVMe and never lands on /scratch at all. So the store +# is not copied, it is simply never remote: zero staging cost, zero NFS random +# reads, and -- the reason this was chosen over per-chunk staging -- the +# per-tile scratch high-water drops by the whole 5.6 GB store (~190 GiB across +# 34 concurrent tiles). Scratch quota, not speed, is what caps batch size. +# +# WHAT STAYS ON SHARED STORAGE, and it is everything that is DAG currency: +# every manifest (the success sentinels), every verdict log, the ngmix chunk +# dirs merge_sep_cats gathers, and final_cat on the PERSISTENT root. Only the +# bulk intra-tile intermediate is node-local. That split is possible because +# ShapePipe's configs set input and output paths independently -- see +# config_tile_PiViVi_.ini (OUTPUT_DIR = $SP_VIGNET_OUT) and +# config_tile_Ng_template.ini and config_tile_Mc.ini ($NGMIX_VIGNET_DIR), and +# config_tile_Ng_template.ini alone for $SP_WCS_DIR. +# +# TWO node-local stores, not one: the ~5-8 GB vignette store and the 11.3 MB +# WCS sqlite, the same fix applied at the two ends of the size distribution +# (tile_local()'s docstring carries the thread-state measurement). +# +# WHY IT NEEDS THE GROUP. The store is node-local, so it exists only on the +# machine that wrote it and only while that job lives. Unfused, tile_vignets and +# tile_ngmix are different jobs on (usually) different nodes, and the store would +# have to be on shared storage -- which is the whole problem. Fused, all members +# run in one allocation on one node, so the store the first member writes is +# simply there for the rest. (Verified live: group members share the directory.) +# +# HOW THE PATH GETS IN HERE. Not through the environment: profiles/nibi passes +# --bind /local and tile_local() below DERIVES the path from the tile wildcard. +# Why nothing can be communicated instead is on that profile line. +# +# THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, +# not one chunk, because the store dies with the job. At ~1 h per fused tile +# that is a cheap trade for the scratch it buys. Noted, not engineered around. +# +# It lives in each rule's `pre_run`, i.e. in THAT RULE's params.pre. The shared +# prologue (unit_pre) and completeness.py are untouched by design: both are +# fingerprinted into every rule, and changing either would invalidate the whole +# campaign, including the 117 finished exposure chains. +# +# EDITING THIS FUNCTION INVALIDATES FINISHED TILES. READ THIS BEFORE YOU DO. +# The returned string lands in `params.pre`, `params` is an active rerun trigger +# in profiles/nibi/config.yaml, and `params.pre` is a non-derived param (its +# lambda takes only `wc`), so snakemake records it and compares it. Any edit +# here therefore reschedules tile_vignets, all eight tile_ngmix chunks and +# tile_make_cat for EVERY tile the campaign already finished. +# +# On a fresh campaign that is free. On a RESUME it is destructive, and the +# mechanism is worth spelling out because it is not obvious: +# * clean_exposure has already reclaimed the exposure stores those tiles read, +# and tile_finished() deliberately drops the manifests of a finished tile's +# exposures, so the rerun is UNSATISFIABLE -- PiViVi runs against dangling +# symlinks and the group fails; +# * a failed group job's postprocess(error=True) fans out over every member in +# every toposort level and removes each member's EXISTING outputs. One of +# those is tile_make_cat's final_cat on the persistent root. So the failure +# deletes the science product of a tile that was finished and correct; +# * with final_cat gone, tile_finished() flips and the tile re-declares its +# whole exposure edge set, rebuilding the reclaimed chains from VOS. That is +# the rerun avalanche D5 exists to prevent, arriving through the params +# trigger instead of through inputs. +# So: land changes to this function BETWEEN campaigns. An edit cannot reach a +# RUNNING one (the launch code snapshot, bin/sp) -- the hazard is the `sp run` +# after it, which is a resume against finished tiles. If that resume is genuinely +# needed, run it once with +# `--rerun-triggers mtime code software-env`, accepting that clean_exposure's +# consumer-set staleness detection (which rides on params) is off for that +# invocation. +def tile_local(tile): + """The node-local prologue, as bash, for one tile. + + A FUNCTION of the tile rather than a constant, because the path has to be + literal by the time apptainer sees it. Nothing can be COMMUNICATED into the + container -- `$SLURM_TMPDIR`, `--env`, `APPTAINERENV_*` and + `{resources.tmpdir}` all fail, one of them in production; the post-mortem is + on the `--bind /local` line of profiles/nibi/config.yaml. + + So the path is derived instead. `/local/scratch` on nibi is + `drwxrwxrwt root:root` -- world-writable with the sticky bit, verified in + the container (probe job 20798618) -- and the tile id is a wildcard + snakemake substitutes at DAG time, so the shell string carries a concrete + path with no `$` left for anything to escape. One group job per tile means + the name cannot collide; the sticky bit means nobody else can remove it. + + What we give up is Slurm's own cleanup of `$SLURM_TMPDIR`. TILE_VIGNET_FRESH + reclaims a stale directory on the next attempt for the same tile, the trap + in the last group member removes it on the way out, and the sweep below + catches what a hard kill leaves behind -- on a shared node that last one is + manners, not housekeeping. + + NOT `/tmp`: inside the container that is a 378 GB tmpfs, i.e. RAM charged to + the job's cgroup, so a 5.6 GB store there would be paid for twice. + + TWO STORES, AND THE SECOND IS 11 MB. `$SP_WCS_DIR` holds one file, + `log_exp_headers-.sqlite`, and staging it is the rest of the same fix + rather than a refinement of it. Moving the 5.6 GB vignette store off NFS + bought 4.3x and did not solve the problem: thread-state sampling of the + fused job (3,200 samples across 8 chunks, campaign smk-g4 job 20799387) put + every chunk at 56% on CPU and 44% in `rpc_wait_bit_killable`, with 147 of + 176 D-state samples inside NFS RPC and exactly one hot NFS file still open. + ngmix reads that file once per object PER EPOCH for the WCS, so what costs + is the operation count and not the byte count -- a network round trip per + read. The two stores are the same intervention at the two ends of the size + distribution, and the small one is worth ~1.8x on the rule that is 99.3% of + tile-side core-hours. + + A copy, never a symlink: a link resolves straight back to NFS. The prologue + runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in + `tile_merge_cats`, which has no pre_run), so the staging is attempted nine + times per tile, eight of them concurrent siblings in one toposort level. It + is copy-to-a-temp-name plus `mv -f` -- an all-or-nothing publish, so a + reader never sees a partial file -- and it is UNCONDITIONAL, + no `cp -u` and no already-there test. An interrupted `cp` leaves a truncated + destination whose mtime is NEWER than the source, so `-u` would skip it + forever, and nothing downstream would catch it: TILE_VIGNET_FRESH and + TILE_VIGNET_REQUIRED both look only at the vignette store, and a truncated + WCS store reaches ngmix as wrong astrometry rather than as an error. Nine + unconditional copies of 11 MB per tile is not a cost worth reasoning about; + a staleness rule would be. Renaming over a file a sibling chunk already has + open is safe -- the open descriptor keeps the old inode, identical in + content. + + `tile_merge_headers` is upstream of the whole group, so the file exists by + the time any member runs; failing loudly if it does not is correct, because + the alternative is ngmix silently reading a different tree. + """ + # `log_exp_headers--.sqlite`, named from the tile in ShapePipe's + # dashed image-number form -- spelled out rather than globbed so a missing + # file is one precise error instead of an empty expansion. unit_num() owns + # that convention and supplies the SEPARATING DASH itself, so there is none + # in the literals below. + tile_num = unit_num(tile) + return f''' +if [ ! -d /local/scratch ]; then + echo "tile_shape: node-local storage unavailable." >&2 + echo " /local/scratch is not a directory inside the container." >&2 + echo " profiles/nibi/config.yaml apptainer-args must carry --bind /local" >&2 + exit 1 +fi +export SP_LOCAL="/local/scratch/sp-{tile}" +export SP_VIGNET_OUT="$SP_LOCAL/output" +export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" +export SP_WCS_DIR="$SP_LOCAL/wcs" +mkdir -p "$SP_VIGNET_OUT" "$SP_WCS_DIR" || {{ + echo "tile_shape: cannot create $SP_LOCAL on this node." >&2 + exit 1 +}} +# the WCS store -- see the docstring. Copy to a temp name and rename, because +# `cp` is not atomic and the eight chunks share this destination. +sp_wcs_src="$SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output/log_exp_headers{tile_num}.sqlite" +cp "$sp_wcs_src" "$SP_WCS_DIR/.log_exp_headers.$$" && \ +mv -f "$SP_WCS_DIR/.log_exp_headers.$$" \ + "$SP_WCS_DIR/log_exp_headers{tile_num}.sqlite" || {{ + rm -f "$SP_WCS_DIR/.log_exp_headers.$$" + echo "tile_shape: could not stage the WCS store." >&2 + echo " source: $sp_wcs_src" >&2 + echo " dest: $SP_WCS_DIR" >&2 + echo " check that tile_merge_headers ran, and that /local/scratch has room." >&2 + exit 1 +}} +find /local/scratch -maxdepth 1 -name 'sp-*.*' -user "$(id -u)" -mmin +1440 \ + -exec rm -rf {{}} + 2>/dev/null || true +''' + + +# Every member of a group job must agree on a STRING resource or snakemake +# raises "Resource slurm_extra is a string but not all jobs in group require +# the same value" (resources.py::_is_string_resource). So the tmp-disk request +# is one constant on all four members, not a property of tile_ngmix alone. +# +# --tmp is a node SELECTION FLOOR, not a reservation: Slurm matches it against +# the node's configured TmpDisk (nibi: 1.67-12 TB, TmpFS=/local) and does not +# decrement it per job. It buys exactly one thing, and it is the thing worth +# having -- the fused job can no longer land on a node with no usable local +# disk, which is the one configuration in which the whole design fails. 16 GB +# against a measured 5.6 GB store leaves room for an unusually rich tile, and +# filters no nibi node today, so it costs no queue time. Resources are not a +# rerun trigger, so adding it invalidates nothing. +TILE_SLURM_EXTRA = "--tmp=16000" + +# One label for the whole shape chain; the composition arithmetic is in this +# file's docstring. The group's declared runtime is a SUM along the chain and +# that sum gates partitions on nibi, so each member's runtime below is measured +# p99 plus margin, not the old defensive ceiling. +TILE_GROUP = "tile_shape" + +# Cleanup, on the LAST member only. $SLURM_TMPDIR is Slurm's to reclaim, but +# /local/scratch on nibi demonstrably carries stale directories from +# long-finished jobs, so the epilog is not on its own reliable. tile_make_cat is +# the last reader of the store, so its EXIT trap is the earliest moment the +# 5.6 GB can go. It must NOT be set by the earlier members: their shells exit +# while the store is still needed. Residual failure mode accepted: SIGKILL +# (OOM-kill, node death) skips the trap, and if the epilog also misses it the +# store leaks until the node drains -- 5.6 GB against a 3.3 TB disk. +TILE_CLEAN = r""" +trap 'rm -rf "$SP_LOCAL"' EXIT +""" + +# tile_vignets ONLY. unit_pre clears each stage's own run dir on the shared root +# ("the job clears its run dir at start" — ShapePipe's FileHandler raises on an +# existing one); the node-local root needs the same treatment, and only the rule +# that WRITES the store may do it. Putting this in TILE_LOCAL would have +# tile_ngmix delete the store it is about to read. +TILE_VIGNET_FRESH = r""" +rm -rf "$SP_VIGNET_OUT/run_sp_tile_PiViVi" +""" + +# THE SINGLE WRITER OF THE ngmix CHUNK PARTITION, and tile_vignets ONLY. +# +# The ranges are materialised once per tile, here, and every chunk then looks +# its own row up (TILE_NGMIX_RANGE_READ below). tile_vignets is the first member +# of the fused group and strictly precedes all chunks in its DAG, so this is the +# one place in the group where a single process can write something all chunks +# read — no flock, no first-one-wins race. The sexcat it reads is tile_detect's, +# upstream of the whole group and on the SHARED root ($SP_RUN); only the OUTPUT +# is node-local. +# +# Group-internal plumbing, deliberately not a rule output: the file lives and +# dies with the group job exactly as the vignette store does, so a chunk can +# only ever read what its own group job wrote. ngmix_range.py owns the rest. +NGMIX_RANGES = "$SP_LOCAL/ngmix_ranges.json" +TILE_NGMIX_RANGES = ( + f'python {SCRIPTS}/ngmix_range.py --run-dir "$SP_RUN" ' + f'--n-chunks {NGMIX_CHUNKS} --write "{NGMIX_RANGES}" || exit 1' +) + +# THE FUSE'S ONE SHARP EDGE, made loud rather than mysterious. +# +# The vignette store is not a declared output any more, so snakemake cannot see +# it. tile_vignets' manifest CAN be up to date while the store does not exist on +# this node — concretely, after a fused group job dies part-way: the manifest is +# a success and survives, so a LATER invocation re-plans the group with only the +# surviving members and the store is simply not there. (An in-flight `retries:` +# resubmission is safe: it reruns the same member set, tile_vignets included.) +# +# Recovery is one line, and the message says it: delete the tile's +# tile_vignets.json and resume — that puts tile_vignets back in the group and +# the store comes back with it. The alternative fixes are worse: temp()-ing the +# vignets manifest would break clean_exposure, which keys reclamation +# eligibility on exactly that file. +TILE_VIGNET_REQUIRED = r""" +if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output" ]; then + echo "tile_shape: the node-local vignette store is missing." >&2 + echo " expected: $NGMIX_VIGNET_DIR" >&2 + echo " This means tile_vignets did NOT run in this group job — its manifest" >&2 + echo " was already satisfied, most likely because a previous fused job for" >&2 + echo " this tile died after tile_vignets succeeded. The store lives and dies" >&2 + echo " with the job, so it cannot be inherited." >&2 + echo " FIX: rm \"\$SP_RUN/manifests/tile_vignets.json\" and resume." >&2 + exit 1 +fi +""" + + + + +def tile_exp(wc): + return tile_exposures(wc.tile) + +# --- the tile->exposure edge, and why it is cut for finished tiles ---------- +# +# THE cascade fix. clean_exposure deletes the exposure's manifests on purpose: +# that is what makes a tile appended later rebuild the chain instead of running +# against an empty store. But those manifests are the tile side's inputs, and an +# exposure is read by ~7-10 tiles. So the moment ONE tile's chain rebuilt an +# exposure, every other tile reading it saw "input files updated by another job" +# and reran — and that rerun rebuilt ITS exposures, which reran THEIR other +# consumers, propagating across the whole exposure-overlap connected component. +# On fixture t4, asking for one damaged tile scheduled all four tiles' chains. +# +# Two mechanisms, and only the second one actually cuts it: +# +# 1. ancient() on every exposure-manifest edge. Correct on its own terms — a +# tile has no business rerunning because an exposure manifest is NEWER — and +# it is what keeps a pure-mtime disturbance (a re-touched manifest, a +# restored backup) from waking finished tiles. But ancient() governs +# TIMESTAMPS only. Snakemake propagates "my input is produced by a job that +# will run in this DAG" separately, and ancient does not suppress it +# (measured: t4 counts were identical with ancient alone). +# +# 2. Cutting the RECLAIMED edges of a FINISHED tile — the mechanism that works. +# A tile whose final_cat is on disk needs nothing further from its +# exposures: it has already extracted everything it will ever read. So for +# such a tile the input list drops the manifests that are GONE, and the +# propagation has nowhere to go. An UNfinished tile keeps its full edge set +# and therefore still drags in — and rebuilds — every exposure it needs, +# which is the accepted price of a late append, unchanged. +# +# Only the missing ones are dropped, never a manifest that still exists: a +# campaign that has cleaned nothing then declares exactly the edges it always +# did, and the cut cannot perturb it. +# +# The marker is final_cat, not the tile's own vignets manifest: on a tile whose +# catalogue was lost, the vignets manifest still exists while the vignette store +# (temp()) does not, so keying on vignets would cut the edge on exactly the tile +# that has to rerun, and run it against a deleted exposure store. +# +# THE CUT REQUIRES THE `input` RERUN-TRIGGER TO BE OFF (profiles/nibi sets the +# trigger list). Dropping an input is itself a change in the set of input files, +# which that trigger reads as a reason to rerun — reinstating the very cascade, +# now as "Set of input files has changed", and running finished tiles against a +# store that is gone. Measured on fixture t4, one damaged tile of four: 82 jobs +# with neither fix, 70 with the cut but the trigger on, 28 with both (= exactly +# the damaged tile's own chain, its two exposures, and the clean jobs). +# +# The cost, stated plainly: `--forcerun` on a tile whose final_cat exists will +# NOT rebuild its reclaimed exposures, because those edges are not in the DAG. +# Delete that tile's final_cat first and the whole chain comes back. +# +# What none of this weakens: clean_exposure's own inputs are neither ancient nor +# cut, so a tile that really did rebuild its vignets still reschedules the cleans +# of the exposures it read, and a grown consumer set still travels through +# params.consumers. +def tile_finished(tile): + return Path(final_cat(tile)).exists() + + +def exp_manifests(wc, stage): + paths = [exp_manifest(e, stage) for e in tile_exp(wc)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] + +def tile_exp_split(wc): return exp_manifests(wc, "exp_split") +def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_psf(wc) + + +# Build the per-tile symlink forest. Declaring the exposure manifests as input +# makes this wait on its exposures; the forest itself is only the $SP_EXP view. +# Its output stays a directory() (it has no ShapePipe run dir and no manifest — +# it is not a shapepipe_run at all). +rule tile_exp_forest: + group: "tile_gather" + input: + tile_exp_all + output: + forest = directory(f"{TILE_DIR}/exp_forest") + params: + cmd = lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB}"), + # build_forest.py's own content hash rides here and nowhere else. + script_hash = FOREST_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 + shell: + # --forest {output} lives in the shell string: snakemake formats shell + # ONCE, so an {output} placeholder inside params.cmd would survive + # literally and every forest job would race one './{output}'. + "{params.cmd} --forest {output.forest}" + +# Merge single-exposure WCS headers into the tile-level sqlite +# (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). +# Reads headers-*.npy through the forest -> the split manifests are the edge. +rule tile_merge_headers: + group: "tile_gather" + input: + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + # config_tile_Mh_exp.ini reads run_sp_tile_Fe output, which the PREPARE + # phase produced. Declaring the Fe manifest gives the COMPUTE DAG a + # regeneration path for it instead of a silent dependency on a + # previous invocation (prepare.smk is included in every parse, so the + # rule exists here too). Normally a satisfied no-op. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" + log: + f"{TILE_DIR}/logs/tile_merge_headers.json" + params: + pre = lambda wc: unit_pre("tile_merge_headers", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") + +# SExtractor object detection on the tile. +rule tile_detect: + input: + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_detect.json" + log: + f"{TILE_DIR}/logs/tile_detect.json" + params: + pre = lambda wc: unit_pre("tile_detect", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 + shell: + sp_shell("tile_detect", "config_tile_Sx.ini") + +# Configured PSF interpolation to galaxies + vignet postage stamps: the last +# stage that reads exposure products, and the bulk intra-tile intermediate. The store it +# writes is node-local (see TILE_LOCAL above). +rule tile_vignets: + group: TILE_GROUP + input: + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + psf = tile_exp_psf, + # config_tile_PiViVi_.ini reads run_sp_tile_Fe output — same reason as + # tile_merge_headers above. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + # THE MANIFEST IS THE ONLY DECLARED OUTPUT. The vignette store used to + # ride along as a second temp(directory()) so native temp() would + # reclaim it; node-local, it cannot be declared at all — and need not + # be. It was never DAG currency, its readers are all inside this group + # job, and TILE_CLEAN's trap reclaims it. Lost affordance: `--notemp` + # can no longer keep it for debugging. + manifest = f"{TILE_DIR}/manifests/tile_vignets.json", + log: + f"{TILE_DIR}/logs/tile_vignets.json" + params: + pre = lambda wc: unit_pre("tile_vignets", wc.tile, + forest=forest_dir(wc.tile), + pre_run=[tile_local(wc.tile), TILE_VIGNET_FRESH, + TILE_NGMIX_RANGES]), + script_hash = SCRIPT_HASH, + # tile_vignets now PRODUCES the chunk partition (TILE_NGMIX_RANGES), so + # it carries the splitter's fingerprint too. Same hash, same constant — + # the Snakefile's NGMIX_RANGE_HASH argues what it guards, and + # tile_ngmix's copy below carries the mid-campaign-edit warning. + range_hash = NGMIX_RANGE_HASH + # 8, not 16, for the same reason tile_ngmix is 1: `-b {threads}` is SMP + # batch size over input FILE SETS, and a tile is one set -- this run's own + # log says "Batch size: 16 / Total number of processes: 1". 16 was the + # widest member and therefore set the whole GROUP's cpus_per_task; at 8 the + # group asks exactly what the eight-chunk ngmix wave needs. Billing is + # unchanged either way (nibi is MAX_TRES and the 112 GB memory term is + # 28 core-equivalents, well above both), but the group now packs onto a + # node in 8 cores instead of 16. + threads: 8 + resources: + mem_mb = lambda wc, attempt: 32000 * attempt, + # Measured median 3m29s, max 5:25 (this branch: 2m38s on 198.305). + # 20 min is p99 plus a wide margin, and it is a term in the GROUP's + # runtime sum, so the old defensive 240 is not free any more. + runtime = 20, + slurm_extra = TILE_SLURM_EXTRA + shell: + # The completeness check is pointed at the NODE-LOCAL run root; see + # sp_shell's check_args for what the two flags do. + sp_shell("tile_vignets", f"config_tile_PiViVi_{PSF_MODEL}.ini", + check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') + +# ngmix shape measurement — N chunks per tile (D4). Each chunk LOOKS UP its own +# CLOSED object-ID range in the file tile_vignets materialised at the top of this +# group job (TILE_NGMIX_RANGES); the ranges are knowable only at EXECUTION time, +# from this tile's own sexcat, which is why a params function cannot supply them. +# Closed, not open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour +# straggler's root cause (ngmix treats id_obj_max <= 0 as unbounded). +# +# Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and +# merge_sep_cats — DAG-serialised after all chunks — is the gather. +rule tile_ngmix: + group: TILE_GROUP + input: + # The manifest, and only the manifest. The vignette store is no longer a + # declared input because it is no longer a declared output: it is + # node-local, produced by tile_vignets earlier in THIS SAME group job + # (see TILE_LOCAL). The manifest was always the real edge. + vignets = rules.tile_vignets.output.manifest, + sx = rules.tile_detect.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", + # STAYS ON SCRATCH, unlike the vignette store. It is DAG currency: + # tile_merge_cats reads it, and merge_sep_cats derives chunks 2..N from + # chunk 1's path. It is also small (~300 KB/chunk), so it costs the + # scratch high-water nothing worth chasing. temp() still reclaims it + # once merge_cats has run. + chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), + log: + f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" + params: + pre = lambda wc: unit_pre("tile_ngmix", wc.tile, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing sexcat would fall + # through to shapepipe_run with an unset range and fail as something + # else. Capture, check, then eval — the range script fails as itself. + # tile_local FIRST, because it is what exports $SP_LOCAL — and the + # ranges file the lookup reads lives there, written once by + # tile_vignets (TILE_NGMIX_RANGES). This chunk only looks its row + # up; it never recomputes, and the script refuses to. + # + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing ranges file would + # fall through to shapepipe_run with an unset range and fail as + # something else. Capture, check, then eval — the lookup fails as + # itself. + pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, + f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py ' + f'--read "{NGMIX_RANGES}" --chunk {wc.chunk}) || exit 1', + 'eval "$ngmix_range_out"']), + script_hash = SCRIPT_HASH, + # `pre_run` already puts the INVOCATION in params, but the invocation is + # invariant to the script's body, and what that body decides is a + # PARTITION (the Snakefile's NGMIX_RANGE_HASH argues the corruption). + # Carried on tile_vignets too, since the split moved there. + # + # WHY THE PARTITION IS MATERIALISED ONCE, kept as history because the + # design only reads as over-careful until you have seen this. The eight + # chunks used to each run ngmix_range.py in their own shell and trust the + # others to have landed on the same boundaries. A fused group holds them + # open for hours (smk-g4: 6,236-7,762 s elapsed per chunk), and each chunk + # read the script only when its own shell started -- against the LIVE + # checkout, as every job did before the launch code snapshot (bin/sp) -- + # so an edit landed mid-flight was read by some and not others. + # Seen once, live: an invocation four seconds after a rewrite returned + # chunk 5 of 186.307 as 17649..22060 where the other seven had been split + # 17129..21229 — 520 objects orphaned, 831 measured twice, and + # merge_sep_cats concatenates whatever it is handed, so the tile would + # have completed green. (Twelve sequential and thirty-six concurrent runs + # against a stable checkout gave the identical correct partition; the + # race was purely the edit.) That is now STRUCTURALLY closed: tile_vignets + # writes the ranges once and the chunks only look their row up, so there + # is nothing left for them to disagree about. The hash below no longer + # buys sibling agreement — it buys REPRODUCIBILITY across a resume. + # + # WHAT A ONE-RULE FINGERPRINT DOES MID-CAMPAIGN: IT DELETES SCIENCE + # PRODUCTS. Observed when range_hash lived on tile_ngmix ALONE, which is + # no longer the case -- tile_vignets carries it too now that it is the + # rule that runs the splitter, so an edit drags tile_vignets back into + # the group, the store and the ranges are rebuilt, and nothing trips. + # Kept because the mechanism is general and the next single-rule param is + # one edit away. A new param entry replans the fused group with "Params have changed since last + # execution", scheduling the eight chunks, tile_merge_cats and + # tile_make_cat but NOT tile_vignets or tile_detect (their manifests + # exist, so missing_output never queues them). That is exactly the state + # TILE_VIGNET_REQUIRED catches: every chunk trips the guard, the group + # fails, and GroupJob.postprocess(error=True) removes every member's + # EXISTING outputs -- tile_make_cat's final_cat on the persistent root + # among them. + # + # OBSERVED end to end, not derived: SLURM job 20818649 against smk-g5, + # 2026-08-30, 32 seconds. The dry run scheduled exactly that member set + # (the group's SLURM label came back + # tile_shape_tile_make_cat_tile_merge_cats_tile_ngmix, no tile_vignets); + # every chunk tripped the guard; the group failed; and + # final_cat-186.307.fits was GONE from /project afterwards, the tile's + # directory on the persistent root empty. It had never been seen before + # because "Group jobs: inactive (local execution)" puts it out of reach + # of any login-node fixture. (Restored from a copy taken first; design + # and transcript in sp-products/smk-g5/EXPERIMENT_postprocess_deletion.md.) + # + # WORTH SITTING WITH: the guard is what makes this loud rather than + # silent, and the loud failure is precisely what triggers the deletion. + # Still the right trade -- but it argues for never reaching this state, + # not for relaxing the guard. + # + # THE RULE THE INCIDENT LEFT BEHIND: a fingerprint that changes on the + # chunks but NOT on tile_vignets is the dangerous shape. tile_local() + # was always safe for exactly this reason -- it sits in three rules' + # params.pre, so an edit there pulls tile_vignets in and the store is + # rebuilt. range_hash is now the same shape by construction. job_head.sh's + # runbook sweep would not have helped either: it skips any tile with a + # final_cat, exactly the set this breaks. + # + # INVERTED BY clean_tiles, so read the default carefully. With + # reclamation ON the tombstoned tile has lost tile_detect.json and the + # structural "Input files updated by another job" propagation puts + # tile_vignets back in the group (measured at clean_tile below), so the + # store is rebuilt and nothing trips. The SHIPPED DEFAULT + # clean_tiles: false is the dangerous configuration -- the opposite of + # how reclamation reads everywhere else in this file. + # + # So: land this hash, and every later edit to ngmix_range.py, at a + # campaign boundary on a fresh root. The same rule and direct command as + # tile_local()'s -- `--rerun-triggers mtime code software-env`. + range_hash = NGMIX_RANGE_HASH + # ONE core, not four. `-b {threads}` is shapepipe_run's SMP BATCH SIZE + # (pipeline/args.py) -- joblib Parallel(n_jobs=batch_size) over + # filehd.process_list, i.e. parallelism ACROSS INPUT FILE SETS. An ngmix + # chunk is one catalogue, so process_list has exactly one entry: every real + # log says "Batch size: 4 / Total number of processes: 1". There is no + # internal parallelism either (no multiprocessing/Pool/joblib/threading in + # ngmix_package/ngmix.py), and OMP/BLAS are pinned to 1 by both the prologue + # and the profile. The reserved cores 2-4 never had anything to run. + # + # Worth being precise about what this saves. nibi bills + # TRESBillingWeights=CPU=1000,Mem=250G under PriorityFlags=MAX_TRES, i.e. + # max(cores, mem_GB/4). At 14 GB the memory term alone is 3.5 core- + # equivalents, so 4 -> 1 core moves the reservation from 4.0 to 3.5, a 12% + # saving -- NOT 75%. The real core-hour win is the elapsed-time collapse + # from killing the NFS random reads, not this. + threads: 1 + retries: 2 + benchmark: + f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" + resources: + # 5000, down from 14000, and this is the campaign's largest single cost + # saving — but read what the number means before moving it again. + # + # sacct's MaxRSS here is NOT process memory. nibi runs + # JobAcctGatherType=jobacct_gather/cgroup, so it reports the cgroup's + # memory.current, which under cgroup v2 CHARGES PAGE CACHE to the job. + # Cache is reclaimable — the kernel evicts it before it kills anything — + # so the cgroup high-water is an upper bound on what the job NEEDS, not + # a hard requirement. What a tight reservation can still do is squeeze the cache + # that keeps the node-local store resident, which is part of why the + # fused tile is fast. + # + # MEASURED ACROSS 31 TILES (campaign smk-g4, 2026-08-30): cgroup + # high-water 27.40 GiB max, 22.14 GiB mean, against the 109.4 GiB the + # group was reserving. The eight chunks are SIBLINGS in the group's + # toposort so snakemake SUMS their mem_mb; at 5000 the group asks + # 40000 MiB = 39.1 GiB, a 1.43x margin over the worst tile observed. + # + # The ANONYMOUS half of that, which is the part that can actually OOM, + # is the number to argue about, and this repo carries two estimates that + # disagree by 1.8x. Snakemake's psutil benchmark says ~1.25 GiB per + # chunk (~10 GiB for eight); config_tile_Ng_template.ini's own + # SAVE_BATCH note says ~2.3 GB per worker from an A/B test (job + # 17607877), i.e. ~18.4 GiB. Prefer the larger: snakemake samples RSS on + # a 30-second grid (BENCHMARK_INTERVAL), and a SAVE_BATCH = 250 flush + # cycle is exactly the sawtooth such a grid misses. So read the margin + # as ~2.1x over anonymous memory with ~20 GiB left for cache against an + # 8.1 GiB store — comfortable, but not the 4x a psutil-only reading + # would suggest. An OOM is also self-healing: mem_mb scales with + # attempt, so a retry asks 80000. + # + # WHY THIS IS THE BIG ONE. nibi bills max(cores, mem_GB/4), so at + # 112 GB the fused group billed 28 core-equivalents for 8 real cores — + # 3.4x, measured live at 905 billed against 266 allocated. At 40 GB it + # bills 10. That is not a saving so much as a schedule: the account's + # fairshare target is ~250 CE, which buys 9 tiles in flight at 28 and 25 + # at 10, and DR6's wall clock is (tiles / tiles-in-flight) x elapsed. + # + # 4000 is the current lower bound and is NOT recommended yet: it would take the group + # to 32000 MiB, where tile_vignets' own 32000 becomes the binding term + # and the group finally bills its 8 real cores — but that is a 1.14x + # margin over the worst tile measured, and the first thing to give would + # be the page cache holding the store. Take it only with a measurement + # of cache behaviour under pressure, not on the arithmetic alone. + mem_mb = lambda wc, attempt: 5000 * attempt, + # 120 on the FIRST attempt, and ATTEMPT-SCALED after it. MEASURED + # two ways, and the second is why the margin is thinner than it looks: + # * alone (job 20795277, tile 198.305 chunk 1): ~76 min for 3547 + # objects = 1.29 s/object, against a 7h34m median on NFS -- the + # same ~50 min of TotalCPU either way, so the collapse is pure I/O. + # * EIGHT-WIDE on one node (job 20799387, tile 186.307, 4412 + # objects/chunk): 1.54 s/object steady-state, i.e. concurrency + # costs ~19%, and the chunk lands at ~113 min. The campaign's + # largest tile (198.306, 4678 objects/chunk) projects to ~120 -- + # exactly this number, with nothing left over. + # + # Inside a group SLURM enforces only the GROUP's wall (165 min), never + # a member's, so 120 is a budgeting term rather than a kill line and + # the worst tile still lands ~127 min inside 165. What is NOT safe is a + # flat retry: a group that TIMEOUTs re-queues against the identical + # wall and fails identically, burning three 165-minute allocations to + # learn nothing. Scaling with `attempt` keeps the happy path in + # cpubase_bycore_b1 (20+120+10+15 = 165 <= 180) and gives a retry real + # headroom (285 min, which is b2) instead of a rerun of the same + # failure. Attempt 1 is unchanged, so this does not perturb the + # benchmark -- it only makes the failure branch mean something. + runtime = lambda wc, attempt: 120 * attempt, + slurm_extra = TILE_SLURM_EXTRA + shell: + sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + + +def ngmix_manifests(wc): + return [f"{tile_dir(wc.tile)}/manifests/tile_ngmix_{k}.json" + for k in range(1, NGMIX_CHUNKS + 1)] + +def ngmix_chunkdirs(wc): + return [f"{tile_dir(wc.tile)}/output/run_sp_tile_ngmix_Ng{k}u" + for k in range(1, NGMIX_CHUNKS + 1)] + +# The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's +# own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). +rule tile_merge_cats: + group: TILE_GROUP + input: + manifests = ngmix_manifests, + chunkdirs = ngmix_chunkdirs, + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" + log: + f"{TILE_DIR}/logs/tile_merge_cats.json" + params: + pre = lambda wc: unit_pre("tile_merge_cats", wc.tile, + env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + # Measured median 0:15, max 0:42. A term in the group's runtime sum. + runtime = 10, + slurm_extra = TILE_SLURM_EXTRA + shell: + sp_shell("tile_merge_cats", "config_tile_Ms.ini") + +# The run's science product. make_cat also reads the vignette store's +# configured PSF-interpolation output, so it — not ngmix — is the store's last reader. +# +# No protected(): the full default rerun-triggers govern, and protected() only +# ever forced people through a `--forcerun` detour. +rule tile_make_cat: + group: TILE_GROUP + input: + # No store input: it is node-local, written by tile_vignets in this same + # group job. make_cat reads its configured PSF-interpolation output + # through $NGMIX_VIGNET_DIR (config_tile_Mc.ini). + ms = rules.tile_merge_cats.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{PROD_TILE_DIR}/final_cat-{{tile}}.fits", + log: + f"{TILE_DIR}/logs/tile_make_cat.json" + params: + pre = lambda wc: unit_pre("tile_make_cat", wc.tile, + pre_run=[tile_local(wc.tile), TILE_VIGNET_REQUIRED, + TILE_CLEAN]), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + # Measured median 1:33, max 1:50. A term in the group's runtime sum. + runtime = 15, + slurm_extra = TILE_SLURM_EXTRA + shell: + # The plain body plus the catalogue publish: final_cat is a real file, so + # it is a real declared output (and it persists — never temp()). The + # publish is guarded on rc, so a job whose shapepipe_run died never + # publishes a catalogue for a manifest snakemake is about to delete. + sp_shell("tile_make_cat", "config_tile_Mc.ini", + post="if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_tile_Mc/make_cat_runner' + '/output/final_cat*.fits | head -1)" {output.final_cat}\n' + "fi\n") + + +# --- tile reclamation (D5) -------------------------------------------------- +# The tile-side counterpart of clean_exposure, and the differences are the whole +# design. Read that rule's commentary in exposure.smk first; this one only says +# where the tile case departs from it. +# +# WHY IT EXISTS. clean_exposure reclaims exposure stores; nothing reclaimed +# tiles, so a FINISHED tile held 1.19 GiB across 137 inodes on scratch forever +# (measured on 186.307; the per-directory breakdown is in clean_tile.py). At +# DR6's 23,114 tiles that is 26.9 TiB against a 1 TiB quota and 3.17M inodes +# against a 1M one. BOTH BOUNDS BIND, and the byte one binds first: without this +# rule no batch may exceed ~859 tiles. +# +# ELIGIBILITY IS TRIVIAL, AND THAT IS THE POINT: a tile's store has no consumer +# outside that tile, so there is no consumer set, no eligibility test and no +# staleness to detect — only an ordering edge on the tile's own final_cat +# (clean_tile.py opens with the asymmetry; the survivor set is argued there too). +# +# THE INPUT IS final_cat, ON THE PERSISTENT ROOT, AND NOT ancient(). It is +# already the campaign's designated tile-finished marker (see final_cat()'s +# docstring and tile_finished() above), and an ordinary input is what orders the +# clean after the tile — which is what makes reclamation ROLLING: within one +# invocation a tile is reclaimed as soon as its own chain lands, so the scratch +# high-water tracks the tiles in flight instead of the tiles in the batch. That +# is the whole reason the rule is worth having; ancient() would keep the +# dependency but drop the ordering, and the batch would peak at its full size. +# +# SCOPE: TILES_READY only, never every tile in the index — see +# clean_tile_targets() in the Snakefile for what an out-of-scope tombstone drags +# into the DAG. +# +# THE SURVIVING tile_vignets.json DOES NOT TRIP TILE_VIGNET_REQUIRED, AND THE +# REASON IS WHAT GETS DELETED, NOT WHAT GETS KEPT. Read that guard above first. +# A cleaned tile is, on its face, exactly the state it exists to catch: a valid +# tile_vignets manifest over a node-local store that is not there (here because +# it never outlived its job, rather than because a fused group half-ran). +# smk-g4's job_head.sh carries a runbook sweep for that state, and it skips any +# tile with a final_cat — i.e. every tile this rule ever touches — so a cleaned +# tile is invisible to it either way. +# +# Force a cleaned tile back anyway (delete its final_cat) and tile_vignets IS +# rescheduled, so the store is rebuilt and no chunk trips the guard. Measured on +# the fixture: 19 jobs, the whole chain from tile_get_images, tile_vignets inside +# the tile_shape group. Snakemake names the mechanism itself -- +# reason: Input files updated by another job: .../exp_forest, +# .../manifests/tile_find_exposures.json, .../manifests/tile_detect.json +# -- and it is NOT the `mtime` trigger. It is the structural propagation that +# makes every dependent of a rerunning job rerun, the same edge the cascade +# commentary above says ancient() cannot suppress. Worth the distinction: that +# propagation is not in profiles/nibi's rerun-triggers list and so cannot be +# switched off there, where an mtime argument could be. +# +# So the guard stays silent only because tile_detect.json and exp_forest/ GO. +# Counterfactual on the same fixture, upstream manifests restored and mtimes +# controlled so nothing reruns on timestamp -- i.e. the "delete only output/" +# design -- schedules 8 x tile_ngmix + tile_merge_cats + tile_make_cat with +# tile_vignets ABSENT, and every chunk trips the guard. +# ADDING A MANIFEST TO THE SURVIVOR LIST IS THEREFORE NOT FREE: tile_vignets is +# safe because nothing upstream of it survives, and tile_detect.json in +# particular must never join the list. +# +# A LOCALRULE (declared in the Snakefile), same as clean_exposure. It is a DAG +# LEAF, so being local can never make it both a dependency and a dependent of a +# group — no `group:` label here, and none possible. +# +# WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to +# tile_local() mid-campaign reruns finished tiles unsatisfiably because their +# EXPOSURE stores are reclaimed. With this rule on, the tile's own store is gone +# too, so the rerun has even less to stand on. The recommendation is unchanged — +# land those edits between campaigns — but the margin for being wrong is smaller. +rule clean_tile: + input: + # The tile-finished marker, on the persistent root. A lambda rather than + # the PROD_TILE_DIR pattern so there is exactly one definition of this + # path (final_cat() in the Snakefile), the same way clean_exposure keys + # off wildcards.exp alone. + lambda wc: final_cat(wc.tile) + output: + tombstone = f"{TILE_DIR}/cleaned.json" + params: + # clean_tile.py is external to the shell string, so the `code` + # rerun-trigger cannot see it — same reason SCRIPT_HASH exists. + script_hash = CLEAN_TILE_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_tile.py" + " --tile-dir $(dirname {output.tombstone}) --tile {wildcards.tile}" + " --tombstone {output.tombstone}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py new file mode 100644 index 000000000..f780531e1 --- /dev/null +++ b/workflow/scripts/build_forest.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the SP_EXP view). + +A plain script (not a run: block) so the tile chain stays group-compatible. +Reads the tile's exposures from run_index.sqlite and symlinks each exposure's +``exp///output`` into ``///output`` by exact +name (no glob). The 2-char ```` shard level is NOT cosmetic: ShapePipe's +``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its +$SP_EXP glob (``///output/run_sp_*/...``), so a flat +forest makes every tile gather stage fail "No split_exp_runner output found". +(The exposure STORE is sharded the same way, for the filesystem's sake.) +The forest is a convenience view; the DAG edge to the exposures is declared in +the rule's input (tile.smk), not here. +""" + +import argparse +import shutil +import sqlite3 +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + con = sqlite3.connect(args.index, timeout=60) + exps = [r[0] for r in con.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] + con.close() + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e[:2] / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape + dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree — unlink() + # raises IsADirectoryError on it and would kill the job. + if dst.is_symlink() or dst.exists(): + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py new file mode 100644 index 000000000..2dd191306 --- /dev/null +++ b/workflow/scripts/build_index.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Build the run index (``run_index.sqlite``) that drives the compute DAG. + +The index is *parse-time data*, never a rule input: the Snakefile loads it once +at parse time into plain dicts, so extending a run's tile list changes which +jobs exist without touching the mtime chain of completed work. + +It ACCUMULATES ACROSS INVOCATIONS, but is AUTHORITATIVE FOR THE CURRENT TILE +LIST (D1). Precisely: + + * a tile in the current list that has find_exposures output has its ``tiles`` + row replaced and its ``tile_exposures`` edges DELETED AND REBUILT — the Fe + output is the truth, so a tile whose exposure list shrank or changed must + not keep stale edges to exposures it no longer reads; + * a tile NOT in the current list is left completely untouched, which is what + makes the index span the campaign and lets a later ``clean_exposure`` see + every consuming tile; + * ``exposures`` rows only ever accumulate (``INSERT OR IGNORE``). An exposure + no tile references any more is a harmless orphan: nothing reads that table + except by joining through ``tile_exposures``. + +It records: + + tiles(tile_id, ra_dir, n_exp, status) + exposures(exp_id) -- deduplicated union over tiles + tile_exposures(tile_id, exp_id) -- the tile->exposure edges + +The tile->exposure edges are *data-derived*: they are read from each tile's +``find_exposures`` output (``exp_numbers--.txt``), which +``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. +So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME +by the Snakefile of the COMPUTE invocation ONLY — ``SP_PHASE == "compute"``, +which ``bin/sp`` sets — after the PREPARE invocation has produced the tiles' +find_exposures output. No other parse builds anything: a prepare parse or a +passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is +already on disk. (There is no ``sp index`` verb; the CLI below stays for +hand-inspection.) ``build`` below documents the missing-tile policy and the +write ordering. + +Exposure IDs are stored with their trailing single-char suffix stripped +(``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the +exposure-rule wildcard, matching the sharded ``exp///`` store. +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + + +def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: + """Return ``(exp_id, name)`` pairs from one tile's find_exposures output. + + Each line is an exposure *name* like ``2243881p``; the bare base ID (suffix + stripped) is the dedup key everywhere in the DAG, but the original name is + kept in the index — the fabricated per-unit ``exp_numbers`` list must carry + it verbatim (``get_images`` matches ``.fits.fz`` in the store; the + bare ID matches nothing). + """ + pairs = [] + for line in exp_numbers_file.read_text().splitlines(): + name = line.strip() + if not name: + continue + pairs.append((name[:-1] if name[-1].isalpha() else name, name)) + return pairs + + +def exp_list_path(run_dir: Path, tile_id: str) -> Path: + """This tile's find_exposures output, at its deterministic path. + + Sharded store (D2), fixed run dir (RUN_DATETIME=False) — an existence check, + never a glob (no ``ls`` at scale). + """ + idra, iddec = tile_id.split(".") + return (run_dir / "tiles" / tile_id[:2] / tile_id / "output" / + "run_sp_tile_Fe" / "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + + +def build(tile_ids: list[str], run_dir: Path, db_path: Path, + missing_threshold: float | None = 0.0) -> dict: + """Build the index over ``tile_ids``; return a summary dict. + + For each tile, check its ``exp_numbers--.txt`` at the tile's + deterministic ``find_exposures`` run dir (RUN_DATETIME=False, no glob). A + tile whose exposure list is missing is recorded in ``missing.json`` and the + index is built over the rest (a bad tile costs that tile, not the run). The + build is fatal only if the missing fraction exceeds ``missing_threshold``. + + ORDER MATTERS: the threshold is evaluated FIRST, from the missing set, and + the database + ``missing.json`` are written only if it passes. A build that + aborts must leave no trace — an aborted parse that had already mutated + durable state was the bug this ordering fixes. + + The write is idempotent, which is what makes it acceptable that the compute + parse runs it even under ``-n``: re-running over an unchanged tree produces + an identical database. + """ + missing = [t for t in tile_ids if not exp_list_path(run_dir, t).exists()] + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if missing_threshold is not None and frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + + db_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db_path, timeout=60) + # No DROP: the index accumulates across invocations (D1). + con.executescript( + """ + CREATE TABLE IF NOT EXISTS tiles( + tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE IF NOT EXISTS exposures( + exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS tile_exposures( + tile_id TEXT, exp_id TEXT, + PRIMARY KEY (tile_id, exp_id)); + """ + ) + + missing_set = set(missing) + all_exposures: set[tuple[str, str]] = set() + for tile_id in tile_ids: + if tile_id in missing_set: + continue + ra_dir = tile_id.split(".")[0] + exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) + con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", + (tile_id, ra_dir, len(exp_pairs))) + # Replace this tile's edge set wholesale. INSERT OR IGNORE alone only + # ever added, so a tile whose exposure list shrank kept edges to + # exposures it no longer reads — and those stale edges would block it in + # the report and pin those exposures against cleanup. + con.execute("DELETE FROM tile_exposures WHERE tile_id = ?", (tile_id,)) + con.executemany("INSERT INTO tile_exposures VALUES (?,?)", + [(tile_id, exp_id) for exp_id, _ in exp_pairs]) + all_exposures.update(exp_pairs) + + # Exposures accumulate: OR IGNORE, never REPLACE (the name never changes, + # and orphans left by a shrunken tile are harmless). + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + sorted(all_exposures)) + con.commit() + con.close() + + (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) + return {"n_tiles": len(tile_ids) - len(missing), + "n_exposures": len(all_exposures), + "n_missing": len(missing)} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-list", required=True, type=Path, + help="file of tile IDs, one per line") + p.add_argument("--run-dir", required=True, type=Path, + help="$SP_RUN: root of the tiles/ work-dir forest") + p.add_argument("--db", required=True, type=Path, + help="output run_index.sqlite path") + p.add_argument("--missing-threshold", type=float, default=0.0, + help="fatal if the missing-tile fraction exceeds this " + "(default 0.0: any missing tile is fatal)") + args = p.parse_args() + + tile_ids = [ln.strip() for ln in args.tile_list.read_text().splitlines() + if ln.strip()] + summary = build(tile_ids, args.run_dir, args.db, args.missing_threshold) + print(f"run_index: {summary['n_tiles']} tiles, " + f"{summary['n_exposures']} exposures, " + f"{summary['n_missing']} missing -> {args.db}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py new file mode 100644 index 000000000..b763a5573 --- /dev/null +++ b/workflow/scripts/clean_exposure.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Reclaim ONE exposure's store and leave a tombstone (PRD #848 D5, S5). + +Run as the shell of the in-DAG ``clean_exposure`` rule, never by hand: the rule's +``input:`` is every consuming tile's ``tile_vignets`` manifest, so by the time +this executes, every campaign tile that reads this exposure has already extracted +its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. + +What it deletes: the exposure's whole ``output/`` tree (the bulk store — +run_sp_exp_Gie/Sp/SxSePsfPi), its ``manifests/`` and its ``logs/``. That is the +entire exposure store: since PR #847 removed ShapePipe's mask generation there +is no run_sp_exp_Ma tree and no star-catalogue link farm to reclaim beside it. + +Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not +``rmtree``d, so a link into a shared store can never be recursed through. + +Deleting the manifests is deliberate and load-bearing, not tidiness: + + * the manifests are the exposure rules' DECLARED outputs. If they survived, a + tile appended later would find the exposure chain "up to date" and run + tile_vignets against products that are no longer on disk. With them gone the + DAG sees the chain as unbuilt and regenerates it — the accepted cost of a + late append (D5), expressed as ordinary Snakemake bookkeeping rather than as + a special case. + * Snakemake only demands a missing intermediate when something downstream of it + needs to run, so tiles already finished are NOT rerun by their exposures' + manifests vanishing. + +``logs/`` goes with them, and for the same reason rather than for bytes. Each +log holds the completeness verdict of one stage, written on every run and kept by +snakemake through failures; a log left behind would attest "complete" for a store +that is no longer there, contradicting the unbuilt chain the DAG must now see. +Its content for a successful stage is byte-identical to the manifest beside it, +so absorbing the logs into the tombstone would duplicate what the manifests +already carry — they are deleted, not copied. + +Nothing is lost to the report: every ``manifests/*.json`` is copied verbatim into +the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned +exposure's record out of the tombstone — it reports the unit as ``cleaned``, +warn counts and shortfalls intact, instead of "not run". + +The absorption is by GLOB, so it takes whatever is in ``manifests/``, keyed by +file stem; the report re-keys on each manifest's own ``stage`` field. A legacy +``.failed.json`` from the pre-``log:`` convention is therefore carried +through unremarkably — it should never be there (an exposure with a failed stage +has no complete vignets consumer and so is not eligible for cleaning), but it +costs nothing to be right about. + +Order matters, and it is the reverse of the obvious one: the tombstone is +written FIRST, complete, and only then is anything deleted. A crash between the +two leaves a tombstone beside a store that is still there — the next invocation +treats the exposure as cleaned and only the disk is lost. Deleting first would +put the crash window where the manifests are already gone and the record that +replaces them was never written, and the report would be blind to that exposure +forever. + +The exp_psf benchmark tsv lives beside ``manifests/`` and ``logs/``, not inside +either, so it survives this job — it is the measured-memory feed for resource sizing (D4). + +The tombstone records the consumer set it was cleaned against. The rule carries +that same set as a ``params`` value, so when the index grows a new consumer the +tombstone goes stale under the default ``params`` rerun-trigger and the clean job +is rescheduled after the new tile's vignets — the exposure is cleaned once per +consumer set, not once per campaign. +""" + +import argparse +import json +import shutil +import time +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path) + p.add_argument("--exp", required=True) + p.add_argument("--tombstone", required=True, type=Path) + p.add_argument("--consumers", default="", + help="comma-separated tile ids this exposure was cleaned against") + args = p.parse_args() + + consumers = [t for t in args.consumers.split(",") if t] + + # Absorb the manifests before they go: the tombstone becomes the exposure's + # surviving record. + manifests = {} + mdir = args.exp_dir / "manifests" + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + manifests[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + manifests[f.stem] = {"unreadable": str(exc)} + + # is_symlink() first, and OR'd with exists(): exists() follows the link, so + # a dangling link would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs") + targets = [t for t in candidates if t.is_symlink() or t.exists()] + + # Tombstone first, complete — then delete (see the module docstring). + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ + "exp": args.exp, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "consumers": consumers, + "removed": [str(t) for t in targets], + "manifests": manifests, + }, indent=2) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed = [] + for target in targets: + # NEVER rmtree a symlink (see the module docstring). + if target.is_symlink(): + target.unlink() + else: + shutil.rmtree(target) + removed.append(str(target)) + print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " + f"{len(consumers)} consuming tile(s)") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_tile.py b/workflow/scripts/clean_tile.py new file mode 100644 index 000000000..bdeefb794 --- /dev/null +++ b/workflow/scripts/clean_tile.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Reclaim one finished tile's scratch store and leave a tombstone (PRD #848 D5). + +Run as the shell of the in-DAG ``clean_tile`` rule, never by hand: the rule's +``input:`` is the tile's ``final_cat`` on the PERSISTENT root, so by the time +this executes the tile has published its final catalogue. +A tile's scratch store has no reader outside that tile — tiles read exposures, +nothing reads another tile's store — so unlike the exposure case there is no +consumer set to close over and no eligibility test to make. Writer, then +cleaner, and the DAG edge is the whole ordering argument. + +What it deletes: the tile's whole ``/tiles///`` directory. +Measured on 186.307 (smk-g4, a finished 34-tile-campaign tile): 1,279,231,196 +bytes across 137 inodes (71 regular files, 9 symlinks, 57 directories) — +``output/run_sp_tile_Sx`` 745 MB, ``run_sp_tile_Uz`` 382 MB, ``run_sp_tile_Mc`` +46 MB, ``run_sp_tile_Ms`` 39 MB, ``run_sp_tile_Mh_exp`` 11 MB, and a non-``output/`` +remainder of 62 inodes totalling 15,726 bytes. Deleting only ``output/`` is NOT +enough: 62 x 23,114 DR6 tiles is 1.43M inodes against a 1M quota, so the inode +bound binds on its own and the directory has to go as a whole. + +The four retained paths +------------------------ +The retained paths are used by other mechanisms after the tile is complete. +after this tile is done. Nothing is kept for tidiness, and the three that +already exist are required: ``require_survivors`` checks all of them before +anything is deleted, because a silent drift in one of these paths would not show +up as a broken clean — it would show up much later as a campaign that cannot +resume. The fourth, ``cleaned.json``, is this job's own output and is the one +thing here that cannot be pre-checked; it is written first instead (see ORDER +below). + + 1. ``cleaned.json`` — this script's own tombstone, and the tile's surviving + record. It absorbs every manifest verbatim so ``sp report`` can still + report a reclaimed tile as ``cleaned`` rather than as "not run"; see + ``run_report.absorb_tombstones``. + + 2. ``manifests/tile_vignets.json`` — CLEAN_EXPOSURE'S CURRENCY, not ours. + Exposure reclamation keys campaign-wide eligibility on exactly this path: + ``clean_targets()`` tests it for out-of-scope consumers and ``rule + clean_exposure`` declares it as an input for in-scope ones. Deleting it + would silently strand every exposure shared with an out-of-scope tile + (~7-10 tiles read each exposure), and would fail an in-scope + ``clean_exposure`` job outright on a missing input if the tile finished + mid-invocation. It is also not stale in any dangerous sense: it attests + that ``tile_vignets`` succeeded, which stays true forever, and its product + — the vignette store — was node-local and died with its job, so it never + lived on scratch to be contradicted. + + 3. ``manifests/tile_find_exposures.json`` — the PREPARE invocation's target. + ``rule prepare_all_tiles`` declares this manifest for EVERY tile in the + list, and the tile list accumulates across the campaign, so a cleaned tile + is still demanded by every later ``sp run``. Delete it and the whole + ``tile_prep`` group (get_images -> uncompress -> find_exposures) reruns per + cleaned tile per invocation, which does not merely cost jobs: uncompress + writes 382 MB back into the store this job just emptied. + + 4. ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-*.txt`` — + the campaign's tile->exposure edge data, and the reason this is a survivor + rather than a manifest is that NOTHING re-derives it. ``build_index.build()`` + runs at the parse of EVERY compute invocation, over the whole declared tile + list, and checks this exact path (``build_index.exp_list_path``); a tile + whose file is gone counts as missing, and the default + ``SP_MISSING_THRESHOLD`` of 0.0 makes ONE missing tile a fatal parse. So + deleting it does not degrade the campaign, it stops it: the first `sp run` + after the first reclaimed tile cannot build a DAG at all. + Only the one file is kept — ``run_sp_tile_Fe``'s own ``logs/``, ``tmp/`` + and the runner's process log are ordinary residue and go. + +The rest of ``run_sp_tile_Fe``'s parents come along because a file cannot +outlive its directories: 10 inodes per cleaned tile in total (the tile dir, +``cleaned.json``, ``manifests/`` + 2 manifests, and the 4-deep Fe path + its +file), i.e. ~231k inodes at DR6 against the 1M scratch quota — versus 3.2M if +nothing were reclaimed and 1.4M if only ``output/`` were. + +What is lost: the per-tile audit trail for both tools that +read it — ``sp_tilecost.py`` and ``sp_costmodel.py``. They attribute the fused +``tile_shape`` group job's cost per tile by reading the tile's SExtractor +catalogue (NAXIS2 of the sexcat = the object count, the cost model's independent +variable, ~380 MB and unkeepable; sp_costmodel also reads its ``EPOCH_k`` +extensions for the geometric epoch count) and the eight +``tile_ngmix_.benchmark.tsv`` files. The sexcat is gone for good; the +benchmark ROWS are absorbed into the tombstone under ``benchmarks``, because +they are two lines each and they are the measured-memory feed D4 sizes +``mem_mb`` from — exposure.smk moves ``exp_psf``'s benchmark outside +``manifests/`` for exactly this reason. They are absorbed rather than left on +disk (8 more inodes per tile = 185k at DR6 buys nothing a JSON blob does not), +so ``sp_tilecost.py`` will not find them at their old paths: the record +survives, the tool's current reader does not. FOLLOW-UP, deliberately not done +here: teach ``sp_tilecost.py`` to fall back to ``cleaned.json`` for a tile whose +benchmark TSVs are gone. Until it does, per-chunk cost attribution stops at the +first reclaimed tile even though the numbers are still on disk. + +Deletion is symlink-safe (``clean_exposure`` gives the general reason). A +finished tile holds nine symlinks in two classes, and +the second is the one that matters: + + * ``exp_forest///output`` — 7 links into the EXPOSURE stores, + each shared with 7-10 other tiles. Rebuildable, but only by re-running those + chains from VOS. + * ``output/run_sp_tile_Git/get_images_runner/output/CFIS_{image,weight}-*`` + — 2 links into ``$SP_INPUT_TILES``, the staged survey imaging: 621 GB across + 2,536 files, on the backed-up, group-shared input filesystem, and not this + campaign's to lose. get_images RETRIEVE=symlink is what puts them there, so + every tile in every campaign carries a pair. + +Both classes are handed WHOLESALE to ``shutil.rmtree`` — ``exp_forest/`` as a +top-level entry, ``run_sp_tile_Git/`` as one inside ``output/``, which ``prune`` +descends only because the Fe survivor lives there. So the safety rests on +rmtree's own semantics (it unlinks a symlinked entry rather than recursing +through it), NOT on ``prune``'s ``is_symlink()`` test, which fires only for a +link that is itself a direct entry of a level prune walks. Both were verified on +the fixture: every link unlinked, no target followed. Whoever edits ``prune`` +next should know that the worst case is not a scratch store they could rebuild — +it is a rmtree walking into half a terabyte of shared, backed-up survey data. + +Logs are deleted, not absorbed, as in ``clean_exposure``. Here the +duplication is exact: on a finished tile every ``logs/.json`` is +BYTE-IDENTICAL to the ``manifests/.json`` beside it (verified across all +16 stage records of 186.307), because a tile with a failed stage has no +final_cat and so is never cleaned. + +Order: write the tombstone first, then delete — ``clean_exposure``'s docstring +the crash window. + +There is no ``consumers`` field and no consumer-set staleness to detect, because +a tile has no consumers. What reruns this job is the ordinary machinery +(``script_hash``, and ``mtime`` if final_cat is rewritten). A rerun over an +already-pruned tree deletes nothing and, because absorption is ADDITIVE, does +not blank the record either — see ``previous_record``. +""" + +import argparse +import csv +import json +import shutil +import time +from pathlib import Path + + +def survivor_paths(tile_dir: Path, tile: str) -> dict: + """``{what it is: path}`` for the three PRE-EXISTING survivors. + + Three, not four: ``cleaned.json`` is this job's own output and does not + exist yet when this is called. + + The Fe path is spelled out rather than globbed and MUST agree with + ``build_index.exp_list_path`` — that function is what re-reads it at every + compute parse. It is not imported, because this script runs inside the + container as a job shell and stays stdlib-only and import-free like + ``clean_exposure.py``; ``require_survivors`` below turns a drift between the + two into a loud failure on the first tile instead of a fatal parse later. + """ + idra, iddec = tile.split(".") + return { + "clean_exposure eligibility (clean_targets / rule clean_exposure input)": + tile_dir / "manifests" / "tile_vignets.json", + "prepare_all_tiles target (rule prepare_all_tiles input)": + tile_dir / "manifests" / "tile_find_exposures.json", + "index build input (build_index.exp_list_path)": + tile_dir / "output" / "run_sp_tile_Fe" / "find_exposures_runner" + / "output" / f"exp_numbers-{idra}-{iddec}.txt", + } + + +def require_survivors(survivors: dict, tile: str) -> None: + """Abort before deleting anything if the survivor contract is not met. + + Fatal, not a warning. Each of these is read by a mechanism OUTSIDE this + tile, and each failure mode is silent at deletion time and loud much later: + a missing vignets manifest strands shared exposures, a missing Fe manifest + reruns the prepare chain, a missing exposure list makes the next compute + parse exit on the missing-tile threshold. Failing here costs one red job in + a keep-going run (clean_tile is a leaf, so it poisons no cone) and leaves + the store intact for inspection. + """ + missing = {k: p for k, p in survivors.items() if not p.exists()} + if missing: + lines = [f"[clean_tile] {tile}: refusing to reclaim — " + f"{len(missing)} survivor(s) not on disk:"] + lines += [f" {p}\n owned by: {k}" for k, p in missing.items()] + lines.append(" Nothing was deleted. Either this tile's store is not in " + "the state a finished tile should be in, or one of these " + "paths has moved and clean_tile.py has not followed it.") + raise SystemExit("\n".join(lines)) + + +def previous_record(tombstone: Path) -> tuple: + """``(manifests, benchmarks)`` from an existing tombstone, or two empties. + + THE ABSORPTION IS ADDITIVE, and this is why. A second clean of an + already-cleaned tile finds only the two surviving manifests on disk, so a + fresh absorption would overwrite a complete record with a two-entry one and + the tile's history would be gone — silently, and for good. Found by running + the fixture clean twice; not hypothetical, since ``script_hash`` reruns this + job on any edit to this file. So the previous record is the BASE and what is + on disk is laid over it: a rebuilt stage's fresh manifest still wins, and a + reclaimed one keeps the only copy that exists. + """ + if not tombstone.exists(): + return {}, {} + try: + old = json.loads(tombstone.read_text()) + except (OSError, json.JSONDecodeError): + return {}, {} # unreadable: start over rather than refuse + return (dict(old.get("manifests") or {}), + dict(old.get("benchmarks") or {})) + + +def absorb_manifests(mdir: Path) -> dict: + """Every ``manifests/*.json`` verbatim, keyed by file stem. + + By GLOB, so it takes whatever is there; ``run_report.py`` re-keys on each + body's own ``stage`` field. Same shape as ``clean_exposure``'s absorption, + including its tolerance for a legacy ``.failed.json``. + """ + out = {} + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + out[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + out[f.stem] = {"unreadable": str(exc)} + return out + + +def absorb_benchmarks(mdir: Path) -> dict: + """The ngmix chunks' benchmark rows, keyed by file stem. + + Two lines each (header + values), so this is a few hundred bytes for the + campaign's only per-chunk record of runtime, RSS and mean load — the feed D4 + sizes ``tile_ngmix``'s ``mem_mb`` from. Numbers are kept as strings: this is + an archive of what the TSV said, not a re-measurement. + """ + out = {} + if mdir.is_dir(): + for f in sorted(mdir.glob("*.benchmark.tsv")): + try: + rows = list(csv.DictReader(f.read_text().splitlines(), + delimiter="\t")) + except OSError as exc: + out[f.name] = {"unreadable": str(exc)} + continue + if rows: + out[f.name] = dict(rows[0]) + return out + + +def prune(root: Path, keep: set, removed: list) -> None: + """Delete everything under ``root`` except ``keep`` and the dirs leading to it. + + A whitelist walk rather than an rmtree-with-exceptions, so adding a survivor + is one line and can never be half-implemented: a path is kept iff it is a + survivor, recursed into iff it is a real directory on the way to one, and + deleted otherwise. + + ``is_symlink()`` is tested BEFORE ``is_dir()`` and before the recursion — + see the module docstring on the two symlink classes. + """ + ancestors = {p for k in keep for p in k.parents} + for entry in sorted(root.iterdir()): + if entry in keep: + continue + if entry in ancestors and not entry.is_symlink(): + prune(entry, keep, removed) + continue + if entry.is_symlink(): + entry.unlink() + elif entry.is_dir(): + shutil.rmtree(entry) # unlinks nested symlinks, never follows + else: + entry.unlink() + removed.append(str(entry)) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-dir", required=True, type=Path) + p.add_argument("--tile", required=True) + p.add_argument("--tombstone", required=True, type=Path) + args = p.parse_args() + + survivors = survivor_paths(args.tile_dir, args.tile) + require_survivors(survivors, args.tile) + + mdir = args.tile_dir / "manifests" + manifests, benchmarks = previous_record(args.tombstone) + manifests.update(absorb_manifests(mdir)) + benchmarks.update(absorb_benchmarks(mdir)) + + # Tombstone first, complete — then delete (see the module docstring). + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ + "tile": args.tile, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "kept": sorted(str(p) for p in survivors.values()), + "manifests": manifests, + "benchmarks": benchmarks, + }, indent=2, sort_keys=True) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed: list = [] + prune(args.tile_dir, set(survivors.values()) | {args.tombstone}, removed) + print(f"[clean_tile] {args.tile}: removed {len(removed)} path(s); kept " + f"{len(survivors)} survivor(s) + the tombstone") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py new file mode 100644 index 000000000..8391336ee --- /dev/null +++ b/workflow/scripts/completeness.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""The count-based completeness table — the single failure policy. + +This is the ported ``complete_check`` count table from the v2.0 bash layer +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). Across smk-g6 +(127 exposures, 64 tiles, and 512 ngmix chunks), every non-warning runner +produced exactly its ``expect`` count; the v2.0 layer likewise used exact counts, +with only ``psfex_interp`` marked ``:warn``. The formerly ported lower bounds +(and the unsupported ~0.2% setools-attrition claim) therefore have no basis: +setools produced 80/80 in all 127 exposures. ``split_exp`` is also structurally +all-or-nothing because it raises on an HDU-count mismatch. + +A runner below ``expect`` fails its unit unless it has ``warn=True``; such a +shortfall gives the unit status ``warn``. There is no 3-class taxonomy and no +error-signature whitelist. + +This file is also the ``check`` CLI, the second half of every rule's shell line +(PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing +onto it, so the check runs — and the verdict is recorded — even when +``shapepipe_run`` failed:: + + rc=0 + shapepipe_run -c $SP_CONFIG/config_exp_Sp.ini -b {threads} || rc=$? + completeness.py check exp_split {output} --log {log} --job-rc "$rc" || rc=1 + exit $rc + +It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory +runner is below ``expect`` OR ``--job-rc`` is nonzero — the verdict is COMPOSED of +the counts and shapepipe_run's own exit status, because a runner can raise after +the counted ones have written their files. + +The verdict is written to two files with different jobs: + + * the LOG (``--log``, the rule's snakemake ``log:``) gets the full verdict on + EVERY run, success or failure — counts against ``expect``, per-runner detail, + scraped failure reasons, ``job_rc`` when nonzero. Snakemake never deletes a + log file, so it survives the failed job that wrote it and is the post-mortem + evidence ``run_report.py`` reads. + * the MANIFEST (the rule's declared ``output:``) gets that same verdict ONLY + when it is a success. Snakemake deletes a failed job's declared output + natively, so nothing here has to unlink anything. + +``.json`` therefore means "this stage succeeded": a resume cannot schedule +a downstream stage on top of a failed one. The removal of a PREVIOUS success's +manifest is snakemake's to do, not this script's, and the one gap that leaves is +a head process SIGKILLed between the job's failure and that deletion — a +success-named manifest then outlives the failure it no longer describes. The +profile's ``rerun-incomplete`` covers the DAG side, and ``run_report.py`` takes +the WORST status across a stage's log and manifest, so the report is right even +in that window. + +Neither file carries wall-clock, and each is rewritten ONLY when its content +changes — identical on-disk state must leave a byte-identical manifest with an +UNMOVED mtime, or the mtime rerun-trigger churns the cone on every unrelated +``--forcerun``. + +Per-runner fields: + expect nominal file count for a fully complete unit; below it fails + warn if True a shortfall warns instead of failing the unit (bash + ``:warn`` — e.g. psfex_interp on tiles missing some epochs) + subpath count files in ``/output//`` instead of + ``/output/`` (bash ``:rand_split`` — setools split cats) + +Counts are file counts in the runner's output dir, matching the bash +``ls / | wc -l`` semantics (broken symlinks excluded by the caller). +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +# stage -> {runner_subdir: {expect, [warn], [subpath]}} +# exp_psf and tile_vignets are selected by $SP_PSF at check time. +COMPLETENESS = { + # --- tile prepare (phase A) --- + # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 + # for the canfar vos flavor; the nibi symlink configs produce one file per + # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), + # verified against the p3-batch1 baseline tree (100 files / 50 tiles). + "tile_get_images": {"get_images_runner": dict(expect=2)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1)}, + + # --- exposure chain --- + "exp_get_images": {"get_images_runner": dict(expect=3)}, + "exp_split": {"split_exp_runner": dict(expect=121)}, + # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree + # AND the bash baseline (both 120/exposure). + # + # mask_query is one sexcat_ext per CCD — the count the deleted exp_mask + # stage used to carry, now inside this chain because querying a healsparse + # map at ~2k detections needs no rule of its own. + "exp_psf": { + "psfex": { + "sextractor_runner": dict(expect=120), + "mask_query_runner": dict(expect=40), + "setools_runner": dict(expect=80, subpath="rand_split"), + "psfex_runner": dict(expect=80), + "psfex_interp_runner": dict(expect=40, warn=True), + }, + # MCCD counts are derived from config_exp_mccd.ini and its per-CCD + # runners, but this chain has not been exercised through this workflow. + # Keep the expected counts visible while making the unverified branch + # warning-only until a real campaign validates its counts. + "mccd": { + "sextractor_runner": dict(expect=120, warn=True), + "setools_runner": dict(expect=80, warn=True, + subpath="rand_split"), + "mccd_preprocessing_runner": dict(expect=80, warn=True), + # Fit/validation is exposure-wide: one model and one validation + # catalogue, unlike the per-CCD preprocessing outputs. + "mccd_fit_val_runner": dict(expect=2, warn=True), + "merge_starcat_runner": dict(expect=1, warn=True), + # config_exp_mccd enables the ten meanshape and six histogram plots. + "mccd_plots_runner": dict(expect=16, warn=True), + }, + }, + + # --- tile post --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2)}, + "tile_vignets": { + "psfex": { + "psfex_interp_runner": dict(expect=1), + "vignetmaker_runner_run_1": dict(expect=1), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. every vignette feeds ngmix, so the expected count is all-or-nothing. + "vignetmaker_runner_run_2": dict(expect=5), + }, + # MCCD is wired but unvalidated here; retain the expected runner names + # and counts as warnings until a workflow campaign exercises them. + "mccd": { + "mccd_interp_runner": dict(expect=1, warn=True), + "vignetmaker_runner_run_1": dict(expect=1, warn=True), + "vignetmaker_runner_run_2": dict(expect=5, warn=True), + }, + }, + # One check runs inside run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u per chunk, + # so expect=1 is the correct per-chunk count. + "tile_ngmix": {"ngmix_runner": dict(expect=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1)}, +} + + +def count_products(run_dir, runner, spec): + """Count files in ``run_dir//output[/]/`` (live links only). + + scandir, not iterdir: the dirent already says whether an entry is a symlink, + so only the symlinks need the follow-stat that drops dead links. A plain + ``p.exists()`` per entry stats every one of them, and at DR6 scale this runs + once per runner per job over directories of tens to hundreds of files on a + network filesystem. + """ + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + n = 0 + try: + with os.scandir(out) as entries: + for e in entries: + # A dead symlink must not count (the bash + # `ls | wc -l` semantics this ports counted live files only), and + # a symlink is the only entry that can be dead — so it is the + # only one worth a follow-stat. + if not e.is_symlink() or os.path.exists(e.path): + n += 1 + except OSError: + return 0 + return n + + +def check_counts(stage, run_dir): + """Return (ok, details); mandatory shortfalls below expect make ok false. + + ``details`` is a list of (runner, n_found, expect, warn) tuples. + """ + table = COMPLETENESS[stage] + if stage in ("exp_psf", "tile_vignets"): + psf_model = os.environ.get("SP_PSF", "psfex") + try: + table = table[psf_model] + except KeyError as exc: + raise ValueError( + f"Invalid SP_PSF={psf_model!r}; expected one of psfex, mccd." + ) from exc + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + warn = spec.get("warn", False) + details.append((runner, n, spec["expect"], warn)) + if not warn and n < spec["expect"]: + ok = False + return ok, details + + +# --- where a stage writes ------------------------------------------------- +# +# stage -> (level, run_sp_ dir under $SP_RUN/output/). These are the +# committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so +# the check never resolves a run-log. The ngmix entry interpolates the same env +# var its config does, so chunk K's check looks at chunk K's dir. +# +# EVERY ENTRY HERE (and in COMPLETENESS above) HAS A RULE. The table used to +# carry stages that did not — `tile_mask` (run_sp_tile_Ma) and `tile_detect_uc` +# (run_sp_tile_Uc) — and, until PR #847, an `exp_mask` stage that did. ShapePipe +# now generates no masks at all: the sky-fixed healsparse maps are queried per +# object inside the exp_psf and tile_make_cat chains, so masking has no stage +# of its own on either side and is not coming back. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), + "tile_merge_cats": ("tile", "run_sp_tile_Ms"), + "tile_make_cat": ("tile", "run_sp_tile_Mc"), +} + + +# --- failure reasons ------------------------------------------------------ + +# Lines worth showing a human who asks "why is this runner short?". Deliberately +# crude: the point is a pointer into the logs, not a taxonomy (there is no error +# whitelist in this design — the count policy is the policy). +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 # logs are per-CCD; a handful is enough to characterise +MAX_TAIL_LINES = 120 # per file +MAX_REASONS = 3 # per runner + + +def _normalise(line: str) -> str: + """Collapse a log line to its shape, so 40 per-CCD copies dedupe to one.""" + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) # paths differ per CCD + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + """Best-effort, bounded: distinct error-looking lines from a runner's logs. + + Two sources, in order of usefulness: the runner's per-process worker logs + (``/logs/process-*.log`` — where the module's own exception lands), + and the stage's ``logs/log_sp.log`` (where ShapePipe records its error + tally). Sorted, truncated, deduped by shape — a manifest must stay + byte-stable for a given tree. + """ + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +# --- manifest ------------------------------------------------------------- + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + """Count, classify and (on shortfall) scrape. Returns (manifest, ok). + + Stages absent from the table fall back to a zero-output check: any product + anywhere under the stage dir passes, nothing at all fails. + """ + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, + "level": level, + "unit": unit, + "run_dir": str(run_dir), + "stage_dir": str(stage_dir), + "runners": {}, + "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "expect": 1, "warn": False, + "status": "failed", "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_counts(stage, stage_dir) + short = False + for runner, n, expect, warn in details: + below = n < expect + if below: + short = True + status = "complete" if not below else "warn" if warn else "failed" + manifest["runners"][runner] = { + "found": n, "expect": expect, "warn": warn, "status": status, + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "expect": expect, + "warn": warn, "status": status, + "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def write_if_changed(path: Path, text: str) -> None: + """Write only when the bytes differ — see the module docstring on mtime.""" + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != text: + path.write_text(text) + + +def _unit_from_run_dir(run_dir): + """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). + + NOT ``SP_UNIT_NUM``, which carries ShapePipe's dashed numbering form + (``-210-282``) and would put ``210-282`` in the manifest — a key that joins + to nothing. ``run_report`` keys units on the store directory name, which is + exactly this basename, so the two now agree. + """ + return Path(str(run_dir)).name or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="ShapePipe per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--log", type=Path, required=True, + help="the rule's log: path — the verdict is written here every " + "run, success or failure") + c.add_argument("--run-dir", type=Path, default=None, + help="the unit's $SP_RUN (default: the env var)") + c.add_argument("--unit", default=None, + help="override the unit ID (default: basename of $SP_RUN)") + c.add_argument("--stage-dir", default=None, + help="override the run_sp_* subdir (default: the stage table)") + c.add_argument("--job-rc", type=int, default=0, + help="shapepipe_run's exit status, composed into the verdict") + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_run_dir(run_dir) + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + + # The verdict is COMPOSED of two independent statements: the count checks + # (above) and shapepipe_run's own exit status (here). Counts alone are not + # enough — a runner can raise AFTER the counted runners have written their + # files, so the counts are met while the job died. Without this, such a job + # publishes a SUCCESS manifest that snakemake then deletes as a failed job's + # output — and the log would claim "complete" for a stage with no manifest, + # which reads as a bookkeeping bug rather than as the failure it is. + # + # Recorded only when nonzero, which keeps every existing success manifest + # byte-identical (the mtime rerun-trigger reads those bytes). + if args.job_rc != 0: + ok = False + manifest["status"] = "failed" + manifest["job_rc"] = args.job_rc + manifest["failures"].append({ + "runner": "shapepipe_run", "found": 0, "expect": 1, + "warn": False, "status": "failed", + "reasons": [f"shapepipe_run exited {args.job_rc} " + f"(counts met expect)"], + }) + + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + + # The log ALWAYS gets the verdict; the manifest gets it only on success. No + # unlink of anything: snakemake deletes a failed job's declared output, so + # the presence of .json IS the statement "this stage succeeded" and + # the DAG can never build on top of a failure. A failure->success transition + # publishes the manifest; a success->failure has the manifest removed for us, + # and the log is overwritten with the new verdict either way. + write_if_changed(args.log, text) + if ok: + write_if_changed(args.manifest, text) + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "failed": "<-- BELOW expect"} + print(f"[completeness] {runner}: {r['found']}/{r['expect']} " + f"{tag[r['status']]}", file=sys.stderr) + print(f"[completeness] {args.stage} {unit}: {manifest['status']} " + f"-> {args.log}" + (f" + {args.manifest}" if ok else ""), file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py new file mode 100644 index 000000000..afc7019ed --- /dev/null +++ b/workflow/scripts/container.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Manage this user's copy of the ShapePipe container image. + +Two layers, and the second only exists when you ask for one: + +* the **SIF** (``~/.cache/shapepipe/shapepipe.sif``) -- a read-only + copy of the published image, pulled into your own cache. Per-user by + construction: one file, one owner, nobody else's refresh moves the ground + under a running job. +* an optional **sandbox** (``~/.cache/shapepipe/sandbox/``) -- the same image + unpacked into a writable directory, so a ``pip install`` inside it sticks. + The direct path for work that needs a package the image does not carry yet. + +Resolution order, shared by this CLI and by the workflow: **sandbox if it +exists, else the cached SIF if it exists, else the ``container:`` path in +workflow/config.yaml**. That last one is the current shared /project image, so +a checkout with an empty cache behaves exactly as it did before this verb +existed, and a package installed into your sandbox is there for your workflow +jobs too. + +Subcommands, exposed as ``sp container ``:: + + sp container pull # fetch the tag into the cache + sp container status # what is here, and how current it is + sp container sandbox # unpack the SIF into a writable dir + sp container exec # run something inside it + sp container exec --writable # ... with writes that persist + +``pull`` needs the network. Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an ``salloc`` allocation -- never +from a batch job. + +Deliberately **stdlib-only**: it runs on the bare host, outside the container, +where the science stack is not installed, and so must import without it. +""" + +import argparse +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + + +class ContainerError(Exception): + """A misconfiguration the caller must fix (bad override, no image at all). + + Raised rather than ``sys.exit``ed so the Snakefile, which imports this + module at parse time, can turn it into a WorkflowError instead of a + SystemExit. ``main`` below turns it back into a one-line CLI error. + """ + +# The published image. CI pushes one tag per branch, sanitized; `-runtime` is +# the slim variant the workflow runs. +CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" + +# The single source of truth for the fallback image: the workflow's own +# `container:` key, which is also what the Snakefile reads. Written down once, +# here, so the CLI and the workflow cannot disagree about the default. +CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" +CONFIG_KEY = "container" + +# The profile whose `apptainer-args:` every workflow job runs under. `exec` reads +# it at runtime rather than restating it, so a one-off `sp container exec` and a +# job see the same environment (the PYTHONPATH pin above all: a divergence there +# means the one-off imports a different src/ than the workflow does). +PROFILE_FILE = Path(__file__).resolve().parents[2] / "profiles" / "nibi" / "config.yaml" + +# ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto +# a filesystem with room), XDG_CACHE_HOME moves it with everything else. +CACHE_DIR = Path( + os.environ.get("SP_CACHE_DIR") + or Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "shapepipe" +).expanduser() + +# This user's read-only image. Override with ``SP_CONTAINER`` (absolute path). +DEFAULT_SIF = CACHE_DIR / "shapepipe.sif" + +# The optional writable unpacking of it. Override with ``SP_SANDBOX``. +DEFAULT_SANDBOX = CACHE_DIR / "sandbox" + +# Bind mounts for `exec`, matching the nibi profile's apptainer-args (the two +# cluster filesystems this workflow reads and writes, plus the home that holds +# ~/.ssl/cadcproxy.pem). Override wholesale with ``SP_APPTAINER_BINDS``. +DEFAULT_BINDS = "/project,/scratch,/home" + + +def configured_default(): + """Return the ``container:`` path from workflow/config.yaml, or ``None``. + + A deliberately minimal scalar read (the same one bin/sp does in sed): this + module is stdlib-only, so there is no yaml to import. + """ + try: + text = CONFIG_FILE.read_text() + except OSError: + return None + match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) + return match.group(1) if match else None + + +def profile_apptainer_args(): + """Return the profile's ``apptainer-args`` as a token list, or ``[]``. + + Minimal scalar read again (stdlib-only); a missing or unreadable profile + yields ``[]``, which callers replace with their own defaults. + """ + try: + text = PROFILE_FILE.read_text() + except OSError: + return [] + match = re.search(r'^apptainer-args:[ \t]*"(.*)"[ \t]*$', text, re.MULTILINE) + return shlex.split(match.group(1)) if match else [] + + +def local_sif(): + """Return this user's cached image path (may not exist yet).""" + override = os.environ.get("SP_CONTAINER") + return (Path(override) if override else DEFAULT_SIF).expanduser() + + +def local_sandbox(): + """Return this user's writable sandbox directory (may not exist).""" + override = os.environ.get("SP_SANDBOX") + return (Path(override) if override else DEFAULT_SANDBOX).expanduser() + + +def resolve_image(): + """Return ``(path, kind)`` for the image everything should run. + + ``kind`` is ``"sandbox"``, ``"sif"``, ``"configured"`` (the shared + /project image named in config.yaml -- the default when the cache is + empty) or ``"none"``. + """ + sandbox = local_sandbox() + if sandbox.is_dir(): + return str(sandbox), "sandbox" + sif = local_sif() + if sif.exists(): + return str(sif), "sif" + # An override that resolves to nothing is a typo, never an intention: falling + # through to the shared image would run the job against something other than + # what the user named. + if os.environ.get("SP_CONTAINER"): + raise ContainerError( + f"SP_CONTAINER={os.environ['SP_CONTAINER']} does not exist " + f"(resolved to {sif}). Unset it, or point it at an image that does." + ) + default = configured_default() + if default: + return default, "configured" + return "", "none" + + +def image_labels(image): + """Return the image's OCI labels as a dict, or ``{}`` if unreadable. + + Never raises: a missing file, a missing ``apptainer`` or a corrupt image + all mean "we don't know", which every caller treats as non-fatal. + """ + path = Path(image) + if not path.exists() or shutil.which("apptainer") is None: + return {} + try: + out = subprocess.run( + ["apptainer", "inspect", "--labels", str(path)], + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return {} + if out.returncode != 0: + return {} + labels = {} + for line in out.stdout.splitlines(): + key, sep, value = line.partition(":") + if sep: + labels[key.strip()] = value.strip() + return labels + + +def image_revision(image): + """Return the commit the image was built from, or ``None``.""" + return image_labels(image).get("org.opencontainers.image.revision") + + +def _require_apptainer(): + """Exit unless ``apptainer`` is on PATH (bin/sp loads the module).""" + if shutil.which("apptainer") is None: + sys.exit("apptainer is not on PATH (bin/sp loads apptainer/1.4.5)") + + +def _git(*args, cwd=None): + """Run git, returning stripped stdout or ``None`` on any failure.""" + try: + out = subprocess.run( + ["git", *args], capture_output=True, text=True, cwd=cwd, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + +def compare_revision(revision, repo=None): + """Place an image revision relative to this checkout's HEAD. + + One of ``"in-sync"``, ``"behind"`` (the image predates HEAD), ``"ahead"``, + ``"diverged"``, or ``"unknown"`` (no label, no git, or a commit this clone + has never fetched). + """ + if not revision: + return "unknown" + repo = repo or Path(__file__).resolve().parents[2] + head = _git("rev-parse", "HEAD", cwd=repo) + if head is None: + return "unknown" + if head == revision: + return "in-sync" + if _git("cat-file", "-e", f"{revision}^{{commit}}", cwd=repo) is None: + return "unknown" + if _git("merge-base", "--is-ancestor", revision, head, cwd=repo) is not None: + return "behind" + if _git("merge-base", "--is-ancestor", head, revision, cwd=repo) is not None: + return "ahead" + return "diverged" + + +def cmd_pull(args): + """Pull ``--tag`` into the cache, atomically.""" + _require_apptainer() + sif = local_sif() + sif.parent.mkdir(parents=True, exist_ok=True) + # Pull to a sibling temp name and rename: an atomic rename within one + # directory, so an in-flight job sees either the whole old image or the + # whole new one. Pulling in place leaves the file half-written for the many + # minutes the pull takes. Jobs already running hold the old inode open. + tmp = sif.with_name(sif.name + f".pull.{os.getpid()}") + print(f"pulling {args.tag}\n -> {sif}") + try: + subprocess.run( + ["apptainer", "pull", "--force", "--name", str(tmp), args.tag], check=True + ) + os.replace(tmp, sif) + except subprocess.CalledProcessError as exc: + tmp.unlink(missing_ok=True) + sys.exit(f"pull failed ({exc.returncode}); {sif} is unchanged") + except KeyboardInterrupt: + tmp.unlink(missing_ok=True) + raise + labels = image_labels(sif) + print(f"revision: {labels.get('org.opencontainers.image.revision', 'unknown')}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + return 0 + + +def cmd_sandbox(args): + """Unpack the image into a writable directory -- the opt-in direct path.""" + _require_apptainer() + sandbox = local_sandbox() + if sandbox.exists() and not args.force: + sys.exit( + f"sandbox already exists at {sandbox}\n" + "pass --force to discard it and rebuild from a clean image" + ) + source = args.source + if not source: + image, kind = resolve_image() + if kind == "sandbox": + # Rebuilding from the sandbox itself would just re-copy the drift. + image = str(local_sif()) if local_sif().exists() else ( + configured_default() or CONTAINER_URI + ) + source = image or CONTAINER_URI + sandbox.parent.mkdir(parents=True, exist_ok=True) + print(f"building sandbox from {source}\n -> {sandbox}") + # Build beside the target and swap it in, as `pull` does -- and for a + # sharper reason. A half-written .sif fails loudly, but a half-unpacked + # sandbox *directory* is still a directory, so resolve_image() would elect + # it and every job would silently run a broken tree. Staging also means a + # --force rebuild that fails leaves the sandbox you already had intact. + # + # `--fix-perms` so the tree can be deleted again later. No `--fakeroot`: an + # unprivileged build from an existing image goes through user namespaces, + # which is what the Alliance clusters provide. + staging = sandbox.with_name(f"{sandbox.name}.build.{os.getpid()}") + shutil.rmtree(staging, ignore_errors=True) + try: + subprocess.run( + ["apptainer", "build", "--sandbox", "--fix-perms", str(staging), source], + check=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"sandbox build failed ({exc.returncode}); {sandbox} is unchanged") + except (KeyboardInterrupt, OSError): + shutil.rmtree(staging, ignore_errors=True) + raise + + if sandbox.exists(): + print(f"replacing {sandbox}") + shutil.rmtree(sandbox, ignore_errors=True) + if sandbox.exists(): + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"could not remove {sandbox}; remove it by hand and retry") + os.replace(staging, sandbox) + print( + "\nthis sandbox now takes precedence over the SIF everywhere, including " + "workflow jobs.\ninstall into it with: sp container exec --writable pip " + "install \nreset to a clean image with: sp container pull && " + "sp container sandbox --force" + ) + return 0 + + +def cmd_status(args): + """Report which image layer is live, its revision, and how current it is.""" + sif = local_sif() + sandbox = local_sandbox() + # status is the verb you run WHEN something is wrong, so a broken override is + # reported here rather than raised. + try: + active, kind = resolve_image() + except ContainerError as exc: + print(f"active: NONE -- {exc}") + return 1 + + if sif.exists(): + print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") + else: + print(f"SIF: absent ({sif})") + if sandbox.is_dir(): + print(f"sandbox: {sandbox} (writable; may carry local modifications)") + else: + print("sandbox: none") + print(f"configured: {configured_default() or 'unset'} ({CONFIG_FILE})") + + if kind == "none": + print("\nactive: NONE -- no cached image and no container: in config.yaml") + print(f"run: sp container pull --tag {CONTAINER_URI}") + return 1 + + print(f"\nactive: {active} ({kind})") + if kind == "configured" and not Path(active).exists(): + print(" WARNING: that path does not exist on this host") + return 1 + + labels = image_labels(active) + revision = labels.get("org.opencontainers.image.revision") + source = "" + if revision is None and kind == "sandbox" and sif.exists(): + # Some sandbox trees do not carry the original labels through. The SIF + # beside it is the best remaining evidence of what it was built from -- + # a guess, so it is labelled as one rather than printed as fact. + revision = image_revision(sif) + if revision: + source = " (inferred from the SIF beside it, not read from the sandbox)" + print(f"revision: {revision or 'unknown'}{source}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + if kind == "sandbox": + print( + " (that revision is what the sandbox was built from; " + "anything\n installed into it since is in no label)" + ) + verdict = compare_revision(revision) + explain = { + "in-sync": "matches this checkout's HEAD", + "behind": ( + "older than this checkout's HEAD -- `sp container pull` fetches " + f"{CONTAINER_URI}, which is not built from this branch unless you " + "pass --tag" + ), + "ahead": "newer than this checkout's HEAD", + "diverged": "on a different branch from this checkout", + "unknown": "cannot compare (no label, or a commit this clone lacks)", + }[verdict] + print(f"checkout: {verdict} ({explain})") + return 0 + + +def cmd_exec(args): + """Run a command inside the resolved image -- the one-off path.""" + _require_apptainer() + command = [a for a in args.command if a != "--"] + if not command: + sys.exit("nothing to run; pass a command after `exec`") + + # Same environment the workflow's jobs get: the profile's apptainer-args + # verbatim (--cleanenv, the PYTHONPATH pin, --home, its binds). Explicit + # binds still win, and a profile that cannot be read falls back to the old + # standalone defaults. + env_args = profile_apptainer_args() + explicit_binds = args.bind or os.environ.get("SP_APPTAINER_BINDS") + if not env_args: + print( + f"warning: could not read apptainer-args from {PROFILE_FILE}; " + "falling back to --cleanenv and the default binds, which may not " + "match what jobs run under", + file=sys.stderr, + ) + env_args = ["--cleanenv"] + explicit_binds = explicit_binds or DEFAULT_BINDS + if explicit_binds: + env_args += ["--bind", explicit_binds] + + if args.writable: + # A SIF is a read-only filesystem, so `--writable` against one fails + # obscurely; only a sandbox takes writes. + sandbox = local_sandbox() + if not sandbox.is_dir(): + sys.exit( + f"--writable needs a sandbox, and there is none at {sandbox}\n" + "build one with: sp container sandbox" + ) + image, extra = str(sandbox), ["--writable"] + else: + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + extra = [] + + cmd = ["apptainer", "exec", *extra, *env_args, image, *command] + return subprocess.run(cmd).returncode + + +def cmd_resolve(args): + """Print just the resolved image path -- what the Snakefile consumes.""" + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + print(image) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="sp container", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="subcommand", required=True) + + p_pull = sub.add_parser( + "pull", + help="fetch the image into the cache (login node or salloc: compute " + "nodes have no network)", + ) + p_pull.add_argument( + "--tag", default=CONTAINER_URI, help=f"image to pull (default: {CONTAINER_URI})" + ) + p_pull.set_defaults(func=cmd_pull) + + p_status = sub.add_parser( + "status", help="report which image layer is live and how current it is" + ) + p_status.set_defaults(func=cmd_status) + + p_sandbox = sub.add_parser( + "sandbox", help="unpack the image into a writable directory (opt-in)" + ) + p_sandbox.add_argument( + "--source", help="image to unpack (default: the cached SIF, or the config path)" + ) + p_sandbox.add_argument( + "--force", + action="store_true", + help="discard an existing sandbox and rebuild from a clean image", + ) + p_sandbox.set_defaults(func=cmd_sandbox) + + p_exec = sub.add_parser("exec", help="run a command inside the resolved image") + p_exec.add_argument("--bind", help=f"bind mounts (default: {DEFAULT_BINDS})") + p_exec.add_argument( + "--writable", + action="store_true", + help="run against the sandbox so writes (e.g. pip install) persist", + ) + p_exec.add_argument("command", nargs=argparse.REMAINDER) + p_exec.set_defaults(func=cmd_exec) + + p_resolve = sub.add_parser( + "resolve", help="print the resolved image path (what the workflow runs)" + ) + p_resolve.set_defaults(func=cmd_resolve) + + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + try: + return args.func(args) + except ContainerError as exc: + sys.exit(str(exc)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py new file mode 100644 index 000000000..aa745c2ce --- /dev/null +++ b/workflow/scripts/ngmix_range.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""Materialise the ngmix chunk partition once per tile, then serve it. + +TWO MODES, ONE WRITER. The partition is computed ONCE, by tile_vignets — the +first member of the fused tile_shape group — and written to the group's +node-local scratch; each of the n_chunks chunk shells then only LOOKS UP its +own row:: + + # tile_vignets, once (see tile.smk's TILE_NGMIX_RANGES) + ngmix_range.py --run-dir $SP_RUN --n-chunks 8 --write $SP_LOCAL/ngmix_ranges.json + + # each chunk, at its own start + eval "$(ngmix_range.py --read $SP_LOCAL/ngmix_ranges.json --chunk 3)" + # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 + +The file is group-internal plumbing, NOT DAG currency: it lives on $SP_LOCAL and +dies with the group job, exactly like the vignette store. It is deliberately not +a rule output — a chunk can only ever read the file its own group job wrote. + +Read mode does NOT fall back to recomputation when the file is absent; it exits +non-zero. The fallback would restore the very failure this design removes (see +DETERMINISM, below). + +The range is still only knowable at EXECUTION time (PRD D4) — a params function +cannot compute it, because params evaluate before the sexcat exists. + +SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering +[1, N] processes every object exactly once. The bounds are CLOSED, never +ID_OBJ_MAX = -1 — ngmix treats ``id_obj_max <= 0`` as unbounded +(``ngmix_package/ngmix.py:804-806``), so an open-ended last chunk silently +re-measures the whole tile instead of its share; rule tile_ngmix carries the +straggler that taught this. + +CHUNKS ARE BALANCED BY EPOCH-WEIGHTED COST, NOT BY OBJECT COUNT. ngmix fits an +object jointly across every exposure it lands on, so its cost scales with +(object, epoch) pairs, not objects: the campaign measured 0.2714 CPU-s per +pair with an intercept consistent with zero, plus ~0.05 CPU-s per-object of +setup. The old equal-count split therefore produced chunks whose cost varied +1.6x within a single tile, tracking their epoch counts at R^2 0.91-1.000. Those +chunks are siblings in the fused tile_shape group job, which ends when its +slowest member does, so the spread was pure wall clock: 7.7 hours over the +34-tile smk-g4 campaign, and on tile 196.307 the slowest chunk ran +26.3 min past the median and took the group to 97.4% of its wall limit. + +Re-splitting all 34 of that campaign's sexcats: the old boundaries leave the +slowest chunk at 1.131x-1.627x its tile's median predicted cost on tile 200.302; the new ones at 1.0000x-1.0002x, and the sum over tiles of +slowest-chunk cost falls 132,875 -> 113,710 predicted CPU-s, 14.4%. + +Moving the boundaries does not change measurements: ngmix's RNG is seeded per object +from its sky position, so which chunk an object falls in cannot change its +measurement (see ``ngmix_package.ngmix.position_seed``). + +DETERMINISM HERE IS CORRECTNESS, NOT TIDINESS. A tile's chunk ranges are a +PARTITION of its object IDs: if two chunks disagree about the boundaries, +objects are silently measured twice or silently dropped and nothing downstream +notices — merge_sep_cats concatenates whatever it is given. + +SINGLE-WRITER IS WHY THAT NO LONGER DEPENDS ON AGREEMENT. Every chunk reads the +same file, produced by one process from one read of the sexcat, so sibling +chunks cannot disagree even in principle — the boundaries are a fact of the +group job rather than a computation eight processes each have to land on. The +mechanism that closes is a real one: this script used to run independently in +each chunk's shell, and a group holds its chunks open for hours (smk-g4: +6,236-7,762 s of elapsed per chunk), reading — as every job did before the launch +code snapshot (bin/sp) — the LIVE checkout, so an edit landed mid-flight was read +by some chunks and not others. Observed once, live — the transcript is at +tile.smk's range_hash, which is the home for that history. + +The rest of the determinism discipline stays, because it is what makes the +written file trustworthy rather than merely shared: integer arithmetic end to +end (weights scale to milli-epochs so no float ever decides a boundary), and NO +FALLBACK — if the EPOCH extensions cannot be read, or the ranges file is absent +in read mode, this script exits non-zero. A fallback would be the worst +available behaviour precisely because it would apply only to the processes that +hit the failure, shredding the tile's coverage instead of failing it. + +What remains uncovered is the cross-TIME case, and NGMIX_RANGE_HASH (Snakefile) +is what covers it: a RESUME that reruns some chunks after an edit to this script +would otherwise mix old and new boundaries within one tile. +""" + +import argparse +import json +from pathlib import Path + +# Weights are integers in milli-epochs (one epoch = 1000) so every boundary is +# decided by exact integer comparison, identically on every run of the splitter +# — which is now what makes the written file REPRODUCIBLE across attempts and +# resumes, rather than what makes eight siblings agree. +MILLI_EPOCH = 1000 + +# The per-object setup cost, expressed in epochs so one integer weight carries +# both terms of the measured cost law: 0.05 CPU-s of setup / 0.2714 CPU-s per +# (object, epoch) = 0.184 epoch-equivalents. It is what keeps the zero-epoch +# objects (13 of 35,298 on tile 186.307) from weighing nothing — they still +# cost ~6% of a typical object, and a chunk handed thousands of them at no extra cost +# would be a straggler of a new kind. Only the RATIO of the two costs moves a +# boundary, which makes this robust: refitting on 186.307's eight measured +# chunk CPU times with the setup term held fixed gives 0.2619 per GEOMETRIC +# epoch (see object_epochs for why that is below 0.2714), i.e. ALPHA 0.191, +# and every boundary on that tile then moves by at most one object. Zeroing +# ALPHA entirely moves them by at most 44. +ALPHA_MILLI_EPOCHS = 184 + + +def id_ranges(epochs, n_chunks: int) -> list[tuple[int, int]]: + """Split object IDs ``1..len(epochs)`` into ``n_chunks`` closed ranges. + + ``epochs[i]`` is object ``i + 1``'s geometric epoch count. The ranges are + CONTIGUOUS — ``NGMIX_ID_MIN``/``NGMIX_ID_MAX`` is an interval, not a set — + and tile ``[1, n_obj]`` exactly, so every object is measured once. + + The objective is the slowest chunk, not the average one, because the + group job waits for it. So this minimises the maximum chunk weight + exactly: binary-search the smallest feasible capacity, then fill left to + right under it. Ties in that maximum break toward the earlier chunks, + which is arbitrary but fixed, and fixed is the property that matters. + + With ``n_obj < n_chunks`` the first ``n_obj`` chunks take one object each + and the remainder are EMPTY, written ``(n_obj + 1, n_obj)``: ``lo = hi+1`` + is the canonical empty closed interval and preserves the chain + ``ranges[k][0] == ranges[k-1][1] + 1``. Deliberately not ``(1, 0)``, which + is what the old equal-count split emitted here — ``ID_OBJ_MAX = 0`` is + ngmix's unbounded sentinel, so each empty chunk would have re-measured the + ENTIRE tile. + """ + if n_chunks < 1: + raise ValueError(f"n_chunks must be >= 1, got {n_chunks}") + n_obj = len(epochs) + if n_obj < 1: + raise ValueError("cannot split a catalogue of zero objects") + + if n_obj < n_chunks: + return ([(k, k) for k in range(1, n_obj + 1)] + + [(n_obj + 1, n_obj)] * (n_chunks - n_obj)) + + weights = [MILLI_EPOCH * int(e) + ALPHA_MILLI_EPOCHS for e in epochs] + cap = _min_feasible_capacity(weights, n_chunks) + + ranges: list[tuple[int, int]] = [] + start, load = 0, 0 + for i, w in enumerate(weights): + # Close before object i when the current chunk holds something and + # either it cannot take i under `cap`, or the objects still to come + # (n_obj - i) no longer outnumber the chunks still to open — the second + # clause is what guarantees exactly n_chunks non-empty ranges when a + # heavy head would otherwise leave the tail with nothing to hold. + if start < i and (load + w > cap + or n_obj - i < n_chunks - len(ranges)): + ranges.append((start + 1, i)) + start, load = i, 0 + load += w + ranges.append((start + 1, n_obj)) + return ranges + + +def _chunks_needed(weights: list[int], cap: int) -> int: + """Chunks a left-to-right fill uses when none may exceed ``cap``.""" + used, load = 1, 0 + for w in weights: + if load + w > cap: + used, load = used + 1, w + else: + load += w + return used + + +def _min_feasible_capacity(weights: list[int], n_chunks: int) -> int: + """Smallest ``cap`` that fits ``weights`` into ``n_chunks`` chunks. + + ``_chunks_needed`` is monotone non-increasing in ``cap``, so bisection on + the integers ``[max(weights), sum(weights)]`` lands on the exact optimum + without a float ever entering the comparison. + """ + lo, hi = max(weights), sum(weights) + while lo < hi: + mid = (lo + hi) // 2 + if _chunks_needed(weights, mid) <= n_chunks: + hi = mid + else: + lo = mid + 1 + return lo + + +def object_epochs(run_dir: Path): + """Per-object geometric epoch count, from this tile's own sexcat. + + tile_detect's SExtractor post-process (``MAKE_POST_PROCESS`` in + ``config_tile_Sx.ini``) writes one ``EPOCH_`` extension per exposure + overlapping the tile — so the extension COUNT is tile-specific and is + discovered by name, never assumed — each with ``n_obj`` rows in NUMBER + order and ``CCD_N < 0`` where the object misses that exposure. Summing + ``CCD_N >= 0`` across them reproduces the final catalogue's ``N_EPOCH`` + column exactly — checked row by row against 186.307's + ``run_sp_tile_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, + 116,727 pairs, mean 3.31. The post-process is upstream of the whole + tile_shape group, so the extensions always exist by the time ngmix runs; + their absence is a broken tile, not a case to accommodate. + + N_EPOCH is a GEOMETRIC count and mildly over-states the work, because ngmix + drops epochs it cannot fit. The same catalogue's NGMIX_N_EPOCH is lower for + 2,203 of the 35,298 objects and never higher: 113,947 pairs against + 116,727, 2.4%. That gap is the whole of the ~3.4% by which this cost model + over-predicts the tile's measured chunk times — refit on NGMIX_N_EPOCH the + law is 0.2681 CPU-s per pair at R^2 0.997, against 0.2619 at R^2 0.973 on + the geometric count. The better number is unavailable before ngmix runs and + is not worth wanting anyway: splitting on it moves boundaries by up to 177 + objects and improves the slowest chunk by 1.07%. + + Only the EPOCH extensions are read. ``LDAC_OBJECTS`` carries a 10 kB + VIGNET per row (376 MB on 186.307) and pulling it in would cost more than + the straggle this split exists to remove; the EPOCH tables are ~530 kB + each, and its NAXIS2 comes from the header alone as the row-count check. + """ + import numpy as np + from astropy.io import fits + + cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not cats: + raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") + with fits.open(cats[0], memmap=True) as hdul: + epoch_hdus = [h for h in hdul if h.name.startswith("EPOCH")] + if not epoch_hdus: + raise SystemExit( + f"[ngmix_range] FATAL: no EPOCH extensions in {cats[0]}; " + "tile_detect's SExtractor post-process did not run. " + "Refusing to fall back to an equal-object split: it would " + "apply to only the chunks that saw this failure, and the " + "tile's objects would be double-measured or dropped." + ) + if "LDAC_OBJECTS" not in hdul: + raise SystemExit( + f"[ngmix_range] FATAL: no LDAC_OBJECTS in {cats[0]}" + ) + n_obj = int(hdul["LDAC_OBJECTS"].header["NAXIS2"]) + expected = np.arange(1, n_obj + 1, dtype=np.int64) + counts = np.zeros(n_obj, dtype=np.int64) + for hdu in epoch_hdus: + data = hdu.data + # NUMBER is asserted, not assumed: the whole scheme is an ID + # INTERVAL, so a permuted or gappy NUMBER column would make the + # weights describe different objects than the bounds select. + if len(data) != n_obj or not np.array_equal( + np.asarray(data["NUMBER"], dtype=np.int64), expected + ): + raise SystemExit( + f"[ngmix_range] FATAL: {cats[0]}[{hdu.name}] is not " + f"{n_obj} rows of NUMBER = 1..{n_obj} in order" + ) + counts += np.asarray(data["CCD_N"]) >= 0 + return counts + + +def partition(epochs, n_chunks: int) -> dict: + """The whole partition, as the dict that gets serialised to JSON. + + Minimal on purpose: the ranges themselves, plus the two numbers a reader + needs to check that the file is the one it expects (``n_obj`` so a mismatch + against the catalogue is visible, ``n_chunks`` so a chunk index is validated + against what was actually written, not against what the caller believes). + ``chunk`` is 1-based, matching SP_NGMIX_CHUNK and the run-directory suffix. + """ + ranges = id_ranges(epochs, n_chunks) + return { + "n_obj": len(epochs), + "n_chunks": n_chunks, + "chunks": [ + {"chunk": k, "id_min": lo, "id_max": hi} + for k, (lo, hi) in enumerate(ranges, start=1) + ], + } + + +def chunk_range(doc: dict, chunk: int) -> tuple[int, int]: + """Chunk ``chunk``'s closed range out of a ``partition()`` document. + + Validated rather than indexed: a negative ``chunk`` would index from the end + and hand this process some other chunk's range without any error. + """ + n_chunks = doc["n_chunks"] + if not 1 <= chunk <= n_chunks: + raise SystemExit( + f"[ngmix_range] FATAL: --chunk {chunk} outside 1..{n_chunks}" + ) + row = doc["chunks"][chunk - 1] + if row["chunk"] != chunk: + raise SystemExit( + f"[ngmix_range] FATAL: ranges file row {chunk - 1} is chunk " + f"{row['chunk']}, not {chunk}" + ) + return int(row["id_min"]), int(row["id_max"]) + + +def read_ranges(path: Path) -> dict: + """Load the ranges file, or die saying what should have written it.""" + try: + return json.loads(path.read_text()) + except FileNotFoundError: + raise SystemExit( + f"[ngmix_range] FATAL: no ranges file at {path}.\n" + " tile_vignets writes it once per tile, on the node-local scratch " + "this group job shares.\n" + " Its absence means tile_vignets did NOT run in this group job " + "(same cause as a missing vignette store).\n" + " FIX: rm the tile's tile_vignets.json and resume.\n" + " NOT recomputed here on purpose: a per-chunk fallback would let " + "chunks disagree about the partition." + ) + + +def main() -> None: + """Write the whole partition, or emit one chunk's range as bash exports.""" + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--write", type=Path, metavar="JSON", + help="compute the whole partition and write it here") + p.add_argument("--read", type=Path, metavar="JSON", + help="look one chunk's range up in a file --write made") + p.add_argument("--run-dir", type=Path, help="--write: the tile's run root") + p.add_argument("--n-chunks", type=int, help="--write: chunks to split into") + p.add_argument("--chunk", type=int, help="--read: 1-based chunk index") + a = p.parse_args() + + if bool(a.write) == bool(a.read): + raise SystemExit( + "[ngmix_range] FATAL: pass exactly one of --write / --read" + ) + + if a.write: + if a.run_dir is None or a.n_chunks is None: + raise SystemExit( + "[ngmix_range] FATAL: --write needs --run-dir and --n-chunks" + ) + doc = partition(object_epochs(a.run_dir), a.n_chunks) + # Temp name plus rename, the same all-or-nothing publish tile_local() + # uses for the WCS store: a reader must never see a half-written file. + tmp = a.write.with_name(f".{a.write.name}.tmp") + tmp.write_text(json.dumps(doc)) + tmp.replace(a.write) + return + + if a.chunk is None: + raise SystemExit("[ngmix_range] FATAL: --read needs --chunk") + lo, hi = chunk_range(read_ranges(a.read), a.chunk) + print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py new file mode 100644 index 000000000..24c07670b --- /dev/null +++ b/workflow/scripts/run_report.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +"""``sp report`` — the run's success/failure tables, read from the manifests. + +NOT a DAG node. A report rule that declared all tiles' outputs as inputs would be +a descendant of every job, so one hard failure under --keep-going would poison +its cone and the report would never run — the exact scenario it exists for. So it +is a plain script, runnable at any time including mid-run; the Snakefile's +onsuccess/onerror hooks call it so every invocation ends with one. + +It reads two things and nothing else (PRD D3): + + * the **index** (``run_index.sqlite``) — the units the run declared, and the + tile->exposure edges that let an exposure failure be blamed on the tiles it + blocks; + * the **verdicts** written by ``completeness.py check`` — per-runner + found/expect and log-scraped failure reasons — in the unit's + ``logs/`` (every run) and ``manifests/`` (successes only; completeness.py + argues the split). + +Both dirs are read here and the status comes from the file BODY, so a failed +stage speaks through its log while a successful one is corroborated by two +identical files. A unit with neither ran nothing. Also read, for continuity with +stores written before this convention: ``manifests/.failed.json``, which +is where failure evidence used to go. + +A reclaimed exposure has neither dir: ``clean_exposure`` deleted both, after +copying the manifests into ``/cleaned.json``. That tombstone is read as the +unit's record and the unit is reported as **cleaned** — not "not run", and it +blocks no tile. + +A reclaimed TILE is the same story with one asymmetry: ``clean_tile`` deletes +``logs/`` but cannot empty ``manifests/``, because two of those manifests are +currency other mechanisms read (SURVIVING_TILE_STAGES). So the "records on disk +mean a rebuilt chain" test takes that survivor set explicitly — see +``absorb_tombstones``. + +No disk scanning: counting products is the *check's* job, done once at the moment +the products were fresh. A unit with no manifest for a stage is "not run" — which +is a real and distinct answer from "ran and produced nothing". + +Records are discovered by glob (``tiles/*/*/manifests/*.json`` and +``tiles/*/*/logs/*.json``), not by constructed path: units are found rather than +named, so a store holding a unit the index never heard of still reports. The +depth is FIXED at two, because the sharded layout is exactly two levels +(``tiles///``) — a ``**`` walked the whole tree, including every +``output/`` a run has not reclaimed yet, to find records that can only ever be +at one depth. +""" + +import argparse +import json +import sqlite3 +import sys +from collections import defaultdict +from pathlib import Path + +# Stage order per level — the report's column order, and the definition of +# "expected" (a declared unit with no manifest for one of these is not run). +TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] + +# The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names +# the mechanism that owns each). Their presence is therefore NOT evidence that a +# tile's chain was rebuilt, which absorb_tombstones needs to know +# to read a reclaimed tile's record out of its tombstone. +SURVIVING_TILE_STAGES = frozenset({"tile_vignets", "tile_find_exposures"}) + +STATUSES = ("complete", "warn", "failed", "not_run") + + +def _rank(m: dict) -> int: + """How BAD a record is, as a position in STATUSES; unknown sorts worst.""" + return STATUSES.index(m["status"]) if m.get("status") in STATUSES else len(STATUSES) + + +def keep_worst(records: dict, stage: str, m: dict) -> None: + """Collapse the several records of one stage into the WORST of them. + + ONE rule, used by both readers in this module, and that is the point. + Several files map to one (unit, stage) either way: on disk, a stage's + manifest and its byte-identical log, plus the eight ngmix chunks, which all + carry ``stage: "tile_ngmix"`` under per-chunk filenames; inside a tombstone, + those same eight chunks again, keyed by file stem. + + absorb_tombstones used to resolve that collision by first-key-wins + (``setdefault`` over sorted stems), so only ``tile_ngmix_1`` survived and a + ``warn`` on any other chunk disappeared the moment the tile was reclaimed — + ``tile_ngmix ok 0 warn 1`` before the clean, ``ok 1 warn 0`` after, and the + tile silently left the "tiles not complete" table. The data was in the + tombstone the whole time; only the reader dropped it (found in review, + reproduced on 186.307 with chunk 5 flipped to warn). + + What this does NOT fix, because it is not a reclamation bug: a stage's + per-runner ``products`` aggregate is taken from the single surviving record, + so tile_ngmix attrition is counted over one chunk of eight. That is true + before and after a clean, identically — it is the price of collapsing the + chunks to one stage row, and it is the same on both paths by construction + now. + """ + prev = records.get(stage) + if prev is None or _rank(m) > _rank(prev): + records[stage] = m + + +def load_manifests(run_dir: Path, sub: str) -> dict: + """``{unit: {stage: verdict}}`` for one store (``tiles`` or ``exp``). + + Reads BOTH of a unit's record dirs: ``manifests/`` (the rules' declared + outputs, success-only) and ``logs/`` (the rules' ``log:``, written every run + and never deleted by snakemake, so this is where a failure survives). + + The unit key is the record dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the record's own + ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is + not the index's ``210.282``). The stage comes from the body, never the + filename: ngmix chunks share a stage under per-chunk filenames, and a log + names the same stage as the manifest beside it. + + Several files therefore map to one (unit, stage), and the WORST status wins. + That is what collapses the ngmix chunks to one entry, and it is why a + successful stage's two byte-identical records cost nothing while a failure + always speaks. A body with no ``stage`` field is skipped: it is not one of + ours, which is what keeps a stray JSON deeper in the tree inert. + """ + out: dict = defaultdict(dict) + paths = sorted((run_dir / sub).glob("*/*/manifests/*.json")) \ + + sorted((run_dir / sub).glob("*/*/logs/*.json")) + for path in paths: + try: + m = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable record {path}: {exc}", file=sys.stderr) + continue + if not isinstance(m, dict) or "stage" not in m: + continue + unit = path.parent.parent.name + keep_worst(out[unit], m["stage"], m) + return out + + +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict, + survivors: frozenset = frozenset()) -> set: + """Fill in reclaimed units from their ``cleaned.json``; return their ids. + + A cleaned exposure has neither ``manifests/`` nor ``logs/``; every manifest + was copied verbatim into the tombstone first. Read them back, or the report + inverts the truth exactly when reclamation works: the exposure shows as "not + run" and blocks the very tiles whose completion authorised the deletion. + + Manifests on disk win if both exist — that is a re-built chain, and the + tombstone is then a stale record of the previous generation. + + ``survivors`` is what makes that test work for TILES. ``clean_tile`` cannot + empty ``manifests/`` the way ``clean_exposure`` does: two of the manifests + there are currency other mechanisms read (SURVIVING_TILE_STAGES below), so a + reclaimed tile ALWAYS has records on disk. Without this argument the + "manifests win" guard fires on every cleaned tile, the tombstone is ignored, + and a fully reclaimed campaign reports as one that ran two stages and + stopped. A unit counts as REBUILT — and keeps the guard — iff it has a record + for some stage that is not a survivor; the default empty set reproduces the + exposure test exactly. + + KNOWN AND DELIBERATE: the guard is all-or-nothing, so a PARTIALLY rebuilt + cleaned tile loses its whole tombstone record. Force a rerun that restores + tile_detect..tile_make_cat but not the prepare stages (which only the + prepare invocation produces) and tile_get_images / tile_uncompress read + "not run" though they did run and nothing invalidated them. + + Not fixed, because the obvious fix is wrong rather than long. Filling + per-stage from the tombstone would, on a unit whose chain is rebuilding, + report the PREVIOUS generation's "complete" for a stage whose manifest is + absent precisely because it is mid-rerun or failed — a stale complete is + invisibly wrong where "not run" is visibly incomplete, and the same hazard + reaches exposures, where the mixing would be silent and campaign-wide. The + all-or-nothing guard is a generation boundary and is worth more than the + cosmetics. A correct fix needs a per-stage notion of which generation a + record belongs to, which nothing here records today. + """ + cleaned = set() + for path in sorted((run_dir / sub).glob("*/*/cleaned.json")): + unit = path.parent.name + try: + tomb = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) + continue + if any(s not in survivors for s in manifests.get(unit, {})): + continue + # Collapse the tombstone's per-stem records to one per stage FIRST, + # by the same rule the on-disk path uses (keep_worst), and only then + # merge. Reversing those two steps is what lost a warning chunk. + absorbed: dict = {} + for key, m in (tomb.get("manifests") or {}).items(): + if not isinstance(m, dict): + continue + keep_worst(absorbed, m.get("stage", key), m) + for stage, m in absorbed.items(): + # setdefault, not assignment: the surviving on-disk records are the + # live ones and stay authoritative, even though the tombstone's + # copies of them are byte-identical today. + manifests[unit].setdefault(stage, m) + cleaned.add(unit) + return cleaned + + +def shortfalls(m: dict) -> dict: + """``{runner: (found, expect)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"]) + for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} + + +def reasons(m: dict) -> list: + """Flattened failure reasons, runner-tagged, for the report's why column.""" + out = [] + for f in m.get("failures", []): + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')}" + out += [f"{head}: {r}" for r in f["reasons"]] or [head] + return out + + +def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: + """Per-stage counts + named unit lists, for one level. + + ``cleaned`` units are counted by the status their absorbed manifests carry + and additionally listed under ``cleaned``, so a reclaimed campaign reads as + reclaimed rather than as a campaign that never ran. The STATUS is preserved + exactly, including a warn on any one of the eight ngmix chunks (keep_worst + is what makes that true on the tombstone path as well as on disk). + + The per-runner ``products`` aggregate is NOT a per-chunk total: reading it + as a whole-tile figure over-states completeness by 8x (see ``keep_worst``). + """ + per_stage = {} + for stage in stages: + # All five status keys are unit-id LISTS, "complete" included: it used + # to be a bare int, which made it the one key a caller had to special- + # case. The emitted JSON gains the complete-unit list; the printed + # counts are len() of it. + t = {"complete": [], "warn": [], "failed": [], "not_run": [], "cleaned": []} + agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) + for u in units: + m = manifests.get(u, {}).get(stage) + if m is None: + t["not_run"].append(u) + continue + if u in cleaned: + t["cleaned"].append(u) + status = m.get("status", "failed") + status = status if status in ("complete", "warn") else "failed" + t[status].append(u) + if status == "failed": + # Failed units are named above, never folded into the attrition + # aggregate: a whole-unit failure is not per-CCD attrition, and + # mixing them hides real deletion bugs behind a big denominator. + continue + for runner, d in m.get("runners", {}).items(): + a = agg[runner] + a["found"] += d["found"] + a["expect"] += d["expect"] + if d["found"] < d["expect"]: + a["by_unit"][u] = d["expect"] - d["found"] + for a in agg.values(): + if not a["by_unit"]: + del a["by_unit"] + t["products"] = dict(agg) + per_stage[stage] = t + return per_stage + + +def unit_rows(units, stages, manifests) -> list: + """One row per non-clean unit: its first bad stage, shortfalls, why.""" + rows = [] + for u in units: + got = manifests.get(u, {}) + bad = [s for s in stages + if got.get(s) is None or got[s].get("status") != "complete"] + if not bad: + continue + stage = bad[0] + m = got.get(stage) + rows.append({ + "unit": u, + "stage": stage, + "status": "not_run" if m is None else m.get("status", "failed"), + "shortfalls": shortfalls(m) if m else {}, + "reasons": reasons(m) if m else [], + "n_bad_stages": len(bad), + }) + return rows + + +def print_table(title, rows, limit=25): + print(f"\n{title} ({len(rows)} affected)") + if not rows: + print(" — none") + return + print(f" {'unit':<14} {'stage':<20} {'status':<8} why") + for r in rows[:limit]: + short = ", ".join(f"{k} {v[0]}/{v[1]}" for k, v in r["shortfalls"].items()) + why = (r["reasons"][0] if r["reasons"] else short) or "-" + print(f" {r['unit']:<14} {r['stage']:<20} {r['status']:<8} {why[:90]}") + if len(rows) > limit: + print(f" … and {len(rows) - limit} more (see the JSON report)") + + +def print_stage_table(title, per_stage, n_units): + print(f"\n{title} ({n_units} units declared)") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} " + f"{'cleaned':>8} attrition") + for stage, t in per_stage.items(): + att = [f"{r} {a['found']}/{a['expect']}" + for r, a in t["products"].items() if a["found"] < a["expect"]] + print(f" {stage:<20} {len(t['complete']):>6} {len(t['warn']):>6} " + f"{len(t['failed']):>6} {len(t['not_run']):>8} " + f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--status", default="manual") + p.add_argument("--out", type=Path, default=None) + p.add_argument("--limit", type=int, default=25, + help="rows per stdout table (the JSON report is complete)") + args = p.parse_args() + + tiles, exps, tile_exp = [], [], defaultdict(list) + if args.index.exists(): + con = sqlite3.connect(args.index, timeout=60) + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] + for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + tile_exp[tile_id].append(exp_id) + con.close() + else: + print(f"[run_report] no index at {args.index} — reporting manifests only", + file=sys.stderr) + + tile_m = load_manifests(args.run_dir, "tiles") + exp_m = load_manifests(args.run_dir, "exp") + # Reclaimed units speak through their tombstones (D5, S5). Tiles pass the + # survivor set: clean_tile leaves two manifests on disk on purpose, and + # without that argument they would read as a rebuilt chain. + cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) + cleaned_tiles = absorb_tombstones(args.run_dir, "tiles", tile_m, + SURVIVING_TILE_STAGES) + tiles = tiles or sorted(tile_m) + exps = exps or sorted(exp_m) + + missing_json = args.index.parent / "missing.json" + missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] + + report = { + "status": args.status, + "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m, cleaned_tiles), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), + "cleaned_exposures": sorted(cleaned_exp), + "cleaned_tiles": sorted(cleaned_tiles), + "tiles": unit_rows(tiles, TILE_STAGES, tile_m), + "exposures": unit_rows(exps, EXP_STAGES, exp_m), + } + + # Blame propagation: a BLOCKING exposure blocks every tile that reads it. + # Without this, a tile stalled at tile_vignets looks like its own failure. + # + # Blocking means "failed" or "never ran" — NOT "warn". Warn is the expected + # per-CCD attrition (setools rejecting a sparse CCD, psfex_interp short an + # epoch); it is present in essentially every exposure at production scale, so + # counting it here made every exposure block every tile and the table said + # nothing. + # Judged over ALL the exposure's stages, not just the first bad one, so an + # exposure that warns early and fails late still blocks. + # A CLEANED exposure never blocks: its store is gone precisely because every + # consuming tile already had its vignets. Its absorbed manifests are read + # above, so a cleaned exposure that genuinely failed still shows in the + # tables — it just does not get to hold complete tiles hostage. + def _blocks(unit): + if unit in cleaned_exp: + return False + for stage in EXP_STAGES: + m = exp_m.get(unit, {}).get(stage) + if m is None or m.get("status", "failed") == "failed": + return True + return False + + bad_exp = {e for e in exps if _blocks(e)} + blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} + report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} + + done = len(report["tile_stages"]["tile_make_cat"]["complete"]) + report["final_cats"] = {"present": done, "of": len(tiles)} + + out = args.out or (args.index.parent / "run_report.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "") + + (f" ({len(cleaned_tiles)} tiles reclaimed)" if cleaned_tiles else "")) + print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) + print_stage_table("TILES", report["tile_stages"], len(tiles)) + print_table("exposures not complete", report["exposures"], args.limit) + print_table("tiles not complete", report["tiles"], args.limit) + nb = report["tiles_blocked_by_exposures"] + if nb: + print(f"\ntiles waiting on incomplete exposures ({len(nb)})") + for t, e in list(nb.items())[:args.limit]: + print(f" {t:<14} {', '.join(e[:6])}" + + (f" (+{len(e) - 6})" if len(e) > 6 else "")) + print(f"\n[run_report] -> {out}") + + +if __name__ == "__main__": + main()