CosmoStat · cailmdaley · Sep 10, 2026 · Sep 10, 2026 · Sep 10, 2026 · Sep 10, 2026
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -65,10 +65,10 @@ Run via `./pipeline.sh` with flags:
 
 ### Configuration
 Main configuration in `scripts/calibration/params.py` with parameters:
-- `name`: Field/patch identifier
+- `campaign`: Campaign name (the ShapePipe tile list); names the input products
 - `data_dir`: Input data directory
 - `galaxy_cat_path`: Galaxy catalogue path (.fits/.hdf5)
-- `star_cat_path`: Star catalogue path (.fits)
+- `star_cat_path`: Star catalogue path (.hdf5, or legacy .fits)
 
 ### Key Dependencies
 - astropy, numpy, scipy for core calculations

diff --git a/config/calibration/mask_v2.0.yaml b/config/calibration/mask_v2.0.yaml
@@ -0,0 +1,164 @@
+# Config file for masking and calibration, ShapePipe v2 catalogues.
+#
+# ShapePipe v2 replaces the single IMAFLAGS_ISO column with per-reason boolean
+# mask columns MASK_n<bit>, where True means MASKED. They are therefore cut
+# with `kind: equal, value: False` (keep the un-masked objects):
+#
+# Reason bits of the ShapePipe r-band default bitmask:
+#
+#   MASK_n1, MASK_n2  star halos (which is faint and which is bright is
+#                     unconfirmed for the Aug-2026 products)
+#   MASK_n4           stars
+#   MASK_n8           manual galaxy mask
+#   MASK_n64          undocumented reason bit
+#   MASK_n1024        MaxiMask
+#
+# Per-band coverage flags and Pan-STARRS:
+#
+#   MASK_n16, MASK_n32, MASK_n128, MASK_n256   u, g, i, z coverage (no r
+#                     flag: the catalogue is r-selected)
+#   MASK_n2048        no Pan-STARRS z2 coverage
+#
+# The default selection (see sp_validation.galaxy.DEFAULT_MASK_COLUMNS) is
+# n1 + n2 + n4 + n8 + n64 + n1024, whose OR reproduces mask_r, the v1 r-band
+# mask, exactly on the P3 region. The coverage flags and n2048 are listed
+# here but commented out, since ORing all of them masks essentially
+# everything.
+
+# General parameters (can also given on command line)
+params:
+  input_path: unions_shapepipe_comprehensive_2025_v2.0.hdf5
+  cmatrices: False
+  sky_regions: False
+  verbose: True
+
+# Masks
+## Using columns in 'dat' group (ShapePipe flags)
+dat:
+  # SExtractor flags
+  - col_name: FLAGS
+    label: SE FLAGS
+    kind: smaller_equal
+    value: 3
+
+  # Duplicate objects
+  - col_name: overlap
+    label: tile overlap
+    kind: equal
+    value: True
+
+  # ShapePipe masks (boolean, True = masked)
+  - col_name: MASK_n4
+    label: "stars"
+    kind: equal
+    value: False
+  # n1/n2 are the two star-halo bits; the faint/bright assignment is
+  # unconfirmed for the Aug-2026 products.
+  - col_name: MASK_n1
+    label: "star halos (n1)"
+    kind: equal
+    value: False
+  - col_name: MASK_n2
+    label: "star halos (n2)"
+    kind: equal
+    value: False
+  - col_name: MASK_n8
+    label: "manual mask"
+    kind: equal
+    value: False
+  - col_name: MASK_n1024
+    label: "maximask"
+    kind: equal
+    value: False
+
+  # Undocumented reason bit of the r-band default bitmask. Required for the
+  # OR above to reproduce mask_r; it is not a coverage flag.
+  - col_name: MASK_n64
+    label: "reason bit n64"
+    kind: equal
+    value: False
+
+  # Per-band coverage flags and Pan-STARRS z2; enable as required.
+  # Cutting on all of them at once leaves essentially no objects.
+  # - col_name: MASK_n16
+  #   label: "u coverage"
+  #   kind: equal
+  #   value: False
+  # - col_name: MASK_n2048
+  #   label: "no PS-z2"
+  #   kind: equal
+  #   value: False
+  #
+  # v1.X also applied a rough pointing-coverage cut, 'npoint3' >= 3, from an
+  # external post-processing catalogue. ShapePipe v2 emits no such column and
+  # no MASK_n* bit encodes it, so that cut has no v2 counterpart; add it back
+  # here if an external pointing-coverage map is ever joined onto the
+  # catalogue (it would belong in the 'dat_ext' group below).
+
+  # Number of epochs
+  - col_name: N_EPOCH
+    label: r"$n_{\rm epoch}$"
+    kind: greater_equal
+    value: 2
+
+  # Magnitude range
+  - col_name: mag
+    label: mag range
+    kind: range
+    value: [15, 30]
+
+  # ngmix flags
+  - col_name: NGMIX_MCAL_TYPES_FAIL
+    label: "ngmix moments failure"
+    kind: equal
+    value: 0
+
+  # Objects ngmix never fit. ShapePipe's make_cat pre-fills the NGMIX_*
+  # columns with sentinels (G1/G2 = -10, T/FLUX = 0) and overwrites them only
+  # for objects present in the ngmix output, so a never-fit object also keeps
+  # NGMIX_MCAL_FLAGS = 0 and is not caught by any flag cut. In
+  # final_cat_smk-g7.hdf5 that is 18,983 of 1,851,100 objects (1.03%);
+  # admitting them gives mean e1 = -0.107 (std 1.03) against -0.004 (std
+  # 0.22). The -10 cuts below do reject them today, but only because make_cat
+  # happens to use the same literal for the PSF columns: that is an exact
+  # float equality against a sentinel ShapePipe may change, so state the
+  # condition directly. Mirrors the guard in
+  # sp_validation.galaxy.classification_galaxy_ngmix.
+  - col_name: NGMIX_N_EPOCH
+    label: "ngmix never fit"
+    kind: greater_equal
+    value: 1
+
+  # invalid PSF ellipticities
+  - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR
+    label: "bad PSF ellipticity comp 1"
+    kind: not_equal
+    value: -10
+  - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR
+    label: "bad PSF ellipticity comp 2"
+    kind: not_equal
+    value: -10
+
+## Using columns in 'dat_ext' group (post-processing flags)
+## ShapePipe v2 carries the imaging masks in the 'dat' group above, so this
+## group is empty unless external masks are added.
+dat_ext: []
+
+# Metacal parameters
+metacal:
+  # Ellipticity dispersion
+  sigma_eps_prior: 0.34
+
+  # Signal-to-noise range
+  gal_snr_min: 10
+  gal_snr_max: 500
+
+  # Relative-size (hlr / hlr_psf) range
+  gal_rel_size_min: 0.707
+  gal_rel_size_max: 3
+
+  # Correct relative size for ellipticity?
+  gal_size_corr_ell: False
+
+  # Weight for global response matrix, None for unweighted mean
+  global_R_weight: w
diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md
@@ -3,8 +3,8 @@
 ## Science-ready catalogue production
 
 Processing steps of `ShapePipe` output catalogues carried out by the `sp_validation` package to produce science-ready catalogues are:
-1. Extract relevant information from a final `ShapePipe` output catalogue per patch; run basic diagnostic tests, create pre-calibration shear catalogues.
-2. Merge pre-calibration catalogues created in the previous step, e.g. processed by individual patches, into one or more joint catalogues;
+1. Extract relevant information from a final `ShapePipe` output catalogue per campaign; run basic diagnostic tests, create pre-calibration shear catalogues.
+2. Merge pre-calibration catalogues created in the previous step, e.g. processed as individual campaigns, into one or more joint catalogues;
 3. Apply external area and footprint masks. These are the "structural" and the coverage masks.  
 4. Create calibrated galaxy shear catalogue. This step includes the tasks:  
    a. Mask objects using flags and criteria in `ShapePipe` output catalogues and external (e.g. mask) files;  
@@ -19,19 +19,19 @@ This is performed (version > v1.4.1, < v2.0) with the python script `scripts/cal
 
 This script creates three shear catalogues in FITS format:
 - _Basic_ catalogue containing
-  positions, shapes (calibrated +  PSF-leakage corrected), weights (DES), magnitude, patch ID. Masking and galaxy selection are applied.  
+  positions, shapes (calibrated +  PSF-leakage corrected), weights (DES), magnitude, campaign ID. Masking and galaxy selection are applied.  
 - _Extended_ catalogue containing **in addition**
   uncalibrated shapes inverse-variance weights, shear response matrices, SNR, flux, size, PSF quantities. Masking and galaxy selection are applied.  
 - _Comprehensive_ catalogue containing **in addition**
   metacal information (measured sheared quantities), mask information (`shapepipe` pre-processing). Masking and galaxy selection is not applied.
   This catalogue does not contain calibrated shear estimates, since the calibration is carried out after applying masking and selection.  
   This is the main output catalogue that will be processed further.
 
-This step is carried out per patch. Parameters have to be set via the python configuration file `params.py` (template at `scripts/calibration/params.py`).
+This step is carried out per campaign. Parameters have to be set via the python configuration file `params.py` (template at `scripts/calibration/params.py`).
 
 ### 2. Merge catalogues
 
-The patch-wise comprehensive catalogues extracted in the previous step are merged using the script `scripts/calibration/create_joint_comprehensive_cat.py`, which is a front-end
+The per-campaign comprehensive catalogues extracted in the previous step are merged using the script `scripts/calibration/create_joint_comprehensive_cat.py`, which is a front-end
 of the `sp_validation` library class `catalog_builders:JointCat`.
 
 ### 3. Apply external masks
@@ -57,16 +57,16 @@ The following describes the pre-v1.4.2 method to create a joint, calibrated shea
 Summary statistics created by shear validation runs of sub-areas of a survey
 can be combined to create joint summary statistics. This is useful in cases
 where the galaxy catalogue of an entire survey is too large to process, and
-needs to be broken down in smaller patches. This step provides global summary
-statistics from those patches.
+needs to be broken down into smaller campaigns. This step provides global
+summary statistics from those campaigns.
 
 Depending on the type of summary, their combination can be the sum (e.g. for
 number of objects), average, weighted average (e.g. for the additive bias), the
 weighted average of the square (e.g. the ellipticity dispersion), the weighted
 variance (to combine variance estimates), or the weighted variance of the mean
 (to combine mean variance estimates).
 
-In a directory containing the subpatches as subdirectories, and within each
+In a directory containing the campaigns as subdirectories, and within each
 their own output directory (`sp_output`by default in `params.py`) with results
 of the validation runs, type
 ```bash
@@ -89,7 +89,7 @@ calibration outputs can be used to create a combined, globally calibrated shear
 catalogue. The calibration is obtained from the files `R.txt` and `c.txt`
 created above.
 
-In the same directory containing the subpatches as above, type
+In the same directory containing the campaigns as above, type
 ```bash
 create_joint_shape_cat.py
 ```

diff --git a/docs/source/run_validation.md b/docs/source/run_validation.md
@@ -12,7 +12,7 @@ including the sheared values for metacalibration.
 All inputs and settings are contained in the python configuration script
 `scripts/calibration/params.py`, that needs to be edited accordingly.
 The main parameters are:
-- `name`: field or patch name, can be any string. E.g. `P3` for patch 3. 
+- `campaign`: campaign name (the ShapePipe tile list), can be any string. 
 - `data_dir`: input directory for data. Set to `.` for validation run in
   current directory.
 - `galaxy_cat_path`: path to galaxy catalogue, format `.fits`. or `.hdf5`.
@@ -27,8 +27,9 @@ Optional parameters are:
 - `mask_external_path`: path to external mask file, format `.reg`. Set to
   `None` if not required.  
 
-See the script `prepare_patch_for_spval.sh` for an example of copying
-the required input files to where the validation is to be run.
+Link or copy the campaign's merged products -- `final_cat_<campaign>.hdf5`
+and `full_starcat_<campaign>.hdf5` -- into the directory where the validation
+is to be run.
 
 
 ### Run

diff --git a/docs/source/using_the_catalogues.md b/docs/source/using_the_catalogues.md
@@ -16,7 +16,7 @@ how to apply the metacalibration corrections yourself.
 ```{note}
 The examples below target catalogue **v1.0** (April 2022), which is distributed
 as FITS. From ShapePipe catalogue **v1.4.1** onward the merged catalogues ship
-as HDF5 instead; open those with {func}`sp_validation.io.read_hdf5_file` (or
+as HDF5 instead; open those with {func}`sp_validation.catalog.read_campaign_catalogue` (or
 `h5py` / `astropy`) in place of `astropy.io.fits` below — the column names and
 the calibration recipe are unchanged.
 ```
@@ -167,7 +167,7 @@ mask = np.full(len(data_ext), True)
 
 # Other examples:
 # mask = data_ext['mask_extern'] == 0  # LensFit-unmasked regions
-# mask = data_ext['patch'] == 3        # patch P3
+# mask = data_ext['campaign'] == b'W3'   # objects from campaign W3
 # mask = data_ext['mag'] < 23.5        # r-band magnitude cut
 
 n_kept, n_all = np.count_nonzero(mask), len(data_ext)

diff --git a/scripts/calibration/README.md b/scripts/calibration/README.md
@@ -6,14 +6,13 @@ in order. See `docs/source/post_processing.md` for the full prose.
 
 | Step | Script | Does |
 |------|--------|------|
-| 1 | `extract_info.py` | Extract metacal + diagnostic info per patch; create pre-calibration shear catalogues. Configured via `params.py`. |
-| 2 | `create_joint_comprehensive_cat.py` | Merge the patch-wise comprehensive catalogues into one joint catalogue (front-end of `catalog_builders.JointCat`). |
+| 1 | `extract_info.py` | Extract metacal + diagnostic info for one campaign; create pre-calibration shear catalogues. Configured via `params.py`. |
+| 2 | `create_joint_comprehensive_cat.py` | Merge the per-campaign comprehensive catalogues into one joint catalogue (front-end of `catalog_builders.JointCat`). |
 | 3 | `demo_apply_hsp_masks.py` | Add the structural and coverage (HealSparse) masks. |
 | 4 | `calibrate_comprehensive_cat.py` | Galaxy selection + metacalibration. Uses the mask configs in `config/calibration/`. |
 
 `params.py` is the shared parameter template (paths, column names, survey
 constants) imported by `extract_info.py`; copy and edit it per run.
 
-> **v2.0 note (Martin):** this clustering reflects the current (v1.4.x) reduction
-> flow. v2.0 no longer has patches, so step 2 (and the per-patch structure of
-> steps 1/3) will change.
+> **v2.0 note:** ShapePipe v2 has no patches. Step 1 runs per campaign and
+> step 2 merges a list of campaign catalogues (`final_cat_<campaign>.hdf5`).
diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py
@@ -34,10 +34,10 @@
 
 import h5py
 import numpy as np
-from astropy.io import fits
 
 # from sp_validation.catalog import *
 from sp_validation import catalog as spv_cat
+from sp_validation import galaxy
 from sp_validation.calibration import *
 from sp_validation.calibration import metacal
 from sp_validation.galaxy import *
@@ -69,9 +69,7 @@
     dd = np.load(galaxy_cat_path, mmap_mode=mmap_mode)
 else:
     print("Loading galaxy .hdf5 file...")
-    dd = spv_cat.read_hdf5_file(
-        galaxy_cat_path, name, stats_file, param_path=param_list_path
-    )
+    dd = spv_cat.read_campaign_catalogue(galaxy_cat_path, param_path=param_list_path)
 
 n_obj = len(dd)
 print_stats(
@@ -116,7 +114,7 @@
 # ### Load star catalogue
 
 if star_cat_path:
-    d_star = fits.getdata(star_cat_path, hdu_star_cat)
+    d_star = spv_cat.read_star_catalogue(star_cat_path, hdu=hdu_star_cat)
 
 if star_cat_path:
     print_stats("Stars:", stats_file, verbose=verbose)
@@ -141,14 +139,13 @@
 # #### Match to all objects
 
 if star_cat_path:
-    ind_star, mask_area_tiles, n_star_tot = spv_cat.check_matching(
+    ind_star, n_star_tot = spv_cat.check_matching(
         d_star,
         dd,
         ["RA", "DEC"],
         [col_name_ra, col_name_dec],
         thresh,
         stats_file,
-        name=None,
         verbose=verbose,
     )
 
@@ -160,7 +157,7 @@
 
     m_star = (
         (dd["FLAGS"][ind_star] == 0)
-        & (dd["IMAFLAGS_ISO"][ind_star] == 0)
+        & galaxy.mask_cut(dd, mask_columns)[ind_star]
         & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0)
         & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10)
     )
@@ -311,6 +308,7 @@
     gal_mag_faint=gal_mag_faint,
     flags_keep=flags_keep,
     n_epoch_min=n_epoch_min,
+    mask_columns=mask_columns,
 )
 if shape == "ngmix":
     m_gal = classification_galaxy_ngmix(
@@ -490,7 +488,7 @@
 write_tile_id_gal_counts(detection_IDs, galaxy_IDs, shape_IDs, fname)
 
 # +
-# Add all weights (for combining weighted averages of subpatches)
+# Add all weights (for combining weighted averages of sub-samples)
 
 w_tot = np.sum(w)