Wire the UNIONS external healsparse masks into the workflow (MASK_n* columns)

src/shapepipe/pipeline/file_io.py

@@ -1568,7 +1568,7 @@ def _get_fits_col_type(self, col_data):
             col_type = "K"
         elif type(col_data[0]) in [float, np.float16, np.float32, np.float64]:
             col_type = "D"
-        elif type(col_data[0]) is bool:
+        elif type(col_data[0]) in [bool, np.bool_]:
             col_type = "L"
         elif type(col_data[0]) in [str, np.str_]:
             col_type = "A"

workflow/README.md

@@ -221,6 +221,16 @@ profiles/nibi/config.yaml  SLURM executor; apptainer SDM; per-user jobs cap; kee
   catalogue server, staged, or rasterized, which is why the old
   `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are
   gone.
+- **External masks are wired, on the tile side only.** `inputs.masks` is a third
+  input root beside tiles and exposures, exported as `$SP_INPUT_MASKS` and
+  pointing at the UNIONS DR6 ugriz bit ladder: one boolean healsparse map per
+  bit, nside 131072, `True` = masked. `config_tile_Mc.ini` names all 11 of them
+  in `MASK_EXT_PATHS`, so `make_cat` writes `MASK_n1` … `MASK_n2048` and
+  `final_cat.param` carries the matching 11 names. That file and the config
+  hold the bit table and the two caveats (the August bit-0/1 halo swap is
+  unconfirmed, so use `n1|n2` combined; `n2048` is 1 where there is *no*
+  Pan-STARRS z2 data, so an OR over every column masks everything). Nothing
+  cuts on them here.
 - **The index is parse-time data, never a rule input.** Appending tiles
   changes which jobs exist without invalidating completed work.
 - **Exposure products are not `temp()`.** Exposures overlap tiles, so

workflow/Snakefile

@@ -1000,9 +1000,9 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None,
     There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at
     the committed config dir.
 
-    It also exports the configured input roots as ``SP_INPUT_TILES`` and
-    ``SP_INPUT_EXPOSURES`` and the PSF choice as ``SP_PSF`` for the committed
-    ini chain.
+    It also exports the configured input roots as ``SP_INPUT_TILES``,
+    ``SP_INPUT_EXPOSURES`` and ``SP_INPUT_MASKS`` and the PSF choice as
+    ``SP_PSF`` for the committed ini chain.
 
     Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's
     FileHandler raises on an existing run dir, and it is how a rerun never sees
@@ -1021,6 +1021,7 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None,
         f"export SP_CONFIG='{CONFIG_DIR}'",
         f"export SP_INPUT_TILES='{INPUTS['tiles']}'",
         f"export SP_INPUT_EXPOSURES='{INPUTS['exposures']}'",
+        f"export SP_INPUT_MASKS='{INPUTS['masks']}'",
         f"export SP_PSF='{PSF_MODEL}'",
         # Also set via apptainer-args in the profile; kept here so a hand-run of
         # this same line outside snakemake behaves identically.

workflow/config.yaml

@@ -19,11 +19,16 @@
 tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt
 
 # Inputs are the only site-specific data paths; all committed configs consume these
-# through SP_INPUT_TILES and SP_INPUT_EXPOSURES.
+# through SP_INPUT_TILES, SP_INPUT_EXPOSURES and SP_INPUT_MASKS.
 inputs:
   # Pre-staged P3 data (get_images RETRIEVE=symlink).
   tiles: /project/def-mjhudson/unions-wl/tiles
   exposures: /project/def-mjhudson/unions-wl/exposures
+  # UNIONS sky-fixed masks: one boolean healsparse map per bit of the
+  # band-combined ugriz mask (nside 131072, True = masked), queried per object
+  # by make_cat -- never rasterized. Consumed by MASK_EXT_PATHS in
+  # config_tile_Mc.ini, which carries the bit table.
+  masks: /project/def-mjhudson/unions-wl/masks/dr6-2026-08
 
 # The container every job runs inside (apptainer software-deployment in the profile).
 container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif

workflow/config/cfis/config_tile_Mc.ini

@@ -82,3 +82,39 @@ NUMBERING_SCHEME = -000-000
 SM_DO_CLASSIFICATION = False
 
 SHAPE_MEASUREMENT_TYPE = ngmix
+
+# The UNIONS sky-fixed masks, queried once per object and written as one
+# MASK_<label> column per bit onto the final catalogue. $SP_INPUT_MASKS is the
+# `inputs.masks` root the workflow exports (workflow/config.yaml). The products
+# are the post-GSC2-fix band-combined ugriz ladder: ONE BOOLEAN MAP PER BIT,
+# nside 131072, True = masked. Labels are `n<bit>`, so the columns land as
+# MASK_n1 ... MASK_n2048 — matched line for line in final_cat.param.
+#
+# Bit table (Ludo, mask-force telecon 2026-07-21; also PR #847):
+#   n1    bit 0   faint star halo
+#   n2    bit 1   bright star halo
+#   n4    bit 2   star body
+#   n8    bit 3   manual mask
+#   n16   bit 4   u footprint   \
+#   n32   bit 5   g footprint    |  1 = NO data in that filter,
+#   n64   bit 6   r footprint    |  i.e. a coverage flag, not a defect
+#   n128  bit 7   i footprint    |
+#   n256  bit 8   z footprint   /
+#   (bit 9 = tile RA/Dec cut: GEOMETRIC, no product exists — hence 11 maps)
+#   n1024 bit 10  MaxiMask
+#   n2048 bit 11  Pan-STARRS z2 footprint (1 = NO PS z2 data)
+#
+# TWO CAVEATS, both load-bearing downstream:
+#  1. The August regeneration exchanged the content of bits 0 and 1 relative to
+#     July, and the mask authors have not confirmed which is now faint and which
+#     is bright. Use n1|n2 COMBINED; do not attribute either one alone.
+#  2. n2048 is 1 where there is NO PS z2 data, so it reads 1 across whole tiles
+#     outside that footprint — a naive OR over all MASK_ columns masks
+#     everything. Likewise, a position with no map coverage reads boolean False,
+#     indistinguishable from clean. Every cut on these columns is a downstream
+#     decision; nothing here rejects an object.
+#
+# PSF-star selection deliberately does NOT consume these: MASK_PATHS in
+# config_exp_psfex.ini stays commented out, keeping the star diet narrow
+# (instrument flags only). See that file's header.
+MASK_EXT_PATHS = n1:$SP_INPUT_MASKS/mask_ugriz_nside131072_n1.hsp, n2:$SP_INPUT_MASKS/mask_ugriz_nside131072_n2.hsp, n4:$SP_INPUT_MASKS/mask_ugriz_nside131072_n4.hsp, n8:$SP_INPUT_MASKS/mask_ugriz_nside131072_n8.hsp, n16:$SP_INPUT_MASKS/mask_ugriz_nside131072_n16.hsp, n32:$SP_INPUT_MASKS/mask_ugriz_nside131072_n32.hsp, n64:$SP_INPUT_MASKS/mask_ugriz_nside131072_n64.hsp, n128:$SP_INPUT_MASKS/mask_ugriz_nside131072_n128.hsp, n256:$SP_INPUT_MASKS/mask_ugriz_nside131072_n256.hsp, n1024:$SP_INPUT_MASKS/mask_ugriz_nside131072_n1024.hsp, n2048:$SP_INPUT_MASKS/mask_ugriz_nside131072_n2048.hsp

workflow/config/cfis/final_cat.param

@@ -8,26 +8,34 @@ TILE_ID
 
 # flags
 FLAGS
-# NO IMAFLAGS_ISO, AND NO MASK COLUMN AT ALL — READ THIS BEFORE ADDING ONE.
+# NO IMAFLAGS_ISO — the MASK_n* columns below replace it.
 # The tile-side SExtractor runs with FLAG_IMAGE = False and DOT_PARAM_FILE =
 # default_noimaflags.param (config_tile_Sx.ini), so IMAFLAGS_ISO is never
 # written into a tile catalogue and asking for it here only made the merge
 # fail. Instrument flags reach the pipeline on the EXPOSURE side, where
 # exp_split delivers the flag image and SExtractor reads it.
 #
-# Its intended replacement is make_cat's per-band MASK_<band> columns, queried
-# from the sky-fixed healsparse maps named by MASK_EXT_PATHS. THE WORKFLOW SETS
-# NO SUCH PATHS: config_tile_Mc.ini has no MASK_EXT_PATHS entry, so
-# save_mask_ext_data is never called, no MASK_<band> column exists in any tile
-# catalogue this workflow has produced, and smk-g6's carry none (checked).
-# Naming one here would fail every merge on every campaign.
+# Its replacement is make_cat's per-bit MASK_n<bit> columns, queried from the
+# sky-fixed UNIONS healsparse maps named by MASK_EXT_PATHS in
+# config_tile_Mc.ini — which carries the bit table and the caveats. Those 11
+# names and these 11 lines must stay in step: a name here that make_cat did not
+# write fails every merge, on every campaign.
 #
-# So the merged catalogue carries NO mask information today, and that is a
-# CONFIG gap and not a gap in this file: turning it on is setting
-# MASK_EXT_PATHS in config_tile_Mc.ini (`band:path` pairs, the same grammar as
-# the commented MASK_PATHS in config_exp_psfex.ini) and adding the matching
-# MASK_<band> names here, in that order. No healsparse map is staged under
-# /project/def-mjhudson yet.
+# sp_validation still cuts on IMAFLAGS_ISO; that cut has to move onto these
+# columns (issue to follow), and it is NOT a plain OR — n2048 is 1 where there
+# is no Pan-STARRS z2 data, so OR-ing every column masks the whole survey.
+MASK_n1
+MASK_n2
+MASK_n4
+MASK_n8
+MASK_n16
+MASK_n32
+MASK_n64
+MASK_n128
+MASK_n256
+MASK_n1024
+MASK_n2048
+
 NGMIX_MCAL_FLAGS
 
 # PSF ellipticity (original image PSF)