DM-55233: Add prompt_source functors

schemas/prompt_source.yaml

@@ -0,0 +1,271 @@
+# This file defines the mapping between the columns in a single visit+detector
+# source table and their respective DPDD-style column names, as used by
+# `lsst.pipe.tasks.postprocess.TransformSourceTableTask`.
+# See the DPDD for more information about the output: https://lse-163.lsst.io
+funcs:
+    sourceId:
+        functor: Column
+        args: id
+    parentSourceId:
+        functor: Column
+        args: parent
+    # htmId20: not available
+    x:
+        functor: Column
+        args: slot_Centroid_x
+    y:
+        functor: Column
+        args: slot_Centroid_y
+    xErr:
+        functor: SinglePrecisionFloatColumn
+        args: slot_Centroid_xErr
+    yErr:
+        functor: SinglePrecisionFloatColumn
+        args: slot_Centroid_yErr
+    # x_y_Cov: not available
+    ra:
+        functor: RAColumn
+    dec:
+        functor: DecColumn
+    raErr:
+        functor: RAErrColumn
+    decErr:
+        functor: DecErrColumn
+    ra_dec_Cov:
+        functor: RADecCovColumn
+    # One calibrated Calib flux is important:
+    calibFlux:
+        functor: LocalNanojansky
+        args:
+            - slot_CalibFlux_instFlux
+            - slot_CalibFlux_instFluxErr
+            - base_LocalPhotoCalib
+    calibFluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - slot_CalibFlux_instFlux
+            - slot_CalibFlux_instFluxErr
+            - base_LocalPhotoCalib
+    ap09Flux:
+        functor: LocalNanojansky
+        args:
+            - base_CircularApertureFlux_9_0_instFlux
+            - base_CircularApertureFlux_9_0_instFluxErr
+            - base_LocalPhotoCalib
+    ap09FluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - base_CircularApertureFlux_9_0_instFlux
+            - base_CircularApertureFlux_9_0_instFluxErr
+            - base_LocalPhotoCalib
+    ap12Flux:
+        functor: LocalNanojansky
+        args:
+            - base_CircularApertureFlux_12_0_instFlux
+            - base_CircularApertureFlux_12_0_instFluxErr
+            - base_LocalPhotoCalib
+    ap12FluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - base_CircularApertureFlux_12_0_instFlux
+            - base_CircularApertureFlux_12_0_instFluxErr
+            - base_LocalPhotoCalib
+    ap25Flux:
+        functor: LocalNanojansky
+        args:
+            - base_CircularApertureFlux_25_0_instFlux
+            - base_CircularApertureFlux_25_0_instFluxErr
+            - base_LocalPhotoCalib
+    ap25FluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - base_CircularApertureFlux_25_0_instFlux
+            - base_CircularApertureFlux_25_0_instFluxErr
+            - base_LocalPhotoCalib
+    # TODO: When DM-25019 is complete, these should be
+    # changed to use the local value of the background
+    # model, rather than the residual of the background
+    sky:
+        functor: LocalNanojansky
+        args:
+            - base_LocalBackground_instFlux
+            - base_LocalBackground_instFluxErr
+            - base_LocalPhotoCalib
+    skyErr:
+        functor: LocalNanojanskyErr
+        args:
+            - base_LocalBackground_instFlux
+            - base_LocalBackground_instFluxErr
+            - base_LocalPhotoCalib
+    psfFlux:
+        functor: LocalNanojansky
+        args:
+            - slot_PsfFlux_instFlux
+            - slot_PsfFlux_instFluxErr
+            - base_LocalPhotoCalib
+    psfFluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - slot_PsfFlux_instFlux
+            - slot_PsfFlux_instFluxErr
+            - base_LocalPhotoCalib
+
+    # These PS columns do not make sense anymore as named
+    # psX
+    # psXSigma
+    # psY
+    # psYSigma
+    # psFlux_psX_Cov
+    # psFlux_psY_Cov
+    # psX_Y_Cov
+    # psLnL
+    # psChi2
+    # psN
+    # psRa
+    # psRaSigma
+    # psDecl
+    # psDeclSigma
+    # psFlux_psRa_Cov
+    # psFlux_psDecl_Cov
+
+    ixx:
+        functor: SinglePrecisionFloatColumn
+        args: slot_Shape_xx
+    iyy:
+        functor: SinglePrecisionFloatColumn
+        args: slot_Shape_yy
+    ixy:
+        functor: SinglePrecisionFloatColumn
+        args: slot_Shape_xy
+    # DPDD should include Psf Shape
+    ixxPSF:
+        functor: SinglePrecisionFloatColumn
+        args: slot_PsfShape_xx
+    iyyPSF:
+        functor: SinglePrecisionFloatColumn
+        args: slot_PsfShape_yy
+    ixyPSF:
+        functor: SinglePrecisionFloatColumn
+        args: slot_PsfShape_xy
+    ixxDebiasedPSF:
+        functor:  SinglePrecisionFloatColumn
+        args: ext_shapeHSM_HsmPsfMomentsDebiased_xx
+    iyyDebiasedPSF:
+        functor:  SinglePrecisionFloatColumn
+        args: ext_shapeHSM_HsmPsfMomentsDebiased_yy
+    ixyDebiasedPSF:
+        functor:  SinglePrecisionFloatColumn
+        args: ext_shapeHSM_HsmPsfMomentsDebiased_xy
+    # apNann: Replaced by raw Aperture instFluxes in flags section below
+    # apMeanSb: Replaced by raw Aperture instFluxes in flags section below
+    # apMeanSbErr: Replaced by raw Aperture instFluxes in flags section below
+
+    # DPDD does not include gaussianFluxes, however they are used for
+    # the single frame extendedness column which is used for QA.
+    gaussianFlux:
+        functor: LocalNanojansky
+        args:
+            - base_GaussianFlux_instFlux
+            - base_GaussianFlux_instFluxErr
+            - base_LocalPhotoCalib
+    gaussianFluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - base_GaussianFlux_instFlux
+            - base_GaussianFlux_instFluxErr
+            - base_LocalPhotoCalib
+    extendedness:
+        functor: SinglePrecisionFloatColumn
+        args: base_ClassificationExtendedness_value
+    sizeExtendedness:
+        functor: SinglePrecisionFloatColumn
+        args: base_ClassificationSizeExtendedness_value
+flags:
+   - base_CircularApertureFlux_9_0_flag
+   - base_CircularApertureFlux_12_0_flag
+   - base_CircularApertureFlux_25_0_flag
+   - base_NormalizedCompensatedTophatFlux_flag
+   - base_NormalizedCompensatedTophatFlux_instFlux
+   - base_NormalizedCompensatedTophatFlux_instFluxErr
+   - base_ClassificationExtendedness_flag
+   - base_ClassificationSizeExtendedness_flag
+   - base_FootprintArea_value
+   - base_InvalidPsf_flag
+   - base_LocalBackground_instFlux
+   - base_LocalBackground_instFluxErr
+   - base_LocalBackground_flag
+   - base_LocalBackground_flag_noGoodPixels
+   - base_LocalBackground_flag_noPsf
+   - base_PixelFlags_flag_bad
+   - base_PixelFlags_flag_cr
+   - base_PixelFlags_flag_crCenter
+   - base_PixelFlags_flag_edge
+   - base_PixelFlags_flag_interpolated
+   - base_PixelFlags_flag_interpolatedCenter
+   - base_PixelFlags_flag_nodata
+   - base_PixelFlags_flag_offimage
+   - base_PixelFlags_flag_saturated
+   - base_PixelFlags_flag_saturatedCenter
+   - base_PixelFlags_flag_suspect
+   - base_PixelFlags_flag_suspectCenter
+   # Streak flags not yet propagated from compareWarp (DM-46947, DM-43077).
+   # - base_PixelFlags_flag_streak
+   # - base_PixelFlags_flag_streakCenter
+   - base_PsfFlux_apCorr
+   - base_PsfFlux_apCorrErr
+   - base_PsfFlux_area
+   - base_PsfFlux_flag
+   - base_PsfFlux_flag_apCorr
+   - base_PsfFlux_flag_edge
+   - base_PsfFlux_flag_noGoodPixels
+   - base_GaussianFlux_flag
+   - base_SdssCentroid_flag
+   - base_SdssCentroid_flag_almostNoSecondDerivative
+   - base_SdssCentroid_flag_badError
+   - base_SdssCentroid_flag_edge
+   - base_SdssCentroid_flag_noSecondDerivative
+   - base_SdssCentroid_flag_notAtMaximum
+   - base_SdssCentroid_flag_resetToPeak
+   - deblend_deblendedAsPsf
+   - deblend_hasStrayFlux
+   - deblend_masked
+   - deblend_nChild
+   - deblend_parentTooBig
+   - deblend_patchedTemplate
+   - deblend_rampedTemplate
+   - deblend_skipped
+   - deblend_tooManyPeaks
+   - ext_shapeHSM_HsmPsfMomentsDebiased_flag
+   - ext_shapeHSM_HsmPsfMomentsDebiased_flag_no_pixels
+   - ext_shapeHSM_HsmPsfMomentsDebiased_flag_not_contained
+   - ext_shapeHSM_HsmPsfMomentsDebiased_flag_parent_source
+   - ext_shapeHSM_HsmPsfMomentsDebiased_flag_galsim
+   - ext_shapeHSM_HsmPsfMomentsDebiased_flag_edge
+   - sky_source
+   - detect_isPrimary
+
+flag_rename_rules:
+    # Taken from db-meas-forced
+    - ['ext_photometryKron_', '']
+    - ['base_CircularApertureFlux_9_0_flag', 'ap09Flux_flag']
+    - ['base_CircularApertureFlux_12_0_flag', 'ap12Flux_flag']
+    - ['base_CircularApertureFlux_25_0_flag', 'ap25Flux_flag']
+    - ['base_Local', 'local']
+    - ['base_PixelFlags_flag', 'pixelFlags']
+    - ['base_ClassificationE', 'e']
+    - ['base_ClassificationS', 's']
+    - ['base_SdssCentroid', 'centroid']
+    - ['base_Psf', 'psf']
+    - ['base_InvalidPsf_flag', 'invalidPsfFlag']
+    - ['base_GaussianFlux', 'gaussianFlux']
+    - ['base_CircularApertureFlux', 'apFlux']
+    - ['base_NormalizedCompensatedTophatFlux', 'normCompTophatFlux']
+    - ['base_FootprintArea', 'footprintArea']
+    - ['ext_shapeHSM_HsmPsfMomentsDebiased', 'iDebiasedPSF']
+    - ['ext_shapeHSM_Hsm', 'hsm']
+    - ['ext_convolved_', '']
+    - ['undeblended_base', 'undeblended']
+    - ['undeblended_ext_photometryKron', 'undeblended']
+    - ['ext_photometryKron_', '']
+    - ['base_', '']
+    - ['slot_', '']

tests/test_promptSourceSchema.py

@@ -0,0 +1,117 @@
+# This file is part of pipe_tasks.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Check that the prompt_source pipeline produces exactly the columns defined
+by the ``PromptSource`` table in ``sdm_schemas``. Only check column names, not
+datatypes, since that requires running the detection and measurement task
+which can't be imported here.
+"""
+
+import os
+import unittest
+
+import numpy as np
+from astropy.table import Table
+import importlib.resources
+
+import lsst.utils.tests
+from lsst.utils import getPackageDir
+
+from lsst.pipe.tasks.postprocess import TransformSourceTableTask, TransformSourceTableConfig
+from lsst.pipe.tasks.schemaUtils import readSdmSchemaFile
+from lsst.pipe.tasks.split_primary import SplitPrimaryTask
+
+FUNCTOR_FILE = os.path.join(getPackageDir("pipe_tasks"), "schemas", "prompt_source.yaml")
+
+SCHEMA_FILE = importlib.resources.files("lsst.sdm.schemas") / "ap_extra.yaml"
+TABLE_NAME = "PromptSource"
+
+
+class PromptSourceSchemaTestCase(lsst.utils.tests.TestCase):
+    """Check the persisted prompt_source columns against the PromptSource DDL.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+
+        schemaFile = os.path.expandvars(SCHEMA_FILE)
+        if not os.path.exists(schemaFile):
+            raise unittest.SkipTest(f"SDM schema file not available: {SCHEMA_FILE}")
+
+        schema = readSdmSchemaFile(schemaFile)
+        if TABLE_NAME not in schema:
+            raise ValueError(f"Table {TABLE_NAME!r} not in {schemaFile}.")
+
+        cls.schemaColumns = {column.name for column in schema[TABLE_NAME].columns}
+
+        config = TransformSourceTableConfig()
+        config.functorFile = FUNCTOR_FILE
+        transformTask = TransformSourceTableTask(config=config)
+        producedColumns = set(transformTask.funcs.funcDict) | set(config.columnsFromDataId)
+
+        # The final prompt_source table is output by splitPromptSource, which is configured
+        # in the pipeline to drop sky sources and remove the corresponding column.
+        splitConfig = SplitPrimaryTask.ConfigClass()
+        splitConfig.discard_primary_columns = ["sky_source"]
+        splitTask = SplitPrimaryTask(config=splitConfig)
+
+        # SplitPrimaryTask only needs the boolean primary-flag column to mask
+        # rows; the other columns are carried through (or dropped) by name, so
+        # a minimal two-row table with placeholder values is sufficient.
+        data = {}
+        for name in producedColumns:
+            if name == splitConfig.primary_flag_column:
+                data[name] = np.array([True, False])
+            else:
+                data[name] = np.zeros(2)
+        full = Table(data)
+
+        cls.persistedColumns = set(splitTask.run(full=full).primary.colnames)
+
+    def testColumnsConform(self):
+        """The persisted prompt_source columns must equal the schema exactly.
+        """
+        missing = self.schemaColumns - self.persistedColumns
+        extra = self.persistedColumns - self.schemaColumns
+
+        message = (
+            f"persisted prompt_source columns do not conform to the "
+            f"{TABLE_NAME} schema.\n"
+            f"  In schema but not persisted ({len(missing)}): "
+            f"{sorted(missing)}\n"
+            f"  Persisted but not in schema ({len(extra)}): "
+            f"{sorted(extra)}"
+        )
+        self.assertEqual(self.persistedColumns, self.schemaColumns, message)
+
+
+class MemoryTester(lsst.utils.tests.MemoryTestCase):
+    pass
+
+
+def setup_module(module):
+    lsst.utils.tests.init()
+
+
+if __name__ == "__main__":
+    lsst.utils.tests.init()
+    unittest.main()