diff --git a/schemas/prompt_source.yaml b/schemas/prompt_source.yaml new file mode 100644 index 000000000..518de4cfd --- /dev/null +++ b/schemas/prompt_source.yaml @@ -0,0 +1,271 @@ +# This file defines the mapping between the columns in a single visit+detector +# source table and their respective DPDD-style column names, as used by +# `lsst.pipe.tasks.postprocess.TransformSourceTableTask`. +# See the DPDD for more information about the output: https://lse-163.lsst.io +funcs: + sourceId: + functor: Column + args: id + parentSourceId: + functor: Column + args: parent + # htmId20: not available + x: + functor: Column + args: slot_Centroid_x + y: + functor: Column + args: slot_Centroid_y + xErr: + functor: SinglePrecisionFloatColumn + args: slot_Centroid_xErr + yErr: + functor: SinglePrecisionFloatColumn + args: slot_Centroid_yErr + # x_y_Cov: not available + ra: + functor: RAColumn + dec: + functor: DecColumn + raErr: + functor: RAErrColumn + decErr: + functor: DecErrColumn + ra_dec_Cov: + functor: RADecCovColumn + # One calibrated Calib flux is important: + calibFlux: + functor: LocalNanojansky + args: + - slot_CalibFlux_instFlux + - slot_CalibFlux_instFluxErr + - base_LocalPhotoCalib + calibFluxErr: + functor: LocalNanojanskyErr + args: + - slot_CalibFlux_instFlux + - slot_CalibFlux_instFluxErr + - base_LocalPhotoCalib + ap09Flux: + functor: LocalNanojansky + args: + - base_CircularApertureFlux_9_0_instFlux + - base_CircularApertureFlux_9_0_instFluxErr + - base_LocalPhotoCalib + ap09FluxErr: + functor: LocalNanojanskyErr + args: + - base_CircularApertureFlux_9_0_instFlux + - base_CircularApertureFlux_9_0_instFluxErr + - base_LocalPhotoCalib + ap12Flux: + functor: LocalNanojansky + args: + - base_CircularApertureFlux_12_0_instFlux + - base_CircularApertureFlux_12_0_instFluxErr + - base_LocalPhotoCalib + ap12FluxErr: + functor: LocalNanojanskyErr + args: + - base_CircularApertureFlux_12_0_instFlux + - base_CircularApertureFlux_12_0_instFluxErr + - base_LocalPhotoCalib + ap25Flux: + functor: LocalNanojansky + args: + - base_CircularApertureFlux_25_0_instFlux + - base_CircularApertureFlux_25_0_instFluxErr + - base_LocalPhotoCalib + ap25FluxErr: + functor: LocalNanojanskyErr + args: + - base_CircularApertureFlux_25_0_instFlux + - base_CircularApertureFlux_25_0_instFluxErr + - base_LocalPhotoCalib + # TODO: When DM-25019 is complete, these should be + # changed to use the local value of the background + # model, rather than the residual of the background + sky: + functor: LocalNanojansky + args: + - base_LocalBackground_instFlux + - base_LocalBackground_instFluxErr + - base_LocalPhotoCalib + skyErr: + functor: LocalNanojanskyErr + args: + - base_LocalBackground_instFlux + - base_LocalBackground_instFluxErr + - base_LocalPhotoCalib + psfFlux: + functor: LocalNanojansky + args: + - slot_PsfFlux_instFlux + - slot_PsfFlux_instFluxErr + - base_LocalPhotoCalib + psfFluxErr: + functor: LocalNanojanskyErr + args: + - slot_PsfFlux_instFlux + - slot_PsfFlux_instFluxErr + - base_LocalPhotoCalib + + # These PS columns do not make sense anymore as named + # psX + # psXSigma + # psY + # psYSigma + # psFlux_psX_Cov + # psFlux_psY_Cov + # psX_Y_Cov + # psLnL + # psChi2 + # psN + # psRa + # psRaSigma + # psDecl + # psDeclSigma + # psFlux_psRa_Cov + # psFlux_psDecl_Cov + + ixx: + functor: SinglePrecisionFloatColumn + args: slot_Shape_xx + iyy: + functor: SinglePrecisionFloatColumn + args: slot_Shape_yy + ixy: + functor: SinglePrecisionFloatColumn + args: slot_Shape_xy + # DPDD should include Psf Shape + ixxPSF: + functor: SinglePrecisionFloatColumn + args: slot_PsfShape_xx + iyyPSF: + functor: SinglePrecisionFloatColumn + args: slot_PsfShape_yy + ixyPSF: + functor: SinglePrecisionFloatColumn + args: slot_PsfShape_xy + ixxDebiasedPSF: + functor: SinglePrecisionFloatColumn + args: ext_shapeHSM_HsmPsfMomentsDebiased_xx + iyyDebiasedPSF: + functor: SinglePrecisionFloatColumn + args: ext_shapeHSM_HsmPsfMomentsDebiased_yy + ixyDebiasedPSF: + functor: SinglePrecisionFloatColumn + args: ext_shapeHSM_HsmPsfMomentsDebiased_xy + # apNann: Replaced by raw Aperture instFluxes in flags section below + # apMeanSb: Replaced by raw Aperture instFluxes in flags section below + # apMeanSbErr: Replaced by raw Aperture instFluxes in flags section below + + # DPDD does not include gaussianFluxes, however they are used for + # the single frame extendedness column which is used for QA. + gaussianFlux: + functor: LocalNanojansky + args: + - base_GaussianFlux_instFlux + - base_GaussianFlux_instFluxErr + - base_LocalPhotoCalib + gaussianFluxErr: + functor: LocalNanojanskyErr + args: + - base_GaussianFlux_instFlux + - base_GaussianFlux_instFluxErr + - base_LocalPhotoCalib + extendedness: + functor: SinglePrecisionFloatColumn + args: base_ClassificationExtendedness_value + sizeExtendedness: + functor: SinglePrecisionFloatColumn + args: base_ClassificationSizeExtendedness_value +flags: + - base_CircularApertureFlux_9_0_flag + - base_CircularApertureFlux_12_0_flag + - base_CircularApertureFlux_25_0_flag + - base_NormalizedCompensatedTophatFlux_flag + - base_NormalizedCompensatedTophatFlux_instFlux + - base_NormalizedCompensatedTophatFlux_instFluxErr + - base_ClassificationExtendedness_flag + - base_ClassificationSizeExtendedness_flag + - base_FootprintArea_value + - base_InvalidPsf_flag + - base_LocalBackground_instFlux + - base_LocalBackground_instFluxErr + - base_LocalBackground_flag + - base_LocalBackground_flag_noGoodPixels + - base_LocalBackground_flag_noPsf + - base_PixelFlags_flag_bad + - base_PixelFlags_flag_cr + - base_PixelFlags_flag_crCenter + - base_PixelFlags_flag_edge + - base_PixelFlags_flag_interpolated + - base_PixelFlags_flag_interpolatedCenter + - base_PixelFlags_flag_nodata + - base_PixelFlags_flag_offimage + - base_PixelFlags_flag_saturated + - base_PixelFlags_flag_saturatedCenter + - base_PixelFlags_flag_suspect + - base_PixelFlags_flag_suspectCenter + # Streak flags not yet propagated from compareWarp (DM-46947, DM-43077). + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter + - base_PsfFlux_apCorr + - base_PsfFlux_apCorrErr + - base_PsfFlux_area + - base_PsfFlux_flag + - base_PsfFlux_flag_apCorr + - base_PsfFlux_flag_edge + - base_PsfFlux_flag_noGoodPixels + - base_GaussianFlux_flag + - base_SdssCentroid_flag + - base_SdssCentroid_flag_almostNoSecondDerivative + - base_SdssCentroid_flag_badError + - base_SdssCentroid_flag_edge + - base_SdssCentroid_flag_noSecondDerivative + - base_SdssCentroid_flag_notAtMaximum + - base_SdssCentroid_flag_resetToPeak + - deblend_deblendedAsPsf + - deblend_hasStrayFlux + - deblend_masked + - deblend_nChild + - deblend_parentTooBig + - deblend_patchedTemplate + - deblend_rampedTemplate + - deblend_skipped + - deblend_tooManyPeaks + - ext_shapeHSM_HsmPsfMomentsDebiased_flag + - ext_shapeHSM_HsmPsfMomentsDebiased_flag_no_pixels + - ext_shapeHSM_HsmPsfMomentsDebiased_flag_not_contained + - ext_shapeHSM_HsmPsfMomentsDebiased_flag_parent_source + - ext_shapeHSM_HsmPsfMomentsDebiased_flag_galsim + - ext_shapeHSM_HsmPsfMomentsDebiased_flag_edge + - sky_source + - detect_isPrimary + +flag_rename_rules: + # Taken from db-meas-forced + - ['ext_photometryKron_', ''] + - ['base_CircularApertureFlux_9_0_flag', 'ap09Flux_flag'] + - ['base_CircularApertureFlux_12_0_flag', 'ap12Flux_flag'] + - ['base_CircularApertureFlux_25_0_flag', 'ap25Flux_flag'] + - ['base_Local', 'local'] + - ['base_PixelFlags_flag', 'pixelFlags'] + - ['base_ClassificationE', 'e'] + - ['base_ClassificationS', 's'] + - ['base_SdssCentroid', 'centroid'] + - ['base_Psf', 'psf'] + - ['base_InvalidPsf_flag', 'invalidPsfFlag'] + - ['base_GaussianFlux', 'gaussianFlux'] + - ['base_CircularApertureFlux', 'apFlux'] + - ['base_NormalizedCompensatedTophatFlux', 'normCompTophatFlux'] + - ['base_FootprintArea', 'footprintArea'] + - ['ext_shapeHSM_HsmPsfMomentsDebiased', 'iDebiasedPSF'] + - ['ext_shapeHSM_Hsm', 'hsm'] + - ['ext_convolved_', ''] + - ['undeblended_base', 'undeblended'] + - ['undeblended_ext_photometryKron', 'undeblended'] + - ['ext_photometryKron_', ''] + - ['base_', ''] + - ['slot_', ''] diff --git a/tests/test_promptSourceSchema.py b/tests/test_promptSourceSchema.py new file mode 100644 index 000000000..2586c166d --- /dev/null +++ b/tests/test_promptSourceSchema.py @@ -0,0 +1,117 @@ +# This file is part of pipe_tasks. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Check that the prompt_source pipeline produces exactly the columns defined +by the ``PromptSource`` table in ``sdm_schemas``. Only check column names, not +datatypes, since that requires running the detection and measurement task +which can't be imported here. +""" + +import os +import unittest + +import numpy as np +from astropy.table import Table +import importlib.resources + +import lsst.utils.tests +from lsst.utils import getPackageDir + +from lsst.pipe.tasks.postprocess import TransformSourceTableTask, TransformSourceTableConfig +from lsst.pipe.tasks.schemaUtils import readSdmSchemaFile +from lsst.pipe.tasks.split_primary import SplitPrimaryTask + +FUNCTOR_FILE = os.path.join(getPackageDir("pipe_tasks"), "schemas", "prompt_source.yaml") + +SCHEMA_FILE = importlib.resources.files("lsst.sdm.schemas") / "ap_extra.yaml" +TABLE_NAME = "PromptSource" + + +class PromptSourceSchemaTestCase(lsst.utils.tests.TestCase): + """Check the persisted prompt_source columns against the PromptSource DDL. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + + schemaFile = os.path.expandvars(SCHEMA_FILE) + if not os.path.exists(schemaFile): + raise unittest.SkipTest(f"SDM schema file not available: {SCHEMA_FILE}") + + schema = readSdmSchemaFile(schemaFile) + if TABLE_NAME not in schema: + raise ValueError(f"Table {TABLE_NAME!r} not in {schemaFile}.") + + cls.schemaColumns = {column.name for column in schema[TABLE_NAME].columns} + + config = TransformSourceTableConfig() + config.functorFile = FUNCTOR_FILE + transformTask = TransformSourceTableTask(config=config) + producedColumns = set(transformTask.funcs.funcDict) | set(config.columnsFromDataId) + + # The final prompt_source table is output by splitPromptSource, which is configured + # in the pipeline to drop sky sources and remove the corresponding column. + splitConfig = SplitPrimaryTask.ConfigClass() + splitConfig.discard_primary_columns = ["sky_source"] + splitTask = SplitPrimaryTask(config=splitConfig) + + # SplitPrimaryTask only needs the boolean primary-flag column to mask + # rows; the other columns are carried through (or dropped) by name, so + # a minimal two-row table with placeholder values is sufficient. + data = {} + for name in producedColumns: + if name == splitConfig.primary_flag_column: + data[name] = np.array([True, False]) + else: + data[name] = np.zeros(2) + full = Table(data) + + cls.persistedColumns = set(splitTask.run(full=full).primary.colnames) + + def testColumnsConform(self): + """The persisted prompt_source columns must equal the schema exactly. + """ + missing = self.schemaColumns - self.persistedColumns + extra = self.persistedColumns - self.schemaColumns + + message = ( + f"persisted prompt_source columns do not conform to the " + f"{TABLE_NAME} schema.\n" + f" In schema but not persisted ({len(missing)}): " + f"{sorted(missing)}\n" + f" Persisted but not in schema ({len(extra)}): " + f"{sorted(extra)}" + ) + self.assertEqual(self.persistedColumns, self.schemaColumns, message) + + +class MemoryTester(lsst.utils.tests.MemoryTestCase): + pass + + +def setup_module(module): + lsst.utils.tests.init() + + +if __name__ == "__main__": + lsst.utils.tests.init() + unittest.main()