DM-55233: Add standardization steps for prompt_source

bps/clustering/clustering_ApPipe.yaml

@@ -30,7 +30,7 @@ cluster:
     dimensions: visit,detector
     equalDimensions: visit:group
   singleFrameMeasurement:
-    pipetasks: singleFrameDetectAndMeasure
+    pipetasks: singleFrameDetectAndMeasure,standardizePromptSource
     dimensions: visit,detector
   diaSrcDetectorAnalysis:
     pipetasks: analyzeAssociatedDiaSourceTable,analyzeTrailedDiaSourceTable

bps/clustering/clustering_Daytime.yaml

@@ -35,7 +35,7 @@ cluster:
     pipetasks: associateApdb,analyzeDiaSourceAssociationMetrics,analyzeAssociateDiaSourceTiming
     dimensions: visit,detector
   singleFrameMeasurement:
-    pipetasks: singleFrameDetectAndMeasure
+    pipetasks: singleFrameDetectAndMeasure,standardizePromptSource
     dimensions: visit,detector
   diaSrcDetectorAnalysis:
     pipetasks: analyzeAssociatedDiaSourceTable,analyzeTrailedDiaSourceTable
@@ -44,7 +44,7 @@ cluster:
     pipetasks: makeSampledImageSubtractionMetrics,analyzeSampledImageSubtractionMetrics
     dimensions: visit,detector
   consolidateVisit:
-    pipetasks: consolidateVisitSummary,consolidateDiaSourceTable,analyzeVisitSampledImageSubtractionMetrics
+    pipetasks: consolidateVisitSummary,consolidateDiaSourceTable,analyzeVisitSampledImageSubtractionMetrics,consolidatePromptSource,splitPromptSource
     dimensions: visit
 
 

pipelines/LSSTCam/ApPipeWithFakes.yaml

@@ -1,4 +1,4 @@
-description: HSC AP Pipeline with synthetic/fake sources. Templates are inputs.
+description: LSSTCam AP Pipeline with synthetic/fake sources. Templates are inputs.
 # Remember:
 # (0) Ensure median calibration products and template coadds exist for the data being processed
 # (1) Execute `apdb-cli create-sql`, e.g.,
@@ -11,114 +11,24 @@ imports:
   #  that is generated when running scons compilation of the ap_pipe package.
   #  Do not edit this file as changes will be overwritten during the next build.
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeWithFakes.yaml
-    exclude:
-      - isr
-  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeWithIsrTaskLSST.yaml
-    include:
-      - isr
 
 parameters:
   coaddName: goodSeeing
   # APDB config file must be user-specified
   apdb_config:
 
-subsets:
-  apPipeSingleFrame:
-    subset:
-      - isr
-      - calibrateImage
-      - analyzePreliminarySummaryStats
-    description: >-
-      The prompt ApPipe tasks that make up single-frame processing. Not to be confused with the
-      SingleFrame.yaml pipeline, which does more than just ApPipe single frame processing, and
-      is designed as a standalone alternative to ApPipeWithFakes.yaml.
-  processCcd:  # TODO: remove on DM-48428
-    subset:
-      - isr
-      - calibrateImage
-      - analyzePreliminarySummaryStats
-    description: Deprecated alias for apPipeSingleFrame, will be removed after v29.
-  apPipe:
-    subset:
-      - isr
-      - calibrateImage
-      - injectVisit
-      - injectTemplate
-      - loadDiaCatalogs
-      - buildTemplate
-      - subtractImages
-      - detectAndMeasureDiaSource
-      - filterDiaSource
-      - computeReliability
-      - filterDiaSourcePostReliability
-      - standardizeDiaSource
-      - getRegionTimeFromVisit
-      - mpSkyEphemerisQuery
-      - associateApdb
-      - injectedMatchDiaSrc
-      - injectedMatchAssocDiaSrc
-      - consolidateMatchDiaSrc
-      - consolidateMatchAssocDiaSrc
-      - makeSampledImageSubtractionMetrics
-      - consolidateVisitSummary
-      - consolidateDiaSourceTable
-      - analyzeAssociatedDiaSourceTable
-      - analyzeTrailedDiaSourceTable
-      - analyzeDiaFakesDetectorVisitCore
-      - analyzeAssocDiaFakesDetectorVisitCore
-      - analyzeDiaFakesVisitCore
-      - analyzeAssocDiaFakesVisitCore
-      - analyzeImageDifferenceMetrics
-      - analyzeDiaSourceDetectionMetrics
-      - analyzeLoadDiaCatalogsMetrics
-      - analyzeDiaSourceAssociationMetrics
-      - analyzeAssociateDiaSourceTiming
-      - analyzeSampledImageSubtractionMetrics
-      - analyzePreliminarySummaryStats
-      - analyzeVisitSampledImageSubtractionMetrics
-      - singleFrameDetectAndMeasure
-    description: >
-      An alias of ApPipe to use in higher-level pipelines.
-  prompt:
-    subset:
-      - isr
-      - calibrateImage
-      - injectVisit
-      - injectTemplate
-      - buildTemplate
-      - subtractImages
-      - detectAndMeasureDiaSource
-      - filterDiaSource
-      - computeReliability
-      - filterDiaSourcePostReliability
-      - standardizeDiaSource
-      - associateApdb
-      - injectedMatchDiaSrc
-      - consolidateMatchDiaSrc
-      - analyzeAssociatedDiaSourceTable
-      - analyzeTrailedDiaSourceTable
-      - analyzeImageDifferenceMetrics
-      - analyzeDiaSourceDetectionMetrics
-      - analyzeDiaSourceAssociationMetrics
-      - analyzeAssociateDiaSourceTiming
-      - analyzePreliminarySummaryStats
-    description: >
-      Tasks necessary to turn raw images into APDB rows and alerts.
-      Requires preload subset to be run first.
-  afterburner:
-    subset:
-    - analyzeAssocDiaFakesDetectorVisitCore
-    - analyzeDiaFakesDetectorVisitCore
-    - consolidateMatchAssocDiaSrc
-    - injectedMatchAssocDiaSrc
-    - analyzeAssocDiaFakesVisitCore
-    - consolidateVisitSummary
-    - consolidateDiaSourceTable
-    - makeSampledImageSubtractionMetrics
-    - analyzeSampledImageSubtractionMetrics
-    - analyzeDiaFakesVisitCore
-    - analyzeVisitSampledImageSubtractionMetrics
-    - singleFrameDetectAndMeasure
-    description: >
-      Tasks for QA and other non-real-time processing.
-      Requires prompt subset to be run first.
+tasks:
+  # Swap in the LSST-specific ISR task. Overriding the task in place (rather than
+  # excluding `isr` from the import and re-adding it) keeps the `isr` label intact,
+  # so all subsets are inherited from _ingredients/ApPipeWithFakes.yaml and stay in
+  # sync with _ingredients/ApPipe.yaml automatically.
+  isr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      connections.outputExposure: post_isr_image
+      # Some instruments have this already in obs_lsst.
+      # Here we turn it off for AP generally.
+      doBrighterFatter: False
+      # Turn off the amp offset calculation.
+      # It is only used to compute metrics, and takes several seconds.
+      doAmpOffset: False

pipelines/_ingredients/ApPipe.yaml

@@ -201,6 +201,26 @@ tasks:
       connections.sources_footprints: single_visit_star_reprocessed_footprints
       connections.background: preliminary_visit_image_reprocessed_background
       id_generator.release_id: parameters.release_id
+  standardizePromptSource:
+    class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
+    config:
+      connections.inputCatalog: single_visit_star_reprocessed_unstandardized
+      connections.outputCatalog: prompt_source_detector
+      functorFile: $PIPE_TASKS_DIR/schemas/prompt_source.yaml
+  consolidatePromptSource:
+    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
+    config:
+      connections.inputCatalogs: prompt_source_detector
+      connections.outputCatalog: prompt_source_all
+  splitPromptSource:
+    class: lsst.pipe.tasks.split_primary.SplitPrimaryTask
+    config:
+      dimensions: ["visit"]
+      connections.full: prompt_source_all
+      connections.primary: prompt_source
+      connections.nonprimary: prompt_source_nonprimary
+      discard_primary_columns: ["sky_source"]
+      discard_nonprimary_columns: ["sky_source"]
 
 subsets:
   apPipeSingleFrame:
@@ -241,6 +261,9 @@ subsets:
       - consolidateDiaSourceTable
       - analyzeVisitSampledImageSubtractionMetrics
       - singleFrameDetectAndMeasure
+      - standardizePromptSource
+      - consolidatePromptSource
+      - splitPromptSource
     description: >
       An alias of ApPipe to use in higher-level pipelines.
   preload:
@@ -279,6 +302,9 @@ subsets:
       - analyzeSampledImageSubtractionMetrics
       - analyzeVisitSampledImageSubtractionMetrics
       - singleFrameDetectAndMeasure
+      - standardizePromptSource
+      - consolidatePromptSource
+      - splitPromptSource
     description: >
       Tasks for QA and other non-real-time processing.
       Requires prompt subset to be run first.

pipelines/_ingredients/ApPipeWithPreconvolution.yaml

@@ -69,6 +69,9 @@ subsets:
       - consolidateDiaSourceTable
       - analyzeVisitSampledImageSubtractionMetrics
       - singleFrameDetectAndMeasure
+      - standardizePromptSource
+      - consolidatePromptSource
+      - splitPromptSource
     description: >
       An alias of ApPipe to use in higher-level pipelines.
   prompt: