diff --git a/bps/clustering/clustering_ApPipe.yaml b/bps/clustering/clustering_ApPipe.yaml index 72595a47..fe1f98fa 100644 --- a/bps/clustering/clustering_ApPipe.yaml +++ b/bps/clustering/clustering_ApPipe.yaml @@ -30,7 +30,7 @@ cluster: dimensions: visit,detector equalDimensions: visit:group singleFrameMeasurement: - pipetasks: singleFrameDetectAndMeasure + pipetasks: singleFrameDetectAndMeasure,standardizePromptSource dimensions: visit,detector diaSrcDetectorAnalysis: pipetasks: analyzeAssociatedDiaSourceTable,analyzeTrailedDiaSourceTable diff --git a/bps/clustering/clustering_Daytime.yaml b/bps/clustering/clustering_Daytime.yaml index 5ddc0d75..54da43d4 100644 --- a/bps/clustering/clustering_Daytime.yaml +++ b/bps/clustering/clustering_Daytime.yaml @@ -35,7 +35,7 @@ cluster: pipetasks: associateApdb,analyzeDiaSourceAssociationMetrics,analyzeAssociateDiaSourceTiming dimensions: visit,detector singleFrameMeasurement: - pipetasks: singleFrameDetectAndMeasure + pipetasks: singleFrameDetectAndMeasure,standardizePromptSource dimensions: visit,detector diaSrcDetectorAnalysis: pipetasks: analyzeAssociatedDiaSourceTable,analyzeTrailedDiaSourceTable @@ -44,7 +44,7 @@ cluster: pipetasks: makeSampledImageSubtractionMetrics,analyzeSampledImageSubtractionMetrics dimensions: visit,detector consolidateVisit: - pipetasks: consolidateVisitSummary,consolidateDiaSourceTable,analyzeVisitSampledImageSubtractionMetrics + pipetasks: consolidateVisitSummary,consolidateDiaSourceTable,analyzeVisitSampledImageSubtractionMetrics,consolidatePromptSource,splitPromptSource dimensions: visit diff --git a/pipelines/LSSTCam/ApPipeWithFakes.yaml b/pipelines/LSSTCam/ApPipeWithFakes.yaml index 9f47879e..e4ba2010 100644 --- a/pipelines/LSSTCam/ApPipeWithFakes.yaml +++ b/pipelines/LSSTCam/ApPipeWithFakes.yaml @@ -1,4 +1,4 @@ -description: HSC AP Pipeline with synthetic/fake sources. Templates are inputs. +description: LSSTCam AP Pipeline with synthetic/fake sources. Templates are inputs. # Remember: # (0) Ensure median calibration products and template coadds exist for the data being processed # (1) Execute `apdb-cli create-sql`, e.g., @@ -11,114 +11,24 @@ imports: # that is generated when running scons compilation of the ap_pipe package. # Do not edit this file as changes will be overwritten during the next build. - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeWithFakes.yaml - exclude: - - isr - - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeWithIsrTaskLSST.yaml - include: - - isr parameters: coaddName: goodSeeing # APDB config file must be user-specified apdb_config: -subsets: - apPipeSingleFrame: - subset: - - isr - - calibrateImage - - analyzePreliminarySummaryStats - description: >- - The prompt ApPipe tasks that make up single-frame processing. Not to be confused with the - SingleFrame.yaml pipeline, which does more than just ApPipe single frame processing, and - is designed as a standalone alternative to ApPipeWithFakes.yaml. - processCcd: # TODO: remove on DM-48428 - subset: - - isr - - calibrateImage - - analyzePreliminarySummaryStats - description: Deprecated alias for apPipeSingleFrame, will be removed after v29. - apPipe: - subset: - - isr - - calibrateImage - - injectVisit - - injectTemplate - - loadDiaCatalogs - - buildTemplate - - subtractImages - - detectAndMeasureDiaSource - - filterDiaSource - - computeReliability - - filterDiaSourcePostReliability - - standardizeDiaSource - - getRegionTimeFromVisit - - mpSkyEphemerisQuery - - associateApdb - - injectedMatchDiaSrc - - injectedMatchAssocDiaSrc - - consolidateMatchDiaSrc - - consolidateMatchAssocDiaSrc - - makeSampledImageSubtractionMetrics - - consolidateVisitSummary - - consolidateDiaSourceTable - - analyzeAssociatedDiaSourceTable - - analyzeTrailedDiaSourceTable - - analyzeDiaFakesDetectorVisitCore - - analyzeAssocDiaFakesDetectorVisitCore - - analyzeDiaFakesVisitCore - - analyzeAssocDiaFakesVisitCore - - analyzeImageDifferenceMetrics - - analyzeDiaSourceDetectionMetrics - - analyzeLoadDiaCatalogsMetrics - - analyzeDiaSourceAssociationMetrics - - analyzeAssociateDiaSourceTiming - - analyzeSampledImageSubtractionMetrics - - analyzePreliminarySummaryStats - - analyzeVisitSampledImageSubtractionMetrics - - singleFrameDetectAndMeasure - description: > - An alias of ApPipe to use in higher-level pipelines. - prompt: - subset: - - isr - - calibrateImage - - injectVisit - - injectTemplate - - buildTemplate - - subtractImages - - detectAndMeasureDiaSource - - filterDiaSource - - computeReliability - - filterDiaSourcePostReliability - - standardizeDiaSource - - associateApdb - - injectedMatchDiaSrc - - consolidateMatchDiaSrc - - analyzeAssociatedDiaSourceTable - - analyzeTrailedDiaSourceTable - - analyzeImageDifferenceMetrics - - analyzeDiaSourceDetectionMetrics - - analyzeDiaSourceAssociationMetrics - - analyzeAssociateDiaSourceTiming - - analyzePreliminarySummaryStats - description: > - Tasks necessary to turn raw images into APDB rows and alerts. - Requires preload subset to be run first. - afterburner: - subset: - - analyzeAssocDiaFakesDetectorVisitCore - - analyzeDiaFakesDetectorVisitCore - - consolidateMatchAssocDiaSrc - - injectedMatchAssocDiaSrc - - analyzeAssocDiaFakesVisitCore - - consolidateVisitSummary - - consolidateDiaSourceTable - - makeSampledImageSubtractionMetrics - - analyzeSampledImageSubtractionMetrics - - analyzeDiaFakesVisitCore - - analyzeVisitSampledImageSubtractionMetrics - - singleFrameDetectAndMeasure - description: > - Tasks for QA and other non-real-time processing. - Requires prompt subset to be run first. +tasks: + # Swap in the LSST-specific ISR task. Overriding the task in place (rather than + # excluding `isr` from the import and re-adding it) keeps the `isr` label intact, + # so all subsets are inherited from _ingredients/ApPipeWithFakes.yaml and stay in + # sync with _ingredients/ApPipe.yaml automatically. + isr: + class: lsst.ip.isr.IsrTaskLSST + config: + connections.outputExposure: post_isr_image + # Some instruments have this already in obs_lsst. + # Here we turn it off for AP generally. + doBrighterFatter: False + # Turn off the amp offset calculation. + # It is only used to compute metrics, and takes several seconds. + doAmpOffset: False diff --git a/pipelines/_ingredients/ApPipe.yaml b/pipelines/_ingredients/ApPipe.yaml index 84ddfc80..33bb9034 100644 --- a/pipelines/_ingredients/ApPipe.yaml +++ b/pipelines/_ingredients/ApPipe.yaml @@ -201,6 +201,26 @@ tasks: connections.sources_footprints: single_visit_star_reprocessed_footprints connections.background: preliminary_visit_image_reprocessed_background id_generator.release_id: parameters.release_id + standardizePromptSource: + class: lsst.pipe.tasks.postprocess.TransformSourceTableTask + config: + connections.inputCatalog: single_visit_star_reprocessed_unstandardized + connections.outputCatalog: prompt_source_detector + functorFile: $PIPE_TASKS_DIR/schemas/prompt_source.yaml + consolidatePromptSource: + class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask + config: + connections.inputCatalogs: prompt_source_detector + connections.outputCatalog: prompt_source_all + splitPromptSource: + class: lsst.pipe.tasks.split_primary.SplitPrimaryTask + config: + dimensions: ["visit"] + connections.full: prompt_source_all + connections.primary: prompt_source + connections.nonprimary: prompt_source_nonprimary + discard_primary_columns: ["sky_source"] + discard_nonprimary_columns: ["sky_source"] subsets: apPipeSingleFrame: @@ -241,6 +261,9 @@ subsets: - consolidateDiaSourceTable - analyzeVisitSampledImageSubtractionMetrics - singleFrameDetectAndMeasure + - standardizePromptSource + - consolidatePromptSource + - splitPromptSource description: > An alias of ApPipe to use in higher-level pipelines. preload: @@ -279,6 +302,9 @@ subsets: - analyzeSampledImageSubtractionMetrics - analyzeVisitSampledImageSubtractionMetrics - singleFrameDetectAndMeasure + - standardizePromptSource + - consolidatePromptSource + - splitPromptSource description: > Tasks for QA and other non-real-time processing. Requires prompt subset to be run first. diff --git a/pipelines/_ingredients/ApPipeWithPreconvolution.yaml b/pipelines/_ingredients/ApPipeWithPreconvolution.yaml index d7882019..0029bc51 100644 --- a/pipelines/_ingredients/ApPipeWithPreconvolution.yaml +++ b/pipelines/_ingredients/ApPipeWithPreconvolution.yaml @@ -69,6 +69,9 @@ subsets: - consolidateDiaSourceTable - analyzeVisitSampledImageSubtractionMetrics - singleFrameDetectAndMeasure + - standardizePromptSource + - consolidatePromptSource + - splitPromptSource description: > An alias of ApPipe to use in higher-level pipelines. prompt: