From 1ebcef39af7c42a7ca55cd31d29502de1cdcee8a Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Tue, 8 Oct 2024 14:21:48 -0700 Subject: [PATCH] Restrict ApVerify pipelines to prompt portion of ApPipe. ap_verify was originally written as a mockup of the process of ingesting and processing new raws. It makes more sense to treat preprocessing datasets as prior inputs, and this also handles the case where preprocessing can't be run in Jenkins. --- pipelines/DECam/ApVerify.yaml | 4 +++- pipelines/DECam/ApVerifyWithFakes.yaml | 4 +++- pipelines/HSC/ApVerify.yaml | 4 +++- pipelines/HSC/ApVerifyWithFakes.yaml | 4 +++- pipelines/LSSTCam-imSim/ApVerify.yaml | 4 +++- pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml | 4 +++- pipelines/_ingredients/ApVerify.yaml | 2 ++ pipelines/_ingredients/ApVerifyWithFakes.yaml | 2 ++ pipelines/_ingredients/MetricsRuntime.yaml | 16 ---------------- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pipelines/DECam/ApVerify.yaml b/pipelines/DECam/ApVerify.yaml index fd83053f..1c453382 100644 --- a/pipelines/DECam/ApVerify.yaml +++ b/pipelines/DECam/ApVerify.yaml @@ -7,8 +7,10 @@ imports: # Include all metrics from standard pipeline. It's not practical to create # a metrics subset because it would require constant micromanagement. exclude: - - apPipe + - prompt - location: $AP_PIPE_DIR/pipelines/DECam/ApPipe.yaml + include: + - prompt tasks: # ApVerify override removed by excluding apPipe. diaPipe: diff --git a/pipelines/DECam/ApVerifyWithFakes.yaml b/pipelines/DECam/ApVerifyWithFakes.yaml index f5acee8c..f826a6d6 100644 --- a/pipelines/DECam/ApVerifyWithFakes.yaml +++ b/pipelines/DECam/ApVerifyWithFakes.yaml @@ -11,8 +11,10 @@ imports: # Include all metrics from standard pipeline. It's not practical to create # a metrics subset because it would require constant micromanagement. exclude: - - apPipe + - prompt - location: $AP_PIPE_DIR/pipelines/DECam/ApPipeWithFakes.yaml + include: + - prompt tasks: # ApVerify override removed by excluding apPipe. diaPipe: diff --git a/pipelines/HSC/ApVerify.yaml b/pipelines/HSC/ApVerify.yaml index e41763d9..81bbad9a 100644 --- a/pipelines/HSC/ApVerify.yaml +++ b/pipelines/HSC/ApVerify.yaml @@ -7,8 +7,10 @@ imports: # Include all metrics from standard pipeline. It's not practical to create # a metrics subset because it would require constant micromanagement. exclude: - - apPipe + - prompt - location: $AP_PIPE_DIR/pipelines/HSC/ApPipe.yaml + include: + - prompt tasks: # ApVerify override removed by excluding apPipe. diaPipe: diff --git a/pipelines/HSC/ApVerifyWithFakes.yaml b/pipelines/HSC/ApVerifyWithFakes.yaml index d15cd727..e7cb3eca 100644 --- a/pipelines/HSC/ApVerifyWithFakes.yaml +++ b/pipelines/HSC/ApVerifyWithFakes.yaml @@ -8,8 +8,10 @@ imports: # Include all metrics from standard pipeline. It's not practical to create # a metrics subset because it would require constant micromanagement. exclude: - - apPipe + - prompt - location: $AP_PIPE_DIR/pipelines/HSC/ApPipeWithFakes.yaml + include: + - prompt tasks: # ApVerify override removed by excluding apPipe. diaPipe: diff --git a/pipelines/LSSTCam-imSim/ApVerify.yaml b/pipelines/LSSTCam-imSim/ApVerify.yaml index c683e2ba..35cbc3fe 100644 --- a/pipelines/LSSTCam-imSim/ApVerify.yaml +++ b/pipelines/LSSTCam-imSim/ApVerify.yaml @@ -7,8 +7,10 @@ imports: # Include all metrics from standard pipeline. It's not practical to create # a metrics subset because it would require constant micromanagement. exclude: - - apPipe + - prompt - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipe.yaml + include: + - prompt tasks: # ApVerify override removed by excluding apPipe. diaPipe: diff --git a/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml b/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml index c7c401eb..73449227 100644 --- a/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml +++ b/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml @@ -8,8 +8,10 @@ imports: # Include all metrics from standard pipeline. It's not practical to create # a metrics subset because it would require constant micromanagement. exclude: - - apPipe + - prompt - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipeWithFakes.yaml + include: + - prompt tasks: # ApVerify override removed by excluding apPipe. diaPipe: diff --git a/pipelines/_ingredients/ApVerify.yaml b/pipelines/_ingredients/ApVerify.yaml index 4e908403..a55082ee 100644 --- a/pipelines/_ingredients/ApVerify.yaml +++ b/pipelines/_ingredients/ApVerify.yaml @@ -3,6 +3,8 @@ description: Fully instrumented AP pipeline. imports: - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml + include: + - prompt - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsRuntime.yaml exclude: - timing_calibrate diff --git a/pipelines/_ingredients/ApVerifyWithFakes.yaml b/pipelines/_ingredients/ApVerifyWithFakes.yaml index baaa17f9..05d97e7f 100644 --- a/pipelines/_ingredients/ApVerifyWithFakes.yaml +++ b/pipelines/_ingredients/ApVerifyWithFakes.yaml @@ -4,6 +4,8 @@ description: Fully instrumented AP pipeline with fakes imports: - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeWithFakes.yaml + include: + - prompt # Most metrics should not be run with fakes, to avoid bias or contamination. - location: $AP_VERIFY_DIR/pipelines/_ingredients/ConversionsForFakes.yaml tasks: diff --git a/pipelines/_ingredients/MetricsRuntime.yaml b/pipelines/_ingredients/MetricsRuntime.yaml index 1241a0ee..c60076c6 100644 --- a/pipelines/_ingredients/MetricsRuntime.yaml +++ b/pipelines/_ingredients/MetricsRuntime.yaml @@ -90,14 +90,6 @@ tasks: connections.metric: MapDiaSourceTime connections.labelName: transformDiaSrcCat target: transformDiaSrcCat.run - timing_diaCatalogLoader: - class: lsst.verify.tasks.commonMetrics.TimingMetricTask - config: - connections.package: ap_association - connections.metric: LoadDiaCatalogsTime - connections.labelName: loadDiaCatalogs - metadataDimensions: [instrument, group, detector] # TimingMetricTask assumes visit - target: loadDiaCatalogs.run timing_diaPipe_associator: class: lsst.verify.tasks.commonMetrics.TimingMetricTask config: @@ -212,14 +204,6 @@ tasks: connections.metric: MapDiaSourceCpuTime connections.labelName: transformDiaSrcCat target: transformDiaSrcCat.run - cputiming_diaCatalogLoader: - class: lsst.verify.tasks.commonMetrics.CpuTimingMetricTask - config: - connections.package: ap_association - connections.metric: LoadDiaCatalogsCpuTime - connections.labelName: loadDiaCatalogs - metadataDimensions: [instrument, group, detector] # CpuTimingMetricTask assumes visit - target: loadDiaCatalogs.run cputiming_diaPipe_associator: class: lsst.verify.tasks.commonMetrics.CpuTimingMetricTask config: