From 5f8f858e383fb393133d7e6a3cbcb1fd82804c05 Mon Sep 17 00:00:00 2001 From: Bruno Sanchez Date: Mon, 13 May 2024 15:45:37 +0200 Subject: [PATCH 1/3] Fixing ApVerifyWithFakes to use new source_injection package --- pipelines/DECam/ApVerifyWithFakes.yaml | 8 -------- pipelines/HSC/ApVerifyWithFakes.yaml | 8 -------- pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml | 8 -------- pipelines/_ingredients/ApVerifyWithFakes.yaml | 14 +++++++++++--- .../_ingredients/ConversionsForFakes.yaml | 18 +++++++----------- 5 files changed, 18 insertions(+), 38 deletions(-) diff --git a/pipelines/DECam/ApVerifyWithFakes.yaml b/pipelines/DECam/ApVerifyWithFakes.yaml index 352fbd49..106b3614 100644 --- a/pipelines/DECam/ApVerifyWithFakes.yaml +++ b/pipelines/DECam/ApVerifyWithFakes.yaml @@ -8,11 +8,3 @@ instrument: lsst.obs.decam.DarkEnergyCamera description: Fully instrumented AP pipeline with fakes, specialized for DECam imports: - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyWithFakes.yaml - exclude: - - processCcd - # The existing Fakes insertion system is not currently compatible with the - # new CalibrateImageTask, so the fakes pipeline still is on the old tasks. - - location: $AP_PIPE_DIR/pipelines/DECam/ProcessCcdCalibrate.yaml - # Can't use $AP_PIPE_DIR/pipelines/DECam/ApPipeWithFakes.yaml here - # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml - # are hard to separate. diff --git a/pipelines/HSC/ApVerifyWithFakes.yaml b/pipelines/HSC/ApVerifyWithFakes.yaml index 02f5991f..cdec6232 100644 --- a/pipelines/HSC/ApVerifyWithFakes.yaml +++ b/pipelines/HSC/ApVerifyWithFakes.yaml @@ -5,11 +5,3 @@ instrument: lsst.obs.subaru.HyperSuprimeCam description: Fully instrumented AP pipeline with fakes, specialized for HSC imports: - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyWithFakes.yaml - exclude: - - processCcd - # The existing Fakes insertion system is not currently compatible with the - # new CalibrateImageTask, so the fakes pipeline still is on the old tasks. - - location: $AP_PIPE_DIR/pipelines/HSC/ProcessCcdCalibrate.yaml - # Can't use $AP_PIPE_DIR/pipelines/HSC/ApPipeWithFakes.yaml here - # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml - # are hard to separate. diff --git a/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml b/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml index 5e8ef84b..06af1e0b 100644 --- a/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml +++ b/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml @@ -5,11 +5,3 @@ instrument: lsst.obs.lsst.LsstCamImSim description: Fully instrumented AP pipeline with fakes, specialized for LSSTCam-imSim imports: - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyWithFakes.yaml - exclude: - - processCcd - # The existing Fakes insertion system is not currently compatible with the - # new CalibrateImageTask, so the fakes pipeline still is on the old tasks. - - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ProcessCcdCalibrate.yaml - # Can't use $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipeWithFakes.yaml here - # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml - # are hard to separate. diff --git a/pipelines/_ingredients/ApVerifyWithFakes.yaml b/pipelines/_ingredients/ApVerifyWithFakes.yaml index e143f006..905f99d2 100644 --- a/pipelines/_ingredients/ApVerifyWithFakes.yaml +++ b/pipelines/_ingredients/ApVerifyWithFakes.yaml @@ -17,9 +17,17 @@ tasks: contracts: # Metric inputs must match pipeline outputs # Use of ConnectionsClass for templated fields is a workaround for DM-30210 - - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name == + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == apFakesCompletenessMag20t22.connections.ConnectionsClass(config=apFakesCompletenessMag20t22).matchedFakes.name - - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name == + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == apFakesCompletenessMag22t24.connections.ConnectionsClass(config=apFakesCompletenessMag22t24).matchedFakes.name - - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name == + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == apFakesCompletenessMag24t26.connections.ConnectionsClass(config=apFakesCompletenessMag24t26).matchedFakes.name + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == + apFakesCountMag20t22.connections.ConnectionsClass(config=apFakesCountMag20t22).matchedFakes.name + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == + apFakesCountMag22t24.connections.ConnectionsClass(config=apFakesCountMag22t24).matchedFakes.name + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == + apFakesCountMag24t26.connections.ConnectionsClass(config=apFakesCountMag24t26).matchedFakes.name + - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name == + apFakesCount.connections.ConnectionsClass(config=apFakesCount).matchedFakes.name diff --git a/pipelines/_ingredients/ConversionsForFakes.yaml b/pipelines/_ingredients/ConversionsForFakes.yaml index b334b33e..75785abe 100644 --- a/pipelines/_ingredients/ConversionsForFakes.yaml +++ b/pipelines/_ingredients/ConversionsForFakes.yaml @@ -11,21 +11,15 @@ imports: # Fakes pipeline doesn't produce non-fakes DiaSourceTable. - consolidateDiaSourceTable tasks: - # Conversion of fakes_src [afw.table] to fakes_source [Parquet] - writeSourceTable: - class: lsst.pipe.tasks.postprocess.WriteSourceTableTask - config: - connections.catalogType: parameters.fakesType # TODO: TransformSourceTableTask can't be run until we create a functor # config that doesn't depend on shapeHSM. # Merging of fakes_source [detector-level] to fakes_sourceTable_visit [visit-level] consolidateSourceTable: class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask config: - connections.catalogType: parameters.fakesType - # Skip DPDD-ified sourceTable and just use source directly - connections.inputCatalogs: "{catalogType}source" - + # This is output directly by CalibrateImageTask. + connections.inputCatalogs: initial_stars_detector + connections.outputCatalog: initial_stars # Merging of fakes_*Diff_diaSrcTable [detector-level Parquet] to fakes_diaSourceTable [visit-level] consolidateDiaSourceTable: class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask @@ -36,13 +30,12 @@ tasks: # and fakesType is more stable than coaddName. connections.inputCatalogs: "fakes_{catalogType}Diff_diaSrcTable" connections.outputCatalog: fakes_diaSourceTable - # Creation of fakes_visitSummary consolidateVisitSummary: class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask config: connections.calexpType: parameters.fakesType - connections.calexp: "{calexpType}calexp" + connections.calexp: "{calexpType}initial_pvi" connections.visitSummary: "{calexpType}visitSummary" connections.visitSummarySchema: "{calexpType}visitSummary_schema" # Conversion of fakes_visitSummary [visit-level afw.table] to fakes_visitTable [instrument-level Parquet] @@ -50,8 +43,11 @@ tasks: class: lsst.pipe.tasks.postprocess.MakeVisitTableTask config: connections.calexpType: parameters.fakesType + connections.visitSummaries: "{calexpType}visitSummary" # Conversion of fakes_visitSummary [visit-level afw.table] to fakes_ccdVisitTable [instrument-level Parquet] makeCcdVisitTable: class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask config: connections.calexpType: parameters.fakesType + connections.visitSummaryRefs: "{calexpType}visitSummary" + From 6a298b4d57f9e2889024b0fd44c81f253c9a8ba0 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Thu, 11 Jul 2024 13:55:28 -0700 Subject: [PATCH 2/3] Fix E721 lint warning (type comparison) --- python/lsst/ap/verify/workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ap/verify/workspace.py b/python/lsst/ap/verify/workspace.py index 4797007f..6fc04cab 100644 --- a/python/lsst/ap/verify/workspace.py +++ b/python/lsst/ap/verify/workspace.py @@ -84,7 +84,7 @@ def __eq__(self, other): """Test whether two workspaces are of the same type and have the same location. """ - return type(self) == type(other) and self.workDir == other.workDir + return type(self) is type(other) and self.workDir == other.workDir def __repr__(self): """A string representation that can be used to reconstruct the Workspace. From cc4e2614fa733f83281bd81fc6191e7e1142f406 Mon Sep 17 00:00:00 2001 From: BrunoSanchez Date: Tue, 23 Jul 2024 10:52:11 -0700 Subject: [PATCH 3/3] Removed pipelines associated with deprecated Calibrate yamls --- pipelines/DECam/ApVerifyCalibrate.yaml | 26 ------------ pipelines/HSC/ApVerifyCalibrate.yaml | 22 ---------- .../LSSTCam-imSim/ApVerifyCalibrate.yaml | 26 ------------ pipelines/_ingredients/ApVerifyCalibrate.yaml | 28 ------------- .../_ingredients/ConversionsCalibrate.yaml | 41 ------------------- .../_ingredients/MetricsMiscCalibrate.yaml | 41 ------------------- 6 files changed, 184 deletions(-) delete mode 100644 pipelines/DECam/ApVerifyCalibrate.yaml delete mode 100644 pipelines/HSC/ApVerifyCalibrate.yaml delete mode 100644 pipelines/LSSTCam-imSim/ApVerifyCalibrate.yaml delete mode 100644 pipelines/_ingredients/ApVerifyCalibrate.yaml delete mode 100644 pipelines/_ingredients/ConversionsCalibrate.yaml delete mode 100644 pipelines/_ingredients/MetricsMiscCalibrate.yaml diff --git a/pipelines/DECam/ApVerifyCalibrate.yaml b/pipelines/DECam/ApVerifyCalibrate.yaml deleted file mode 100644 index 566debc6..00000000 --- a/pipelines/DECam/ApVerifyCalibrate.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Verification pipeline specialized for DECam. -# This concatenates various lsst.verify metrics to an AP pipeline - -description: Fully instrumented AP pipeline specialized for DECam -imports: - - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrate.yaml - # Include all metrics from standard pipeline. It's not practical to create - # a metrics subset because it would require constant micromanagement. - exclude: - - apPipe - - location: $AP_PIPE_DIR/pipelines/DECam/ApPipeCalibrate.yaml -tasks: - # ApVerify override removed by excluding apPipe. - diaPipe: - class: lsst.ap.association.DiaPipelineTask - config: - doPackageAlerts: True -contracts: - # Must re-declare contracts that cross apPipe and metrics boundary, as - # these were removed on import. - - contract: diaPipe.doConfigureApdb or not totalUnassociatedDiaObjects.doReadMarker - msg: "totalUnassociatedDiaObjects.doReadMarker requires diaPipe.doConfigureApdb" - - (totalUnassociatedDiaObjects.doReadMarker) or (diaPipe.apdb_config_url == totalUnassociatedDiaObjects.apdb_config_url) - # Use of ConnectionsClass for templated fields is a workaround for DM-30210 - - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name == - fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name diff --git a/pipelines/HSC/ApVerifyCalibrate.yaml b/pipelines/HSC/ApVerifyCalibrate.yaml deleted file mode 100644 index f7a25936..00000000 --- a/pipelines/HSC/ApVerifyCalibrate.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Verification pipeline specialized for HSC. -# This concatenates various lsst.verify metrics to an AP pipeline - -description: Fully instrumented AP pipeline specialized for HSC -imports: - - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrate.yaml - # Include all metrics from standard pipeline. It's not practical to create - # a metrics subset because it would require constant micromanagement. - exclude: - - apPipe - - location: $AP_PIPE_DIR/pipelines/HSC/ApPipeCalibrate.yaml -tasks: - # ApVerify override removed by excluding apPipe. - diaPipe: - class: lsst.ap.association.DiaPipelineTask - config: - doPackageAlerts: True -contracts: - # Contracts removed by excluding apPipe - - contract: diaPipe.doConfigureApdb or not totalUnassociatedDiaObjects.doReadMarker - msg: "totalUnassociatedDiaObjects.doReadMarker requires diaPipe.doConfigureApdb" - - (totalUnassociatedDiaObjects.doReadMarker) or (diaPipe.apdb_config_url == totalUnassociatedDiaObjects.apdb_config_url) diff --git a/pipelines/LSSTCam-imSim/ApVerifyCalibrate.yaml b/pipelines/LSSTCam-imSim/ApVerifyCalibrate.yaml deleted file mode 100644 index 72c8788c..00000000 --- a/pipelines/LSSTCam-imSim/ApVerifyCalibrate.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Verification pipeline specialized for the DC2 ImSim simulation. -# This concatenates various lsst.verify metrics to an AP pipeline - -description: Fully instrumented AP pipeline specialized for LSSTCam-imSim -imports: - - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrate.yaml - # Include all metrics from standard pipeline. It's not practical to create - # a metrics subset because it would require constant micromanagement. - exclude: - - apPipe - - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipeCalibrate.yaml -tasks: - # ApVerify override removed by excluding apPipe. - diaPipe: - class: lsst.ap.association.DiaPipelineTask - config: - doPackageAlerts: True -contracts: - # Must re-declare contracts that cross apPipe and metrics boundary, as - # these were removed on import. - - contract: diaPipe.doConfigureApdb or not totalUnassociatedDiaObjects.doReadMarker - msg: "totalUnassociatedDiaObjects.doReadMarker requires diaPipe.doConfigureApdb" - - (totalUnassociatedDiaObjects.doReadMarker) or (diaPipe.apdb_config_url == totalUnassociatedDiaObjects.apdb_config_url) - # Use of ConnectionsClass for templated fields is a workaround for DM-30210 - - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name == - fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name diff --git a/pipelines/_ingredients/ApVerifyCalibrate.yaml b/pipelines/_ingredients/ApVerifyCalibrate.yaml deleted file mode 100644 index 06626d3b..00000000 --- a/pipelines/_ingredients/ApVerifyCalibrate.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Gen 3 pipeline for ap_verify -# This concatenates various lsst.verify metrics to an AP pipeline - -description: Fully instrumented AP pipeline -imports: - - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrate.yaml - # Metrics that should be run without fakes - - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsRuntime.yaml - exclude: - - timing_calibrateImage - - cputiming_calibrateImage - - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsMiscCalibrate.yaml - - location: $AP_VERIFY_DIR/pipelines/_ingredients/ConversionsCalibrate.yaml -tasks: - diaPipe: - class: lsst.ap.association.DiaPipelineTask - config: - # TODO: needed for "providing bulk sample alerts to brokers"; remove once - # we have an alternative. - doPackageAlerts: True -contracts: - - contract: diaPipe.doConfigureApdb or not totalUnassociatedDiaObjects.doReadMarker - msg: "totalUnassociatedDiaObjects.doReadMarker requires diaPipe.doConfigureApdb" - - (totalUnassociatedDiaObjects.doReadMarker) or (diaPipe.apdb_config_url == totalUnassociatedDiaObjects.apdb_config_url) - # Metric inputs must match pipeline outputs - # Use of ConnectionsClass for templated fields is a workaround for DM-30210 - - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name == - fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name diff --git a/pipelines/_ingredients/ConversionsCalibrate.yaml b/pipelines/_ingredients/ConversionsCalibrate.yaml deleted file mode 100644 index 22bc492d..00000000 --- a/pipelines/_ingredients/ConversionsCalibrate.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Add-on pipeline designed to be added to ApVerify or ApPipe for QA purposes. -# -# This pipeline depends on an external pipeline for the coaddName and fakesType -# pipeline parameters. - -description: Type conversion tasks customized for AP pipeline -tasks: - # Conversion of src [afw.table] to source [Parquet] - writeSourceTable: - class: lsst.pipe.tasks.postprocess.WriteSourceTableTask - # TODO: TransformSourceTableTask can't be run until we create a functor - # config that doesn't depend on shapeHSM. - # Merging of source [detector-level] to sourceTable_visit [visit-level] - consolidateSourceTable: - class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask - config: - # Skip DPDD-ified sourceTable and just use source directly - connections.inputCatalogs: "{catalogType}source" - - # Merging of *Diff_diaSrcTable [detector-level Parquet] to diaSourceTable [visit-level] - consolidateDiaSourceTable: - class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask - config: - # Task doesn't support coaddName, so coopt catalogType instead. - connections.catalogType: parameters.coaddName - connections.inputCatalogs: "{catalogType}Diff_diaSrcTable" - connections.outputCatalog: diaSourceTable - - # Creation of visitSummary - consolidateVisitSummary: - class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask - # Conversion of visitSummary [visit-level afw.table] to visitTable [instrument-level Parquet] - makeVisitTable: - class: lsst.pipe.tasks.postprocess.MakeVisitTableTask - config: - connections.visitSummaries: "{calexpType}visitSummary" - # Conversion of visitSummary [visit-level afw.table] to ccdVisitTable [instrument-level Parquet] - makeCcdVisitTable: - class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask - config: - connections.visitSummaryRefs: "{calexpType}visitSummary" diff --git a/pipelines/_ingredients/MetricsMiscCalibrate.yaml b/pipelines/_ingredients/MetricsMiscCalibrate.yaml deleted file mode 100644 index 807c6b2b..00000000 --- a/pipelines/_ingredients/MetricsMiscCalibrate.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Miscellaneous metrics for Alert Production -# In the future, these might be placed in task-specific pipelines (for debugging) -# or grouped by their datasets (to optimize expensive Butler reads) - -description: Miscelaneous AP Pipeline metrics -tasks: - numNewDiaObjects: - class: lsst.ap.association.metrics.NumberNewDiaObjectsMetricTask - config: - connections.labelName: diaPipe # partial name of metadata dataset - numUnassociatedDiaObjects: - class: lsst.ap.association.metrics.NumberUnassociatedDiaObjectsMetricTask - config: - connections.labelName: diaPipe - fracUpdatedDiaObjects: - class: lsst.ap.association.metrics.FractionUpdatedDiaObjectsMetricTask - config: - connections.labelName: diaPipe - numTotalSolarSystemObjects: - class: lsst.ap.association.metrics.NumberSolarSystemObjectsMetricTask - config: - connections.labelName: diaPipe - numAssociatedSsObjects: - class: lsst.ap.association.metrics.NumberAssociatedSolarSystemObjectsMetricTask - config: - connections.labelName: diaPipe - totalUnassociatedDiaObjects: - class: lsst.ap.association.metrics.TotalUnassociatedDiaObjectsMetricTask - config: - doReadMarker: False # Impossible if diaPipe uses new-style config - apdb_config_url: parameters.apdb_config - numSciSources: - class: lsst.ip.diffim.metrics.NumberSciSourcesMetricTask - fracDiaSourcesToSciSources: - class: lsst.ip.diffim.metrics.FractionDiaSourcesToSciSourcesMetricTask - config: - connections.coaddName: parameters.coaddName - numDeblendedSciSources: - class: lsst.pipe.tasks.metrics.NumberDeblendedSourcesMetricTask - numDeblendChildSciSources: - class: lsst.pipe.tasks.metrics.NumberDeblendChildSourcesMetricTask