From b2a4a1e9be65955353960f74b58702f2976f1624 Mon Sep 17 00:00:00 2001 From: rai-harshit Date: Sun, 28 Jan 2024 13:36:03 -0800 Subject: [PATCH] Add runtime metrics for FilterDiaSourceCatalogTask --- pipelines/_ingredients/MetricsRuntime.yaml | 16 ++++++++++++- python/lsst/ap/verify/testPipeline.py | 27 +++++++++++++++++++++- tests/MockApPipe.yaml | 7 ++++++ tests/test_testPipeline.py | 12 ++++++++-- 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pipelines/_ingredients/MetricsRuntime.yaml b/pipelines/_ingredients/MetricsRuntime.yaml index 479e0eaa..79435936 100644 --- a/pipelines/_ingredients/MetricsRuntime.yaml +++ b/pipelines/_ingredients/MetricsRuntime.yaml @@ -119,6 +119,13 @@ tasks: connections.metric: PackageAlertsTime connections.labelName: diaPipe target: diaPipe:alertPackager.run + timing_filterDiaSrcCat: + class: lsst.verify.tasks.commonMetrics.TimingMetricTask + config: + connections.package: ap_association + connections.metric: FilterDiaSourceCatalogTime + connections.labelName: filterDiaSrcCat + target: filterDiaSrcCat.run cputiming_isr: class: lsst.verify.tasks.commonMetrics.CpuTimingMetricTask config: @@ -233,6 +240,13 @@ tasks: connections.metric: PackageAlertsCpuTime connections.labelName: diaPipe target: diaPipe:alertPackager.run + cputiming_filterDiaSrcCat: + class: lsst.verify.tasks.commonMetrics.TimingMetricTask + config: + connections.package: ap_association + connections.metric: FilterDiaSourceCatalogCpuTime + connections.labelName: filterDiaSrcCat + target: filterDiaSrcCat.run memory_apPipe: class: lsst.verify.tasks.commonMetrics.MemoryMetricTask config: @@ -253,4 +267,4 @@ tasks: connections.package: ap_association connections.metric: PackageAlertsMemory connections.labelName: diaPipe - target: diaPipe:alertPackager.run + target: diaPipe:alertPackager.run \ No newline at end of file diff --git a/python/lsst/ap/verify/testPipeline.py b/python/lsst/ap/verify/testPipeline.py index a3f36b0d..efe47257 100644 --- a/python/lsst/ap/verify/testPipeline.py +++ b/python/lsst/ap/verify/testPipeline.py @@ -34,13 +34,14 @@ import lsst.afw.image as afwImage import lsst.afw.math as afwMath import lsst.afw.table as afwTable +from lsst.ap.association import (TransformDiaSourceCatalogConfig, + DiaPipelineConfig, FilterDiaSourceCatalogConfig) from lsst.pipe.base import PipelineTask, Struct from lsst.ip.isr import IsrTaskConfig from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig from lsst.pipe.tasks.calibrate import CalibrateConfig from lsst.meas.transiNet import RBTransiNetConfig -from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig class MockIsrTask(PipelineTask): @@ -434,6 +435,30 @@ def run(self, science, matchedTemplate, difference, ) +class MockFilterDiaSourceCatalogTask(PipelineTask): + """A do-nothing substitute for FilterDiaSourceCatalogTask. + """ + ConfigClass = FilterDiaSourceCatalogConfig + _DefaultName = "notFilterDiaSourceCatalog" + + def run(self, diaSourceCat): + """Produce filtering outputs with no processing. + + Parameters + ---------- + diaSourceCat : `lsst.afw.table.SourceCatalog` + Catalog of sources measured on the difference image. + + Returns + ------- + results : `lsst.pipe.base.Struct` + Results struct with components. + """ + return Struct(filteredDiaSourceCat=afwTable.SourceCatalog(), + rejectedDiaSources=afwTable.SourceCatalog(), + ) + + class MockRBTransiNetTask(PipelineTask): """A do-nothing substitute for RBTransiNetTask. """ diff --git a/tests/MockApPipe.yaml b/tests/MockApPipe.yaml index 95d5b84e..9833038f 100644 --- a/tests/MockApPipe.yaml +++ b/tests/MockApPipe.yaml @@ -35,6 +35,11 @@ tasks: config: connections.coaddName: parameters.coaddName doSkySources: True + filterDiaSrcCat: + class: lsst.ap.verify.testPipeline.MockFilterDiaSourceCatalogTask + config: + doRemoveSkySources: True + connections.coaddName: parameters.coaddName rbClassify: class: lsst.ap.verify.testPipeline.MockRBTransiNetTask config: @@ -70,6 +75,8 @@ contracts: - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name == transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name == + filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name + - filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name == transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name == diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name diff --git a/tests/test_testPipeline.py b/tests/test_testPipeline.py index ae91ace7..90e80320 100644 --- a/tests/test_testPipeline.py +++ b/tests/test_testPipeline.py @@ -38,7 +38,7 @@ from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \ MockCalibrateTask, MockGetTemplateTask, \ MockAlardLuptonSubtractTask, MockDetectAndMeasureTask, MockTransformDiaSourceCatalogTask, \ - MockRBTransiNetTask, MockDiaPipelineTask + MockRBTransiNetTask, MockDiaPipelineTask, MockFilterDiaSourceCatalogTask class MockTaskTestSuite(unittest.TestCase): @@ -121,6 +121,8 @@ def setUpClass(cls): butlerTests.addDatasetType(cls.repo, "deepDiff_matchedExp", cls.visitId.dimensions, "ExposureF") butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrc", cls.visitId.dimensions, "SourceCatalog") butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrcTable", cls.visitId.dimensions, "DataFrame") + butlerTests.addDatasetType(cls.repo, "deepDiff_candidateDiaSrc", cls.visitId.dimensions, + "SourceCatalog") butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.dimensions, "DataFrame") butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.dimensions, "Config") butlerTests.addDatasetType(cls.repo, "deepDiff_assocDiaSrc", cls.visitId.dimensions, "DataFrame") @@ -251,6 +253,12 @@ def testMockDetectAndMeasureTask(self): }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) + def testMockFilterDiaSourceCatalogTask(self): + task = MockFilterDiaSourceCatalogTask() + pipelineTests.assertValidInitOutput(task) + result = task.run(afwTable.SourceCatalog()) + pipelineTests.assertValidOutput(task, result) + def testMockRBTransiNetTask(self): task = MockRBTransiNetTask() pipelineTests.assertValidInitOutput(task) @@ -282,7 +290,7 @@ def testMockTransformDiaSourceCatalogTask(self): result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42) pipelineTests.assertValidOutput(task, result) - self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId) + self.butler.put(afwTable.SourceCatalog(), "deepDiff_candidateDiaSrc", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId,