diff --git a/doc/lsst.ap.verify/index.rst b/doc/lsst.ap.verify/index.rst index d0a25397..e82d669b 100644 --- a/doc/lsst.ap.verify/index.rst +++ b/doc/lsst.ap.verify/index.rst @@ -25,6 +25,7 @@ Using lsst.ap.verify datasets new-metrics command-line-reference + limitations .. _lsst.ap.verify-contributing: diff --git a/doc/lsst.ap.verify/limitations.rst b/doc/lsst.ap.verify/limitations.rst new file mode 100644 index 00000000..ff8c7411 --- /dev/null +++ b/doc/lsst.ap.verify/limitations.rst @@ -0,0 +1,14 @@ +.. py:currentmodule:: lsst.ap.verify + +.. _ap-verify-request: + +################# +Known Limitations +################# + +mpSkyEphemerisQueryTask +======================= + +The AP pipelines now run :py:class:`lsst.ap.association.MPSkyEphemerisQueryTask`, which includes a request to mpSky. +mpSky is not expected to host ephemerides for all test datasets, (or be reachable from Jenkins) so it will often (or always) fail. +When requests fail, mpSkyEphemerisQuery will return nothing, and diaPipe will skip ssoAssociation. diff --git a/python/lsst/ap/verify/testPipeline.py b/python/lsst/ap/verify/testPipeline.py index d0d59d33..529b5a51 100644 --- a/python/lsst/ap/verify/testPipeline.py +++ b/python/lsst/ap/verify/testPipeline.py @@ -35,7 +35,8 @@ import lsst.afw.math as afwMath import lsst.afw.table as afwTable from lsst.ap.association import (LoadDiaCatalogsConfig, TransformDiaSourceCatalogConfig, - DiaPipelineConfig, FilterDiaSourceCatalogConfig) + DiaPipelineConfig, FilterDiaSourceCatalogConfig, + MPSkyEphemerisQueryConfig) from lsst.pipe.base import PipelineTask, Struct from lsst.ip.isr import IsrTaskConfig from lsst.ip.diffim import (GetTemplateConfig, AlardLuptonSubtractConfig, @@ -552,6 +553,16 @@ def run(self, template, science, difference, diaSources, pretrainedModel=None): return Struct(classifications=afwTable.BaseCatalog(afwTable.Schema())) +class MockMPSkyEphemerisQueryTask(PipelineTask): + """A do-nothing substitute for MPSkyEphemerisQueryTask. + """ + _DefaultName = "notMPSkyEphemerisQuery" + ConfigClass = MPSkyEphemerisQueryConfig + + def run(self, predictedRegionTime): + return Struct(ssObjects=pandas.DataFrame()) + + class MockTransformDiaSourceCatalogTask(PipelineTask): """A do-nothing substitute for TransformDiaSourceCatalogTask. """ diff --git a/tests/test_testPipeline.py b/tests/test_testPipeline.py index 133d9911..dbcd7def 100644 --- a/tests/test_testPipeline.py +++ b/tests/test_testPipeline.py @@ -42,7 +42,8 @@ MockCalibrateTask, MockGetTemplateTask, \ MockAlardLuptonSubtractTask, MockDetectAndMeasureTask, MockTransformDiaSourceCatalogTask, \ MockRBTransiNetTask, MockDiaPipelineTask, MockFilterDiaSourceCatalogTask, \ - MockSpatiallySampledMetricsTask, MockLoadDiaCatalogsTask + MockSpatiallySampledMetricsTask, MockMPSkyEphemerisQueryTask, MockLoadDiaCatalogsTask +from lsst.sphgeom import Circle class MockTaskTestSuite(unittest.TestCase): @@ -56,7 +57,6 @@ class MockTaskTestSuite(unittest.TestCase): @classmethod def setUpClass(cls): super().setUpClass() - repoDir = tempfile.mkdtemp() cls.addClassCleanup(shutil.rmtree, repoDir, ignore_errors=True) cls.repo = butlerTests.makeTestRepo(repoDir) @@ -77,6 +77,7 @@ def setUpClass(cls): name=INSTRUMENT, visit_max=256, exposure_max=256, detector_max=128, class_name="lsst.obs.base.instrument_tests.DummyCam", ) + cls.repo.registry.syncDimensionData("instrument", instrumentRecord) butlerTests.addDataIdValue(cls.repo, "physical_filter", PHYSICAL, band=BAND) butlerTests.addDataIdValue(cls.repo, "subfilter", SUB_FILTER) @@ -141,6 +142,8 @@ def setUpClass(cls): butlerTests.addDatasetType(cls.repo, "deepDiff_candidateDiaSrc", cls.visitId.dimensions, "SourceCatalog") butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.dimensions, "DataFrame") + butlerTests.addDatasetType(cls.repo, "preloaded_SsObjects", cls.groupId.dimensions, "DataFrame") + butlerTests.addDatasetType(cls.repo, "regionTimeInfo", cls.groupId.dimensions, "RegionTimeInfo") butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.dimensions, "Config") butlerTests.addDatasetType(cls.repo, "deepDiff_assocDiaSrc", cls.visitId.dimensions, "DataFrame") butlerTests.addDatasetType(cls.repo, "deepDiff_longTrailedSrc", cls.visitId.dimensions, @@ -356,6 +359,21 @@ def testMockRBTransiNetTask(self): }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) + def testMockMPSkyEphemerisQueryTask(self): + task = MockMPSkyEphemerisQueryTask() + pipelineTests.assertValidInitOutput(task) + dummyRegionTimeInfo = RegionTimeInfo(timespan=Timespan.makeEmpty(), region=Circle()) + result = task.run(predictedRegionTime=dummyRegionTimeInfo) + pipelineTests.assertValidOutput(task, result) + + self.butler.put(dummyRegionTimeInfo, "regionTimeInfo", self.groupId) + quantum = pipelineTests.makeQuantum( + task, self.butler, self.groupId, + {"predictedRegionTime": self.groupId, + "ssObjects": self.groupId, + }) + pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) + def testMockTransformDiaSourceCatalogTask(self): task = MockTransformDiaSourceCatalogTask(initInputs=afwTable.SourceCatalog()) pipelineTests.assertValidInitOutput(task) @@ -388,14 +406,14 @@ def testMockDiaPipelineTask(self): self.butler.put(pandas.DataFrame(), "preloaded_diaSources", self.groupId) self.butler.put(pandas.DataFrame(), "preloaded_diaForcedSources", self.groupId) self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable", self.visitId) - self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId) + self.butler.put(pandas.DataFrame(), "preloaded_SsObjects", self.groupId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, {"diaSourceTable": self.visitId, - "solarSystemObjectTable": self.visitId, + "solarSystemObjectTable": self.visitOnlyId, "diffIm": self.visitId, "exposure": self.visitId, "template": self.visitId,