Merge branch 'main' into tickets/DM-42576

pipelines/DECam/ApVerifyWithFakes.yaml

@@ -8,11 +8,3 @@ instrument: lsst.obs.decam.DarkEnergyCamera
 description: Fully instrumented AP pipeline with fakes, specialized for DECam
 imports:
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyWithFakes.yaml
-    exclude:
-      - processCcd
-  # The existing Fakes insertion system is not currently compatible with the
-  # new CalibrateImageTask, so the fakes pipeline still is on the old tasks.
-  - location: $AP_PIPE_DIR/pipelines/DECam/ProcessCcdCalibrate.yaml
-  # Can't use $AP_PIPE_DIR/pipelines/DECam/ApPipeWithFakes.yaml here
-  # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml
-  # are hard to separate.

pipelines/HSC/ApVerifyWithFakes.yaml

@@ -5,11 +5,3 @@ instrument: lsst.obs.subaru.HyperSuprimeCam
 description: Fully instrumented AP pipeline with fakes, specialized for HSC
 imports:
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyWithFakes.yaml
-    exclude:
-      - processCcd
-  # The existing Fakes insertion system is not currently compatible with the
-  # new CalibrateImageTask, so the fakes pipeline still is on the old tasks.
-  - location: $AP_PIPE_DIR/pipelines/HSC/ProcessCcdCalibrate.yaml
-  # Can't use $AP_PIPE_DIR/pipelines/HSC/ApPipeWithFakes.yaml here
-  # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml
-  # are hard to separate.

pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml

@@ -5,11 +5,3 @@ instrument: lsst.obs.lsst.LsstCamImSim
 description: Fully instrumented AP pipeline with fakes, specialized for LSSTCam-imSim
 imports:
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyWithFakes.yaml
-    exclude:
-      - processCcd
-  # The existing Fakes insertion system is not currently compatible with the
-  # new CalibrateImageTask, so the fakes pipeline still is on the old tasks.
-  - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ProcessCcdCalibrate.yaml
-  # Can't use $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipeWithFakes.yaml here
-  # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml
-  # are hard to separate.

pipelines/_ingredients/ApVerifyWithFakes.yaml

@@ -17,9 +17,17 @@ tasks:
 contracts:
   # Metric inputs must match pipeline outputs
   # Use of ConnectionsClass for templated fields is a workaround for DM-30210
-  - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name ==
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
       apFakesCompletenessMag20t22.connections.ConnectionsClass(config=apFakesCompletenessMag20t22).matchedFakes.name
-  - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name ==
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
       apFakesCompletenessMag22t24.connections.ConnectionsClass(config=apFakesCompletenessMag22t24).matchedFakes.name
-  - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name ==
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
       apFakesCompletenessMag24t26.connections.ConnectionsClass(config=apFakesCompletenessMag24t26).matchedFakes.name
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
+      apFakesCountMag20t22.connections.ConnectionsClass(config=apFakesCountMag20t22).matchedFakes.name
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
+      apFakesCountMag22t24.connections.ConnectionsClass(config=apFakesCountMag22t24).matchedFakes.name
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
+      apFakesCountMag24t26.connections.ConnectionsClass(config=apFakesCountMag24t26).matchedFakes.name
+  - injectedMatch.connections.ConnectionsClass(config=injectedMatch).matchedDiaSources.name ==
+      apFakesCount.connections.ConnectionsClass(config=apFakesCount).matchedFakes.name

pipelines/_ingredients/ConversionsForFakes.yaml

@@ -11,21 +11,15 @@ imports:
       # Fakes pipeline doesn't produce non-fakes DiaSourceTable.
       - consolidateDiaSourceTable
 tasks:
-  # Conversion of fakes_src [afw.table] to fakes_source [Parquet]
-  writeSourceTable:
-    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
-    config:
-      connections.catalogType: parameters.fakesType
   # TODO: TransformSourceTableTask can't be run until we create a functor
   # config that doesn't depend on shapeHSM.
   # Merging of fakes_source [detector-level] to fakes_sourceTable_visit [visit-level]
   consolidateSourceTable:
     class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
     config:
-      connections.catalogType: parameters.fakesType
-      # Skip DPDD-ified sourceTable and just use source directly
-      connections.inputCatalogs: "{catalogType}source"
-
+      # This is output directly by CalibrateImageTask.
+      connections.inputCatalogs: initial_stars_detector
+      connections.outputCatalog: initial_stars
   # Merging of fakes_*Diff_diaSrcTable [detector-level Parquet] to fakes_diaSourceTable [visit-level]
   consolidateDiaSourceTable:
     class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
@@ -36,22 +30,24 @@ tasks:
       # and fakesType is more stable than coaddName.
       connections.inputCatalogs: "fakes_{catalogType}Diff_diaSrcTable"
       connections.outputCatalog: fakes_diaSourceTable
-
   # Creation of fakes_visitSummary
   consolidateVisitSummary:
     class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask
     config:
       connections.calexpType: parameters.fakesType
-      connections.calexp: "{calexpType}calexp"
+      connections.calexp: "{calexpType}initial_pvi"
       connections.visitSummary: "{calexpType}visitSummary"
       connections.visitSummarySchema: "{calexpType}visitSummary_schema"
   # Conversion of fakes_visitSummary [visit-level afw.table] to fakes_visitTable [instrument-level Parquet]
   makeVisitTable:
     class: lsst.pipe.tasks.postprocess.MakeVisitTableTask
     config:
       connections.calexpType: parameters.fakesType
+      connections.visitSummaries: "{calexpType}visitSummary"
   # Conversion of fakes_visitSummary [visit-level afw.table] to fakes_ccdVisitTable [instrument-level Parquet]
   makeCcdVisitTable:
     class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask
     config:
       connections.calexpType: parameters.fakesType
+      connections.visitSummaryRefs: "{calexpType}visitSummary"
+