Merge pull request #60 from lsst/tickets/DM-44040

DM 44040: Update dimensions-config version from 4 to 7.

.gitattributes

@@ -4,6 +4,5 @@
 *.sqlite3 filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text
-*.parq filter=lfs diff=lfs merge=lfs -text
 # Too small to be worth tracking
 scripts/rows_as_votable_1500407676_2451.vot.gz !filter !diff !merge

config/export.yaml

@@ -1,6 +1,6 @@
 description: Butler Data Repository Export
 version: 1.0.2
-universe_version: 4
+universe_version: 7
 universe_namespace: daf_butler
 data:
 - type: dimension
@@ -21,6 +21,17 @@ data:
     tract_max: 18938
     patch_nx_max: 6
     patch_ny_max: 6
+- type: dimension
+  element: day_obs
+  records:
+  - instrument: DECam
+    id: 20150217
+    datetime_begin: !butler_time/tai/iso '2015-02-17 12:00:00.000000000'
+    datetime_end: !butler_time/tai/iso '2015-02-18 12:00:00.000000000'
+  - instrument: DECam
+    id: 20150308
+    datetime_begin: !butler_time/tai/iso '2015-03-08 12:00:00.000000000'
+    datetime_end: !butler_time/tai/iso '2015-03-09 12:00:00.000000000'
 - type: dimension
   element: detector
   records:
@@ -396,6 +407,15 @@ data:
     name_in_raft: '31'
     raft: N
     purpose: SCIENCE
+- type: dimension
+  element: group
+  records:
+  - instrument: DECam
+    name: '411371'
+  - instrument: DECam
+    name: '411420'
+  - instrument: DECam
+    name: '419802'
 - type: dimension
   element: physical_filter
   records:
@@ -422,18 +442,17 @@ data:
   records:
   - instrument: DECam
     id: 411371
+    day_obs: 20150217
+    group: '411371'
     physical_filter: g DECam SDSS c0001 4720.0 1520.0
     obs_id: ct4m20150218t052637
     exposure_time: 86.0
     dark_time: 87.1173
     observation_type: science
     observation_reason: science
-    day_obs: 20150217
     seq_num: 411371
     seq_start: 411371
     seq_end: 411371
-    group_name: '411371'
-    group_id: 411371
     target_name: Blind15A_42
     science_program: 2015A-0608
     tracking_ra: 155.47028496089578
@@ -442,22 +461,22 @@ data:
     azimuth: 353.1027
     zenith_angle: 25.289999999999992
     has_simulated: false
+    can_see_sky: true
     datetime_begin: !butler_time/tai/iso '2015-02-18 05:27:12.433035000'
     datetime_end: !butler_time/tai/iso '2015-02-18 05:29:25.000000000'
   - instrument: DECam
     id: 411420
+    day_obs: 20150217
+    group: '411420'
     physical_filter: g DECam SDSS c0001 4720.0 1520.0
     obs_id: ct4m20150218t070355
     exposure_time: 86.0
     dark_time: 87.3794801
     observation_type: science
     observation_reason: science
-    day_obs: 20150217
     seq_num: 411420
     seq_start: 411420
     seq_end: 411420
-    group_name: '411420'
-    group_id: 411420
     target_name: Blind15A_40
     science_program: 2015A-0608
     tracking_ra: 155.12027235395627
@@ -466,22 +485,22 @@ data:
     azimuth: 306.579
     zenith_angle: 35.09
     has_simulated: false
+    can_see_sky: true
     datetime_begin: !butler_time/tai/iso '2015-02-18 07:04:30.532568000'
     datetime_end: !butler_time/tai/iso '2015-02-18 07:06:54.000000000'
   - instrument: DECam
     id: 419802
+    day_obs: 20150308
+    group: '419802'
     physical_filter: g DECam SDSS c0001 4720.0 1520.0
     obs_id: ct4m20150309t055516
     exposure_time: 102.0
     dark_time: 103.1057501
     observation_type: science
     observation_reason: science
-    day_obs: 20150308
     seq_num: 419802
     seq_start: 419802
     seq_end: 419802
-    group_name: '419802'
-    group_id: 419802
     target_name: Blind15A_40
     science_program: 2015A-0608
     tracking_ra: 155.12015152062799
@@ -490,6 +509,7 @@ data:
     azimuth: 304.7488
     zenith_angle: 36.15
     has_simulated: false
+    can_see_sky: true
     datetime_begin: !butler_time/tai/iso '2015-03-09 05:55:51.522985000'
     datetime_end: !butler_time/tai/iso '2015-03-09 05:58:23.000000000'
 - type: dimension
@@ -570,9 +590,9 @@ data:
   records:
   - instrument: DECam
     id: 411371
+    day_obs: 20150217
     physical_filter: g DECam SDSS c0001 4720.0 1520.0
     name: ct4m20150218t052637
-    day_obs: 20150217
     seq_num: 411371
     exposure_time: 86.0
     target_name: Blind15A_42
@@ -586,9 +606,9 @@ data:
     datetime_end: !butler_time/tai/iso '2015-02-18 05:29:25.000000000'
   - instrument: DECam
     id: 411420
+    day_obs: 20150217
     physical_filter: g DECam SDSS c0001 4720.0 1520.0
     name: ct4m20150218t070355
-    day_obs: 20150217
     seq_num: 411420
     exposure_time: 86.0
     target_name: Blind15A_40
@@ -602,9 +622,9 @@ data:
     datetime_end: !butler_time/tai/iso '2015-02-18 07:06:54.000000000'
   - instrument: DECam
     id: 419802
+    day_obs: 20150308
     physical_filter: g DECam SDSS c0001 4720.0 1520.0
     name: ct4m20150309t055516
-    day_obs: 20150308
     seq_num: 419802
     exposure_time: 102.0
     target_name: Blind15A_40
@@ -9282,6 +9302,7 @@ data:
   dimensions:
   - band
   - instrument
+  - day_obs
   - physical_filter
   - visit
   storage_class: DataFrame

scripts/generate_all_gen3.sh

@@ -123,7 +123,7 @@ python "${SCRIPT_DIR}/generate_ephemerides_gen3.py"
 butler collection-chain "${DATASET_REPO}" sso sso/cached
 butler collection-chain "${DATASET_REPO}" DECam/defaults templates/goodSeeing skymaps DECam/calib refcats sso \
     models ${INJECTION_CATALOG_COLLECTION}
-python "${SCRIPT_DIR}/make_preloaded_export.py" --dataset ap_verify_ci_hits2015
+python "${SCRIPT_DIR}/make_preloaded_export.py"
 
 echo "Gen 3 preloaded repository complete."
 echo "All preloaded data products are accessible through the DECam/defaults collection."

scripts/make_preloaded_export.py

@@ -1,48 +1,67 @@
 #!/usr/bin/env python
+# This file is part of ap_verify_ci_hits2015.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Make the butler export yaml file to be used by ap_verify runs.
+
+This script must be run after **any** change to the preloaded repository;
+otherwise, ingestion may fail or the changes may not be visible.
+"""
 
-import argparse
 import logging
 import os
 import sys
 
 import lsst.log
 import lsst.skymap
 import lsst.daf.butler as daf_butler
-import lsst.ap.verify as ap_verify
 
 
-def _make_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--dataset", required=True,
-                        help="The name of the dataset as recognized by ap_verify.py.")
-    return parser
+# Avoid explicit references to dataset package to maximize portability.
+SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
+REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "preloaded"))
+CONFIG_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "config"))
 
 
 def main():
     # Ensure logs from tasks are visible
     logging.basicConfig(level=logging.INFO, stream=sys.stdout)
     lsst.log.configure_pylog_MDC("DEBUG", MDC_class=None)
 
-    args = _make_parser().parse_args()
-    dataset = ap_verify.dataset.Dataset(args.dataset)
-    gen3_repo = os.path.join(dataset.datasetRoot, "preloaded")
-
-    logging.info("Exporting Gen 3 registry to configure new repos...")
-    _export_for_copy(dataset, gen3_repo)
+    logging.info("Exporting registry to configure new repos...")
+    _export_for_copy(REPO_DIR, CONFIG_DIR)
 
 
-def _export_for_copy(dataset, repo):
-    """Export a Gen 3 repository so that a dataset can make copies later.
+def _export_for_copy(repo, export_dir):
+    """Export a butler repository so that a dataset can make copies later.
 
     Parameters
     ----------
-    dataset : `lsst.ap.verify.dataset.Dataset`
-        The dataset needing the ability to copy the repository.
     repo : `str`
-        The location of the Gen 3 repository.
+        The location of the repository.
+    export_dir : `str`
+        The location at which to create the export file.
     """
     butler = daf_butler.Butler(repo)
-    with butler.export(directory=dataset.configLocation, format="yaml") as contents:
+    with butler.export(directory=export_dir, format="yaml") as contents:
         # Need all detectors, even those without data, for visit definition
         contents.saveDataIds(butler.registry.queryDataIds({"detector"}).expanded())
         contents.saveDatasets(butler.registry.queryDatasets(datasetType=..., collections=...))
@@ -57,8 +76,12 @@ def _export_for_copy(dataset, repo):
         contents.saveCollection(lsst.skymap.BaseSkyMap.SKYMAP_RUN_COLLECTION_NAME)
         # Dataset export exports visits, but need matching visit definitions as
         # well (DefineVisitsTask won't add them back in).
+        contents.saveDimensionData("day_obs",
+                                   butler.registry.queryDimensionRecords("day_obs"))
         contents.saveDimensionData("exposure",
                                    butler.registry.queryDimensionRecords("exposure"))
+        contents.saveDimensionData("group",
+                                   butler.registry.queryDimensionRecords("group"))
         contents.saveDimensionData("visit_definition",
                                    butler.registry.queryDimensionRecords("visit_definition"))
         contents.saveDimensionData("visit_detector_region",