From 6e736d7de275594a90fb05d9a7f09337862c3bc2 Mon Sep 17 00:00:00 2001 From: Lee Kelvin Date: Mon, 16 Sep 2024 05:37:47 -0700 Subject: [PATCH] Remove ApTemplate from ap_pipe/ap_verify and docs --- doc/ap_verify_ci_hits2015/index.rst | 2 +- pipelines/ApTemplate.yaml | 8 -- scripts/generate_all_gen3.sh | 8 -- scripts/generate_calibs_gen3_template.sh | 107 ------------------ scripts/generate_templates_gen3.sh | 133 ----------------------- scripts/import_templates_gen3.py | 6 +- 6 files changed, 2 insertions(+), 262 deletions(-) delete mode 100644 pipelines/ApTemplate.yaml delete mode 100755 scripts/generate_calibs_gen3_template.sh delete mode 100755 scripts/generate_templates_gen3.sh diff --git a/doc/ap_verify_ci_hits2015/index.rst b/doc/ap_verify_ci_hits2015/index.rst index 58f5fce..9e099c9 100644 --- a/doc/ap_verify_ci_hits2015/index.rst +++ b/doc/ap_verify_ci_hits2015/index.rst @@ -14,7 +14,7 @@ It is intended for continuous integration. Using ap_verify_ci_hits2015 =========================== -This dataset is intended for fast integration testing of difference imaging analysis, using "deep" coadd templates, by ``ap_verify``. +This dataset is intended for fast integration testing of difference imaging analysis, using "goodSeeing" coadd templates, by ``ap_verify``. Because of the small data volume it is not recommended for "stress tests" or other large-scale testing. .. _ap_verify_ci_hits2015-contents: diff --git a/pipelines/ApTemplate.yaml b/pipelines/ApTemplate.yaml deleted file mode 100644 index 4de5dee..0000000 --- a/pipelines/ApTemplate.yaml +++ /dev/null @@ -1,8 +0,0 @@ -description: Alert Production template building pipeline specialized for HiTS-2015 -# This pipeline assumes the working repo has raws, calibs, refcats, and a skymap, -# and that you have already run the RunIsrForCrosstalkSources.yaml pipeline. - -imports: - - location: $AP_PIPE_DIR/pipelines/DECam/ApTemplate.yaml -parameters: - coaddName: goodSeeing diff --git a/scripts/generate_all_gen3.sh b/scripts/generate_all_gen3.sh index b273853..003e456 100755 --- a/scripts/generate_all_gen3.sh +++ b/scripts/generate_all_gen3.sh @@ -87,14 +87,6 @@ INJECTION_CATALOG_COLLECTION="fake-injection-catalog" "${SCRIPT_DIR}/generate_calibs_gen3_science.sh" -b ${SCRATCH_REPO} -c "${CALIB_COLLECTION_SCI}" -######################################## -# Prepare templates - -"${SCRIPT_DIR}/generate_calibs_gen3_template.sh" -b ${SCRATCH_REPO} -c "${CALIB_COLLECTION_TMP}" -"${SCRIPT_DIR}/generate_templates_gen3.sh" -b ${SCRATCH_REPO} -c "${CALIB_COLLECTION_TMP}-calib" \ - -o "${TEMPLATE_COLLECTION}" - - ######################################## # Repository creation diff --git a/scripts/generate_calibs_gen3_template.sh b/scripts/generate_calibs_gen3_template.sh deleted file mode 100755 index 0d3f24e..0000000 --- a/scripts/generate_calibs_gen3_template.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# This file is part of ap_verify_ci_hits2015. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Script for automatically generating template calibs for this dataset. It -# takes roughly 4 hours to run on lsst-devl. -# This script should be run before generating templates in order to -# self-consistently use all pipeline improvements. -# -# Example: -# $ nohup generate_calibs_gen3_template.sh -c "u/me/DM-123456" & -# produces certified calibs in /repo/main in the u/me/DM-123456-calib -# collection. See generate_calibs_gen3_template.sh -h for more options. - - -# Common definitions -SCRIPT_DIR="$( dirname -- "${BASH_SOURCE[0]}" )" -source "${SCRIPT_DIR}/generate_calibs_gen3.sh" - - -######################################## -# Raw calibs to process - -# Syntax matters -- use -# https://pipelines.lsst.io/v/daily/modules/lsst.daf.butler/queries.html#dimension-expressions -# syntax, with no trailing comma. - -declare -A EXPOSURES_BIAS -EXPOSURES_BIAS[20140228]='288902, 288903, 288904, 288905, 288906, 288907, 288908, 288909, 288910, - 288911, 288912' -EXPOSURES_BIAS[20140301]='289119, 289120, 289121, 289122, 289123, 289124, 289125, 289126, 289127, - 289128, 289129' -EXPOSURES_BIAS[20140302]='289328, 289329, 289330, 289331, 289332, 289333, 289334, 289335, 289336, - 289337, 289338' -EXPOSURES_BIAS[20140303]='289532, 289533, 289534, 289535, 289536, 289537, 289538, 289539, 289540, - 289541, 289542' -EXPOSURES_BIAS[20140304]='289742, 289743, 289744, 289745, 289746, 289747, 289748, 289749, 289750, - 289751, 289752' - -declare -A VALIDITIES_BIAS -VALIDITIES_BIAS[20140228]='--begin-date 2014-02-15T00:00:00 --end-date 2014-02-28T23:59:59' -VALIDITIES_BIAS[20140301]='--begin-date 2014-03-01T00:00:00 --end-date 2014-03-01T23:59:59' -VALIDITIES_BIAS[20140302]='--begin-date 2014-03-02T00:00:00 --end-date 2014-03-02T23:59:59' -VALIDITIES_BIAS[20140303]='--begin-date 2014-03-03T00:00:00 --end-date 2014-03-03T23:59:59' -VALIDITIES_BIAS[20140304]='--begin-date 2014-03-04T00:00:00 --end-date 2014-03-15T23:59:59' - -declare -A EXPOSURES_FLAT_g_c0001 -EXPOSURES_FLAT_g_c0001[20140228]='288847, 288848, 288849, 288850, 288851, 288852, 288853, 288854, - 288855, 288856, 288857' -EXPOSURES_FLAT_g_c0001[20140301]='289130, 289131, 289132, 289133, 289134, 289135, 289136, 289137, - 289138, 289139, 289140' -EXPOSURES_FLAT_g_c0001[20140302]='289339, 289340, 289341, 289342, 289343, 289344, 289345, 289346, - 289347, 289348, 289349' -EXPOSURES_FLAT_g_c0001[20140303]='289543, 289544, 289545, 289546, 289547, 289548, 289549, 289550, - 289551, 289552, 289553' -EXPOSURES_FLAT_g_c0001[20140304]='289753, 289754, 289755, 289756, 289757, 289758, 289759, 289760, - 289761, 289762, 289763' - -declare -A VALIDITIES_FLAT_g_c0001 -VALIDITIES_FLAT_g_c0001[20140228]='--begin-date 2014-02-15T00:00:00 --end-date 2014-02-28T23:59:59' -VALIDITIES_FLAT_g_c0001[20140301]='--begin-date 2014-03-01T00:00:00 --end-date 2014-03-01T23:59:59' -VALIDITIES_FLAT_g_c0001[20140302]='--begin-date 2014-03-02T00:00:00 --end-date 2014-03-02T23:59:59' -VALIDITIES_FLAT_g_c0001[20140303]='--begin-date 2014-03-03T00:00:00 --end-date 2014-03-03T23:59:59' -VALIDITIES_FLAT_g_c0001[20140304]='--begin-date 2014-03-04T00:00:00 --end-date 2014-03-15T23:59:59' - -EXPOSURES_CROSSTALK=$(join_by , "${EXPOSURES_FLAT_g_c0001[@]}") - - -######################################## -# Command-line options - -parse_args "$@" - - -######################################## -# Generate calibs - -do_crosstalk $BUTLER_REPO $COLLECT_ROOT "$EXPOSURES_CROSSTALK" - -# Associative array hack from https://stackoverflow.com/questions/4069188/ -do_bias $BUTLER_REPO $COLLECT_ROOT "$(declare -p EXPOSURES_BIAS)" "$(declare -p VALIDITIES_BIAS)" -do_flat $BUTLER_REPO $COLLECT_ROOT \ - "$(declare -p EXPOSURES_FLAT_g_c0001)" "$(declare -p VALIDITIES_FLAT_g_c0001)" - - -######################################## -# Final summary - -echo "Biases and flats stored in ${BUTLER_REPO}:${COLLECT_ROOT}-calib" diff --git a/scripts/generate_templates_gen3.sh b/scripts/generate_templates_gen3.sh deleted file mode 100755 index d4b8c48..0000000 --- a/scripts/generate_templates_gen3.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# This file is part of ap_verify_ci_hits2015. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Script for automatically generating differencing templates for this dataset. -# It takes roughly 5 hours to run on lsst-devl. -# Running this script allows for templates to incorporate pipeline -# improvements. It makes no attempt to update the set of input exposures; they -# are hard-coded into the file. -# -# Example: -# $ nohup generate_templates_gen3.sh -c "u/me/DM-123456-calib" -o "u/me/DM-123456-template" & -# produces templates in /repo/main in the u/me/DM-123456-template collection. -# See generate_templates_gen3.sh -h for more options. - - -# Abort script on any error -set -e - - -######################################## -# Raw template exposures to process - -# Exposure filter is a safeguard against queries that can't be constrained by -# tract/patch, or against inclusion of non-HiTS data. Patch filter specifies -# the area covered by the CI dataset in the decam_rings_v1 skymap. - -FIELDS="'Blind14A_09', 'Blind14A_10'" -EXPOSURES="288934, 288935, 288975, 288976, 289015, 289016, 289055, 289056, 289160, 289161, 289201, - 289202, 289242, 289243, 289283, 289284, 289367, 289368, 289408, 289409, 289449, 289450, - 289492, 289493, 289572, 289573, 289613, 289614, 289655, 289656, 289696, 289697, 289782, - 289783, 289818, 289820, 289828, 289870, 289871, 289912, 289913" -declare -A PATCHES -PATCHES[8363]="33, 34" -PATCHES[8604]="0, 1, 2, 6, 7, 8" -PATCHES[8605]="5, 11" -DATE_CUTOFF=20141231 - - -######################################## -# Command-line options - -print_error() { - >&2 echo "$@" -} - -usage() { - print_error - print_error "Usage: $0 [-b BUTLER_REPO] -c ROOT_COLLECTION [-h]" - print_error - print_error "Specific options:" - print_error " -b Butler repo URI, defaults to /repo/main" - print_error " -c input calib collection for template processing" - print_error " -o template collection name; will also appear in final repo" - print_error " -h show this message" - exit 1 -} - -parse_args() { - while getopts "b:c:o:h" option $@; do - case "$option" in - b) BUTLER_REPO="$OPTARG";; - c) CALIBS="$OPTARG";; - o) TEMPLATES="$OPTARG";; - h) usage;; - *) usage;; - esac - done - if [[ -z "${BUTLER_REPO}" ]]; then - BUTLER_REPO="/repo/main" - fi - if [[ -z "${CALIBS}" ]]; then - print_error "$0: mandatory argument -- c" - usage - exit 1 - fi - if [[ -z "${TEMPLATES}" ]]; then - print_error "$0: mandatory argument -- o" - usage - exit 1 - fi -} -parse_args "$@" - - -######################################## -# Generate templates - -FILTER="instrument='DECam' AND skymap='decam_rings_v1' - AND exposure IN (${EXPOSURES}) AND detector NOT IN (2, 61) - AND (" -OR="" -for tract in ${!PATCHES[*]}; do - FILTER="${FILTER} ${OR} (tract=${tract} AND patch IN (${PATCHES[${tract}]}))" - OR="OR" -done -FILTER="${FILTER})" - - -pipetask run -j 12 -d "${FILTER}" -b ${BUTLER_REPO} -i DECam/defaults,${CALIBS} -o ${TEMPLATES} \ - -p $AP_PIPE_DIR/pipelines/DECam/RunIsrForCrosstalkSources.yaml - -# Run single-frame processing and coaddition separately, so that isolated -# errors in SFP do not prevent coaddition from running. Instead, coadds will -# use all successful runs, ignoring any failures. -pipetask run -j 12 -d "${FILTER}" -b ${BUTLER_REPO} -o ${TEMPLATES} \ - -p $AP_VERIFY_CI_HITS2015_DIR/pipelines/ApTemplate.yaml#singleFrameAp -pipetask run -j 12 -d "${FILTER}" -b ${BUTLER_REPO} -o ${TEMPLATES} \ - -p $AP_VERIFY_CI_HITS2015_DIR/pipelines/ApTemplate.yaml#makeTemplate - - -######################################## -# Final summary - -echo "Templates stored in ${BUTLER_REPO}:${TEMPLATES}" diff --git a/scripts/import_templates_gen3.py b/scripts/import_templates_gen3.py index 0291567..40d9cee 100644 --- a/scripts/import_templates_gen3.py +++ b/scripts/import_templates_gen3.py @@ -22,14 +22,10 @@ """Script for copying templates appropriate for these fields. -The datasets can be from any source, including the generate_templates_gen3.sh -script in this directory. - Example: $ python import_templates_gen3.py -t "u/me/DM-123456-template" imports goodSeeing templates from u/me/DM-123456-template in /repo/main to -templates/goodSeeing in this dataset's preloaded repo. See -generate_templates_gen3.sh -h for more options. +templates/goodSeeing in this dataset's preloaded repo. """ import argparse