Skip to content

Commit

Permalink
Merge pull request #751 from Keck-DataReductionPipelines/develop
Browse files Browse the repository at this point in the history
Version 2.5.3
  • Loading branch information
bjfultn authored Dec 19, 2023
2 parents 74b36ac + 70c053d commit 82cf70d
Show file tree
Hide file tree
Showing 73 changed files with 4,140 additions and 1,230 deletions.
97 changes: 97 additions & 0 deletions AWH_notebooks/QC Example - L0 Data Products Check.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "76101924",
"metadata": {},
"source": [
"This notebook provides an example running a quality control (QC) method with the KPF DRP."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "ce403f32",
"metadata": {},
"outputs": [],
"source": [
"# Import packages for reading files\n",
"\n",
"from modules.Utils.kpf_parse import get_datecode\n",
"from kpfpipe.models.level0 import KPF0\n",
"from modules.quality_control.src.quality_control import QCL0"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "8240bcca",
"metadata": {},
"outputs": [],
"source": [
"# Example files\n",
"\n",
"# Good L0 files\n",
"#ObsID = 'KP.20230830.33530.82' # file with no missing data products\n",
"#ObsID = 'KP.20230829.76026.81' # file with no missing data products\n",
"\n",
"# Bad L0 file\n",
"ObsID = 'KP.20231108.77769.16' # file with missing Red\n",
"\n",
"# Read the file\n",
"L0_filename = '/data/L0/' + get_datecode(ObsID) + '/' + ObsID + '.fits'\n",
"L0 = KPF0.from_fits(L0_filename)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "850c1449",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Data products that are supposed to be in this L0 file: ['Green', 'Red', 'ExpMeter', 'Telemetry', 'Pyrheliometer']\n",
"Data products in L0 file: ['Green', 'ExpMeter', 'Telemetry', 'Pyrheliometer']\n",
"Red not present in L0 file. QC(L0_data_products_check) failed.\n",
"---->add_qc_keyword_to_header: qc_name, keyword, value, comment = L0_data_products_check, DATAPRL0, False, QC: L0 data present check\n",
"QC result: False\n"
]
}
],
"source": [
"# Test of 'L0_data_products_check'\n",
"\n",
"qcl0 = QCL0(L0)\n",
"qc_name = 'L0_data_products_check'\n",
"qc_value = qcl0.L0_data_products_check(L0)\n",
"qcl0.add_qc_keyword_to_header(qc_name,qc_value)\n",
"L0_new = qcl0.fits_object\n",
"print('QC result: ' + str(L0_new.header['PRIMARY']['DATAPRL0']))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN mkdir /code && \
mkdir /outputs && \
apt-get --yes update && \
apt install build-essential -y --no-install-recommends && \
apt-get install --yes git vim emacs nano && \
apt-get install --yes git vim emacs nano parallel && \
/usr/local/bin/python -m pip install --upgrade pip && \
cd /code/KPF-Pipeline && \
mkdir -p logs && \
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ If you use the KPF Data Reduction Pipeline in your research, please cite the fol

If there is no place to include the relevant citations in the text of the publication, please include the following acknowledgment (in LaTeX using the [provided BibTeX entry](kpf_bibliography.bib)):

"This research made use of the KPF Data Reduction Pipeline \citep{kpf:gibson2020}."
"This research made use of the KPF Data Reduction Pipeline \citep{kpf:gibson2020}."
4 changes: 2 additions & 2 deletions configs/framework_multi.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ temp_directory = "temp"
#
# no_event_event = None
#
no_event_event = Event('no_event', None)
# no_event_event = Event('exit', None)
# no_event_event = Event('no_event', None)
no_event_event = Event('wait', None)

#
# How long to wait before sending no_event_event
Expand Down
1 change: 1 addition & 0 deletions configs/kpf_drp.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -181,4 +181,5 @@ hk_spectral_extraction = modules/ca_hk/configs/default_hk.cfg
quicklook = modules/quicklook/configs/default.cfg
bias_subtraction = modules/bias_subtraction/configs/default.cfg
flat_fielding = modules/flat_fielding/configs/default.cfg
quality_control = modules/quality_control/configs/default.cfg

11 changes: 6 additions & 5 deletions configs/kpf_masters_drp.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ channel_datasec_ncols_green = 2040
channel_datasec_nrows_green = 2040
channel_datasec_ncols_red = 2040
channel_datasec_nrows_red = 2040
# These two paths will have KPFPIPE_TEST_DATA/ prepended.
channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
prescan_reg = [0,4]
Expand All @@ -43,7 +42,7 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
# 20230619 had OBJECT=test with just one frame, so we handle it specially as follows:
bias_objects = ['autocal-bias']
# The above with explicit bias_objects = ['autocal-bias'] is the best policy for automated daily runs.
masterbias_path = /testdata/kpf_master_bias.fits
masterbias_path = /data/masters/pool/kpf_master_bias.fits

[DARK]
exptime_minimum = 300.0
Expand All @@ -53,12 +52,14 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
#dark_objects = []
dark_objects = ['autocal-dark']
# The above with explicit dark_objects = ['autocal-dark'] is the best policy for automated daily runs.
masterdark_path = /testdata/kpf_master_dark.fits
masterdark_path = /data/masters/pool/kpf_master_dark.fits

[FLAT]
n_sigma = 2.3
lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
masterflat_path = /testdata/kpf_master_flat.fits
flat_objects = ['autocal-flat-all']
# The above with explicit flat_objects = ['autocal-flat-all'] is the best policy for automated daily runs.
masterflat_path = /data/masters/pool/kpf_master_flat.fits
# Smooth lamp pattern made from stack of 100 frames taken on 20230628.
# A smoothing kernel 200-pixels wide (along dispersion dimension) by
# 1-pixel high (along cross-dispersion dimension) is used for computing
Expand All @@ -77,7 +78,7 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD']
arclamp_objects = []
# Generic output filename masterarclamp_path will be automatically modified with date after kpf_ prefix
# and suffix _arclamp_object before .fits by the code. E.g., kpf_20230106_master_arclamp_LFC_all.fits
masterarclamp_path = /testdata/kpf_master_arclamp.fits
masterarclamp_path = /data/masters/pool/kpf_master_arclamp.fits
# Set skip_flattening = 0 to not skip; otherwise, set to 1 to skip
skip_flattening = 0

Expand Down
11 changes: 6 additions & 5 deletions configs/kpf_masters_drp_before20230623.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ channel_datasec_ncols_green = 2040
channel_datasec_nrows_green = 2040
channel_datasec_ncols_red = 2040
channel_datasec_nrows_red = 2040
# These two paths will have KPFPIPE_TEST_DATA/ prepended.
channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
prescan_reg = [0,4]
Expand All @@ -43,7 +42,7 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
# 20230619 had OBJECT=test with just one frame, so we handle it specially as follows:
bias_objects = ['autocal-bias']
# The above with explicit bias_objects = ['autocal-bias'] is the best policy for automated daily runs.
masterbias_path = /testdata/kpf_master_bias.fits
masterbias_path = /data/masters/pool/kpf_master_bias.fits

[DARK]
exptime_minimum = 300.0
Expand All @@ -53,12 +52,14 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
#dark_objects = []
dark_objects = ['autocal-dark']
# The above with explicit dark_objects = ['autocal-dark'] is the best policy for automated daily runs.
masterdark_path = /testdata/kpf_master_dark.fits
masterdark_path = /data/masters/pool/kpf_master_dark.fits

[FLAT]
n_sigma = 2.3
lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
masterflat_path = /testdata/kpf_master_flat.fits
flat_objects = ['autocal-flat-all']
# The above with explicit flat_objects = ['autocal-flat-all'] is the best policy for automated daily runs.
masterflat_path = /data/masters/pool/kpf_master_flat.fits
# Smooth lamp pattern made from stack of 100 frames taken on 20230628.
# A smoothing kernel 200-pixels wide (along dispersion dimension) by
# 1-pixel high (along cross-dispersion dimension) is used for computing
Expand All @@ -79,7 +80,7 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD']
arclamp_objects = []
# Generic output filename masterarclamp_path will be automatically modified with date after kpf_ prefix
# and suffix _arclamp_object before .fits by the code. E.g., kpf_20230106_master_arclamp_LFC_all.fits
masterarclamp_path = /testdata/kpf_master_arclamp.fits
masterarclamp_path = /data/masters/pool/kpf_master_arclamp.fits
# Set skip_flattening = 0 to not skip; otherwise, set to 1 to skip
skip_flattening = 0

Expand Down
11 changes: 6 additions & 5 deletions configs/kpf_masters_drp_from20230624to20230730.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ channel_datasec_ncols_green = 2040
channel_datasec_nrows_green = 2040
channel_datasec_ncols_red = 2040
channel_datasec_nrows_red = 2040
# These two paths will have KPFPIPE_TEST_DATA/ prepended.
channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
prescan_reg = [0,4]
Expand All @@ -43,7 +42,7 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
# 20230619 had OBJECT=test with just one frame, so we handle it specially as follows:
bias_objects = ['autocal-bias']
# The above with explicit bias_objects = ['autocal-bias'] is the best policy for automated daily runs.
masterbias_path = /testdata/kpf_master_bias.fits
masterbias_path = /data/masters/pool/kpf_master_bias.fits

[DARK]
exptime_minimum = 300.0
Expand All @@ -53,12 +52,14 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
#dark_objects = []
dark_objects = ['autocal-dark']
# The above with explicit dark_objects = ['autocal-dark'] is the best policy for automated daily runs.
masterdark_path = /testdata/kpf_master_dark.fits
masterdark_path = /data/masters/pool/kpf_master_dark.fits

[FLAT]
n_sigma = 2.3
lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
masterflat_path = /testdata/kpf_master_flat.fits
flat_objects = ['autocal-flat-all']
# The above with explicit flat_objects = ['autocal-flat-all'] is the best policy for automated daily runs.
masterflat_path = /data/masters/pool/kpf_master_flat.fits
# Smooth lamp pattern made from stack of 100 frames taken on 20230628.
# A smoothing kernel 200-pixels wide (along dispersion dimension) by
# 1-pixel high (along cross-dispersion dimension) is used for computing
Expand All @@ -79,7 +80,7 @@ lev0_ffi_exts = ['GREEN_CCD','RED_CCD']
arclamp_objects = []
# Generic output filename masterarclamp_path will be automatically modified with date after kpf_ prefix
# and suffix _arclamp_object before .fits by the code. E.g., kpf_20230106_master_arclamp_LFC_all.fits
masterarclamp_path = /testdata/kpf_master_arclamp.fits
masterarclamp_path = /data/masters/pool/kpf_master_arclamp.fits
# Set skip_flattening = 0 to not skip; otherwise, set to 1 to skip
skip_flattening = 0

Expand Down
13 changes: 8 additions & 5 deletions configs/kpf_masters_l1.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,16 @@ input_dir_root = /data/masters/

# for module process:
do_l0_to_2d = False
do_order_trace = True
do_order_trace = False
do_spectral_extraction = True
do_outlier_rejection = False
do_rv = True
do_rv_reweighting = False
do_rv = False
do_rv_reweighting = True
do_hk = True
do_wavecopy_in_sp = True
do_qlp = False
do_bk_subtraction = True
do_bc = True
do_bc = False
outlier_mask_path = None

# the subdirectory containing order trace result, L1 data, L2 data, L2 reweighted data, qlp, bary data
Expand Down Expand Up @@ -145,7 +145,10 @@ channel_datasec_ncols_red = 2040
channel_datasec_nrows_red = 2040
channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
masterbias_path = /data/masters/master_bias_20221022.fits
do_db_query_for_master_files = True
masterbias_path = /data/masters/20230730/kpf_20230730_master_bias.fits
masterdark_path = /data/masters/20230730/kpf_20230730_master_dark.fits
masterflat_path = /data/masters/20230730/kpf_20230730_master_flat.fits
pl_overscan_reg_green = [2040,2140]
srl_overscan_reg_green = [2040,2140]
prescan_reg = [0,4]
Expand Down
14 changes: 7 additions & 7 deletions configs/master_arclamp.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@
[LOGGER]
start_log = True
log_path = logs/pipe_log.log
log_level = info
log_level = debug
log_verbose = True

[ARGUMENT]
data_type = KPF
n_sigma = 2.1
all_fits_files_path = /data/KP*.fits
all_fits_files_path = /data/2D/20231116/KP*.fits
lev0_ffi_exts = ['GREEN_CCD','RED_CCD']
arclamp_object = autocal-une-sky
masterbias_path = /testdata/kpf_master_bias.fits
masterdark_path = /testdata/kpf_master_dark.fits
masterflat_path = /testdata/kpf_master_flat.fits
masterarclamp_path = /testdata/kpf_master_arclamp_une-sky.fits
arclamp_object = autocal-une-cal-eve
masterbias_path = /data/masters/pool/kpf_20231116_master_bias_autocal-bias.fits
masterdark_path = /data/masters/pool/kpf_20231116_master_dark_autocal-dark.fits
masterflat_path = /data/masters/pool/kpf_20231116_master_flat.fits
masterarclamp_path = /testdata/kpf_20231116_master_arclamp_autocal-une-cal-eve.fits

[MODULE_CONFIGS]
master_arclamp = modules/master_arclamp/configs/default.cfg
Expand Down
20 changes: 12 additions & 8 deletions configs/quality_control_example.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,19 @@ log_directory = /data/logs/
[ARGUMENT]
data_type = KPF
# L0, 2D, L1, L2 are possible choices for data_level_str.
#data_level_str = L0
#input_fits_filename = /data/L0/20230927/KP.20230927.82330.61.fits
#output_fits_filename = /data/2D/20230927/KP.20230927.82330.61_2D.fits
data_level_str = L1
input_fits_filename = /data/masters/20230927/kpf_20230927_master_flat_L1.fits
output_fits_filename = /data/masters/20230927/kpf_20230927_master_flat_L1_QC.fits

[MODULE_CONFIGS]
quality_control = modules/quality_control/configs/default.cfg
# Junked observation
#data_level_str = L1
#input_fits_filename = /data/L1/20230608/KP.20230608.68393.35_L1.fits
#output_fits_filename = /testdata/L1/20230608/KP.20230608.68393.35_L1.fits

#data_level_str = L1
#input_fits_filename = /data/L1/20231030/KP.20231030.00336.79_L1.fits
#output_fits_filename = /testdata/L1/20231030/KP.20231030.00336.79_L1.fits

data_level_str = L0
input_fits_filename = /data/L0/20230829/KP.20230829.76026.81.fits
output_fits_filename = /testdata/L0/20230829/KP.20230829.76026.81.fits

[MODULE_CONFIGS]
quality_control = modules/quality_control/configs/default.cfg
2 changes: 1 addition & 1 deletion configs/quicklook_match.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ log_directory = /data/logs_QLP/

[ARGUMENT]
# see quicklook_match.recipe for a description of how to set fullpath
fullpath = '/data/??/202?????/KP.20231105.?????.??*.fits'
fullpath = '/data/L0/202?????/KP.20231204.5????.??*.fits'
#fullpath = '/data/masters/20230429/*.fits'

[MODULE_CONFIGS]
Expand Down
19 changes: 19 additions & 0 deletions configs/write_headers_match_files.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
[LOGGER]
start_log = True
log_path = logs/pipe_log.log
log_level = info
log_verbose = True
log_directory = /data/logs_QLP/

[ARGUMENT]
# see write_headers_match_files.cfg for a description of how to set fullpath
fullpath = '/data/L1/202?????/KP.20231104.39907.04*.fits'
output_dir = /testdata/
do_dark_curr = True # dark current headers
do_gdhdr = True # guider headers
do_emhdr = True # exposure meter headers
do_snr = True # L1 SNR headers
do_fr = True # L1 orderlet flux ratios headers

[MODULE_CONFIGS]
quicklook = modules/quicklook/configs/default.cfg
Loading

0 comments on commit 82cf70d

Please sign in to comment.