Merge pull request #868 from Keck-DataReductionPipelines/develop

Version 2.7.0

README.md

@@ -19,4 +19,3 @@ If you use the KPF Data Reduction Pipeline in your research, please cite the fol
 If there is no place to include the relevant citations in the text of the publication, please include the following acknowledgment (in LaTeX using the [provided BibTeX entry](kpf_bibliography.bib)):
 
 "This research made use of the KPF Data Reduction Pipeline \citep{kpf:gibson2020}."
-

caldates/master_wls.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, /data/reference_fits/430LFCWLS.fits,
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/Era2_RoughWLS_woCAL.fits,

caldates/order_mask.csv

@@ -0,0 +1,4 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+0.5,    2000-01-01 00:00:00, 2024-02-03 00:00:00, /data/reference_fits/kpf_20230730_order_mask_untrimmed_made20230804.fits,
+1.5,    2024-02-03 00:00:01, 2024-02-11 00:00:00, /data/reference_fits/kpf_20240206_order_mask_untrimmed_made20240206.fits,
+2.0,    2024-02-11 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/kpf_20240211_order_mask_untrimmed_made20240212.fits,

caldates/order_trace.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, "['/data/reference_fits/kpf_20230920_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230920_master_flat_RED_CCD.csv']"
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, "['/data/reference_fits/kpf_20240211_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20240211_master_flat_GREEN_CCD.csv']"

caldates/smooth_lamp_pattern.csv

@@ -0,0 +1,6 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+0.5,    2000-01-01 00:00:00, 2023-06-23 00:00:00, /data/reference_fits/kpf_20230619_smooth_lamp_made20230817_float32.fits,
+0.6,    2023-06-23 00:00:01, 2023-07-30 00:00:00, /data/reference_fits/kpf_20230628_smooth_lamp_made20230803_float32.fits, 
+1.0,    2023-07-30 00:00:01, 2024-02-03 00:00:00, /data/reference_fits/kpf_20230804_smooth_lamp_made20230808_float32.fits,
+1.5,    2024-02-03 00:00:01, 2024-02-11 00:00:00, /data/reference_fits/kpf_20240206_smooth_lamp_made20240206.fits,
+2.0,    2024-02-11 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/kpf_20240211_smooth_lamp_made20240212.fits,

caldates/start_order.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, "[-1, 0]"
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, "[0, 1]"

caldates/trace_flat.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, /data/reference_fits/kpf_20230730_master_flat.fits,
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/kpf_20240211_master_flat.fits,

configs/kpf_drp.cfg

@@ -43,8 +43,6 @@ ccd_list = ['GREEN_CCD', 'RED_CCD']
 #flat_file = KP.20221107.04689.77
 #flat_file = /data/reference_fits/kpf_20230730_master_flat.fits
 flat_file = /data/reference_fits/kpf_20240211_master_flat.fits
-order_trace_flat = /data/reference_fits/kpf_20240211_master_flat.fits
-order_trace_files = ['/data/reference_fits/kpf_20240211_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20240211_master_flat_RED_CCD.csv']
 fitting_poly_degree = 3
 ccd_idx = [0, 1]
 # number of pixels to ignore between orderlets during extraction
@@ -62,7 +60,6 @@ orderlet_gap_pixels = 1
 #    - note: order_per_ccd, start_order, orderlet_names should (the outer list), orderlet_widths_ccds (the outer list) have size as that of ccd_list.
 #            the inner lists of orderlet_names and orderlet_widths_ccds are with the same size.
 orders_per_ccd=[35,32]
-start_order = [0, 1]
 orderlet_names = [['GREEN_SKY_FLUX', 'GREEN_SCI_FLUX1', 'GREEN_SCI_FLUX2', 'GREEN_SCI_FLUX3', 'GREEN_CAL_FLUX'], ['RED_SKY_FLUX', 'RED_SCI_FLUX1', 'RED_SCI_FLUX2', 'RED_SCI_FLUX3', 'RED_CAL_FLUX']]
 #orderlet_widths_ccds = [[],[]]
 orderlet_widths_ccds = [[-1, -1, -1, 1, -1],[-1, -1, -1, -1, -1]]
@@ -82,8 +79,7 @@ do_db_query_for_one_nearest_wls_master_file = True
 cal_file_level = 1
 # contentbitmask = 3 means require at least GREEN and RED CCDs in the WLS master file database-queried nearest in time.
 contentbitmask = 3
-cal_type_pairs = [['WLS','autocal-lfc-all-eve'], ['WLS', 'autocal-lfc-all-morn'],
-                  ['WLS', 'autocal-thar-all-eve'], ['WLS', 'autocal-thar-all-morn']]
+cal_type_pairs = [['WLS','autocal-lfc-all-eve'], ['WLS', 'autocal-thar-all-eve']]
 # cal_type_pairs = [['WLS', 'autocal-thar-all-eve']]
 # Maximum start-date age of WLS file relative to context.date_dir at 00:00:00 UT, otherwise fall back on wls_fits.
 max_cal_file_age = '4 days'
@@ -128,7 +124,7 @@ do_order_trace = False
 do_spectral_extraction = True
 do_rv = True
 do_rv_reweighting = True
-do_hk = True
+do_hk = True 
 do_wavecopy_in_sp = True
 do_bk_subtraction = True
 do_bc = True
@@ -146,9 +142,6 @@ channel_datasec_nrows_red = 2040
 channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
 channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
 do_db_query_for_master_files = True
-masterbias_path = /data/masters/20240223/kpf_20240223_master_bias_autocal-bias.fits
-masterdark_path = /data/masters/20240223/kpf_20240223_master_dark_autocal-dark.fits
-masterflat_path = /data/masters/20240223/kpf_20240223_master_flat.fits
 bad_pixel_masks = []
 prescan_reg = [0,4]
 # overscan_method = clippedmean is n_sigma clipping and average over entire post-overscan strip.
@@ -183,4 +176,12 @@ quicklook = modules/quicklook/configs/default.cfg
 bias_subtraction = modules/bias_subtraction/configs/default.cfg
 flat_fielding = modules/flat_fielding/configs/default.cfg
 quality_control = modules/quality_control/configs/default.cfg
-
+calibration_lookup = modules/calibration_lookup/configs/default.cfg
+
+[WLS_INTERPOLATION]
+cal_type = 'WLS'
+object_before = 'autocal-lfc-all-eve'
+object_after = 'autocal-lfc-all-morn'
+#master_file_before_default = /masters/20240128/kpf_20240128_master_WLS_autocal-etalon-all-eve_L1.fits
+#master_file_after_default = /masters/20240129/kpf_20240129_master_WLS_autocal-etalon-all-morn_L1.fits
+do_db_query_for_before_after_master_files = True   

configs/kpf_masters_l1.cfg

@@ -59,8 +59,11 @@ ccd_list = ['GREEN_CCD', 'RED_CCD']
 flat_file = *_master_flat.fits
 # order_trace_files = ['', '']
 # flat_file = /data/masters/20230411/kpf_20230411_master_flat.fits
-order_trace_flat = /data/reference_fits/kpf_20230920_master_flat.fits
-order_trace_files = [ '/data/reference_fits/kpf_20230920_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230920_master_flat_RED_CCD.csv']
+# Make sure the following reference_fits files exist in master pipeline sandbox (~/sbx/reference_fits).
+order_trace_flat = /data/reference_fits/kpf_20240225_master_flat.fits
+order_trace_files = [ '/data/reference_fits/kpf_20240225_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20240225_master_flat_RED_CCD.csv']
+#order_trace_flat = /data/reference_fits/kpf_20230920_master_flat.fits
+#order_trace_files = [ '/data/reference_fits/kpf_20230920_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230920_master_flat_RED_CCD.csv']
 fitting_poly_degree = 3
 ccd_idx = [0, 1]
 # number of pixels to ignore between orderlets during extraction
@@ -76,7 +79,6 @@ orderlet_gap_pixels = 1
 #        ex: start_order = [-1, -1] means the traces of sky fiber of first order for both green and red are missing in the order trace result.
 #    - note: order_per_ccd, start_order, orderlet_names should have size as that of ccd_list.
 orders_per_ccd=[35,32]
-start_order = [0, 1]
 orderlet_names = [['GREEN_SKY_FLUX', 'GREEN_SCI_FLUX1', 'GREEN_SCI_FLUX2', 'GREEN_SCI_FLUX3', 'GREEN_CAL_FLUX'], ['RED_SKY_FLUX', 'RED_SCI_FLUX1', 'RED_SCI_FLUX2', 'RED_SCI_FLUX3', 'RED_CAL_FLUX']]
 orderlet_widths_ccds = [[-1, -1, -1, 1, -1],[-1, -1, -1, -1, -1]]
 
@@ -88,7 +90,8 @@ extraction_method = optimal
 #   - fits with wavelength calibration data
 #	  wls_fits: [ <wavelength solution file for each ccd>].
 #	  wave_to_ext: [ <extensions containing wavelength solution data for each ccd>]
-wls_fits = ['reference_fits/kpf_20230531_master_WLS_autocal-thar-all-eve_L1.fits', 'reference_fits/kpf_20230531_master_WLS_autocal-thar-all-eve_L1.fits']
+wls_fits = ['reference_fits/kpf_20230531_master_WLS_autocal-thar-all-eve_L1.fits',
+            'reference_fits/kpf_20230531_master_WLS_autocal-thar-all-eve_L1.fits']
 wave_to_ext = [['GREEN_SCI_WAVE1', 'GREEN_SCI_WAVE2', 'GREEN_SCI_WAVE3', 'GREEN_SKY_WAVE', 'GREEN_CAL_WAVE'], ['RED_SCI_WAVE1', 'RED_SCI_WAVE2', 'RED_SCI_WAVE3', 'RED_SKY_WAVE', 'RED_CAL_WAVE']]
 # Parameters for querying DB for WLS
 do_db_query_for_one_nearest_wls_master_file = True
@@ -147,9 +150,12 @@ channel_datasec_nrows_red = 2040
 channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
 channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
 do_db_query_for_master_files = True
-masterbias_path = /data/masters/20230730/kpf_20230730_master_bias_autocal-bias.fits
-masterdark_path = /data/masters/20230730/kpf_20230730_master_dark_autocal-dark.fits
-masterflat_path = /data/masters/20230730/kpf_20230730_master_flat.fits
+masterbias_path = /data/masters/20240228/kpf_20240228_master_bias_autocal-bias.fits
+masterdark_path = /data/masters/20240228/kpf_20240228_master_dark_autocal-dark.fits
+masterflat_path = /data/masters/20240228/kpf_20240228_master_flat.fits
+#masterbias_path = /data/masters/20230730/kpf_20230730_master_bias_autocal-bias.fits
+#masterdark_path = /data/masters/20230730/kpf_20230730_master_dark_autocal-dark.fits
+#masterflat_path = /data/masters/20230730/kpf_20230730_master_flat.fits
 pl_overscan_reg_green = [2040,2140]
 srl_overscan_reg_green = [2040,2140]
 prescan_reg = [0,4]

configs/quicklook_match.cfg

@@ -6,6 +6,9 @@ log_verbose = True
 log_directory = /data/logs_QLP/
 
 [ARGUMENT]
+# set outdir to `/testdata/QLP/` to run in a private environment
+#            or '/data/QLP/' for production mode
+outdir= '/data/QLP/'
 # see quicklook_match.recipe for a description of how to set fullpath
 fullpath = '/data/L1/202?????/KP.20240116.?????.??*.fits'
 #fullpath = '/data/masters/20230429/*.fits'

configs/test_wls_interpolation.cfg

@@ -0,0 +1,29 @@
+# This is an example config showing use of the WLS interpolation framework
+
+## Pipeline logger configurations
+[LOGGER]
+start_log = True
+log_path = pipeline.log
+log_level = info
+log_verbose = True
+log_directory = /data/logs/
+
+[ARGUMENT]
+data_type = KPF
+overwrite = True
+
+# path to input and output data
+output_dir = /testdata/
+input_dir = /data/
+
+# Files
+wls1_file = 'masters/20240101/kpf_20240101_master_WLS_autocal-lfc-all-eve_L1.fits'
+wls2_file = 'masters/20240101/kpf_20240101_master_WLS_autocal-lfc-all-morn_L1.fits'
+obsid_l1 = 'KP.20240101.24368.88' # stellar observation (10476)
+
+# WAVE extensions
+wave_ext = ['GREEN_SCI_WAVE1', 'GREEN_SCI_WAVE2', 'GREEN_SCI_WAVE3', 'GREEN_SKY_WAVE', 'GREEN_CAL_WAVE', 'RED_SCI_WAVE1', 'RED_SCI_WAVE2', 'RED_SCI_WAVE3', 'RED_SKY_WAVE', 'RED_CAL_WAVE']
+
+[MODULE_CONFIGS]
+wls = modules/wavelength_cal/configs/default.cfg
+calibration_lookup = modules/calibration_lookup/configs/default.cfg

configs/wls_auto.cfg

@@ -18,7 +18,7 @@ thar_search_string = -thar-all*_L1
 # thar_search_string = nothing
 lfc_search_string = lfc-all*_L1
 etalon_search_string = etalon-all*L1
-save_wl_pixel_toggle = True
+save_wl_pixel_toggle = 1
 
 quicklook = 0
 # f0_key = 5220000000.

cronjobs/cleanOldFilesFromDisk.pl

@@ -9,16 +9,17 @@
 my @op0 = `$cmd0`;
 if (@op0) { print "Output from [$cmd0]=[@op0]\n"; }
 
+my $olderthanndays = 3;   # Remove files older than 3 days.
 
 my $dir1 = "/data/user/rlaher/sbx/L0";
-&removeOldSubDirs(3, $dir1);
+&removeOldSubDirs($olderthanndays, $dir1);
 
 
 my $dir2 = "/data/user/rlaher/sbx/2D";
-&removeOldSubDirs(3, $dir2);
+&removeOldSubDirs($olderthanndays, $dir2);
 
 my $dir3 = "/data/user/rlaher/sbx/masters/wlpixelfiles";
-&removeOldFiles(5, $dir3);
+&removeOldFiles($olderthanndays, $dir3);
 
 my $cmd1 = "df -h /data/user/rlaher/sbx";
 

cronjobs/kpfmastersruncmd_l0.pl

@@ -107,7 +107,7 @@
 # Initialize fixed parameters and read command-line parameter.
 
 my $iam = 'kpfmastersruncmd_l0.pl';
-my $version = '2.1';
+my $version = '2.2';
 
 my $procdate = shift @ARGV;                  # YYYYMMDD command-line parameter.
 
@@ -132,6 +132,11 @@
     $config = $configenvar;
 }
 
+my $pythonscript = 'scripts/make_smooth_lamp_pattern_new.py';
+
+my ($pylogfileDir, $pylogfileBase) = $pythonscript =~ /(.+)\/(.+)\.py/;
+my $pylogfile = $pylogfileBase . '_' . $procdate . '.out';
+
 
 # Get database parameters from ~/.pgpass file.
 
@@ -192,11 +197,14 @@
              "rm -rf /data/masters/${procdate}\n" .
              "find /data/masters/pool/kpf_????????_master_*fits -mtime +7 -exec rm {} +\n" .
              "kpf -r $recipe  -c $config --date ${procdate}\n" .
+             "python $pythonscript /data/masters/pool/kpf_${procdate}_master_flat.fits /data/masters/pool/kpf_${procdate}_smooth_lamp.fits >& ${pylogfile}\n" .
              "mkdir -p /masters/${procdate}\n" .
              "sleep 3\n" .
              "cp -p /data/masters/pool/kpf_${procdate}* /masters/${procdate}\n" .
              "chown root:root /masters/${procdate}/*\n" .
              "cp -p /data/logs/${procdate}/pipeline_${procdate}.log /masters/${procdate}/pipeline_masters_drp_l0_${procdate}.log\n" .
+             "cp -p /code/KPF-Pipeline/${pylogfile} /masters/${procdate}\n" .
+             "rm /code/KPF-Pipeline/${pylogfile}\n" .
              "exit\n";
 my $makescriptcmd = "echo \"$script\" > $dockercmdscript";
 `$makescriptcmd`;

cronjobs/kpfmastersruncmd_l1.pl

@@ -104,7 +104,7 @@
 # Initialize fixed parameters and read command-line parameter.
 
 my $iam = 'kpfmastersruncmd_l1.pl';
-my $version = '1.9';
+my $version = '2.0';
 
 my $procdate = shift @ARGV;                  # YYYYMMDD command-line parameter.
 
@@ -175,6 +175,7 @@
              "git config --global --add safe.directory /code/KPF-Pipeline\n" .
              "mkdir -p /data/masters/${procdate}\n" .
              "cp -pr /masters/${procdate}/kpf_${procdate}*.fits /data/masters/${procdate}\n" .
+             "rm /data/masters/${procdate}/kpf_${procdate}_smooth_lamp.fits\n" .
              "kpf -r $recipe  -c $config --date ${procdate}\n" .
              "cp -p /data/masters/${procdate}/* /masters/${procdate}\n" .
              "cp -p /data/logs/${procdate}/pipeline_${procdate}.log /masters/${procdate}/pipeline_masters_drp_l1_${procdate}.log\n" .

database/configs/query_db_before_after_master_files.cfg

@@ -0,0 +1,25 @@
+# Pipeline logger configurations
+[LOGGER]
+start_log = True
+log_path = pipeline.log
+log_level = info
+log_verbose = True
+log_directory = /data/logs/
+
+[ARGUMENT]
+data_type = KPF
+obs_date_time = '2024-01-28 04:15:17'
+cal_file_level = 1
+contentbitmask = 3
+cal_type = 'WLS'
+object_before = 'autocal-etalon-all-eve'
+object_after = 'autocal-etalon-all-morn'
+max_cal_file_age = '5 days'
+master_file_before_default = /masters/20240128/kpf_20240128_master_WLS_autocal-etalon-all-eve_L1.fits
+master_file_after_default = /masters/20240129/kpf_20240129_master_WLS_autocal-etalon-all-morn_L1.fits
+
+[MODULE_CONFIGS]
+query_db_before_after_master_files = database/modules/query_db_before_after_master_files/configs/default.cfg
+
+
+

database/modules/query_db_before_after_master_files/configs/default.cfg

@@ -0,0 +1,14 @@
+## Default configuration for QueryDBOneNearestMasterFileFramework primitive
+[LOGGER]
+start_log = True
+log_path = logs/query_db_one_nearest_master_file_framework_debug.log
+log_level = debug
+log_verbose = True
+
+
+## Module related parameters
+[PARAM]
+cal_file_level = 0
+contentbitmask = 7
+cal_type = 'Bias'
+max_cal_file_age = '1000 days'