Merge pull request #896 from Keck-DataReductionPipelines/feature/wls_…

…interp_plumbing

Feature/wls interp plumbing

caldates/master_wls.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, /data/reference_fits/430LFCWLS.fits,
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/Era2_RoughWLS_woCAL.fits,

caldates/order_mask.csv

@@ -0,0 +1,4 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+0.5,    2000-01-01 00:00:00, 2024-02-03 00:00:00, /data/reference_fits/kpf_20230730_order_mask_untrimmed_made20230804.fits,
+1.5,    2024-02-03 00:00:01, 2024-02-11 00:00:00, /data/reference_fits/kpf_20240206_order_mask_untrimmed_made20240206.fits,
+2.0,    2024-02-11 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/kpf_20240211_order_mask_untrimmed_made20240212.fits,

caldates/order_trace.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, "['/data/reference_fits/kpf_20230920_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230920_master_flat_RED_CCD.csv']"
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, "['/data/reference_fits/kpf_20240211_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20240211_master_flat_GREEN_CCD.csv']"

caldates/smooth_lamp_pattern.csv

@@ -0,0 +1,6 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+0.5,    2000-01-01 00:00:00, 2023-06-23 00:00:00, /data/reference_fits/kpf_20230619_smooth_lamp_made20230817_float32.fits,
+0.6,    2023-06-23 00:00:01, 2023-07-30 00:00:00, /data/reference_fits/kpf_20230628_smooth_lamp_made20230803_float32.fits, 
+1.0,    2023-07-30 00:00:01, 2024-02-03 00:00:00, /data/reference_fits/kpf_20230804_smooth_lamp_made20230808_float32.fits,
+1.5,    2024-02-03 00:00:01, 2024-02-11 00:00:00, /data/reference_fits/kpf_20240206_smooth_lamp_made20240206.fits,
+2.0,    2024-02-11 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/kpf_20240211_smooth_lamp_made20240212.fits,

caldates/start_order.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, "[-1, 0]"
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, "[0, 1]"

caldates/trace_flat.csv

@@ -0,0 +1,3 @@
+CALTAG, UT_start_date,       UT_end_date,         CALPATH,  comment
+1.0,    2000-01-01 00:00:00, 2024-02-03 00:00:00, /data/reference_fits/kpf_20230730_master_flat.fits,
+2.0,    2024-02-03 00:00:01, 3000-01-01 00:00:00, /data/reference_fits/kpf_20240211_master_flat.fits,

configs/kpf_drp.cfg

@@ -124,7 +124,7 @@ do_order_trace = False
 do_spectral_extraction = True
 do_rv = True
 do_rv_reweighting = True
-do_hk = True
+do_hk = True 
 do_wavecopy_in_sp = True
 do_bk_subtraction = True
 do_bc = True
@@ -142,9 +142,6 @@ channel_datasec_nrows_red = 2040
 channel_orientation_ref_path_red = /code/KPF-Pipeline/static/kpfsim_ccd_orient_red.txt
 channel_orientation_ref_path_green = /code/KPF-Pipeline/static/kpfsim_ccd_orient_green.txt
 do_db_query_for_master_files = True
-masterbias_path = /data/masters/20240223/kpf_20240223_master_bias_autocal-bias.fits
-masterdark_path = /data/masters/20240223/kpf_20240223_master_dark_autocal-dark.fits
-masterflat_path = /data/masters/20240223/kpf_20240223_master_flat.fits
 bad_pixel_masks = []
 prescan_reg = [0,4]
 # overscan_method = clippedmean is n_sigma clipping and average over entire post-overscan strip.
@@ -179,4 +176,12 @@ quicklook = modules/quicklook/configs/default.cfg
 bias_subtraction = modules/bias_subtraction/configs/default.cfg
 flat_fielding = modules/flat_fielding/configs/default.cfg
 quality_control = modules/quality_control/configs/default.cfg
-
+calibration_lookup = modules/calibration_lookup/configs/default.cfg
+
+[WLS_INTERPOLATION]
+cal_type = 'WLS'
+object_before = 'autocal-lfc-all-eve'
+object_after = 'autocal-lfc-all-morn'
+#master_file_before_default = /masters/20240128/kpf_20240128_master_WLS_autocal-etalon-all-eve_L1.fits
+#master_file_after_default = /masters/20240129/kpf_20240129_master_WLS_autocal-etalon-all-morn_L1.fits
+do_db_query_for_before_after_master_files = True   

configs/test_wls_interpolation.cfg

@@ -26,3 +26,4 @@ wave_ext = ['GREEN_SCI_WAVE1', 'GREEN_SCI_WAVE2', 'GREEN_SCI_WAVE3', 'GREEN_SKY_
 
 [MODULE_CONFIGS]
 wls = modules/wavelength_cal/configs/default.cfg
+calibration_lookup = modules/calibration_lookup/configs/default.cfg

database/modules/query_db_before_after_master_files/src/query_db_before_after_master_files_framework.py

@@ -10,22 +10,11 @@
 from kpfpipe.models.level0 import KPF0
 from kpfpipe.primitives.level0 import KPF0_Primitive
 from keckdrpframework.models.arguments import Arguments
+from database.modules.query_db_nearest_master_files.src.query_db_nearest_master_files_framework import md5
 
 # Global read-only variables
 DEFAULT_CFG_PATH = 'database/modules/query_db_before_after_master_files/configs/default.cfg'
 
-def md5(fname):
-    hash_md5 = hashlib.md5()
-
-    try:
-        with open(fname, "rb") as f:
-            for chunk in iter(lambda: f.read(4096), b""):
-                hash_md5.update(chunk)
-        return hash_md5.hexdigest()
-    except:
-        print("*** Error: Cannot open file =",fname,"; quitting...")
-        exit(65)
-
 class QueryDBBeforeAfterMasterFilesFramework(KPF0_Primitive):
 
     """

modules/Utils/data_handler.py

@@ -260,7 +260,7 @@ def __init__(self,
         KPF_Primitive.__init__(self, action, context)
 
         self.kpfobj = action.args[0]
-        self.key_list = [str] if isinstance(action.args[1], str) else action.args[1]
+        self.key_list = [action.args[1]] if isinstance(action.args[1], str) else action.args[1]
 
         self.logger = None
         if not self.logger:

modules/calibration_lookup/configs/default.cfg

@@ -0,0 +1,37 @@
+[LOGGER]
+log = True
+log_path = logs/calibrations.log
+log_level = debug
+log_verbose = True
+
+[PARAM]
+date_files = {'rough_wls': 'caldates/master_wls.csv',
+              'order_trace': 'caldates/order_trace.csv',
+              'start_order': 'caldates/start_order.csv',
+              'order_mask': 'caldates/order_mask.csv',
+              'smooth_lamp_pattern': 'caldates/smooth_lamp_pattern.csv',
+              'trace_flat': 'caldates/trace_flat.csv'}
+
+# List of possible calibration types and how to look them up
+# possible values are 'database' | 'file' | 'wls'
+lookup_map = {'bias': 'database',
+              'dark': 'database',
+              'flat': 'database',
+              'wls': 'wls',  # wls has a special lookup to get before/after solutions
+              'rough_wls': 'file',
+              'order_trace': 'file',
+              'start_order': 'file',
+              'order_mask': 'file',
+              'smooth_lamp_pattern': 'file',
+              'trace_flat': 'file'}
+
+defaults = {'bias': '/data/reference_fits/kpf_20240223_master_bias_autocal-bias.fits',
+            'dark': '/data/reference_fits/kpf_20240223_master_dark_autocal-dark.fits',
+            'flat': '/data/reference_fits/kpf_20240223_master_flat.fits',
+            'wls': ['/data/reference_fits/Era2_RoughWLS_woCAL.fits', '/data/reference_fits/Era2_RoughWLS_woCAL.fits']}
+
+db_cal_file_levels = [0, 0, 0]
+db_cal_types = [['Bias','autocal-bias'],['Dark', 'autocal-dark'],['Flat','autocal-flat-all']]
+
+wls_cal_types = [['WLS','autocal-lfc-all'], ['WLS', 'autocal-thar-all']]
+max_cal_age = '3 days'