Merge pull request #706 from Keck-DataReductionPipelines/develop

Version 2.5.1

.gitignore

@@ -5,11 +5,14 @@ examples/V1/recipe_results/
 *~
 *.DS_Store
 .idea/
-#logs/
+logs/
 receipts/
 __pycache__/
 *.py[cod]
 *$py.class
+*.ipynb_checkpoints
+*.log
+*.txt
 
 # C extensions
 *.so

configs/framework_multi.cfg

@@ -23,6 +23,8 @@ monitor_interval = 1 # sec
 # In second.
 #
 event_timeout = 3600
+# event_timeout = 5
+
 
 #
 # Default event to trigger on new files
@@ -76,7 +78,8 @@ temp_directory = "temp"
 #
 # no_event_event = None
 #
-no_event_event = Event ('no_event', None)
+no_event_event = Event('no_event', None)
+# no_event_event = Event('exit', None)
 
 #
 # How long to wait before sending no_event_event
@@ -103,5 +106,5 @@ http_defaultFile = ""
 #
 want_multiprocessing = True
 queue_manager_hostname = 'localhost'
-queue_manager_portnr = 50131
+queue_manager_portnr = 50111
 queue_manager_auth_code = b"a very long authentication code" 

configs/kpf_drp.cfg

@@ -16,13 +16,12 @@ output_dir_flat = /data/
 input_dir_root = /data/2D/
 masters_dir = /data/masters/
 
-# the subdirectory containing order trace result, L1 data, L2 data, L2 reweighted data, qlp, bary data
+# the subdirectory containing order trace result, L1 data, L2 data, L2 reweighted data, bary data
 # output_trace = order_trace/
 output_trace = masters/
 output_extraction = L1/
 output_rv = L2/
 output_rv_reweighting=''
-output_qlp = QLP/
 output_barycorr = bary/
 
 # output_clip containing files with polygon clipping information when rectification_method is 'normal' or vertical'
@@ -42,11 +41,14 @@ ccd_list = ['GREEN_CCD', 'RED_CCD']
 #    flat_file: flat file for order trace process. updated upon new flat file.
 #    ccd_idx: index in ccd_list for the ccd to be processed in DRP recipe
 #flat_file = KP.20221107.04689.77
-flat_file = /data/reference_fits/kpf_20230730_master_flat.fits
-order_trace_flat = /data/reference_fits/kpf_20230730_master_flat.fits
-order_trace_files = [ '/data/reference_fits/kpf_20230730_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230730_master_flat_RED_CCD.csv']
+#flat_file = /data/reference_fits/kpf_20230730_master_flat.fits
+flat_file = /data/reference_fits/kpf_20230920_master_flat.fits
+order_trace_flat = /data/reference_fits/kpf_20230920_master_flat.fits
+order_trace_files = [ '/data/reference_fits/kpf_20230920_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230920_master_flat_RED_CCD.csv']
 fitting_poly_degree = 3
 ccd_idx = [0, 1]
+# number of pixels to ignore between orderlets during extraction
+orderlet_gap_pixels = 1
 
 # for spectral extraction:
 #    - update orders_per_ccd, start_order, orderlet_names per new order trace result
@@ -60,7 +62,7 @@ ccd_idx = [0, 1]
 #    - note: order_per_ccd, start_order, orderlet_names should (the outer list), orderlet_widths_ccds (the outer list) have size as that of ccd_list.
 #            the inner lists of orderlet_names and orderlet_widths_ccds are with the same size.
 orders_per_ccd=[35,32]
-start_order = [-1, -1]
+start_order = [-1, 0]
 orderlet_names = [['GREEN_SKY_FLUX', 'GREEN_SCI_FLUX1', 'GREEN_SCI_FLUX2', 'GREEN_SCI_FLUX3', 'GREEN_CAL_FLUX'], ['RED_SKY_FLUX', 'RED_SCI_FLUX1', 'RED_SCI_FLUX2', 'RED_SCI_FLUX3', 'RED_CAL_FLUX']]
 #orderlet_widths_ccds = [[],[]]
 orderlet_widths_ccds = [[-1, -1, -1, 1, -1],[-1, -1, -1, -1, -1]]
@@ -98,6 +100,10 @@ reweighting_enable_masks = [['espresso'], ['espresso']]
 ccf_ext = ['GREEN_CCF', 'RED_CCF']
 rv_ext = RV
 static_ccf_ratio = ['/code/KPF-Pipeline/static/static_green_ccf_ratio_2.csv', '/code/KPF-Pipeline/static/static_red_ccf_ratio_2.csv']
+# starting and ending location for CCF calculation, >= 0, position relative to left end of the image,
+#                                                   < 0,  position relative to the right end of the image
+rv_start_x = 500
+rv_end_x = -500
 
 # for ca_hk:
 #    hk_fiber_list: [<CA-HK spectrometer fibers>]
@@ -112,8 +118,8 @@ hk_extract_exts = ['CA_HK_SCI', 'CA_HK_SKY']
 hk_wave_exts = ['CA_HK_SCI_WAVE', 'CA_HK_SKY_WAVE']
 hk_dark_fits = masters/KP.20221029.21537.28.fits
 hk_bias_fits = None
-hk_trace_path = masters/kpfMaster_HKOrderBounds20220909.csv
-hk_wavelength_path = ["masters/kpfMaster_HKwave20220909_sci.csv", "masters/kpfMaster_HKwave20220909_sky.csv"]
+hk_trace_path = masters/kpfMaster_HKOrderBounds20230818.csv
+hk_wavelength_path = ["masters/kpfMaster_HKwave20230818_sci.csv", "masters/kpfMaster_HKwave20230818_sky.csv"]
 
 # for module process:
 do_l0_to_2d = True
@@ -123,10 +129,10 @@ do_rv = True
 do_rv_reweighting = True
 do_hk = True
 do_wavecopy_in_sp = True
-#do_qlp = True
 do_bk_subtraction = True
 do_bc = True
 do_outlier_rejection = True
+outlier_mask_path = None
 
 # for L0->2D process
 [WATCHFOR_L0]
@@ -153,6 +159,7 @@ overscan_clip = 5
 overscan_order = 1
 lev0_ffi_ext_red = ['RED_CCD']
 lev0_ffi_ext_green = ['GREEN_CCD']
+lev0_ffi_ext_cahk = ['CA_HK']
 quicklook = False
 gain_keyword = 'CCDGAIN'
 input_dir = /data/L0/

configs/kpf_drp_local.cfg

@@ -122,6 +122,8 @@ do_wavecopy_in_sp = True
 do_qlp = False
 do_bk_subtraction = True
 do_bc = True
+do_outlier_rejection = False
+outlier_mask_path = None
 
 # for L0->2D process
 [WATCHFOR_L0]

configs/kpf_masters_drp.cfg

@@ -39,15 +39,20 @@ output_dir = /data/2D
 n_sigma = 2.1
 lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
 # Leave empty list for bias_objects for recipe to automatically fill in.
-bias_objects = []
+#bias_objects = []
+# 20230619 had OBJECT=test with just one frame, so we handle it specially as follows:
+bias_objects = ['autocal-bias']
+# The above with explicit bias_objects = ['autocal-bias'] is the best policy for automated daily runs.
 masterbias_path = /testdata/kpf_master_bias.fits
 
 [DARK]
 exptime_minimum = 300.0
 n_sigma = 2.2
 lev0_ffi_exts = ['GREEN_CCD','RED_CCD','CA_HK']
 # Leave empty list for dark_objects for recipe to automatically fill in.
-dark_objects = []
+#dark_objects = []
+dark_objects = ['autocal-dark']
+# The above with explicit dark_objects = ['autocal-dark'] is the best policy for automated daily runs.
 masterdark_path = /testdata/kpf_master_dark.fits
 
 [FLAT]

configs/kpf_masters_l1.cfg

@@ -22,14 +22,15 @@ input_dir_root = /data/masters/
 do_l0_to_2d = False
 do_order_trace = True
 do_spectral_extraction = True
-do_outlier_rejection = True
+do_outlier_rejection = False
 do_rv = True
 do_rv_reweighting = False
 do_hk = True
 do_wavecopy_in_sp = True
 do_qlp = False
 do_bk_subtraction = True
 do_bc = True
+outlier_mask_path = None
 
 # the subdirectory containing order trace result, L1 data, L2 data, L2 reweighted data, qlp, bary data
 output_trace = /
@@ -58,10 +59,12 @@ ccd_list = ['GREEN_CCD', 'RED_CCD']
 flat_file = *_master_flat.fits
 # order_trace_files = ['', '']
 # flat_file = /data/masters/20230411/kpf_20230411_master_flat.fits
-order_trace_flat = /data/reference_fits/kpf_20230730_master_flat.fits
-order_trace_files = [ '/data/reference_fits/kpf_20230730_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230730_master_flat_RED_CCD.csv']
+order_trace_flat = /data/reference_fits/kpf_20230920_master_flat.fits
+order_trace_files = [ '/data/reference_fits/kpf_20230920_master_flat_GREEN_CCD.csv', '/data/reference_fits/kpf_20230920_master_flat_RED_CCD.csv']
 fitting_poly_degree = 3
 ccd_idx = [0, 1]
+# number of pixels to ignore between orderlets during extraction
+orderlet_gap_pixels = 1
 
 # for spectral extraction:
 #    - update orders_per_ccd, start_order, orderlet_names per new order trace result
@@ -73,7 +76,7 @@ ccd_idx = [0, 1]
 #        ex: start_order = [-1, -1] means the traces of sky fiber of first order for both green and red are missing in the order trace result.
 #    - note: order_per_ccd, start_order, orderlet_names should have size as that of ccd_list.
 orders_per_ccd=[35,32]
-start_order = [-1, -1]
+start_order = [-1, 0]
 orderlet_names = [['GREEN_SKY_FLUX', 'GREEN_SCI_FLUX1', 'GREEN_SCI_FLUX2', 'GREEN_SCI_FLUX3', 'GREEN_CAL_FLUX'], ['RED_SKY_FLUX', 'RED_SCI_FLUX1', 'RED_SCI_FLUX2', 'RED_SCI_FLUX3', 'RED_CAL_FLUX']]
 orderlet_widths_ccds = [[-1, -1, -1, 1, -1],[-1, -1, -1, -1, -1]]
 
@@ -109,6 +112,10 @@ reweighting_enable_masks = [['espresso'], ['espresso']]
 ccf_ext = ['GREEN_CCF', 'RED_CCF']
 rv_ext = RV
 static_ccf_ratio = ['/code/KPF-Pipeline/static/static_green_ccf_ratio_2.csv', '/code/KPF-Pipeline/static/static_red_ccf_ratio_2.csv']
+# starting and ending location for CCF calculation, >= 0, position relative to left end of the image,
+#                                                   < 0,  position relative to the right end of the image
+rv_start_x = 500
+rv_end_x = -500
 
 # for ca_hk:
 #    hk_fiber_list: [<CA-HK spectrometer fibers>]
@@ -123,8 +130,10 @@ hk_extract_exts = ['CA_HK_SCI', 'CA_HK_SKY']
 hk_wave_exts = ['CA_HK_SCI_WAVE', 'CA_HK_SKY_WAVE']
 hk_dark_fits = masters/KP.20221029.21537.28.fits
 hk_bias_fits = None
-hk_trace_path = masters/kpfMaster_HKOrderBounds20220909.csv
-hk_wavelength_path = ["masters/kpfMaster_HKwave20220909_sci.csv", "masters/kpfMaster_HKwave20220909_sky.csv"]
+#hk_trace_path = masters/kpfMaster_HKOrderBounds20220909.csv
+hk_trace_path = masters/kpfMaster_HKOrderBounds20230818.csv
+#hk_wavelength_path = ["masters/kpfMaster_HKwave20220909_sci.csv", "masters/kpfMaster_HKwave20220909_sky.csv"]
+hk_wavelength_path = ["masters/kpfMaster_HKwave20230818_sci.csv", "masters/kpfMaster_HKwave20230818_sky.csv"]
 
 # for L0->2D process
 [WATCHFOR_L0]

configs/quality_control_example.cfg

@@ -0,0 +1,23 @@
+# Pipeline logger configurations
+[LOGGER]
+start_log = True
+log_path = pipeline.log
+log_level = info
+log_verbose = True
+log_directory = /data/logs/
+
+[ARGUMENT]
+data_type = KPF
+# L0, 2D, L1, L2 are possible choices for data_level_str.
+#data_level_str = L0
+#input_fits_filename = /data/L0/20230927/KP.20230927.82330.61.fits
+#output_fits_filename = /data/2D/20230927/KP.20230927.82330.61_2D.fits
+data_level_str = L1
+input_fits_filename = /data/masters/20230927/kpf_20230927_master_flat_L1.fits
+output_fits_filename = /data/masters/20230927/kpf_20230927_master_flat_L1_QC.fits
+
+[MODULE_CONFIGS]
+quality_control = modules/quality_control/configs/default.cfg
+
+
+

configs/quicklook_match.cfg

@@ -7,8 +7,8 @@ log_directory = /data/logs_QLP/
 
 [ARGUMENT]
 # see quicklook_match.recipe for a description of how to set fullpath
-fullpath = '/data/??/202?????/KP.20230813.53844.48*.fits'
-#fullpath = '/data/??/202?????/KP.20230813.51416.47*.fits'
+fullpath = '/data/L0/202?????/KP.20230102.10546.01*.fits'
+#fullpath = '/data/masters/20230419/*.fits'
 
 [MODULE_CONFIGS]
 quicklook = modules/quicklook/configs/default.cfg

cronjobs/kpfmasters_wls_auto.pl

@@ -19,7 +19,6 @@
 use strict;
 use warnings;
 use File::Copy;
-use File::Path qw/make_path/;
 
 select STDERR; $| = 1; select STDOUT; $| = 1;
 
@@ -93,7 +92,7 @@
 # Initialize fixed parameters and read command-line parameter.
 
 my $iam = 'kpfmasters_wls_auto.pl';
-my $version = '1.5';
+my $version = '1.6';
 
 my $procdate = shift @ARGV;                  # YYYYMMDD command-line parameter.
 
@@ -134,17 +133,21 @@
              "make init\n" .
              "export PYTHONUNBUFFERED=1\n" .
              "git config --global --add safe.directory /code/KPF-Pipeline\n" .
+             "mkdir -p /data/masters/${procdate}\n" .
+             "cp -pr /masters/${procdate}/kpf_${procdate}*L1.fits /data/masters/${procdate}\n" .
              "kpf -r $recipe  -c $config --date ${procdate}\n" .
+             "cp -p /data/masters/${procdate}/*master_WLS* /masters/${procdate}\n" .
+             "mkdir -p /masters/${procdate}/wlpixelfiles\n" .
+             "cp -p /data/masters/wlpixelfiles/*kpf_${procdate}* /masters/${procdate}/wlpixelfiles\n" .
+             "cp -p /code/KPF-Pipeline/pipeline_${procdate}.log /masters/${procdate}/pipeline_wls_auto_${procdate}.log\n" .
+             "rm /code/KPF-Pipeline/pipeline_${procdate}.log\n" .
              "exit\n";
 my $makescriptcmd = "echo \"$script\" > $dockercmdscript";
 `$makescriptcmd`;
 `chmod +x $dockercmdscript`;
 
-`mkdir -p $sbxdir`;
-`cp -pr ${mastersdir}/${procdate}/kpf_${procdate}*L1.fits $sbxdir`;
-
 my $dockerruncmd = "docker run -d --name $containername " .
-                   "-v ${codedir}:/code/KPF-Pipeline -v ${testdatadir}:/testdata -v $sandbox:/data " .
+                   "-v ${codedir}:/code/KPF-Pipeline -v ${testdatadir}:/testdata -v $sandbox:/data -v ${mastersdir}:/masters " .
                    "$containerimage bash ./$dockercmdscript";
 print "Executing $dockerruncmd\n";
 my $opdockerruncmd = `$dockerruncmd`;
@@ -182,63 +185,6 @@
 $icheckpoint++;
 
 
-# Directory to store products should already exist because
-# cronjob kpfmastersruncmd_l0.pl ran before.
-
-my $destdir  = "${mastersdir}/$procdate";
-
-if (! (-e $destdir)) {
-    print "*** Error: Product directory does not exist ($destdir): $!\n";
-    exit(64);
-}
-
-sleep(30);
-
-my $globfiles = "$sbxdir/*master_WLS*";
-
-my @files  = glob("$globfiles");
-
-foreach my $file (@files) {
-    my $destfile = "$destdir/$file";
-    if (! (-e $destfile)) {
-        if (! (copy($file, $destdir))) {
-            print "*** Warning: couldn't copy $file to $destdir ($!); " .
-                "skipping...\n";
-        } else {
-            print "Copied $file to $destdir\n";
-        }
-    }
-}
-
-
-# Make directory to store wlpixelfiles.
-
-my $destdir2  = "${mastersdir}/$procdate/wlpixelfiles";
-
-if (! (-e $destdir2)) {
-    if (! make_path($destdir2)) {
-        die "*** Error: Could not make directory ($destdir2): $!\n";
-    } else {
-        print "Made new directory $destdir2\n";
-    }
-}
-
-sleep(30);
-
-my $globfiles2 = "${sandbox}/masters/wlpixelfiles/*kpf_${procdate}*";
-
-my @files2  = glob("$globfiles2");
-
-foreach my $file (@files2) {
-    if (! (copy($file, $destdir2))) {
-        print "*** Warning: couldn't copy $file to $destdir2 ($!); " .
-            "skipping...\n";
-    } else {
-        print "Copied $file to $destdir2\n";
-    }
-}
-
-
 # Log end time.
 
 $endscript = time();
@@ -249,21 +195,4 @@
 print "Terminating normally...\n";
 
 
-# Copy log file from runtime directory to product directory.
-
-my ($logfileBase) = $iam =~ /(.+)\.pl/;
-
-my $logfile = $logdir . '/' . $logfileBase . '_' . $procdate . '.out';
-
-if (-e $logfile) {
-
-    if (! (copy($logfile, $destdir))) {
-        die "*** Warning: couldn't copy $logfile to $destdir ($!); " .
-            "quitting...\n";
-    } else {
-        print "Copied $logfile to $destdir\n";
-    }
-}
-
-
 exit(0);