Wavelength correction improvements

.gitignore

@@ -5,4 +5,5 @@
 *__pycache__*
 /build
 *.egg
-/dist
+/dist
+/graphs

kcwidrp/configs/framework.cfg

@@ -101,7 +101,7 @@ http_defaultFile = ""
 # The queue_manager_hostname is used for clients to connect to the server so share the event queue.
 # The authentication code is arbitrary.
 #
-want_multiprocessing = True
+want_multiprocessing = False
 queue_manager_hostname = "localhost"
 queue_manager_portnr = 50101
 queue_manager_auth_code = b"kcwi"

kcwidrp/configs/rti.cfg

@@ -8,7 +8,6 @@ rti_retry_time = 5 # How many seconds to wait before trying connection again
 rti_user = ''
 rti_pass = ''
 # RTI API parameters
-rti_ingesttype = 'lev1'
 rti_reingest = False
 rti_testonly = False
 rti_dev = True

kcwidrp/core/kcwi_proctab.py

@@ -135,7 +135,6 @@ def update_proctab(self, frame, suffix='raw', newtype=None, filename=""):
                        filename]
         else:
             new_row = None
-        # print("Attempting to add %s" % str(new_row))
         self.proctab.add_row(new_row)
         self.proctab = unique(self.proctab, keys=['CID', 'FRAMENO', 'TYPE'],
                               keep='last')

kcwidrp/primitives/CorrectGain.py

@@ -31,7 +31,6 @@ def _perform(self):
         # Header keyword to update
         key = 'GAINCOR'
         keycom = 'Gain corrected?'
-        # print(self.action.args.ccddata.header)
         number_of_amplifiers = self.action.args.namps
         bsec, dsec, tsec, direc, amps, aoff = self.action.args.map_ccd
         namps = len(amps)

kcwidrp/primitives/ExtractArcs.py

@@ -87,7 +87,8 @@ def _perform(self):
         self.action.args.contbar_image_number = self.context.trace['CBARSNO']
         self.action.args.contbar_image = self.context.trace['CBARSFL']
         self.action.args.arc_number = self.action.args.ccddata.header['FRAMENO']
-        self.action.args.arc_image = self.action.args.ccddata.header['OFNAME']
+        self.action.args.arc_image = self.action.args.name
+        # self.action.args.arc_image = self.action.args.ccddata.header['OFNAME']
 
         self.action.args.source_control_points = trace['src']
         self.action.args.destination_control_points = trace['dst']

kcwidrp/primitives/FindBars.py

@@ -201,7 +201,8 @@ def _perform(self):
         self.action.args.contbar_image_number = \
             self.action.args.ccddata.header['FRAMENO']
         self.action.args.contbar_image = \
-            self.action.args.ccddata.header['OFNAME']
+            self.action.args.name
+            # self.action.args.ccddata.header['OFNAME']
 
         log_string = FindBars.__module__
         self.action.args.ccddata.header['HISTORY'] = log_string

kcwidrp/primitives/FluxCalibrate.py

@@ -95,7 +95,7 @@ def _perform(self):
                     sz[0] != mcsz[0]:
                 self.logger.warning("wavelength scales not identical, "
                                     "resampling standard")
-                print(w0, mcw0, dw, mcdw, wav[-1], mcwav[-1], sz[0], mcsz[0])
+                self.logger.info(w0, mcw0, dw, mcdw, wav[-1], mcwav[-1], sz[0], mcsz[0])
                 mcint = interp1d(mcwav, mcal, kind='cubic',
                                  fill_value='extrapolate')
                 mscal = mcint(wav) * 1.e16 / expt

kcwidrp/primitives/MakeInvsens.py

@@ -333,7 +333,7 @@ def _perform(self):
         wlm0 = wgoo0
         wlm1 = wgoo1
         # interactively set wavelength limits
-        if self.config.instrument.plot_level >= 1:
+        if self.config.instrument.plot_level > 1:
             print("CHECKING WAVELENGTH LIMITS")
             print("Current WL limits: %.1f - %.1f Angstroms "
                   "(blue vertical lines)" % (wlm0, wlm1))
@@ -387,7 +387,8 @@ def _perform(self):
         local_lmfile = os.path.basename(stdfile).split('.fits')[0] + '.lmsk'
         if not os.path.exists(local_lmfile):
             # then in stds directory
-            lmfile = stdfile.split('.fits')[0] + '.lmsk'
+            # lmfile = stdfile.split('.fits')[0] + '.lmsk'
+            lmfile = strip_fname(stdfile) + '.lmsk'
             if not os.path.exists(lmfile):
                 lmfile = None
         else:
@@ -428,7 +429,7 @@ def _perform(self):
                 else:
                     print("bad line: %s" % lmws)
         # Now interactively identify lines if requested
-        if self.config.instrument.plot_level >= 1:
+        if self.config.instrument.plot_level >= 2:
             yran = [np.min(obsspec[wl_good]), np.max(obsspec[wl_good])]
             # source = ColumnDataSource(data=dict(x=w, y=obsspec))
             print("MASKING SHARP FEATURES: ABSORPTION LINES/COSMIC RAYS")
@@ -580,7 +581,7 @@ def _perform(self):
                           line_color='black', line_dash='dashdot')
                 set_plot_lims(peff, xlim=[wall0, wall1], ylim=yran)
                 bokeh_plot(peff, self.context.bokeh_session)
-                if self.config.instrument.plot_level >= 1:
+                if self.config.instrument.plot_level >= 2:
                     input("Next? <cr>: ")
                 else:
                     time.sleep(2. * self.config.instrument.plot_pause)
@@ -601,7 +602,7 @@ def _perform(self):
                 pivs.line([wlm1, wlm1], yran, line_color='green')
                 set_plot_lims(pivs, xlim=[wall0, wall1], ylim=yran)
                 bokeh_plot(pivs, self.context.bokeh_session)
-                if self.config.instrument.plot_level >= 1:
+                if self.config.instrument.plot_level >= 2:
                     input("Next? <cr>: ")
                 else:
                     time.sleep(2. * self.config.instrument.plot_pause)
@@ -620,7 +621,7 @@ def _perform(self):
                 pcal.line([wlm1, wlm1], yran, line_color='green')
                 set_plot_lims(pcal, xlim=[wall0, wall1], ylim=yran)
                 bokeh_plot(pcal, self.context.bokeh_session)
-                if self.config.instrument.plot_level >= 1:
+                if self.config.instrument.plot_level >= 2:
                     input("Next? <cr>: ")
                 else:
                     time.sleep(2. * self.config.instrument.plot_pause)
@@ -650,7 +651,7 @@ def _perform(self):
                           line_color='black', line_dash='dashed')
                 set_plot_lims(prsd, xlim=[wall0, wall1], ylim=yran)
                 bokeh_plot(prsd, self.context.bokeh_session)
-                if self.config.instrument.plot_level >= 1:
+                if self.config.instrument.plot_level >= 2:
                     qstr = input("Current fit order = %d, "
                                  "New fit order? <int>, <cr> - done: " % ford)
                     if len(qstr) <= 0:

kcwidrp/primitives/MakeMasterArc.py

@@ -1,6 +1,6 @@
 from keckdrpframework.primitives.base_img import BaseImg
 from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \
-    kcwi_fits_writer, strip_fname  # , get_master_name
+    kcwi_fits_writer, strip_fname , get_unique_STATEID_master_name
 
 import os
 import ccdproc
@@ -58,14 +58,15 @@ def _perform(self):
 
         combine_list = list(self.combine_list['filename'])
         # get master arc output name
-        maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+        # maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
         if self.action.args.min_files > 1:
             stack = []
             stackf = []
             for arc in combine_list:
                 # get arc intensity (int) image file name in redux directory
-                stackf.append(arc.split('.fits')[0] + '_int.fits')
-                arcfn = os.path.join(args.in_directory, stackf[-1])
+                stackf.append(strip_fname(arc) + '_int.fits')
+                arcfn = os.path.join(self.config.instrument.cwd,
+                self.config.instrument.output_directory, stackf[-1])
                 # using [0] gets just the image data
                 stack.append(kcwi_fits_reader(arcfn)[0])
 
@@ -86,15 +87,18 @@ def _perform(self):
                                                          "stack input file")
             stacked.header['HISTORY'] = log_string
             self.action.args.ccddata = stacked
+            # maname = get_unique_STATEID_master_name(stacked, suffix)
+            maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
 
             kcwi_fits_writer(stacked, output_file=maname,
                              output_dir=self.config.instrument.output_directory)
             self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
                                                 newtype=args.new_type,
-                                                filename=stacked.header[
-                                                    'OFNAME'])
-            self.action.args.name = stacked.header['OFNAME']
+                                                filename=self.action.args.name) ### HERE
+            # self.action.args.name = maname
+            # self.action.args.name = stacked.header['OFNAME']
         else:
+            maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
             self.action.args.ccddata.header['IMTYPE'] = args.new_type
             self.action.args.ccddata.header['HISTORY'] = log_string
             kcwi_fits_writer(self.action.args.ccddata, output_file=maname,

kcwidrp/primitives/MakeMasterBias.py

@@ -1,7 +1,7 @@
 from keckdrpframework.models.arguments import Arguments
 from keckdrpframework.primitives.base_img import BaseImg
 from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \
-    kcwi_fits_writer, parse_imsec, strip_fname
+    kcwi_fits_writer, parse_imsec, strip_fname, get_unique_CCD_master_name
 from kcwidrp.core.bokeh_plotting import bokeh_plot
 from kcwidrp.core.kcwi_plotting import save_plot
 
@@ -64,15 +64,17 @@ def _perform(self):
         combine_list = list(self.combine_list['filename'])
         # get master bias output name
         # mbname = combine_list[-1].split('.fits')[0] + '_' + suffix + '.fits'
-        mbname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+        # mbname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+
         # mbname = master_bias_name(self.action.args.ccddata)
         bsec, dsec, tsec, direc, amps, aoff = self.action.args.map_ccd
 
         # loop over amps
         stack = []
         stackf = []
         for bias in combine_list:
-            inbias = bias.split('.fits')[0] + '_intb.fits'
+            # inbias = bias.split('.fits')[0] + '_intb.fits'
+            inbias = strip_fname(bias) + '_intb.fits'
             stackf.append(inbias)
             # using [0] drops the table and leaves just the image
             stack.append(kcwi_fits_reader(
@@ -158,11 +160,14 @@ def _perform(self):
         stacked.header['HISTORY'] = log_string
         self.logger.info(log_string)
 
+        mbname = get_unique_CCD_master_name(stacked)
+
+
         kcwi_fits_writer(stacked, output_file=mbname,
                          output_dir=self.config.instrument.output_directory)
         self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
                                             newtype=self.action.args.new_type,
-                                            filename=stacked.header['OFNAME'])
+                                            filename=self.action.args.name)
         self.context.proctab.write_proctab(tfil=self.config.instrument.procfile)
         return Arguments(name=mbname)
     # END: class ProcessBias()

kcwidrp/primitives/MakeMasterContbars.py

@@ -1,6 +1,6 @@
 from keckdrpframework.primitives.base_img import BaseImg
 from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \
-    kcwi_fits_writer, strip_fname  # , get_master_name
+    kcwi_fits_writer, strip_fname , get_unique_STATEID_master_name
 
 import os
 import ccdproc
@@ -57,15 +57,14 @@ def _perform(self):
         log_string = MakeMasterContbars.__module__
 
         combine_list = list(self.combine_list['filename'])
-        # get master arc output name
-        maname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+
         if self.action.args.min_files > 1:
             stack = []
             stackf = []
             for cbs in combine_list:
                 # get arc intensity (int) image file name in redux directory
-                stackf.append(cbs.split('.fits')[0] + '_int.fits')
-                cbsfn = os.path.join(args.in_directory, stackf[-1])
+                stackf.append(strip_fname(cbs) + '_int.fits')
+                cbsfn = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, stackf[-1])
                 # using [0] gets just the image data
                 stack.append(kcwi_fits_reader(cbsfn)[0])
 
@@ -87,17 +86,20 @@ def _perform(self):
             stacked.header['HISTORY'] = log_string
             self.action.args.ccddata = stacked
 
-            kcwi_fits_writer(stacked, output_file=maname,
+            # get master arc output name
+            mcbars_name = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+
+            kcwi_fits_writer(stacked, output_file=mcbars_name,
                              output_dir=self.config.instrument.output_directory)
             self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
                                                 newtype=args.new_type,
-                                                filename=stacked.header[
-                                                    'OFNAME'])
-            self.action.args.name = stacked.header['OFNAME']
+                                                filename=self.action.args.name) ### HERE
+            # self.action.args.name = stacked.header['OFNAME']
         else:
+            mcbars_name = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
             self.action.args.ccddata.header['IMTYPE'] = args.new_type
             self.action.args.ccddata.header['HISTORY'] = log_string
-            kcwi_fits_writer(self.action.args.ccddata, output_file=maname,
+            kcwi_fits_writer(self.action.args.ccddata, output_file=mcbars_name,
                              output_dir=self.config.instrument.output_directory)
             self.context.proctab.update_proctab(frame=self.action.args.ccddata,
                                                 suffix=suffix,

kcwidrp/primitives/MakeMasterDark.py

@@ -1,6 +1,6 @@
 from keckdrpframework.primitives.base_img import BaseImg
 from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_reader, \
-    kcwi_fits_writer, strip_fname  # , get_master_name
+    kcwi_fits_writer, strip_fname, get_unique_STATEID_master_name  # , get_master_name
 
 import os
 import ccdproc
@@ -54,13 +54,15 @@ def _perform(self):
 
         combine_list = list(self.combine_list['filename'])
         # get master dark output name
-        mdname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+        # mdname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
+        mdname = get_unique_STATEID_master_name(self.action.args.ccddata, suffix="mdark")
         stack = []
         stackf = []
         for dark in combine_list:
             # get dark intensity (int) image file name in redux directory
-            stackf.append(dark.split('.fits')[0] + '_int.fits')
-            darkfn = os.path.join(args.in_directory, stackf[-1])
+            stackf.append(strip_fname(dark) + '_int.fits')
+            darkfn = os.path.join(self.config.instrument.cwd,
+                self.config.instrument.output_directory, stackf[-1])
             # using [0] gets just the image data
             stack.append(kcwi_fits_reader(darkfn)[0])
 
@@ -87,7 +89,7 @@ def _perform(self):
                          output_dir=self.config.instrument.output_directory)
         self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
                                             newtype=args.new_type,
-                                            filename=stacked.header['OFNAME'])
+                                            filename=self.action.args.name) ### HERE
         self.context.proctab.write_proctab(tfil=self.config.instrument.procfile)
         return self.action.args
     # END: class MakeMasterDark()

kcwidrp/primitives/MakeMasterFlat.py

@@ -99,7 +99,7 @@ def _perform(self):
             self.logger.error("Geometry not solved!")
             return self.action.args
 
-        mroot = strip_fname(tab['filename'][-1])
+        mroot = strip_fname(tab['filename'][-1]) # ??????????????????
 
         # Wavelength map image
         wmf = mroot + '_wavemap.fits'
@@ -435,7 +435,7 @@ def _perform(self):
                 trm = int(len(deriv)/5)
                 deriv = deriv[trm:-trm]
                 xvals = fpoints[trm:-trm]
-                peaks, _ = find_peaks(deriv, height=20)
+                peaks, properties = find_peaks(deriv, height=20)
                 self.logger.info("%d Peak(s) found" % len(peaks))
 
                 p = figure(title=plab +
@@ -466,11 +466,16 @@ def _perform(self):
                 bokeh_plot(p, self.context.bokeh_session)
                 if len(peaks) != 1:
                     self.logger.warning("Single peak not found!")
-                    print("Please indicate the integer pixel value of the peak")
-                    spk = input("Peak? <int>: ")
-                    while not spk.isnumeric():
+                    if self.config.instrument.plot_level >= 2:
+                        print("Please indicate the integer pixel value of the peak")
                         spk = input("Peak? <int>: ")
-                    ipk = int(spk)
+                        while not spk.isnumeric():
+                            spk = input("Peak? <int>: ")
+                        ipk = int(spk)
+                    else:
+                        # Pick the highest peak, using the peaks properties
+                        self.logger.warning("Picking the highest peak. This may not be the correct peak!")
+                        ipk = properties['peak_heights'].argmax()
                 else:
                     ipk = peaks[0]
                 apk = xvals[ipk]
@@ -915,7 +920,7 @@ def _perform(self):
             ratio.flat[qq] = 1.0
 
         # get master flat output name
-        mfname = stack_list[0].split('.fits')[0] + '_' + suffix + '.fits'
+        mfname = strip_fname(stack_list[0]) + '_' + suffix + '.fits'
 
         log_string = MakeMasterFlat.__module__
         stacked.header['IMTYPE'] = self.action.args.new_type
@@ -933,9 +938,10 @@ def _perform(self):
                          output_dir=self.config.instrument.output_directory)
         self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
                                             newtype=self.action.args.new_type,
-                                            filename=stacked.header['OFNAME'])
+                                            filename=stack_list[0]) ### HERE
         self.context.proctab.write_proctab(tfil=self.config.instrument.procfile)
-        self.action.args.name = stacked.header['OFNAME']
+        # self.action.args.name = stacked.header['OFNAME']
+        # self.action.args.name = mfname
 
         self.logger.info(log_string)
         return self.action.args