diff --git a/configs/kpf_masters_l1.cfg b/configs/kpf_masters_l1.cfg index a5881c72c..ceae00968 100644 --- a/configs/kpf_masters_l1.cfg +++ b/configs/kpf_masters_l1.cfg @@ -174,6 +174,14 @@ quicklook = False gain_keyword = 'CCDGAIN' n_sigma_read_noise = 3.0 +[WLS_INTERPOLATION] +cal_type = 'WLS' +object_before = 'autocal-lfc-all-eve' +object_after = 'autocal-lfc-all-morn' +#master_file_before_default = /masters/20240128/kpf_20240128_master_WLS_autocal-etalon-all-eve_L1.fits +#master_file_after_default = /masters/20240129/kpf_20240129_master_WLS_autocal-etalon-all-morn_L1.fits +do_db_query_for_before_after_master_files = True + # config file associated with the modules [MODULE_CONFIGS] order_trace = modules/order_trace/configs/default_recipe_kpf_20220505.cfg diff --git a/cronjobs/cleanOldFilesFromDisk.pl b/cronjobs/cleanOldFilesFromDisk.pl index 3861f15b5..ba11d40c2 100755 --- a/cronjobs/cleanOldFilesFromDisk.pl +++ b/cronjobs/cleanOldFilesFromDisk.pl @@ -42,8 +42,9 @@ sub removeOldSubDirs { closedir DIR; foreach my $file (@files) { print "file = $file\n"; - if (($file eq ".") or ($file eq "..") or ($file =~ /\.+/)) { + if (($file eq ".") or ($file eq "..") or ($file =~ /^\.+/)) { print "Skipping file---->[$file]\n"; + next; } $file = $dir . "/" . $file; if ((-d $file) and (! ($file =~ /^.+\/\..*$/))) { @@ -71,8 +72,9 @@ sub removeOldFiles { closedir DIR; foreach my $file (@files) { print "file = $file\n"; - if (($file eq ".") or ($file eq "..") or ($file =~ /\.+/)) { + if (($file eq ".") or ($file eq "..") or ($file =~ /^\.+/)) { print "Skipping file---->[$file]\n"; + next; } $file = $dir . "/" . $file; if ((-f $file) and (! ($file =~ /^.+\/\..*$/))) { diff --git a/cronjobs/updateSoftwareAndReferenceFits.pl b/cronjobs/updateSoftwareAndReferenceFits.pl new file mode 100755 index 000000000..d913b0290 --- /dev/null +++ b/cronjobs/updateSoftwareAndReferenceFits.pl @@ -0,0 +1,96 @@ +#! /usr/local/bin/perl + +use strict; +use warnings; + +select STDERR; $| = 1; select STDOUT; $| = 1; + +my $swdir = '/data/user/rlaher/git/KPF-Pipeline'; +&updateSoftwareWithGitPull($swdir); + +my $privaterefdir = '/data/user/rlaher/sbx/reference_fits'; +my $publicrefdir = '/data/kpf/reference_fits'; +&updateReferenceFits($privaterefdir,$publicrefdir); + +exit 0; + + +#------------------------------------------ +# Update software with git pull. + +sub updateSoftwareWithGitPull { + + my ($workdir) = @_; + + + # Change directory. + + print "Changing to workdir = $workdir\n"; + + if (! chdir "$workdir") { + print "*** Warning: Could not change to $workdir; sleep for 10 seconds and try again...\n"; + sleep(10); + if (! chdir "$workdir") { + die "*** Error: Could not change to $workdir; quitting...\n"; + } + } + + my $cmd0 = "git pull"; + print "Executing [$cmd0]...\n"; + my @op0 = `$cmd0`; + if (@op0) { print "Output from [$cmd0]=[@op0]\n"; } +} + + +#------------------------------------------ +# Update files in private reference_fits. + +sub updateReferenceFits { + + my ($dir,$dir2) = @_; + + + # Change to private directory. + + print "Changing to private reference_fits directory = $dir\n"; + + if (! chdir "$dir") { + print "*** Warning: Could not change to $dir; sleep for 10 seconds and try again...\n"; + sleep(10); + if (! chdir "$dir") { + die "*** Error: Could not change to $dir; quitting...\n"; + } + } + + print "dir = $dir\n"; + opendir(DIR, "$dir"); + my @files = readdir DIR; + closedir DIR; + + print "dir2 = $dir2\n"; + opendir(DIR, "$dir2"); + my @files2 = readdir DIR; + closedir DIR; + + foreach my $file2 (@files2) { + print "file2 = $file2\n"; + if (($file2 eq ".") or ($file2 eq "..") or ($file2 =~ /^\.+/)) { + print "1 Skipping file2---->[$file2]\n"; + next; + } + + # Check if file2 exists in private (current) directory. + # If not, copy it from the public directory. + + if (-e $file2) { + print "$file2 already exists in private directory; skipping...\n"; + } else { + $file2 = $dir2 . "/" . $file2; + + my $cmd0 = "cp -p $file2 ."; + print "Executing [$cmd0]...\n"; + my @op0 = `$cmd0`; + if (@op0) { print "Output from [$cmd0]=[@op0]\n"; } + } + } +} diff --git a/database/schema/kpfOpsRoles.sql b/database/schema/kpfOpsRoles.sql index 69e17de39..8ef2bb781 100644 --- a/database/schema/kpfOpsRoles.sql +++ b/database/schema/kpfOpsRoles.sql @@ -29,3 +29,7 @@ GRANT kpfporole to bfulton; -- Make it so Andrew can run the master-files pipeline. GRANT kpfporole to howard; + +-- Make it so Howard Isaacson has a read-only database role (per his request). +REVOKE kpfporole from isaacson; +GRANT kpfreadrole to isaacson; diff --git a/docs/source/tutorials/KPF_Data_Tutorial_Quality_Control.ipynb b/docs/source/tutorials/KPF_Data_Tutorial_Quality_Control.ipynb new file mode 100644 index 000000000..87c704798 --- /dev/null +++ b/docs/source/tutorials/KPF_Data_Tutorial_Quality_Control.ipynb @@ -0,0 +1,1038 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "635b80ed", + "metadata": {}, + "source": [ + "# Import Packages" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ed65195a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from modules.Utils.kpf_parse import get_kpf_data, HeaderParse\n", + "import modules.quality_control.src.quality_control as qc\n", + "from modules.quality_control.src.quality_control import test_all_QCs" + ] + }, + { + "cell_type": "markdown", + "id": "197ede5c", + "metadata": {}, + "source": [ + "# List QC Methods\n", + "The method `list_qc_metrics()` lists the available QC methods that can be applied to the different data levels." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b5877a3c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1mQuality Control tests for L0:\u001b[0m\n", + " \u001b[1mQC Name:\u001b[0m not_junk_check\n", + " \u001b[1mDescription:\u001b[0m Check if file is not in list of junk files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m NOTJUNK\n", + " \u001b[1mComment:\u001b[0m QC: Not in list of junk files\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m L0_data_products_check\n", + " \u001b[1mDescription:\u001b[0m Check if expected L0 data products are present with non-zero array sizes.\n", + " \u001b[1mData levels:\u001b[0m ['L0']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m DATAPRL0\n", + " \u001b[1mComment:\u001b[0m QC: L0 data present\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m L0_header_keywords_present_check\n", + " \u001b[1mDescription:\u001b[0m Check if expected L0 header keywords are present.\n", + " \u001b[1mData levels:\u001b[0m ['L0']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m KWRDPRL0\n", + " \u001b[1mComment:\u001b[0m QC: L0 keywords present\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m L0_datetime_checks\n", + " \u001b[1mDescription:\u001b[0m Check for timing consistency in L0 header keywords and Exp Meter table.\n", + " \u001b[1mData levels:\u001b[0m ['L0']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m TIMCHKL0\n", + " \u001b[1mComment:\u001b[0m QC: L0 times consistent\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m exposure_meter_not_saturated_check\n", + " \u001b[1mDescription:\u001b[0m Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.\n", + " \u001b[1mData levels:\u001b[0m ['L0']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m EMSAT\n", + " \u001b[1mComment:\u001b[0m QC: EM not saturated\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m exposure_meter_flux_not_negative_check\n", + " \u001b[1mDescription:\u001b[0m Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.\n", + " \u001b[1mData levels:\u001b[0m ['L0']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m EMNEG\n", + " \u001b[1mComment:\u001b[0m QC: EM not negative flux\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m add_kpfera\n", + " \u001b[1mDescription:\u001b[0m Not a QC test. The QC module is used to add the KPFERA keyword to all files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m float\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m KPFERA\n", + " \u001b[1mComment:\u001b[0m Current era of KPF observations\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + "\u001b[1mQuality Control tests for 2D:\u001b[0m\n", + " \u001b[1mQC Name:\u001b[0m not_junk_check\n", + " \u001b[1mDescription:\u001b[0m Check if file is not in list of junk files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m NOTJUNK\n", + " \u001b[1mComment:\u001b[0m QC: Not in list of junk files\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m D2_lfc_flux_check\n", + " \u001b[1mDescription:\u001b[0m Check if an LFC frame that goes into a master has sufficient flux\n", + " \u001b[1mData levels:\u001b[0m ['2D']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['LFC']\n", + " \u001b[1mKeyword:\u001b[0m LFC2DFOK\n", + " \u001b[1mComment:\u001b[0m QC: LFC flux meets threshold of 4000 counts\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_2D_bias_low_flux_check\n", + " \u001b[1mDescription:\u001b[0m Check if flux is low in bias exposure.\n", + " \u001b[1mData levels:\u001b[0m ['2D']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['Bias']\n", + " \u001b[1mKeyword:\u001b[0m LOWBIAS\n", + " \u001b[1mComment:\u001b[0m QC: 2D bias low flux check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_2D_dark_low_flux_check\n", + " \u001b[1mDescription:\u001b[0m Check if flux is low in dark exposure.\n", + " \u001b[1mData levels:\u001b[0m ['2D']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['Dark']\n", + " \u001b[1mKeyword:\u001b[0m LOWDARK\n", + " \u001b[1mComment:\u001b[0m QC: 2D dark low flux check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_2D_CaHK_check\n", + " \u001b[1mDescription:\u001b[0m Check if CaHK CCD data is present with expected array sizes.\n", + " \u001b[1mData levels:\u001b[0m ['2D']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m CaHKPR2D\n", + " \u001b[1mComment:\u001b[0m QC: 2D CaHK data present check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_2D_red_green_check\n", + " \u001b[1mDescription:\u001b[0m Check if red/green CCD data is present with expected array sizes.\n", + " \u001b[1mData levels:\u001b[0m ['2D']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m DATAPR2D\n", + " \u001b[1mComment:\u001b[0m QC: 2D red and green data present check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m add_kpfera\n", + " \u001b[1mDescription:\u001b[0m Not a QC test. The QC module is used to add the KPFERA keyword to all files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m float\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m KPFERA\n", + " \u001b[1mComment:\u001b[0m Current era of KPF observations\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + "\u001b[1mQuality Control tests for L1:\u001b[0m\n", + " \u001b[1mQC Name:\u001b[0m not_junk_check\n", + " \u001b[1mDescription:\u001b[0m Check if file is not in list of junk files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m NOTJUNK\n", + " \u001b[1mComment:\u001b[0m QC: Not in list of junk files\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m monotonic_wavelength_solution_check\n", + " \u001b[1mDescription:\u001b[0m Check if wavelength solution is monotonic.\n", + " \u001b[1mData levels:\u001b[0m ['L1']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m MONOTWLS\n", + " \u001b[1mComment:\u001b[0m QC: Monotonic wavelength-solution\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_L1_red_green_check\n", + " \u001b[1mDescription:\u001b[0m Check if red/green data are present in L1 with expected shapes.\n", + " \u001b[1mData levels:\u001b[0m ['L1']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m DATAPRL1\n", + " \u001b[1mComment:\u001b[0m QC: L1 red and green data present check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_L1_CaHK_check\n", + " \u001b[1mDescription:\u001b[0m Check if CaHK data is present in L1 with expected shape.\n", + " \u001b[1mData levels:\u001b[0m ['L1']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m CaHKPRL1\n", + " \u001b[1mComment:\u001b[0m QC: L1 CaHK present check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m add_kpfera\n", + " \u001b[1mDescription:\u001b[0m Not a QC test. The QC module is used to add the KPFERA keyword to all files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m float\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m KPFERA\n", + " \u001b[1mComment:\u001b[0m Current era of KPF observations\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + "\u001b[1mQuality Control tests for L2:\u001b[0m\n", + " \u001b[1mQC Name:\u001b[0m not_junk_check\n", + " \u001b[1mDescription:\u001b[0m Check if file is not in list of junk files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m NOTJUNK\n", + " \u001b[1mComment:\u001b[0m QC: Not in list of junk files\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m L2_datetime_checks\n", + " \u001b[1mDescription:\u001b[0m Check for timing consistency in L2 files.\n", + " \u001b[1mData levels:\u001b[0m ['L2']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m TIMCHKL2\n", + " \u001b[1mComment:\u001b[0m QC: L2 times consistent\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m data_L2_check\n", + " \u001b[1mDescription:\u001b[0m Check if all data are present in L2.\n", + " \u001b[1mData levels:\u001b[0m ['L2']\n", + " \u001b[1mData type:\u001b[0m int\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m DATAPRL2\n", + " \u001b[1mComment:\u001b[0m QC: L2 data present check\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + " \u001b[1mQC Name:\u001b[0m add_kpfera\n", + " \u001b[1mDescription:\u001b[0m Not a QC test. The QC module is used to add the KPFERA keyword to all files.\n", + " \u001b[1mData levels:\u001b[0m ['L0', '2D', 'L1', 'L2']\n", + " \u001b[1mData type:\u001b[0m float\n", + " \u001b[1mSpectrum type:\u001b[0m ['all']\n", + " \u001b[1mKeyword:\u001b[0m KPFERA\n", + " \u001b[1mComment:\u001b[0m Current era of KPF observations\n", + " \u001b[1mDatabase column:\u001b[0m None\n", + "\n", + "None\n" + ] + } + ], + "source": [ + "myQCdef = qc.QCDefinitions()\n", + "QC_lists = myQCdef.list_qc_metrics()\n", + "print(QC_lists)" + ] + }, + { + "cell_type": "markdown", + "id": "9d28be81", + "metadata": {}, + "source": [ + "# Run all QC tests on all data levels of several KPF observations\n", + "The method `test_all_QCs(kpf_object, data_level)` runs the QC tests on a kpf_object for the stated data_level." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ad8b062", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1mL0 QC tests on KP.20230701.49940.99\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Star\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: Method L0_data_products_check does not exist in qc_obj or another AttributeError occurred: 'KPF0' object has no attribute 'GUIDER_AVG'\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: Method L0_datetime_checks does not exist in qc_obj or another AttributeError occurred: 'KPF0' object has no attribute 'GUIDER_AVG'\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2023-07-01 13:52:20.\n", + "INFO: QC result: \u001b[1;32m1.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20230701.49940.99\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Star\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Star not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type Star not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type Star not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2023-07-01 13:52:20.\n", + "INFO: QC result: \u001b[1;32m1.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20230701.49940.99\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Star\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2023-07-01 13:52:20.\n", + "INFO: QC result: \u001b[1;32m1.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20230701.49940.99\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Star\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2023-07-01 13:52:20.\n", + "INFO: QC result: \u001b[1;32m1.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240525.77699.94\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Sun\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-25 21:34:59.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240525.77699.94\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Sun\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Sun not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type Sun not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type Sun not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-25 21:34:59.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240525.77699.94\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Sun\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-25 21:34:59.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240525.77699.94\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO: Spectrum type: Sun\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-25 21:34:59.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240528.04120.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Bias\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:08:40.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240528.04120.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Bias\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Bias not in list of spectrum types: ['LFC']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_bias_low_flux_check\u001b[0m (Check if flux is low in bias exposure.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type Bias not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:08:40.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240528.04120.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Bias\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:08:40.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240528.08502.04\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Dark\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:21:42.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240528.08502.04\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Dark\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Dark not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type Dark not in list of spectrum types: ['Bias']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_dark_low_flux_check\u001b[0m (Check if flux is low in dark exposure.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:21:42.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240528.08502.04\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Dark\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:21:42.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240527.84455.02\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Flat\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 23:27:35.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240527.84455.02\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO: Spectrum type: Flat\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Flat not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type Flat not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type Flat not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 23:27:35.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240527.84455.02\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Flat\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 23:27:35.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240527.84455.02\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Flat\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 23:27:35.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240526.11989.38\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: LFC\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-26 03:19:49.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240526.11989.38\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: LFC\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mD2_lfc_flux_check\u001b[0m (Check if an LFC frame that goes into a master has sufficient flux)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type LFC not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type LFC not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-26 03:19:49.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240526.11989.38\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: LFC\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-26 03:19:49.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240526.11989.38\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: LFC\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-26 03:19:49.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240528.07447.61\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: ThAr\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:04:07.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240528.07447.61\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO: Spectrum type: ThAr\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type ThAr not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type ThAr not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type ThAr not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:04:07.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240528.07447.61\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: ThAr\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:04:07.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240528.07447.61\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: ThAr\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:04:07.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240527.11183.00\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: UNe\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 03:06:23.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240527.11183.00\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: UNe\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type UNe not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type UNe not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type UNe not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 03:06:23.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240527.11183.00\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: UNe\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 03:06:23.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240527.11183.00\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: UNe\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 03:06:23.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240528.05681.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:34:41.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240528.05681.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Etalon not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type Etalon not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type Etalon not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:34:41.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240528.05681.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:34:41.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240528.05681.26\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 01:34:41.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240527.51851.54\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 14:24:11.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240527.51851.54\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type Etalon not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type Etalon not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type Etalon not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 14:24:11.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240527.51851.54\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 14:24:11.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240527.51851.54\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: Etalon\n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-27 14:24:11.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL0 QC tests on KP.20240528.07772.51\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: \n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_data_products_check\u001b[0m (Check if expected L0 data products are present with non-zero array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_header_keywords_present_check\u001b[0m (Check if expected L0 header keywords are present.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL0_datetime_checks\u001b[0m (Check for timing consistency in L0 header keywords and Exp Meter table.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_not_saturated_check\u001b[0m (Check if 2+ reduced EM pixels are within 90% of saturation in EM-SCI or EM-SKY.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mexposure_meter_flux_not_negative_check\u001b[0m (Check for negative flux in the EM-SCI and EM-SKY by looking for 20 consecuitive pixels in the summed spectra with negative flux.)\n", + "INFO: QC result: \u001b[1;31mFalse\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:09:32.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1m2D QC tests on KP.20240528.07772.51\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO: Spectrum type: \n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Not running QC: D2_lfc_flux_check (Check if an LFC frame that goes into a master has sufficient flux) because spectrum type not in list of spectrum types: ['LFC']\n", + "INFO: Not running QC: data_2D_bias_low_flux_check (Check if flux is low in bias exposure.) because spectrum type not in list of spectrum types: ['Bias']\n", + "INFO: Not running QC: data_2D_dark_low_flux_check (Check if flux is low in dark exposure.) because spectrum type not in list of spectrum types: ['Dark']\n", + "INFO: Running QC: \u001b[1;34mdata_2D_CaHK_check\u001b[0m (Check if CaHK CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_2D_red_green_check\u001b[0m (Check if red/green CCD data is present with expected array sizes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:09:32.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL1 QC tests on KP.20240528.07772.51\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: \n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mmonotonic_wavelength_solution_check\u001b[0m (Check if wavelength solution is monotonic.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_red_green_check\u001b[0m (Check if red/green data are present in L1 with expected shapes.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L1_CaHK_check\u001b[0m (Check if CaHK data is present in L1 with expected shape.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:09:32.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n", + "\u001b[1mL2 QC tests on KP.20240528.07772.51\u001b[0m\n", + "\u001b[1m***********************************\u001b[0m\n", + "INFO: Spectrum type: \n", + "INFO: Running QC: \u001b[1;34mnot_junk_check\u001b[0m (Check if file is not in list of junk files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mL2_datetime_checks\u001b[0m (Check for timing consistency in L2 files.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34mdata_L2_check\u001b[0m (Check if all data are present in L2.)\n", + "INFO: QC result: \u001b[1;32mTrue\u001b[0m (True = pass)\n", + "INFO: Running QC: \u001b[1;34madd_kpfera\u001b[0m (Not a QC test. The QC module is used to add the KPFERA keyword to all files.)\n", + "INFO: The datetime of ObsID is 2024-05-28 02:09:32.\n", + "INFO: QC result: \u001b[1m2.0\u001b[0m (True = pass)\n", + "\n" + ] + } + ], + "source": [ + "# Define data levels to loop over\n", + "data_levels = ['L0', '2D', 'L1', 'L2']\n", + "\n", + "# Define list of observations to loop over - one of every spectrum type\n", + "ObsIDs = [\n", + " 'KP.20230701.49940.99', # 185144 (stellar observation)\n", + " 'KP.20240525.77699.94', # Socal\n", + " 'KP.20240528.04120.26', # autocal-bias\n", + " 'KP.20240528.08502.04', # autocal-dark\n", + " 'KP.20240527.84455.02', # autocal-flat-all\n", + " 'KP.20240526.11989.38', # autocal-lfc-all-eve\n", + " 'KP.20240528.07447.61', # autocal-thar-all-eve\n", + " 'KP.20240527.11183.00', # autocal-une-all-eve\n", + " 'KP.20240528.05681.26', # autocal-etalon-all-eve\n", + " 'KP.20240527.51851.54', # autocal-etalon-all-night\n", + " 'KP.20240528.07772.51', # autocal-thar-hk\n", + " ]\n", + "\n", + "# Run QC tests\n", + "for ObsID in ObsIDs:\n", + " for data_level in data_levels:\n", + " if os.path.isfile(get_kpf_data(ObsID, data_level, return_kpf_object=False)):\n", + " print(f'\\033[1m{data_level} QC tests on {ObsID}\\033[0m')\n", + " print(f'\\033[1m***********************************\\033[0m')\n", + " kpf_object = get_kpf_data(ObsID, data_level)\n", + " test_all_QCs(kpf_object, data_level)\n", + " print()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modules/Utils/kpf_parse.py b/modules/Utils/kpf_parse.py index 93379e83b..d73b5fdc8 100644 --- a/modules/Utils/kpf_parse.py +++ b/modules/Utils/kpf_parse.py @@ -2,6 +2,9 @@ import pandas as pd from astropy.io import fits from datetime import datetime, timedelta +from kpfpipe.models.level0 import KPF0 +from kpfpipe.models.level1 import KPF1 +from kpfpipe.models.level2 import KPF2 class KPFParse: @@ -400,7 +403,7 @@ def get_data_products_L2(L2): Telemetry, Config, Receipt Args: - L1 - a KPF L2 object + L2 - a KPF L2 object Returns: data_products in a L2 file @@ -434,3 +437,95 @@ def get_data_products_L2(L2): if L2['RECEIPT'].size > 1: data_products.append('Receipt') return data_products + + +def hasattr_with_wildcard(obj, pattern): + regex = re.compile(pattern) + return any(regex.match(attr) for attr in dir(obj)) + +def get_kpf_level(kpf_object): + """ + Returns a string with the KPF level ('L0', '2D', 'L1', 'L2') corresponding + to the input KPF pubject + + Args: + kpf_object - a KPF object + + Returns: + kpf_level ('L0', '2D', 'L1', 'L2') + """ + + # L2 if there's an extension that starts with 'CCF' or 'RV' + if hasattr(kpf_object, 'RV'): + return 'L2' + if hasattr(kpf_object, 'CCF'): + return 'L2' + + # elif L1 if there's an extension that includes 'WAVE' + if hasattr_with_wildcard(kpf_object, r'.*WAVE.*'): + return 'L1' + + # elif 2D if GREEN_CCD or RED_CCD has non-zero size + if hasattr(kpf_object, 'GREEN_CCD'): + if kpf_object['GREEN_CCD'].size > 1: + return '2D' + if hasattr(kpf_object, 'RED_CCD'): + if kpf_object['RED_CCD'].size > 1: + return '2D' + + # elif L0 if one of the standard extensions is present with non-zero size + L0_attrs = ['GREEN_AMP1', 'RED_AMP1', 'CA_HK', 'EXPMETER_SCI', 'GUIDER_AVG', 'GUIDER_CUBE_ORIGINS'] + for L0_attr in L0_attrs: + if hasattr(kpf_object, L0_attr): + if kpf_object[L0_attr].size: + return 'L0' + + return None + +def get_kpf_data(ObsID, data_level, data_dir='/data', return_kpf_object=True): + """ + Returns the full path of a KPF object or the KPF object itself + with a specific data level + + Args: + ObsID - e.g., 'KP.20230617.61836.73' + data_level - 'L0', '2D', 'L1', or 'L2' + data_dir - directory that contains L0/, 2D/, L1/, L2/ + return_kpf_object - if True, return kpf_object; if false, return path to object + + Returns: + kpf_object + or + full path of file, e.g., /data/2D/20230701/KP.20230701.49940.99_2D.fits + """ + try: + datecode = get_datecode(ObsID) + if data_level == 'L0': + fullpath = data_dir + '/L0/' + get_datecode(ObsID) + '/' + ObsID + '.fits' + if return_kpf_object: + return_object = KPF0.from_fits(fullpath) + else: + return_object = fullpath + elif data_level == '2D': + fullpath = data_dir + '/2D/' + get_datecode(ObsID) + '/' + ObsID + '_2D.fits' + if return_kpf_object: + return_object = KPF0.from_fits(fullpath) + else: + return_object = fullpath + elif data_level == 'L1': + fullpath = data_dir + '/L1/' + get_datecode(ObsID) + '/' + ObsID + '_L1.fits' + if return_kpf_object: + return_object = KPF1.from_fits(fullpath) + else: + return_object = fullpath + elif data_level == 'L2': + fullpath = data_dir + '/L2/' + get_datecode(ObsID) + '/' + ObsID + '_L2.fits' + if return_kpf_object: + return_object = KPF2.from_fits(fullpath) + else: + return_object = fullpath + except: + return None + + return return_object + diff --git a/modules/calibration_lookup/src/alg.py b/modules/calibration_lookup/src/alg.py index 19c5c29c4..7bd6e5053 100644 --- a/modules/calibration_lookup/src/alg.py +++ b/modules/calibration_lookup/src/alg.py @@ -358,7 +358,14 @@ def run_query(cur,rep,query_template, contentbitmask, log): return exit_code,results_list def extract_from_db_results(results, cal_type): - if results[0] == 1: + if cal_type.lower() == 'wls': + results_list = [None, None] + if results[0] == 0: + results_list[0] = results[1][6] + if results[2] == 0: + results_list[1] = results[3][6] + return results_list + elif results[0] == 1: return '' elif cal_type.lower() == 'wls': return [results[1][6], results[3][6]] @@ -429,8 +436,13 @@ def lookup(self): output_cals[cal] = self.defaults[cal] elif lookup == 'wls': wls_results = query_wls(self.datetime, self.wls_cal_types, self.max_age, self.log) - if wls_results[0] == 0 and wls_results[2] == 0: - output_cals[cal] = extract_from_db_results(wls_results, cal) + if wls_results[0] == 0 or wls_results[2] == 0: + wls_files = extract_from_db_results(wls_results, cal) + if wls_files[0] == None: + wls_files[0] = wls_files[1] + if wls_files[1] == None: + wls_files[1] = wls_files[0] + output_cals[cal] = wls_files else: output_cals[cal] = self.defaults[cal] diff --git a/modules/master_flat/src/master_flat_framework.py b/modules/master_flat/src/master_flat_framework.py index 9563ad389..d43136735 100644 --- a/modules/master_flat/src/master_flat_framework.py +++ b/modules/master_flat/src/master_flat_framework.py @@ -208,32 +208,6 @@ def _perform(self): return Arguments(exit_list) - # Era-specific parameters. Override input arguments. - - self.logger.info('Override smoothlamppattern_path and ordermask_path with era-specific settings...') - - era_file = 'static/kpfera_definitions.csv' - config_file = 'configs/era_specific.cfg' - self.config = ConfigClass(config_file) - - self.eras = pd.read_csv(era_file, dtype='str', - sep='\s*,\s*') - - dt = datetime.strptime(obsdate, "%Y%m%d") - for i,row in self.eras.iterrows(): - start = datetime.strptime(row['UT_start_date'], "%Y-%m-%d %H:%M:%S") - end = datetime.strptime(row['UT_end_date'], "%Y-%m-%d %H:%M:%S") - if dt > start and dt <= end: - break - - era = row['KPFERA'] - self.logger.info('era = {}'.format(era)) - smoothlamppattern_path_options = eval(self.config.ARGUMENTS["smoothlamppattern_path"]) - self.smoothlamppattern_path = smoothlamppattern_path_options[era] - ordermask_path_options = eval(self.config.ARGUMENTS["ordermask_path"]) - self.ordermask_path = ordermask_path_options[era] - - # Optionally override self.smoothlamppattern_path from input argument with environment-variable setting. smoothlamppattern_envar = getenv('SMOOTH_LAMP_PATTERN') @@ -259,7 +233,9 @@ def _perform(self): self.logger.debug('Finished loading order-mask data from FITS file = {}'.format(self.ordermask_path)) - # Get master calibration files. + # Get required bias and dark master calibration files from database, if those provided by input arguments to not exist. + # This code section is needed because this cannot be easily handled via module/calibration_lookup in kpf_masters_drp.recipe. + # This code section is only necessary in the rare case where no daily bias and/or dark data are available. dbh = db.KPFDB() # Open database connection (if needed for fallback master calibration file) diff --git a/modules/quality_control/src/quality_control.py b/modules/quality_control/src/quality_control.py index ac82f1dd4..4af516776 100644 --- a/modules/quality_control/src/quality_control.py +++ b/modules/quality_control/src/quality_control.py @@ -5,7 +5,7 @@ from datetime import datetime from scipy.ndimage import convolve1d from modules.Utils.utils import DummyLogger -from modules.Utils.kpf_parse import HeaderParse, get_data_products_L0, get_datetime_obsid +from modules.Utils.kpf_parse import HeaderParse, get_data_products_L0, get_datetime_obsid, get_kpf_level """ This module contains classes for KPF data quality control (QC). Various QC metrics are defined in @@ -57,6 +57,65 @@ def avg_data_with_clipping(data_array,n_sigma = 3.0): return avg,std,cnt +def test_all_QCs(kpf_object, data_type): + """ + Method to loop over all QC tests for the data level of the input KPF object + (an L0, 2D, L1, or L2 object). This method is useful for testing (e.g., + in a Jupyter Notebook). To run the QCs in a recipe, use methods in + quality_control_framework.py + """ + + logger = DummyLogger() + + data_level = get_kpf_level(kpf_object) + + # Define QC object + if data_level == 'L0': + qc_obj = QCL0(kpf_object) + elif data_level == '2D': + qc_obj = QC2D(kpf_object) + elif data_level == 'L1': + qc_obj = QCL1(kpf_object) + elif data_level == 'L2': + qc_obj = QCL2(kpf_object) + else: + print('data_level is not L0, 2D, L1, or L2. Exiting.') + + if data_level != None: + + # Get a list of QC method names appropriate for the data level + qc_names = [] + for qc_name in qc_obj.qcdefinitions.names: + if data_level in qc_obj.qcdefinitions.kpf_data_levels[qc_name]: + qc_names.append(qc_name) + + # Run the QC tests and add result to keyword to header + primary_header = HeaderParse(kpf_object, 'PRIMARY') + this_spectrum_type = primary_header.get_name(use_star_names=False) + logger.info(f'Spectrum type: {this_spectrum_type}') + for qc_name in qc_names: + try: + spectrum_types = qc_obj.qcdefinitions.spectrum_types[qc_name] + if (this_spectrum_type in spectrum_types) or ('all' in spectrum_types): + logger.info(f'Running QC: \033[1;34m{qc_name}\033[0m ({qc_obj.qcdefinitions.descriptions[qc_name]})') + method = getattr(qc_obj, qc_name) # get method with the name 'qc_name' + qc_value = method() # evaluate method + if qc_value == True: + color_code = '\033[1;32m' # green + elif qc_value == False: + color_code = '\033[1;31m' # red + else: + color_code = '\033[1m' + logger.info(f'QC result: {color_code}{qc_value}\033[0m (True = pass)') + qc_obj.add_qc_keyword_to_header(qc_name, qc_value) + else: + logger.info(f'Not running QC: {qc_name} ({qc_obj.qcdefinitions.descriptions[qc_name]}) because spectrum type {this_spectrum_type} not in list of spectrum types: {spectrum_types}') + except AttributeError as e: + logger.info(f'Method {qc_name} does not exist in qc_obj or another AttributeError occurred: {e}') + pass + except Exception as e: + logger.info(f'An error occurred when executing {qc_name}:', str(e)) + pass ##################################################################### @@ -205,7 +264,7 @@ def __init__(self, logger=None): name9 = 'data_2D_bias_low_flux_check' self.names.append(name9) self.kpf_data_levels[name9] = ['2D'] - self.descriptions[name9] = 'Check to see if flux is low in bias exposure.' + self.descriptions[name9] = 'Check if flux is low in bias exposure.' self.data_types[name9] = 'int' self.spectrum_types[name9] = ['Bias', ] self.fits_keywords[name9] = 'LOWBIAS' @@ -215,7 +274,7 @@ def __init__(self, logger=None): name10 = 'data_2D_dark_low_flux_check' self.names.append(name10) self.kpf_data_levels[name10] = ['2D'] - self.descriptions[name10] = 'Check to see if flux is low in dark exposure.' + self.descriptions[name10] = 'Check if flux is low in dark exposure.' self.data_types[name10] = 'int' self.spectrum_types[name10] = ['Dark', ] self.fits_keywords[name10] = 'LOWDARK' @@ -227,7 +286,7 @@ def __init__(self, logger=None): self.kpf_data_levels[name11] = ['L1'] self.data_types[name11] = 'int' self.spectrum_types[name11] = ['all', ] - self.descriptions[name11] = 'Check to see if red and green data are present in L1 with expected shapes.' + self.descriptions[name11] = 'Check if red/green data are present in L1 with expected shapes.' self.fits_keywords[name11] = 'DATAPRL1' self.fits_comments[name11] = 'QC: L1 red and green data present check' self.db_columns[name11] = None @@ -235,7 +294,7 @@ def __init__(self, logger=None): name12 = 'data_L1_CaHK_check' self.names.append(name12) self.kpf_data_levels[name12] = ['L1'] - self.descriptions[name12] = 'Check to see if CaHK data is present in L1 with expected shape.' + self.descriptions[name12] = 'Check if CaHK data is present in L1 with expected shape.' self.data_types[name12] = 'int' self.spectrum_types[name12] = ['all', ] self.fits_keywords[name12] = 'CaHKPRL1' @@ -245,7 +304,7 @@ def __init__(self, logger=None): name13 = 'data_L2_check' self.names.append(name13) self.kpf_data_levels[name13] = ['L2'] - self.descriptions[name13] = 'Check to see if all data is present in L2.' + self.descriptions[name13] = 'Check if all data are present in L2.' self.data_types[name13] = 'int' self.spectrum_types[name13] = ['all', ] self.fits_keywords[name13] = 'DATAPRL2' @@ -255,7 +314,7 @@ def __init__(self, logger=None): name14 = 'data_2D_CaHK_check' self.names.append(name14) self.kpf_data_levels[name14] = ['2D'] - self.descriptions[name14] = 'Check to see if CaHK CCD data is present with expected array sizes.' + self.descriptions[name14] = 'Check if CaHK CCD data is present with expected array sizes.' self.data_types[name14] = 'int' self.spectrum_types[name14] = ['all', ] self.fits_keywords[name14] = 'CaHKPR2D' @@ -265,7 +324,7 @@ def __init__(self, logger=None): name15 = 'data_2D_red_green_check' self.names.append(name15) self.kpf_data_levels[name15] = ['2D'] - self.descriptions[name15] = 'Check to see if red and green CCD data is present with expected array sizes.' + self.descriptions[name15] = 'Check if red/green CCD data is present with expected array sizes.' self.data_types[name15] = 'int' self.spectrum_types[name15] = ['all', ] self.fits_keywords[name15] = 'DATAPR2D' @@ -929,7 +988,7 @@ def __init__(self,kpf_object): def data_2D_red_green_check(self,debug=False): """ - This Quality Control function checks to see if the 2D data exists for both + This Quality Control function checks if the 2D data exists for both the red and green chips and checks that the sizes of the arrays are as expected. Args: @@ -984,7 +1043,7 @@ def data_2D_red_green_check(self,debug=False): def data_2D_CaHK_check(self,debug=False): """ - This Quality Control function checks to see if the 2D data exists for the + This Quality Control function checks if the 2D data exists for the Ca H&K chip and checks that the size of the array is as expected. Args: @@ -1024,7 +1083,7 @@ def data_2D_CaHK_check(self,debug=False): def data_2D_bias_low_flux_check(self,debug=False): """ - This Quality Control function checks to see if the flux is low + This Quality Control function checks if the flux is low (mean flux < 10) for a bias exposure. Args: @@ -1062,7 +1121,7 @@ def data_2D_bias_low_flux_check(self,debug=False): def data_2D_dark_low_flux_check(self,debug=False): """ - This Quality Control function checks to see if the flux is low + This Quality Control function checks if the flux is low (mean flux < 10) for a dark exposure. Args: @@ -1145,7 +1204,7 @@ def __init__(self,kpf_object): def monotonic_wavelength_solution_check(self,debug=False): """ - This Quality Control function checks to see if a wavelength solution is + This Quality Control function checks if a wavelength solution is monotonic, specifically if wavelength decreases (or stays constant) with increasing array index. @@ -1231,7 +1290,7 @@ def monotonic_wavelength_solution_check(self,debug=False): def data_L1_red_green_check(self,debug=False): """ - This Quality Control function checks to see if the red and green data + This Quality Control function checks if the red and green data are present in an L1 file, and that all array sizes are as expected. Args: @@ -1319,7 +1378,7 @@ def data_L1_red_green_check(self,debug=False): def data_L1_CaHK_check(self,debug=False): """ - This Quality Control function checks to see if the green and red data + This Quality Control function checks if the green and red data are present in an L1 file, and that all array sizes are as expected. Args: @@ -1386,7 +1445,7 @@ def __init__(self,kpf_object): def data_L2_check(self,debug=False): """ - This Quality Control function checks to see if all of the + This Quality Control function checks if all of the expected data (telemetry, CCFs, and RVs) are present. Args: diff --git a/modules/quicklook/src/analyze_l0.py b/modules/quicklook/src/analyze_l0.py index 02040819c..58bbdcd6d 100644 --- a/modules/quicklook/src/analyze_l0.py +++ b/modules/quicklook/src/analyze_l0.py @@ -151,11 +151,11 @@ def determine_regions(self): self.red_present = True - def measure_read_noise_overscan(self, nparallel=30, nserial=50, nsigma=5.0): + def measure_read_noise_overscan(self, nparallel=30, nserial=50, nsigma=5.0, verbose=False): """ Measure read noise in the overscan region of a KPF CCD image. Read noise is measured as the standard deviation of the pixel values, - after applying an n-sigma iterative outlier rejection. + after applying an n-sigma outlier rejection. Args: nparallel (integer) - overscan length in parallel direction @@ -192,6 +192,8 @@ def measure_read_noise_overscan(self, nparallel=30, nserial=50, nsigma=5.0): overscan_region = data[5:2040 + nparallel-5,2044 + 10:2044 + nserial-10] vals = self.reject_outliers(overscan_region.flat, nsigma) self.read_noise_overscan[region] = gain * np.std(vals) + if verbose: + self.logger.info(f'Read noise({region}) = {self.read_noise_overscan[region]}') def plot_L0_stitched_image(self, chip=None, fig_path=None, show_plot=False): diff --git a/modules/quicklook/src/analyze_time_series.py b/modules/quicklook/src/analyze_time_series.py index 07878fc04..6aa6438af 100644 --- a/modules/quicklook/src/analyze_time_series.py +++ b/modules/quicklook/src/analyze_time_series.py @@ -58,6 +58,7 @@ class AnalyzeTimeSeries: To-do: * Add database for masters (separate from ObsIDs?) + * Method to return the avg, std., etc. for a DB column over a time range, with conditions (e.g., fast-read mode only) * Check if the plot doesn't have data and don't generate if so * Make plots of temperature vs. RV for various types of RVs * Add standard plots of flux vs. time for cals (all types?), stars, and solar -- highlight Junked files @@ -699,8 +700,12 @@ def get_keyword_types(self, level): 'EMNEG': 'float', # Quality Control: 1 = Exp Meter not negative flux; 0 = 20+ consecutive pixels in summed spectra with negative flux 'RNRED1': 'float', # Read noise for RED_AMP1 [e-] (first amplifier region on Red CCD) 'RNRED2': 'float', # Read noise for RED_AMP2 [e-] (second amplifier region on Red CCD) + 'RNRED3': 'float', # Read noise for RED_AMP3 [e-] (third amplifier region on Red CCD) + 'RNRED4': 'float', # Read noise for RED_AMP4 [e-] (fourth amplifier region on Red CCD) 'RNGREEN1': 'float', # Read noise for GREEN_AMP1 [e-] (first amplifier region on Green CCD) 'RNGREEN2': 'float', # Read noise for GREEN_AMP2 [e-] (second amplifier region on Green CCD) + 'RNGREEN3': 'float', # Read noise for GREEN_AMP3 [e-] (third amplifier region on Green CCD) + 'RNGREEN4': 'float', # Read noise for GREEN_AMP4 [e-] (fourth amplifier region on Green CCD) 'GREENTRT': 'float', # Green CCD read time [sec] 'REDTRT': 'float', # Red CCD read time [sec] 'READSPED': 'string', # Categorization of CCD read speed ('regular' or 'fast') @@ -1469,15 +1474,19 @@ def plot_standard_time_series(self, plot_name, start_date=None, end_date=None, elif plot_name=='ccd_readnoise': dict1 = {'col': 'RNGREEN1', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'Green CCD 1', 'marker': '.', 'linewidth': 0.5, 'color': 'darkgreen'}} dict2 = {'col': 'RNGREEN2', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'Green CCD 2', 'marker': '.', 'linewidth': 0.5, 'color': 'forestgreen'}} + dict1b= {'col': 'RNGREEN3', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'Green CCD 3', 'marker': '.', 'linewidth': 0.5, 'color': 'limegreen'}} + dict2b= {'col': 'RNGREEN4', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'Green CCD 4', 'marker': '.', 'linewidth': 0.5, 'color': 'lime'}} dict3 = {'col': 'RNRED1', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'RED CCD 1', 'marker': '.', 'linewidth': 0.5, 'color': 'darkred'}} dict4 = {'col': 'RNRED2', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'RED CCD 2', 'marker': '.', 'linewidth': 0.5, 'color': 'firebrick'}} - thispanelvars = [dict1, dict2] + dict3b= {'col': 'RNRED3', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'RED CCD 3', 'marker': '.', 'linewidth': 0.5, 'color': 'indianred'}} + dict4b= {'col': 'RNRED4', 'plot_type': 'plot', 'unit': 'e-', 'plot_attr': {'label': 'RED CCD 4', 'marker': '.', 'linewidth': 0.5, 'color': 'lightcoral'}} + thispanelvars = [dict1, dict2, dict1b, dict2b] thispaneldict = {'ylabel': 'Green CCD\nRead Noise [e-]', 'not_junk': 'true', 'legend_frac_size': 0.25} readnoisepanel1 = {'panelvars': thispanelvars, 'paneldict': thispaneldict} - thispanelvars = [dict3, dict4] + thispanelvars = [dict3, dict4, dict3b, dict4b] thispaneldict = {'ylabel': 'Red CCD\nRead Noise [e-]', 'title': 'CCD Read Noise', 'not_junk': 'true', @@ -1505,8 +1514,8 @@ def plot_standard_time_series(self, plot_name, start_date=None, end_date=None, # Red CCD panel - Dark current dict1 = {'col': 'FLXCOLLR', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Coll-side', 'marker': '.', 'linewidth': 0.5, 'color': 'darkred'}} - dict2 = {'col': 'FLXECHR', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Ech-side', 'marker': '.', 'linewidth': 0.5, 'color': 'firebrick'}} - dict3 = {'col': 'FLXREG1R', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Region 1', 'marker': '.', 'linewidth': 0.5, 'color': 'lightcoral'}} + dict2 = {'col': 'FLXECHR', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Ech-side', 'marker': '.', 'linewidth': 0.5, 'color': 'firebrick'}} + dict3 = {'col': 'FLXREG1R', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Region 1', 'marker': '.', 'linewidth': 0.5, 'color': 'lightcoral'}} dict4 = {'col': 'FLXREG2R', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Region 2', 'marker': '.', 'linewidth': 0.5, 'color': 'lightcoral'}} dict5 = {'col': 'FLXREG3R', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Region 3', 'marker': '.', 'linewidth': 0.5, 'color': 'lightcoral'}} dict6 = {'col': 'FLXREG4R', 'plot_type': 'plot', 'unit': 'e-/hr', 'plot_attr': {'label': 'Region 4', 'marker': '.', 'linewidth': 0.5, 'color': 'lightcoral'}} diff --git a/modules/quicklook/src/diagnostics.py b/modules/quicklook/src/diagnostics.py index 0da2d05c3..14ec95537 100644 --- a/modules/quicklook/src/diagnostics.py +++ b/modules/quicklook/src/diagnostics.py @@ -17,7 +17,7 @@ def add_headers_dark_current_2D(D2, logger=None): """ - Compute the read noise for dark files and adds keywords to the 2D object headers + Compute the dark current for dark files and adds keywords to the 2D object headers Keywords: FLXREG1G - Dark current [e-/hr] - Green CCD region 1 - coords = [1690:1990,1690:1990] diff --git a/recipes/kpf_masters_drp.recipe b/recipes/kpf_masters_drp.recipe index 3e7e78d21..150a333eb 100644 --- a/recipes/kpf_masters_drp.recipe +++ b/recipes/kpf_masters_drp.recipe @@ -13,11 +13,15 @@ from modules.master_flat.src.master_flat_framework import MasterFlatFramework from modules.master_arclamp.src.master_arclamp_framework import MasterArclampFramework from modules.quality_control.src.quality_control_framework import QualityControlFramework from modules.quicklook.src.diagnostics_framework import DiagnosticsFramework +from modules.calibration_lookup.src.calibration_lookup import CalibrationLookup flat_objects = config.FLAT.flat_objects # Normally standard_flat_object = "autocal-flat-all" standard_flat_object = flat_objects[0] +smoothlamppattern_path = config.FLAT.smoothlamppattern_path +ordermask_path = config.FLAT.ordermask_path + data_type = config.ARGUMENT.data_type overwrite = config.ARGUMENT.overwrite date_dir = context.date_dir @@ -27,6 +31,12 @@ if date_dir is None: else: search_string = config.PICK_INPUTS_MASTERS_DRP.input_dir + "/" + date_dir + "/" + '*.fits' + dt_string = date_dir[0] + date_dir[1] + date_dir[2] + date_dir[3] +\ + "-" + date_dir[4] + date_dir[5] + "-" + date_dir[6] + date_dir[7] + "T00:00:00.000000" + cals = CalibrationLookup(dt_string) + ordermask_path = cals['order_mask'] + smoothlamppattern_path = cals['smooth_lamp_pattern'] + exptime_minimum = config.DARK.exptime_minimum all_bias_files_list,\ @@ -166,8 +176,6 @@ masterdark_path = config.DARK.masterdark_path flat_n_sigma = config.FLAT.n_sigma flat_lev0_ffi_exts = config.FLAT.lev0_ffi_exts masterflat_path = config.FLAT.masterflat_path -smoothlamppattern_path = config.FLAT.smoothlamppattern_path -ordermask_path = config.FLAT.ordermask_path standard_bias_object = "autocal-bias" standard_dark_object = "autocal-dark"