diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ade4cbd --- /dev/null +++ b/.gitignore @@ -0,0 +1,111 @@ +*.pyc +.ipynb_checkpoints/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# ipynb +.ipynb_checkpoints/ +__pycache__/ diff --git a/README.md b/README.md index 479578f..08e5bda 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,4 @@ Tools and tips to process Keck Observatory AO Telemetry - A library of database commands was written to interface to IDL - Extracted telemetry corresponding to NIRC2 images is dumped to IDL ".sav" files for archiving - Basic description: https://www2.keck.hawaii.edu/optics/ao/ngwfc/trsqueries.html + diff --git a/dependencies.txt b/dependencies.txt new file mode 100755 index 0000000..e69de29 diff --git a/katan/compiler.py b/katan/compiler.py new file mode 100644 index 0000000..19b0eaa --- /dev/null +++ b/katan/compiler.py @@ -0,0 +1,238 @@ +### compiler.py : Compiler for all desired data (nirc2, weather, seeing, telemetry, temperature) +### Author : Emily Ramey +### Date : 6/3/21 + +from . import times +from . import strehl +from . import mkwc +from . import telemetry as telem +from . import temperature as temp +from . import templates + +import numpy as np +import pandas as pd +import os + +# Check whether PyYAML is installed +try: + import yaml +except: + raise ValueError("PyYAML not installed. Please install from https://anaconda.org/anaconda/pyyaml") + +### Use importlib for package resources +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +# All data types that can be compiled by this package +accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] + +# Default parameter file for data keywords +default_parfile = 'keyword_defaults.yaml' + +# Shorthand / nicknames for data types +expand = { + 'temp': ['k2AO', 'k2L4', 'k2ENV'], + 'seeing': ['mass', 'dimm', 'masspro'], + 'weather': ['cfht'], + 'telemetry': ['telem'], + 'all': accept_labels, +} + +# Utility functions +def check_dtypes(data_types, file_paths): + """ + Checks user-input data types and file paths and re-formats if necessary. + data_types: list of data types or combined types + file_paths: dictionary with data_type: file_path pairs + returns: list of properly formatted data types + """ + new_dtypes = [] + + # Edge case: only temperature dir specified + if 'temp' in file_paths and os.path.isdir(file_paths['temp']): + file_paths['k2AO'] = file_paths['temp']+'k2AOtemps/' + file_paths['k2L4'] = file_paths['temp']+'k2L4temps/' + file_paths['k2ENV'] = file_paths['temp']+'k2envMet/' + + # Check requested data types + for dtype in data_types: + # Check for nicknames + if dtype in expand: + new_dtypes.extend(expand[dtype]) + elif dtype in accept_labels: # can get data + new_dtypes.append(dtype) + + # Return cleaned list + return new_dtypes + +def load_default_parfile(): + """ + Loads default parameter file from package. + returns: dictionary of dtypes, with sub-dictionaries of data labels:bool/string + """ + try: # Load from templates module + file = pkg_resources.open_text(templates, default_parfile) + default_params = yaml.load(file, Loader=yaml.FullLoader) + except: # Raise error and request param file + raise ValueError("Unable to load default parameters. Please specify a parameter file.") + + return default_params + +def read_params(param_file): + """ + Reads parameters from the specified file or returns default parameters if no file is specified. + param_file: path to parameter file, as a string, or None to return default + returns: dictionary of dtypes, with sub-dictionaries of data labels:bool/string + """ + if param_file is None: # load default + params = load_default_parfile() + elif isinstance(param_file, str) and os.path.exists(param_file): # Load user-specified + try: # Open stream and read as YAML + with open(param_file, 'r') as stream: + params = yaml.load(stream, Loader=yaml.FullLoader) + except: # Unable to load + raise ValueError(f"Failed to load {param_file}. Please check that PyYAML is installed \ + and that the file is formatted correctly.") + else: # Invalid input + raise ValueError(f"{param_file} is not a valid parameter file.") + + return params + +############################################# +######## Interface with other modules ####### +############################################# + +# Have files with acceptable columns for each data type so you can check inputs +# Accept column inputs to the data function and take optional arguments in combine_strehl + +def data_func(dtype, data_dir=None): + """ + Returns the function to get data for the specified dtype + and whether data needs to be matched to nirc2 mjds + """ + # MKWC seeing (mass, dimm, masspro) or weather (cfht) files + if dtype in ['cfht']+expand['seeing']: + return lambda files,mjds: mkwc.from_nirc2(mjds, dtype, data_dir), True + # Temperature files (k2AO, k2L4, or k2ENV) + if dtype in expand['temp']: + return lambda files,mjds: temp.from_mjds(mjds, dtype, data_dir), True + # Telemetry files (matched to NIRC2 filenames) + if dtype=='telem': + return lambda files,mjds: telem.from_nirc2(mjds, files, data_dir), False + +def change_cols(data, params): + """ + Changes / filters columns according to the parameters passed. + data: a dataframe with columns for a certain dtype + params: a list of column names to values + A False value means the column will be omitted + A True value means the column will be included as-is + A string value means the column will be re-named to that value + """ + # Drop bad columns first + good_cols = [col for col,val in params.items() if (val and col in data.columns)] + new_data = data[good_cols].copy() + + # Re-map column names + col_mapper = {col: new_col for col,new_col in params.items() if isinstance(new_col, str)} + new_data.rename(columns=col_mapper, inplace=True) + + return new_data + +################################ +###### Compiler Functions ###### +################################ + +def match_data(mjd1, mjd2): + """ + Matches secondary MJD values to those of their closest primary observations. + mjd1: Nx1 array/series of primary MJDs + mjd2: Mx1 array/series of secondary MJDs + returns: Nx1 array of indices in mjd2 which best match entries of mjd1 + """ + # Edge case + if mjd1.empty or mjd2.empty: + return None + + # Get mjds from dataframes + mjd1 = np.array(mjd1).reshape(-1,1) # column vector + mjd2 = np.array(mjd2).reshape(1,-1) # row vector + + # Take difference of mjd1 with mjd2 + diffs = np.abs(mjd1 - mjd2) + # Find smallest difference for each original mjd + idxs = np.argmin(diffs, axis=1) + + # Return indices of matches in mjd2 + return idxs + + +# data_types can contain: 'chft', 'mass', 'dimm', 'masspro', +# 'telem', 'k2AO', 'k2L4', or 'k2ENV' +# 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and +# ['mass', 'dimm', 'masspro'], respectively +def combine_strehl(strehl_file, data_types, file_paths={}, test=False, + param_file=None): + """ + Combines and matches data from a certain Strehl file with other specified data types. + NIRC2 files must be in the same directory as the Strehl file. + strehl_file: a string containing the file path for a Strehl file + data_types: a list of data types to match with the Strehl file + file_paths: a dictionary mapping data types to file paths for the relevant data + test: bool, if True the compiler will match only the first 3 lines of the Strehl file + param_file: string, path to a parameter file (see templates/keyword_defaults.yaml) + if None, default parameters will be used + returns: a fully matched dataset with Strehl, NIRC2, and other secondary data + """ + ### Check file paths parameter dict, load if yaml + if not isinstance(file_paths, dict) and os.path.isfile(file_paths): + with open(file_paths) as file: + file_paths = yaml.load(file, loader=yaml.FullLoader) + # Check file paths are valid + for dtype, data_dir in file_paths.items(): + if not os.path.isdir(data_dir): + raise ValueError((f"The path specified for {dtype} data ({data_dir}) " + "is not a valid directory.")) + + ### Sanitize data types + data_types = check_dtypes(data_types, file_paths) + + ### Read in parameter file + params = read_params(param_file) + + ### Read in Strehl file + nirc2_data = strehl.from_strehl(strehl_file) + + if test: # Take only first few files + nirc2_data = nirc2_data.loc[:3] + + # Full data container + all_data = [nirc2_data.reset_index(drop=True)] + + # Loop through and get data + for dtype in data_types: + # Get data directory + data_dir = file_paths[dtype] if dtype in file_paths else None + # Fetch data function from dictionary + get_data, match = data_func(dtype, data_dir) # Data retrieval function + # Get other data from strehl info + other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) + + # Change or omit selected columns + other_data = change_cols(other_data, params[dtype]) + + if match: # Needs to be matched + if other_data.empty: # No data found + other_data = pd.DataFrame(columns=other_data.columns, index=range(len(nirc2_data))) + else: # Get indices of matched data + idxs = match_data(nirc2_data.nirc2_mjd, other_data[dtype+'_mjd']) + other_data = other_data.iloc[idxs] + + # Add to all data + all_data.append(other_data.reset_index(drop=True)) + + # Concatenate new data with nirc2 + return pd.concat(all_data, axis=1) \ No newline at end of file diff --git a/katan/mkwc.py b/katan/mkwc.py new file mode 100644 index 0000000..6aa7edc --- /dev/null +++ b/katan/mkwc.py @@ -0,0 +1,198 @@ +### mkwc_util.py : Contains utilities for extracting and processing data from the MKWC website +### Author : Emily Ramey +### Date : 6/1/2021 + +import os +import numpy as np +import pandas as pd +from . import times + +### NOTE: Seeing data is STORED by UT, with data in HST +### CFHT data is STORED by HST, with data in HST + +mkwc_url = 'http://mkwc.ifa.hawaii.edu/' +year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' +# Cutoff for daily-vs-yearly CFHT files on MKWC website +cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST + +# Time columns from MKWC data +time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] + +# Data-specific fields +data_types = { + 'cfht': { + 'web_pat': mkwc_url+'archive/wx/cfht/indiv-days/cfht-wx.{}.dat', + 'file_pat': "{}cfht-wx.{}.dat", + 'data_cols': ['wind_speed', 'wind_direction', 'temperature', + 'relative_humidity', 'pressure'], + }, + 'mass': { + 'web_pat': mkwc_url+'current/seeing/mass/{}.mass.dat', + 'file_pat': '{}mass/{}.mass.dat', + 'data_cols': ['mass'], + }, + 'dimm': { + 'web_pat': mkwc_url+'current/seeing/dimm/{}.dimm.dat', + 'file_pat': '{}dimm/{}.dimm.dat', + 'data_cols': ['dimm'], + }, + 'masspro': { + 'web_pat': mkwc_url+'current/seeing/masspro/{}.masspro.dat', + 'file_pat': '{}masspro/{}.masspro.dat', + 'data_cols': ['masspro_half', 'masspro_1', 'masspro_2', + 'masspro_4', 'masspro_8', 'masspro_16', 'masspro'], + }, +} + +# Mix and match data & time columns +for dtype in data_types: + # CFHT files don't have seconds + tcols = time_cols if dtype != 'cfht' else time_cols[:-1] + # Format web columns + data_types[dtype]['web_cols'] = tcols+data_types[dtype]['data_cols'] + # Format file columns + data_types[dtype]['cols'] = [dtype+'_mjd']+data_types[dtype]['data_cols'] + # Different file storage timezones + data_types[dtype]['file_zone'] = 'hst' if dtype=='cfht' else 'utc' + +############################# +######### Functions ######### +############################# + +def cfht_from_year(datestrings, year): + """ + Gets pre-2012 MKWC data from the year-long file instead of the by-date files. + datestrings: dates to pull data from, as {yyyymmdd} strings + year: the year to pull datestring data from + returns: dataframe with cfht data from requested dates + """ + # Get year-long file URL + url = year_url.format(year) + + # Read in data + try: + web_cols = data_types['cfht']['web_cols'] + year_data = pd.read_csv(url, delim_whitespace=True, header=None, + names=web_cols, usecols=range(len(web_cols))) + except: # No data, return blank + return pd.DataFrame(columns=data_types['cfht']['cols']) + + # Full dataset + all_data = [pd.DataFrame(columns=data_types['cfht']['cols'])] + + # Slice up dataframe + for ds in datestrings: + month, day = int(ds[4:6]), int(ds[6:]) + # Get data by month and day + df = year_data.loc[(year_data.month==month) & (year_data.day==day)].copy() + # Format columns + if not df.empty: + format_columns(df, 'cfht') + # Add to full dataset + all_data.append(df) + + return pd.concat(all_data) + +def format_columns(df, dtype): + """ + Changes columns (in place) from date/time to MJD. + df: MKWC dataframe with date and time columns + dtype: specific data type (mass, dimm, masspro, cfht) + """ + # Get MJDs from HST values + mjds = times.table_to_mjd(df, columns=time_cols, zone='hst') + df[dtype+'_mjd'] = mjds + + # Drop old times + df.drop(columns=time_cols, inplace=True, errors='ignore') + +def from_url(datestring, dtype): + """ + Pulls weather or seeing file from MKWC website. + datestring: date to pull data for, in {yyyymmdd} format + dtype: cfht, mass, dimm, or masspro + returns: dataframe with MKWC data + """ + # Format URL + url = data_types[dtype]['web_pat'].format(datestring) + + # Read data + try: # Check if data is there + web_cols = data_types[dtype]['web_cols'] # MKWC Weather columns + df = pd.read_csv(url, delim_whitespace = True, header=None, + names=web_cols, usecols=range(len(web_cols))) + except: # otherwise return blank df + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Get mjd from time columns + format_columns(df, dtype) + + return df + +def from_file(filename, dtype, data_dir='./'): + """ + Pulls weather or seeing file from local directory + """ + # Read in CSV + df = pd.read_csv(filename) + + # Check formatting + if 'mjd' in df.columns: + df.rename(columns={'mjd':dtype+'_mjd'}, inplace=True) + elif dtype+'_mjd' not in df.columns: # No MJD + try: # Change to MJD, if times + format_columns(df, dtype) + except: # No time info, return blank + return pd.DataFrame() + + return df + +def from_nirc2(mjds, dtype, data_dir='./'): + """ + Compiles a list of cfht or seeing observations based on MJDs + mjds: list of NIRC2 MJDs to pull data from (assumes one per day) + dtype: cfht, mass, dimm, or masspro + data_dir: location of downloaded weather/seeing files, if applicable + returns: dataframe containing all data from MJDs specified + """ + + # Get datestrings + dts = times.mjd_to_dt(mjds, zone=data_types[dtype]['file_zone']) + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + # No duplicates + datestrings = pd.Series(np.unique(datestrings)) + + # Blank data structure + all_data = [pd.DataFrame(columns=data_types[dtype]['cols'])] + + # Check for pre-2012 cfht files + if dtype=='cfht' and any(mjds < cfht_cutoff): + # Get datetimes + pre_2012 = datestrings[mjds < cfht_cutoff] + + # Compile data by year + for yr in np.unique(pre_2012.str[:4]): + ds = pre_2012[pre_2012.str[:4]==yr] + df = cfht_from_year(ds, yr) + # Append to full dataset + all_data.append(df) + + # Find data for each file + for ds in datestrings: + # Get local filename + filename = data_types[dtype]['file_pat'].format(data_dir, ds) + + # Check for local files + if os.path.isfile(filename): + df = from_file(filename, dtype) + + else: # Pull from the web + df = from_url(ds, dtype) + + # Save to local file: TODO + + # Add to data + all_data.append(df) + + # Return concatenated dataframe + return pd.concat(all_data, ignore_index=True) \ No newline at end of file diff --git a/katan/strehl.py b/katan/strehl.py new file mode 100644 index 0000000..8810199 --- /dev/null +++ b/katan/strehl.py @@ -0,0 +1,68 @@ +### nirc2_util.py : Functions for processing nirc2 data and strehl files +### Author : Emily Ramey +### Date : 6/2/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits + +from . import times + +strehl_cols = ['nirc2_file', 'strehl', 'rms_err', 'fwhm', 'nirc2_mjd'] +default_keys = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', + 'AOLBFWHM', 'WSFRRT', 'LSAMPPWR', 'LGRMSWF', 'AOAOAMED', 'TUBETEMP'] + +def from_filename(nirc2_file, data, i, header_kws): + """ + Gets nirc2 header values from a filename and loads values (in place) into data. + nirc2_file: string, path to nirc2 file to load + data: partially filled dataframe (line i will be overwritten) + i: index at which to load NIRC2 data + header_kws: header keywords to load into data + """ + # Check for valid file + if not os.path.isfile(nirc2_file): + return + # Open nirc2 file + with fits.open(nirc2_file) as file: + nirc2_hdr = file[0].header + + # Get fields from header + for kw in header_kws: + # load DataFrame value + data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) + +def from_strehl(strehl_file, header_kws=default_keys): + """ + Gets NIRC2 header data based on contents of a Strehl file. + strehl_file: string, path to Strehl file + header_kws: header keywords to pull from NIRC2 data files + returns: dataframe with Strehl and NIRC2 data + """ + # Get directory name + data_dir = os.path.dirname(strehl_file) + # Retrieve Strehl data + strehl_data = pd.read_csv(strehl_file, delim_whitespace = True, + header = None, skiprows = 1, names=strehl_cols) + # Add true file path + strehl_data['nirc2_file'] = data_dir + "/" + strehl_data['nirc2_file'] + + # Add decimal year + strehl_data['dec_year'] = times.mjd_to_yr(strehl_data.nirc2_mjd) + + # Add nirc2 columns + for col in header_kws: + strehl_data[col.lower()] = np.nan + + # Loop through nirc2 files + for i,nirc2_file in enumerate(strehl_data.nirc2_file): + # Load header data into df + from_filename(nirc2_file, strehl_data, i, header_kws) + + # Return data + return strehl_data + + + diff --git a/katan/telemetry.py b/katan/telemetry.py new file mode 100644 index 0000000..f6c5559 --- /dev/null +++ b/katan/telemetry.py @@ -0,0 +1,221 @@ +### telem_util.py: For code that interacts with Keck AO Telemetry files +### Author: Emily Ramey +### Date: 11/18/2020 + +import numpy as np +import pandas as pd +import glob +from scipy.io import readsav + +from . import times +from . import templates + +### For importing package files +### May need to edit this later if there are import issues with Py<37 +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +### Path to telemetry files [EDIT THIS] +# telem_dir = '/g/lu/data/keck_telemetry/' + +# Sub-aperture maps +wfs_file = "sub_ap_map.txt" +act_file = "act.txt" + +# regex to match telem filenumbers +filenum_match = ".*c(\d+).fits" +filename_match = "{}{}/**/n?{}*LGS*.sav" + +TIME_DELTA = 0.001 # About 100 seconds in mjd + +RESID_CUTOFF = 349 +TT_IDXS = [349,350] +DEFOCUS = 351 +LAMBDA = 2.1 # microns + +cols = ['telem_file', + 'telem_mjd', + 'TT_mean', + 'TT_std', + 'DM_mean', + 'DM_std', + 'rmswfe_mean', + 'rmswfe_std', + 'strehl_telem'] + +def read_map(filename): + """ Reads a map of actuators or sub-apertures from a file """ + file = pkg_resources.open_text(templates, filename) + return pd.read_csv(file, delim_whitespace=True, header=None).to_numpy() + +def get_times(telem_data, start=None): + """ Pulls timestamps from a telemetry file in seconds from start """ + if start is None: + start = telem_data.a.timestamp[0][0] + + return (telem_data.a.timestamp[0]-start)/1e7 + +def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): + """ + Returns the locations of the valid actuators in the actuator array + resids: Nx349 residual wavefront array (microns) + ints: Nx304 intensity array (any units) + N: Number of timestamps + """ + # Check inputs + N = ints.shape[0] # Num timestamps + + # Aggregate intensities over all timestamps + med_ints = np.median(ints, axis=0) + + # Fill WFS map with aggregated intensities + int_map = wfs_map.copy() + int_map[np.where(int_map==1)] = med_ints + + # Find lenslets with greatest intensity + idxs = np.flip(np.argsort(int_map, axis=None))[:num_aps] # flat idxs of sort + idxs = np.unravel_index(idxs, wfs_map.shape) # 2D idxs of sort + + # Mask for good sub-ap values + good_aps = np.zeros(wfs_map.shape, dtype=int) + good_aps[idxs] = 1 + good_aps = good_aps * wfs_map # Just in case + + # Mask for good actuator values + good_acts = np.pad(good_aps, ((1,1),(1,1))) + good_acts = (good_acts[1:,1:] | good_acts[1:,:-1] + | good_acts[:-1,:-1] | good_acts[:-1,1:]) * act_map + + return good_acts + +def tt2um(tt_as): + """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ + D = 10.5e6 # telescope diameter in microns + tt = tt_as*4.8e-6 # TT error in radians + tt_err = np.sqrt(D**2 / 12 * (tt[:,0]**2 + tt[:,1]**2)) + return tt_err + +def rms_acts(act_resids, ints): + """ Clips bad actuators and averages for each timestamp """ + N = act_resids.shape[0] # num timestamps + + # Read in actuator map + act_map = read_map(act_file) + + # Get good actuator mask from intensities + mask = resid_mask(ints) + flat_map = ~mask[np.where(act_map==1)].astype(bool) # flatten + flat_map = np.tile(flat_map, (N,1)) + + # Mask the bad actuators + good_acts = np.ma.masked_array(act_resids, flat_map) + # Average resids for each timestep + act_rms = np.sqrt((good_acts**2).mean(axis=1) - good_acts.mean(axis=1)**2) + + return act_rms.compressed() + +######################################## +######### Processing Files ############# +######################################## + +def get_mjd(telem): + """ + Validates a telemetry file against an MJD value. + telem: structure returned from readsav + returns: MJD of telemetry file start + """ + # Get timestamp + tstamp = telem.tstamp_str_start.decode('utf-8') + # Convert to MJD + telem_mjd = times.str_to_mjd(tstamp, fmt='isot') + + # Returns telem mjd + return telem_mjd + +def extract_telem(file, data, idx, check_mjd=None): + """ + Extracts telemetry values from a file to a dataframe. + file: file path to telemetry, as string + data: dataframe to load new telemetry into + idx: index in dataframe receive new telemetry data + check_mjd: MJD to match with telemetry file (no matching if none) + """ + # Read IDL file + telem = readsav(file) + + # Make sure MJD matches + telem_mjd = get_mjd(telem) + if check_mjd is not None: + delta = np.abs(telem_mjd-check_mjd) + if delta > TIME_DELTA: + return False + + # Get residuals and intensities + act_resids = telem.a.residualwavefront[0][:, :RESID_CUTOFF] + tt_resids = telem.a.residualwavefront[0][:,TT_IDXS] + ints = telem.a.subapintensity[0] # Sub-aperture intensities + + # Convert TT resids to microns + tt_microns = tt2um(tt_resids) + # Get RMS resids from the actuator array + act_rms = rms_acts(act_resids, ints) + + # Total RMS Wavefront Error + rmswfe = np.sqrt(tt_microns**2 + act_rms**2) + + # Strehl calculation + strehl = np.exp(-(2*np.pi*rmswfe/LAMBDA)**2) + + # Assemble aggregate data + data.loc[idx, cols] = [ + file, + telem_mjd, + np.mean(tt_microns), + np.std(tt_microns), + np.mean(act_rms), + np.std(act_rms), + np.mean(rmswfe), + np.std(rmswfe), + np.mean(strehl), + ] + + return True + +def from_nirc2(mjds, nirc2_filenames, telem_dir): + """ + Gets a table of telemetry information from a set of mjds and NIRC2 filenames. + mjds: NIRC2 mjds to match to telemetry files + nirc2_filenames: NIRC2 filenames to match to telemetry files + telem_dir: path to directory containing telemetry files + returns: dataframe of telemetry, in same order as NIRC2 MJDs passed + """ + N = len(mjds) # number of data points + # Get file numbers + filenums = nirc2_filenames.str.extract(filenum_match, expand=False) + filenums = filenums.str[1:] # First digit doesn't always match + + # Get datestrings + dts = times.mjd_to_dt(mjds) # HST or UTC??? + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + + # Set up dataframe + data = pd.DataFrame(columns=cols, index=range(N)) + + # Find telemetry for each file + for i in range(N): + # Get filename and number + fn, ds, mjd = filenums[i], datestrings[i], mjds[i] + # Search for correct file + file_pat = filename_match.format(telem_dir, ds, fn) + all_files = glob.glob(file_pat, recursive=True) + + # Extract the first file that matches the MJD to data + for file in all_files: + success = extract_telem(file, data, i, check_mjd=mjd) + if success: break + + return data + \ No newline at end of file diff --git a/katan/temperature.py b/katan/temperature.py new file mode 100644 index 0000000..654b0e6 --- /dev/null +++ b/katan/temperature.py @@ -0,0 +1,288 @@ +### temp_util.py: Functions relating to Keck-II temperature data +### Main reads in Keck II temperature files (AO, env, and L4) and saves each to a FITS file +### Author: Emily Ramey +### Date: 01/04/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits +from astropy.table import Table + +from . import times + +verbose = True + +# data_dir = "/u/emily_ramey/work/Keck_Performance/data/" +# temp_dir = data_dir+"temp_data_2/" + +col_dict = { + 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV + 'AO_benchT2': 'k1:ao:env:benchtemp2_Raw', + 'k1:ao:env:elect_vault_t': 'k1:ao:env:elect_vault:T', + 'k0:met:humidityStats.VAL': 'k0:met:humidityStats', + "k2:ao:env:ETroomtemp_Raw": "El_roomT", # k2AO + "k2:ao:env:DMracktemp_Raw": "DM_rackT", + "k2:ao:env:KCAMtemp2_Raw": "KCAM_T2", + "k2:ao:env:OBrighttemp_Raw": "OB_rightT", + "k2:ao:env:phototemp_Raw": "photometricT", + "k2:ao:env:OBlefttemp_Raw": "OB_leftT", + "k2:ao:env:KCAMtemp1_Raw": "KCAM_T1", + "k2:ao:env:ACAMtemp_Raw": "AOA_camT", + "k2:ao:env:LStemp_Raw": "LGS_temp", # k2L4 + "k2:ao:env:LSenchumdity_Raw": "LGS_enclosure_hum", + "k2:ao:env:LShumidity_Raw": "LGS_humidity", + "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" +} + +# file_pats = { # File name patterns +# # 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", +# # 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", +# # 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", +# 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", +# 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", +# 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", +# } + +data_types = { + 'k2AO': { + 'file_pat': "{}{}/AO_temps.log", + 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', + 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', + 'k2AO_mjd'], + }, + 'k2L4': { + 'file_pat': "{}/{}/L4_env.log", + 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', + 'LGS_temp', 'k2L4_mjd'], + }, + 'k2ENV': { + 'file_pat': "{}/{}/envMet.arT", + 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', + 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', + 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', + 'k0:met:out:windSpeedMaxStats', 'k0:met:out:windSpeedMaxmph', + 'k0:met:out:windSpeedmph', 'k0:met:outTempDwptDiff', + 'k0:met:pressureRaw', 'k0:met:pressureStats', 'k0:met:tempMax', + 'k0:met:tempMin', 'k0:met:tempRaw', 'k2:dcs:sec:acsDwptDiff', + 'k2:dcs:sec:acsTemp', 'k2:dcs:sec:secDwptDiff', 'k2:dcs:sec:secondaryTemp', + 'k2:met:humidityRaw','k2:met:humidityStats', 'k2:met:tempRaw', 'k2:met:tempStats', + 'k2:met:windAzRaw', 'k2:met:windElRaw', 'k2:met:windSpeedMaxmph', + 'k2:met:windSpeedMinmph', 'k2:met:windSpeedRaw', 'k2:met:windSpeedStats', + 'k2:met:windSpeedmph', 'k2ENV_mjd'], + } +} + +# def get_columns(): +# all_data = search_files() +# for name, data_files in all_data.items(): +# columns = set() +# for i,file in enumerate(data_files): +# try: +# df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, +# na_values=['***'], error_bad_lines=False, warn_bad_lines=False, +# ).replace('"', regex=True) +# except: +# if verbose: +# print(f"Warning: read failed for file {file}") +# continue + +# if len(df.columns)==1: # no header +# if verbose: +# print(f"Skipping file {file}, no header") +# continue # skip for now +# df.columns = [col.replace('"', '').strip() for col in df.columns] +# if "UNIXDate" not in df.columns: +# if verbose: +# print(f"Skipping file {file}, columns not as expected") +# continue # skip for now +# for col in df.columns: +# columns.add(col) +# all_data[name] = columns + +# return all_data + + +# def search_files(file_pats=file_pats): +# """ Finds all filenames matching the given file patterns """ +# all_filenames = {} +# for name, search in file_pats.items(): + +# filenames = glob.glob(search, recursive=True) +# all_filenames[name] = filenames +# if verbose: +# print(f"Done {name}, length: {len(filenames)}") + +# return all_filenames + +def collect_data(data_files, col_dict=col_dict): + """ + Takes a list of file names and reads them into a pandas dataframe. + data_files: list of filenames + col_dict: dictionary of columns to rename + returns: dataframe of temperature data from files + """ + if isinstance(data_files, dict): # Return dict w/dataframes + new_files = {} + for name, files in data_files.items(): + if isinstance(files, list): + # Recurse on each list of files + new_files[name] = collect_data(files) + return new_files + + all_dfs = [pd.DataFrame()] + for i,file in enumerate(data_files): + if not os.path.isfile(file): + continue + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + + if col_dict is not None: + df = df.rename(columns=col_dict) + + all_dfs.append(df) + + data = pd.concat(all_dfs, ignore_index=True, sort=True) + return data + +def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): + """ + Parses specified date and time columns and returns a cleaned data table. + """ + new_data = data.copy() + for label,cols in date_cols.items(): + date_col, time_col = cols + # Parse dates and times, coercing invalid strings to NaN + datetimes = (pd.to_datetime(data[date_col], exact=False, errors='coerce') + + pd.to_timedelta(data[time_col], errors='coerce')) + new_data = new_data.drop(columns=cols) + new_data[label] = datetimes + + return new_data + +def clean_dates(data, date_cols=['HST', 'UNIX']): + """ Removes any rows containing invalid dates in date_cols """ + new_data = data.copy() + for col in date_cols: + new_data = new_data[~np.isnan(new_data[col])] + + return new_data + +def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): + """ Casts columns to a numeric data type """ + if data_cols is None: + data_cols = [col for col in data.columns if col not in non_numeric] + + # Cast data to numeric type, coercing invalid values to NaN + new_data = data.copy() + for col in data_cols: + new_data[col] = pd.to_numeric(new_data[col], errors='coerce') + + return new_data + +# def to_fits(data, filename, str_cols=['HST', 'UNIX']): +# """ Writes a FITS file from the given temperature array """ +# fits_data = data.copy() +# for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: +# if col in fits_data.columns: +# fits_data = fits_data.drop(columns=[col]) +# for col in str_cols: +# if col in fits_data.columns: +# fits_data[col] = fits_data[col].astype(str) +# # Assuming the data columns are already numeric +# fits_data = Table.from_pandas(fits_data) +# fits_data.write(filename) + +# return + +# def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): +# """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ +# data = Table.read(filename).to_pandas() + +# # Fix NaNs, because astropy is dumb sometimes +# data[data==1e+20] = np.nan + +# if date_cols is None: return data + +# for col in date_cols: +# if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings +# data[col] = data[col].str.decode("utf-8") +# data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') + +# if str_cols is None: return data + +# for col in str_cols: +# if col in data.columns and isinstance(data[col][0], bytes): +# data[col] = data[col].str.decode("utf-8") + +# return data + +# def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): +# """ +# Reads in all data matching file patterns (file_pats), combines them into one table, +# cleans them, and saves them to a FITS file +# """ +# # Find all files matching pattern +# all_filenames = search_files(file_pats) +# # Read all data into one table (per dictionary label) +# all_data = collect_data(all_filenames) + +# for name, data in all_data.items(): +# data = parse_dates(data) # Parse date cols into datetimes +# data = clean_dates(data) # Remove invalid dates +# data = clean_data(data) # Casts other cols to numeric +# # Save combined/cleaned data to FITS file +# filename = location+name+".fits" +# to_fits(data, filename) + +# return + +def from_mjds(mjds, dtype, data_dir): + """ + Gets temp data of input type from the specified MJDs. + mjds: list of Modified Julian Dates to pull data from + dtype: k2AO, k2L4, or k2ENV file type + data_dir: path to directory containing temperature data + returns: dataframe of relevant temperature data + """ + # Get pd datetimes in HST + dts = times.mjd_to_dt(mjds, zone='hst') + # Format list + datestrings = dts.strftime("%y/%m/%d") + datestrings = np.unique(datestrings) # one file per date + # Get relevant filenames + filenames = [data_types[dtype]['file_pat'].format(data_dir, ds) for ds in datestrings] + # Get data from filenames + data = collect_data(filenames) + + # Empty dataframe + if data.empty: + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Convert dates & times to MJDs + data["datetime"] = data['HSTdate']+' '+data['HSTtime'] + mjds = times.table_to_mjd(data, columns='datetime', zone='hst') + # Add to dataframe + data[dtype+"_mjd"] = mjds + # Drop other date & time cols + data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], + inplace=True, errors='ignore') + + return data \ No newline at end of file diff --git a/katan/templates/.ipynb_checkpoints/__init__-checkpoint.py b/katan/templates/.ipynb_checkpoints/__init__-checkpoint.py new file mode 100755 index 0000000..e69de29 diff --git a/katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml b/katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml new file mode 100644 index 0000000..26925a8 --- /dev/null +++ b/katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml @@ -0,0 +1,127 @@ +### This is the default YAML file for the KATAN compiler +### Note: columns are not in the order they are read in from data +### Change 'True' to an alias for the column, if desired, or set to False to exclude + +--- +### Strehl file columns +strehl: + 'nirc2_mjd': True # required + 'nirc2_file': True + 'strehl': True + 'rms_err': True + 'fwhm': True + +### NIRC2 file columns +nirc2: + 'AIRMASS': 'airmass' + 'ITIME': 'itime' + 'COADDS': 'coadds' + 'FWINAME': 'fwiname' + 'AZ': 'az' + 'DMGAIN': 'dmgain' + 'DTGAIN': 'dtgain' + 'AOLBFWHM': 'aolbfwhm' + 'WSFRRT': 'wsfrrt' + 'LSAMPPWR': 'lsamppwr' + 'LGRMSWF': 'lgrmswf' + 'AOAOAMED': 'aoaomed' + 'TUBETEMP': 'tubetemp' + +### Telemetry file columns +telem: + 'telem_file': True + 'telem_mjd': True # required + 'TT_mean': True + 'TT_std': True + 'DM_mean': True + 'DM_std': True + 'rmswfe_mean': True + 'rmswfe_std': True + 'strehl_telem': True + +### Weather file columns +cfht: + 'cfht_mjd': True # required + 'wind_speed': True + 'wind_direction': True + 'temperature': True + 'relative_humidity': True + 'pressure': True + +### MASS seeing columns +mass: + 'mass_mjd': True # required + 'mass': True + +### DIMM seeing columns +dimm: + 'dimm_mjd': True # required + 'dimm': True + +### MASSPRO seeing columns +masspro: + 'masspro_mjd': True # required + 'masspro': True + 'masspro_half': True + 'masspro_1': True + 'masspro_2': True + 'masspro_4': True + 'masspro_8': True + 'masspro_16': True + +### k2AO temperature columns +k2AO: + 'k2AO_mjd': True # required + 'AOA_camT': True + 'DM_rackT': False + 'El_roomT': False + 'KCAM_T1': False + 'KCAM_T2': False + 'OB_leftT': "k2ao_nirc2_temp" + 'OB_rightT': "k2ao_wfs_temp" + 'photometricT': True + +### k2L4 temperature columns +k2L4: + 'k2L4_mjd': True # required + 'LGS_enclosure_hum': False + 'LGS_enclosure_temp': True + 'LGS_humidity': False + 'LGS_temp': True + +### k2ENV temperature columns +k2ENV: + 'k2ENV_mjd': True # required + 'k0:met:dewpointMax': False + 'k0:met:dewpointMin': False + 'k0:met:dewpointRaw': False + 'k0:met:humidityRaw': "k0_humidity" + 'k0:met:humidityStats': False + 'k0:met:out:windDirection': "k0_wind_dir" + 'k0:met:out:windDirectionMax': False + 'k0:met:out:windDirectionMin': False + 'k0:met:out:windSpeedMaxStats': False + 'k0:met:out:windSpeedMaxmph': False + 'k0:met:out:windSpeedmph': "k0_wind_speed" + 'k0:met:outTempDwptDiff': False + 'k0:met:pressureRaw': "k0_pressure" + 'k0:met:pressureStats': False + 'k0:met:tempMax': False + 'k0:met:tempMin': False + 'k0:met:tempRaw': "k0_temperature" + 'k2:dcs:sec:acsDwptDiff': False + 'k2:dcs:sec:acsTemp': "k2_primary_temp" + 'k2:dcs:sec:secDwptDiff': False + 'k2:dcs:sec:secondaryTemp': "k2_secondary_temp" + 'k2:met:humidityRaw': "k2_dome_humidity" + 'k2:met:humidityStats': False + 'k2:met:tempRaw': "k2_dome_temp" + 'k2:met:tempStats': False + 'k2:met:windAzRaw': False + 'k2:met:windElRaw': False + 'k2:met:windSpeedMaxmph': False + 'k2:met:windSpeedMinmph': False + 'k2:met:windSpeedRaw': False # investigate + 'k2:met:windSpeedStats': False + 'k2:met:windSpeedmph': False +... \ No newline at end of file diff --git a/katan/templates/__init__.py b/katan/templates/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/katan/templates/act.txt b/katan/templates/act.txt new file mode 100755 index 0000000..90ea449 --- /dev/null +++ b/katan/templates/act.txt @@ -0,0 +1,21 @@ +0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/katan/templates/keyword_defaults.yaml b/katan/templates/keyword_defaults.yaml new file mode 100644 index 0000000..26925a8 --- /dev/null +++ b/katan/templates/keyword_defaults.yaml @@ -0,0 +1,127 @@ +### This is the default YAML file for the KATAN compiler +### Note: columns are not in the order they are read in from data +### Change 'True' to an alias for the column, if desired, or set to False to exclude + +--- +### Strehl file columns +strehl: + 'nirc2_mjd': True # required + 'nirc2_file': True + 'strehl': True + 'rms_err': True + 'fwhm': True + +### NIRC2 file columns +nirc2: + 'AIRMASS': 'airmass' + 'ITIME': 'itime' + 'COADDS': 'coadds' + 'FWINAME': 'fwiname' + 'AZ': 'az' + 'DMGAIN': 'dmgain' + 'DTGAIN': 'dtgain' + 'AOLBFWHM': 'aolbfwhm' + 'WSFRRT': 'wsfrrt' + 'LSAMPPWR': 'lsamppwr' + 'LGRMSWF': 'lgrmswf' + 'AOAOAMED': 'aoaomed' + 'TUBETEMP': 'tubetemp' + +### Telemetry file columns +telem: + 'telem_file': True + 'telem_mjd': True # required + 'TT_mean': True + 'TT_std': True + 'DM_mean': True + 'DM_std': True + 'rmswfe_mean': True + 'rmswfe_std': True + 'strehl_telem': True + +### Weather file columns +cfht: + 'cfht_mjd': True # required + 'wind_speed': True + 'wind_direction': True + 'temperature': True + 'relative_humidity': True + 'pressure': True + +### MASS seeing columns +mass: + 'mass_mjd': True # required + 'mass': True + +### DIMM seeing columns +dimm: + 'dimm_mjd': True # required + 'dimm': True + +### MASSPRO seeing columns +masspro: + 'masspro_mjd': True # required + 'masspro': True + 'masspro_half': True + 'masspro_1': True + 'masspro_2': True + 'masspro_4': True + 'masspro_8': True + 'masspro_16': True + +### k2AO temperature columns +k2AO: + 'k2AO_mjd': True # required + 'AOA_camT': True + 'DM_rackT': False + 'El_roomT': False + 'KCAM_T1': False + 'KCAM_T2': False + 'OB_leftT': "k2ao_nirc2_temp" + 'OB_rightT': "k2ao_wfs_temp" + 'photometricT': True + +### k2L4 temperature columns +k2L4: + 'k2L4_mjd': True # required + 'LGS_enclosure_hum': False + 'LGS_enclosure_temp': True + 'LGS_humidity': False + 'LGS_temp': True + +### k2ENV temperature columns +k2ENV: + 'k2ENV_mjd': True # required + 'k0:met:dewpointMax': False + 'k0:met:dewpointMin': False + 'k0:met:dewpointRaw': False + 'k0:met:humidityRaw': "k0_humidity" + 'k0:met:humidityStats': False + 'k0:met:out:windDirection': "k0_wind_dir" + 'k0:met:out:windDirectionMax': False + 'k0:met:out:windDirectionMin': False + 'k0:met:out:windSpeedMaxStats': False + 'k0:met:out:windSpeedMaxmph': False + 'k0:met:out:windSpeedmph': "k0_wind_speed" + 'k0:met:outTempDwptDiff': False + 'k0:met:pressureRaw': "k0_pressure" + 'k0:met:pressureStats': False + 'k0:met:tempMax': False + 'k0:met:tempMin': False + 'k0:met:tempRaw': "k0_temperature" + 'k2:dcs:sec:acsDwptDiff': False + 'k2:dcs:sec:acsTemp': "k2_primary_temp" + 'k2:dcs:sec:secDwptDiff': False + 'k2:dcs:sec:secondaryTemp': "k2_secondary_temp" + 'k2:met:humidityRaw': "k2_dome_humidity" + 'k2:met:humidityStats': False + 'k2:met:tempRaw': "k2_dome_temp" + 'k2:met:tempStats': False + 'k2:met:windAzRaw': False + 'k2:met:windElRaw': False + 'k2:met:windSpeedMaxmph': False + 'k2:met:windSpeedMinmph': False + 'k2:met:windSpeedRaw': False # investigate + 'k2:met:windSpeedStats': False + 'k2:met:windSpeedmph': False +... \ No newline at end of file diff --git a/katan/templates/sub_ap_map.txt b/katan/templates/sub_ap_map.txt new file mode 100755 index 0000000..4dfc98e --- /dev/null +++ b/katan/templates/sub_ap_map.txt @@ -0,0 +1,20 @@ +0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/katan/times.py b/katan/times.py new file mode 100644 index 0000000..fe42060 --- /dev/null +++ b/katan/times.py @@ -0,0 +1,76 @@ +### time_util.py : to handle time formatting changes between MJD, HST, and UTC +### Author : Emily Ramey +### Date: 5/26/21 + +# Preamble +import pandas as pd +import numpy as np +import time +import pytz as tz +from datetime import datetime, timezone +from astropy.time import Time, TimezoneInfo +from astropy import units as u, constants as c + +hst = tz.timezone('US/Hawaii') + +### Conversion / utility functions +def mjd_to_dt(mjds, zone='utc'): + """ + Converts MJDs to HST or UTC date. + mjds: list or series of Modified Julian Dates + zone: time zone ('utc' or 'hst') to return + returns: datetime objects in the given time zone + """ + # Convert mjds -> astropy times -> datetimes + dts = Time(mjds, format='mjd', scale='utc').to_datetime() + # Convert to pandas + dts = pd.to_datetime(dts).tz_localize(tz.utc) + if zone=='hst': + dts = dts.tz_convert("US/Hawaii") + return dts + +def table_to_mjd(table, columns, zone='utc'): + """ + Converts any date and time columns in a table to mjds. + table: dataframe containing dates and times + columns: date or time column labels + zone: time zone of the dates/times + returns: Modified Julian Dates of the dates/times in table + """ + # Safety check for list of columns + if not isinstance(columns, str): + columns = [col for col in columns if col in table.columns] + # Convert to datetimes + dts = pd.to_datetime(table[columns], errors='coerce') + + if zone=='hst':# Convert to UTC + dts = dts.dt.tz_localize(hst) + dts = dts.dt.tz_convert(tz.utc) + + # Masked invalid values + dts = np.ma.masked_array(dts, mask=dts.isnull()) + + # Convert to astropy + times = Time(dts, format='datetime', scale='utc') + # return MJDs + return np.ma.getdata(times.mjd) + +def str_to_mjd(datestrings, fmt): + """ + Converts astropy-formatted date/time strings (in UTC) to MJD values. + datestrings: dates to convert, as strings + fmt: format specifier for the input datestrings + returns: an array of MJDs for the given datestrings + """ + # Get astropy times + times = Time(datestrings, format=fmt, scale='utc') + # Convert to mjd + return times.mjd + +def mjd_to_yr(mjds): + """ + Converts MJD to Decimal Year. + mjds: a list or series of Modified Julian Dates + returns: decimal years correpsonding to the given MJDs + """ + return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file