From e839033c3cf38c8fb70613348d0d5bf1240b32d4 Mon Sep 17 00:00:00 2001 From: Jessica Lu Date: Thu, 27 May 2021 15:42:15 -0700 Subject: [PATCH 01/12] Changing to main --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 479578f..08e5bda 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,4 @@ Tools and tips to process Keck Observatory AO Telemetry - A library of database commands was written to interface to IDL - Extracted telemetry corresponding to NIRC2 images is dumped to IDL ".sav" files for archiving - Basic description: https://www2.keck.hawaii.edu/optics/ao/ngwfc/trsqueries.html + From 60bdc55f051c860c57d10841efd80c23d290f354 Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 10 Jun 2021 13:52:04 -0700 Subject: [PATCH 02/12] Added main files --- __pycache__/compiler.cpython-37.pyc | Bin 0 -> 3110 bytes __pycache__/nirc2_util.cpython-37.pyc | Bin 0 -> 1912 bytes __pycache__/telem_util.cpython-37.pyc | Bin 0 -> 4373 bytes __pycache__/time_util.cpython-37.pyc | Bin 0 -> 1804 bytes compiler.py | 155 ++++++++++++++ mkwc_util.py | 186 +++++++++++++++++ nirc2_util.py | 87 ++++++++ telem_util.py | 283 ++++++++++++++++++++++++++ temp_util.py | 277 +++++++++++++++++++++++++ time_util.py | 56 +++++ 10 files changed, 1044 insertions(+) create mode 100644 __pycache__/compiler.cpython-37.pyc create mode 100644 __pycache__/nirc2_util.cpython-37.pyc create mode 100644 __pycache__/telem_util.cpython-37.pyc create mode 100644 __pycache__/time_util.cpython-37.pyc create mode 100644 compiler.py create mode 100644 mkwc_util.py create mode 100644 nirc2_util.py create mode 100644 telem_util.py create mode 100644 temp_util.py create mode 100644 time_util.py diff --git a/__pycache__/compiler.cpython-37.pyc b/__pycache__/compiler.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb0f6823785a8acf2e33f5f95eaea6c94c4b68eb GIT binary patch literal 3110 zcmb7GOK%*<5uTpc&de@X6h%EK2gXD|9IxS!5#bm}APBbQ5D1bmfJDf^Ai-uh-Mc&F zyxh|(a+%rWAO+Edhun3ba?7tse#cyM2y_l|@x}S7hol+#mcgudJwH} zlUv;84tIHvd)%LT(XeYzt!`v$i6IYOSBYArBz88ncsqVKlT?cJf*HSg5%y>M$-saa_d( z7d+ewlT5?~T3K4Fu$+WER&l5nm8ipsEc0+)3mK+GHCN%tZryGs)k+k+U5S%KRBD{X z`y#7bH?As(;axy>#@bF46yj*ib-Q(R{dYsIz^$)7*qU#NJk1tk8RufL^-GZ)ep>EP zgoKidt)$GWG!t^OTC{^?hS767w<$716ozS=f$6bT=CPqEuR(GdMhd5mkHT3$Me!1F znj&NzA-hc>y!RJl!UzyAJnl@)C(yvCyFp_Q^;qxf821iM`I!5V89X3;-bZi;Cw~Zo z5WkBEtC zfR1>q4MOHj5bGL4Fg;?DChB976zJA_kj42vj~{NL_aA)5$vgUx$4m3FBEl4l-NU@<$-|#&& z8sh+S8g^jFRbm&#&P?nWH<`HXwYlshOiLhxgh9t{aKGcm$ksPugbJQH!ZZn_J`sAg76hSi?g}tP<8=rkVhzpu+}kV!h30I(@PuHQPnHqbX_Obh=MYjtGd z4w!ANu{CRG4lQ{P^1Ype&Vhb?d@eqHCjUf3-x`Xo-B|K_jeTHp#?3#2JRmXU`^swA zbGBp{u{lWO6y&mWj!zl}o|a2jIgR_A{nlLemR{3?mW!|V++6yNCto%GfhYf{{DT0k zz8a{N#(oZa+`;+z#k*=q+Ku11+`VmV7??+z*O~z9`;Bw9ir)FUn(H5YVZhq;#-W^d zN;lm<`CE7XkIH;MEkNnSF&&k3VCXnL4ih2iG{YxKidhyC5p*yqH62~ytIPDT*&XLE z9zWXF`gNGkfhBMi>}Zk$rH4F~B2lGWgfXZL)OTfta|>_i_U;iFxGXF1P%kO7d0y15 zlOjnSNA?(JF7H4K7jA>9wbs$374meuTZtLjthyoqyPUHKq^fsP{$c1!<<8Db!ZXxsxH!y9a zKAU;g&z$em+5o3_@-zf4DZl?eE~WSqtfMG764>-B1i=eC$Ir0_jK$$JeI|TlXv?!(M5CT-ahvAX0!6Sc&cC=E9SSGWvod)B}r+VYrZ;*j+LgT8)-v5~pAGrAf1D$T`suF?$eOEl{X=#i*%DA4att)tT) z=74j%J~3eA(~F?%ucD6|HqWbte23aDJs0D;^P1qJQ(+e_%4_ literal 0 HcmV?d00001 diff --git a/__pycache__/nirc2_util.cpython-37.pyc b/__pycache__/nirc2_util.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9c378a7bc083f0ae29d60a86b2d4d0368f79f0a GIT binary patch literal 1912 zcmZ8iO>f&a7^bKX%U_9;rc0A`ov?L44HPlZ0>cJmL)|9n(#38UdvHN)(^QeRmB^M$ z(oO8jr}nVRuR;$-FR7$0Mu9Is^pfKHJnuu@uU1P4%HQ38etiqQ zf7Rf!RH1VZs`wKOMbwB9$y;@BA#NWsW`TvDVaDN&1*S()1Fh*YRU z3j<85)TKpQqGehc*kp-T=@MP0HM&Aq=@q(0uhNg`I=x0W234}m%6)@w(vM#oq(-mP zPhKOkLT}Jdp;T^vp#f&)l94YXT@EPaC z>YofpSuuAR9Z^zf?jN?Aoer`0yZfzO;yifV+}Y`n!d`E`-Gm9;B*fWiJ#6l`q3!C{ zZ9YD@zt{V&MVww|@9?lo+=EWD_2fzKkQ5Fc9=1BYJpxYfw03uh+x_wWZg;o!Bs5g$ zP;96U)CW+-BQOzK7^n9nIx|vqZtS3+?=0}?ZHb|Gb74yJ%t~=;p5uij(UE=Tq}I8? zH&ZJ$QoN0xZ}!mhS`VcPUt|82*I@w@WCBV zVeHX}vrr~{=G6-sPDECSBOzfnt6hMtu+%HF(#6aqkeLIUkD#?+FwC4tD7U(uxoTG* z)|1j8NvLK*5L+KWaUQd*{K3Y5AzlLn-rjCZ8*CKCGoQmDXN~V!c-&07EM_An`K*z1 zHkDDlHJN2qp+ofbu3|&sxC9277OpuZqk=0&4f9o)2~9N*MFC!fszLt+Ism0{h85tV zrczVG45L443o|t--Tq13E-c(djMe03qJOPOY;-YD@R1c#c!!s08HLXXSI$ zS7dytRsa#^{Fztspgd$!c>4I>kOh=!N~Ve(o{YS}yM)&Z1dtKRd=)SkCjk|TKjj>a zW#Wkm3!{F-s7L#H?>1LCRj+Ur9WN_WQ3iv_tYpH*nUjAsE9I?!d?GS)z+{HUlgu0k zt=ExlBcm>CgQsFDl83@Oj>@rmXuH!I4ws^ZWzt0cJ%Q$n2qITgbK z9@fkf<~LycF8$S63s5x^zd`3c6AMF{M^*|5QcO*0Q|oto0T`8Pyaa#lGQ<)Ho?bu2fOXdbOe035;S*} zxv7i}T&Z4%r@#Ke+0{M6b;`Qiql8Xl_N{_TsN{5w{DBy-g)PH{SeD?aS(Pj9%70FA B?Kl7c literal 0 HcmV?d00001 diff --git a/__pycache__/telem_util.cpython-37.pyc b/__pycache__/telem_util.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01772e013ab05abdc34163eca8cabe277cfb3d65 GIT binary patch literal 4373 zcmZ8kO>7&-72ciw;c`XNw4~VbPm)biC#I7~t&=oqU^td7J4j+HX{;p8qA|1LEUBgV zmzkk0DNCS$lcTRWOULxqdoR5eMGF)~(8HXH_GF-!0_mkFkbZAgWCbn3H#2YF%KuU10~*S(K__xazSx2%8AV0x9&xrV>|D+IT=-Lqobv@CW^>&7lx*7N#)?AwJ+ zDGs<3m$@5E+M_b{gxSE!EfPRga;}blL>pbLh9TuPDm3W?4;{{&hCwQHo zkV6%M4E7!+}}I9(pl+^ zS9qe5mF={(-Bf8W?WaoYuI$y9msgVEQB6&bH=D_**-u8xYDevrl9n=^>hh)5#b5Fl z8_VrX$-RXyCo7jOtqk7!a&qbR+jlQ5<58Xee*3S#+}Zr#t?V<~va)+5nh>2o{gE!~ zS{#~ZH``e+)#ak!@A9~`vC-_O$sqR8kqU~nJBJE%NHHh+@5X$P>IgXq&?mTk#*?|RTxqGiiACi}%B*K29VLQTZ$B7yW84X1Q zSG=5zQlZ8ol~G#^`!J+;k2cmDOxJnZ%lgd+TbWAbC~2kI-@ z2Wr?HjQgWqoJ&9p43d8OttAMn|9NR;ypr~_-fmMsySwrp&S-VGF*~HfO=FeymPfmX zi{foB($*M+#Y6?NW%@Kc{F}YMgh6g|dpi(Ixz&XiI6DJ~pRp$J$8?E4D*R+8IP2+R z4K@V<0aANrl5s>K<}go}JE>}#DfIr*r^Q%c*O{oJ_xPU>EtizI$cJQLw);19Xj|OI z23=)$S#ju(*u=~2edoZM__>$+%HDT#pF7*^z`kN(Rbg0vT{NdW6G54aE8a$e>RcfSVUNVc)sOylxWp}9^v zGb$um9nsBZLdUQ^)&y*Wz2Aost`AcHa}cSmH0ljoiOPlpz_x7^_mf`6kDwW6MRrqo zM~M*0u2}`p&bWLtTHkr))t^OXG){WagXDgSQyr+nG`iT&S{TZPnUoDwG3CdvM(C#l znW^2WWir@}#siQ28?3>^0#rsE z?LYnjS7Q#~*+O_~HW+EYpYkjjXty;)ka!QaQju!6-OEP0l!%TT3Z<*#fxxbM&1}Ha z9qsL32Fteqo6PFOYAVsO|^*F$jumagZ8ap@RJ2DPB z!yz|T<)YV8@;I!|M0{e&)NYkBNvL z6Ezi-Fg?c8&}dhFAymVSJuH*JuWOnq75f;aLLB!Rj4qG+zs5WnKv>L$lisuik;9Dk z=+2A>eucq|2bi)Idx*^{C%4hMxvSiLFLyYj_&;Oq`>J$cpRgt#^a1#Ral|WP7~^gk z5pCrjMn3M&5o;`1-3k(k`^e7yX^*r^}eNC z1M$&MbcG-X;F`D>4Kj5pP5_jjwd+oeJI>r&Hy%vj%B8#lVk%4>Ka6M-d&48hT}Ba^Tw+E@*4q`D0{*B54cz4 zF2r-q5!W!DrL7tUlvU3+L!PDL@i_41bJAkBPz=ag0+-+A}=I}!9 zcD*j(h!}^b5bw4+Oe+B4v3n|@$e#p8kHmz2Ss_*7NuioUf-65{lQ}%Al84*{W6Y5} zO@OXITjefJp|)t{l|@UPI0v3t!0Y2~m4X8n^*lVlN$sCJZ0q^l;kan)dF60iwDr7t zV0X$$rt>*d3vG7A;vP=c-}`BFb4LjR&yiy_M>s1)Obw=nQ4&!R(Wbg7TEA7q4Ta1` zV^I+M7?+#q&`z=`b-3})otw?Io3}SsMHTnslhb@ROLw~R;my16tTk_ZwDJC}Tbe4< zch-J!H>SEc8G-SOTxYn)V7Le-C*2vL=*zT!d-cxswN*j%t{qgU+Cg%rywsD88d4HP zq-$y)2Wdl>TU%6<11{n!y)gj_2{7#eim%ZtmcfZfA`;aTz*Xd8@j4m!MK!Gbq7pui z#%fx4v#5~2!wk6$VVP(Q>>8-lMdn$uD^7qIJ>@**GMiBrn*B~W;^&w#quLV~5ZxN= z{uZ4>>@u7tN-hQDQW-_r1@)#WVt_ZtkS<{c1ibHa4?Rznx`84jf}pvNIsQ(!7_C{K z-2;xDyoAEZGtwgIpun93lQQ8JXf5XfsCiCRyH#Me!~?8R*?BgvVE)-jRn_up?gQT; zFF$f8wY(<2`M~<(n@_ASKK{hYeIPE}doOxB1=>I($tH?>Dxs#;H!>U;x^IfB=@)=Z z71`j(Fb7<2(1;2FC60`4fmwWj0XGbVukZO6KE2Stz)=X70D^IUAhl;IkPVoYMcNt* zncYu8^XqVB;{*Vc4hT9zmk!TPSLqp4gy2nDTjB}~br~-}yu_6bVB99Q5|jaTc@0r< zi=skMrJy~L3_7W5?4~s8QxVQ z8+4?&O^>iM&DsOaFxSBf#f0W4QE@rx^-QXg;ytn~$#JWNd~+P9A$>zM2bpMH{sDH9 zvH*&HR!313+MzR#v~iwQ%&2a=R>(vb77ZH}<9g#s?ZG{x-6Gh1a(^O?H^k?#iUZlo zM!U<|(C|X>J<_CaD&tE8869|&XCf{kxtUHtWgzWayo?pZrzGx?xK4twFV-ONt$Op; z`!_ziE8Zp*Io#xE^Qls#XR1>QuDC`$iY4>zM_dq+&V2g>SNcOf?xk1h`&C|pC<3v< l0>^cz@(HN^VJ=DAwOvNxdi3YBzzs`F4}wx~CJ4h9{|9Nad}aUu literal 0 HcmV?d00001 diff --git a/__pycache__/time_util.cpython-37.pyc b/__pycache__/time_util.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35e48602f1406121f30fc6a9394d717af2fb166a GIT binary patch literal 1804 zcmZ`)OK;mo5a#YuBt<vY!?9b@m*lSPy3psUWB|p#yg`M3U&JJh3`DXQ5uNOM-{CfPyx5Rgx zKXG&QFj#y7L;VbbJDlWBOzcc!W@k5c?c9nzm}%}8e(V=P91yf;dAkVXu;|1cyLa<$ z(Tn@VMm!*n@Rk8@U6Oc{Z*ZRn=Oo_Z1K#E#tnTm*@18sHHt+F1%y&6Ca)z6KK^)Fk zLuvw^XrZ%0m;mLxDg|iWW7Kp}qUMY8qylZLE;Fr6JFQBklTxdB``OXxaq?r5Wro!{ zO)0vC2lH*gIDnx(2BDn>vgVYt3$`Se&Vp*XBn`bJOQzikM8BX7;qDogj~cr4FdxuR zh}D~a6dhFM3n8_No>n|N$pnv{)OnVa(IG^?qlS^LqQ^(aQ6-~i#|IJSgj&Oj36}wU zAENh8Dp@4jv{ahp!m#g^z7_w_moC_uj;_YnkkBay#eAGsa3q@xL-pLcGa+aJ1`KM* zj0>Mp#zm+M<68ah$h+X|_5D#j5=EBJ#xg0yZ1kl_&pxe=MJ@`Vf;E)40O+q9HgT6N?E%!xu6-R$+PKrN-ESDu zi?=V>!bOYaM&nKfmq@5^zH?I~#4k!p9$|h_TIF?7Dl8Gg6r}_L#s|rQnBRe+-Ue|B zBzeBIAPsrtOq?@Mx()e_{@@<1;2aX;ubn|T>F$Sy!4|YFoqU%I3%R@peqM)rd8Tw) z71RCq(K&v}bLNq?LyRGi-91}XwGWFcW8-S$ z=lP0{{;AM_XCm8hVAzuH;Ndq>AZ&6Q1mq7mE1Un2tSk@))4iQ99#mT(92OFw6%9z2 z^w~D+Q#l0ffA=o>$KHkB1X4lap(iI;a5n$^rr-(WTuo;WR~@FIJ&^*I-VwaY%2O5X zWF_{dZPlkw4x<-IUJDid8%}ByU**f<4tiSqFzlqzatA_M)f#)u7Tknjv99X~lnQC( zU}pDWz9t+J@A_84D%`%6Y*z04y_>8sN+9i_NV6izqi;kaBOBKL#SFo&Y-eB!l|Z!r z8R=D*&*UR`Ynka#`2f~%z}{5r>7MbX&_ju`WHPO?E~Yd20T>z=>uucW3~nP!Cd*5_ z3`}=jid{5W0f`wKf324u?no`+9}4DlMN7O~ED0=LHkw;;-@qK~6&0^@v5#I9ax8$` Q34a6p(U65= MAX_LEN: +# continue + + # Search epoch for valid strehl file + for file in strehl_filenames: + strehl_file = strehl_pat.format(epoch, file) + # If good, add to dict + if os.path.isfile(strehl_file): + good_files[epoch] = strehl_file + # Returns {epoch:strehl} dict + return good_files + +def from_filename(nirc2_file, data, i): + """ + Gets nirc2 header values from a filename as dict + or loads values into specified df + """ + # Check for valid file + if not os.path.isfile(nirc2_file): + return + # Open nirc2 file + with fits.open(nirc2_file) as file: + nirc2_hdr = file[0].header + + # Get fields from header + for kw in header_kws: + # load DataFrame value + data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) + +def from_strehl(strehl_file): + """ Gets NIRC2 header data based on contents of Strehl file """ + # Get directory name + data_dir = os.path.dirname(strehl_file) + # Retrieve Strehl data + strehl_data = pd.read_csv(strehl_file, delim_whitespace = True, + header = None, skiprows = 1, names=strehl_cols) + # Add true file path + strehl_data['nirc2_file'] = data_dir + "/" + strehl_data['nirc2_file'] + + # Add decimal year + strehl_data['dec_year'] = times.mjd_to_yr(strehl_data.nirc2_mjd) + + # Add nirc2 columns + for col in header_kws: + strehl_data[col.lower()] = np.nan + + # Loop through nirc2 files + for i,nirc2_file in enumerate(strehl_data.nirc2_file): + # Load header data into df + from_filename(nirc2_file, strehl_data, i) + + # Return data + return strehl_data + + + diff --git a/telem_util.py b/telem_util.py new file mode 100644 index 0000000..7ca7e51 --- /dev/null +++ b/telem_util.py @@ -0,0 +1,283 @@ +### telem_util.py: For code that interacts with Keck AO Telemetry files +### Author: Emily Ramey +### Date: 11/18/2020 + +import numpy as np +import pandas as pd +import glob +from scipy.io import readsav + +import time_util as times + +### Path to telemetry files [EDIT THIS] +telem_dir = '/g/lu/data/keck_telemetry/' + +# Sub-aperture maps +map_dir = "../ao_telemetry/" +wfs_file = map_dir+"sub_ap_map.txt" +act_file = map_dir+"act.txt" + +filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers +filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" + +TIME_DELTA = 0.001 # About 100 seconds in mjd + +RESID_CUTOFF = 349 +TT_IDXS = [349,350] +DEFOCUS = 351 +LAMBDA = 2.1 # microns + +cols = ['telem_file', + 'telem_mjd', + 'TT_mean', + 'TT_std', + 'DM_mean', + 'DM_std', + 'rmswfe_mean', + 'rmswfe_std', + 'strehl_telem'] + +def read_map(filename): + """ Reads a map of actuators or sub-apertures from a file """ + return pd.read_csv(filename, delim_whitespace=True, header=None).to_numpy() + +def get_times(telem_data, start=None): + """ Pulls timestamps from a telemetry file in seconds from start """ + if start is None: + start = telem_data.a.timestamp[0][0] + + return (telem_data.a.timestamp[0]-start)/1e7 + +def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): + """ + Return the locations of the valid actuators in the actuator array + resids: Nx349 residual wavefront array (microns) + ints: Nx304 intensity array (any units) + N: Number of timestamps + """ + # Check inputs + N = ints.shape[0] # Num timestamps + + # Aggregate intensities over all timestamps + med_ints = np.median(ints, axis=0) + + # Fill WFS map with aggregated intensities + int_map = wfs_map.copy() + int_map[np.where(int_map==1)] = med_ints + + # Find lenslets with greatest intensity + idxs = np.flip(np.argsort(int_map, axis=None))[:num_aps] # flat idxs of sort + idxs = np.unravel_index(idxs, wfs_map.shape) # 2D idxs of sort + + # Mask for good sub-ap values + good_aps = np.zeros(wfs_map.shape, dtype=int) + good_aps[idxs] = 1 + good_aps = good_aps * wfs_map # Just in case + + # Mask for good actuator values + good_acts = np.pad(good_aps, ((1,1),(1,1))) + good_acts = (good_acts[1:,1:] | good_acts[1:,:-1] + | good_acts[:-1,:-1] | good_acts[:-1,1:]) * act_map + + return good_acts + +# def wfs_error2(resids, ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file)): +# """ Calculates the variance in the wavefront (microns^2) produced by the WFS """ +# # Get residual mask for brightest sub-aps +# rmask = resid_mask(resids, ints) + +# # Square residuals +# sig2_resid = np.mean(resids**2, axis=0) +# # Load into actuator grid +# resid_grid = act_map.astype(float) +# resid_grid[np.where(resid_grid==1)] = sig2_resid + +# # Mask out actuators next to dimmer apertures +# sig2_masked = resid_grid * rmask +# # Sum values and return +# return np.sum(sig2_masked) + +# def tt_error2(tt_resids): ### TODO: fix this so it's not unitless +# """ Calculate mean tip-tilt residual variance in microns """ +# wvln = 0.658 # wavelength (microns) +# D = 10.5 * 1e6 # telescope diameter (microns) + +# # Magnitude of residuals +# sig_alpha2 = (tt_resids[:,0] + tt_resids[:,1])**2 # x + y TT residual variance +# sig_w2 = (np.pi*D/wvln/2.0)**2 * sig_alpha2 # TT resids in microns^2 + +# return np.mean(sig_w2) + +# def total_WFE(telem_data): +# resids = telem_data.a.residualwavefront[0][:, :RESID_CUTOFF] * 0.6 # WFS resids, microns, Nx349 +# ints = telem_data.a.subapintensity[0] # Sub-ap intensities, Nx304 +# tt_resids = telem_data.a.residualwavefront[0][:, TT_IDXS] * np.pi / (180*3600) # TT resids, radians, Nx2 +# defocus = [0] #telem_data.a.residualwavefront[0][:, DEFOCUS] # Defocus resids, microns, Nx1 + +# return wfs_error2(resids, ints) + tt_error2(tt_resids) + np.mean(defocus**2) + +# def mask_residuals_old(telem_data, num_aps=236, wfs_file=wfs_file, act_file=act_file): +# """ Really freakin complicated array logic to mask out all invalid actuators """ +# # Get data +# resids = telem_data.a.residualwavefront[0][:, :349]*0.6 # microns +# intensities = telem_data.a.subapintensity[0] # ADU +# N = intensities.shape[0] +# # Get hardware maps +# wfs_map = read_map(wfs_file) +# act_map = read_map(act_file) + +# # Get X and Y values of sub-aps and replicate them for all timestamps +# wfs_x, wfs_y = np.where(wfs_map==1) +# wfs_x, wfs_y = np.tile(wfs_x, (N,1)), np.tile(wfs_y, (N,1)) + +# # Get valid indices for each timestep +# idxs = np.flip(np.argsort(intensities, axis=1), axis=1)[:,:num_aps] +# valid_x = np.take_along_axis(wfs_x, idxs, axis=1) +# valid_y = np.take_along_axis(wfs_y, idxs, axis=1) +# valid_z = np.tile(np.arange(N), (num_aps,1)).T + +# # Put 1s at each valid index +# valid_saps = np.zeros((N, 20, 20), int) +# valid_saps[valid_z, valid_x, valid_y] = 1 # TODO: flip this back +# # Pad each sheet (timestamp) with zeros at the edges +# check = valid_saps.reshape(N, 20*20).sum(axis=1) +# if any(check!=236): +# print("Shape mismatch in valid sub-ap array") +# valid_saps = np.pad(valid_saps, ((0,0),(1,1),(1,1))) + +# # Get (potentially)valid actuators for sub-aps +# valid_acts = (valid_saps[:,1:,1:]|valid_saps[:,1:,:-1]| +# valid_saps[:,:-1,:-1]|valid_saps[:,:-1,1:]) # 4 corners of sub-aps +# # Multiply by actuator map to remove any non-actuator positions +# valid_acts = valid_acts * np.tile(act_map, (N,1,1)) + +# # Get number of valid actuators in each frame (can vary due to edge cases) +# valid_act_nums = valid_acts.reshape(N,21*21).sum(axis=1) + +# # Map residuals to actuator positions +# resid_vals = np.tile(act_map, (N,1,1)).astype(float) +# resid_vals[np.where(resid_vals==1)] = resids.flatten() + +# # Mask out invalid actuators +# valid_acts = valid_acts * resid_vals +# rms_resids = valid_acts.reshape(N, 21*21) + +# # Take the RMS residual for each frame +# rms_resids = np.sqrt((rms_resids**2).sum(axis=1)/valid_act_nums) + +# return rms_resids + +def tt2um(tt_as): + """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ + D = 10.5e6 # telescope diameter in microns + tt = tt_as*4.8e-6 # TT error in radians + tt_err = np.sqrt(D**2 / 12 * (tt[:,0]**2 + tt[:,1]**2)) + return tt_err + +def rms_acts(act_resids, ints): + """ Clips bad actuators and averages for each timestamp """ + N = act_resids.shape[0] # num timestamps + + # Read in actuator map + act_map = read_map(act_file) + + # Get good actuator mask from intensities + mask = resid_mask(ints) + flat_map = ~mask[np.where(act_map==1)].astype(bool) # flatten + flat_map = np.tile(flat_map, (N,1)) + + # Mask the bad actuators + good_acts = np.ma.masked_array(act_resids, flat_map) + # Average resids for each timestep + act_rms = np.sqrt((good_acts**2).mean(axis=1) - good_acts.mean(axis=1)**2) + + return act_rms.compressed() + +######################################## +######### Processing Files ############# +######################################## + +# Do some telemetry files need to be cropped around the observation? + +def get_mjd(telem): + """ Validates a telemetry file against an MJD value """ + # Get timestamp + tstamp = telem.tstamp_str_start.decode('utf-8') + # Convert to MJD + telem_mjd = times.str_to_mjd(tstamp, fmt='isot') + + # Return telem mjd if they match, else return False + return telem_mjd + +def extract_telem(file, data, idx, check_mjd=None): + """ Extracts telemetry values from a file to a dataframe """ + # Read IDL file + telem = readsav(file) + + # Make sure MJD matches + telem_mjd = get_mjd(telem) + if check_mjd is not None: + delta = np.abs(telem_mjd-check_mjd) + if delta > TIME_DELTA: + return False + + # Get residuals and intensities + act_resids = telem.a.residualwavefront[0][:, :RESID_CUTOFF] + tt_resids = telem.a.residualwavefront[0][:,TT_IDXS] + ints = telem.a.subapintensity[0] # Sub-aperture intensities + + # Convert TT resids to microns + tt_microns = tt2um(tt_resids) + # Get RMS resids from the actuator array + act_rms = rms_acts(act_resids, ints) + + # Total RMS Wavefront Error + rmswfe = np.sqrt(tt_microns**2 + act_rms**2) + + # Strehl calculation + strehl = np.exp(-(2*np.pi*rmswfe/LAMBDA)**2) + + # Assemble aggregate data + data.loc[idx, cols] = [ + file, + telem_mjd, + np.mean(tt_microns), + np.std(tt_microns), + np.mean(act_rms), + np.std(act_rms), + np.mean(rmswfe), + np.std(rmswfe), + np.mean(strehl), + ] + + return True + +def from_nirc2(mjds, filenames): + """ Gets a table of telemetry information from a set of mjds and file numbers """ + N = len(mjds) # number of data points + # Get file numbers + filenums = filenames.str.extract(filenum_match, expand=False) + filenums = filenums.str[1:] # First digit doesn't always match + + # Get datestrings + dts = times.mjd_to_dt(mjds) # HST or UTC??? + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + + # Set up dataframe + data = pd.DataFrame(columns=cols, index=range(N)) + + # Find telemetry for each file + for i in range(N): + # Get filename and number + fn, ds, mjd = filenums[i], datestrings[i], mjds[i] + # Search for correct file + file_pat = filename_match.format(ds, fn) + all_files = glob.glob(file_pat, recursive=True) + + # Grab the first file that matches the MJD + for file in all_files: + success = extract_telem(file, data, i, check_mjd=mjd) + if success: break + + return data + \ No newline at end of file diff --git a/temp_util.py b/temp_util.py new file mode 100644 index 0000000..18c3053 --- /dev/null +++ b/temp_util.py @@ -0,0 +1,277 @@ +### temp_util.py: Functions relating to Keck-II temperature data +### Main reads in Keck II temperature files (AO, env, and L4) and saves each to a FITS file +### Author: Emily Ramey +### Date: 01/04/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits +from astropy.table import Table + +import time_util as times + +verbose = True + +data_dir = "/u/emily_ramey/work/Keck_Performance/data/" +temp_dir = data_dir+"temp_data_2/" + +col_dict = { + 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV + 'AO_benchT2': 'k1:ao:env:benchtemp2_Raw', + 'k1:ao:env:elect_vault_t': 'k1:ao:env:elect_vault:T', + 'k0:met:humidityStats.VAL': 'k0:met:humidityStats', + "k2:ao:env:ETroomtemp_Raw": "El_roomT", # k2AO + "k2:ao:env:DMracktemp_Raw": "DM_rackT", + "k2:ao:env:KCAMtemp2_Raw": "KCAM_T2", + "k2:ao:env:OBrighttemp_Raw": "OB_rightT", + "k2:ao:env:phototemp_Raw": "photometricT", + "k2:ao:env:OBlefttemp_Raw": "OB_leftT", + "k2:ao:env:KCAMtemp1_Raw": "KCAM_T1", + "k2:ao:env:ACAMtemp_Raw": "AOA_camT", + "k2:ao:env:LStemp_Raw": "LGS_temp", # k2L4 + "k2:ao:env:LSenchumdity_Raw": "LGS_enclosure_hum", + "k2:ao:env:LShumidity_Raw": "LGS_humidity", + "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" +} + +file_pats = { # File name patterns +# 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", +# 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", +# 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", + 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", + 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", + 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", +} + +data_types = { + 'k2AO': { + 'file_pat': temp_dir+"k2AOtemps/{}/AO_temps.log", + 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', + 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', + 'k2AO_mjd'], + }, + 'k2L4': { + 'file_pat': temp_dir+"k2L4temps/{}/L4_env.log", + 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', + 'LGS_temp', 'k2L4_mjd'], + }, + 'k2ENV': { + 'file_pat': temp_dir+"k2envMet/{}/envMet.arT", + 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', + 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', + 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', + 'k0:met:out:windSpeedMaxStats', 'k0:met:out:windSpeedMaxmph', + 'k0:met:out:windSpeedmph', 'k0:met:outTempDwptDiff', + 'k0:met:pressureRaw', 'k0:met:pressureStats', 'k0:met:tempMax', + 'k0:met:tempMin', 'k0:met:tempRaw', 'k2:dcs:sec:acsDwptDiff', + 'k2:dcs:sec:acsTemp', 'k2:dcs:sec:secDwptDiff', 'k2:dcs:sec:secondaryTemp', + 'k2:met:humidityRaw','k2:met:humidityStats', 'k2:met:tempRaw', 'k2:met:tempStats', + 'k2:met:windAzRaw', 'k2:met:windElRaw', 'k2:met:windSpeedMaxmph', + 'k2:met:windSpeedMinmph', 'k2:met:windSpeedRaw', 'k2:met:windSpeedStats', + 'k2:met:windSpeedmph', 'k2ENV_mjd'], + } +} + +def get_columns(): + all_data = search_files() + for name, data_files in all_data.items(): + columns = set() + for i,file in enumerate(data_files): + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + for col in df.columns: + columns.add(col) + all_data[name] = columns + + return all_data + + +def search_files(file_pats=file_pats): + """ Finds all filenames matching the given file patterns """ + all_filenames = {} + for name, search in file_pats.items(): + filenames = glob.glob(search, recursive=True) + all_filenames[name] = filenames + if verbose: + print(f"Done {name}, length: {len(filenames)}") + + return all_filenames + +def collect_data(data_files, col_dict=col_dict): + """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ + if isinstance(data_files, dict): # Return dict w/dataframes + new_files = {} + for name, files in data_files.items(): + if isinstance(files, list): + # Recurse on each list of files + new_files[name] = collect_data(files) + return new_files + + all_dfs = [pd.DataFrame()] + for i,file in enumerate(data_files): + if not os.path.isfile(file): + continue + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + + if col_dict is not None: + df = df.rename(columns=col_dict) + + all_dfs.append(df) + + data = pd.concat(all_dfs, ignore_index=True, sort=True) + return data + +def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): + """ Parses specified date and time columns and returns a cleaned data table """ + new_data = data.copy() + for label,cols in date_cols.items(): + date_col, time_col = cols + # Parse dates and times, coercing invalid strings to NaN + datetimes = (pd.to_datetime(data[date_col], exact=False, errors='coerce') + + pd.to_timedelta(data[time_col], errors='coerce')) + new_data = new_data.drop(columns=cols) + new_data[label] = datetimes + + return new_data + +def clean_dates(data, date_cols=['HST', 'UNIX']): + """ Removes any rows containing invalid dates in date_cols """ + new_data = data.copy() + for col in date_cols: + new_data = new_data[~np.isnan(new_data[col])] + + return new_data + +def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): + """ Casts columns to a numeric data type """ + if data_cols is None: + data_cols = [col for col in data.columns if col not in non_numeric] + + # Cast data to numeric type, coercing invalid values to NaN + new_data = data.copy() + for col in data_cols: + new_data[col] = pd.to_numeric(new_data[col], errors='coerce') + + return new_data + +def to_fits(data, filename, str_cols=['HST', 'UNIX']): + """ Writes a FITS file from the given temperature array """ + fits_data = data.copy() + for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: + if col in fits_data.columns: + fits_data = fits_data.drop(columns=[col]) + for col in str_cols: + if col in fits_data.columns: + fits_data[col] = fits_data[col].astype(str) + # Assuming the data columns are already numeric + fits_data = Table.from_pandas(fits_data) + fits_data.write(filename) + + return + +def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): + """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ + data = Table.read(filename).to_pandas() + + # Fix NaNs, because astropy is dumb sometimes + data[data==1e+20] = np.nan + + if date_cols is None: return data + + for col in date_cols: + if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings + data[col] = data[col].str.decode("utf-8") + data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') + + if str_cols is None: return data + + for col in str_cols: + if col in data.columns and isinstance(data[col][0], bytes): + data[col] = data[col].str.decode("utf-8") + + return data + +def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): + """ + Reads in all data matching file patterns (file_pats), combines them into one table, + cleans them, and saves them to a FITS file + """ + # Find all files matching pattern + all_filenames = search_files(file_pats) + # Read all data into one table (per dictionary label) + all_data = collect_data(all_filenames) + + for name, data in all_data.items(): + data = parse_dates(data) # Parse date cols into datetimes + data = clean_dates(data) # Remove invalid dates + data = clean_data(data) # Casts other cols to numeric + # Save combined/cleaned data to FITS file + filename = location+name+".fits" + to_fits(data, filename) + + return + +def from_mjds(mjds, dtype): + """ Gets temp data of input type from the specified MJDs """ + # Get pd datetimes in HST + dts = times.mjd_to_dt(mjds, zone='hst') + # Format list + datestrings = dts.strftime("%y/%m/%d") + datestrings = np.unique(datestrings) # one file per date + # Get relevant filenames + filenames = [data_types[dtype]['file_pat'].format(ds) for ds in datestrings] + # Get data from filenames + data = collect_data(filenames) + + # Empty dataframe + if data.empty: + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Convert dates & times to MJDs + data["datetime"] = data['HSTdate']+' '+data['HSTtime'] + mjds = times.table_to_mjd(data, columns='datetime', zone='hst') + # Add to dataframe + data[dtype+"_mjd"] = mjds + # Drop other date & time cols + data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], + inplace=True, errors='ignore') + + return data + +if __name__=='__main__': + pass \ No newline at end of file diff --git a/time_util.py b/time_util.py new file mode 100644 index 0000000..4974d2c --- /dev/null +++ b/time_util.py @@ -0,0 +1,56 @@ +### time_util.py : to handle time formatting changes between MJD, HST, and UTC +### Author : Emily Ramey +### Date: 5/26/21 + +# Preamble +import pandas as pd +import numpy as np +import time +import pytz as tz +from datetime import datetime, timezone +from astropy.time import Time, TimezoneInfo +from astropy import units as u, constants as c + +hst = tz.timezone('US/Hawaii') + +### Conversion / utility functions +def mjd_to_dt(mjds, zone='utc'): + """ Converts Modified Julian Date(s) to HST or UTC date(s) """ + # Convert mjds -> astropy times -> datetimes + dts = Time(mjds, format='mjd', scale='utc').to_datetime() + # Convert to pandas + dts = pd.to_datetime(dts).tz_localize(tz.utc) + if zone=='hst': + dts = dts.tz_convert("US/Hawaii") + return dts + +def table_to_mjd(table, columns, zone='utc'): + """ Converts date and time columns to mjds """ + # Safety check for list of columns + if not isinstance(columns, str): + columns = [col for col in columns if col in table.columns] + # Convert to datetimes + dts = pd.to_datetime(table[columns], errors='coerce') + + if zone=='hst':# Convert to UTC + dts = dts.dt.tz_localize(hst) + dts = dts.dt.tz_convert(tz.utc) + + # Masked invalid values + dts = np.ma.masked_array(dts, mask=dts.isnull()) + + # Convert to astropy + times = Time(dts, format='datetime', scale='utc') + # return MJDs + return np.ma.getdata(times.mjd) + +def str_to_mjd(datestrings, fmt): + """ Converts astropy-formatted date/time strings (in UTC) to MJD values """ + # Get astropy times + times = Time(datestrings, format=fmt, scale='utc') + # Convert to mjd + return times.mjd + +def mjd_to_yr(mjds): + """ Converts MJD to Decimal Year """ + return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file From c32d5aba9b0689871616d4891433adc873a22d16 Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 10 Jun 2021 14:28:36 -0700 Subject: [PATCH 03/12] moved code to katan --- katan/act.txt | 21 ++++ katan/compiler.py | 159 ++++++++++++++++++++++++ katan/mkwc_util.py | 184 ++++++++++++++++++++++++++++ katan/nirc2_util.py | 87 +++++++++++++ katan/sub_ap_map.txt | 20 +++ katan/telem_util.py | 283 +++++++++++++++++++++++++++++++++++++++++++ katan/temp_util.py | 277 ++++++++++++++++++++++++++++++++++++++++++ katan/time_util.py | 56 +++++++++ 8 files changed, 1087 insertions(+) create mode 100755 katan/act.txt create mode 100644 katan/compiler.py create mode 100644 katan/mkwc_util.py create mode 100644 katan/nirc2_util.py create mode 100755 katan/sub_ap_map.txt create mode 100644 katan/telem_util.py create mode 100644 katan/temp_util.py create mode 100644 katan/time_util.py diff --git a/katan/act.txt b/katan/act.txt new file mode 100755 index 0000000..90ea449 --- /dev/null +++ b/katan/act.txt @@ -0,0 +1,21 @@ +0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/katan/compiler.py b/katan/compiler.py new file mode 100644 index 0000000..38c2592 --- /dev/null +++ b/katan/compiler.py @@ -0,0 +1,159 @@ +### compiler.py : Compiler for all desired data (nirc2, weather, seeing, telemetry, temperature) +### Author : Emily Ramey +### Date : 6/3/21 + +import time_util as times +import nirc2_util as nirc2 +import telem_util as telem +import temp_util as temp +import mkwc_util as mkwc + +import numpy as np +import pandas as pd + +# All data types that can be compiled by this package +accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] + +# Shorthand / nicknames for data types +expand = { + 'temp': ['k2AO', 'k2L4', 'k2ENV'], + 'seeing': ['mass', 'dimm', 'masspro'], + 'weather': ['cfht'], + 'telemetry': ['telem'], + 'all': accept_labels, +} + +# Utility functions +def check_dtypes(data_types): + """ Returns an expanded / cleaned list of data types from user input """ + new_dtypes = [] + # Check requested data + for dtype in data_types: + # Check for nicknames + if dtype in expand: + new_dtypes.extend(expand[dtype]) + elif dtype in accept_labels: # can get data + new_dtypes.append(dtype) + # Return cleaned list + return new_dtypes + +############################################# +######## Interface with other modules ####### +############################################# + +def data_func(dtype): + """ + Returns the function to get data for the specified dtype + and whether data needs to be matched to nirc2 mjds + """ + if dtype in ['cfht']+expand['seeing']: + return lambda files,mjds: mkwc.from_nirc2(mjds,dtype), True + if dtype in expand['temp']: + return lambda files,mjds: temp.from_mjds(mjds,dtype), True + if dtype=='telem': + return lambda files,mjds: telem.from_nirc2(mjds,files), False + + +################################ +###### Compiler Functions ###### +################################ + +def match_data(mjd1, mjd2): + """ + Matches data1 with its closest points in data2, by mjd values + Returns an array containing data1 with corresponding rows from data2 + """ + # Edge case + if mjd1.empty or mjd2.empty: + return None + + # Get mjds from dataframes + mjd1 = np.array(mjd1).reshape(-1,1) # column vector + mjd2 = np.array(mjd2).reshape(1,-1) # row vector + + # Take difference of mjd1 with mjd2 + diffs = np.abs(mjd1 - mjd2) + # Find smallest difference for each original mjd + idxs = np.argmin(diffs, axis=1) + + # Return indices of matches in mjd2 + return idxs + + +# data_types can contain: 'chft', 'mass', 'dimm', 'masspro', +# 'telem', 'k2AO', 'k2L4', or 'k2ENV' +# 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and +# ['mass', 'dimm', 'masspro'], respectively +def combine_strehl(strehl_file, data_types, check=True, test=False): + """ + Combines and matches data from a certain Strehl file with other specified data types. + NIRC2 files must be in the same directory as the Strehl file. + """ + + nirc2_data = nirc2.from_strehl(strehl_file) + + if test: # Take first few files + nirc2_data = nirc2_data.loc[:3] + + if check: # Sanitize user input + data_types = check_dtypes(data_types) + + # Full data container + all_data = [nirc2_data.reset_index(drop=True)] + + # Loop through and get data + for dtype in data_types: + get_data, match = data_func(dtype) # Data retrieval function + # Get other data from strehl info + other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) + + if match: # Needs to be matched + if other_data.empty: # No data found + other_data = pd.DataFrame(columns=other_data.columns, index=range(len(nirc2_data))) + else: # Get indices of matched data + idxs = match_data(nirc2_data.nirc2_mjd, other_data[dtype+'_mjd']) + other_data = other_data.iloc[idxs] + +# # Edge case: no data +# if idxs is not None: +# other_data = other_data.iloc[idxs] + + # Add to all data + all_data.append(other_data.reset_index(drop=True)) + + # Concatenate new data with nirc2 + return pd.concat(all_data, axis=1) + +################################# +######### Full Compiler ######### +################################# + +def compile_all(data_dir, data_types=['all'], test=False, save=False): + """ Compiles and matches requested data from all strehl files in a given nirc2 directory """ + + # Expand requested data types + data_types = check_dtypes(data_types) + + # Get all epochs in requested folder + epochs = nirc2.search_epochs(data_dir=data_dir) + + # Full data container + all_data = [] + + # Compile data for each epoch + for epoch, strehl_file in epochs.items(): + data = combine_strehl(strehl_file, data_types, check=False, + test=test) + # Add epoch name to columns + data['epoch'] = epoch + + # Append to full data + all_data.append(data) + + all_data = pd.concat(all_data, ignore_index=True) + + if save: + all_data.to_csv(save, index=False) + + # Concatenate all data + return all_data \ No newline at end of file diff --git a/katan/mkwc_util.py b/katan/mkwc_util.py new file mode 100644 index 0000000..7b626c3 --- /dev/null +++ b/katan/mkwc_util.py @@ -0,0 +1,184 @@ +### mkwc_util.py : Contains utilities for extracting and processing data from the MKWC website +### Author : Emily Ramey +### Date : 6/1/2021 + +import os +import numpy as np +import pandas as pd +import time_util as times + +### NOTE: Seeing data is STORED by UT, with data in HST +### CFHT data is STORED by HST, with data in HST + +# cfht_dir = "/u/emily_ramey/work/Keck_Performance/data/weather_data/" +mkwc_url = 'http://mkwc.ifa.hawaii.edu/' +year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' +cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST + +# seeing_dir = "/u/emily_ramey/work/Keck_Performance/data/seeing_data" + +# Time columns from MKWC data +time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] + +# Data-specific fields +data_types = { + 'cfht': { + 'web_pat': mkwc_url+'archive/wx/cfht/indiv-days/cfht-wx.{}.dat', + 'file_pat': "{}cfht-wx.{}.dat", + 'data_cols': ['wind_speed', 'wind_direction', 'temperature', + 'relative_humidity', 'pressure'], + }, + 'mass': { + 'web_pat': mkwc_url+'current/seeing/mass/{}.mass.dat', + 'file_pat': '{}mass/{}.mass.dat', + 'data_cols': ['mass'], + }, + 'dimm': { + 'web_pat': mkwc_url+'current/seeing/dimm/{}.dimm.dat', + 'file_pat': '{}dimm/{}.dimm.dat', + 'data_cols': ['dimm'], + }, + 'masspro': { + 'web_pat': mkwc_url+'current/seeing/masspro/{}.masspro.dat', + 'file_pat': '{}masspro/{}.masspro.dat', + 'data_cols': ['masspro_half', 'masspro_1', 'masspro_2', + 'masspro_4', 'masspro_8', 'masspro_16', 'masspro'], + }, +} + +# Mix and match data & time columns +for dtype in data_types: + # CFHT files don't have seconds + tcols = time_cols if dtype != 'cfht' else time_cols[:-1] + # Format web columns + data_types[dtype]['web_cols'] = tcols+data_types[dtype]['data_cols'] + # Format file columns + data_types[dtype]['cols'] = [dtype+'_mjd']+data_types[dtype]['data_cols'] + # Different file storage timezones + data_types[dtype]['file_zone'] = 'hst' if dtype=='cfht' else 'utc' + +############################# +######### Functions ######### +############################# + +def cfht_from_year(datestrings, year): + """ + Gets pre-2012 MKWC data from the year-long file + instead of the by-date files + """ + # Get year-long file URL + url = year_url.format(year) + + # Read in data + try: + web_cols = data_types['cfht']['web_cols'] + year_data = pd.read_csv(url, delim_whitespace=True, header=None, + names=web_cols, usecols=range(len(web_cols))) + except: # No data, return blank + return pd.DataFrame(columns=data_types['cfht']['cols']) + + # Full dataset + all_data = [pd.DataFrame(columns=data_types['cfht']['cols'])] + + # Slice up dataframe + for ds in datestrings: + month, day = int(ds[4:6]), int(ds[6:]) + # Get data by month and day + df = year_data.loc[(year_data.month==month) & (year_data.day==day)].copy() + # Format columns + if not df.empty: + format_columns(df, 'cfht') + # Add to full dataset + all_data.append(df) + + return pd.concat(all_data) + +def format_columns(df, dtype): + """ Changes columns (in place) from time_cols to MJD """ + # Get MJDs from HST values + mjds = times.table_to_mjd(df, columns=time_cols, zone='hst') + df[dtype+'_mjd'] = mjds + + # Drop old times + df.drop(columns=time_cols, inplace=True, errors='ignore') + +def from_url(datestring, dtype): + """ Pulls cfht or seeing file from MKWC website """ + # Format URL + url = data_types[dtype]['web_pat'].format(datestring) + + # Read data + try: # Check if data is there + web_cols = data_types[dtype]['web_cols'] # MKWC Weather columns + df = pd.read_csv(url, delim_whitespace = True, header=None, + names=web_cols, usecols=range(len(web_cols))) + except: # otherwise return blank df + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Get mjd from time columns + format_columns(df, dtype) + + return df + +def from_file(filename, dtype, data_dir='./'): + """ Pulls cfht or seeing file from local directory """ + # Read in CSV + df = pd.read_csv(filename) + + # Check formatting + if 'mjd' in df.columns: + df.rename(columns={'mjd':dtype+'_mjd'}, inplace=True) + elif dtype+'_mjd' not in df.columns: # No MJD + try: # Change to MJD, if times + format_columns(df, dtype) + except: # No time info, return blank + return pd.DataFrame() + + return df + +def from_nirc2(mjds, dtype, data_dir='./'): + """ + Compiles a list of cfht or seeing observations based on MJDs + note: does not compare MJDs; assumption is inputs are rounded to nearest day + """ + + # Get datestrings + dts = times.mjd_to_dt(mjds, zone=data_types[dtype]['file_zone']) + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + # No duplicates + datestrings = pd.Series(np.unique(datestrings)) + + # Blank data structure + all_data = [pd.DataFrame(columns=data_types[dtype]['cols'])] + + # Check for pre-2012 cfht files + if dtype=='cfht' and any(mjds < cfht_cutoff): + # Get datetimes + pre_2012 = datestrings[mjds < cfht_cutoff] + + # Compile data by year + for yr in np.unique(pre_2012.str[:4]): + ds = pre_2012[pre_2012.str[:4]==yr] + df = cfht_from_year(ds, yr) + # Append to full dataset + all_data.append(df) + + # Find data for each file + for ds in datestrings: + # Get local filename + filename = data_types[dtype]['file_pat'].format(data_dir, ds) + + # Check for local files + if os.path.isfile(filename): + df = from_file(filename, dtype) + + else: # Pull from the web + df = from_url(ds, dtype) + + # Save to local file: TODO + + # Add to data + all_data.append(df) + + # Return concatenated dataframe + return pd.concat(all_data, ignore_index=True) \ No newline at end of file diff --git a/katan/nirc2_util.py b/katan/nirc2_util.py new file mode 100644 index 0000000..f34174b --- /dev/null +++ b/katan/nirc2_util.py @@ -0,0 +1,87 @@ +### nirc2_util.py : Functions for processing nirc2 data and strehl files +### Author : Emily Ramey +### Date : 6/2/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits + +import time_util as times + +MAX_LEN = 10 # Max char length of epoch name + +nirc2_dir = "/g/lu/data/gc/lgs_data/" +strehl_pat = nirc2_dir+"{}/clean/kp/{}" + +strehl_filenames = ['strehl_source.txt', 'irs33N.strehl'] +strehl_cols = ['nirc2_file', 'strehl', 'rms_err', 'fwhm', 'nirc2_mjd'] +header_kws = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', + 'AOLBFWHM', 'WSFRRT', 'LSAMPPWR', 'LGRMSWF', 'AOAOAMED', 'TUBETEMP'] + +# Note: need to get better at identifying which strehl files we want +# Alternatively, we could have people provide a list +def search_epochs(data_dir=nirc2_dir): + """ Searches for valid epoch names in NIRC2 data directory """ + # Epochs with valid strehl files + good_files = {} + # Loop through all sub-directories + for epoch in os.listdir(data_dir): +# # Only valid if {yr}{month}lgs +# if len(epoch) >= MAX_LEN: +# continue + + # Search epoch for valid strehl file + for file in strehl_filenames: + strehl_file = strehl_pat.format(epoch, file) + # If good, add to dict + if os.path.isfile(strehl_file): + good_files[epoch] = strehl_file + # Returns {epoch:strehl} dict + return good_files + +def from_filename(nirc2_file, data, i): + """ + Gets nirc2 header values from a filename as dict + or loads values into specified df + """ + # Check for valid file + if not os.path.isfile(nirc2_file): + return + # Open nirc2 file + with fits.open(nirc2_file) as file: + nirc2_hdr = file[0].header + + # Get fields from header + for kw in header_kws: + # load DataFrame value + data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) + +def from_strehl(strehl_file): + """ Gets NIRC2 header data based on contents of Strehl file """ + # Get directory name + data_dir = os.path.dirname(strehl_file) + # Retrieve Strehl data + strehl_data = pd.read_csv(strehl_file, delim_whitespace = True, + header = None, skiprows = 1, names=strehl_cols) + # Add true file path + strehl_data['nirc2_file'] = data_dir + "/" + strehl_data['nirc2_file'] + + # Add decimal year + strehl_data['dec_year'] = times.mjd_to_yr(strehl_data.nirc2_mjd) + + # Add nirc2 columns + for col in header_kws: + strehl_data[col.lower()] = np.nan + + # Loop through nirc2 files + for i,nirc2_file in enumerate(strehl_data.nirc2_file): + # Load header data into df + from_filename(nirc2_file, strehl_data, i) + + # Return data + return strehl_data + + + diff --git a/katan/sub_ap_map.txt b/katan/sub_ap_map.txt new file mode 100755 index 0000000..4dfc98e --- /dev/null +++ b/katan/sub_ap_map.txt @@ -0,0 +1,20 @@ +0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 +1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 +0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 +0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 +0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/katan/telem_util.py b/katan/telem_util.py new file mode 100644 index 0000000..30bbd76 --- /dev/null +++ b/katan/telem_util.py @@ -0,0 +1,283 @@ +### telem_util.py: For code that interacts with Keck AO Telemetry files +### Author: Emily Ramey +### Date: 11/18/2020 + +import numpy as np +import pandas as pd +import glob +from scipy.io import readsav + +import time_util as times + +### Path to telemetry files [EDIT THIS] +telem_dir = '/g/lu/data/keck_telemetry/' + +# Sub-aperture maps +map_dir = "./" +wfs_file = map_dir+"sub_ap_map.txt" +act_file = map_dir+"act.txt" + +filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers +filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" + +TIME_DELTA = 0.001 # About 100 seconds in mjd + +RESID_CUTOFF = 349 +TT_IDXS = [349,350] +DEFOCUS = 351 +LAMBDA = 2.1 # microns + +cols = ['telem_file', + 'telem_mjd', + 'TT_mean', + 'TT_std', + 'DM_mean', + 'DM_std', + 'rmswfe_mean', + 'rmswfe_std', + 'strehl_telem'] + +def read_map(filename): + """ Reads a map of actuators or sub-apertures from a file """ + return pd.read_csv(filename, delim_whitespace=True, header=None).to_numpy() + +def get_times(telem_data, start=None): + """ Pulls timestamps from a telemetry file in seconds from start """ + if start is None: + start = telem_data.a.timestamp[0][0] + + return (telem_data.a.timestamp[0]-start)/1e7 + +def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): + """ + Return the locations of the valid actuators in the actuator array + resids: Nx349 residual wavefront array (microns) + ints: Nx304 intensity array (any units) + N: Number of timestamps + """ + # Check inputs + N = ints.shape[0] # Num timestamps + + # Aggregate intensities over all timestamps + med_ints = np.median(ints, axis=0) + + # Fill WFS map with aggregated intensities + int_map = wfs_map.copy() + int_map[np.where(int_map==1)] = med_ints + + # Find lenslets with greatest intensity + idxs = np.flip(np.argsort(int_map, axis=None))[:num_aps] # flat idxs of sort + idxs = np.unravel_index(idxs, wfs_map.shape) # 2D idxs of sort + + # Mask for good sub-ap values + good_aps = np.zeros(wfs_map.shape, dtype=int) + good_aps[idxs] = 1 + good_aps = good_aps * wfs_map # Just in case + + # Mask for good actuator values + good_acts = np.pad(good_aps, ((1,1),(1,1))) + good_acts = (good_acts[1:,1:] | good_acts[1:,:-1] + | good_acts[:-1,:-1] | good_acts[:-1,1:]) * act_map + + return good_acts + +# def wfs_error2(resids, ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file)): +# """ Calculates the variance in the wavefront (microns^2) produced by the WFS """ +# # Get residual mask for brightest sub-aps +# rmask = resid_mask(resids, ints) + +# # Square residuals +# sig2_resid = np.mean(resids**2, axis=0) +# # Load into actuator grid +# resid_grid = act_map.astype(float) +# resid_grid[np.where(resid_grid==1)] = sig2_resid + +# # Mask out actuators next to dimmer apertures +# sig2_masked = resid_grid * rmask +# # Sum values and return +# return np.sum(sig2_masked) + +# def tt_error2(tt_resids): ### TODO: fix this so it's not unitless +# """ Calculate mean tip-tilt residual variance in microns """ +# wvln = 0.658 # wavelength (microns) +# D = 10.5 * 1e6 # telescope diameter (microns) + +# # Magnitude of residuals +# sig_alpha2 = (tt_resids[:,0] + tt_resids[:,1])**2 # x + y TT residual variance +# sig_w2 = (np.pi*D/wvln/2.0)**2 * sig_alpha2 # TT resids in microns^2 + +# return np.mean(sig_w2) + +# def total_WFE(telem_data): +# resids = telem_data.a.residualwavefront[0][:, :RESID_CUTOFF] * 0.6 # WFS resids, microns, Nx349 +# ints = telem_data.a.subapintensity[0] # Sub-ap intensities, Nx304 +# tt_resids = telem_data.a.residualwavefront[0][:, TT_IDXS] * np.pi / (180*3600) # TT resids, radians, Nx2 +# defocus = [0] #telem_data.a.residualwavefront[0][:, DEFOCUS] # Defocus resids, microns, Nx1 + +# return wfs_error2(resids, ints) + tt_error2(tt_resids) + np.mean(defocus**2) + +# def mask_residuals_old(telem_data, num_aps=236, wfs_file=wfs_file, act_file=act_file): +# """ Really freakin complicated array logic to mask out all invalid actuators """ +# # Get data +# resids = telem_data.a.residualwavefront[0][:, :349]*0.6 # microns +# intensities = telem_data.a.subapintensity[0] # ADU +# N = intensities.shape[0] +# # Get hardware maps +# wfs_map = read_map(wfs_file) +# act_map = read_map(act_file) + +# # Get X and Y values of sub-aps and replicate them for all timestamps +# wfs_x, wfs_y = np.where(wfs_map==1) +# wfs_x, wfs_y = np.tile(wfs_x, (N,1)), np.tile(wfs_y, (N,1)) + +# # Get valid indices for each timestep +# idxs = np.flip(np.argsort(intensities, axis=1), axis=1)[:,:num_aps] +# valid_x = np.take_along_axis(wfs_x, idxs, axis=1) +# valid_y = np.take_along_axis(wfs_y, idxs, axis=1) +# valid_z = np.tile(np.arange(N), (num_aps,1)).T + +# # Put 1s at each valid index +# valid_saps = np.zeros((N, 20, 20), int) +# valid_saps[valid_z, valid_x, valid_y] = 1 # TODO: flip this back +# # Pad each sheet (timestamp) with zeros at the edges +# check = valid_saps.reshape(N, 20*20).sum(axis=1) +# if any(check!=236): +# print("Shape mismatch in valid sub-ap array") +# valid_saps = np.pad(valid_saps, ((0,0),(1,1),(1,1))) + +# # Get (potentially)valid actuators for sub-aps +# valid_acts = (valid_saps[:,1:,1:]|valid_saps[:,1:,:-1]| +# valid_saps[:,:-1,:-1]|valid_saps[:,:-1,1:]) # 4 corners of sub-aps +# # Multiply by actuator map to remove any non-actuator positions +# valid_acts = valid_acts * np.tile(act_map, (N,1,1)) + +# # Get number of valid actuators in each frame (can vary due to edge cases) +# valid_act_nums = valid_acts.reshape(N,21*21).sum(axis=1) + +# # Map residuals to actuator positions +# resid_vals = np.tile(act_map, (N,1,1)).astype(float) +# resid_vals[np.where(resid_vals==1)] = resids.flatten() + +# # Mask out invalid actuators +# valid_acts = valid_acts * resid_vals +# rms_resids = valid_acts.reshape(N, 21*21) + +# # Take the RMS residual for each frame +# rms_resids = np.sqrt((rms_resids**2).sum(axis=1)/valid_act_nums) + +# return rms_resids + +def tt2um(tt_as): + """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ + D = 10.5e6 # telescope diameter in microns + tt = tt_as*4.8e-6 # TT error in radians + tt_err = np.sqrt(D**2 / 12 * (tt[:,0]**2 + tt[:,1]**2)) + return tt_err + +def rms_acts(act_resids, ints): + """ Clips bad actuators and averages for each timestamp """ + N = act_resids.shape[0] # num timestamps + + # Read in actuator map + act_map = read_map(act_file) + + # Get good actuator mask from intensities + mask = resid_mask(ints) + flat_map = ~mask[np.where(act_map==1)].astype(bool) # flatten + flat_map = np.tile(flat_map, (N,1)) + + # Mask the bad actuators + good_acts = np.ma.masked_array(act_resids, flat_map) + # Average resids for each timestep + act_rms = np.sqrt((good_acts**2).mean(axis=1) - good_acts.mean(axis=1)**2) + + return act_rms.compressed() + +######################################## +######### Processing Files ############# +######################################## + +# Do some telemetry files need to be cropped around the observation? + +def get_mjd(telem): + """ Validates a telemetry file against an MJD value """ + # Get timestamp + tstamp = telem.tstamp_str_start.decode('utf-8') + # Convert to MJD + telem_mjd = times.str_to_mjd(tstamp, fmt='isot') + + # Return telem mjd if they match, else return False + return telem_mjd + +def extract_telem(file, data, idx, check_mjd=None): + """ Extracts telemetry values from a file to a dataframe """ + # Read IDL file + telem = readsav(file) + + # Make sure MJD matches + telem_mjd = get_mjd(telem) + if check_mjd is not None: + delta = np.abs(telem_mjd-check_mjd) + if delta > TIME_DELTA: + return False + + # Get residuals and intensities + act_resids = telem.a.residualwavefront[0][:, :RESID_CUTOFF] + tt_resids = telem.a.residualwavefront[0][:,TT_IDXS] + ints = telem.a.subapintensity[0] # Sub-aperture intensities + + # Convert TT resids to microns + tt_microns = tt2um(tt_resids) + # Get RMS resids from the actuator array + act_rms = rms_acts(act_resids, ints) + + # Total RMS Wavefront Error + rmswfe = np.sqrt(tt_microns**2 + act_rms**2) + + # Strehl calculation + strehl = np.exp(-(2*np.pi*rmswfe/LAMBDA)**2) + + # Assemble aggregate data + data.loc[idx, cols] = [ + file, + telem_mjd, + np.mean(tt_microns), + np.std(tt_microns), + np.mean(act_rms), + np.std(act_rms), + np.mean(rmswfe), + np.std(rmswfe), + np.mean(strehl), + ] + + return True + +def from_nirc2(mjds, filenames): + """ Gets a table of telemetry information from a set of mjds and file numbers """ + N = len(mjds) # number of data points + # Get file numbers + filenums = filenames.str.extract(filenum_match, expand=False) + filenums = filenums.str[1:] # First digit doesn't always match + + # Get datestrings + dts = times.mjd_to_dt(mjds) # HST or UTC??? + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + + # Set up dataframe + data = pd.DataFrame(columns=cols, index=range(N)) + + # Find telemetry for each file + for i in range(N): + # Get filename and number + fn, ds, mjd = filenums[i], datestrings[i], mjds[i] + # Search for correct file + file_pat = filename_match.format(ds, fn) + all_files = glob.glob(file_pat, recursive=True) + + # Grab the first file that matches the MJD + for file in all_files: + success = extract_telem(file, data, i, check_mjd=mjd) + if success: break + + return data + \ No newline at end of file diff --git a/katan/temp_util.py b/katan/temp_util.py new file mode 100644 index 0000000..18c3053 --- /dev/null +++ b/katan/temp_util.py @@ -0,0 +1,277 @@ +### temp_util.py: Functions relating to Keck-II temperature data +### Main reads in Keck II temperature files (AO, env, and L4) and saves each to a FITS file +### Author: Emily Ramey +### Date: 01/04/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits +from astropy.table import Table + +import time_util as times + +verbose = True + +data_dir = "/u/emily_ramey/work/Keck_Performance/data/" +temp_dir = data_dir+"temp_data_2/" + +col_dict = { + 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV + 'AO_benchT2': 'k1:ao:env:benchtemp2_Raw', + 'k1:ao:env:elect_vault_t': 'k1:ao:env:elect_vault:T', + 'k0:met:humidityStats.VAL': 'k0:met:humidityStats', + "k2:ao:env:ETroomtemp_Raw": "El_roomT", # k2AO + "k2:ao:env:DMracktemp_Raw": "DM_rackT", + "k2:ao:env:KCAMtemp2_Raw": "KCAM_T2", + "k2:ao:env:OBrighttemp_Raw": "OB_rightT", + "k2:ao:env:phototemp_Raw": "photometricT", + "k2:ao:env:OBlefttemp_Raw": "OB_leftT", + "k2:ao:env:KCAMtemp1_Raw": "KCAM_T1", + "k2:ao:env:ACAMtemp_Raw": "AOA_camT", + "k2:ao:env:LStemp_Raw": "LGS_temp", # k2L4 + "k2:ao:env:LSenchumdity_Raw": "LGS_enclosure_hum", + "k2:ao:env:LShumidity_Raw": "LGS_humidity", + "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" +} + +file_pats = { # File name patterns +# 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", +# 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", +# 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", + 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", + 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", + 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", +} + +data_types = { + 'k2AO': { + 'file_pat': temp_dir+"k2AOtemps/{}/AO_temps.log", + 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', + 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', + 'k2AO_mjd'], + }, + 'k2L4': { + 'file_pat': temp_dir+"k2L4temps/{}/L4_env.log", + 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', + 'LGS_temp', 'k2L4_mjd'], + }, + 'k2ENV': { + 'file_pat': temp_dir+"k2envMet/{}/envMet.arT", + 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', + 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', + 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', + 'k0:met:out:windSpeedMaxStats', 'k0:met:out:windSpeedMaxmph', + 'k0:met:out:windSpeedmph', 'k0:met:outTempDwptDiff', + 'k0:met:pressureRaw', 'k0:met:pressureStats', 'k0:met:tempMax', + 'k0:met:tempMin', 'k0:met:tempRaw', 'k2:dcs:sec:acsDwptDiff', + 'k2:dcs:sec:acsTemp', 'k2:dcs:sec:secDwptDiff', 'k2:dcs:sec:secondaryTemp', + 'k2:met:humidityRaw','k2:met:humidityStats', 'k2:met:tempRaw', 'k2:met:tempStats', + 'k2:met:windAzRaw', 'k2:met:windElRaw', 'k2:met:windSpeedMaxmph', + 'k2:met:windSpeedMinmph', 'k2:met:windSpeedRaw', 'k2:met:windSpeedStats', + 'k2:met:windSpeedmph', 'k2ENV_mjd'], + } +} + +def get_columns(): + all_data = search_files() + for name, data_files in all_data.items(): + columns = set() + for i,file in enumerate(data_files): + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + for col in df.columns: + columns.add(col) + all_data[name] = columns + + return all_data + + +def search_files(file_pats=file_pats): + """ Finds all filenames matching the given file patterns """ + all_filenames = {} + for name, search in file_pats.items(): + filenames = glob.glob(search, recursive=True) + all_filenames[name] = filenames + if verbose: + print(f"Done {name}, length: {len(filenames)}") + + return all_filenames + +def collect_data(data_files, col_dict=col_dict): + """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ + if isinstance(data_files, dict): # Return dict w/dataframes + new_files = {} + for name, files in data_files.items(): + if isinstance(files, list): + # Recurse on each list of files + new_files[name] = collect_data(files) + return new_files + + all_dfs = [pd.DataFrame()] + for i,file in enumerate(data_files): + if not os.path.isfile(file): + continue + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + + if col_dict is not None: + df = df.rename(columns=col_dict) + + all_dfs.append(df) + + data = pd.concat(all_dfs, ignore_index=True, sort=True) + return data + +def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): + """ Parses specified date and time columns and returns a cleaned data table """ + new_data = data.copy() + for label,cols in date_cols.items(): + date_col, time_col = cols + # Parse dates and times, coercing invalid strings to NaN + datetimes = (pd.to_datetime(data[date_col], exact=False, errors='coerce') + + pd.to_timedelta(data[time_col], errors='coerce')) + new_data = new_data.drop(columns=cols) + new_data[label] = datetimes + + return new_data + +def clean_dates(data, date_cols=['HST', 'UNIX']): + """ Removes any rows containing invalid dates in date_cols """ + new_data = data.copy() + for col in date_cols: + new_data = new_data[~np.isnan(new_data[col])] + + return new_data + +def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): + """ Casts columns to a numeric data type """ + if data_cols is None: + data_cols = [col for col in data.columns if col not in non_numeric] + + # Cast data to numeric type, coercing invalid values to NaN + new_data = data.copy() + for col in data_cols: + new_data[col] = pd.to_numeric(new_data[col], errors='coerce') + + return new_data + +def to_fits(data, filename, str_cols=['HST', 'UNIX']): + """ Writes a FITS file from the given temperature array """ + fits_data = data.copy() + for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: + if col in fits_data.columns: + fits_data = fits_data.drop(columns=[col]) + for col in str_cols: + if col in fits_data.columns: + fits_data[col] = fits_data[col].astype(str) + # Assuming the data columns are already numeric + fits_data = Table.from_pandas(fits_data) + fits_data.write(filename) + + return + +def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): + """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ + data = Table.read(filename).to_pandas() + + # Fix NaNs, because astropy is dumb sometimes + data[data==1e+20] = np.nan + + if date_cols is None: return data + + for col in date_cols: + if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings + data[col] = data[col].str.decode("utf-8") + data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') + + if str_cols is None: return data + + for col in str_cols: + if col in data.columns and isinstance(data[col][0], bytes): + data[col] = data[col].str.decode("utf-8") + + return data + +def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): + """ + Reads in all data matching file patterns (file_pats), combines them into one table, + cleans them, and saves them to a FITS file + """ + # Find all files matching pattern + all_filenames = search_files(file_pats) + # Read all data into one table (per dictionary label) + all_data = collect_data(all_filenames) + + for name, data in all_data.items(): + data = parse_dates(data) # Parse date cols into datetimes + data = clean_dates(data) # Remove invalid dates + data = clean_data(data) # Casts other cols to numeric + # Save combined/cleaned data to FITS file + filename = location+name+".fits" + to_fits(data, filename) + + return + +def from_mjds(mjds, dtype): + """ Gets temp data of input type from the specified MJDs """ + # Get pd datetimes in HST + dts = times.mjd_to_dt(mjds, zone='hst') + # Format list + datestrings = dts.strftime("%y/%m/%d") + datestrings = np.unique(datestrings) # one file per date + # Get relevant filenames + filenames = [data_types[dtype]['file_pat'].format(ds) for ds in datestrings] + # Get data from filenames + data = collect_data(filenames) + + # Empty dataframe + if data.empty: + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Convert dates & times to MJDs + data["datetime"] = data['HSTdate']+' '+data['HSTtime'] + mjds = times.table_to_mjd(data, columns='datetime', zone='hst') + # Add to dataframe + data[dtype+"_mjd"] = mjds + # Drop other date & time cols + data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], + inplace=True, errors='ignore') + + return data + +if __name__=='__main__': + pass \ No newline at end of file diff --git a/katan/time_util.py b/katan/time_util.py new file mode 100644 index 0000000..4974d2c --- /dev/null +++ b/katan/time_util.py @@ -0,0 +1,56 @@ +### time_util.py : to handle time formatting changes between MJD, HST, and UTC +### Author : Emily Ramey +### Date: 5/26/21 + +# Preamble +import pandas as pd +import numpy as np +import time +import pytz as tz +from datetime import datetime, timezone +from astropy.time import Time, TimezoneInfo +from astropy import units as u, constants as c + +hst = tz.timezone('US/Hawaii') + +### Conversion / utility functions +def mjd_to_dt(mjds, zone='utc'): + """ Converts Modified Julian Date(s) to HST or UTC date(s) """ + # Convert mjds -> astropy times -> datetimes + dts = Time(mjds, format='mjd', scale='utc').to_datetime() + # Convert to pandas + dts = pd.to_datetime(dts).tz_localize(tz.utc) + if zone=='hst': + dts = dts.tz_convert("US/Hawaii") + return dts + +def table_to_mjd(table, columns, zone='utc'): + """ Converts date and time columns to mjds """ + # Safety check for list of columns + if not isinstance(columns, str): + columns = [col for col in columns if col in table.columns] + # Convert to datetimes + dts = pd.to_datetime(table[columns], errors='coerce') + + if zone=='hst':# Convert to UTC + dts = dts.dt.tz_localize(hst) + dts = dts.dt.tz_convert(tz.utc) + + # Masked invalid values + dts = np.ma.masked_array(dts, mask=dts.isnull()) + + # Convert to astropy + times = Time(dts, format='datetime', scale='utc') + # return MJDs + return np.ma.getdata(times.mjd) + +def str_to_mjd(datestrings, fmt): + """ Converts astropy-formatted date/time strings (in UTC) to MJD values """ + # Get astropy times + times = Time(datestrings, format=fmt, scale='utc') + # Convert to mjd + return times.mjd + +def mjd_to_yr(mjds): + """ Converts MJD to Decimal Year """ + return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file From a58874a05f80b49fa348652d7ff8f0bc4396ce24 Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 10 Jun 2021 14:29:12 -0700 Subject: [PATCH 04/12] moved to katan --- .../compiler-checkpoint.py | 4 ++++ .../mkwc_util-checkpoint.py | 18 ++++++++---------- .../nirc2_util-checkpoint.py | 0 .../telem_util-checkpoint.py | 2 +- .../temp_util-checkpoint.py | 0 .../time_util-checkpoint.py | 0 6 files changed, 13 insertions(+), 11 deletions(-) rename compiler.py => .ipynb_checkpoints/compiler-checkpoint.py (97%) rename mkwc_util.py => .ipynb_checkpoints/mkwc_util-checkpoint.py (93%) rename nirc2_util.py => .ipynb_checkpoints/nirc2_util-checkpoint.py (100%) rename telem_util.py => .ipynb_checkpoints/telem_util-checkpoint.py (99%) rename temp_util.py => .ipynb_checkpoints/temp_util-checkpoint.py (100%) rename time_util.py => .ipynb_checkpoints/time_util-checkpoint.py (100%) diff --git a/compiler.py b/.ipynb_checkpoints/compiler-checkpoint.py similarity index 97% rename from compiler.py rename to .ipynb_checkpoints/compiler-checkpoint.py index 6500ed3..38c2592 100644 --- a/compiler.py +++ b/.ipynb_checkpoints/compiler-checkpoint.py @@ -11,11 +11,15 @@ import numpy as np import pandas as pd +# All data types that can be compiled by this package accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] +# Shorthand / nicknames for data types expand = { 'temp': ['k2AO', 'k2L4', 'k2ENV'], 'seeing': ['mass', 'dimm', 'masspro'], + 'weather': ['cfht'], + 'telemetry': ['telem'], 'all': accept_labels, } diff --git a/mkwc_util.py b/.ipynb_checkpoints/mkwc_util-checkpoint.py similarity index 93% rename from mkwc_util.py rename to .ipynb_checkpoints/mkwc_util-checkpoint.py index 9357193..7b626c3 100644 --- a/mkwc_util.py +++ b/.ipynb_checkpoints/mkwc_util-checkpoint.py @@ -11,13 +11,11 @@ ### CFHT data is STORED by HST, with data in HST # cfht_dir = "/u/emily_ramey/work/Keck_Performance/data/weather_data/" -cfht_dir = './' mkwc_url = 'http://mkwc.ifa.hawaii.edu/' year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST # seeing_dir = "/u/emily_ramey/work/Keck_Performance/data/seeing_data" -seeing_dir = './' # Time columns from MKWC data time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] @@ -26,23 +24,23 @@ data_types = { 'cfht': { 'web_pat': mkwc_url+'archive/wx/cfht/indiv-days/cfht-wx.{}.dat', - 'file_pat': cfht_dir+"cfht-wx.{}.dat", + 'file_pat': "{}cfht-wx.{}.dat", 'data_cols': ['wind_speed', 'wind_direction', 'temperature', 'relative_humidity', 'pressure'], }, 'mass': { 'web_pat': mkwc_url+'current/seeing/mass/{}.mass.dat', - 'file_pat': seeing_dir+'mass/{}.mass.dat', + 'file_pat': '{}mass/{}.mass.dat', 'data_cols': ['mass'], }, 'dimm': { 'web_pat': mkwc_url+'current/seeing/dimm/{}.dimm.dat', - 'file_pat': seeing_dir+'dimm/{}.dimm.dat', + 'file_pat': '{}dimm/{}.dimm.dat', 'data_cols': ['dimm'], }, 'masspro': { 'web_pat': mkwc_url+'current/seeing/masspro/{}.masspro.dat', - 'file_pat': seeing_dir+'masspro/{}.masspro.dat', + 'file_pat': '{}masspro/{}.masspro.dat', 'data_cols': ['masspro_half', 'masspro_1', 'masspro_2', 'masspro_4', 'masspro_8', 'masspro_16', 'masspro'], }, @@ -105,7 +103,7 @@ def format_columns(df, dtype): df.drop(columns=time_cols, inplace=True, errors='ignore') def from_url(datestring, dtype): - """ Pulls cfht file from MKWC website """ + """ Pulls cfht or seeing file from MKWC website """ # Format URL url = data_types[dtype]['web_pat'].format(datestring) @@ -122,7 +120,7 @@ def from_url(datestring, dtype): return df -def from_file(filename, dtype): +def from_file(filename, dtype, data_dir='./'): """ Pulls cfht or seeing file from local directory """ # Read in CSV df = pd.read_csv(filename) @@ -138,7 +136,7 @@ def from_file(filename, dtype): return df -def from_nirc2(mjds, dtype): +def from_nirc2(mjds, dtype, data_dir='./'): """ Compiles a list of cfht or seeing observations based on MJDs note: does not compare MJDs; assumption is inputs are rounded to nearest day @@ -168,7 +166,7 @@ def from_nirc2(mjds, dtype): # Find data for each file for ds in datestrings: # Get local filename - filename = data_types[dtype]['file_pat'].format(ds) + filename = data_types[dtype]['file_pat'].format(data_dir, ds) # Check for local files if os.path.isfile(filename): diff --git a/nirc2_util.py b/.ipynb_checkpoints/nirc2_util-checkpoint.py similarity index 100% rename from nirc2_util.py rename to .ipynb_checkpoints/nirc2_util-checkpoint.py diff --git a/telem_util.py b/.ipynb_checkpoints/telem_util-checkpoint.py similarity index 99% rename from telem_util.py rename to .ipynb_checkpoints/telem_util-checkpoint.py index 7ca7e51..30bbd76 100644 --- a/telem_util.py +++ b/.ipynb_checkpoints/telem_util-checkpoint.py @@ -13,7 +13,7 @@ telem_dir = '/g/lu/data/keck_telemetry/' # Sub-aperture maps -map_dir = "../ao_telemetry/" +map_dir = "./" wfs_file = map_dir+"sub_ap_map.txt" act_file = map_dir+"act.txt" diff --git a/temp_util.py b/.ipynb_checkpoints/temp_util-checkpoint.py similarity index 100% rename from temp_util.py rename to .ipynb_checkpoints/temp_util-checkpoint.py diff --git a/time_util.py b/.ipynb_checkpoints/time_util-checkpoint.py similarity index 100% rename from time_util.py rename to .ipynb_checkpoints/time_util-checkpoint.py From 9fa9ef553f35cfcd1c55d4efe0ff76c401163612 Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Fri, 18 Jun 2021 14:18:22 -0700 Subject: [PATCH 05/12] Added param files --- .../dependencies-checkpoint.txt | 0 dependencies.txt | 0 .../.ipynb_checkpoints/compiler-checkpoint.py | 157 ++++++++++ .../mkwc-checkpoint.py} | 2 +- .../strehl-checkpoint.py} | 37 +-- .../telemetry-checkpoint.py} | 21 +- .../temperature-checkpoint.py} | 2 +- .../times-checkpoint.py} | 0 katan/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 135 bytes katan/__pycache__/compiler.cpython-37.pyc | Bin 0 -> 3454 bytes katan/__pycache__/mkwc.cpython-37.pyc | Bin 0 -> 3710 bytes katan/__pycache__/strehl.cpython-37.pyc | Bin 0 -> 1457 bytes katan/__pycache__/telemetry.cpython-37.pyc | Bin 0 -> 4551 bytes katan/__pycache__/temperature.cpython-37.pyc | Bin 0 -> 8092 bytes katan/__pycache__/times.cpython-37.pyc | Bin 0 -> 1806 bytes katan/compiler.py | 96 +++--- katan/mkwc.py | 184 +++++++++++ katan/strehl.py | 60 ++++ katan/telemetry.py | 292 ++++++++++++++++++ katan/temperature.py | 277 +++++++++++++++++ .../.ipynb_checkpoints/__init__-checkpoint.py | 0 .../keyword_defaults-checkpoint.yaml | 127 ++++++++ katan/templates/__init__.py | 0 .../__pycache__/__init__.cpython-37.pyc | Bin 0 -> 145 bytes katan/{ => templates}/act.txt | 0 katan/templates/keyword_defaults.yaml | 127 ++++++++ katan/{ => templates}/sub_ap_map.txt | 0 katan/times.py | 56 ++++ 28 files changed, 1349 insertions(+), 89 deletions(-) create mode 100755 .ipynb_checkpoints/dependencies-checkpoint.txt create mode 100755 dependencies.txt create mode 100644 katan/.ipynb_checkpoints/compiler-checkpoint.py rename katan/{mkwc_util.py => .ipynb_checkpoints/mkwc-checkpoint.py} (99%) rename katan/{nirc2_util.py => .ipynb_checkpoints/strehl-checkpoint.py} (57%) rename katan/{telem_util.py => .ipynb_checkpoints/telemetry-checkpoint.py} (94%) rename katan/{temp_util.py => .ipynb_checkpoints/temperature-checkpoint.py} (99%) rename katan/{time_util.py => .ipynb_checkpoints/times-checkpoint.py} (100%) create mode 100644 katan/__pycache__/__init__.cpython-37.pyc create mode 100644 katan/__pycache__/compiler.cpython-37.pyc create mode 100644 katan/__pycache__/mkwc.cpython-37.pyc create mode 100644 katan/__pycache__/strehl.cpython-37.pyc create mode 100644 katan/__pycache__/telemetry.cpython-37.pyc create mode 100644 katan/__pycache__/temperature.cpython-37.pyc create mode 100644 katan/__pycache__/times.cpython-37.pyc create mode 100644 katan/mkwc.py create mode 100644 katan/strehl.py create mode 100644 katan/telemetry.py create mode 100644 katan/temperature.py create mode 100755 katan/templates/.ipynb_checkpoints/__init__-checkpoint.py create mode 100644 katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml create mode 100755 katan/templates/__init__.py create mode 100644 katan/templates/__pycache__/__init__.cpython-37.pyc rename katan/{ => templates}/act.txt (100%) create mode 100644 katan/templates/keyword_defaults.yaml rename katan/{ => templates}/sub_ap_map.txt (100%) create mode 100644 katan/times.py diff --git a/.ipynb_checkpoints/dependencies-checkpoint.txt b/.ipynb_checkpoints/dependencies-checkpoint.txt new file mode 100755 index 0000000..e69de29 diff --git a/dependencies.txt b/dependencies.txt new file mode 100755 index 0000000..e69de29 diff --git a/katan/.ipynb_checkpoints/compiler-checkpoint.py b/katan/.ipynb_checkpoints/compiler-checkpoint.py new file mode 100644 index 0000000..08e4ad6 --- /dev/null +++ b/katan/.ipynb_checkpoints/compiler-checkpoint.py @@ -0,0 +1,157 @@ +### compiler.py : Compiler for all desired data (nirc2, weather, seeing, telemetry, temperature) +### Author : Emily Ramey +### Date : 6/3/21 + +from . import times +from . import strehl +from . import mkwc +from . import telemetry as telem +from . import temperature as temp +from . import templates + +import numpy as np +import pandas as pd +import os + +# Check dependencies +try: + import yaml +except: + raise ValueError("PyYAML not installed. Please install from https://anaconda.org/anaconda/pyyaml") + +### For importing package files +### May need to edit this later if there are import issues with Py<37 +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +# All data types that can be compiled by this package +accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] + +default_parfile = 'keyword_defaults.yaml' + +### Load default parameter file +try: # Load from templates module + file = pkg_resources.open_text(templates, default_parfile) + default_params = yaml.load(file, Loader=yaml.FullLoader) +except: # Warn user + default_params = {} + print("WARNING: Unable to load default parameters. Please specify a parameter file when compiling.") + +# Shorthand / nicknames for data types +expand = { + 'temp': ['k2AO', 'k2L4', 'k2ENV'], + 'seeing': ['mass', 'dimm', 'masspro'], + 'weather': ['cfht'], + 'telemetry': ['telem'], + 'all': accept_labels, +} + +# Utility functions +def check_dtypes(data_types): + """ Returns an expanded / cleaned list of data types from user input """ + new_dtypes = [] + # Check requested data + for dtype in data_types: + # Check for nicknames + if dtype in expand: + new_dtypes.extend(expand[dtype]) + elif dtype in accept_labels: # can get data + new_dtypes.append(dtype) + # Return cleaned list + return new_dtypes + +############################################# +######## Interface with other modules ####### +############################################# + +# Have files with acceptable columns for each data type so you can check inputs +# Accept column inputs to the data function and take optional arguments in combine_strehl + +def data_func(dtype): + """ + Returns the function to get data for the specified dtype + and whether data needs to be matched to nirc2 mjds + """ + if dtype in ['cfht']+expand['seeing']: # MKWC + return lambda files,mjds: mkwc.from_nirc2(mjds,dtype), True + if dtype in expand['temp']: # Temperature + return lambda files,mjds: temp.from_mjds(mjds,dtype), True + if dtype=='telem': # Telemetry + return lambda files,mjds: telem.from_nirc2(mjds,files), False + + +################################ +###### Compiler Functions ###### +################################ + +def match_data(mjd1, mjd2): + """ + Matches data1 with its closest points in data2, by mjd values + Returns an array containing data1 with corresponding rows from data2 + """ + # Edge case + if mjd1.empty or mjd2.empty: + return None + + # Get mjds from dataframes + mjd1 = np.array(mjd1).reshape(-1,1) # column vector + mjd2 = np.array(mjd2).reshape(1,-1) # row vector + + # Take difference of mjd1 with mjd2 + diffs = np.abs(mjd1 - mjd2) + # Find smallest difference for each original mjd + idxs = np.argmin(diffs, axis=1) + + # Return indices of matches in mjd2 + return idxs + + +# data_types can contain: 'chft', 'mass', 'dimm', 'masspro', +# 'telem', 'k2AO', 'k2L4', or 'k2ENV' +# 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and +# ['mass', 'dimm', 'masspro'], respectively +def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=False, + params=default_params): + """ + Combines and matches data from a certain Strehl file with other specified data types. + NIRC2 files must be in the same directory as the Strehl file. + """ + # Check file paths parameter & open if yaml + if not isinstance(file_paths, dict) and os.path.isfile(file_paths): + with open(file_paths) as file: + file_paths = yaml.load(file, loader=yaml.FullLoader) + ### Add a catch for YAML not being installed if file specified + + # Read in Strehl file + nirc2_data = strehl.from_strehl(strehl_file) + + if test: # Take first few files + nirc2_data = nirc2_data.loc[:3] + + if check: # Sanitize user input + data_types = check_dtypes(data_types) + + # Full data container + all_data = [nirc2_data.reset_index(drop=True)] + + # Loop through and get data + for dtype in data_types: + get_data, match = data_func(dtype) # Data retrieval function + # Get other data from strehl info + other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) + + if match: # Needs to be matched + if other_data.empty: # No data found + other_data = pd.DataFrame(columns=other_data.columns, index=range(len(nirc2_data))) + else: # Get indices of matched data + idxs = match_data(nirc2_data.nirc2_mjd, other_data[dtype+'_mjd']) + other_data = other_data.iloc[idxs] + + # Add to all data + all_data.append(other_data.reset_index(drop=True)) + + # Concatenate new data with nirc2 + return pd.concat(all_data, axis=1) \ No newline at end of file diff --git a/katan/mkwc_util.py b/katan/.ipynb_checkpoints/mkwc-checkpoint.py similarity index 99% rename from katan/mkwc_util.py rename to katan/.ipynb_checkpoints/mkwc-checkpoint.py index 7b626c3..65aa637 100644 --- a/katan/mkwc_util.py +++ b/katan/.ipynb_checkpoints/mkwc-checkpoint.py @@ -5,7 +5,7 @@ import os import numpy as np import pandas as pd -import time_util as times +from . import times ### NOTE: Seeing data is STORED by UT, with data in HST ### CFHT data is STORED by HST, with data in HST diff --git a/katan/nirc2_util.py b/katan/.ipynb_checkpoints/strehl-checkpoint.py similarity index 57% rename from katan/nirc2_util.py rename to katan/.ipynb_checkpoints/strehl-checkpoint.py index f34174b..9846ac3 100644 --- a/katan/nirc2_util.py +++ b/katan/.ipynb_checkpoints/strehl-checkpoint.py @@ -8,40 +8,13 @@ import glob from astropy.io import fits -import time_util as times +from . import times -MAX_LEN = 10 # Max char length of epoch name - -nirc2_dir = "/g/lu/data/gc/lgs_data/" -strehl_pat = nirc2_dir+"{}/clean/kp/{}" - -strehl_filenames = ['strehl_source.txt', 'irs33N.strehl'] strehl_cols = ['nirc2_file', 'strehl', 'rms_err', 'fwhm', 'nirc2_mjd'] -header_kws = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', +default_keys = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', 'AOLBFWHM', 'WSFRRT', 'LSAMPPWR', 'LGRMSWF', 'AOAOAMED', 'TUBETEMP'] -# Note: need to get better at identifying which strehl files we want -# Alternatively, we could have people provide a list -def search_epochs(data_dir=nirc2_dir): - """ Searches for valid epoch names in NIRC2 data directory """ - # Epochs with valid strehl files - good_files = {} - # Loop through all sub-directories - for epoch in os.listdir(data_dir): -# # Only valid if {yr}{month}lgs -# if len(epoch) >= MAX_LEN: -# continue - - # Search epoch for valid strehl file - for file in strehl_filenames: - strehl_file = strehl_pat.format(epoch, file) - # If good, add to dict - if os.path.isfile(strehl_file): - good_files[epoch] = strehl_file - # Returns {epoch:strehl} dict - return good_files - -def from_filename(nirc2_file, data, i): +def from_filename(nirc2_file, data, i, header_kws): """ Gets nirc2 header values from a filename as dict or loads values into specified df @@ -58,7 +31,7 @@ def from_filename(nirc2_file, data, i): # load DataFrame value data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) -def from_strehl(strehl_file): +def from_strehl(strehl_file, header_kws=default_keys): """ Gets NIRC2 header data based on contents of Strehl file """ # Get directory name data_dir = os.path.dirname(strehl_file) @@ -78,7 +51,7 @@ def from_strehl(strehl_file): # Loop through nirc2 files for i,nirc2_file in enumerate(strehl_data.nirc2_file): # Load header data into df - from_filename(nirc2_file, strehl_data, i) + from_filename(nirc2_file, strehl_data, i, header_kws) # Return data return strehl_data diff --git a/katan/telem_util.py b/katan/.ipynb_checkpoints/telemetry-checkpoint.py similarity index 94% rename from katan/telem_util.py rename to katan/.ipynb_checkpoints/telemetry-checkpoint.py index 30bbd76..3757ee7 100644 --- a/katan/telem_util.py +++ b/katan/.ipynb_checkpoints/telemetry-checkpoint.py @@ -7,15 +7,23 @@ import glob from scipy.io import readsav -import time_util as times +from . import times +from . import templates + +### For importing package files +### May need to edit this later if there are import issues with Py<37 +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources ### Path to telemetry files [EDIT THIS] telem_dir = '/g/lu/data/keck_telemetry/' # Sub-aperture maps -map_dir = "./" -wfs_file = map_dir+"sub_ap_map.txt" -act_file = map_dir+"act.txt" +wfs_file = "sub_ap_map.txt" +act_file = "act.txt" filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" @@ -39,7 +47,8 @@ def read_map(filename): """ Reads a map of actuators or sub-apertures from a file """ - return pd.read_csv(filename, delim_whitespace=True, header=None).to_numpy() + file = pkg_resources.open_text(templates, filename) + return pd.read_csv(file, delim_whitespace=True, header=None).to_numpy() def get_times(telem_data, start=None): """ Pulls timestamps from a telemetry file in seconds from start """ @@ -206,7 +215,7 @@ def get_mjd(telem): # Convert to MJD telem_mjd = times.str_to_mjd(tstamp, fmt='isot') - # Return telem mjd if they match, else return False + # Returns telem mjd return telem_mjd def extract_telem(file, data, idx, check_mjd=None): diff --git a/katan/temp_util.py b/katan/.ipynb_checkpoints/temperature-checkpoint.py similarity index 99% rename from katan/temp_util.py rename to katan/.ipynb_checkpoints/temperature-checkpoint.py index 18c3053..9c672e0 100644 --- a/katan/temp_util.py +++ b/katan/.ipynb_checkpoints/temperature-checkpoint.py @@ -10,7 +10,7 @@ from astropy.io import fits from astropy.table import Table -import time_util as times +from . import times verbose = True diff --git a/katan/time_util.py b/katan/.ipynb_checkpoints/times-checkpoint.py similarity index 100% rename from katan/time_util.py rename to katan/.ipynb_checkpoints/times-checkpoint.py diff --git a/katan/__pycache__/__init__.cpython-37.pyc b/katan/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f28df4013162b34e912dc6e15cb29b97195a24d5 GIT binary patch literal 135 zcmZ?b<>g`k0*U5B35-DcF^B^LAOQy;E@lA|DGb33nv8xc8Hzx{2;!H4eyM(HZe~tp zd{JU&ryk0@&Ee@O9{FKt1R6CGK IpMjVG0CNN%EdT%j literal 0 HcmV?d00001 diff --git a/katan/__pycache__/compiler.cpython-37.pyc b/katan/__pycache__/compiler.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04bb419b7d1f02198940e59a9b5d221101dedbd3 GIT binary patch literal 3454 zcmb7GOLN@D5yk-6T`cw?isGA+1?S*ot6Zeba$G5uOJ!NIQc7e-krc^>+S=!q*&d%}M9nd`Z3@kQB>41t`q)^fbC>x_iE!PkKGq zz|%Fq{oPl~hVgG2oIehTkMR|cm|<{(v)D+O&lu4rXwx?}ZGpCYOVc)J+iz&P0lMk8 zG~ER4_zvh6cjC71>b~Q5b>H)QvzqGr{pUuy#0+}H72ZC9y)TT^{M?u=XxWA2s=vq# zNL{xLaqYmB|EQ()?2^A&OLLcZc$fEhpI_k%{OZ{8m-(W8fzV$}7`f6aC4$ zpmu{_dujTc(7IM@iwu7g_Ltt+uQiRWS}%Hc%%GQvWxo8^hJT%}@YNTFe}k{_b@XrX z4Zex~Ccn;a@atl8Wb?Q9%@@q~xbwsqc(37AW5BA0iV{(PXckI}{TP&;92|z|+e*YD z5lYT5=qQooLI!FoMLn0~aiB!;8hVV~pRk#5u~qmCB1-+uK1JgjvditxS$D2HW{;79_EkNiPY~Db&ooA{Svant7ur7G55x{i3qN z(Z2HSBq$2s=24RPEuwRo`S!uionKM=@B_bbu=DBeGi;J}a=d&XW`~*NLoPgE5tPF{Zq;A7ESt@Q9Gb#n#Rj7rdFQ7kaoa zQZLMsJc^@qyftW4Hu(^a76J^vbtteEA!P#&tSsy_q?piS;Un%{e8tCT#>Si-eOMaz zjIXxlW@#$ppef%e*(p;-X|5Zukf*NNj5U z!*LvrTO~*S=ugIoaf{n89Nrk2PauKW<~q&Y)^n}PbKE&F{c*`L=d|`u`@$K&fS5LHG!~JUU7efq2Dle%B=6Iwk2x7BjS2oK z*~E|w9LH9%Qxhx3apoqsUYYZn!!!pxa2RCl5{G;Za&uF$Nuz`;w_lobtF#d4#_?b3 z%=%0xPocBweeW<*`(C6906?ae5?ysv|JD(WOV&ukz_AZ0KE zyrn9LQoz||tzjmmC~{ysO~~x<3`)p1J33o;U;h`6JgY%=a1<4TX4L>Zs##^GdDYPR zs}}U^2e|+|?h(4kaT28i8^Ax|@9TC4h!TxPNZ5!U72k_q<>~`7B*S0Azcp4gyUey0 z*`n1q`jwVoL2|9^!G5=NPS-Iyv#U$R*Ut@T+bA9G@b*iB2>1AJb>`g7 zlD#NJMIdmhF{nXQ6XC(*yE|IG@RBLA z5FiIV8XZv)JRV6As!Yzj02m0&ykw!)K;@fiw2-^63F4oEVRM<~z_M1D#nU7$sz!ur zcjVhcgRwR(6J}R@ro0sBAJ;x%a1W zT;CcbS@k;_vg$ijWdR3u5%N1oV4;TEygvLWh6LP0((AWtnil0jNoD4|YTtv4?o&CG z?-H*egLI4o5Q|jjs&7Z6eUavxIDcWF27SL%Th?kN0X|S27KZh0siBJ|-9HLdHT5M1 zThvQ9TMnHN-@zS3pB0Vzm4kw%hgCzz+;?>}Yl$Dj#`jNKABCFoWLPg|3X$R+G)8NM zxopFBO&itOW{%ajT-G&-Zn6z?iMiH_CGSCU@OEWLDqPyDx<2Xqkp_23hg4S-*#cEH z(rJ><Z<2F+Ac~fh%An@<`!T2QIk)lt z;$q0~{QWO~`&_Ru_75sNdTPjA!<*l*7~_mv1J){2t7V(A({j1Zoq;#>TfW7(+p35P z_f}Xd;C`#hD?H#;9`XfV+w@x@+7?$Buk#~(i67<5{Me?)kALl;{sihz@|8_%UOt8L zX@2IQd=};B`1AZ6KhGon0)LUev{|9P{N=CR*20bw3pdzjgH}z{`0AD;zZGE`ijZ6U zmEYT^S!+=&iUmc)b^I$4Rm-__#Se7t2mBxhI<2JIzkFr;J(K-dVpt zTj#ObJb&lYPyc=EU)Qud5wXJ2Rzmx^=wvCMHlCa?NqKT_ z4FmEg=AJIypV5+!S8Ija=-tjoLhx4I4v+r9BH z;fb2)U?fGJqp{)W%C@-M9>q!r-DDt4u8OaT+nsEXPtSM8Qi@bHb0Ly+vpJ0Oyov9n zls4rwT*kA^;3r(LRa-*}w8+zJo+zNIVke8bf)p)ZfXS>zI#qhyw09{vZ9tRtrbLXuF^ zZhP407(;odmn-dzRYzA1z|$-h9UJ}WA;t%Iuj9=%jADCMVeMNpdvZb9+yYC$nVF-U zzFUCb+$|jCDZg+YSPv}j?Rqo!4olBijDF}%4YG7IBE-r;Nt!DW^C;_@E_WxOi4vwmUiL7@A&&#Z+qZCHOI!?+VSBrmD3Ooi z4rtfI1OmiNV{lzp#zezGE>B_zB%p%Ac$nr7U6ddCRC!bfw3qf+4z%CRW?M{AAdooTpg?0ua)zusLqMH&6X(vhH0E4U}JaNP zunSgL+*x5)S;6`aT3qgNf7d}6tH5!B=|*(DN7&}k9HZ#9B#lM`oN8n4Hjr<_y+~!z z$3MFfNrX_=uqAC*yF@y90@>TzPd3wxl&@^6B&nH4`$Ec0=5QTJTh=ud--RBk%y79% z@nKGGF0&Dazd^k`PM!e8>jQB@xHNB3ZVl#hyg3~?bC#@{W9kl|Ay^LB-$OwG2=Ep! zRDjJ@psXuZ!0tN-D&Q*bf2e@n_vbS`R)I?479~Gj1L<1yi}7FpKykuRCZkf{OTrl< z8eT%ta_}dTbOAZwmOMkHvt*tl^E{c7MF!MHAYY=Y7sp=arZvc~kP$X`$D3L0Eu3FWQNm6z>#`|J}| zI6&8%Ui^RPLJs0VR7UJfP9p8VgGvvbJFUEeZ8rQv_!(FB5skrIgPy?=QfDH!sU-k) zKoZc#+|LBCY5*i*y-#422!33QO>Y<=6A^ON~;H4&ynCI`@ zG;{d<1-iXbMB29yv({&o!Y>ZhQd~m+K-G$%s64O{lf1pfc?>GRy{?KgSBBHtd0KLVU7P-~ z3_sVi;RvASQ5+4D9GvPt$*t^eF62F2#j-Sy?#8)5G)qaWauXKQOo=NI&oBkDP`cqL zmcq2Z6X7~DMzFxpD9Iye))+CAnq@XlIc9?FrI2Va3Bhz?CX}afZVgLbMl$tRf3rGV z<%el%0_H7w9?fzQ<~D+0tyIT0?v`SwaiWyMDtwcKkSoJ$9Uv5SNf3>5YX1`<6N-lE zNc-b7`C=@LXKE)-Cwjrekj_|T-L7^p(u4=OPOFkYXgkA&2^Y{Fq&X+QG|p!@ESu=i zSTKR76o;EM!r>~H6iVa2nyf?P{b?dQ8-GA^PCHnsWv*4N=Sq?c+`jk{x<=NB#=EfaLp!l4J6hDB&`JbD7AGDDRKrl*f6A z3NzTX0{SaKdL>^#7x{BCKPF=olK9&4l(~jb)GTRD@>POB>mK?9^Dz`#bZ~W;@$o>s uOT5hAg$XSB)qEG$^_DyzWJWx8C=Cu-#A`3KPvLjYKIaEfPz{#DrT+rp>%EQu literal 0 HcmV?d00001 diff --git a/katan/__pycache__/strehl.cpython-37.pyc b/katan/__pycache__/strehl.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5088ee2cb5c9226a3ee0795981514691fe73728a GIT binary patch literal 1457 zcmYjQOK&4Z5bo}I`jt4@u-PmJ280B@#L^y+kRpWh#1MOfZDiB}%*5S}$DW6`XS|75 zK+30sH?hqs=uzPull%FDmv)*7J`JfOqIATCpyuRu8p%@O63rNOTJ2t^)$dcQJ!p5% zsD1Lhb$Z&T&iSz2ZNUa-pvJ`S7a_wTJ!lUT;9%%YLhKbv5i! z=klW0=?~8-IKkIBJEiX6yW_LLS?6kOs=#4DZBYs43C#2-5E+``4aw0xPU|Yi%-ER9 z`mMe;bCZz+G_%&$5UTzy>Y?w^-KV*6ihlIgc5ZF#+|HbtdrxvaE9E4&H|2XYAxG%N zXX*r0%y@O}l|2|2JWD;Dy*K4P| zy&8%&H8pF$z~VpF)+94#z!<}fFe9^=`Kz@ya>A^AlpD-GK!rX+nZ0px1K7f_?;N4K zr#S{Dch_cCnw4`SH#e1g1l)Z+M0X!&Rq(C7RvwRkAZu%FXIr@qURI7@n;5+ahky?Z zyRSYk$nCa!C(jl8M*X4@2|q4Z6aBY#OGlGad%f=yn=p zJYD!>E>uONo6duUNTB|fDpmRpH#JApWqdqZabGmHg$tfSHMMkDAe5gZ3npCweKs1W zFJ+}D{b-zosZcLLmf!`BvShRpLe*7lDY31DVnddAyo|W;Gp=vbWHg+AadcLPnLixJvenx>3fv#va)<#3!)d z*wP6?0hZBXr6ZO0!jGAsiVxtwG;hM>nwEV4OctxdASs$2=}g6k;6kg6PyA(=jplrn n>TERh{{mBeEOd14QIxP{$iGmb>(E?Wd{t5G3VZj{s9G=0!7=?00j#4WT1x@NH0av_V;EbT2LwM?97`tvpeto z-tWED^?E%p@cZq5zWw>%pD~QT(%|q{Mdx)~!Z!_r8O$6Qk*V)iWb3;VIk;N`H}fLT zG?>ktK_&AeEI{8yzZz9dW2dRLgKQ?MnZ}1ke%>*X`c}X zR*9O-k4~^^G?&b>;It9Vvzh25t3{8o8msrL=oD*2r`c?@z?$p?n`86rBzugVVyD?c zuNs|UXV}>ZFYe@0r`0rvL~VWID3kH5BCe~ z`|Kj_Pp~WOY4!{>LUxI@AbFCRcZ}Av56M1jEmKu^5;GCseF*ay7`rmf5`j)tCfR5Z z%S7y+UFt0j#!D=g@zPe(*=oyVkYtJEJ4?F_G2Up$qjnaL7Uj0wt;8LvJN3oOolBpv zE3L(DD#h;n7n7ySmzVO_zL;Eo`;9x77xB#AzrXp%A8v2F`&#-bo}PY2O&d+`ci+;_ zcs&aAliJ;MkkAX^sqL)KqRRSuJ4@m`^3V|yf|c6`0(3|)!!vQOo0P)fU_cW!A$hVn zD38#gL(|W~<>I=5ON3~ov1dxFF!rs9Deb;fn9O7rvme-d?!EyFI=ep#-y_q7afq{r z!)}O^k7GIHA{_D%W_d0iC0veqBEl{oW>BPagzIZ9OEp{)ckMq55#D2%?*hqcZis?Hb_^U=647Z6kDO8dPUzB?WaM5qBL<7_lF z)AUtJgQa;WlFl%Pt+gG6jJXWgI61F}%lQ3VXBo#gV_<~POPjTi;0wO#Bsl{lx7fA+xYQ z_5{TJ-QNTue(xm!aUROeBpeJou}p_KAl}u2yYV1pN01L+k=|h_gfZvwj@|`;PMLTm zT-*M^i$4zaXdDm1d-2@_Ud?5x8eYoM4u+zo7p1u@mpuPsh<=ibRPG#V#`#V-&Ot7v z?i%(UXB!FE?{QsvRalF>E>wv;eYXK%>X=`2xVpE zBjsfYOXFNQogu=+y|ALksW8nn}#CELo~P54kK8}F~|QOsI_ za_C%9U1?+u4cYgNl$M~dUX4zzu- zNNiohVZ{O(!w>9+?VEG$DQCuTNnh;N9@R^JS1r z+1?K0oQ3G@&}_^*V1ZBv{C6H(?%lm#|L&hp{^`|+L_!a1Xi7>rbmMtQlp{XpvgJfB zwuxca6=g{oc^D;}AGaEmu8y-`Vx92O7?uMgy<+kg(4W#C%}jaV7Z^-=z>-KA_insQgtX( ztDrN?MnI)VSj$%d?OQSef0{PEL=9>;QpXOF-vvep^!zdjb%Y>`ANQCKSsQzaUqMSy zwll22_JQ)VX47t1Ph0#akWJA|kxvNM=)QmsId2bn3kHSr`f!}i^47#EES=?F2tOc- zqdF*z!(*8#?!{>?KwWuw`^}XQWH?U3U3Vn<%IVI!BzU9XKsR2=}2J!tGQ-bg6g>*H+7@p@3O$o#ezl+T=PmlpSvf6|CR7 zeY3rC^X>IzK9Bk6#9_9ZCOcJo@8+FbEA1QaufMaps;Hj5wer!Oh^py$1imjbon{`5 z;4+l#WP61AFIC>#%eSwuEc2JKy|Pi)DjTVpveG~xfM-eU=)%WF&QR6PCY9|#3%^b; zOn5@}O9sHB0*Sn`gjQZzJ|72TovKrcX^q=IV}+pDH*_fa zW&=FxAnh!eHQPskp0Y1EmT6H6n*N=#`8rlid6v#VT&tn`Yjh5vYr&bQwn)@f z6q~w&0o-iOxC9xn@1Dn8^jul#`;veNc4i*xDQ~-9j#i9MKLd>IqJpZ))vZCTjmmc7 zPpSl4fVEoq;N=-v@7Dp@3iGi?ZTqRBhV^GBb=fHDg$H;Ctop#2G>QiQ`d#DmuRkb{|u&V%>pk;PRQY^ za^O2vr)N+Ff;Z`Gmb6sG3lJ?aseCARsaWR3D5|=Gpjf4#;A=E?d7SqW)zF(jTJEB} z@?^^C4Tc-q!+5wu$5PcpRiWu0oL@d4wGxv;&1lkHn~eyfR!C@=$i4msmIzI15R zG#w+bcn*!0iE?qR^^|g9p3zPj>mHdu7TOxT1FgsxopiLbm<}~9C_|gD{Aim`vk_{; zLAtSsI6M5Zn|3+cuA9D3*#iMUQ#Hs!UW+{(+svgN={3_P+PpFBhS*7L; zYJQ3apT;*=-?{Pr9sU-H$W=PE>o1%#t5bPXk__bz?IV5aBSB7?mFrI?|5`R=<3aK& xeTxcuLK(s}SgnSjh9!}O9NRG+DxWOfnlz$-Kl91PR zWLvKo^4F}G_|=+LD^tm+G)8aQt!yQWy1~q5t~FK}L*1(6S*9}XjWc^jsT5e2<=7a@ zvvF2n6RgNiut|23O|esKn$56TcKVxQWx^}4GcyXxv$8xlqp(-L(JMtz&Z9nsx#O5M zfw{E8tBovkzR}S81N2T~l^Lw}TEpy7UkCLKc7eUgeu$kfvJ$(*-eQ;8+iZ@_vv=4P zw!q$HSJ{u)d+ayZ`|QVTk^LsS#(u)Cv!AjL*b=+JK5YDeR+HyA!Is(0FO|xqSFEdS zh28p6t(^2GQ2&VCMtzFiVIMa%c9-2lx%!QUp8M<*^qgY9#nu|sUT0;rrcs=ATH>i2@%oXl@CojBKyGr%y!hC0T`^uslE_%W4;)WO0HmUPUb<8aDp%~~dpKZiyIje9 z{Nd8QYI&iOy?>+1{l=y!SMu%6P=uJz{Tjxiqv_Q_?~T1uv6t>IRckISbobVJm5y3D zK{Xt_8AdzYt77L$o=E*8R3?uOph=}-CAW0HD(_pqGE!gY9DlW{05^`g+T4SsyXaZy zbMR)F?kV}3a&XYIe2?xu=$RSLAx02AW$o!<$Nd|UFUQT1j2d)|9N_*9V%ebQm~B$o zP7k+P>g_KjJ2j-=T^}rSW|T@wveP2z_OxVh$Ghu&ruVdDqzAJG%bnX^Sh`QGX#SG? zKo+t&*9;pSYwB!c!@^p(x2PsVyZ%^+J!q}anBFfvd>{~ei(Gl_I7%yhqnCwPV9Z| zg_-MZ&tsS+trh$~x?{h$)!qbSj_IS0>0w71f?eKgi)FuFA6%);y@>3SuG9yKJ}uq0 z&|85dPd7Ny#-T^sv#DM8BVu6O7{ce_sJLOw8*I! ziRu0?jl}|$tTYf(4Nl&4Bp+5gX2IaHr3%gW>b_+0C%T}^19 z>-+yC^jLeUg8uiQ8(s6@vM{#H*o05i8Y@%%vJyZfmRp=u! zd{@{k7n?s?72J`w!c$fBgf9YhPU52{}6pxa?#<~&8fQp zD*S}pA;@w%g>~$Nts1@d*kvaOo%CciiV^6K4sYSRj&F1hMFW?6Jk?cW^-uG);@pWFsTn|mkGfsucr@Us~8-McTu9Tajbg=Itq&{}>l zzl|s(m?vBCxGMkvbM1rga@U%EBx+%+eZ5LYO3&AMwc&{>*4}9a(cHCWSOfE}qtYLn z*q^T6ezfcguXFmc(;JBE3dfBcZ@&#(^jImM+y{+!fo3e$erj-FGw=IeG=VetYjy)+ic3%DKP>>T1igE zQF1GOkycTC3e}p}fz|mLP$Z+$f_R^dja>5{2+;xxMYYsP`PI~-TENFr4V1Q4z;992 z(3|Vk@~G!E{tjjhL~Npw!B^VuC#d*JSKVKM~%=ES%O3kI>sOm%{*xo4f3Qy$!Ls=#)(E;KYg1Xoeg70;>E?Dt?Hf zWF}eSCOPLQ87JmbQ*O=QqFGid?Q~d5C(Rr(oWF|ZHoj2-g`yc?9~cW}p3`Jsjr5QH zD6Ja9S2A%$0${8N^{K|MgPq#`C9qSI>^u*4>Rm%9u_1Ky&!$X^wO9vu%z|awx+2U& z>$%cUyXOACi%blF0#yDQGp%k$*jw4d9Dz@?`D6{iX>I|Ez6Ut-1K_krfODMxxhMcQ zGXvmU>jCFPY;P5Nz?q0~zJ_sFeFQkydccWs5;*N@5x|Mob_S_O;mDjmbgsoJJF%Tb z{~H*2064`81tElijvf7;+q1 z^79z|zW^n2N(op1^}OE*LgY@cZf`#^qL7QyRFd-}KZpR|z<-0p)w5C(S&|>!p1+Q* zk}QzfT_Mr%*Ql+9QDVSsHxtW`$Y?n|PNcUzYUC+!T3niSV!7?M7qG-aJXUi>DaS9< zayE9X(lIL8z}ri~Jw*VM7LO_!iOltgzfIfBQ87;w4dT!XAk43!XBXe-0~Crz&gffyt6S^;z9YDwm!^)j~dWQEdlDS^~!mC*#Fbqq-i@?rzbp`eR= zv&0;RN($LPn+NdDY1nlqMVr{bui3SPwnbrzc^c05;QcOM6Kur;U{Bly1Lvs z`X#1E1QCj6L-4jXsZK%s@UNeN#_v$^F^Yk}C>$IJjI?YZFefH}pY6hr3iVJUmY-Ih zs}ss;I1lMZ;S@@0=Y40*YlXW6hrxkEkt3KL2p0h(+^8QQPV`f0L}*E|$h$iwo!`fX zi59dG)A*Ghy_s=jWBBi7I=G0k@e~x-hCkp;3j-W5Kqq`^o z^Xj(E&;7oFH!mpLmv*ddAs2Y6B7Bs20WyKtnOS)dL4&0}k;L)K16U zts)!r5(Q2_z!-_WAH==1!5Nyoz}F{7{0hD0nWj8u_x(B0k^h-L;l z>q@!t$C&zpbg+`7%7`#G-1;x+M0pe>o++I~bVlP}pnXI{#1V-YZB%*+sPDhQNCD5u zz)s&%w=G_X^(`$nkc4T#0iBd^|6;6Rlru8wYAUirT^P?19!im2C0~AM#rkv1L!y+4 zRr2wS#AtMW zV}!f|^7?LVsp;}o4>(o<1ElvU)7MagkBCT55Y-FLNmr9G+8t3M1qq3{N2lvcLcZ=$ zXn|Bsa4C`sf=Urd968ovGLR>Y_&Y2eQLvC&3%Q0IKGT5yoW>iVj~xB~aHAta9H7Xr zu#SK&#Xt)RFcGKg0CcdOa^GuYgBon`BH7UX9cgPfU~6V7HU^Xt7J|@6X?rVkh&;P5 zNRi#j^~RYp50~}{o17z8AlLmpx$ZTo7FT;}F$U%N>ZKjNI&Oz2U-hip?yLdXTx1A9 zEf+cgJ15snxmtkS3Qd@MDx%BM(MTMYG2PJLT=eThpHy-hAN;{z?Z5IDr zu;WK)Ae$gqMUocdd82FW--wM~zVTkq^XsvN9t+t<`fH}YNaIZWi$sOr*Mt>Yc$UHY z&66_^5f8v#_0AtUIR}43=^%w8CpygDNKlYXFK4?*O__6*NP4XedZ-u)$0>!A-v4sC zumrGlKq}eDCI2SlcM0(Wl#u7>m~+Im)lx<#U^0G6!vg7a$s&DfUiY1u9hOME{175oH02WozW>ZTe_=wV?43)Pa*H(A}&UJv(!_e>Eo#n_$^vy`XVHzEX zw_{&8n7`1Pzrc>TVu}r}YQh64G5EZ6xBm~$|V#$9- ziJp(fBZ>T<7NgD=oHM-5z&XDn4GoW#$36_n7_jFo_$}=Do$Bz9@SDbk6a!*EdY$1< zh-%QYr;N_|ZL}*En(JPzj6_8yLWr|yaFPMj==rM;Fm(AOi@{aE3lq|v)M@n;WGwiu zCp%u64VYlQI7p%8Cz!`MP0mZBq3Lu2a%Ck%#i&Pohx&DfSB^0XX6eB#Xhi%ynmbT# zT$09d88)QoLzDjtw1^@<1=vHbX;QOw3-QCOTGaB;b2zJ~=uPZuwS_pdS}nc8uM@#U z)Pj&)f~3_$ObLez)SH8wMHqfC=ZBoaBs{?M2=X2hab*_Ei8b=+MzY*RqK1c2Qb4Nl zW$Gm(mRyi8S0jdhjLl1=lqNYiH#zWXbr|IhwI#8ywL-Sj^sbZHM*AqFUBk91kk4xd z-h6DGGP>{o*cl5?Zh6y6F#+7O)SN8oA&x#op7fHamqfM GQ~w7syK`&+ literal 0 HcmV?d00001 diff --git a/katan/__pycache__/times.cpython-37.pyc b/katan/__pycache__/times.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50875482158a9d12a9ff79290bb62fab7d949699 GIT binary patch literal 1806 zcmZ`)OK&7K5N_K&-90^zJT|*L1uem9Gl$J^S|JogK!UU^T7;DBftFA(Z71}kAA{{# zG@~8~v60wI4*Y}3kw3tn;2-$PiNCNXsyvg2c=X6+ySiLm_0?CNPkX)CgU9~<$G0T( zygzYs^)Og`3`746LU@7{UP|0dQ|4wr_1)Y`1DI(MmSGx}Q5q4nXGOb=)41%U9k=(3 zZrMxw#IU*}I-+~-rCXvW`Y_)Wzm)@Kjx*!d`BnxKz5kx3~{eVFXm)pMzgPM+2xKgp#?o=l3ISIGfHzpV#}sguWtM@g-cXGi-9=7d_y%8^hJ zd>^6@PHI(hV_Q1og|zH@ZEnRs2$c_Zwqxq!Ye?u=fMRxB)Nmx9NlVSlxwA270R{|e zz^o6S(bh+(EE7im=Bc~j?A62JWGKtLm>w%$%IWY6nVo%7AIUp9Nkn3zUISf*z~Y18;=sNB2?PL440%EI+rM}x1ff2U8gk&h zAai2qh`ogOZvc^me?jN0p`ry)f7P&&zi62Nz-De8nrPIqA|X&oMj&8(kS>V&9T@se z5T`(sZ#L(oAuqj=cNQqWA-~ce{KF-n17gFKGbkwC`M@&Rg0^M&cZGDIt9#(*RlHl| z+GKS(-g_6FKyu(&d|K~_tZN$?p+rD~;B7mmW#FS$Nb=x0SJ zh4oP#e7j-eT<5MSGGKrnM^-2*whvX$R9c6@*j|;1;SvvxAVn=dJ}}lVgj_H z5$TdX+hTpH?t}KfyBGaq_d<69si5%Cl_M-Tn|*dua1J@wo3AJ1LENA3|Ez8oSID+{R$Bs_PJx z8foQWW)EP#CL9v)`d-2++`gAwR{rdro2)QOAnk$7@{$+H*OIHmh4p_iL$FKRDVRbf z5bb|Pde!Aq^$6ZNW_na@!x|2_yJ|Jvv%wg8h-*hC+o~qzc&gq5L+fL`Q-3^#`^b^W z@e(hCw7V+B4w@{1#EjE$rB?wuAQJ`uQ7~sqTH@v6NZ{~t(cFsr2IgqDti_~|d+0?Y T$0E3$@VAaX8nZZ#x1#WG944$S literal 0 HcmV?d00001 diff --git a/katan/compiler.py b/katan/compiler.py index 38c2592..08e4ad6 100644 --- a/katan/compiler.py +++ b/katan/compiler.py @@ -2,18 +2,44 @@ ### Author : Emily Ramey ### Date : 6/3/21 -import time_util as times -import nirc2_util as nirc2 -import telem_util as telem -import temp_util as temp -import mkwc_util as mkwc +from . import times +from . import strehl +from . import mkwc +from . import telemetry as telem +from . import temperature as temp +from . import templates import numpy as np import pandas as pd +import os + +# Check dependencies +try: + import yaml +except: + raise ValueError("PyYAML not installed. Please install from https://anaconda.org/anaconda/pyyaml") + +### For importing package files +### May need to edit this later if there are import issues with Py<37 +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources # All data types that can be compiled by this package accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] +default_parfile = 'keyword_defaults.yaml' + +### Load default parameter file +try: # Load from templates module + file = pkg_resources.open_text(templates, default_parfile) + default_params = yaml.load(file, Loader=yaml.FullLoader) +except: # Warn user + default_params = {} + print("WARNING: Unable to load default parameters. Please specify a parameter file when compiling.") + # Shorthand / nicknames for data types expand = { 'temp': ['k2AO', 'k2L4', 'k2ENV'], @@ -41,16 +67,19 @@ def check_dtypes(data_types): ######## Interface with other modules ####### ############################################# +# Have files with acceptable columns for each data type so you can check inputs +# Accept column inputs to the data function and take optional arguments in combine_strehl + def data_func(dtype): """ Returns the function to get data for the specified dtype and whether data needs to be matched to nirc2 mjds """ - if dtype in ['cfht']+expand['seeing']: + if dtype in ['cfht']+expand['seeing']: # MKWC return lambda files,mjds: mkwc.from_nirc2(mjds,dtype), True - if dtype in expand['temp']: + if dtype in expand['temp']: # Temperature return lambda files,mjds: temp.from_mjds(mjds,dtype), True - if dtype=='telem': + if dtype=='telem': # Telemetry return lambda files,mjds: telem.from_nirc2(mjds,files), False @@ -84,13 +113,20 @@ def match_data(mjd1, mjd2): # 'telem', 'k2AO', 'k2L4', or 'k2ENV' # 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and # ['mass', 'dimm', 'masspro'], respectively -def combine_strehl(strehl_file, data_types, check=True, test=False): +def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=False, + params=default_params): """ Combines and matches data from a certain Strehl file with other specified data types. NIRC2 files must be in the same directory as the Strehl file. """ + # Check file paths parameter & open if yaml + if not isinstance(file_paths, dict) and os.path.isfile(file_paths): + with open(file_paths) as file: + file_paths = yaml.load(file, loader=yaml.FullLoader) + ### Add a catch for YAML not being installed if file specified - nirc2_data = nirc2.from_strehl(strehl_file) + # Read in Strehl file + nirc2_data = strehl.from_strehl(strehl_file) if test: # Take first few files nirc2_data = nirc2_data.loc[:3] @@ -113,47 +149,9 @@ def combine_strehl(strehl_file, data_types, check=True, test=False): else: # Get indices of matched data idxs = match_data(nirc2_data.nirc2_mjd, other_data[dtype+'_mjd']) other_data = other_data.iloc[idxs] - -# # Edge case: no data -# if idxs is not None: -# other_data = other_data.iloc[idxs] # Add to all data all_data.append(other_data.reset_index(drop=True)) # Concatenate new data with nirc2 - return pd.concat(all_data, axis=1) - -################################# -######### Full Compiler ######### -################################# - -def compile_all(data_dir, data_types=['all'], test=False, save=False): - """ Compiles and matches requested data from all strehl files in a given nirc2 directory """ - - # Expand requested data types - data_types = check_dtypes(data_types) - - # Get all epochs in requested folder - epochs = nirc2.search_epochs(data_dir=data_dir) - - # Full data container - all_data = [] - - # Compile data for each epoch - for epoch, strehl_file in epochs.items(): - data = combine_strehl(strehl_file, data_types, check=False, - test=test) - # Add epoch name to columns - data['epoch'] = epoch - - # Append to full data - all_data.append(data) - - all_data = pd.concat(all_data, ignore_index=True) - - if save: - all_data.to_csv(save, index=False) - - # Concatenate all data - return all_data \ No newline at end of file + return pd.concat(all_data, axis=1) \ No newline at end of file diff --git a/katan/mkwc.py b/katan/mkwc.py new file mode 100644 index 0000000..65aa637 --- /dev/null +++ b/katan/mkwc.py @@ -0,0 +1,184 @@ +### mkwc_util.py : Contains utilities for extracting and processing data from the MKWC website +### Author : Emily Ramey +### Date : 6/1/2021 + +import os +import numpy as np +import pandas as pd +from . import times + +### NOTE: Seeing data is STORED by UT, with data in HST +### CFHT data is STORED by HST, with data in HST + +# cfht_dir = "/u/emily_ramey/work/Keck_Performance/data/weather_data/" +mkwc_url = 'http://mkwc.ifa.hawaii.edu/' +year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' +cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST + +# seeing_dir = "/u/emily_ramey/work/Keck_Performance/data/seeing_data" + +# Time columns from MKWC data +time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] + +# Data-specific fields +data_types = { + 'cfht': { + 'web_pat': mkwc_url+'archive/wx/cfht/indiv-days/cfht-wx.{}.dat', + 'file_pat': "{}cfht-wx.{}.dat", + 'data_cols': ['wind_speed', 'wind_direction', 'temperature', + 'relative_humidity', 'pressure'], + }, + 'mass': { + 'web_pat': mkwc_url+'current/seeing/mass/{}.mass.dat', + 'file_pat': '{}mass/{}.mass.dat', + 'data_cols': ['mass'], + }, + 'dimm': { + 'web_pat': mkwc_url+'current/seeing/dimm/{}.dimm.dat', + 'file_pat': '{}dimm/{}.dimm.dat', + 'data_cols': ['dimm'], + }, + 'masspro': { + 'web_pat': mkwc_url+'current/seeing/masspro/{}.masspro.dat', + 'file_pat': '{}masspro/{}.masspro.dat', + 'data_cols': ['masspro_half', 'masspro_1', 'masspro_2', + 'masspro_4', 'masspro_8', 'masspro_16', 'masspro'], + }, +} + +# Mix and match data & time columns +for dtype in data_types: + # CFHT files don't have seconds + tcols = time_cols if dtype != 'cfht' else time_cols[:-1] + # Format web columns + data_types[dtype]['web_cols'] = tcols+data_types[dtype]['data_cols'] + # Format file columns + data_types[dtype]['cols'] = [dtype+'_mjd']+data_types[dtype]['data_cols'] + # Different file storage timezones + data_types[dtype]['file_zone'] = 'hst' if dtype=='cfht' else 'utc' + +############################# +######### Functions ######### +############################# + +def cfht_from_year(datestrings, year): + """ + Gets pre-2012 MKWC data from the year-long file + instead of the by-date files + """ + # Get year-long file URL + url = year_url.format(year) + + # Read in data + try: + web_cols = data_types['cfht']['web_cols'] + year_data = pd.read_csv(url, delim_whitespace=True, header=None, + names=web_cols, usecols=range(len(web_cols))) + except: # No data, return blank + return pd.DataFrame(columns=data_types['cfht']['cols']) + + # Full dataset + all_data = [pd.DataFrame(columns=data_types['cfht']['cols'])] + + # Slice up dataframe + for ds in datestrings: + month, day = int(ds[4:6]), int(ds[6:]) + # Get data by month and day + df = year_data.loc[(year_data.month==month) & (year_data.day==day)].copy() + # Format columns + if not df.empty: + format_columns(df, 'cfht') + # Add to full dataset + all_data.append(df) + + return pd.concat(all_data) + +def format_columns(df, dtype): + """ Changes columns (in place) from time_cols to MJD """ + # Get MJDs from HST values + mjds = times.table_to_mjd(df, columns=time_cols, zone='hst') + df[dtype+'_mjd'] = mjds + + # Drop old times + df.drop(columns=time_cols, inplace=True, errors='ignore') + +def from_url(datestring, dtype): + """ Pulls cfht or seeing file from MKWC website """ + # Format URL + url = data_types[dtype]['web_pat'].format(datestring) + + # Read data + try: # Check if data is there + web_cols = data_types[dtype]['web_cols'] # MKWC Weather columns + df = pd.read_csv(url, delim_whitespace = True, header=None, + names=web_cols, usecols=range(len(web_cols))) + except: # otherwise return blank df + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Get mjd from time columns + format_columns(df, dtype) + + return df + +def from_file(filename, dtype, data_dir='./'): + """ Pulls cfht or seeing file from local directory """ + # Read in CSV + df = pd.read_csv(filename) + + # Check formatting + if 'mjd' in df.columns: + df.rename(columns={'mjd':dtype+'_mjd'}, inplace=True) + elif dtype+'_mjd' not in df.columns: # No MJD + try: # Change to MJD, if times + format_columns(df, dtype) + except: # No time info, return blank + return pd.DataFrame() + + return df + +def from_nirc2(mjds, dtype, data_dir='./'): + """ + Compiles a list of cfht or seeing observations based on MJDs + note: does not compare MJDs; assumption is inputs are rounded to nearest day + """ + + # Get datestrings + dts = times.mjd_to_dt(mjds, zone=data_types[dtype]['file_zone']) + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + # No duplicates + datestrings = pd.Series(np.unique(datestrings)) + + # Blank data structure + all_data = [pd.DataFrame(columns=data_types[dtype]['cols'])] + + # Check for pre-2012 cfht files + if dtype=='cfht' and any(mjds < cfht_cutoff): + # Get datetimes + pre_2012 = datestrings[mjds < cfht_cutoff] + + # Compile data by year + for yr in np.unique(pre_2012.str[:4]): + ds = pre_2012[pre_2012.str[:4]==yr] + df = cfht_from_year(ds, yr) + # Append to full dataset + all_data.append(df) + + # Find data for each file + for ds in datestrings: + # Get local filename + filename = data_types[dtype]['file_pat'].format(data_dir, ds) + + # Check for local files + if os.path.isfile(filename): + df = from_file(filename, dtype) + + else: # Pull from the web + df = from_url(ds, dtype) + + # Save to local file: TODO + + # Add to data + all_data.append(df) + + # Return concatenated dataframe + return pd.concat(all_data, ignore_index=True) \ No newline at end of file diff --git a/katan/strehl.py b/katan/strehl.py new file mode 100644 index 0000000..9846ac3 --- /dev/null +++ b/katan/strehl.py @@ -0,0 +1,60 @@ +### nirc2_util.py : Functions for processing nirc2 data and strehl files +### Author : Emily Ramey +### Date : 6/2/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits + +from . import times + +strehl_cols = ['nirc2_file', 'strehl', 'rms_err', 'fwhm', 'nirc2_mjd'] +default_keys = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', + 'AOLBFWHM', 'WSFRRT', 'LSAMPPWR', 'LGRMSWF', 'AOAOAMED', 'TUBETEMP'] + +def from_filename(nirc2_file, data, i, header_kws): + """ + Gets nirc2 header values from a filename as dict + or loads values into specified df + """ + # Check for valid file + if not os.path.isfile(nirc2_file): + return + # Open nirc2 file + with fits.open(nirc2_file) as file: + nirc2_hdr = file[0].header + + # Get fields from header + for kw in header_kws: + # load DataFrame value + data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) + +def from_strehl(strehl_file, header_kws=default_keys): + """ Gets NIRC2 header data based on contents of Strehl file """ + # Get directory name + data_dir = os.path.dirname(strehl_file) + # Retrieve Strehl data + strehl_data = pd.read_csv(strehl_file, delim_whitespace = True, + header = None, skiprows = 1, names=strehl_cols) + # Add true file path + strehl_data['nirc2_file'] = data_dir + "/" + strehl_data['nirc2_file'] + + # Add decimal year + strehl_data['dec_year'] = times.mjd_to_yr(strehl_data.nirc2_mjd) + + # Add nirc2 columns + for col in header_kws: + strehl_data[col.lower()] = np.nan + + # Loop through nirc2 files + for i,nirc2_file in enumerate(strehl_data.nirc2_file): + # Load header data into df + from_filename(nirc2_file, strehl_data, i, header_kws) + + # Return data + return strehl_data + + + diff --git a/katan/telemetry.py b/katan/telemetry.py new file mode 100644 index 0000000..3757ee7 --- /dev/null +++ b/katan/telemetry.py @@ -0,0 +1,292 @@ +### telem_util.py: For code that interacts with Keck AO Telemetry files +### Author: Emily Ramey +### Date: 11/18/2020 + +import numpy as np +import pandas as pd +import glob +from scipy.io import readsav + +from . import times +from . import templates + +### For importing package files +### May need to edit this later if there are import issues with Py<37 +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +### Path to telemetry files [EDIT THIS] +telem_dir = '/g/lu/data/keck_telemetry/' + +# Sub-aperture maps +wfs_file = "sub_ap_map.txt" +act_file = "act.txt" + +filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers +filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" + +TIME_DELTA = 0.001 # About 100 seconds in mjd + +RESID_CUTOFF = 349 +TT_IDXS = [349,350] +DEFOCUS = 351 +LAMBDA = 2.1 # microns + +cols = ['telem_file', + 'telem_mjd', + 'TT_mean', + 'TT_std', + 'DM_mean', + 'DM_std', + 'rmswfe_mean', + 'rmswfe_std', + 'strehl_telem'] + +def read_map(filename): + """ Reads a map of actuators or sub-apertures from a file """ + file = pkg_resources.open_text(templates, filename) + return pd.read_csv(file, delim_whitespace=True, header=None).to_numpy() + +def get_times(telem_data, start=None): + """ Pulls timestamps from a telemetry file in seconds from start """ + if start is None: + start = telem_data.a.timestamp[0][0] + + return (telem_data.a.timestamp[0]-start)/1e7 + +def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): + """ + Return the locations of the valid actuators in the actuator array + resids: Nx349 residual wavefront array (microns) + ints: Nx304 intensity array (any units) + N: Number of timestamps + """ + # Check inputs + N = ints.shape[0] # Num timestamps + + # Aggregate intensities over all timestamps + med_ints = np.median(ints, axis=0) + + # Fill WFS map with aggregated intensities + int_map = wfs_map.copy() + int_map[np.where(int_map==1)] = med_ints + + # Find lenslets with greatest intensity + idxs = np.flip(np.argsort(int_map, axis=None))[:num_aps] # flat idxs of sort + idxs = np.unravel_index(idxs, wfs_map.shape) # 2D idxs of sort + + # Mask for good sub-ap values + good_aps = np.zeros(wfs_map.shape, dtype=int) + good_aps[idxs] = 1 + good_aps = good_aps * wfs_map # Just in case + + # Mask for good actuator values + good_acts = np.pad(good_aps, ((1,1),(1,1))) + good_acts = (good_acts[1:,1:] | good_acts[1:,:-1] + | good_acts[:-1,:-1] | good_acts[:-1,1:]) * act_map + + return good_acts + +# def wfs_error2(resids, ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file)): +# """ Calculates the variance in the wavefront (microns^2) produced by the WFS """ +# # Get residual mask for brightest sub-aps +# rmask = resid_mask(resids, ints) + +# # Square residuals +# sig2_resid = np.mean(resids**2, axis=0) +# # Load into actuator grid +# resid_grid = act_map.astype(float) +# resid_grid[np.where(resid_grid==1)] = sig2_resid + +# # Mask out actuators next to dimmer apertures +# sig2_masked = resid_grid * rmask +# # Sum values and return +# return np.sum(sig2_masked) + +# def tt_error2(tt_resids): ### TODO: fix this so it's not unitless +# """ Calculate mean tip-tilt residual variance in microns """ +# wvln = 0.658 # wavelength (microns) +# D = 10.5 * 1e6 # telescope diameter (microns) + +# # Magnitude of residuals +# sig_alpha2 = (tt_resids[:,0] + tt_resids[:,1])**2 # x + y TT residual variance +# sig_w2 = (np.pi*D/wvln/2.0)**2 * sig_alpha2 # TT resids in microns^2 + +# return np.mean(sig_w2) + +# def total_WFE(telem_data): +# resids = telem_data.a.residualwavefront[0][:, :RESID_CUTOFF] * 0.6 # WFS resids, microns, Nx349 +# ints = telem_data.a.subapintensity[0] # Sub-ap intensities, Nx304 +# tt_resids = telem_data.a.residualwavefront[0][:, TT_IDXS] * np.pi / (180*3600) # TT resids, radians, Nx2 +# defocus = [0] #telem_data.a.residualwavefront[0][:, DEFOCUS] # Defocus resids, microns, Nx1 + +# return wfs_error2(resids, ints) + tt_error2(tt_resids) + np.mean(defocus**2) + +# def mask_residuals_old(telem_data, num_aps=236, wfs_file=wfs_file, act_file=act_file): +# """ Really freakin complicated array logic to mask out all invalid actuators """ +# # Get data +# resids = telem_data.a.residualwavefront[0][:, :349]*0.6 # microns +# intensities = telem_data.a.subapintensity[0] # ADU +# N = intensities.shape[0] +# # Get hardware maps +# wfs_map = read_map(wfs_file) +# act_map = read_map(act_file) + +# # Get X and Y values of sub-aps and replicate them for all timestamps +# wfs_x, wfs_y = np.where(wfs_map==1) +# wfs_x, wfs_y = np.tile(wfs_x, (N,1)), np.tile(wfs_y, (N,1)) + +# # Get valid indices for each timestep +# idxs = np.flip(np.argsort(intensities, axis=1), axis=1)[:,:num_aps] +# valid_x = np.take_along_axis(wfs_x, idxs, axis=1) +# valid_y = np.take_along_axis(wfs_y, idxs, axis=1) +# valid_z = np.tile(np.arange(N), (num_aps,1)).T + +# # Put 1s at each valid index +# valid_saps = np.zeros((N, 20, 20), int) +# valid_saps[valid_z, valid_x, valid_y] = 1 # TODO: flip this back +# # Pad each sheet (timestamp) with zeros at the edges +# check = valid_saps.reshape(N, 20*20).sum(axis=1) +# if any(check!=236): +# print("Shape mismatch in valid sub-ap array") +# valid_saps = np.pad(valid_saps, ((0,0),(1,1),(1,1))) + +# # Get (potentially)valid actuators for sub-aps +# valid_acts = (valid_saps[:,1:,1:]|valid_saps[:,1:,:-1]| +# valid_saps[:,:-1,:-1]|valid_saps[:,:-1,1:]) # 4 corners of sub-aps +# # Multiply by actuator map to remove any non-actuator positions +# valid_acts = valid_acts * np.tile(act_map, (N,1,1)) + +# # Get number of valid actuators in each frame (can vary due to edge cases) +# valid_act_nums = valid_acts.reshape(N,21*21).sum(axis=1) + +# # Map residuals to actuator positions +# resid_vals = np.tile(act_map, (N,1,1)).astype(float) +# resid_vals[np.where(resid_vals==1)] = resids.flatten() + +# # Mask out invalid actuators +# valid_acts = valid_acts * resid_vals +# rms_resids = valid_acts.reshape(N, 21*21) + +# # Take the RMS residual for each frame +# rms_resids = np.sqrt((rms_resids**2).sum(axis=1)/valid_act_nums) + +# return rms_resids + +def tt2um(tt_as): + """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ + D = 10.5e6 # telescope diameter in microns + tt = tt_as*4.8e-6 # TT error in radians + tt_err = np.sqrt(D**2 / 12 * (tt[:,0]**2 + tt[:,1]**2)) + return tt_err + +def rms_acts(act_resids, ints): + """ Clips bad actuators and averages for each timestamp """ + N = act_resids.shape[0] # num timestamps + + # Read in actuator map + act_map = read_map(act_file) + + # Get good actuator mask from intensities + mask = resid_mask(ints) + flat_map = ~mask[np.where(act_map==1)].astype(bool) # flatten + flat_map = np.tile(flat_map, (N,1)) + + # Mask the bad actuators + good_acts = np.ma.masked_array(act_resids, flat_map) + # Average resids for each timestep + act_rms = np.sqrt((good_acts**2).mean(axis=1) - good_acts.mean(axis=1)**2) + + return act_rms.compressed() + +######################################## +######### Processing Files ############# +######################################## + +# Do some telemetry files need to be cropped around the observation? + +def get_mjd(telem): + """ Validates a telemetry file against an MJD value """ + # Get timestamp + tstamp = telem.tstamp_str_start.decode('utf-8') + # Convert to MJD + telem_mjd = times.str_to_mjd(tstamp, fmt='isot') + + # Returns telem mjd + return telem_mjd + +def extract_telem(file, data, idx, check_mjd=None): + """ Extracts telemetry values from a file to a dataframe """ + # Read IDL file + telem = readsav(file) + + # Make sure MJD matches + telem_mjd = get_mjd(telem) + if check_mjd is not None: + delta = np.abs(telem_mjd-check_mjd) + if delta > TIME_DELTA: + return False + + # Get residuals and intensities + act_resids = telem.a.residualwavefront[0][:, :RESID_CUTOFF] + tt_resids = telem.a.residualwavefront[0][:,TT_IDXS] + ints = telem.a.subapintensity[0] # Sub-aperture intensities + + # Convert TT resids to microns + tt_microns = tt2um(tt_resids) + # Get RMS resids from the actuator array + act_rms = rms_acts(act_resids, ints) + + # Total RMS Wavefront Error + rmswfe = np.sqrt(tt_microns**2 + act_rms**2) + + # Strehl calculation + strehl = np.exp(-(2*np.pi*rmswfe/LAMBDA)**2) + + # Assemble aggregate data + data.loc[idx, cols] = [ + file, + telem_mjd, + np.mean(tt_microns), + np.std(tt_microns), + np.mean(act_rms), + np.std(act_rms), + np.mean(rmswfe), + np.std(rmswfe), + np.mean(strehl), + ] + + return True + +def from_nirc2(mjds, filenames): + """ Gets a table of telemetry information from a set of mjds and file numbers """ + N = len(mjds) # number of data points + # Get file numbers + filenums = filenames.str.extract(filenum_match, expand=False) + filenums = filenums.str[1:] # First digit doesn't always match + + # Get datestrings + dts = times.mjd_to_dt(mjds) # HST or UTC??? + datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 + + # Set up dataframe + data = pd.DataFrame(columns=cols, index=range(N)) + + # Find telemetry for each file + for i in range(N): + # Get filename and number + fn, ds, mjd = filenums[i], datestrings[i], mjds[i] + # Search for correct file + file_pat = filename_match.format(ds, fn) + all_files = glob.glob(file_pat, recursive=True) + + # Grab the first file that matches the MJD + for file in all_files: + success = extract_telem(file, data, i, check_mjd=mjd) + if success: break + + return data + \ No newline at end of file diff --git a/katan/temperature.py b/katan/temperature.py new file mode 100644 index 0000000..9c672e0 --- /dev/null +++ b/katan/temperature.py @@ -0,0 +1,277 @@ +### temp_util.py: Functions relating to Keck-II temperature data +### Main reads in Keck II temperature files (AO, env, and L4) and saves each to a FITS file +### Author: Emily Ramey +### Date: 01/04/2021 + +import os +import numpy as np +import pandas as pd +import glob +from astropy.io import fits +from astropy.table import Table + +from . import times + +verbose = True + +data_dir = "/u/emily_ramey/work/Keck_Performance/data/" +temp_dir = data_dir+"temp_data_2/" + +col_dict = { + 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV + 'AO_benchT2': 'k1:ao:env:benchtemp2_Raw', + 'k1:ao:env:elect_vault_t': 'k1:ao:env:elect_vault:T', + 'k0:met:humidityStats.VAL': 'k0:met:humidityStats', + "k2:ao:env:ETroomtemp_Raw": "El_roomT", # k2AO + "k2:ao:env:DMracktemp_Raw": "DM_rackT", + "k2:ao:env:KCAMtemp2_Raw": "KCAM_T2", + "k2:ao:env:OBrighttemp_Raw": "OB_rightT", + "k2:ao:env:phototemp_Raw": "photometricT", + "k2:ao:env:OBlefttemp_Raw": "OB_leftT", + "k2:ao:env:KCAMtemp1_Raw": "KCAM_T1", + "k2:ao:env:ACAMtemp_Raw": "AOA_camT", + "k2:ao:env:LStemp_Raw": "LGS_temp", # k2L4 + "k2:ao:env:LSenchumdity_Raw": "LGS_enclosure_hum", + "k2:ao:env:LShumidity_Raw": "LGS_humidity", + "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" +} + +file_pats = { # File name patterns +# 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", +# 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", +# 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", + 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", + 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", + 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", +} + +data_types = { + 'k2AO': { + 'file_pat': temp_dir+"k2AOtemps/{}/AO_temps.log", + 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', + 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', + 'k2AO_mjd'], + }, + 'k2L4': { + 'file_pat': temp_dir+"k2L4temps/{}/L4_env.log", + 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', + 'LGS_temp', 'k2L4_mjd'], + }, + 'k2ENV': { + 'file_pat': temp_dir+"k2envMet/{}/envMet.arT", + 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', + 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', + 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', + 'k0:met:out:windSpeedMaxStats', 'k0:met:out:windSpeedMaxmph', + 'k0:met:out:windSpeedmph', 'k0:met:outTempDwptDiff', + 'k0:met:pressureRaw', 'k0:met:pressureStats', 'k0:met:tempMax', + 'k0:met:tempMin', 'k0:met:tempRaw', 'k2:dcs:sec:acsDwptDiff', + 'k2:dcs:sec:acsTemp', 'k2:dcs:sec:secDwptDiff', 'k2:dcs:sec:secondaryTemp', + 'k2:met:humidityRaw','k2:met:humidityStats', 'k2:met:tempRaw', 'k2:met:tempStats', + 'k2:met:windAzRaw', 'k2:met:windElRaw', 'k2:met:windSpeedMaxmph', + 'k2:met:windSpeedMinmph', 'k2:met:windSpeedRaw', 'k2:met:windSpeedStats', + 'k2:met:windSpeedmph', 'k2ENV_mjd'], + } +} + +def get_columns(): + all_data = search_files() + for name, data_files in all_data.items(): + columns = set() + for i,file in enumerate(data_files): + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + for col in df.columns: + columns.add(col) + all_data[name] = columns + + return all_data + + +def search_files(file_pats=file_pats): + """ Finds all filenames matching the given file patterns """ + all_filenames = {} + for name, search in file_pats.items(): + filenames = glob.glob(search, recursive=True) + all_filenames[name] = filenames + if verbose: + print(f"Done {name}, length: {len(filenames)}") + + return all_filenames + +def collect_data(data_files, col_dict=col_dict): + """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ + if isinstance(data_files, dict): # Return dict w/dataframes + new_files = {} + for name, files in data_files.items(): + if isinstance(files, list): + # Recurse on each list of files + new_files[name] = collect_data(files) + return new_files + + all_dfs = [pd.DataFrame()] + for i,file in enumerate(data_files): + if not os.path.isfile(file): + continue + try: + df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, + na_values=['***'], error_bad_lines=False, warn_bad_lines=False, + ).replace('"', regex=True) + except: + if verbose: + print(f"Warning: read failed for file {file}") + continue + + if len(df.columns)==1: # no header + if verbose: + print(f"Skipping file {file}, no header") + continue # skip for now + df.columns = [col.replace('"', '').strip() for col in df.columns] + if "UNIXDate" not in df.columns: + if verbose: + print(f"Skipping file {file}, columns not as expected") + continue # skip for now + + if col_dict is not None: + df = df.rename(columns=col_dict) + + all_dfs.append(df) + + data = pd.concat(all_dfs, ignore_index=True, sort=True) + return data + +def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): + """ Parses specified date and time columns and returns a cleaned data table """ + new_data = data.copy() + for label,cols in date_cols.items(): + date_col, time_col = cols + # Parse dates and times, coercing invalid strings to NaN + datetimes = (pd.to_datetime(data[date_col], exact=False, errors='coerce') + + pd.to_timedelta(data[time_col], errors='coerce')) + new_data = new_data.drop(columns=cols) + new_data[label] = datetimes + + return new_data + +def clean_dates(data, date_cols=['HST', 'UNIX']): + """ Removes any rows containing invalid dates in date_cols """ + new_data = data.copy() + for col in date_cols: + new_data = new_data[~np.isnan(new_data[col])] + + return new_data + +def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): + """ Casts columns to a numeric data type """ + if data_cols is None: + data_cols = [col for col in data.columns if col not in non_numeric] + + # Cast data to numeric type, coercing invalid values to NaN + new_data = data.copy() + for col in data_cols: + new_data[col] = pd.to_numeric(new_data[col], errors='coerce') + + return new_data + +def to_fits(data, filename, str_cols=['HST', 'UNIX']): + """ Writes a FITS file from the given temperature array """ + fits_data = data.copy() + for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: + if col in fits_data.columns: + fits_data = fits_data.drop(columns=[col]) + for col in str_cols: + if col in fits_data.columns: + fits_data[col] = fits_data[col].astype(str) + # Assuming the data columns are already numeric + fits_data = Table.from_pandas(fits_data) + fits_data.write(filename) + + return + +def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): + """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ + data = Table.read(filename).to_pandas() + + # Fix NaNs, because astropy is dumb sometimes + data[data==1e+20] = np.nan + + if date_cols is None: return data + + for col in date_cols: + if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings + data[col] = data[col].str.decode("utf-8") + data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') + + if str_cols is None: return data + + for col in str_cols: + if col in data.columns and isinstance(data[col][0], bytes): + data[col] = data[col].str.decode("utf-8") + + return data + +def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): + """ + Reads in all data matching file patterns (file_pats), combines them into one table, + cleans them, and saves them to a FITS file + """ + # Find all files matching pattern + all_filenames = search_files(file_pats) + # Read all data into one table (per dictionary label) + all_data = collect_data(all_filenames) + + for name, data in all_data.items(): + data = parse_dates(data) # Parse date cols into datetimes + data = clean_dates(data) # Remove invalid dates + data = clean_data(data) # Casts other cols to numeric + # Save combined/cleaned data to FITS file + filename = location+name+".fits" + to_fits(data, filename) + + return + +def from_mjds(mjds, dtype): + """ Gets temp data of input type from the specified MJDs """ + # Get pd datetimes in HST + dts = times.mjd_to_dt(mjds, zone='hst') + # Format list + datestrings = dts.strftime("%y/%m/%d") + datestrings = np.unique(datestrings) # one file per date + # Get relevant filenames + filenames = [data_types[dtype]['file_pat'].format(ds) for ds in datestrings] + # Get data from filenames + data = collect_data(filenames) + + # Empty dataframe + if data.empty: + return pd.DataFrame(columns=data_types[dtype]['cols']) + + # Convert dates & times to MJDs + data["datetime"] = data['HSTdate']+' '+data['HSTtime'] + mjds = times.table_to_mjd(data, columns='datetime', zone='hst') + # Add to dataframe + data[dtype+"_mjd"] = mjds + # Drop other date & time cols + data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], + inplace=True, errors='ignore') + + return data + +if __name__=='__main__': + pass \ No newline at end of file diff --git a/katan/templates/.ipynb_checkpoints/__init__-checkpoint.py b/katan/templates/.ipynb_checkpoints/__init__-checkpoint.py new file mode 100755 index 0000000..e69de29 diff --git a/katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml b/katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml new file mode 100644 index 0000000..f1a4c26 --- /dev/null +++ b/katan/templates/.ipynb_checkpoints/keyword_defaults-checkpoint.yaml @@ -0,0 +1,127 @@ +### This is the default YAML file for the KATAN compiler +### Note: columns are not in the order they are read in from data +### Change 'True' to an alias for the column, if desired, or set to False to exclude + +--- +### Strehl file columns +strehl: + 'nirc2_mjd': True # required + 'nirc2_file': True + 'strehl': True + 'rms_err': True + 'fwhm': True + +### NIRC2 file columns +nirc2: + 'AIRMASS': 'airmass' + 'ITIME': 'itime' + 'COADDS': 'coadds' + 'FWINAME': 'fwiname' + 'AZ': 'az' + 'DMGAIN': 'dmgain' + 'DTGAIN': 'dtgain' + 'AOLBFWHM': 'aolbfwhm' + 'WSFRRT': 'wsfrrt' + 'LSAMPPWR': 'lsamppwr' + 'LGRMSWF': 'lgrmswf' + 'AOAOAMED': 'aoaomed' + 'TUBETEMP': 'tubetemp' + +### Telemetry file columns +telemetry: + 'telem_file': True + 'telem_mjd': True # required + 'TT_mean': True + 'TT_std': True + 'DM_mean': True + 'DM_std': True + 'rmswfe_mean': True + 'rmswfe_std': True + 'strehl_telem': True + +### Weather file columns +cfht: + 'cfht_mjd': True # required + 'wind_speed': True + 'wind_direction': True + 'temperature': True + 'relative_humidity': True + 'pressure': True + +### MASS seeing columns +mass: + 'mass_mjd': True # required + 'mass': True + +### DIMM seeing columns +dimm: + 'dimm_mjd': True # required + 'dimm': True + +### MASSPRO seeing columns +masspro: + 'masspro_mjd': True # required + 'masspro': True + 'masspro_half': True + 'masspro_1': True + 'masspro_2': True + 'masspro_4': True + 'masspro_8': True + 'masspro_16': True + +### k2AO temperature columns +k2AO: + 'k2AO_mjd': True # required + 'AOA_camT': True + 'DM_rackT': True + 'El_roomT': True + 'KCAM_T1': True + 'KCAM_T2': True + 'OB_leftT': True + 'OB_rightT': True + 'photometricT': True + +### k2L4 temperature columns +k2L4: + 'k2L4_mjd': True # required + 'LGS_enclosure_hum': True + 'LGS_enclosure_temp': True + 'LGS_humidity': True + 'LGS_temp': True + +### k2ENV temperature columns +k2ENV: + 'k2ENV_mjd': True # required + 'k0:met:dewpointMax': True + 'k0:met:dewpointMin': True + 'k0:met:dewpointRaw': True + 'k0:met:humidityRaw': True + 'k0:met:humidityStats': True + 'k0:met:out:windDirection': True + 'k0:met:out:windDirectionMax': True + 'k0:met:out:windDirectionMin': True + 'k0:met:out:windSpeedMaxStats': True + 'k0:met:out:windSpeedMaxmph': True + 'k0:met:out:windSpeedmph': True + 'k0:met:outTempDwptDiff': True + 'k0:met:pressureRaw': True + 'k0:met:pressureStats': True + 'k0:met:tempMax': True + 'k0:met:tempMin': True + 'k0:met:tempRaw': True + 'k2:dcs:sec:acsDwptDiff': True + 'k2:dcs:sec:acsTemp': True + 'k2:dcs:sec:secDwptDiff': True + 'k2:dcs:sec:secondaryTemp': True + 'k2:met:humidityRaw': True + 'k2:met:humidityStats': True + 'k2:met:tempRaw': True + 'k2:met:tempStats': True + 'k2:met:windAzRaw': True + 'k2:met:windElRaw': True + 'k2:met:windSpeedMaxmph': True + 'k2:met:windSpeedMinmph': True + 'k2:met:windSpeedRaw': True + 'k2:met:windSpeedStats': True + 'k2:met:windSpeedmph': True +... \ No newline at end of file diff --git a/katan/templates/__init__.py b/katan/templates/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/katan/templates/__pycache__/__init__.cpython-37.pyc b/katan/templates/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97ddbcb4e0241171600f08eb07679bc17f321eca GIT binary patch literal 145 zcmZ?b<>g`kf;+oUCxGb3AOZ#$feZ&AE@lA|DGb33nv8xc8Hzx{2;!HOeyM(HZe~tp zd{JUK~7>xYO#KNd}dx|NqoFsLFFwD So80`A(wtN~keQ!>m;nH6ek3^n literal 0 HcmV?d00001 diff --git a/katan/act.txt b/katan/templates/act.txt similarity index 100% rename from katan/act.txt rename to katan/templates/act.txt diff --git a/katan/templates/keyword_defaults.yaml b/katan/templates/keyword_defaults.yaml new file mode 100644 index 0000000..f1a4c26 --- /dev/null +++ b/katan/templates/keyword_defaults.yaml @@ -0,0 +1,127 @@ +### This is the default YAML file for the KATAN compiler +### Note: columns are not in the order they are read in from data +### Change 'True' to an alias for the column, if desired, or set to False to exclude + +--- +### Strehl file columns +strehl: + 'nirc2_mjd': True # required + 'nirc2_file': True + 'strehl': True + 'rms_err': True + 'fwhm': True + +### NIRC2 file columns +nirc2: + 'AIRMASS': 'airmass' + 'ITIME': 'itime' + 'COADDS': 'coadds' + 'FWINAME': 'fwiname' + 'AZ': 'az' + 'DMGAIN': 'dmgain' + 'DTGAIN': 'dtgain' + 'AOLBFWHM': 'aolbfwhm' + 'WSFRRT': 'wsfrrt' + 'LSAMPPWR': 'lsamppwr' + 'LGRMSWF': 'lgrmswf' + 'AOAOAMED': 'aoaomed' + 'TUBETEMP': 'tubetemp' + +### Telemetry file columns +telemetry: + 'telem_file': True + 'telem_mjd': True # required + 'TT_mean': True + 'TT_std': True + 'DM_mean': True + 'DM_std': True + 'rmswfe_mean': True + 'rmswfe_std': True + 'strehl_telem': True + +### Weather file columns +cfht: + 'cfht_mjd': True # required + 'wind_speed': True + 'wind_direction': True + 'temperature': True + 'relative_humidity': True + 'pressure': True + +### MASS seeing columns +mass: + 'mass_mjd': True # required + 'mass': True + +### DIMM seeing columns +dimm: + 'dimm_mjd': True # required + 'dimm': True + +### MASSPRO seeing columns +masspro: + 'masspro_mjd': True # required + 'masspro': True + 'masspro_half': True + 'masspro_1': True + 'masspro_2': True + 'masspro_4': True + 'masspro_8': True + 'masspro_16': True + +### k2AO temperature columns +k2AO: + 'k2AO_mjd': True # required + 'AOA_camT': True + 'DM_rackT': True + 'El_roomT': True + 'KCAM_T1': True + 'KCAM_T2': True + 'OB_leftT': True + 'OB_rightT': True + 'photometricT': True + +### k2L4 temperature columns +k2L4: + 'k2L4_mjd': True # required + 'LGS_enclosure_hum': True + 'LGS_enclosure_temp': True + 'LGS_humidity': True + 'LGS_temp': True + +### k2ENV temperature columns +k2ENV: + 'k2ENV_mjd': True # required + 'k0:met:dewpointMax': True + 'k0:met:dewpointMin': True + 'k0:met:dewpointRaw': True + 'k0:met:humidityRaw': True + 'k0:met:humidityStats': True + 'k0:met:out:windDirection': True + 'k0:met:out:windDirectionMax': True + 'k0:met:out:windDirectionMin': True + 'k0:met:out:windSpeedMaxStats': True + 'k0:met:out:windSpeedMaxmph': True + 'k0:met:out:windSpeedmph': True + 'k0:met:outTempDwptDiff': True + 'k0:met:pressureRaw': True + 'k0:met:pressureStats': True + 'k0:met:tempMax': True + 'k0:met:tempMin': True + 'k0:met:tempRaw': True + 'k2:dcs:sec:acsDwptDiff': True + 'k2:dcs:sec:acsTemp': True + 'k2:dcs:sec:secDwptDiff': True + 'k2:dcs:sec:secondaryTemp': True + 'k2:met:humidityRaw': True + 'k2:met:humidityStats': True + 'k2:met:tempRaw': True + 'k2:met:tempStats': True + 'k2:met:windAzRaw': True + 'k2:met:windElRaw': True + 'k2:met:windSpeedMaxmph': True + 'k2:met:windSpeedMinmph': True + 'k2:met:windSpeedRaw': True + 'k2:met:windSpeedStats': True + 'k2:met:windSpeedmph': True +... \ No newline at end of file diff --git a/katan/sub_ap_map.txt b/katan/templates/sub_ap_map.txt similarity index 100% rename from katan/sub_ap_map.txt rename to katan/templates/sub_ap_map.txt diff --git a/katan/times.py b/katan/times.py new file mode 100644 index 0000000..4974d2c --- /dev/null +++ b/katan/times.py @@ -0,0 +1,56 @@ +### time_util.py : to handle time formatting changes between MJD, HST, and UTC +### Author : Emily Ramey +### Date: 5/26/21 + +# Preamble +import pandas as pd +import numpy as np +import time +import pytz as tz +from datetime import datetime, timezone +from astropy.time import Time, TimezoneInfo +from astropy import units as u, constants as c + +hst = tz.timezone('US/Hawaii') + +### Conversion / utility functions +def mjd_to_dt(mjds, zone='utc'): + """ Converts Modified Julian Date(s) to HST or UTC date(s) """ + # Convert mjds -> astropy times -> datetimes + dts = Time(mjds, format='mjd', scale='utc').to_datetime() + # Convert to pandas + dts = pd.to_datetime(dts).tz_localize(tz.utc) + if zone=='hst': + dts = dts.tz_convert("US/Hawaii") + return dts + +def table_to_mjd(table, columns, zone='utc'): + """ Converts date and time columns to mjds """ + # Safety check for list of columns + if not isinstance(columns, str): + columns = [col for col in columns if col in table.columns] + # Convert to datetimes + dts = pd.to_datetime(table[columns], errors='coerce') + + if zone=='hst':# Convert to UTC + dts = dts.dt.tz_localize(hst) + dts = dts.dt.tz_convert(tz.utc) + + # Masked invalid values + dts = np.ma.masked_array(dts, mask=dts.isnull()) + + # Convert to astropy + times = Time(dts, format='datetime', scale='utc') + # return MJDs + return np.ma.getdata(times.mjd) + +def str_to_mjd(datestrings, fmt): + """ Converts astropy-formatted date/time strings (in UTC) to MJD values """ + # Get astropy times + times = Time(datestrings, format=fmt, scale='utc') + # Convert to mjd + return times.mjd + +def mjd_to_yr(mjds): + """ Converts MJD to Decimal Year """ + return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file From b86dc4d466d5edc444b17db59693953d8404bd34 Mon Sep 17 00:00:00 2001 From: Jessica Lu Date: Mon, 21 Jun 2021 10:23:09 -0700 Subject: [PATCH 06/12] Clean up of files that shouldn't be in the repo (caches and pynb checkpoints) --- .gitignore | 108 +++++++ .ipynb_checkpoints/compiler-checkpoint.py | 159 ---------- .../dependencies-checkpoint.txt | 0 .ipynb_checkpoints/mkwc_util-checkpoint.py | 184 ----------- .ipynb_checkpoints/nirc2_util-checkpoint.py | 87 ------ .ipynb_checkpoints/telem_util-checkpoint.py | 283 ----------------- .ipynb_checkpoints/temp_util-checkpoint.py | 277 ----------------- .ipynb_checkpoints/time_util-checkpoint.py | 56 ---- .../.ipynb_checkpoints/compiler-checkpoint.py | 157 ---------- katan/.ipynb_checkpoints/mkwc-checkpoint.py | 184 ----------- katan/.ipynb_checkpoints/strehl-checkpoint.py | 60 ---- .../telemetry-checkpoint.py | 292 ------------------ .../temperature-checkpoint.py | 277 ----------------- katan/.ipynb_checkpoints/times-checkpoint.py | 56 ---- katan/__pycache__/__init__.cpython-37.pyc | Bin 135 -> 0 bytes katan/__pycache__/compiler.cpython-37.pyc | Bin 3454 -> 0 bytes katan/__pycache__/mkwc.cpython-37.pyc | Bin 3710 -> 0 bytes katan/__pycache__/strehl.cpython-37.pyc | Bin 1457 -> 0 bytes katan/__pycache__/telemetry.cpython-37.pyc | Bin 4551 -> 0 bytes katan/__pycache__/temperature.cpython-37.pyc | Bin 8092 -> 0 bytes katan/__pycache__/times.cpython-37.pyc | Bin 1806 -> 0 bytes .../__pycache__/__init__.cpython-37.pyc | Bin 145 -> 0 bytes 22 files changed, 108 insertions(+), 2072 deletions(-) create mode 100644 .gitignore delete mode 100644 .ipynb_checkpoints/compiler-checkpoint.py delete mode 100755 .ipynb_checkpoints/dependencies-checkpoint.txt delete mode 100644 .ipynb_checkpoints/mkwc_util-checkpoint.py delete mode 100644 .ipynb_checkpoints/nirc2_util-checkpoint.py delete mode 100644 .ipynb_checkpoints/telem_util-checkpoint.py delete mode 100644 .ipynb_checkpoints/temp_util-checkpoint.py delete mode 100644 .ipynb_checkpoints/time_util-checkpoint.py delete mode 100644 katan/.ipynb_checkpoints/compiler-checkpoint.py delete mode 100644 katan/.ipynb_checkpoints/mkwc-checkpoint.py delete mode 100644 katan/.ipynb_checkpoints/strehl-checkpoint.py delete mode 100644 katan/.ipynb_checkpoints/telemetry-checkpoint.py delete mode 100644 katan/.ipynb_checkpoints/temperature-checkpoint.py delete mode 100644 katan/.ipynb_checkpoints/times-checkpoint.py delete mode 100644 katan/__pycache__/__init__.cpython-37.pyc delete mode 100644 katan/__pycache__/compiler.cpython-37.pyc delete mode 100644 katan/__pycache__/mkwc.cpython-37.pyc delete mode 100644 katan/__pycache__/strehl.cpython-37.pyc delete mode 100644 katan/__pycache__/telemetry.cpython-37.pyc delete mode 100644 katan/__pycache__/temperature.cpython-37.pyc delete mode 100644 katan/__pycache__/times.cpython-37.pyc delete mode 100644 katan/templates/__pycache__/__init__.cpython-37.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b56c81e --- /dev/null +++ b/.gitignore @@ -0,0 +1,108 @@ +*.pyc +.ipynb_checkpoints/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + diff --git a/.ipynb_checkpoints/compiler-checkpoint.py b/.ipynb_checkpoints/compiler-checkpoint.py deleted file mode 100644 index 38c2592..0000000 --- a/.ipynb_checkpoints/compiler-checkpoint.py +++ /dev/null @@ -1,159 +0,0 @@ -### compiler.py : Compiler for all desired data (nirc2, weather, seeing, telemetry, temperature) -### Author : Emily Ramey -### Date : 6/3/21 - -import time_util as times -import nirc2_util as nirc2 -import telem_util as telem -import temp_util as temp -import mkwc_util as mkwc - -import numpy as np -import pandas as pd - -# All data types that can be compiled by this package -accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] - -# Shorthand / nicknames for data types -expand = { - 'temp': ['k2AO', 'k2L4', 'k2ENV'], - 'seeing': ['mass', 'dimm', 'masspro'], - 'weather': ['cfht'], - 'telemetry': ['telem'], - 'all': accept_labels, -} - -# Utility functions -def check_dtypes(data_types): - """ Returns an expanded / cleaned list of data types from user input """ - new_dtypes = [] - # Check requested data - for dtype in data_types: - # Check for nicknames - if dtype in expand: - new_dtypes.extend(expand[dtype]) - elif dtype in accept_labels: # can get data - new_dtypes.append(dtype) - # Return cleaned list - return new_dtypes - -############################################# -######## Interface with other modules ####### -############################################# - -def data_func(dtype): - """ - Returns the function to get data for the specified dtype - and whether data needs to be matched to nirc2 mjds - """ - if dtype in ['cfht']+expand['seeing']: - return lambda files,mjds: mkwc.from_nirc2(mjds,dtype), True - if dtype in expand['temp']: - return lambda files,mjds: temp.from_mjds(mjds,dtype), True - if dtype=='telem': - return lambda files,mjds: telem.from_nirc2(mjds,files), False - - -################################ -###### Compiler Functions ###### -################################ - -def match_data(mjd1, mjd2): - """ - Matches data1 with its closest points in data2, by mjd values - Returns an array containing data1 with corresponding rows from data2 - """ - # Edge case - if mjd1.empty or mjd2.empty: - return None - - # Get mjds from dataframes - mjd1 = np.array(mjd1).reshape(-1,1) # column vector - mjd2 = np.array(mjd2).reshape(1,-1) # row vector - - # Take difference of mjd1 with mjd2 - diffs = np.abs(mjd1 - mjd2) - # Find smallest difference for each original mjd - idxs = np.argmin(diffs, axis=1) - - # Return indices of matches in mjd2 - return idxs - - -# data_types can contain: 'chft', 'mass', 'dimm', 'masspro', -# 'telem', 'k2AO', 'k2L4', or 'k2ENV' -# 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and -# ['mass', 'dimm', 'masspro'], respectively -def combine_strehl(strehl_file, data_types, check=True, test=False): - """ - Combines and matches data from a certain Strehl file with other specified data types. - NIRC2 files must be in the same directory as the Strehl file. - """ - - nirc2_data = nirc2.from_strehl(strehl_file) - - if test: # Take first few files - nirc2_data = nirc2_data.loc[:3] - - if check: # Sanitize user input - data_types = check_dtypes(data_types) - - # Full data container - all_data = [nirc2_data.reset_index(drop=True)] - - # Loop through and get data - for dtype in data_types: - get_data, match = data_func(dtype) # Data retrieval function - # Get other data from strehl info - other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) - - if match: # Needs to be matched - if other_data.empty: # No data found - other_data = pd.DataFrame(columns=other_data.columns, index=range(len(nirc2_data))) - else: # Get indices of matched data - idxs = match_data(nirc2_data.nirc2_mjd, other_data[dtype+'_mjd']) - other_data = other_data.iloc[idxs] - -# # Edge case: no data -# if idxs is not None: -# other_data = other_data.iloc[idxs] - - # Add to all data - all_data.append(other_data.reset_index(drop=True)) - - # Concatenate new data with nirc2 - return pd.concat(all_data, axis=1) - -################################# -######### Full Compiler ######### -################################# - -def compile_all(data_dir, data_types=['all'], test=False, save=False): - """ Compiles and matches requested data from all strehl files in a given nirc2 directory """ - - # Expand requested data types - data_types = check_dtypes(data_types) - - # Get all epochs in requested folder - epochs = nirc2.search_epochs(data_dir=data_dir) - - # Full data container - all_data = [] - - # Compile data for each epoch - for epoch, strehl_file in epochs.items(): - data = combine_strehl(strehl_file, data_types, check=False, - test=test) - # Add epoch name to columns - data['epoch'] = epoch - - # Append to full data - all_data.append(data) - - all_data = pd.concat(all_data, ignore_index=True) - - if save: - all_data.to_csv(save, index=False) - - # Concatenate all data - return all_data \ No newline at end of file diff --git a/.ipynb_checkpoints/dependencies-checkpoint.txt b/.ipynb_checkpoints/dependencies-checkpoint.txt deleted file mode 100755 index e69de29..0000000 diff --git a/.ipynb_checkpoints/mkwc_util-checkpoint.py b/.ipynb_checkpoints/mkwc_util-checkpoint.py deleted file mode 100644 index 7b626c3..0000000 --- a/.ipynb_checkpoints/mkwc_util-checkpoint.py +++ /dev/null @@ -1,184 +0,0 @@ -### mkwc_util.py : Contains utilities for extracting and processing data from the MKWC website -### Author : Emily Ramey -### Date : 6/1/2021 - -import os -import numpy as np -import pandas as pd -import time_util as times - -### NOTE: Seeing data is STORED by UT, with data in HST -### CFHT data is STORED by HST, with data in HST - -# cfht_dir = "/u/emily_ramey/work/Keck_Performance/data/weather_data/" -mkwc_url = 'http://mkwc.ifa.hawaii.edu/' -year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' -cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST - -# seeing_dir = "/u/emily_ramey/work/Keck_Performance/data/seeing_data" - -# Time columns from MKWC data -time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] - -# Data-specific fields -data_types = { - 'cfht': { - 'web_pat': mkwc_url+'archive/wx/cfht/indiv-days/cfht-wx.{}.dat', - 'file_pat': "{}cfht-wx.{}.dat", - 'data_cols': ['wind_speed', 'wind_direction', 'temperature', - 'relative_humidity', 'pressure'], - }, - 'mass': { - 'web_pat': mkwc_url+'current/seeing/mass/{}.mass.dat', - 'file_pat': '{}mass/{}.mass.dat', - 'data_cols': ['mass'], - }, - 'dimm': { - 'web_pat': mkwc_url+'current/seeing/dimm/{}.dimm.dat', - 'file_pat': '{}dimm/{}.dimm.dat', - 'data_cols': ['dimm'], - }, - 'masspro': { - 'web_pat': mkwc_url+'current/seeing/masspro/{}.masspro.dat', - 'file_pat': '{}masspro/{}.masspro.dat', - 'data_cols': ['masspro_half', 'masspro_1', 'masspro_2', - 'masspro_4', 'masspro_8', 'masspro_16', 'masspro'], - }, -} - -# Mix and match data & time columns -for dtype in data_types: - # CFHT files don't have seconds - tcols = time_cols if dtype != 'cfht' else time_cols[:-1] - # Format web columns - data_types[dtype]['web_cols'] = tcols+data_types[dtype]['data_cols'] - # Format file columns - data_types[dtype]['cols'] = [dtype+'_mjd']+data_types[dtype]['data_cols'] - # Different file storage timezones - data_types[dtype]['file_zone'] = 'hst' if dtype=='cfht' else 'utc' - -############################# -######### Functions ######### -############################# - -def cfht_from_year(datestrings, year): - """ - Gets pre-2012 MKWC data from the year-long file - instead of the by-date files - """ - # Get year-long file URL - url = year_url.format(year) - - # Read in data - try: - web_cols = data_types['cfht']['web_cols'] - year_data = pd.read_csv(url, delim_whitespace=True, header=None, - names=web_cols, usecols=range(len(web_cols))) - except: # No data, return blank - return pd.DataFrame(columns=data_types['cfht']['cols']) - - # Full dataset - all_data = [pd.DataFrame(columns=data_types['cfht']['cols'])] - - # Slice up dataframe - for ds in datestrings: - month, day = int(ds[4:6]), int(ds[6:]) - # Get data by month and day - df = year_data.loc[(year_data.month==month) & (year_data.day==day)].copy() - # Format columns - if not df.empty: - format_columns(df, 'cfht') - # Add to full dataset - all_data.append(df) - - return pd.concat(all_data) - -def format_columns(df, dtype): - """ Changes columns (in place) from time_cols to MJD """ - # Get MJDs from HST values - mjds = times.table_to_mjd(df, columns=time_cols, zone='hst') - df[dtype+'_mjd'] = mjds - - # Drop old times - df.drop(columns=time_cols, inplace=True, errors='ignore') - -def from_url(datestring, dtype): - """ Pulls cfht or seeing file from MKWC website """ - # Format URL - url = data_types[dtype]['web_pat'].format(datestring) - - # Read data - try: # Check if data is there - web_cols = data_types[dtype]['web_cols'] # MKWC Weather columns - df = pd.read_csv(url, delim_whitespace = True, header=None, - names=web_cols, usecols=range(len(web_cols))) - except: # otherwise return blank df - return pd.DataFrame(columns=data_types[dtype]['cols']) - - # Get mjd from time columns - format_columns(df, dtype) - - return df - -def from_file(filename, dtype, data_dir='./'): - """ Pulls cfht or seeing file from local directory """ - # Read in CSV - df = pd.read_csv(filename) - - # Check formatting - if 'mjd' in df.columns: - df.rename(columns={'mjd':dtype+'_mjd'}, inplace=True) - elif dtype+'_mjd' not in df.columns: # No MJD - try: # Change to MJD, if times - format_columns(df, dtype) - except: # No time info, return blank - return pd.DataFrame() - - return df - -def from_nirc2(mjds, dtype, data_dir='./'): - """ - Compiles a list of cfht or seeing observations based on MJDs - note: does not compare MJDs; assumption is inputs are rounded to nearest day - """ - - # Get datestrings - dts = times.mjd_to_dt(mjds, zone=data_types[dtype]['file_zone']) - datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 - # No duplicates - datestrings = pd.Series(np.unique(datestrings)) - - # Blank data structure - all_data = [pd.DataFrame(columns=data_types[dtype]['cols'])] - - # Check for pre-2012 cfht files - if dtype=='cfht' and any(mjds < cfht_cutoff): - # Get datetimes - pre_2012 = datestrings[mjds < cfht_cutoff] - - # Compile data by year - for yr in np.unique(pre_2012.str[:4]): - ds = pre_2012[pre_2012.str[:4]==yr] - df = cfht_from_year(ds, yr) - # Append to full dataset - all_data.append(df) - - # Find data for each file - for ds in datestrings: - # Get local filename - filename = data_types[dtype]['file_pat'].format(data_dir, ds) - - # Check for local files - if os.path.isfile(filename): - df = from_file(filename, dtype) - - else: # Pull from the web - df = from_url(ds, dtype) - - # Save to local file: TODO - - # Add to data - all_data.append(df) - - # Return concatenated dataframe - return pd.concat(all_data, ignore_index=True) \ No newline at end of file diff --git a/.ipynb_checkpoints/nirc2_util-checkpoint.py b/.ipynb_checkpoints/nirc2_util-checkpoint.py deleted file mode 100644 index f34174b..0000000 --- a/.ipynb_checkpoints/nirc2_util-checkpoint.py +++ /dev/null @@ -1,87 +0,0 @@ -### nirc2_util.py : Functions for processing nirc2 data and strehl files -### Author : Emily Ramey -### Date : 6/2/2021 - -import os -import numpy as np -import pandas as pd -import glob -from astropy.io import fits - -import time_util as times - -MAX_LEN = 10 # Max char length of epoch name - -nirc2_dir = "/g/lu/data/gc/lgs_data/" -strehl_pat = nirc2_dir+"{}/clean/kp/{}" - -strehl_filenames = ['strehl_source.txt', 'irs33N.strehl'] -strehl_cols = ['nirc2_file', 'strehl', 'rms_err', 'fwhm', 'nirc2_mjd'] -header_kws = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', - 'AOLBFWHM', 'WSFRRT', 'LSAMPPWR', 'LGRMSWF', 'AOAOAMED', 'TUBETEMP'] - -# Note: need to get better at identifying which strehl files we want -# Alternatively, we could have people provide a list -def search_epochs(data_dir=nirc2_dir): - """ Searches for valid epoch names in NIRC2 data directory """ - # Epochs with valid strehl files - good_files = {} - # Loop through all sub-directories - for epoch in os.listdir(data_dir): -# # Only valid if {yr}{month}lgs -# if len(epoch) >= MAX_LEN: -# continue - - # Search epoch for valid strehl file - for file in strehl_filenames: - strehl_file = strehl_pat.format(epoch, file) - # If good, add to dict - if os.path.isfile(strehl_file): - good_files[epoch] = strehl_file - # Returns {epoch:strehl} dict - return good_files - -def from_filename(nirc2_file, data, i): - """ - Gets nirc2 header values from a filename as dict - or loads values into specified df - """ - # Check for valid file - if not os.path.isfile(nirc2_file): - return - # Open nirc2 file - with fits.open(nirc2_file) as file: - nirc2_hdr = file[0].header - - # Get fields from header - for kw in header_kws: - # load DataFrame value - data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) - -def from_strehl(strehl_file): - """ Gets NIRC2 header data based on contents of Strehl file """ - # Get directory name - data_dir = os.path.dirname(strehl_file) - # Retrieve Strehl data - strehl_data = pd.read_csv(strehl_file, delim_whitespace = True, - header = None, skiprows = 1, names=strehl_cols) - # Add true file path - strehl_data['nirc2_file'] = data_dir + "/" + strehl_data['nirc2_file'] - - # Add decimal year - strehl_data['dec_year'] = times.mjd_to_yr(strehl_data.nirc2_mjd) - - # Add nirc2 columns - for col in header_kws: - strehl_data[col.lower()] = np.nan - - # Loop through nirc2 files - for i,nirc2_file in enumerate(strehl_data.nirc2_file): - # Load header data into df - from_filename(nirc2_file, strehl_data, i) - - # Return data - return strehl_data - - - diff --git a/.ipynb_checkpoints/telem_util-checkpoint.py b/.ipynb_checkpoints/telem_util-checkpoint.py deleted file mode 100644 index 30bbd76..0000000 --- a/.ipynb_checkpoints/telem_util-checkpoint.py +++ /dev/null @@ -1,283 +0,0 @@ -### telem_util.py: For code that interacts with Keck AO Telemetry files -### Author: Emily Ramey -### Date: 11/18/2020 - -import numpy as np -import pandas as pd -import glob -from scipy.io import readsav - -import time_util as times - -### Path to telemetry files [EDIT THIS] -telem_dir = '/g/lu/data/keck_telemetry/' - -# Sub-aperture maps -map_dir = "./" -wfs_file = map_dir+"sub_ap_map.txt" -act_file = map_dir+"act.txt" - -filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers -filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" - -TIME_DELTA = 0.001 # About 100 seconds in mjd - -RESID_CUTOFF = 349 -TT_IDXS = [349,350] -DEFOCUS = 351 -LAMBDA = 2.1 # microns - -cols = ['telem_file', - 'telem_mjd', - 'TT_mean', - 'TT_std', - 'DM_mean', - 'DM_std', - 'rmswfe_mean', - 'rmswfe_std', - 'strehl_telem'] - -def read_map(filename): - """ Reads a map of actuators or sub-apertures from a file """ - return pd.read_csv(filename, delim_whitespace=True, header=None).to_numpy() - -def get_times(telem_data, start=None): - """ Pulls timestamps from a telemetry file in seconds from start """ - if start is None: - start = telem_data.a.timestamp[0][0] - - return (telem_data.a.timestamp[0]-start)/1e7 - -def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): - """ - Return the locations of the valid actuators in the actuator array - resids: Nx349 residual wavefront array (microns) - ints: Nx304 intensity array (any units) - N: Number of timestamps - """ - # Check inputs - N = ints.shape[0] # Num timestamps - - # Aggregate intensities over all timestamps - med_ints = np.median(ints, axis=0) - - # Fill WFS map with aggregated intensities - int_map = wfs_map.copy() - int_map[np.where(int_map==1)] = med_ints - - # Find lenslets with greatest intensity - idxs = np.flip(np.argsort(int_map, axis=None))[:num_aps] # flat idxs of sort - idxs = np.unravel_index(idxs, wfs_map.shape) # 2D idxs of sort - - # Mask for good sub-ap values - good_aps = np.zeros(wfs_map.shape, dtype=int) - good_aps[idxs] = 1 - good_aps = good_aps * wfs_map # Just in case - - # Mask for good actuator values - good_acts = np.pad(good_aps, ((1,1),(1,1))) - good_acts = (good_acts[1:,1:] | good_acts[1:,:-1] - | good_acts[:-1,:-1] | good_acts[:-1,1:]) * act_map - - return good_acts - -# def wfs_error2(resids, ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file)): -# """ Calculates the variance in the wavefront (microns^2) produced by the WFS """ -# # Get residual mask for brightest sub-aps -# rmask = resid_mask(resids, ints) - -# # Square residuals -# sig2_resid = np.mean(resids**2, axis=0) -# # Load into actuator grid -# resid_grid = act_map.astype(float) -# resid_grid[np.where(resid_grid==1)] = sig2_resid - -# # Mask out actuators next to dimmer apertures -# sig2_masked = resid_grid * rmask -# # Sum values and return -# return np.sum(sig2_masked) - -# def tt_error2(tt_resids): ### TODO: fix this so it's not unitless -# """ Calculate mean tip-tilt residual variance in microns """ -# wvln = 0.658 # wavelength (microns) -# D = 10.5 * 1e6 # telescope diameter (microns) - -# # Magnitude of residuals -# sig_alpha2 = (tt_resids[:,0] + tt_resids[:,1])**2 # x + y TT residual variance -# sig_w2 = (np.pi*D/wvln/2.0)**2 * sig_alpha2 # TT resids in microns^2 - -# return np.mean(sig_w2) - -# def total_WFE(telem_data): -# resids = telem_data.a.residualwavefront[0][:, :RESID_CUTOFF] * 0.6 # WFS resids, microns, Nx349 -# ints = telem_data.a.subapintensity[0] # Sub-ap intensities, Nx304 -# tt_resids = telem_data.a.residualwavefront[0][:, TT_IDXS] * np.pi / (180*3600) # TT resids, radians, Nx2 -# defocus = [0] #telem_data.a.residualwavefront[0][:, DEFOCUS] # Defocus resids, microns, Nx1 - -# return wfs_error2(resids, ints) + tt_error2(tt_resids) + np.mean(defocus**2) - -# def mask_residuals_old(telem_data, num_aps=236, wfs_file=wfs_file, act_file=act_file): -# """ Really freakin complicated array logic to mask out all invalid actuators """ -# # Get data -# resids = telem_data.a.residualwavefront[0][:, :349]*0.6 # microns -# intensities = telem_data.a.subapintensity[0] # ADU -# N = intensities.shape[0] -# # Get hardware maps -# wfs_map = read_map(wfs_file) -# act_map = read_map(act_file) - -# # Get X and Y values of sub-aps and replicate them for all timestamps -# wfs_x, wfs_y = np.where(wfs_map==1) -# wfs_x, wfs_y = np.tile(wfs_x, (N,1)), np.tile(wfs_y, (N,1)) - -# # Get valid indices for each timestep -# idxs = np.flip(np.argsort(intensities, axis=1), axis=1)[:,:num_aps] -# valid_x = np.take_along_axis(wfs_x, idxs, axis=1) -# valid_y = np.take_along_axis(wfs_y, idxs, axis=1) -# valid_z = np.tile(np.arange(N), (num_aps,1)).T - -# # Put 1s at each valid index -# valid_saps = np.zeros((N, 20, 20), int) -# valid_saps[valid_z, valid_x, valid_y] = 1 # TODO: flip this back -# # Pad each sheet (timestamp) with zeros at the edges -# check = valid_saps.reshape(N, 20*20).sum(axis=1) -# if any(check!=236): -# print("Shape mismatch in valid sub-ap array") -# valid_saps = np.pad(valid_saps, ((0,0),(1,1),(1,1))) - -# # Get (potentially)valid actuators for sub-aps -# valid_acts = (valid_saps[:,1:,1:]|valid_saps[:,1:,:-1]| -# valid_saps[:,:-1,:-1]|valid_saps[:,:-1,1:]) # 4 corners of sub-aps -# # Multiply by actuator map to remove any non-actuator positions -# valid_acts = valid_acts * np.tile(act_map, (N,1,1)) - -# # Get number of valid actuators in each frame (can vary due to edge cases) -# valid_act_nums = valid_acts.reshape(N,21*21).sum(axis=1) - -# # Map residuals to actuator positions -# resid_vals = np.tile(act_map, (N,1,1)).astype(float) -# resid_vals[np.where(resid_vals==1)] = resids.flatten() - -# # Mask out invalid actuators -# valid_acts = valid_acts * resid_vals -# rms_resids = valid_acts.reshape(N, 21*21) - -# # Take the RMS residual for each frame -# rms_resids = np.sqrt((rms_resids**2).sum(axis=1)/valid_act_nums) - -# return rms_resids - -def tt2um(tt_as): - """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ - D = 10.5e6 # telescope diameter in microns - tt = tt_as*4.8e-6 # TT error in radians - tt_err = np.sqrt(D**2 / 12 * (tt[:,0]**2 + tt[:,1]**2)) - return tt_err - -def rms_acts(act_resids, ints): - """ Clips bad actuators and averages for each timestamp """ - N = act_resids.shape[0] # num timestamps - - # Read in actuator map - act_map = read_map(act_file) - - # Get good actuator mask from intensities - mask = resid_mask(ints) - flat_map = ~mask[np.where(act_map==1)].astype(bool) # flatten - flat_map = np.tile(flat_map, (N,1)) - - # Mask the bad actuators - good_acts = np.ma.masked_array(act_resids, flat_map) - # Average resids for each timestep - act_rms = np.sqrt((good_acts**2).mean(axis=1) - good_acts.mean(axis=1)**2) - - return act_rms.compressed() - -######################################## -######### Processing Files ############# -######################################## - -# Do some telemetry files need to be cropped around the observation? - -def get_mjd(telem): - """ Validates a telemetry file against an MJD value """ - # Get timestamp - tstamp = telem.tstamp_str_start.decode('utf-8') - # Convert to MJD - telem_mjd = times.str_to_mjd(tstamp, fmt='isot') - - # Return telem mjd if they match, else return False - return telem_mjd - -def extract_telem(file, data, idx, check_mjd=None): - """ Extracts telemetry values from a file to a dataframe """ - # Read IDL file - telem = readsav(file) - - # Make sure MJD matches - telem_mjd = get_mjd(telem) - if check_mjd is not None: - delta = np.abs(telem_mjd-check_mjd) - if delta > TIME_DELTA: - return False - - # Get residuals and intensities - act_resids = telem.a.residualwavefront[0][:, :RESID_CUTOFF] - tt_resids = telem.a.residualwavefront[0][:,TT_IDXS] - ints = telem.a.subapintensity[0] # Sub-aperture intensities - - # Convert TT resids to microns - tt_microns = tt2um(tt_resids) - # Get RMS resids from the actuator array - act_rms = rms_acts(act_resids, ints) - - # Total RMS Wavefront Error - rmswfe = np.sqrt(tt_microns**2 + act_rms**2) - - # Strehl calculation - strehl = np.exp(-(2*np.pi*rmswfe/LAMBDA)**2) - - # Assemble aggregate data - data.loc[idx, cols] = [ - file, - telem_mjd, - np.mean(tt_microns), - np.std(tt_microns), - np.mean(act_rms), - np.std(act_rms), - np.mean(rmswfe), - np.std(rmswfe), - np.mean(strehl), - ] - - return True - -def from_nirc2(mjds, filenames): - """ Gets a table of telemetry information from a set of mjds and file numbers """ - N = len(mjds) # number of data points - # Get file numbers - filenums = filenames.str.extract(filenum_match, expand=False) - filenums = filenums.str[1:] # First digit doesn't always match - - # Get datestrings - dts = times.mjd_to_dt(mjds) # HST or UTC??? - datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 - - # Set up dataframe - data = pd.DataFrame(columns=cols, index=range(N)) - - # Find telemetry for each file - for i in range(N): - # Get filename and number - fn, ds, mjd = filenums[i], datestrings[i], mjds[i] - # Search for correct file - file_pat = filename_match.format(ds, fn) - all_files = glob.glob(file_pat, recursive=True) - - # Grab the first file that matches the MJD - for file in all_files: - success = extract_telem(file, data, i, check_mjd=mjd) - if success: break - - return data - \ No newline at end of file diff --git a/.ipynb_checkpoints/temp_util-checkpoint.py b/.ipynb_checkpoints/temp_util-checkpoint.py deleted file mode 100644 index 18c3053..0000000 --- a/.ipynb_checkpoints/temp_util-checkpoint.py +++ /dev/null @@ -1,277 +0,0 @@ -### temp_util.py: Functions relating to Keck-II temperature data -### Main reads in Keck II temperature files (AO, env, and L4) and saves each to a FITS file -### Author: Emily Ramey -### Date: 01/04/2021 - -import os -import numpy as np -import pandas as pd -import glob -from astropy.io import fits -from astropy.table import Table - -import time_util as times - -verbose = True - -data_dir = "/u/emily_ramey/work/Keck_Performance/data/" -temp_dir = data_dir+"temp_data_2/" - -col_dict = { - 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV - 'AO_benchT2': 'k1:ao:env:benchtemp2_Raw', - 'k1:ao:env:elect_vault_t': 'k1:ao:env:elect_vault:T', - 'k0:met:humidityStats.VAL': 'k0:met:humidityStats', - "k2:ao:env:ETroomtemp_Raw": "El_roomT", # k2AO - "k2:ao:env:DMracktemp_Raw": "DM_rackT", - "k2:ao:env:KCAMtemp2_Raw": "KCAM_T2", - "k2:ao:env:OBrighttemp_Raw": "OB_rightT", - "k2:ao:env:phototemp_Raw": "photometricT", - "k2:ao:env:OBlefttemp_Raw": "OB_leftT", - "k2:ao:env:KCAMtemp1_Raw": "KCAM_T1", - "k2:ao:env:ACAMtemp_Raw": "AOA_camT", - "k2:ao:env:LStemp_Raw": "LGS_temp", # k2L4 - "k2:ao:env:LSenchumdity_Raw": "LGS_enclosure_hum", - "k2:ao:env:LShumidity_Raw": "LGS_humidity", - "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" -} - -file_pats = { # File name patterns -# 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", -# 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", -# 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", - 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", - 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", - 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", -} - -data_types = { - 'k2AO': { - 'file_pat': temp_dir+"k2AOtemps/{}/AO_temps.log", - 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', - 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', - 'k2AO_mjd'], - }, - 'k2L4': { - 'file_pat': temp_dir+"k2L4temps/{}/L4_env.log", - 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', - 'LGS_temp', 'k2L4_mjd'], - }, - 'k2ENV': { - 'file_pat': temp_dir+"k2envMet/{}/envMet.arT", - 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', - 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', - 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', - 'k0:met:out:windSpeedMaxStats', 'k0:met:out:windSpeedMaxmph', - 'k0:met:out:windSpeedmph', 'k0:met:outTempDwptDiff', - 'k0:met:pressureRaw', 'k0:met:pressureStats', 'k0:met:tempMax', - 'k0:met:tempMin', 'k0:met:tempRaw', 'k2:dcs:sec:acsDwptDiff', - 'k2:dcs:sec:acsTemp', 'k2:dcs:sec:secDwptDiff', 'k2:dcs:sec:secondaryTemp', - 'k2:met:humidityRaw','k2:met:humidityStats', 'k2:met:tempRaw', 'k2:met:tempStats', - 'k2:met:windAzRaw', 'k2:met:windElRaw', 'k2:met:windSpeedMaxmph', - 'k2:met:windSpeedMinmph', 'k2:met:windSpeedRaw', 'k2:met:windSpeedStats', - 'k2:met:windSpeedmph', 'k2ENV_mjd'], - } -} - -def get_columns(): - all_data = search_files() - for name, data_files in all_data.items(): - columns = set() - for i,file in enumerate(data_files): - try: - df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, - na_values=['***'], error_bad_lines=False, warn_bad_lines=False, - ).replace('"', regex=True) - except: - if verbose: - print(f"Warning: read failed for file {file}") - continue - - if len(df.columns)==1: # no header - if verbose: - print(f"Skipping file {file}, no header") - continue # skip for now - df.columns = [col.replace('"', '').strip() for col in df.columns] - if "UNIXDate" not in df.columns: - if verbose: - print(f"Skipping file {file}, columns not as expected") - continue # skip for now - for col in df.columns: - columns.add(col) - all_data[name] = columns - - return all_data - - -def search_files(file_pats=file_pats): - """ Finds all filenames matching the given file patterns """ - all_filenames = {} - for name, search in file_pats.items(): - filenames = glob.glob(search, recursive=True) - all_filenames[name] = filenames - if verbose: - print(f"Done {name}, length: {len(filenames)}") - - return all_filenames - -def collect_data(data_files, col_dict=col_dict): - """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ - if isinstance(data_files, dict): # Return dict w/dataframes - new_files = {} - for name, files in data_files.items(): - if isinstance(files, list): - # Recurse on each list of files - new_files[name] = collect_data(files) - return new_files - - all_dfs = [pd.DataFrame()] - for i,file in enumerate(data_files): - if not os.path.isfile(file): - continue - try: - df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, - na_values=['***'], error_bad_lines=False, warn_bad_lines=False, - ).replace('"', regex=True) - except: - if verbose: - print(f"Warning: read failed for file {file}") - continue - - if len(df.columns)==1: # no header - if verbose: - print(f"Skipping file {file}, no header") - continue # skip for now - df.columns = [col.replace('"', '').strip() for col in df.columns] - if "UNIXDate" not in df.columns: - if verbose: - print(f"Skipping file {file}, columns not as expected") - continue # skip for now - - if col_dict is not None: - df = df.rename(columns=col_dict) - - all_dfs.append(df) - - data = pd.concat(all_dfs, ignore_index=True, sort=True) - return data - -def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): - """ Parses specified date and time columns and returns a cleaned data table """ - new_data = data.copy() - for label,cols in date_cols.items(): - date_col, time_col = cols - # Parse dates and times, coercing invalid strings to NaN - datetimes = (pd.to_datetime(data[date_col], exact=False, errors='coerce') + - pd.to_timedelta(data[time_col], errors='coerce')) - new_data = new_data.drop(columns=cols) - new_data[label] = datetimes - - return new_data - -def clean_dates(data, date_cols=['HST', 'UNIX']): - """ Removes any rows containing invalid dates in date_cols """ - new_data = data.copy() - for col in date_cols: - new_data = new_data[~np.isnan(new_data[col])] - - return new_data - -def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): - """ Casts columns to a numeric data type """ - if data_cols is None: - data_cols = [col for col in data.columns if col not in non_numeric] - - # Cast data to numeric type, coercing invalid values to NaN - new_data = data.copy() - for col in data_cols: - new_data[col] = pd.to_numeric(new_data[col], errors='coerce') - - return new_data - -def to_fits(data, filename, str_cols=['HST', 'UNIX']): - """ Writes a FITS file from the given temperature array """ - fits_data = data.copy() - for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: - if col in fits_data.columns: - fits_data = fits_data.drop(columns=[col]) - for col in str_cols: - if col in fits_data.columns: - fits_data[col] = fits_data[col].astype(str) - # Assuming the data columns are already numeric - fits_data = Table.from_pandas(fits_data) - fits_data.write(filename) - - return - -def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): - """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ - data = Table.read(filename).to_pandas() - - # Fix NaNs, because astropy is dumb sometimes - data[data==1e+20] = np.nan - - if date_cols is None: return data - - for col in date_cols: - if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings - data[col] = data[col].str.decode("utf-8") - data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') - - if str_cols is None: return data - - for col in str_cols: - if col in data.columns and isinstance(data[col][0], bytes): - data[col] = data[col].str.decode("utf-8") - - return data - -def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): - """ - Reads in all data matching file patterns (file_pats), combines them into one table, - cleans them, and saves them to a FITS file - """ - # Find all files matching pattern - all_filenames = search_files(file_pats) - # Read all data into one table (per dictionary label) - all_data = collect_data(all_filenames) - - for name, data in all_data.items(): - data = parse_dates(data) # Parse date cols into datetimes - data = clean_dates(data) # Remove invalid dates - data = clean_data(data) # Casts other cols to numeric - # Save combined/cleaned data to FITS file - filename = location+name+".fits" - to_fits(data, filename) - - return - -def from_mjds(mjds, dtype): - """ Gets temp data of input type from the specified MJDs """ - # Get pd datetimes in HST - dts = times.mjd_to_dt(mjds, zone='hst') - # Format list - datestrings = dts.strftime("%y/%m/%d") - datestrings = np.unique(datestrings) # one file per date - # Get relevant filenames - filenames = [data_types[dtype]['file_pat'].format(ds) for ds in datestrings] - # Get data from filenames - data = collect_data(filenames) - - # Empty dataframe - if data.empty: - return pd.DataFrame(columns=data_types[dtype]['cols']) - - # Convert dates & times to MJDs - data["datetime"] = data['HSTdate']+' '+data['HSTtime'] - mjds = times.table_to_mjd(data, columns='datetime', zone='hst') - # Add to dataframe - data[dtype+"_mjd"] = mjds - # Drop other date & time cols - data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], - inplace=True, errors='ignore') - - return data - -if __name__=='__main__': - pass \ No newline at end of file diff --git a/.ipynb_checkpoints/time_util-checkpoint.py b/.ipynb_checkpoints/time_util-checkpoint.py deleted file mode 100644 index 4974d2c..0000000 --- a/.ipynb_checkpoints/time_util-checkpoint.py +++ /dev/null @@ -1,56 +0,0 @@ -### time_util.py : to handle time formatting changes between MJD, HST, and UTC -### Author : Emily Ramey -### Date: 5/26/21 - -# Preamble -import pandas as pd -import numpy as np -import time -import pytz as tz -from datetime import datetime, timezone -from astropy.time import Time, TimezoneInfo -from astropy import units as u, constants as c - -hst = tz.timezone('US/Hawaii') - -### Conversion / utility functions -def mjd_to_dt(mjds, zone='utc'): - """ Converts Modified Julian Date(s) to HST or UTC date(s) """ - # Convert mjds -> astropy times -> datetimes - dts = Time(mjds, format='mjd', scale='utc').to_datetime() - # Convert to pandas - dts = pd.to_datetime(dts).tz_localize(tz.utc) - if zone=='hst': - dts = dts.tz_convert("US/Hawaii") - return dts - -def table_to_mjd(table, columns, zone='utc'): - """ Converts date and time columns to mjds """ - # Safety check for list of columns - if not isinstance(columns, str): - columns = [col for col in columns if col in table.columns] - # Convert to datetimes - dts = pd.to_datetime(table[columns], errors='coerce') - - if zone=='hst':# Convert to UTC - dts = dts.dt.tz_localize(hst) - dts = dts.dt.tz_convert(tz.utc) - - # Masked invalid values - dts = np.ma.masked_array(dts, mask=dts.isnull()) - - # Convert to astropy - times = Time(dts, format='datetime', scale='utc') - # return MJDs - return np.ma.getdata(times.mjd) - -def str_to_mjd(datestrings, fmt): - """ Converts astropy-formatted date/time strings (in UTC) to MJD values """ - # Get astropy times - times = Time(datestrings, format=fmt, scale='utc') - # Convert to mjd - return times.mjd - -def mjd_to_yr(mjds): - """ Converts MJD to Decimal Year """ - return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file diff --git a/katan/.ipynb_checkpoints/compiler-checkpoint.py b/katan/.ipynb_checkpoints/compiler-checkpoint.py deleted file mode 100644 index 08e4ad6..0000000 --- a/katan/.ipynb_checkpoints/compiler-checkpoint.py +++ /dev/null @@ -1,157 +0,0 @@ -### compiler.py : Compiler for all desired data (nirc2, weather, seeing, telemetry, temperature) -### Author : Emily Ramey -### Date : 6/3/21 - -from . import times -from . import strehl -from . import mkwc -from . import telemetry as telem -from . import temperature as temp -from . import templates - -import numpy as np -import pandas as pd -import os - -# Check dependencies -try: - import yaml -except: - raise ValueError("PyYAML not installed. Please install from https://anaconda.org/anaconda/pyyaml") - -### For importing package files -### May need to edit this later if there are import issues with Py<37 -try: - import importlib.resources as pkg_resources -except ImportError: - # Try backported to PY<37 `importlib_resources`. - import importlib_resources as pkg_resources - -# All data types that can be compiled by this package -accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] - -default_parfile = 'keyword_defaults.yaml' - -### Load default parameter file -try: # Load from templates module - file = pkg_resources.open_text(templates, default_parfile) - default_params = yaml.load(file, Loader=yaml.FullLoader) -except: # Warn user - default_params = {} - print("WARNING: Unable to load default parameters. Please specify a parameter file when compiling.") - -# Shorthand / nicknames for data types -expand = { - 'temp': ['k2AO', 'k2L4', 'k2ENV'], - 'seeing': ['mass', 'dimm', 'masspro'], - 'weather': ['cfht'], - 'telemetry': ['telem'], - 'all': accept_labels, -} - -# Utility functions -def check_dtypes(data_types): - """ Returns an expanded / cleaned list of data types from user input """ - new_dtypes = [] - # Check requested data - for dtype in data_types: - # Check for nicknames - if dtype in expand: - new_dtypes.extend(expand[dtype]) - elif dtype in accept_labels: # can get data - new_dtypes.append(dtype) - # Return cleaned list - return new_dtypes - -############################################# -######## Interface with other modules ####### -############################################# - -# Have files with acceptable columns for each data type so you can check inputs -# Accept column inputs to the data function and take optional arguments in combine_strehl - -def data_func(dtype): - """ - Returns the function to get data for the specified dtype - and whether data needs to be matched to nirc2 mjds - """ - if dtype in ['cfht']+expand['seeing']: # MKWC - return lambda files,mjds: mkwc.from_nirc2(mjds,dtype), True - if dtype in expand['temp']: # Temperature - return lambda files,mjds: temp.from_mjds(mjds,dtype), True - if dtype=='telem': # Telemetry - return lambda files,mjds: telem.from_nirc2(mjds,files), False - - -################################ -###### Compiler Functions ###### -################################ - -def match_data(mjd1, mjd2): - """ - Matches data1 with its closest points in data2, by mjd values - Returns an array containing data1 with corresponding rows from data2 - """ - # Edge case - if mjd1.empty or mjd2.empty: - return None - - # Get mjds from dataframes - mjd1 = np.array(mjd1).reshape(-1,1) # column vector - mjd2 = np.array(mjd2).reshape(1,-1) # row vector - - # Take difference of mjd1 with mjd2 - diffs = np.abs(mjd1 - mjd2) - # Find smallest difference for each original mjd - idxs = np.argmin(diffs, axis=1) - - # Return indices of matches in mjd2 - return idxs - - -# data_types can contain: 'chft', 'mass', 'dimm', 'masspro', -# 'telem', 'k2AO', 'k2L4', or 'k2ENV' -# 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and -# ['mass', 'dimm', 'masspro'], respectively -def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=False, - params=default_params): - """ - Combines and matches data from a certain Strehl file with other specified data types. - NIRC2 files must be in the same directory as the Strehl file. - """ - # Check file paths parameter & open if yaml - if not isinstance(file_paths, dict) and os.path.isfile(file_paths): - with open(file_paths) as file: - file_paths = yaml.load(file, loader=yaml.FullLoader) - ### Add a catch for YAML not being installed if file specified - - # Read in Strehl file - nirc2_data = strehl.from_strehl(strehl_file) - - if test: # Take first few files - nirc2_data = nirc2_data.loc[:3] - - if check: # Sanitize user input - data_types = check_dtypes(data_types) - - # Full data container - all_data = [nirc2_data.reset_index(drop=True)] - - # Loop through and get data - for dtype in data_types: - get_data, match = data_func(dtype) # Data retrieval function - # Get other data from strehl info - other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) - - if match: # Needs to be matched - if other_data.empty: # No data found - other_data = pd.DataFrame(columns=other_data.columns, index=range(len(nirc2_data))) - else: # Get indices of matched data - idxs = match_data(nirc2_data.nirc2_mjd, other_data[dtype+'_mjd']) - other_data = other_data.iloc[idxs] - - # Add to all data - all_data.append(other_data.reset_index(drop=True)) - - # Concatenate new data with nirc2 - return pd.concat(all_data, axis=1) \ No newline at end of file diff --git a/katan/.ipynb_checkpoints/mkwc-checkpoint.py b/katan/.ipynb_checkpoints/mkwc-checkpoint.py deleted file mode 100644 index 65aa637..0000000 --- a/katan/.ipynb_checkpoints/mkwc-checkpoint.py +++ /dev/null @@ -1,184 +0,0 @@ -### mkwc_util.py : Contains utilities for extracting and processing data from the MKWC website -### Author : Emily Ramey -### Date : 6/1/2021 - -import os -import numpy as np -import pandas as pd -from . import times - -### NOTE: Seeing data is STORED by UT, with data in HST -### CFHT data is STORED by HST, with data in HST - -# cfht_dir = "/u/emily_ramey/work/Keck_Performance/data/weather_data/" -mkwc_url = 'http://mkwc.ifa.hawaii.edu/' -year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' -cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST - -# seeing_dir = "/u/emily_ramey/work/Keck_Performance/data/seeing_data" - -# Time columns from MKWC data -time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] - -# Data-specific fields -data_types = { - 'cfht': { - 'web_pat': mkwc_url+'archive/wx/cfht/indiv-days/cfht-wx.{}.dat', - 'file_pat': "{}cfht-wx.{}.dat", - 'data_cols': ['wind_speed', 'wind_direction', 'temperature', - 'relative_humidity', 'pressure'], - }, - 'mass': { - 'web_pat': mkwc_url+'current/seeing/mass/{}.mass.dat', - 'file_pat': '{}mass/{}.mass.dat', - 'data_cols': ['mass'], - }, - 'dimm': { - 'web_pat': mkwc_url+'current/seeing/dimm/{}.dimm.dat', - 'file_pat': '{}dimm/{}.dimm.dat', - 'data_cols': ['dimm'], - }, - 'masspro': { - 'web_pat': mkwc_url+'current/seeing/masspro/{}.masspro.dat', - 'file_pat': '{}masspro/{}.masspro.dat', - 'data_cols': ['masspro_half', 'masspro_1', 'masspro_2', - 'masspro_4', 'masspro_8', 'masspro_16', 'masspro'], - }, -} - -# Mix and match data & time columns -for dtype in data_types: - # CFHT files don't have seconds - tcols = time_cols if dtype != 'cfht' else time_cols[:-1] - # Format web columns - data_types[dtype]['web_cols'] = tcols+data_types[dtype]['data_cols'] - # Format file columns - data_types[dtype]['cols'] = [dtype+'_mjd']+data_types[dtype]['data_cols'] - # Different file storage timezones - data_types[dtype]['file_zone'] = 'hst' if dtype=='cfht' else 'utc' - -############################# -######### Functions ######### -############################# - -def cfht_from_year(datestrings, year): - """ - Gets pre-2012 MKWC data from the year-long file - instead of the by-date files - """ - # Get year-long file URL - url = year_url.format(year) - - # Read in data - try: - web_cols = data_types['cfht']['web_cols'] - year_data = pd.read_csv(url, delim_whitespace=True, header=None, - names=web_cols, usecols=range(len(web_cols))) - except: # No data, return blank - return pd.DataFrame(columns=data_types['cfht']['cols']) - - # Full dataset - all_data = [pd.DataFrame(columns=data_types['cfht']['cols'])] - - # Slice up dataframe - for ds in datestrings: - month, day = int(ds[4:6]), int(ds[6:]) - # Get data by month and day - df = year_data.loc[(year_data.month==month) & (year_data.day==day)].copy() - # Format columns - if not df.empty: - format_columns(df, 'cfht') - # Add to full dataset - all_data.append(df) - - return pd.concat(all_data) - -def format_columns(df, dtype): - """ Changes columns (in place) from time_cols to MJD """ - # Get MJDs from HST values - mjds = times.table_to_mjd(df, columns=time_cols, zone='hst') - df[dtype+'_mjd'] = mjds - - # Drop old times - df.drop(columns=time_cols, inplace=True, errors='ignore') - -def from_url(datestring, dtype): - """ Pulls cfht or seeing file from MKWC website """ - # Format URL - url = data_types[dtype]['web_pat'].format(datestring) - - # Read data - try: # Check if data is there - web_cols = data_types[dtype]['web_cols'] # MKWC Weather columns - df = pd.read_csv(url, delim_whitespace = True, header=None, - names=web_cols, usecols=range(len(web_cols))) - except: # otherwise return blank df - return pd.DataFrame(columns=data_types[dtype]['cols']) - - # Get mjd from time columns - format_columns(df, dtype) - - return df - -def from_file(filename, dtype, data_dir='./'): - """ Pulls cfht or seeing file from local directory """ - # Read in CSV - df = pd.read_csv(filename) - - # Check formatting - if 'mjd' in df.columns: - df.rename(columns={'mjd':dtype+'_mjd'}, inplace=True) - elif dtype+'_mjd' not in df.columns: # No MJD - try: # Change to MJD, if times - format_columns(df, dtype) - except: # No time info, return blank - return pd.DataFrame() - - return df - -def from_nirc2(mjds, dtype, data_dir='./'): - """ - Compiles a list of cfht or seeing observations based on MJDs - note: does not compare MJDs; assumption is inputs are rounded to nearest day - """ - - # Get datestrings - dts = times.mjd_to_dt(mjds, zone=data_types[dtype]['file_zone']) - datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 - # No duplicates - datestrings = pd.Series(np.unique(datestrings)) - - # Blank data structure - all_data = [pd.DataFrame(columns=data_types[dtype]['cols'])] - - # Check for pre-2012 cfht files - if dtype=='cfht' and any(mjds < cfht_cutoff): - # Get datetimes - pre_2012 = datestrings[mjds < cfht_cutoff] - - # Compile data by year - for yr in np.unique(pre_2012.str[:4]): - ds = pre_2012[pre_2012.str[:4]==yr] - df = cfht_from_year(ds, yr) - # Append to full dataset - all_data.append(df) - - # Find data for each file - for ds in datestrings: - # Get local filename - filename = data_types[dtype]['file_pat'].format(data_dir, ds) - - # Check for local files - if os.path.isfile(filename): - df = from_file(filename, dtype) - - else: # Pull from the web - df = from_url(ds, dtype) - - # Save to local file: TODO - - # Add to data - all_data.append(df) - - # Return concatenated dataframe - return pd.concat(all_data, ignore_index=True) \ No newline at end of file diff --git a/katan/.ipynb_checkpoints/strehl-checkpoint.py b/katan/.ipynb_checkpoints/strehl-checkpoint.py deleted file mode 100644 index 9846ac3..0000000 --- a/katan/.ipynb_checkpoints/strehl-checkpoint.py +++ /dev/null @@ -1,60 +0,0 @@ -### nirc2_util.py : Functions for processing nirc2 data and strehl files -### Author : Emily Ramey -### Date : 6/2/2021 - -import os -import numpy as np -import pandas as pd -import glob -from astropy.io import fits - -from . import times - -strehl_cols = ['nirc2_file', 'strehl', 'rms_err', 'fwhm', 'nirc2_mjd'] -default_keys = ['AIRMASS', 'ITIME', 'COADDS', 'FWINAME', 'AZ', 'DMGAIN', 'DTGAIN', - 'AOLBFWHM', 'WSFRRT', 'LSAMPPWR', 'LGRMSWF', 'AOAOAMED', 'TUBETEMP'] - -def from_filename(nirc2_file, data, i, header_kws): - """ - Gets nirc2 header values from a filename as dict - or loads values into specified df - """ - # Check for valid file - if not os.path.isfile(nirc2_file): - return - # Open nirc2 file - with fits.open(nirc2_file) as file: - nirc2_hdr = file[0].header - - # Get fields from header - for kw in header_kws: - # load DataFrame value - data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) - -def from_strehl(strehl_file, header_kws=default_keys): - """ Gets NIRC2 header data based on contents of Strehl file """ - # Get directory name - data_dir = os.path.dirname(strehl_file) - # Retrieve Strehl data - strehl_data = pd.read_csv(strehl_file, delim_whitespace = True, - header = None, skiprows = 1, names=strehl_cols) - # Add true file path - strehl_data['nirc2_file'] = data_dir + "/" + strehl_data['nirc2_file'] - - # Add decimal year - strehl_data['dec_year'] = times.mjd_to_yr(strehl_data.nirc2_mjd) - - # Add nirc2 columns - for col in header_kws: - strehl_data[col.lower()] = np.nan - - # Loop through nirc2 files - for i,nirc2_file in enumerate(strehl_data.nirc2_file): - # Load header data into df - from_filename(nirc2_file, strehl_data, i, header_kws) - - # Return data - return strehl_data - - - diff --git a/katan/.ipynb_checkpoints/telemetry-checkpoint.py b/katan/.ipynb_checkpoints/telemetry-checkpoint.py deleted file mode 100644 index 3757ee7..0000000 --- a/katan/.ipynb_checkpoints/telemetry-checkpoint.py +++ /dev/null @@ -1,292 +0,0 @@ -### telem_util.py: For code that interacts with Keck AO Telemetry files -### Author: Emily Ramey -### Date: 11/18/2020 - -import numpy as np -import pandas as pd -import glob -from scipy.io import readsav - -from . import times -from . import templates - -### For importing package files -### May need to edit this later if there are import issues with Py<37 -try: - import importlib.resources as pkg_resources -except ImportError: - # Try backported to PY<37 `importlib_resources`. - import importlib_resources as pkg_resources - -### Path to telemetry files [EDIT THIS] -telem_dir = '/g/lu/data/keck_telemetry/' - -# Sub-aperture maps -wfs_file = "sub_ap_map.txt" -act_file = "act.txt" - -filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers -filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" - -TIME_DELTA = 0.001 # About 100 seconds in mjd - -RESID_CUTOFF = 349 -TT_IDXS = [349,350] -DEFOCUS = 351 -LAMBDA = 2.1 # microns - -cols = ['telem_file', - 'telem_mjd', - 'TT_mean', - 'TT_std', - 'DM_mean', - 'DM_std', - 'rmswfe_mean', - 'rmswfe_std', - 'strehl_telem'] - -def read_map(filename): - """ Reads a map of actuators or sub-apertures from a file """ - file = pkg_resources.open_text(templates, filename) - return pd.read_csv(file, delim_whitespace=True, header=None).to_numpy() - -def get_times(telem_data, start=None): - """ Pulls timestamps from a telemetry file in seconds from start """ - if start is None: - start = telem_data.a.timestamp[0][0] - - return (telem_data.a.timestamp[0]-start)/1e7 - -def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): - """ - Return the locations of the valid actuators in the actuator array - resids: Nx349 residual wavefront array (microns) - ints: Nx304 intensity array (any units) - N: Number of timestamps - """ - # Check inputs - N = ints.shape[0] # Num timestamps - - # Aggregate intensities over all timestamps - med_ints = np.median(ints, axis=0) - - # Fill WFS map with aggregated intensities - int_map = wfs_map.copy() - int_map[np.where(int_map==1)] = med_ints - - # Find lenslets with greatest intensity - idxs = np.flip(np.argsort(int_map, axis=None))[:num_aps] # flat idxs of sort - idxs = np.unravel_index(idxs, wfs_map.shape) # 2D idxs of sort - - # Mask for good sub-ap values - good_aps = np.zeros(wfs_map.shape, dtype=int) - good_aps[idxs] = 1 - good_aps = good_aps * wfs_map # Just in case - - # Mask for good actuator values - good_acts = np.pad(good_aps, ((1,1),(1,1))) - good_acts = (good_acts[1:,1:] | good_acts[1:,:-1] - | good_acts[:-1,:-1] | good_acts[:-1,1:]) * act_map - - return good_acts - -# def wfs_error2(resids, ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file)): -# """ Calculates the variance in the wavefront (microns^2) produced by the WFS """ -# # Get residual mask for brightest sub-aps -# rmask = resid_mask(resids, ints) - -# # Square residuals -# sig2_resid = np.mean(resids**2, axis=0) -# # Load into actuator grid -# resid_grid = act_map.astype(float) -# resid_grid[np.where(resid_grid==1)] = sig2_resid - -# # Mask out actuators next to dimmer apertures -# sig2_masked = resid_grid * rmask -# # Sum values and return -# return np.sum(sig2_masked) - -# def tt_error2(tt_resids): ### TODO: fix this so it's not unitless -# """ Calculate mean tip-tilt residual variance in microns """ -# wvln = 0.658 # wavelength (microns) -# D = 10.5 * 1e6 # telescope diameter (microns) - -# # Magnitude of residuals -# sig_alpha2 = (tt_resids[:,0] + tt_resids[:,1])**2 # x + y TT residual variance -# sig_w2 = (np.pi*D/wvln/2.0)**2 * sig_alpha2 # TT resids in microns^2 - -# return np.mean(sig_w2) - -# def total_WFE(telem_data): -# resids = telem_data.a.residualwavefront[0][:, :RESID_CUTOFF] * 0.6 # WFS resids, microns, Nx349 -# ints = telem_data.a.subapintensity[0] # Sub-ap intensities, Nx304 -# tt_resids = telem_data.a.residualwavefront[0][:, TT_IDXS] * np.pi / (180*3600) # TT resids, radians, Nx2 -# defocus = [0] #telem_data.a.residualwavefront[0][:, DEFOCUS] # Defocus resids, microns, Nx1 - -# return wfs_error2(resids, ints) + tt_error2(tt_resids) + np.mean(defocus**2) - -# def mask_residuals_old(telem_data, num_aps=236, wfs_file=wfs_file, act_file=act_file): -# """ Really freakin complicated array logic to mask out all invalid actuators """ -# # Get data -# resids = telem_data.a.residualwavefront[0][:, :349]*0.6 # microns -# intensities = telem_data.a.subapintensity[0] # ADU -# N = intensities.shape[0] -# # Get hardware maps -# wfs_map = read_map(wfs_file) -# act_map = read_map(act_file) - -# # Get X and Y values of sub-aps and replicate them for all timestamps -# wfs_x, wfs_y = np.where(wfs_map==1) -# wfs_x, wfs_y = np.tile(wfs_x, (N,1)), np.tile(wfs_y, (N,1)) - -# # Get valid indices for each timestep -# idxs = np.flip(np.argsort(intensities, axis=1), axis=1)[:,:num_aps] -# valid_x = np.take_along_axis(wfs_x, idxs, axis=1) -# valid_y = np.take_along_axis(wfs_y, idxs, axis=1) -# valid_z = np.tile(np.arange(N), (num_aps,1)).T - -# # Put 1s at each valid index -# valid_saps = np.zeros((N, 20, 20), int) -# valid_saps[valid_z, valid_x, valid_y] = 1 # TODO: flip this back -# # Pad each sheet (timestamp) with zeros at the edges -# check = valid_saps.reshape(N, 20*20).sum(axis=1) -# if any(check!=236): -# print("Shape mismatch in valid sub-ap array") -# valid_saps = np.pad(valid_saps, ((0,0),(1,1),(1,1))) - -# # Get (potentially)valid actuators for sub-aps -# valid_acts = (valid_saps[:,1:,1:]|valid_saps[:,1:,:-1]| -# valid_saps[:,:-1,:-1]|valid_saps[:,:-1,1:]) # 4 corners of sub-aps -# # Multiply by actuator map to remove any non-actuator positions -# valid_acts = valid_acts * np.tile(act_map, (N,1,1)) - -# # Get number of valid actuators in each frame (can vary due to edge cases) -# valid_act_nums = valid_acts.reshape(N,21*21).sum(axis=1) - -# # Map residuals to actuator positions -# resid_vals = np.tile(act_map, (N,1,1)).astype(float) -# resid_vals[np.where(resid_vals==1)] = resids.flatten() - -# # Mask out invalid actuators -# valid_acts = valid_acts * resid_vals -# rms_resids = valid_acts.reshape(N, 21*21) - -# # Take the RMS residual for each frame -# rms_resids = np.sqrt((rms_resids**2).sum(axis=1)/valid_act_nums) - -# return rms_resids - -def tt2um(tt_as): - """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ - D = 10.5e6 # telescope diameter in microns - tt = tt_as*4.8e-6 # TT error in radians - tt_err = np.sqrt(D**2 / 12 * (tt[:,0]**2 + tt[:,1]**2)) - return tt_err - -def rms_acts(act_resids, ints): - """ Clips bad actuators and averages for each timestamp """ - N = act_resids.shape[0] # num timestamps - - # Read in actuator map - act_map = read_map(act_file) - - # Get good actuator mask from intensities - mask = resid_mask(ints) - flat_map = ~mask[np.where(act_map==1)].astype(bool) # flatten - flat_map = np.tile(flat_map, (N,1)) - - # Mask the bad actuators - good_acts = np.ma.masked_array(act_resids, flat_map) - # Average resids for each timestep - act_rms = np.sqrt((good_acts**2).mean(axis=1) - good_acts.mean(axis=1)**2) - - return act_rms.compressed() - -######################################## -######### Processing Files ############# -######################################## - -# Do some telemetry files need to be cropped around the observation? - -def get_mjd(telem): - """ Validates a telemetry file against an MJD value """ - # Get timestamp - tstamp = telem.tstamp_str_start.decode('utf-8') - # Convert to MJD - telem_mjd = times.str_to_mjd(tstamp, fmt='isot') - - # Returns telem mjd - return telem_mjd - -def extract_telem(file, data, idx, check_mjd=None): - """ Extracts telemetry values from a file to a dataframe """ - # Read IDL file - telem = readsav(file) - - # Make sure MJD matches - telem_mjd = get_mjd(telem) - if check_mjd is not None: - delta = np.abs(telem_mjd-check_mjd) - if delta > TIME_DELTA: - return False - - # Get residuals and intensities - act_resids = telem.a.residualwavefront[0][:, :RESID_CUTOFF] - tt_resids = telem.a.residualwavefront[0][:,TT_IDXS] - ints = telem.a.subapintensity[0] # Sub-aperture intensities - - # Convert TT resids to microns - tt_microns = tt2um(tt_resids) - # Get RMS resids from the actuator array - act_rms = rms_acts(act_resids, ints) - - # Total RMS Wavefront Error - rmswfe = np.sqrt(tt_microns**2 + act_rms**2) - - # Strehl calculation - strehl = np.exp(-(2*np.pi*rmswfe/LAMBDA)**2) - - # Assemble aggregate data - data.loc[idx, cols] = [ - file, - telem_mjd, - np.mean(tt_microns), - np.std(tt_microns), - np.mean(act_rms), - np.std(act_rms), - np.mean(rmswfe), - np.std(rmswfe), - np.mean(strehl), - ] - - return True - -def from_nirc2(mjds, filenames): - """ Gets a table of telemetry information from a set of mjds and file numbers """ - N = len(mjds) # number of data points - # Get file numbers - filenums = filenames.str.extract(filenum_match, expand=False) - filenums = filenums.str[1:] # First digit doesn't always match - - # Get datestrings - dts = times.mjd_to_dt(mjds) # HST or UTC??? - datestrings = dts.strftime("%Y%m%d") # e.g. 20170826 - - # Set up dataframe - data = pd.DataFrame(columns=cols, index=range(N)) - - # Find telemetry for each file - for i in range(N): - # Get filename and number - fn, ds, mjd = filenums[i], datestrings[i], mjds[i] - # Search for correct file - file_pat = filename_match.format(ds, fn) - all_files = glob.glob(file_pat, recursive=True) - - # Grab the first file that matches the MJD - for file in all_files: - success = extract_telem(file, data, i, check_mjd=mjd) - if success: break - - return data - \ No newline at end of file diff --git a/katan/.ipynb_checkpoints/temperature-checkpoint.py b/katan/.ipynb_checkpoints/temperature-checkpoint.py deleted file mode 100644 index 9c672e0..0000000 --- a/katan/.ipynb_checkpoints/temperature-checkpoint.py +++ /dev/null @@ -1,277 +0,0 @@ -### temp_util.py: Functions relating to Keck-II temperature data -### Main reads in Keck II temperature files (AO, env, and L4) and saves each to a FITS file -### Author: Emily Ramey -### Date: 01/04/2021 - -import os -import numpy as np -import pandas as pd -import glob -from astropy.io import fits -from astropy.table import Table - -from . import times - -verbose = True - -data_dir = "/u/emily_ramey/work/Keck_Performance/data/" -temp_dir = data_dir+"temp_data_2/" - -col_dict = { - 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV - 'AO_benchT2': 'k1:ao:env:benchtemp2_Raw', - 'k1:ao:env:elect_vault_t': 'k1:ao:env:elect_vault:T', - 'k0:met:humidityStats.VAL': 'k0:met:humidityStats', - "k2:ao:env:ETroomtemp_Raw": "El_roomT", # k2AO - "k2:ao:env:DMracktemp_Raw": "DM_rackT", - "k2:ao:env:KCAMtemp2_Raw": "KCAM_T2", - "k2:ao:env:OBrighttemp_Raw": "OB_rightT", - "k2:ao:env:phototemp_Raw": "photometricT", - "k2:ao:env:OBlefttemp_Raw": "OB_leftT", - "k2:ao:env:KCAMtemp1_Raw": "KCAM_T1", - "k2:ao:env:ACAMtemp_Raw": "AOA_camT", - "k2:ao:env:LStemp_Raw": "LGS_temp", # k2L4 - "k2:ao:env:LSenchumdity_Raw": "LGS_enclosure_hum", - "k2:ao:env:LShumidity_Raw": "LGS_humidity", - "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" -} - -file_pats = { # File name patterns -# 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", -# 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", -# 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", - 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", - 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", - 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", -} - -data_types = { - 'k2AO': { - 'file_pat': temp_dir+"k2AOtemps/{}/AO_temps.log", - 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', - 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', - 'k2AO_mjd'], - }, - 'k2L4': { - 'file_pat': temp_dir+"k2L4temps/{}/L4_env.log", - 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', - 'LGS_temp', 'k2L4_mjd'], - }, - 'k2ENV': { - 'file_pat': temp_dir+"k2envMet/{}/envMet.arT", - 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', - 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', - 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', - 'k0:met:out:windSpeedMaxStats', 'k0:met:out:windSpeedMaxmph', - 'k0:met:out:windSpeedmph', 'k0:met:outTempDwptDiff', - 'k0:met:pressureRaw', 'k0:met:pressureStats', 'k0:met:tempMax', - 'k0:met:tempMin', 'k0:met:tempRaw', 'k2:dcs:sec:acsDwptDiff', - 'k2:dcs:sec:acsTemp', 'k2:dcs:sec:secDwptDiff', 'k2:dcs:sec:secondaryTemp', - 'k2:met:humidityRaw','k2:met:humidityStats', 'k2:met:tempRaw', 'k2:met:tempStats', - 'k2:met:windAzRaw', 'k2:met:windElRaw', 'k2:met:windSpeedMaxmph', - 'k2:met:windSpeedMinmph', 'k2:met:windSpeedRaw', 'k2:met:windSpeedStats', - 'k2:met:windSpeedmph', 'k2ENV_mjd'], - } -} - -def get_columns(): - all_data = search_files() - for name, data_files in all_data.items(): - columns = set() - for i,file in enumerate(data_files): - try: - df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, - na_values=['***'], error_bad_lines=False, warn_bad_lines=False, - ).replace('"', regex=True) - except: - if verbose: - print(f"Warning: read failed for file {file}") - continue - - if len(df.columns)==1: # no header - if verbose: - print(f"Skipping file {file}, no header") - continue # skip for now - df.columns = [col.replace('"', '').strip() for col in df.columns] - if "UNIXDate" not in df.columns: - if verbose: - print(f"Skipping file {file}, columns not as expected") - continue # skip for now - for col in df.columns: - columns.add(col) - all_data[name] = columns - - return all_data - - -def search_files(file_pats=file_pats): - """ Finds all filenames matching the given file patterns """ - all_filenames = {} - for name, search in file_pats.items(): - filenames = glob.glob(search, recursive=True) - all_filenames[name] = filenames - if verbose: - print(f"Done {name}, length: {len(filenames)}") - - return all_filenames - -def collect_data(data_files, col_dict=col_dict): - """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ - if isinstance(data_files, dict): # Return dict w/dataframes - new_files = {} - for name, files in data_files.items(): - if isinstance(files, list): - # Recurse on each list of files - new_files[name] = collect_data(files) - return new_files - - all_dfs = [pd.DataFrame()] - for i,file in enumerate(data_files): - if not os.path.isfile(file): - continue - try: - df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, - na_values=['***'], error_bad_lines=False, warn_bad_lines=False, - ).replace('"', regex=True) - except: - if verbose: - print(f"Warning: read failed for file {file}") - continue - - if len(df.columns)==1: # no header - if verbose: - print(f"Skipping file {file}, no header") - continue # skip for now - df.columns = [col.replace('"', '').strip() for col in df.columns] - if "UNIXDate" not in df.columns: - if verbose: - print(f"Skipping file {file}, columns not as expected") - continue # skip for now - - if col_dict is not None: - df = df.rename(columns=col_dict) - - all_dfs.append(df) - - data = pd.concat(all_dfs, ignore_index=True, sort=True) - return data - -def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): - """ Parses specified date and time columns and returns a cleaned data table """ - new_data = data.copy() - for label,cols in date_cols.items(): - date_col, time_col = cols - # Parse dates and times, coercing invalid strings to NaN - datetimes = (pd.to_datetime(data[date_col], exact=False, errors='coerce') + - pd.to_timedelta(data[time_col], errors='coerce')) - new_data = new_data.drop(columns=cols) - new_data[label] = datetimes - - return new_data - -def clean_dates(data, date_cols=['HST', 'UNIX']): - """ Removes any rows containing invalid dates in date_cols """ - new_data = data.copy() - for col in date_cols: - new_data = new_data[~np.isnan(new_data[col])] - - return new_data - -def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): - """ Casts columns to a numeric data type """ - if data_cols is None: - data_cols = [col for col in data.columns if col not in non_numeric] - - # Cast data to numeric type, coercing invalid values to NaN - new_data = data.copy() - for col in data_cols: - new_data[col] = pd.to_numeric(new_data[col], errors='coerce') - - return new_data - -def to_fits(data, filename, str_cols=['HST', 'UNIX']): - """ Writes a FITS file from the given temperature array """ - fits_data = data.copy() - for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: - if col in fits_data.columns: - fits_data = fits_data.drop(columns=[col]) - for col in str_cols: - if col in fits_data.columns: - fits_data[col] = fits_data[col].astype(str) - # Assuming the data columns are already numeric - fits_data = Table.from_pandas(fits_data) - fits_data.write(filename) - - return - -def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): - """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ - data = Table.read(filename).to_pandas() - - # Fix NaNs, because astropy is dumb sometimes - data[data==1e+20] = np.nan - - if date_cols is None: return data - - for col in date_cols: - if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings - data[col] = data[col].str.decode("utf-8") - data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') - - if str_cols is None: return data - - for col in str_cols: - if col in data.columns and isinstance(data[col][0], bytes): - data[col] = data[col].str.decode("utf-8") - - return data - -def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): - """ - Reads in all data matching file patterns (file_pats), combines them into one table, - cleans them, and saves them to a FITS file - """ - # Find all files matching pattern - all_filenames = search_files(file_pats) - # Read all data into one table (per dictionary label) - all_data = collect_data(all_filenames) - - for name, data in all_data.items(): - data = parse_dates(data) # Parse date cols into datetimes - data = clean_dates(data) # Remove invalid dates - data = clean_data(data) # Casts other cols to numeric - # Save combined/cleaned data to FITS file - filename = location+name+".fits" - to_fits(data, filename) - - return - -def from_mjds(mjds, dtype): - """ Gets temp data of input type from the specified MJDs """ - # Get pd datetimes in HST - dts = times.mjd_to_dt(mjds, zone='hst') - # Format list - datestrings = dts.strftime("%y/%m/%d") - datestrings = np.unique(datestrings) # one file per date - # Get relevant filenames - filenames = [data_types[dtype]['file_pat'].format(ds) for ds in datestrings] - # Get data from filenames - data = collect_data(filenames) - - # Empty dataframe - if data.empty: - return pd.DataFrame(columns=data_types[dtype]['cols']) - - # Convert dates & times to MJDs - data["datetime"] = data['HSTdate']+' '+data['HSTtime'] - mjds = times.table_to_mjd(data, columns='datetime', zone='hst') - # Add to dataframe - data[dtype+"_mjd"] = mjds - # Drop other date & time cols - data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], - inplace=True, errors='ignore') - - return data - -if __name__=='__main__': - pass \ No newline at end of file diff --git a/katan/.ipynb_checkpoints/times-checkpoint.py b/katan/.ipynb_checkpoints/times-checkpoint.py deleted file mode 100644 index 4974d2c..0000000 --- a/katan/.ipynb_checkpoints/times-checkpoint.py +++ /dev/null @@ -1,56 +0,0 @@ -### time_util.py : to handle time formatting changes between MJD, HST, and UTC -### Author : Emily Ramey -### Date: 5/26/21 - -# Preamble -import pandas as pd -import numpy as np -import time -import pytz as tz -from datetime import datetime, timezone -from astropy.time import Time, TimezoneInfo -from astropy import units as u, constants as c - -hst = tz.timezone('US/Hawaii') - -### Conversion / utility functions -def mjd_to_dt(mjds, zone='utc'): - """ Converts Modified Julian Date(s) to HST or UTC date(s) """ - # Convert mjds -> astropy times -> datetimes - dts = Time(mjds, format='mjd', scale='utc').to_datetime() - # Convert to pandas - dts = pd.to_datetime(dts).tz_localize(tz.utc) - if zone=='hst': - dts = dts.tz_convert("US/Hawaii") - return dts - -def table_to_mjd(table, columns, zone='utc'): - """ Converts date and time columns to mjds """ - # Safety check for list of columns - if not isinstance(columns, str): - columns = [col for col in columns if col in table.columns] - # Convert to datetimes - dts = pd.to_datetime(table[columns], errors='coerce') - - if zone=='hst':# Convert to UTC - dts = dts.dt.tz_localize(hst) - dts = dts.dt.tz_convert(tz.utc) - - # Masked invalid values - dts = np.ma.masked_array(dts, mask=dts.isnull()) - - # Convert to astropy - times = Time(dts, format='datetime', scale='utc') - # return MJDs - return np.ma.getdata(times.mjd) - -def str_to_mjd(datestrings, fmt): - """ Converts astropy-formatted date/time strings (in UTC) to MJD values """ - # Get astropy times - times = Time(datestrings, format=fmt, scale='utc') - # Convert to mjd - return times.mjd - -def mjd_to_yr(mjds): - """ Converts MJD to Decimal Year """ - return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file diff --git a/katan/__pycache__/__init__.cpython-37.pyc b/katan/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index f28df4013162b34e912dc6e15cb29b97195a24d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmZ?b<>g`k0*U5B35-DcF^B^LAOQy;E@lA|DGb33nv8xc8Hzx{2;!H4eyM(HZe~tp zd{JU&ryk0@&Ee@O9{FKt1R6CGK IpMjVG0CNN%EdT%j diff --git a/katan/__pycache__/compiler.cpython-37.pyc b/katan/__pycache__/compiler.cpython-37.pyc deleted file mode 100644 index 04bb419b7d1f02198940e59a9b5d221101dedbd3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3454 zcmb7GOLN@D5yk-6T`cw?isGA+1?S*ot6Zeba$G5uOJ!NIQc7e-krc^>+S=!q*&d%}M9nd`Z3@kQB>41t`q)^fbC>x_iE!PkKGq zz|%Fq{oPl~hVgG2oIehTkMR|cm|<{(v)D+O&lu4rXwx?}ZGpCYOVc)J+iz&P0lMk8 zG~ER4_zvh6cjC71>b~Q5b>H)QvzqGr{pUuy#0+}H72ZC9y)TT^{M?u=XxWA2s=vq# zNL{xLaqYmB|EQ()?2^A&OLLcZc$fEhpI_k%{OZ{8m-(W8fzV$}7`f6aC4$ zpmu{_dujTc(7IM@iwu7g_Ltt+uQiRWS}%Hc%%GQvWxo8^hJT%}@YNTFe}k{_b@XrX z4Zex~Ccn;a@atl8Wb?Q9%@@q~xbwsqc(37AW5BA0iV{(PXckI}{TP&;92|z|+e*YD z5lYT5=qQooLI!FoMLn0~aiB!;8hVV~pRk#5u~qmCB1-+uK1JgjvditxS$D2HW{;79_EkNiPY~Db&ooA{Svant7ur7G55x{i3qN z(Z2HSBq$2s=24RPEuwRo`S!uionKM=@B_bbu=DBeGi;J}a=d&XW`~*NLoPgE5tPF{Zq;A7ESt@Q9Gb#n#Rj7rdFQ7kaoa zQZLMsJc^@qyftW4Hu(^a76J^vbtteEA!P#&tSsy_q?piS;Un%{e8tCT#>Si-eOMaz zjIXxlW@#$ppef%e*(p;-X|5Zukf*NNj5U z!*LvrTO~*S=ugIoaf{n89Nrk2PauKW<~q&Y)^n}PbKE&F{c*`L=d|`u`@$K&fS5LHG!~JUU7efq2Dle%B=6Iwk2x7BjS2oK z*~E|w9LH9%Qxhx3apoqsUYYZn!!!pxa2RCl5{G;Za&uF$Nuz`;w_lobtF#d4#_?b3 z%=%0xPocBweeW<*`(C6906?ae5?ysv|JD(WOV&ukz_AZ0KE zyrn9LQoz||tzjmmC~{ysO~~x<3`)p1J33o;U;h`6JgY%=a1<4TX4L>Zs##^GdDYPR zs}}U^2e|+|?h(4kaT28i8^Ax|@9TC4h!TxPNZ5!U72k_q<>~`7B*S0Azcp4gyUey0 z*`n1q`jwVoL2|9^!G5=NPS-Iyv#U$R*Ut@T+bA9G@b*iB2>1AJb>`g7 zlD#NJMIdmhF{nXQ6XC(*yE|IG@RBLA z5FiIV8XZv)JRV6As!Yzj02m0&ykw!)K;@fiw2-^63F4oEVRM<~z_M1D#nU7$sz!ur zcjVhcgRwR(6J}R@ro0sBAJ;x%a1W zT;CcbS@k;_vg$ijWdR3u5%N1oV4;TEygvLWh6LP0((AWtnil0jNoD4|YTtv4?o&CG z?-H*egLI4o5Q|jjs&7Z6eUavxIDcWF27SL%Th?kN0X|S27KZh0siBJ|-9HLdHT5M1 zThvQ9TMnHN-@zS3pB0Vzm4kw%hgCzz+;?>}Yl$Dj#`jNKABCFoWLPg|3X$R+G)8NM zxopFBO&itOW{%ajT-G&-Zn6z?iMiH_CGSCU@OEWLDqPyDx<2Xqkp_23hg4S-*#cEH z(rJ><Z<2F+Ac~fh%An@<`!T2QIk)lt z;$q0~{QWO~`&_Ru_75sNdTPjA!<*l*7~_mv1J){2t7V(A({j1Zoq;#>TfW7(+p35P z_f}Xd;C`#hD?H#;9`XfV+w@x@+7?$Buk#~(i67<5{Me?)kALl;{sihz@|8_%UOt8L zX@2IQd=};B`1AZ6KhGon0)LUev{|9P{N=CR*20bw3pdzjgH}z{`0AD;zZGE`ijZ6U zmEYT^S!+=&iUmc)b^I$4Rm-__#Se7t2mBxhI<2JIzkFr;J(K-dVpt zTj#ObJb&lYPyc=EU)Qud5wXJ2Rzmx^=wvCMHlCa?NqKT_ z4FmEg=AJIypV5+!S8Ija=-tjoLhx4I4v+r9BH z;fb2)U?fGJqp{)W%C@-M9>q!r-DDt4u8OaT+nsEXPtSM8Qi@bHb0Ly+vpJ0Oyov9n zls4rwT*kA^;3r(LRa-*}w8+zJo+zNIVke8bf)p)ZfXS>zI#qhyw09{vZ9tRtrbLXuF^ zZhP407(;odmn-dzRYzA1z|$-h9UJ}WA;t%Iuj9=%jADCMVeMNpdvZb9+yYC$nVF-U zzFUCb+$|jCDZg+YSPv}j?Rqo!4olBijDF}%4YG7IBE-r;Nt!DW^C;_@E_WxOi4vwmUiL7@A&&#Z+qZCHOI!?+VSBrmD3Ooi z4rtfI1OmiNV{lzp#zezGE>B_zB%p%Ac$nr7U6ddCRC!bfw3qf+4z%CRW?M{AAdooTpg?0ua)zusLqMH&6X(vhH0E4U}JaNP zunSgL+*x5)S;6`aT3qgNf7d}6tH5!B=|*(DN7&}k9HZ#9B#lM`oN8n4Hjr<_y+~!z z$3MFfNrX_=uqAC*yF@y90@>TzPd3wxl&@^6B&nH4`$Ec0=5QTJTh=ud--RBk%y79% z@nKGGF0&Dazd^k`PM!e8>jQB@xHNB3ZVl#hyg3~?bC#@{W9kl|Ay^LB-$OwG2=Ep! zRDjJ@psXuZ!0tN-D&Q*bf2e@n_vbS`R)I?479~Gj1L<1yi}7FpKykuRCZkf{OTrl< z8eT%ta_}dTbOAZwmOMkHvt*tl^E{c7MF!MHAYY=Y7sp=arZvc~kP$X`$D3L0Eu3FWQNm6z>#`|J}| zI6&8%Ui^RPLJs0VR7UJfP9p8VgGvvbJFUEeZ8rQv_!(FB5skrIgPy?=QfDH!sU-k) zKoZc#+|LBCY5*i*y-#422!33QO>Y<=6A^ON~;H4&ynCI`@ zG;{d<1-iXbMB29yv({&o!Y>ZhQd~m+K-G$%s64O{lf1pfc?>GRy{?KgSBBHtd0KLVU7P-~ z3_sVi;RvASQ5+4D9GvPt$*t^eF62F2#j-Sy?#8)5G)qaWauXKQOo=NI&oBkDP`cqL zmcq2Z6X7~DMzFxpD9Iye))+CAnq@XlIc9?FrI2Va3Bhz?CX}afZVgLbMl$tRf3rGV z<%el%0_H7w9?fzQ<~D+0tyIT0?v`SwaiWyMDtwcKkSoJ$9Uv5SNf3>5YX1`<6N-lE zNc-b7`C=@LXKE)-Cwjrekj_|T-L7^p(u4=OPOFkYXgkA&2^Y{Fq&X+QG|p!@ESu=i zSTKR76o;EM!r>~H6iVa2nyf?P{b?dQ8-GA^PCHnsWv*4N=Sq?c+`jk{x<=NB#=EfaLp!l4J6hDB&`JbD7AGDDRKrl*f6A z3NzTX0{SaKdL>^#7x{BCKPF=olK9&4l(~jb)GTRD@>POB>mK?9^Dz`#bZ~W;@$o>s uOT5hAg$XSB)qEG$^_DyzWJWx8C=Cu-#A`3KPvLjYKIaEfPz{#DrT+rp>%EQu diff --git a/katan/__pycache__/strehl.cpython-37.pyc b/katan/__pycache__/strehl.cpython-37.pyc deleted file mode 100644 index 5088ee2cb5c9226a3ee0795981514691fe73728a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1457 zcmYjQOK&4Z5bo}I`jt4@u-PmJ280B@#L^y+kRpWh#1MOfZDiB}%*5S}$DW6`XS|75 zK+30sH?hqs=uzPull%FDmv)*7J`JfOqIATCpyuRu8p%@O63rNOTJ2t^)$dcQJ!p5% zsD1Lhb$Z&T&iSz2ZNUa-pvJ`S7a_wTJ!lUT;9%%YLhKbv5i! z=klW0=?~8-IKkIBJEiX6yW_LLS?6kOs=#4DZBYs43C#2-5E+``4aw0xPU|Yi%-ER9 z`mMe;bCZz+G_%&$5UTzy>Y?w^-KV*6ihlIgc5ZF#+|HbtdrxvaE9E4&H|2XYAxG%N zXX*r0%y@O}l|2|2JWD;Dy*K4P| zy&8%&H8pF$z~VpF)+94#z!<}fFe9^=`Kz@ya>A^AlpD-GK!rX+nZ0px1K7f_?;N4K zr#S{Dch_cCnw4`SH#e1g1l)Z+M0X!&Rq(C7RvwRkAZu%FXIr@qURI7@n;5+ahky?Z zyRSYk$nCa!C(jl8M*X4@2|q4Z6aBY#OGlGad%f=yn=p zJYD!>E>uONo6duUNTB|fDpmRpH#JApWqdqZabGmHg$tfSHMMkDAe5gZ3npCweKs1W zFJ+}D{b-zosZcLLmf!`BvShRpLe*7lDY31DVnddAyo|W;Gp=vbWHg+AadcLPnLixJvenx>3fv#va)<#3!)d z*wP6?0hZBXr6ZO0!jGAsiVxtwG;hM>nwEV4OctxdASs$2=}g6k;6kg6PyA(=jplrn n>TERh{{mBeEOd14QIxP{$iGmb>(E?Wd{t5G3VZj{s9G=0!7=?00j#4WT1x@NH0av_V;EbT2LwM?97`tvpeto z-tWED^?E%p@cZq5zWw>%pD~QT(%|q{Mdx)~!Z!_r8O$6Qk*V)iWb3;VIk;N`H}fLT zG?>ktK_&AeEI{8yzZz9dW2dRLgKQ?MnZ}1ke%>*X`c}X zR*9O-k4~^^G?&b>;It9Vvzh25t3{8o8msrL=oD*2r`c?@z?$p?n`86rBzugVVyD?c zuNs|UXV}>ZFYe@0r`0rvL~VWID3kH5BCe~ z`|Kj_Pp~WOY4!{>LUxI@AbFCRcZ}Av56M1jEmKu^5;GCseF*ay7`rmf5`j)tCfR5Z z%S7y+UFt0j#!D=g@zPe(*=oyVkYtJEJ4?F_G2Up$qjnaL7Uj0wt;8LvJN3oOolBpv zE3L(DD#h;n7n7ySmzVO_zL;Eo`;9x77xB#AzrXp%A8v2F`&#-bo}PY2O&d+`ci+;_ zcs&aAliJ;MkkAX^sqL)KqRRSuJ4@m`^3V|yf|c6`0(3|)!!vQOo0P)fU_cW!A$hVn zD38#gL(|W~<>I=5ON3~ov1dxFF!rs9Deb;fn9O7rvme-d?!EyFI=ep#-y_q7afq{r z!)}O^k7GIHA{_D%W_d0iC0veqBEl{oW>BPagzIZ9OEp{)ckMq55#D2%?*hqcZis?Hb_^U=647Z6kDO8dPUzB?WaM5qBL<7_lF z)AUtJgQa;WlFl%Pt+gG6jJXWgI61F}%lQ3VXBo#gV_<~POPjTi;0wO#Bsl{lx7fA+xYQ z_5{TJ-QNTue(xm!aUROeBpeJou}p_KAl}u2yYV1pN01L+k=|h_gfZvwj@|`;PMLTm zT-*M^i$4zaXdDm1d-2@_Ud?5x8eYoM4u+zo7p1u@mpuPsh<=ibRPG#V#`#V-&Ot7v z?i%(UXB!FE?{QsvRalF>E>wv;eYXK%>X=`2xVpE zBjsfYOXFNQogu=+y|ALksW8nn}#CELo~P54kK8}F~|QOsI_ za_C%9U1?+u4cYgNl$M~dUX4zzu- zNNiohVZ{O(!w>9+?VEG$DQCuTNnh;N9@R^JS1r z+1?K0oQ3G@&}_^*V1ZBv{C6H(?%lm#|L&hp{^`|+L_!a1Xi7>rbmMtQlp{XpvgJfB zwuxca6=g{oc^D;}AGaEmu8y-`Vx92O7?uMgy<+kg(4W#C%}jaV7Z^-=z>-KA_insQgtX( ztDrN?MnI)VSj$%d?OQSef0{PEL=9>;QpXOF-vvep^!zdjb%Y>`ANQCKSsQzaUqMSy zwll22_JQ)VX47t1Ph0#akWJA|kxvNM=)QmsId2bn3kHSr`f!}i^47#EES=?F2tOc- zqdF*z!(*8#?!{>?KwWuw`^}XQWH?U3U3Vn<%IVI!BzU9XKsR2=}2J!tGQ-bg6g>*H+7@p@3O$o#ezl+T=PmlpSvf6|CR7 zeY3rC^X>IzK9Bk6#9_9ZCOcJo@8+FbEA1QaufMaps;Hj5wer!Oh^py$1imjbon{`5 z;4+l#WP61AFIC>#%eSwuEc2JKy|Pi)DjTVpveG~xfM-eU=)%WF&QR6PCY9|#3%^b; zOn5@}O9sHB0*Sn`gjQZzJ|72TovKrcX^q=IV}+pDH*_fa zW&=FxAnh!eHQPskp0Y1EmT6H6n*N=#`8rlid6v#VT&tn`Yjh5vYr&bQwn)@f z6q~w&0o-iOxC9xn@1Dn8^jul#`;veNc4i*xDQ~-9j#i9MKLd>IqJpZ))vZCTjmmc7 zPpSl4fVEoq;N=-v@7Dp@3iGi?ZTqRBhV^GBb=fHDg$H;Ctop#2G>QiQ`d#DmuRkb{|u&V%>pk;PRQY^ za^O2vr)N+Ff;Z`Gmb6sG3lJ?aseCARsaWR3D5|=Gpjf4#;A=E?d7SqW)zF(jTJEB} z@?^^C4Tc-q!+5wu$5PcpRiWu0oL@d4wGxv;&1lkHn~eyfR!C@=$i4msmIzI15R zG#w+bcn*!0iE?qR^^|g9p3zPj>mHdu7TOxT1FgsxopiLbm<}~9C_|gD{Aim`vk_{; zLAtSsI6M5Zn|3+cuA9D3*#iMUQ#Hs!UW+{(+svgN={3_P+PpFBhS*7L; zYJQ3apT;*=-?{Pr9sU-H$W=PE>o1%#t5bPXk__bz?IV5aBSB7?mFrI?|5`R=<3aK& xeTxcuLK(s}SgnSjh9!}O9NRG+DxWOfnlz$-Kl91PR zWLvKo^4F}G_|=+LD^tm+G)8aQt!yQWy1~q5t~FK}L*1(6S*9}XjWc^jsT5e2<=7a@ zvvF2n6RgNiut|23O|esKn$56TcKVxQWx^}4GcyXxv$8xlqp(-L(JMtz&Z9nsx#O5M zfw{E8tBovkzR}S81N2T~l^Lw}TEpy7UkCLKc7eUgeu$kfvJ$(*-eQ;8+iZ@_vv=4P zw!q$HSJ{u)d+ayZ`|QVTk^LsS#(u)Cv!AjL*b=+JK5YDeR+HyA!Is(0FO|xqSFEdS zh28p6t(^2GQ2&VCMtzFiVIMa%c9-2lx%!QUp8M<*^qgY9#nu|sUT0;rrcs=ATH>i2@%oXl@CojBKyGr%y!hC0T`^uslE_%W4;)WO0HmUPUb<8aDp%~~dpKZiyIje9 z{Nd8QYI&iOy?>+1{l=y!SMu%6P=uJz{Tjxiqv_Q_?~T1uv6t>IRckISbobVJm5y3D zK{Xt_8AdzYt77L$o=E*8R3?uOph=}-CAW0HD(_pqGE!gY9DlW{05^`g+T4SsyXaZy zbMR)F?kV}3a&XYIe2?xu=$RSLAx02AW$o!<$Nd|UFUQT1j2d)|9N_*9V%ebQm~B$o zP7k+P>g_KjJ2j-=T^}rSW|T@wveP2z_OxVh$Ghu&ruVdDqzAJG%bnX^Sh`QGX#SG? zKo+t&*9;pSYwB!c!@^p(x2PsVyZ%^+J!q}anBFfvd>{~ei(Gl_I7%yhqnCwPV9Z| zg_-MZ&tsS+trh$~x?{h$)!qbSj_IS0>0w71f?eKgi)FuFA6%);y@>3SuG9yKJ}uq0 z&|85dPd7Ny#-T^sv#DM8BVu6O7{ce_sJLOw8*I! ziRu0?jl}|$tTYf(4Nl&4Bp+5gX2IaHr3%gW>b_+0C%T}^19 z>-+yC^jLeUg8uiQ8(s6@vM{#H*o05i8Y@%%vJyZfmRp=u! zd{@{k7n?s?72J`w!c$fBgf9YhPU52{}6pxa?#<~&8fQp zD*S}pA;@w%g>~$Nts1@d*kvaOo%CciiV^6K4sYSRj&F1hMFW?6Jk?cW^-uG);@pWFsTn|mkGfsucr@Us~8-McTu9Tajbg=Itq&{}>l zzl|s(m?vBCxGMkvbM1rga@U%EBx+%+eZ5LYO3&AMwc&{>*4}9a(cHCWSOfE}qtYLn z*q^T6ezfcguXFmc(;JBE3dfBcZ@&#(^jImM+y{+!fo3e$erj-FGw=IeG=VetYjy)+ic3%DKP>>T1igE zQF1GOkycTC3e}p}fz|mLP$Z+$f_R^dja>5{2+;xxMYYsP`PI~-TENFr4V1Q4z;992 z(3|Vk@~G!E{tjjhL~Npw!B^VuC#d*JSKVKM~%=ES%O3kI>sOm%{*xo4f3Qy$!Ls=#)(E;KYg1Xoeg70;>E?Dt?Hf zWF}eSCOPLQ87JmbQ*O=QqFGid?Q~d5C(Rr(oWF|ZHoj2-g`yc?9~cW}p3`Jsjr5QH zD6Ja9S2A%$0${8N^{K|MgPq#`C9qSI>^u*4>Rm%9u_1Ky&!$X^wO9vu%z|awx+2U& z>$%cUyXOACi%blF0#yDQGp%k$*jw4d9Dz@?`D6{iX>I|Ez6Ut-1K_krfODMxxhMcQ zGXvmU>jCFPY;P5Nz?q0~zJ_sFeFQkydccWs5;*N@5x|Mob_S_O;mDjmbgsoJJF%Tb z{~H*2064`81tElijvf7;+q1 z^79z|zW^n2N(op1^}OE*LgY@cZf`#^qL7QyRFd-}KZpR|z<-0p)w5C(S&|>!p1+Q* zk}QzfT_Mr%*Ql+9QDVSsHxtW`$Y?n|PNcUzYUC+!T3niSV!7?M7qG-aJXUi>DaS9< zayE9X(lIL8z}ri~Jw*VM7LO_!iOltgzfIfBQ87;w4dT!XAk43!XBXe-0~Crz&gffyt6S^;z9YDwm!^)j~dWQEdlDS^~!mC*#Fbqq-i@?rzbp`eR= zv&0;RN($LPn+NdDY1nlqMVr{bui3SPwnbrzc^c05;QcOM6Kur;U{Bly1Lvs z`X#1E1QCj6L-4jXsZK%s@UNeN#_v$^F^Yk}C>$IJjI?YZFefH}pY6hr3iVJUmY-Ih zs}ss;I1lMZ;S@@0=Y40*YlXW6hrxkEkt3KL2p0h(+^8QQPV`f0L}*E|$h$iwo!`fX zi59dG)A*Ghy_s=jWBBi7I=G0k@e~x-hCkp;3j-W5Kqq`^o z^Xj(E&;7oFH!mpLmv*ddAs2Y6B7Bs20WyKtnOS)dL4&0}k;L)K16U zts)!r5(Q2_z!-_WAH==1!5Nyoz}F{7{0hD0nWj8u_x(B0k^h-L;l z>q@!t$C&zpbg+`7%7`#G-1;x+M0pe>o++I~bVlP}pnXI{#1V-YZB%*+sPDhQNCD5u zz)s&%w=G_X^(`$nkc4T#0iBd^|6;6Rlru8wYAUirT^P?19!im2C0~AM#rkv1L!y+4 zRr2wS#AtMW zV}!f|^7?LVsp;}o4>(o<1ElvU)7MagkBCT55Y-FLNmr9G+8t3M1qq3{N2lvcLcZ=$ zXn|Bsa4C`sf=Urd968ovGLR>Y_&Y2eQLvC&3%Q0IKGT5yoW>iVj~xB~aHAta9H7Xr zu#SK&#Xt)RFcGKg0CcdOa^GuYgBon`BH7UX9cgPfU~6V7HU^Xt7J|@6X?rVkh&;P5 zNRi#j^~RYp50~}{o17z8AlLmpx$ZTo7FT;}F$U%N>ZKjNI&Oz2U-hip?yLdXTx1A9 zEf+cgJ15snxmtkS3Qd@MDx%BM(MTMYG2PJLT=eThpHy-hAN;{z?Z5IDr zu;WK)Ae$gqMUocdd82FW--wM~zVTkq^XsvN9t+t<`fH}YNaIZWi$sOr*Mt>Yc$UHY z&66_^5f8v#_0AtUIR}43=^%w8CpygDNKlYXFK4?*O__6*NP4XedZ-u)$0>!A-v4sC zumrGlKq}eDCI2SlcM0(Wl#u7>m~+Im)lx<#U^0G6!vg7a$s&DfUiY1u9hOME{175oH02WozW>ZTe_=wV?43)Pa*H(A}&UJv(!_e>Eo#n_$^vy`XVHzEX zw_{&8n7`1Pzrc>TVu}r}YQh64G5EZ6xBm~$|V#$9- ziJp(fBZ>T<7NgD=oHM-5z&XDn4GoW#$36_n7_jFo_$}=Do$Bz9@SDbk6a!*EdY$1< zh-%QYr;N_|ZL}*En(JPzj6_8yLWr|yaFPMj==rM;Fm(AOi@{aE3lq|v)M@n;WGwiu zCp%u64VYlQI7p%8Cz!`MP0mZBq3Lu2a%Ck%#i&Pohx&DfSB^0XX6eB#Xhi%ynmbT# zT$09d88)QoLzDjtw1^@<1=vHbX;QOw3-QCOTGaB;b2zJ~=uPZuwS_pdS}nc8uM@#U z)Pj&)f~3_$ObLez)SH8wMHqfC=ZBoaBs{?M2=X2hab*_Ei8b=+MzY*RqK1c2Qb4Nl zW$Gm(mRyi8S0jdhjLl1=lqNYiH#zWXbr|IhwI#8ywL-Sj^sbZHM*AqFUBk91kk4xd z-h6DGGP>{o*cl5?Zh6y6F#+7O)SN8oA&x#op7fHamqfM GQ~w7syK`&+ diff --git a/katan/__pycache__/times.cpython-37.pyc b/katan/__pycache__/times.cpython-37.pyc deleted file mode 100644 index 50875482158a9d12a9ff79290bb62fab7d949699..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1806 zcmZ`)OK&7K5N_K&-90^zJT|*L1uem9Gl$J^S|JogK!UU^T7;DBftFA(Z71}kAA{{# zG@~8~v60wI4*Y}3kw3tn;2-$PiNCNXsyvg2c=X6+ySiLm_0?CNPkX)CgU9~<$G0T( zygzYs^)Og`3`746LU@7{UP|0dQ|4wr_1)Y`1DI(MmSGx}Q5q4nXGOb=)41%U9k=(3 zZrMxw#IU*}I-+~-rCXvW`Y_)Wzm)@Kjx*!d`BnxKz5kx3~{eVFXm)pMzgPM+2xKgp#?o=l3ISIGfHzpV#}sguWtM@g-cXGi-9=7d_y%8^hJ zd>^6@PHI(hV_Q1og|zH@ZEnRs2$c_Zwqxq!Ye?u=fMRxB)Nmx9NlVSlxwA270R{|e zz^o6S(bh+(EE7im=Bc~j?A62JWGKtLm>w%$%IWY6nVo%7AIUp9Nkn3zUISf*z~Y18;=sNB2?PL440%EI+rM}x1ff2U8gk&h zAai2qh`ogOZvc^me?jN0p`ry)f7P&&zi62Nz-De8nrPIqA|X&oMj&8(kS>V&9T@se z5T`(sZ#L(oAuqj=cNQqWA-~ce{KF-n17gFKGbkwC`M@&Rg0^M&cZGDIt9#(*RlHl| z+GKS(-g_6FKyu(&d|K~_tZN$?p+rD~;B7mmW#FS$Nb=x0SJ zh4oP#e7j-eT<5MSGGKrnM^-2*whvX$R9c6@*j|;1;SvvxAVn=dJ}}lVgj_H z5$TdX+hTpH?t}KfyBGaq_d<69si5%Cl_M-Tn|*dua1J@wo3AJ1LENA3|Ez8oSID+{R$Bs_PJx z8foQWW)EP#CL9v)`d-2++`gAwR{rdro2)QOAnk$7@{$+H*OIHmh4p_iL$FKRDVRbf z5bb|Pde!Aq^$6ZNW_na@!x|2_yJ|Jvv%wg8h-*hC+o~qzc&gq5L+fL`Q-3^#`^b^W z@e(hCw7V+B4w@{1#EjE$rB?wuAQJ`uQ7~sqTH@v6NZ{~t(cFsr2IgqDti_~|d+0?Y T$0E3$@VAaX8nZZ#x1#WG944$S diff --git a/katan/templates/__pycache__/__init__.cpython-37.pyc b/katan/templates/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index 97ddbcb4e0241171600f08eb07679bc17f321eca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 145 zcmZ?b<>g`kf;+oUCxGb3AOZ#$feZ&AE@lA|DGb33nv8xc8Hzx{2;!HOeyM(HZe~tp zd{JUK~7>xYO#KNd}dx|NqoFsLFFwD So80`A(wtN~keQ!>m;nH6ek3^n From 51e57f925a59c9362e270c64bcf3f92fa2592bce Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 12 Aug 2021 15:08:57 -0700 Subject: [PATCH 07/12] Incorporated keyword defaults --- .../.ipynb_checkpoints/compiler-checkpoint.py | 19 ++++ katan/__pycache__/compiler.cpython-37.pyc | Bin 3454 -> 4259 bytes katan/compiler.py | 19 ++++ .../keyword_defaults-checkpoint.yaml | 82 +++++++++--------- katan/templates/keyword_defaults.yaml | 82 +++++++++--------- 5 files changed, 120 insertions(+), 82 deletions(-) diff --git a/katan/.ipynb_checkpoints/compiler-checkpoint.py b/katan/.ipynb_checkpoints/compiler-checkpoint.py index 08e4ad6..c84c8d8 100644 --- a/katan/.ipynb_checkpoints/compiler-checkpoint.py +++ b/katan/.ipynb_checkpoints/compiler-checkpoint.py @@ -82,6 +82,22 @@ def data_func(dtype): if dtype=='telem': # Telemetry return lambda files,mjds: telem.from_nirc2(mjds,files), False +def change_cols(data, params): + """ + Changes / filters columns according to the parameters passed. + A False entry means the column will be omitted + A True entry means the column will be included as-is + A string entry means the column will be re-named + """ + # Drop bad columns first + bad_cols = [col for col,val in params.items() if not val] + data = data.drop(bad_cols, errors='ignore') + + # Re-map column names + col_mapper = {col: new_col for col,new_col in params.items() if isinstance(new_col, str)} + data.rename(columns=col_mapper, inplace=True) + + return data ################################ ###### Compiler Functions ###### @@ -143,6 +159,9 @@ def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=F # Get other data from strehl info other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) + # Change or omit selected columns + change_cols(other_data, params[dtype]) + if match: # Needs to be matched if other_data.empty: # No data found other_data = pd.DataFrame(columns=other_data.columns, index=range(len(nirc2_data))) diff --git a/katan/__pycache__/compiler.cpython-37.pyc b/katan/__pycache__/compiler.cpython-37.pyc index 04bb419b7d1f02198940e59a9b5d221101dedbd3..fa324c4dd8722ba54d83757dcb537cec0fa1ab97 100644 GIT binary patch delta 1411 zcmah|&2Jk;6yG(gq8ZpdyUq9&HJR&f5Mo5WVXM0o@X`-SVSI@MkXTvinjGKlFkw7o0LR z>nrWpr;AiYQYZD2+oYHcG1=4WJkagKWg=NfYe=esYzNWU82*~{G>Sx7F&vZI)|w$p zcsO8nGbMAEQxmI6&Hf!0JU@~nX!MtRE$~(3wOgI*8#*jOW*hsH-T>K6^iy-!g0_bk zT}R~~+tl`V!nPE3NNpiyTdEhb%wUv3mJ}6VXIa54`!0o?aa3t(vXe+Bfhf-~cFyfAy(WYx_2VJBdo zK)5n~Bw8x9xNLV)TMFHWx`lfdvDC&_#9FH9Y%xB-rs-mc%@*r)5QQ_;2hgTN|HANd zo!anV4xRH*G?9p$J^9n#1K3SMr(9b=i^jTVqtL5}ltWrZQVt{!9hJQo!-}!-Z8DJ? zzZ$&~=i^MA8#NB}Pz=c@aenu_{55)Y_X29AXe27f*#kHLZsSb z^zyNDzlwGRZt=_!JOD6m-}%x&@3G{E+%K189q+9ZIfDWr3Ay;DX>GQ4xzvxb4su?L zarrumYWW$asga(#FVROTYoJn<*&@VjjHZV$eEHi za3ep^Y*GJqzhFUE+|X|{@$>4VPxQ%>W*XXNk&A8C4WiA{&Ze0T&SxagX@jz66PgXG m(;bql$#CJ#S_9QQoPERr{7iy>Y$#gEdf|<1-YGd%8~z4`jbp9= delta 636 zcmX9*zi-n}5cc_J;yO+m+Wb!A6p=J4Quz_0ZU`DWFrbK5LJCx^$#cMwQEL`oly*E};y{&be+Gb<^fIr4@WN7eC0fZ zHg|$aLo2-q*YNL98?58+L3(hFb{oNFkO`F;%mB#?eyX`!K{iwdsZa~!NXc?}ZpKd& zRGAACd2XtOu`hW#C6I`u1$+hI-n@Or(Q5^Asr}fGM1)6PgO8Nn$Qch^Kay-0S?{~z z>2QEBP+twe6;_IoYWue92d3Mz$mc{@GbkPL9*W{^GcXUaIkfOi#!1+8Us;jhST5cn zkMx6BBzq>^`(&ZthdQ~^w_9VX^lKD2B?BrHM3omorx_D;K?RLxy$l6@15_a+;5Jn@ zs-&L0vt6YvyXYd*$NO}^w-_`TEqT~>1M(s1Hy<$a9)lKzu8yp}Id+26|5k94ZL$o? b4A^UMo7_#+piR1|5=i7->V~u)E6DH Date: Thu, 12 Aug 2021 15:21:18 -0700 Subject: [PATCH 08/12] Added keyword support --- katan/compiler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/katan/compiler.py b/katan/compiler.py index c84c8d8..12edf33 100644 --- a/katan/compiler.py +++ b/katan/compiler.py @@ -90,14 +90,14 @@ def change_cols(data, params): A string entry means the column will be re-named """ # Drop bad columns first - bad_cols = [col for col,val in params.items() if not val] - data = data.drop(bad_cols, errors='ignore') + good_cols = [col for col,val in params.items() if (val and col in data.columns)] + new_data = data[good_cols].copy() # Re-map column names col_mapper = {col: new_col for col,new_col in params.items() if isinstance(new_col, str)} - data.rename(columns=col_mapper, inplace=True) + new_data.rename(columns=col_mapper, inplace=True) - return data + return new_data ################################ ###### Compiler Functions ###### @@ -160,7 +160,7 @@ def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=F other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) # Change or omit selected columns - change_cols(other_data, params[dtype]) + other_data = change_cols(other_data, params[dtype]) if match: # Needs to be matched if other_data.empty: # No data found From 7e040ab420861d0f2adb71002a1e44e830ab042d Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 19 Aug 2021 12:46:21 -0700 Subject: [PATCH 09/12] Added support for external parameter files --- katan/compiler.py | 53 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/katan/compiler.py b/katan/compiler.py index 12edf33..ac1c251 100644 --- a/katan/compiler.py +++ b/katan/compiler.py @@ -20,7 +20,7 @@ raise ValueError("PyYAML not installed. Please install from https://anaconda.org/anaconda/pyyaml") ### For importing package files -### May need to edit this later if there are import issues with Py<37 +### Note: may need to edit this later if there are import issues with Py<37 try: import importlib.resources as pkg_resources except ImportError: @@ -32,13 +32,30 @@ default_parfile = 'keyword_defaults.yaml' -### Load default parameter file -try: # Load from templates module - file = pkg_resources.open_text(templates, default_parfile) - default_params = yaml.load(file, Loader=yaml.FullLoader) -except: # Warn user - default_params = {} - print("WARNING: Unable to load default parameters. Please specify a parameter file when compiling.") +def load_default_parfile(): + """ Loads default parameter file and returns a dictionary """ + try: # Load from templates module + file = pkg_resources.open_text(templates, default_parfile) + default_params = yaml.load(file, Loader=yaml.FullLoader) + except: # Raise error and request param file + raise ValueError("Unable to load default parameters. Please specify a parameter file.") + + return default_params + +def read_params(param_file): + """ Reads parameters from the specified file or returns default parameters """ + if param_file is None: # load default + params = load_default_parfile() + elif isinstance(param_file, str) and os.path.exists(param_file): # Load user-specified + try: + params = yaml.load(param_file, Loader=yaml.FullLoader) + except: # Unable to load + raise ValueError(f"Failed to load {param_file}. Please check that PyYAML is installed \ + and that the file is formatted correctly.") + else: # Invalid input + raise ValueError(f"{param_file} is not a valid parameter file.") + + return params # Shorthand / nicknames for data types expand = { @@ -130,26 +147,30 @@ def match_data(mjd1, mjd2): # 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and # ['mass', 'dimm', 'masspro'], respectively def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=False, - params=default_params): + param_file=None): """ Combines and matches data from a certain Strehl file with other specified data types. NIRC2 files must be in the same directory as the Strehl file. """ - # Check file paths parameter & open if yaml + ### Check file paths parameter dict, load if yaml if not isinstance(file_paths, dict) and os.path.isfile(file_paths): with open(file_paths) as file: file_paths = yaml.load(file, loader=yaml.FullLoader) - ### Add a catch for YAML not being installed if file specified + ### Add a catch for YAML not being installed if file specified? + + ### Sanitize data types + if check: + data_types = check_dtypes(data_types) - # Read in Strehl file + ### Read parameter file + params = read_params(param_file) + + ### Read in Strehl file nirc2_data = strehl.from_strehl(strehl_file) - if test: # Take first few files + if test: # Take only first few files nirc2_data = nirc2_data.loc[:3] - if check: # Sanitize user input - data_types = check_dtypes(data_types) - # Full data container all_data = [nirc2_data.reset_index(drop=True)] From d0293b13a3c186eca4d114fea1ee82fb75d5d889 Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 19 Aug 2021 14:32:46 -0700 Subject: [PATCH 10/12] Added telem and temp support for external file directories --- katan/compiler.py | 43 +++++++++++++++-------- katan/telemetry.py | 12 +++---- katan/temperature.py | 83 ++++++++++++++++++++++---------------------- 3 files changed, 77 insertions(+), 61 deletions(-) diff --git a/katan/compiler.py b/katan/compiler.py index ac1c251..252cde6 100644 --- a/katan/compiler.py +++ b/katan/compiler.py @@ -47,8 +47,9 @@ def read_params(param_file): if param_file is None: # load default params = load_default_parfile() elif isinstance(param_file, str) and os.path.exists(param_file): # Load user-specified - try: - params = yaml.load(param_file, Loader=yaml.FullLoader) + try: # Open stream and read as YAML + with open(param_file, 'r') as stream: + params = yaml.load(stream, Loader=yaml.FullLoader) except: # Unable to load raise ValueError(f"Failed to load {param_file}. Please check that PyYAML is installed \ and that the file is formatted correctly.") @@ -67,16 +68,24 @@ def read_params(param_file): } # Utility functions -def check_dtypes(data_types): +def check_dtypes(data_types, file_paths): """ Returns an expanded / cleaned list of data types from user input """ new_dtypes = [] - # Check requested data + + # Edge case: only temperature dir specified + if 'temp' in file_paths and os.path.isdir(file_paths['temp']): + file_paths['k2AO'] = file_paths['temp']+'k2AOtemps/' + file_paths['k2L4'] = file_paths['temp']+'k2L4temps/' + file_paths['k2ENV'] = file_paths['temp']+'k2envMet/' + + # Check requested data types for dtype in data_types: # Check for nicknames if dtype in expand: new_dtypes.extend(expand[dtype]) elif dtype in accept_labels: # can get data new_dtypes.append(dtype) + # Return cleaned list return new_dtypes @@ -87,17 +96,17 @@ def check_dtypes(data_types): # Have files with acceptable columns for each data type so you can check inputs # Accept column inputs to the data function and take optional arguments in combine_strehl -def data_func(dtype): +def data_func(dtype, data_dir=None): """ Returns the function to get data for the specified dtype and whether data needs to be matched to nirc2 mjds """ if dtype in ['cfht']+expand['seeing']: # MKWC - return lambda files,mjds: mkwc.from_nirc2(mjds,dtype), True + return lambda files,mjds: mkwc.from_nirc2(mjds, dtype, data_dir), True if dtype in expand['temp']: # Temperature - return lambda files,mjds: temp.from_mjds(mjds,dtype), True + return lambda files,mjds: temp.from_mjds(mjds, dtype, data_dir), True if dtype=='telem': # Telemetry - return lambda files,mjds: telem.from_nirc2(mjds,files), False + return lambda files,mjds: telem.from_nirc2(mjds, files, data_dir), False def change_cols(data, params): """ @@ -146,7 +155,7 @@ def match_data(mjd1, mjd2): # 'telem', 'k2AO', 'k2L4', or 'k2ENV' # 'temp' or 'seeing' will be expanded to ['k2AO', 'k2L4', 'k2ENV'] and # ['mass', 'dimm', 'masspro'], respectively -def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=False, +def combine_strehl(strehl_file, data_types, file_paths={}, test=False, param_file=None): """ Combines and matches data from a certain Strehl file with other specified data types. @@ -156,13 +165,16 @@ def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=F if not isinstance(file_paths, dict) and os.path.isfile(file_paths): with open(file_paths) as file: file_paths = yaml.load(file, loader=yaml.FullLoader) - ### Add a catch for YAML not being installed if file specified? + # Check file paths are valid + for dtype, data_dir in file_paths.items(): + if not os.path.isdir(data_dir): + raise ValueError((f"The path specified for {dtype} data ({data_dir}) " + "is not a valid directory.")) ### Sanitize data types - if check: - data_types = check_dtypes(data_types) + data_types = check_dtypes(data_types, file_paths) - ### Read parameter file + ### Read in parameter file params = read_params(param_file) ### Read in Strehl file @@ -176,7 +188,10 @@ def combine_strehl(strehl_file, data_types, file_paths=False, check=True, test=F # Loop through and get data for dtype in data_types: - get_data, match = data_func(dtype) # Data retrieval function + # Get data directory + data_dir = file_paths[dtype] if dtype in file_paths else None + # Fetch data function from dictionary + get_data, match = data_func(dtype, data_dir) # Data retrieval function # Get other data from strehl info other_data = get_data(nirc2_data.nirc2_file, nirc2_data.nirc2_mjd) diff --git a/katan/telemetry.py b/katan/telemetry.py index 3757ee7..db37923 100644 --- a/katan/telemetry.py +++ b/katan/telemetry.py @@ -19,14 +19,14 @@ import importlib_resources as pkg_resources ### Path to telemetry files [EDIT THIS] -telem_dir = '/g/lu/data/keck_telemetry/' +# telem_dir = '/g/lu/data/keck_telemetry/' # Sub-aperture maps wfs_file = "sub_ap_map.txt" act_file = "act.txt" filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers -filename_match = telem_dir+"{}/**/n?{}*LGS*.sav" +filename_match = "{}{}/**/n?{}*LGS*.sav" TIME_DELTA = 0.001 # About 100 seconds in mjd @@ -261,11 +261,11 @@ def extract_telem(file, data, idx, check_mjd=None): return True -def from_nirc2(mjds, filenames): +def from_nirc2(mjds, nirc2_filenames, telem_dir): """ Gets a table of telemetry information from a set of mjds and file numbers """ N = len(mjds) # number of data points # Get file numbers - filenums = filenames.str.extract(filenum_match, expand=False) + filenums = nirc2_filenames.str.extract(filenum_match, expand=False) filenums = filenums.str[1:] # First digit doesn't always match # Get datestrings @@ -280,10 +280,10 @@ def from_nirc2(mjds, filenames): # Get filename and number fn, ds, mjd = filenums[i], datestrings[i], mjds[i] # Search for correct file - file_pat = filename_match.format(ds, fn) + file_pat = filename_match.format(telem_dir, ds, fn) all_files = glob.glob(file_pat, recursive=True) - # Grab the first file that matches the MJD + # Extract the first file that matches the MJD to data for file in all_files: success = extract_telem(file, data, i, check_mjd=mjd) if success: break diff --git a/katan/temperature.py b/katan/temperature.py index 9c672e0..0614920 100644 --- a/katan/temperature.py +++ b/katan/temperature.py @@ -14,8 +14,8 @@ verbose = True -data_dir = "/u/emily_ramey/work/Keck_Performance/data/" -temp_dir = data_dir+"temp_data_2/" +# data_dir = "/u/emily_ramey/work/Keck_Performance/data/" +# temp_dir = data_dir+"temp_data_2/" col_dict = { 'AO_benchT1': 'k1:ao:env:benchtemp1_Raw', # k2ENV @@ -36,29 +36,29 @@ "k2:ao:env:LSenctemp_Raw": "LGS_enclosure_temp" } -file_pats = { # File name patterns -# 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", -# 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", -# 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", - 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", - 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", - 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", -} +# file_pats = { # File name patterns +# # 'k1AOfiles': temp_dir+"k1AOtemps/*/*/*/AO_bench_temps.log", +# # 'k1LTAfiles': temp_dir+"k1LTAtemps/*/*/*/LTA_temps.log", +# # 'k1ENVfiles': temp_dir+"k1envMet/*/*/*/envMet.arT", +# 'k2AO': temp_dir+"k2AOtemps/*/*/*/AO_temps.log", +# 'k2L4': temp_dir+"k2L4temps/*/*/*/L4_env.log", +# 'k2ENV': temp_dir+"k2envMet/*/*/*/envMet.arT", +# } data_types = { 'k2AO': { - 'file_pat': temp_dir+"k2AOtemps/{}/AO_temps.log", + 'file_pat': "{}{}/AO_temps.log", 'cols': ['AOA_camT', 'DM_rackT', 'El_roomT', 'KCAM_T1', 'KCAM_T2', 'OB_leftT', 'OB_rightT', 'photometricT', 'k2AO_mjd'], }, 'k2L4': { - 'file_pat': temp_dir+"k2L4temps/{}/L4_env.log", + 'file_pat': "{}/{}/L4_env.log", 'cols': ['LGS_enclosure_hum', 'LGS_enclosure_temp', 'LGS_humidity', 'LGS_temp', 'k2L4_mjd'], }, 'k2ENV': { - 'file_pat': temp_dir+"k2envMet/{}/envMet.arT", + 'file_pat': "{}/{}/envMet.arT", 'cols': ['k0:met:dewpointMax', 'k0:met:dewpointMin', 'k0:met:dewpointRaw', 'k0:met:humidityRaw', 'k0:met:humidityStats', 'k0:met:out:windDirection', 'k0:met:out:windDirectionMax', 'k0:met:out:windDirectionMin', @@ -104,16 +104,17 @@ def get_columns(): return all_data -def search_files(file_pats=file_pats): - """ Finds all filenames matching the given file patterns """ - all_filenames = {} - for name, search in file_pats.items(): - filenames = glob.glob(search, recursive=True) - all_filenames[name] = filenames - if verbose: - print(f"Done {name}, length: {len(filenames)}") +# def search_files(file_pats=file_pats): +# """ Finds all filenames matching the given file patterns """ +# all_filenames = {} +# for name, search in file_pats.items(): + +# filenames = glob.glob(search, recursive=True) +# all_filenames[name] = filenames +# if verbose: +# print(f"Done {name}, length: {len(filenames)}") - return all_filenames +# return all_filenames def collect_data(data_files, col_dict=col_dict): """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ @@ -226,27 +227,27 @@ def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInval return data -def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): - """ - Reads in all data matching file patterns (file_pats), combines them into one table, - cleans them, and saves them to a FITS file - """ - # Find all files matching pattern - all_filenames = search_files(file_pats) - # Read all data into one table (per dictionary label) - all_data = collect_data(all_filenames) +# def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): +# """ +# Reads in all data matching file patterns (file_pats), combines them into one table, +# cleans them, and saves them to a FITS file +# """ +# # Find all files matching pattern +# all_filenames = search_files(file_pats) +# # Read all data into one table (per dictionary label) +# all_data = collect_data(all_filenames) - for name, data in all_data.items(): - data = parse_dates(data) # Parse date cols into datetimes - data = clean_dates(data) # Remove invalid dates - data = clean_data(data) # Casts other cols to numeric - # Save combined/cleaned data to FITS file - filename = location+name+".fits" - to_fits(data, filename) +# for name, data in all_data.items(): +# data = parse_dates(data) # Parse date cols into datetimes +# data = clean_dates(data) # Remove invalid dates +# data = clean_data(data) # Casts other cols to numeric +# # Save combined/cleaned data to FITS file +# filename = location+name+".fits" +# to_fits(data, filename) - return +# return -def from_mjds(mjds, dtype): +def from_mjds(mjds, dtype, data_dir): """ Gets temp data of input type from the specified MJDs """ # Get pd datetimes in HST dts = times.mjd_to_dt(mjds, zone='hst') @@ -254,7 +255,7 @@ def from_mjds(mjds, dtype): datestrings = dts.strftime("%y/%m/%d") datestrings = np.unique(datestrings) # one file per date # Get relevant filenames - filenames = [data_types[dtype]['file_pat'].format(ds) for ds in datestrings] + filenames = [data_types[dtype]['file_pat'].format(data_dir, ds) for ds in datestrings] # Get data from filenames data = collect_data(filenames) From 6aa3bbeb4200defcd3b3c5f771ab67d4b0a4e7de Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Mon, 30 Aug 2021 13:22:56 -0700 Subject: [PATCH 11/12] Starting OO-changes --- katan/compiler.py | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/katan/compiler.py b/katan/compiler.py index 252cde6..6b04c50 100644 --- a/katan/compiler.py +++ b/katan/compiler.py @@ -32,32 +32,6 @@ default_parfile = 'keyword_defaults.yaml' -def load_default_parfile(): - """ Loads default parameter file and returns a dictionary """ - try: # Load from templates module - file = pkg_resources.open_text(templates, default_parfile) - default_params = yaml.load(file, Loader=yaml.FullLoader) - except: # Raise error and request param file - raise ValueError("Unable to load default parameters. Please specify a parameter file.") - - return default_params - -def read_params(param_file): - """ Reads parameters from the specified file or returns default parameters """ - if param_file is None: # load default - params = load_default_parfile() - elif isinstance(param_file, str) and os.path.exists(param_file): # Load user-specified - try: # Open stream and read as YAML - with open(param_file, 'r') as stream: - params = yaml.load(stream, Loader=yaml.FullLoader) - except: # Unable to load - raise ValueError(f"Failed to load {param_file}. Please check that PyYAML is installed \ - and that the file is formatted correctly.") - else: # Invalid input - raise ValueError(f"{param_file} is not a valid parameter file.") - - return params - # Shorthand / nicknames for data types expand = { 'temp': ['k2AO', 'k2L4', 'k2ENV'], @@ -89,6 +63,32 @@ def check_dtypes(data_types, file_paths): # Return cleaned list return new_dtypes +def load_default_parfile(): + """ Loads default parameter file and returns a dictionary """ + try: # Load from templates module + file = pkg_resources.open_text(templates, default_parfile) + default_params = yaml.load(file, Loader=yaml.FullLoader) + except: # Raise error and request param file + raise ValueError("Unable to load default parameters. Please specify a parameter file.") + + return default_params + +def read_params(param_file): + """ Reads parameters from the specified file or returns default parameters """ + if param_file is None: # load default + params = load_default_parfile() + elif isinstance(param_file, str) and os.path.exists(param_file): # Load user-specified + try: # Open stream and read as YAML + with open(param_file, 'r') as stream: + params = yaml.load(stream, Loader=yaml.FullLoader) + except: # Unable to load + raise ValueError(f"Failed to load {param_file}. Please check that PyYAML is installed \ + and that the file is formatted correctly.") + else: # Invalid input + raise ValueError(f"{param_file} is not a valid parameter file.") + + return params + ############################################# ######## Interface with other modules ####### ############################################# From d04e13aef9b9c1311b1f2adc048c985547ebe180 Mon Sep 17 00:00:00 2001 From: Emily Ramey Date: Thu, 2 Sep 2021 16:37:20 -0700 Subject: [PATCH 12/12] Added documentation --- katan/compiler.py | 54 +++++++++++++----- katan/mkwc.py | 32 ++++++++--- katan/strehl.py | 14 ++++- katan/telemetry.py | 115 ++++++++----------------------------- katan/temperature.py | 132 +++++++++++++++++++++++-------------------- katan/times.py | 28 +++++++-- 6 files changed, 191 insertions(+), 184 deletions(-) diff --git a/katan/compiler.py b/katan/compiler.py index 6b04c50..19b0eaa 100644 --- a/katan/compiler.py +++ b/katan/compiler.py @@ -13,14 +13,13 @@ import pandas as pd import os -# Check dependencies +# Check whether PyYAML is installed try: import yaml except: raise ValueError("PyYAML not installed. Please install from https://anaconda.org/anaconda/pyyaml") -### For importing package files -### Note: may need to edit this later if there are import issues with Py<37 +### Use importlib for package resources try: import importlib.resources as pkg_resources except ImportError: @@ -30,6 +29,7 @@ # All data types that can be compiled by this package accept_labels = ['cfht', 'mass', 'dimm', 'masspro', 'k2AO', 'k2L4', 'k2ENV', 'telem'] +# Default parameter file for data keywords default_parfile = 'keyword_defaults.yaml' # Shorthand / nicknames for data types @@ -43,7 +43,12 @@ # Utility functions def check_dtypes(data_types, file_paths): - """ Returns an expanded / cleaned list of data types from user input """ + """ + Checks user-input data types and file paths and re-formats if necessary. + data_types: list of data types or combined types + file_paths: dictionary with data_type: file_path pairs + returns: list of properly formatted data types + """ new_dtypes = [] # Edge case: only temperature dir specified @@ -64,7 +69,10 @@ def check_dtypes(data_types, file_paths): return new_dtypes def load_default_parfile(): - """ Loads default parameter file and returns a dictionary """ + """ + Loads default parameter file from package. + returns: dictionary of dtypes, with sub-dictionaries of data labels:bool/string + """ try: # Load from templates module file = pkg_resources.open_text(templates, default_parfile) default_params = yaml.load(file, Loader=yaml.FullLoader) @@ -74,7 +82,11 @@ def load_default_parfile(): return default_params def read_params(param_file): - """ Reads parameters from the specified file or returns default parameters """ + """ + Reads parameters from the specified file or returns default parameters if no file is specified. + param_file: path to parameter file, as a string, or None to return default + returns: dictionary of dtypes, with sub-dictionaries of data labels:bool/string + """ if param_file is None: # load default params = load_default_parfile() elif isinstance(param_file, str) and os.path.exists(param_file): # Load user-specified @@ -101,19 +113,24 @@ def data_func(dtype, data_dir=None): Returns the function to get data for the specified dtype and whether data needs to be matched to nirc2 mjds """ - if dtype in ['cfht']+expand['seeing']: # MKWC + # MKWC seeing (mass, dimm, masspro) or weather (cfht) files + if dtype in ['cfht']+expand['seeing']: return lambda files,mjds: mkwc.from_nirc2(mjds, dtype, data_dir), True - if dtype in expand['temp']: # Temperature + # Temperature files (k2AO, k2L4, or k2ENV) + if dtype in expand['temp']: return lambda files,mjds: temp.from_mjds(mjds, dtype, data_dir), True - if dtype=='telem': # Telemetry + # Telemetry files (matched to NIRC2 filenames) + if dtype=='telem': return lambda files,mjds: telem.from_nirc2(mjds, files, data_dir), False def change_cols(data, params): """ Changes / filters columns according to the parameters passed. - A False entry means the column will be omitted - A True entry means the column will be included as-is - A string entry means the column will be re-named + data: a dataframe with columns for a certain dtype + params: a list of column names to values + A False value means the column will be omitted + A True value means the column will be included as-is + A string value means the column will be re-named to that value """ # Drop bad columns first good_cols = [col for col,val in params.items() if (val and col in data.columns)] @@ -131,8 +148,10 @@ def change_cols(data, params): def match_data(mjd1, mjd2): """ - Matches data1 with its closest points in data2, by mjd values - Returns an array containing data1 with corresponding rows from data2 + Matches secondary MJD values to those of their closest primary observations. + mjd1: Nx1 array/series of primary MJDs + mjd2: Mx1 array/series of secondary MJDs + returns: Nx1 array of indices in mjd2 which best match entries of mjd1 """ # Edge case if mjd1.empty or mjd2.empty: @@ -160,6 +179,13 @@ def combine_strehl(strehl_file, data_types, file_paths={}, test=False, """ Combines and matches data from a certain Strehl file with other specified data types. NIRC2 files must be in the same directory as the Strehl file. + strehl_file: a string containing the file path for a Strehl file + data_types: a list of data types to match with the Strehl file + file_paths: a dictionary mapping data types to file paths for the relevant data + test: bool, if True the compiler will match only the first 3 lines of the Strehl file + param_file: string, path to a parameter file (see templates/keyword_defaults.yaml) + if None, default parameters will be used + returns: a fully matched dataset with Strehl, NIRC2, and other secondary data """ ### Check file paths parameter dict, load if yaml if not isinstance(file_paths, dict) and os.path.isfile(file_paths): diff --git a/katan/mkwc.py b/katan/mkwc.py index 65aa637..6aa7edc 100644 --- a/katan/mkwc.py +++ b/katan/mkwc.py @@ -10,13 +10,11 @@ ### NOTE: Seeing data is STORED by UT, with data in HST ### CFHT data is STORED by HST, with data in HST -# cfht_dir = "/u/emily_ramey/work/Keck_Performance/data/weather_data/" mkwc_url = 'http://mkwc.ifa.hawaii.edu/' year_url = mkwc_url+'archive/wx/cfht/cfht-wx.{}.dat' +# Cutoff for daily-vs-yearly CFHT files on MKWC website cfht_cutoff = 55927.41666667 # 01/01/2012 12:00 am HST -# seeing_dir = "/u/emily_ramey/work/Keck_Performance/data/seeing_data" - # Time columns from MKWC data time_cols = ['year', 'month', 'day', 'hour', 'minute', 'second'] @@ -63,8 +61,10 @@ def cfht_from_year(datestrings, year): """ - Gets pre-2012 MKWC data from the year-long file - instead of the by-date files + Gets pre-2012 MKWC data from the year-long file instead of the by-date files. + datestrings: dates to pull data from, as {yyyymmdd} strings + year: the year to pull datestring data from + returns: dataframe with cfht data from requested dates """ # Get year-long file URL url = year_url.format(year) @@ -94,7 +94,11 @@ def cfht_from_year(datestrings, year): return pd.concat(all_data) def format_columns(df, dtype): - """ Changes columns (in place) from time_cols to MJD """ + """ + Changes columns (in place) from date/time to MJD. + df: MKWC dataframe with date and time columns + dtype: specific data type (mass, dimm, masspro, cfht) + """ # Get MJDs from HST values mjds = times.table_to_mjd(df, columns=time_cols, zone='hst') df[dtype+'_mjd'] = mjds @@ -103,7 +107,12 @@ def format_columns(df, dtype): df.drop(columns=time_cols, inplace=True, errors='ignore') def from_url(datestring, dtype): - """ Pulls cfht or seeing file from MKWC website """ + """ + Pulls weather or seeing file from MKWC website. + datestring: date to pull data for, in {yyyymmdd} format + dtype: cfht, mass, dimm, or masspro + returns: dataframe with MKWC data + """ # Format URL url = data_types[dtype]['web_pat'].format(datestring) @@ -121,7 +130,9 @@ def from_url(datestring, dtype): return df def from_file(filename, dtype, data_dir='./'): - """ Pulls cfht or seeing file from local directory """ + """ + Pulls weather or seeing file from local directory + """ # Read in CSV df = pd.read_csv(filename) @@ -139,7 +150,10 @@ def from_file(filename, dtype, data_dir='./'): def from_nirc2(mjds, dtype, data_dir='./'): """ Compiles a list of cfht or seeing observations based on MJDs - note: does not compare MJDs; assumption is inputs are rounded to nearest day + mjds: list of NIRC2 MJDs to pull data from (assumes one per day) + dtype: cfht, mass, dimm, or masspro + data_dir: location of downloaded weather/seeing files, if applicable + returns: dataframe containing all data from MJDs specified """ # Get datestrings diff --git a/katan/strehl.py b/katan/strehl.py index 9846ac3..8810199 100644 --- a/katan/strehl.py +++ b/katan/strehl.py @@ -16,8 +16,11 @@ def from_filename(nirc2_file, data, i, header_kws): """ - Gets nirc2 header values from a filename as dict - or loads values into specified df + Gets nirc2 header values from a filename and loads values (in place) into data. + nirc2_file: string, path to nirc2 file to load + data: partially filled dataframe (line i will be overwritten) + i: index at which to load NIRC2 data + header_kws: header keywords to load into data """ # Check for valid file if not os.path.isfile(nirc2_file): @@ -32,7 +35,12 @@ def from_filename(nirc2_file, data, i, header_kws): data.loc[i,kw.lower()] = nirc2_hdr.get(kw, np.nan) def from_strehl(strehl_file, header_kws=default_keys): - """ Gets NIRC2 header data based on contents of Strehl file """ + """ + Gets NIRC2 header data based on contents of a Strehl file. + strehl_file: string, path to Strehl file + header_kws: header keywords to pull from NIRC2 data files + returns: dataframe with Strehl and NIRC2 data + """ # Get directory name data_dir = os.path.dirname(strehl_file) # Retrieve Strehl data diff --git a/katan/telemetry.py b/katan/telemetry.py index db37923..f6c5559 100644 --- a/katan/telemetry.py +++ b/katan/telemetry.py @@ -25,7 +25,8 @@ wfs_file = "sub_ap_map.txt" act_file = "act.txt" -filenum_match = ".*c(\d+).fits" # regex to match telem filenumbers +# regex to match telem filenumbers +filenum_match = ".*c(\d+).fits" filename_match = "{}{}/**/n?{}*LGS*.sav" TIME_DELTA = 0.001 # About 100 seconds in mjd @@ -59,7 +60,7 @@ def get_times(telem_data, start=None): def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num_aps=236): """ - Return the locations of the valid actuators in the actuator array + Returns the locations of the valid actuators in the actuator array resids: Nx349 residual wavefront array (microns) ints: Nx304 intensity array (any units) N: Number of timestamps @@ -90,92 +91,6 @@ def resid_mask(ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file), num return good_acts -# def wfs_error2(resids, ints, wfs_map=read_map(wfs_file), act_map=read_map(act_file)): -# """ Calculates the variance in the wavefront (microns^2) produced by the WFS """ -# # Get residual mask for brightest sub-aps -# rmask = resid_mask(resids, ints) - -# # Square residuals -# sig2_resid = np.mean(resids**2, axis=0) -# # Load into actuator grid -# resid_grid = act_map.astype(float) -# resid_grid[np.where(resid_grid==1)] = sig2_resid - -# # Mask out actuators next to dimmer apertures -# sig2_masked = resid_grid * rmask -# # Sum values and return -# return np.sum(sig2_masked) - -# def tt_error2(tt_resids): ### TODO: fix this so it's not unitless -# """ Calculate mean tip-tilt residual variance in microns """ -# wvln = 0.658 # wavelength (microns) -# D = 10.5 * 1e6 # telescope diameter (microns) - -# # Magnitude of residuals -# sig_alpha2 = (tt_resids[:,0] + tt_resids[:,1])**2 # x + y TT residual variance -# sig_w2 = (np.pi*D/wvln/2.0)**2 * sig_alpha2 # TT resids in microns^2 - -# return np.mean(sig_w2) - -# def total_WFE(telem_data): -# resids = telem_data.a.residualwavefront[0][:, :RESID_CUTOFF] * 0.6 # WFS resids, microns, Nx349 -# ints = telem_data.a.subapintensity[0] # Sub-ap intensities, Nx304 -# tt_resids = telem_data.a.residualwavefront[0][:, TT_IDXS] * np.pi / (180*3600) # TT resids, radians, Nx2 -# defocus = [0] #telem_data.a.residualwavefront[0][:, DEFOCUS] # Defocus resids, microns, Nx1 - -# return wfs_error2(resids, ints) + tt_error2(tt_resids) + np.mean(defocus**2) - -# def mask_residuals_old(telem_data, num_aps=236, wfs_file=wfs_file, act_file=act_file): -# """ Really freakin complicated array logic to mask out all invalid actuators """ -# # Get data -# resids = telem_data.a.residualwavefront[0][:, :349]*0.6 # microns -# intensities = telem_data.a.subapintensity[0] # ADU -# N = intensities.shape[0] -# # Get hardware maps -# wfs_map = read_map(wfs_file) -# act_map = read_map(act_file) - -# # Get X and Y values of sub-aps and replicate them for all timestamps -# wfs_x, wfs_y = np.where(wfs_map==1) -# wfs_x, wfs_y = np.tile(wfs_x, (N,1)), np.tile(wfs_y, (N,1)) - -# # Get valid indices for each timestep -# idxs = np.flip(np.argsort(intensities, axis=1), axis=1)[:,:num_aps] -# valid_x = np.take_along_axis(wfs_x, idxs, axis=1) -# valid_y = np.take_along_axis(wfs_y, idxs, axis=1) -# valid_z = np.tile(np.arange(N), (num_aps,1)).T - -# # Put 1s at each valid index -# valid_saps = np.zeros((N, 20, 20), int) -# valid_saps[valid_z, valid_x, valid_y] = 1 # TODO: flip this back -# # Pad each sheet (timestamp) with zeros at the edges -# check = valid_saps.reshape(N, 20*20).sum(axis=1) -# if any(check!=236): -# print("Shape mismatch in valid sub-ap array") -# valid_saps = np.pad(valid_saps, ((0,0),(1,1),(1,1))) - -# # Get (potentially)valid actuators for sub-aps -# valid_acts = (valid_saps[:,1:,1:]|valid_saps[:,1:,:-1]| -# valid_saps[:,:-1,:-1]|valid_saps[:,:-1,1:]) # 4 corners of sub-aps -# # Multiply by actuator map to remove any non-actuator positions -# valid_acts = valid_acts * np.tile(act_map, (N,1,1)) - -# # Get number of valid actuators in each frame (can vary due to edge cases) -# valid_act_nums = valid_acts.reshape(N,21*21).sum(axis=1) - -# # Map residuals to actuator positions -# resid_vals = np.tile(act_map, (N,1,1)).astype(float) -# resid_vals[np.where(resid_vals==1)] = resids.flatten() - -# # Mask out invalid actuators -# valid_acts = valid_acts * resid_vals -# rms_resids = valid_acts.reshape(N, 21*21) - -# # Take the RMS residual for each frame -# rms_resids = np.sqrt((rms_resids**2).sum(axis=1)/valid_act_nums) - -# return rms_resids - def tt2um(tt_as): """ Calculates TT residuals in microns from tt_x and tt_y in arcsec """ D = 10.5e6 # telescope diameter in microns @@ -206,10 +121,12 @@ def rms_acts(act_resids, ints): ######### Processing Files ############# ######################################## -# Do some telemetry files need to be cropped around the observation? - def get_mjd(telem): - """ Validates a telemetry file against an MJD value """ + """ + Validates a telemetry file against an MJD value. + telem: structure returned from readsav + returns: MJD of telemetry file start + """ # Get timestamp tstamp = telem.tstamp_str_start.decode('utf-8') # Convert to MJD @@ -219,7 +136,13 @@ def get_mjd(telem): return telem_mjd def extract_telem(file, data, idx, check_mjd=None): - """ Extracts telemetry values from a file to a dataframe """ + """ + Extracts telemetry values from a file to a dataframe. + file: file path to telemetry, as string + data: dataframe to load new telemetry into + idx: index in dataframe receive new telemetry data + check_mjd: MJD to match with telemetry file (no matching if none) + """ # Read IDL file telem = readsav(file) @@ -262,7 +185,13 @@ def extract_telem(file, data, idx, check_mjd=None): return True def from_nirc2(mjds, nirc2_filenames, telem_dir): - """ Gets a table of telemetry information from a set of mjds and file numbers """ + """ + Gets a table of telemetry information from a set of mjds and NIRC2 filenames. + mjds: NIRC2 mjds to match to telemetry files + nirc2_filenames: NIRC2 filenames to match to telemetry files + telem_dir: path to directory containing telemetry files + returns: dataframe of telemetry, in same order as NIRC2 MJDs passed + """ N = len(mjds) # number of data points # Get file numbers filenums = nirc2_filenames.str.extract(filenum_match, expand=False) diff --git a/katan/temperature.py b/katan/temperature.py index 0614920..654b0e6 100644 --- a/katan/temperature.py +++ b/katan/temperature.py @@ -74,34 +74,34 @@ } } -def get_columns(): - all_data = search_files() - for name, data_files in all_data.items(): - columns = set() - for i,file in enumerate(data_files): - try: - df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, - na_values=['***'], error_bad_lines=False, warn_bad_lines=False, - ).replace('"', regex=True) - except: - if verbose: - print(f"Warning: read failed for file {file}") - continue +# def get_columns(): +# all_data = search_files() +# for name, data_files in all_data.items(): +# columns = set() +# for i,file in enumerate(data_files): +# try: +# df = pd.read_csv(file, header=1, skiprows=[2], quoting=3, skipinitialspace=True, +# na_values=['***'], error_bad_lines=False, warn_bad_lines=False, +# ).replace('"', regex=True) +# except: +# if verbose: +# print(f"Warning: read failed for file {file}") +# continue - if len(df.columns)==1: # no header - if verbose: - print(f"Skipping file {file}, no header") - continue # skip for now - df.columns = [col.replace('"', '').strip() for col in df.columns] - if "UNIXDate" not in df.columns: - if verbose: - print(f"Skipping file {file}, columns not as expected") - continue # skip for now - for col in df.columns: - columns.add(col) - all_data[name] = columns +# if len(df.columns)==1: # no header +# if verbose: +# print(f"Skipping file {file}, no header") +# continue # skip for now +# df.columns = [col.replace('"', '').strip() for col in df.columns] +# if "UNIXDate" not in df.columns: +# if verbose: +# print(f"Skipping file {file}, columns not as expected") +# continue # skip for now +# for col in df.columns: +# columns.add(col) +# all_data[name] = columns - return all_data +# return all_data # def search_files(file_pats=file_pats): @@ -117,7 +117,12 @@ def get_columns(): # return all_filenames def collect_data(data_files, col_dict=col_dict): - """ Takes a list or dict w/lists of file names and reads them into a pandas dataframe """ + """ + Takes a list of file names and reads them into a pandas dataframe. + data_files: list of filenames + col_dict: dictionary of columns to rename + returns: dataframe of temperature data from files + """ if isinstance(data_files, dict): # Return dict w/dataframes new_files = {} for name, files in data_files.items(): @@ -158,7 +163,9 @@ def collect_data(data_files, col_dict=col_dict): return data def parse_dates(data, date_cols={'HST': ['HSTdate', 'HSTtime'], 'UNIX': ['UNIXDate', 'UNIXTime']}): - """ Parses specified date and time columns and returns a cleaned data table """ + """ + Parses specified date and time columns and returns a cleaned data table. + """ new_data = data.copy() for label,cols in date_cols.items(): date_col, time_col = cols @@ -190,42 +197,42 @@ def clean_data(data, data_cols=None, non_numeric=['HST', 'UNIX']): return new_data -def to_fits(data, filename, str_cols=['HST', 'UNIX']): - """ Writes a FITS file from the given temperature array """ - fits_data = data.copy() - for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: - if col in fits_data.columns: - fits_data = fits_data.drop(columns=[col]) - for col in str_cols: - if col in fits_data.columns: - fits_data[col] = fits_data[col].astype(str) - # Assuming the data columns are already numeric - fits_data = Table.from_pandas(fits_data) - fits_data.write(filename) +# def to_fits(data, filename, str_cols=['HST', 'UNIX']): +# """ Writes a FITS file from the given temperature array """ +# fits_data = data.copy() +# for col in ['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']: +# if col in fits_data.columns: +# fits_data = fits_data.drop(columns=[col]) +# for col in str_cols: +# if col in fits_data.columns: +# fits_data[col] = fits_data[col].astype(str) +# # Assuming the data columns are already numeric +# fits_data = Table.from_pandas(fits_data) +# fits_data.write(filename) - return +# return -def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): - """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ - data = Table.read(filename).to_pandas() +# def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInvalram', 'k0:met:GEunitSvcAlarm']): +# """ Reads in a fits file, converts to pandas, and parses date columns (if specified) """ +# data = Table.read(filename).to_pandas() - # Fix NaNs, because astropy is dumb sometimes - data[data==1e+20] = np.nan +# # Fix NaNs, because astropy is dumb sometimes +# data[data==1e+20] = np.nan - if date_cols is None: return data +# if date_cols is None: return data - for col in date_cols: - if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings - data[col] = data[col].str.decode("utf-8") - data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') +# for col in date_cols: +# if isinstance(data[col][0], bytes): # Cast bytes to utf-8 strings +# data[col] = data[col].str.decode("utf-8") +# data[col] = pd.to_datetime(data[col], exact=False, errors='coerce') - if str_cols is None: return data +# if str_cols is None: return data - for col in str_cols: - if col in data.columns and isinstance(data[col][0], bytes): - data[col] = data[col].str.decode("utf-8") +# for col in str_cols: +# if col in data.columns and isinstance(data[col][0], bytes): +# data[col] = data[col].str.decode("utf-8") - return data +# return data # def combine_and_save(file_pats=file_pats, location=temp_dir, filename=None): # """ @@ -248,7 +255,13 @@ def from_fits(filename, date_cols=['HST', 'UNIX'], str_cols=['k0:met:GEUnitInval # return def from_mjds(mjds, dtype, data_dir): - """ Gets temp data of input type from the specified MJDs """ + """ + Gets temp data of input type from the specified MJDs. + mjds: list of Modified Julian Dates to pull data from + dtype: k2AO, k2L4, or k2ENV file type + data_dir: path to directory containing temperature data + returns: dataframe of relevant temperature data + """ # Get pd datetimes in HST dts = times.mjd_to_dt(mjds, zone='hst') # Format list @@ -272,7 +285,4 @@ def from_mjds(mjds, dtype, data_dir): data.drop(columns=["HSTdate", "HSTtime", "UNIXDate", "UNIXTime", "datetime", 'mjdSec'], inplace=True, errors='ignore') - return data - -if __name__=='__main__': - pass \ No newline at end of file + return data \ No newline at end of file diff --git a/katan/times.py b/katan/times.py index 4974d2c..fe42060 100644 --- a/katan/times.py +++ b/katan/times.py @@ -15,7 +15,12 @@ ### Conversion / utility functions def mjd_to_dt(mjds, zone='utc'): - """ Converts Modified Julian Date(s) to HST or UTC date(s) """ + """ + Converts MJDs to HST or UTC date. + mjds: list or series of Modified Julian Dates + zone: time zone ('utc' or 'hst') to return + returns: datetime objects in the given time zone + """ # Convert mjds -> astropy times -> datetimes dts = Time(mjds, format='mjd', scale='utc').to_datetime() # Convert to pandas @@ -25,7 +30,13 @@ def mjd_to_dt(mjds, zone='utc'): return dts def table_to_mjd(table, columns, zone='utc'): - """ Converts date and time columns to mjds """ + """ + Converts any date and time columns in a table to mjds. + table: dataframe containing dates and times + columns: date or time column labels + zone: time zone of the dates/times + returns: Modified Julian Dates of the dates/times in table + """ # Safety check for list of columns if not isinstance(columns, str): columns = [col for col in columns if col in table.columns] @@ -45,12 +56,21 @@ def table_to_mjd(table, columns, zone='utc'): return np.ma.getdata(times.mjd) def str_to_mjd(datestrings, fmt): - """ Converts astropy-formatted date/time strings (in UTC) to MJD values """ + """ + Converts astropy-formatted date/time strings (in UTC) to MJD values. + datestrings: dates to convert, as strings + fmt: format specifier for the input datestrings + returns: an array of MJDs for the given datestrings + """ # Get astropy times times = Time(datestrings, format=fmt, scale='utc') # Convert to mjd return times.mjd def mjd_to_yr(mjds): - """ Converts MJD to Decimal Year """ + """ + Converts MJD to Decimal Year. + mjds: a list or series of Modified Julian Dates + returns: decimal years correpsonding to the given MJDs + """ return Time(mjds, format='mjd', scale='utc').decimalyear \ No newline at end of file