From 020dea06adb64cd3124b84bee55c938437f0f104 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 8 Dec 2023 17:09:29 +0100 Subject: [PATCH] Added working skeleton code for the TFS-specific TIFF parser, which fishes successfully metadata and converts into standardized python dict, ods document added to collect IKZ feedback how to map specific TFS using assumptions onto specific concepts in NXem to consume in e.g. OASIS --- image_tiff_tfs_to_nexus.ods | Bin 0 -> 17720 bytes imgs.batch.sh | 2 +- imgs.dev.ipynb | 7 +- .../readers/em/subparsers/image_tiff_tfs.py | 130 ++++++++++++++++++ .../em/subparsers/image_tiff_tfs_cfg.py | 62 +++++++++ .../readers/em/subparsers/nxs_imgs.py | 10 +- .../readers/em/utils/image_utils.py | 40 ++++++ 7 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 image_tiff_tfs_to_nexus.ods create mode 100644 pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py create mode 100644 pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py create mode 100644 pynxtools/dataconverter/readers/em/utils/image_utils.py diff --git a/image_tiff_tfs_to_nexus.ods b/image_tiff_tfs_to_nexus.ods new file mode 100644 index 0000000000000000000000000000000000000000..f3fc491b4a8bac27c6a828b0b0de8a3732fc185e GIT binary patch literal 17720 zcmb8X1z23kwl<8r6C8pBcXxLU8r-3AcXxuj1rP2P+#P}ich?XsxXa&UCTHfHJLmiF zEuMy=s+QK;?_RsRRx8MWL%ab2fdK(Ibq`eww&Dt>2LS}Uo6I4S%a&wTBeK^%Ax zkk=PBkf(~7v$dg(fw`3-+d4eP!Jd}V2{15 z!U+TfZbM2`NX0GVIP=({oD)IRyX^W&kE>O~Mh;(!-GqO2imMYl#(;JTM#1QOe12XH zTwyy~9vNig?K9lXc(%Y`j|bSZZPIs>!y;R|CyJhHwV3aSXXJYn+i!W!7ScHgZjPTU zyuWMZ{WOXWdEOp|A{-=)ZOs1kn2fV*W;r-GdET~MGvN!F63T!vn$50l)_HT=7W1R? zOSh>(gL|mSxrm8mUd0{CJzbU4yKVlS7&|#;#>n`IRbKDFWnoQk$ZI8HQkB{v9PjK? zY(_Pt4oA;1z`7_0Is9ArJdt^;3OzuD?Y+&j_^L zMn-3|p=Nw4_UL*B!$vQpFpFA(crz`a(^aG?86!8=Zn^1J;0)$KeOhw|>m*Pr5xN5?53k48n~__>)YQ7Bl5^4lNE&{+jxh7E{ddmyDy z=q2M9{mgjLvQp5X`mDLUWTKp$03`rH>}YHzqBm5=g0f#djLFHHK{wCLF~Ei6MxjLU zJW&!{Mf@QWdj-*g;Nyw3V@Yd4g)(WF?hr$>CM{DK7j`I31tU%Pc0wzW>JdWAC5(ia z@lc-OLgwiUa*9EjG@xK9L;Oi$et8TQO0;dOhEb1Z@ni}NoP?c7mbgbgkCDONLk;2- z3n4TW)S%_+fp{HO8Xh4OeHXxO#^MjkG?KPSCX}5YcU9Eo*EC_htL29LmpTyOB)Vtn z>a0XroU0L118g@n1 zd3n~U_Bs5`N5#`(LCawn10Bd3pA6xi;}84r>QBP^yJ=Bfrc|UWdvG=05BvEHIz#Ve z={BP+p2ZWN;Xr3k#OXHgO#xo5=BZx6X#$cyo6$D;Z#%oUFpmRwu;}w&B>`P^m)hb; zTh9B5qHl{7KXzNINj&k1v(rcq zMW2E1QQ(elFnMhut!}*i?kAn?w1K;V00@(XJqY0e43bYREG;=WW0FkAS_a~P{1Seq z66Riw4k`m(HQeIuU&4p11Xpy5lo1+m$q+(^mMz5q>We)d zNVe@0D-0DR)y21|c7biX9&axoYyzH(D6IZsF*WEL*R zuEbC1Do?2;$riZv1v7kSucK?ou~CR%|1^ON0j58$Hv`CJ3js!NfIUlNnYm@ln4PEx zyb2Kt9~0-(UNzkiV(1`1Vqo>vEnZN4q!8%jIIbM8krn9OZT?La{olhuM6d@1Dm`2e3Y9`%M^dEU=Vk+#GB z`#yExzZu=ceO~qx#sB7)!8xnR%Rl3L%waZ_<5C(x#z{Isnc`-w7ih-QL!QM5Z?iT{ z$vQ!6K1rylb3s)iA)maf1x5WrQZ(p2ZlfV(A*mLVZDu$uh}O&P4R*>Q_91FNZ;P@)S{P->{v zEZ8XPvD&+zqaP4UlfZ!&6EK9u-^)pufNymn&&Fijr{DvlFi^M9B$oWtS_DMi0X#QO zA|Zao?=k9j7X>%16^E^>FcEw$V$} zTP$Q#e_Nb?34(2+N+}po8sTC|fp|{a6fp`K*>U~WbGYxUe?bOu2dx+ezG!~SN9KY_z7$jkY7qqydc)a`pOkUgv zJU8maR>hN64;cKCz%a=m>={s)=BIgxy1~LQyLozzL36H)0hv{HgyvTy?#uZQ_~)Jh zae-M#gB<0d&7|Jpfv_Cj3mvVOw5gSxgheut4eGh1g-a@GVl_$ zUSwY2f`9?uAOwG7gOrUR>uy|c(5w6B&&3NY6^Mb6V@TbqGAIh1!!O2Hseb>80N}Y{ zk%Z(xIpQM6DREVlvxuQe-6jG5QmAhjeQR&_t>$d|7vQdHvNB%q7ii^az#pH;`hb&o zZBBJp#4QEj1Mf1^yq` zVn^SzT2*eq8ytA~Dk@oZ#&q%1JDxFiA3-FP4mB|pB!f;ym`|qfYmsYNPG1z~{71$-zmHhmR3P}Rypw#L|U3O1xo zCwm_Y$Odk|q0<_2XQnSN3;{IZ^Nwrds+z+LGm$v$M{T=tEBHUb3LB*t1M5M1iQlBJ zw#(1$tKaFWSBRATM~mK7eZfcCb!$Oc*5fi;Z>88}qX$1w`y$XuA5%+*@9Eus)yK!- zpax6c2bo8dkzo>De(Lvb`hBYmqm3(v3svy0exPS#GKesh06kFWm6SjWJy2kNaSqQ> z9;-bZka5UC+G&SqaJJ5{&h2ZGXeZ9KnI?R7OuW>fiW(?KNuix>k6KK!8)SJIZXs4e zwsOY{IsF#bsv7n#_+bMP^TyOOUau+kHQy-z=BLm%({_?x3*vd6VE@n)+kAzY_7gBvy0E!sQ4M8w)_5)2|G_FlA7SU=#8bQ%T*=2wdTI`g|B06UHVCns|o zQ%8C?YpY!i8Rcb3l#cIeO->v>AmpkBMUm)W8tgf=z2wbN1r2=tW}0j)U+jIJqDdo| zh2ey!jFCOTELrs$AI`3NzZ1Uoo?G&ZeRKEqfWWx|(+-PoSnb`RjnATwtv7Zj&){x+ zi@xXnM}E86n4uKrVb3b4|5Rqvwo)IK7fIum65qI|V#Z|M71f%Dk$_hmwQIiEXYX?imJ6t6fuV(VXU#iFpGbBZnD z`0~6ha>yMo6@0;O(bXmq`}SFS)FAgtusKRP5HD2WcxXtH(wfH3UBf;6rE{`JY6{GE zxz6Q9eJc%c5r4p3W>qsr;94#ns!g^&-wABnfy=#aTQf>}>xdgn!NR z7fC-f1H{LwVp$$z??ce&-TPg(x?Cwadq9 z)K_TyZkGdmC{Q8xYqzZi>I%`BNt$kKZD!aPQFgm*D}jzPDiL!M-^$&=r=9@~bj71i zO_FD@Zi;v`A3O)OU{cdPbKwb6n1z$~0k@KG%(Li9|QvaS`Z9$>N! zYbqGHm}#KE1}4A{hb?GiPATYJg*=KkK@8|@phBhy-{fsT%f76)=}_eKP#M^mL9KZq z-irWc6jE>BWex-}t)7d4-a%q1_vq*s@>-W;F)p93Dm=#;Yn`bc-gZ{bRwb7Y;5}?% zoRNh?Uh!_ow`uztc!}InlQJ}#JoJN}h9`mdO8S5vWGdSyNe0oEEE>~|A7fSLYKQ1t z9h`=FcDCy^*3-nr?S1E+4y;Z+TQ$3|b)#l(V<@RTpu%V8X7KrR^ybN!=NU^u1{$Vs zo+I}K90cT&00iW}0|)R?9K^`h#_9FJ9f%vpnOcs^oXLahH_F9rAC!6OX(DbSidL1Q z8IF$zeyALgvP`of*Fnf+6iOmWElT;`L%PMeam7Q%ha`*g?~s6y+9SO{_+Nl`Wk270 zpCDv~5iVqT`jBM7QGjnnw}&0SAD8doT|tOS=zM-86mXZ9zy6dV*LLRce#MUSdL{Om z=d(m>w?>Ai&zH+pmT|bFjnVo&Qcn0m)3t!6ABv`nQ>K&kTZu%En@3)5Dx989k{uzf zZ%2psqU+zfb~*q@F8dmFUHCpF6}%sN`@D1PaeC8*!|y(GcoB@T6s+mxYISpSk)VN1 z|4lqEB_(AV>u%8aZXo0`ka;u&LJ5+nmGc=4?gPU(?vQf9}R0;=p< zWUgGkq|e_fYlO?3)Ufzr%}_1q(f4d9hTkSk&tEv1I`heMG`aKEK4v;&VtkVk%Q~XR zaF6G6u?en4yqU*dogCe-YDKoxxx3J*>L1?IX1hM_|7J2!P&??E)mTc+L*%oNcOV_` zIdW?~b&5eNqVv{8@=^BeY;MbF1gDDkWQ@3I(qSt!89UyT{BBmH44Zz*gu{H+*`nNnSqb;~P|rngl2Mk6t=Es$cva{0U`JEO z>28PVTxDNo(1@vnukV&3CqmqAtrAjRM(Znj)pL+{=rE3^+ z$-?xy3ucP=-6lji2V;^*&pa5S|&o9_(xOkU-ClDy00nczd!VR#qzd5__msG`PFL6PDl%CD})^uN`7te-R#59UJ2o4dk2Re^B=Uhxa|IniPd{A_ENO_9A&(>}4X^r-*+tDyofw6k!sf30A z4yo821LJsh^`?ze#VpR=Wjlw=2sts*pkvWdyc9LO#3S#$dE!%ZWmn7r^b1k)I4uoM zEq~f|0rA5dckAv*7l=RtYI`U(Lz~u-*cnbC!GSEE#a#&STZlQYrcb7i)$I4P59M%G zE8kgDq7W+Vuod)bu(6L12zddn-2ca5x5#$g;pR++2U?+H7 ztUKh4RG#9V)CJ7IO@8;5-AB>YT*cMAGmGRbFwI_2=03_{sMDq=_~1bgZ&tW?E>&2$ z7+!I3w(pKqqg~um=&nSY%%yeGMaSU!rKDB{pJGfyoxjZa(#W#+D-w0vy`GmPD%+}+ z@u*LE2-?xNq;;h765pCtSnJ)q4&Gy8H=hnb6w+mKjtF$sk7b(l{;817(VOJyMMl&( z8NO^hkwoZ)x(3Df`Y%F`q6Vnq=08|me1x~=XBMxSFrKCgp`xo{ zWroAv^cOlUke4@xn#3C@%U(!wXcW3gz+Bc|l7xR#gON@4$wPYJGw@xgU+EiE6{x ziIp`*YKl${bgqqk<>gNjAZt`tD5iyUbe7Ym3XN6GEaxcr1Ym_nVzSqpcje8u4`V6j z)^~94V&%Y>R`KH!Lj#f)z2+)-NGPmv)#sg*W;l3pf!&SQt|=F=3&%PG>?(8PQvBe2 zuY{~1l^4~$O1eFn{-~b!DOkQ-it$|#684tA=cOxiXv~w{_!skR&i5l8l;LmthTzwA zwNiN0&|bihVtR(9nut2EYC>|-S(olI(0aH4L5|cg-&Ih*m9%@jpW~s<7{Af=OQ`v3 zyxDT8O`~bXC9`z#Ttg&#t|G6j`biF3MPY2zgb$@yBa2qGy4!l@L-lG=ar#Mjzz3p}R1&b~fBv*P;iRC^H!u zwV7e@N(_qH)2*qRZ}M$LwHG%WFIUSyf*;2(1fD#91p9`)Jjkuds`y?bLcZi8!{m9+KafaB0-d)m{c;n3XJ zFoSp^Sdni0@SY{+Zv3zwNi}<)^wP=UOmp1}!viB)qYs*DEUjm1hI5QCCTcWw29K%&qVXD1 zlm~fMVjsu|jQE$kX`Y6*Rd>Lj7%HjRkUm)WQP+Ev8bg5%TP>XI%?_C`#dil^)NF6% z_vX}a2%UiL?4dUP>=sS_mA#XfvL;^egv!c;t;32hdee(Z z0XyaOR%oi4({ZsuI;w}=`~&MJGqqT?8XkK#DmeVy{NRz&{4h4-RBR0;lZ%0QtnUfi z8U|}q7wFR}Vv3Va#W$k$dXAGDiNfTizllrwVg?h(Io7*TxVh)d9$v9X;ksnNd(!& zs7KwDYs#Cvnl`JknQRwwRE@^4 z^J!>ZfL_aIa5PwQ5r*Os9VUc~g`#AjmQALLhSIRep!M4cR!tSAPo&QF=x2w_grbz9 zmer)y)kH?oa^yzgt+(-a3LyOax$dl*eUro1f8u%0h>IG)YRD#?;KJPNMhwU~T&&)N z33OM)_9C3<ZTfuvp|ff{mR0{48V@tW844LJiL?@3Zlb0}JUP&BBNtnTVy0K_ z0u8Z&l4!_m@kAD+vw7k(Jr2mV8bIouq8lvmySmoZk%+U-IKYrd7-=Y)R6QjsTlI=2 zyM+_O@W4aL$XO$evBr595o~~v5*_xNRP_@Bb}LS_rZ?{)nQD7No)5}Zw#zohs}kH$ z;vk4gN-#+@moD`1Vt&Fs4-=(PxiH#d^KaZgS zLtd;KXB>n*iPX;pX*d_PC5u3tPLg$>w{PAFQHqy-H2j1v(WrN!OTcD9uXK?MCv4IP zdTklA7He&B;j?MbiRu115<&>41UE6P@9PoGeyu@5jjE#YoV?SA^c=oSt(@;Kh-x{^ z$k}nk{O{DZS!(drR56{y4fx|VLwcyJMVzISF`TW%+3OE+pLxD~bmxplu*Yx49UPLm z`UzAy|IY19N+=dmDq&aq6SLe=alm5ziRra+=qlU=5h0E+zXRJ2IxO(xLZNQTJtHd4 zyyeO=jMnzuB?zPy$Vyz;;fOB_Yq_7+%P=Zj85>r{bsxJ`jN8v+iEv1S9N5s_xPmjg zV0yxE((kiH6fE92T1uZxG*ZO7`^6%F5;xppb5)&2 zn&PzuVvJhMt#SC=qh%%=X+tj?F;{GU6N{Ewv zB--g3i4@`B+|`8QWz^Q@)0n{_J(MWHilvI54xLssS85AlCuN*$$-+CRWjFol;UIJp zEhfn3^lcOusLiV&n|&>NmOdEM9JXRq#{BTpn=TnpMMF$ZcKtqmqxt!vzU4u#&mAz` zD}0n$`Q~j_t{|^i$$;{?p={JDsVami>nR&02CL@)h^?sb1mW zZ22&Fy$Xy)iSuXGfp58P_S&c9boMhPU#}@3*e#aJ_>8qa+yov6rMzvgH)3bI@jj(F zcS1>mjwwNTt9-ao^QBp=B9D9LYvYyGhh{YaGj~s~?WhzX_?BH_6Kc`X8ScJvGb_dV z&~N)}%S|SoI`u^AZ^H|P-^f*nOgbdnC|GXW>=zld_vgM#f{rXf!B#%>V{)5Sk~;MTrag++(CW>NKhk_D-+8W&UG7?#ECVNV6St80j#Y}C$IYZ!An(OR7#u$XNQO7 z4lJe~b#-y3UaGxF_4uG)B7~@sEtH67^5=x)G(=>A2nS&$ZMENpNJJrRPcncu*MV8B zo>!A$7|K`>G$bylCt6(;atIjv2xOxRxK&_z_T9HpTWw)GjF}McI4*34O5 z&dN}#L=G?dwwqW;d33Zq`h!xDSj6hUxjrFc55Fee5qc_2_%(>;oY}as7z-OQ%RGI; zg8Ak%xJDE2sYKvLj4JeVA9+BJ^_1NVS8kzs;$G;Ny7wd3EMKhLtK*>z$SROj!{s0V z%lVt$@|bG*$e3TG-6s;Ea;Peh(xj~*087O`Y?<9P;$*2>BSpEAM&C2M3;TKa?n}OV zaT1gwO$AcDv{ew4@TKmk#^9vsfBL5;aXa2S~4-!cVm?0er<(D{WDcaX_Fz z#+oLkFPvXmL%#hc$7K2f@De0!G56-v!_#Y-bZ<&UvS(>W;t1q-RvzU7LA}Wgnd5iK zyAWg{2r%&=FoW_wF8nkNe~kj+*^X~i^>f}mG!+RINWm0AOFLX=UW@9k@ixg>waAf2 z-;r8NW3}pHgPEWQ;6b3RT3w&7!>Xo#Z6h{`B3nDr8in3k`s4BZ3f}d8SiJtqc}!&5 zDYWp1vl*`b&ufe)xz>DSh@XK+0(X-c)W#(jD4YZ&b{+&^DZmhMy(qlVF=Ea1dB>8% zN&GA^;`A1&NCf|VRqrMcRIkFf1AMN8`Stxe-Gsh(8*&HrUvjI{vZ29Y9=%KjZ+NA7HH7F}ag=g%^L3q? z#Z537)Sp&lf1z=BU(URY;2zr@Oaqkn%~wClSj8wXMS*BTaK{b;Hk-?{NK9NC-Gx1o zg=2W+FHA(nMv?@Udbkgd-|0s}S7UE0cQVZV^u?~FdOv#r6pR|$W$hPzLA%H&)>lHwFmWkqISBGNTf*(Qr~m{-LljWT>r zKR&hoR7h)Pi5{gqz0zpip@nrrS5w4dENXa4ga$55D$XYQ1=HS&ob=}a!Nj4b{H6Ed z9S6HTx&ITiOGr>i88(UYpk{w}Tkcu3(w`UMWH*)#3v20f*EcO74 zbCh@JB;qFZ1@dKPI>w|Z{gz8^<#gUG?OV~(R|7@;gcC-N!r}Y`G`_uvu_e-Tc%e__P$ob5~ zk^oih{JP-I{CeJ}wRB_R)``aH$>?;xI zN-=xEVQA8B4p_>p8%uu}BVUr;<#q*Lshdj{Id|!HYZpiLpFt@pr~kmwcS6V)iDhh6 z?Q6%klx3+V8wGy>F6(hV)$7)VQQmgjb|YQ6Y_YutK1udH2{Y~! z5A7=Wg5@J3KZ(2Gx!7}fKIQ2Ir7CW`ZGXQKPUbUsvh54iP#`A5c_XQqKQwF2RID3K zWyo;xB@6A^b6`9)=`6HyNx1f?Bt&YQy1kr=cmi^@N_yS%ej}Oy>4$f|DoSsfyW(L2 z$ZEFkOUiu+Irdv#zU=rHI|~imYOP0*U@4&q<&&z4NSDx}i;Q?!uYtGgpFuo~UHBHk zs=1(Oo?xf1M`(viL7*n6+gtXjXF+<(ocCWqA2i9O)K4`<6*#NTV{K3PkWaGJu#NR+ zh{ke-VKjuOmO9{UisZhPcjo0Ho6ca`5c&$h7kuQbaHb z7XHMZYRmjnWVf4d$vx2HMWYa!I(_=lx3*M5=2I!UK zd^X0r23gV$9JL9%l$aNbSZm!CHb)Dc%Gac%V{=L{L;kF{9Kp`@l*!g!w_-|jD_hR) za`%ReK7iaGLatP7VwgN7DDnV{ z`m^L_&P)Uc?&dWx_Jxx%VS&1jZ84S;)4t~crwKo!q);|~dXhRC@Z$8jInYI>a5Uxv zSZ!GeMV|MYm$Pe(RF4CJ?;In-Tih%ZA#fP~%X;|YxQw1%aukwuE^4#H%953E>t{UM^R_%hJ)g}l7_5;4;lg*LyY zYXt>a#d$~Kxer@!+r^iY&KWST4}KI-@q3 zupHurc1YDk-iLiu5|P;wJ{7!ps@C%4#^dd*j5JC8aP$3Asn_ zXbQ1dw?f}2eprqhHMV;8N2|>WcQN6f!g{EMD!JeYGD;#Dg}&%^b~V?c3X>zF9b_vi zJIH+uM!_l-_=zff?r=ItZ9a;OH={R)pU6tZ1wCyZACYXm}9fIU|gJMHDyCJ?u=6i+Yb;Du7r^He^Qq)2W zC}bjo%}hMLe+r@3PJ)juCL&TS1C3va=pfP|MZWa*!@LI}ML1K_F4j&Ig`^78wC;!W z+#Y1#^8}4r_Wp{R@ieuQdAxM7`*;wXwbv~k6HunTA`aOcr5UxXK#UnRR`O;NeUL@u zz;F~(+Py<=!n|r2QdR}!Q?ZT~R$Q}yM8D?;dfuD>H{ptp6wLH=sOt`Ey6rpHfSTmBHuNgUDyB$5Hx-3P*bS+yHISiUG0V^D(UsxqUPKgE^}nj%l|#wO$@P@BWJ6 zV09Ign&Md$8I3xcyhbX#UX~|2^Ip3jQ|SzC1w40xD;onY!dpf$DXjEPx&Hg2=-lkO z)a!+FYyF`2c2e5`Xt*$BbruS_NAsvGZt2?+h@KDz)phM2I*KQ~qZHkvj^q^xKw7sP z|5M>ICAV7@ zw@DaqC3Wj8(xm{#n$U|MtU$z#%5V)WnefNXsFekE^D%~vnGN2}I z>xsh1JN=k|a;tJC&)=><;!^r7zpjehb<-WwO&A#Kk_%m@|E5(_07+hPo!i4s6?tYQ zeS)PRy8!QT)BWPML^`|fES`LNa4btgXtg`f8Zs~f@~roYYWuy%OA`RvW2U*2U*|qQ zYSovGp0?0mUAF9FKl8Zan(?Wk$b~d`dfxu$GxTfJeb&GyNyHQgDeUkb1x7;AOadN47%(iZX+ej=5ARHa)rN<+q8JDevM=*s zwdr15EPctlJwsaNU9?h13hcX?5rY7lZ0gCAH2NIHmfA0uaX5DFlTmURQt%3*-+PR? zLx(OcXd=HbeQF!C%YYhVXoujCIi;cbSRq8uSQM!Lz6xqj-&;()lf0*pZ@R2{X7JM2 zwgP9T$AUX__R>O~Za#2Sy)7puY&${ME4^=C8vJYjs8ej}kJ)QLU|oIloyf)TS^4|c zh5$msqaqYfgdE2>HUd)1mMNxhX=@n`nsMV&U6@+%+CkFLut~a63I%To3JQ}m`Gik< z&iX`gZG3Fn8PgU#T)kam)*$Rbi3uQtTjpxWDLn(1EI!(3*sWzY)Rwe<6Qg&E)!v`| zDmK@c*EV^Ih@InJsJ2Ot;d6!W0Nv+OUu_L)*PGm*ASjtf+!xs~_z{tD$5sXY4*Xm*z5PQl*hC(S6 z4A&g50jFX(y4`?NZwZI2Lf-OfSEg($Uhc?MerRX86siRIGcZM9ml*;v02=;wE zEIe+5cr|1wA|r83Mzs5J`deFuxR`l+N&ziiGlelzA)cG)mS~KD^I0S|3FT)cq4o;ZP;V;v0oa z0pKqp7l4B!u%3jEgo&P!ga}|`WNQpO$HqtUK}DR7lSF_Yj@Q=2#M}tL{kz134p@s~ zKm;VUapeAEijTzE!G`-+F$%Yhfi=L9+sTOg?^P+>e_zf0OVA%9ZdT?tmV6{;PEK~* z3=FQWuJo=f^tKMB3`|^HTnxWvN&XaNZ1k^ncFqn~zg8I=F#xOpuXQhu3{3P)48PaD zYWTlZ>UHhkRkF3U{giu*x2e{>;Gl|a4idiDTA?r zlL4KJIl%QD3DKW^{F}p>N%((z_f-vk-e1a?+n76<8(7g90V{c&Y#sRJ3>++-9f@Rm z+sq6NjLdi${&CeGw7+P8+err$4B%&GWM-jbVy0u{RAFN0W@P4OWBZf(-`DXP8*%>w z0TUZH3o|dn@5$Fy|5c~I&HbSXz!+$SjVYb6v%{}~C4L1JCT2Mn4&^^J_#gBCPG;n8 zWCd{K2a5k??eEop4*#*%6kr1YG6%Xy9=K8Rzcz+}l^z%gtfU`o%-x9o8h+hBJv#?m z3xJUm11l2~t2Gm;u(P?9F&!5t6VsnU{^Ius|9?fiqZ7~sM<;V5qF>5585mjt=!|Tg zZJhW>SV;aDGy+%wx9pEG4i=6-M{NyXCH@{~Bw^s^{WF5NUs?V2-@j!28vgG`#$J!4 zIy$)nee-$=uWDG^F0-RNcjyrpcU6NXQ7y{hdDaBd_g1HnW8YtACR3)CiNcWrD4wo( zl=4Y5;wxMTTIfLI_itCuHhZ_v%C&XYv6>qxgq6O{`LM!!$5el!NCa5EtS)B<%Di}7 zjA6(t<0gFQ*)AoLg;e6(5%t&@-6?&m{g9+j6HS&x^AVmzjanCYknww+MV^{T>5^f9 zoHNHpLnFzJMQQDTi`SXU{V3GR^?L|@S?4+RiMr#g^OA2j%*aY_``RdLawTX_9EbVK zpE8wp;c|L68F^?;w2dh>=~KQ&vIlKLsi~516qAaRQg|?jJtv<%$82e*^X-pvq)D498TX2ZV_h3=`J@(ABl=d zav@{ndyFy$qqf?x(GcxW!9)eGxpxUP_r+qwI|Ks~wTnMWa7w~U#!R$c%hvSa=KJ#i#$BR_zr4L{Ku}-viuP?#Pdp4NBJ93bRU zIQF4xZ!*Ttu5Gxsb#8bxt!T`l+SFEavcG2EdhfP(%OY_;OD3rVZ1i{{Pd-qNlo2cT zv({8MB9oOk*5OSV-uI$l4X!-yeSu)Ax3YGxw-XY_Mt$SZz5$C`4o+1Po`au%`POUl z$Cx{nSvT}`|KcD;GBow>M4KmV@lXzn^16M0lsNeG@{>d8{TXW}bcy7d*sU8!=hI1h zGnf1bSSL0Qt4mvXyG_XoHCp@@bz@X=gjHCY2!0>wxHmLIO;>8yEaospQdgu=ajoNoXr}+EY+wc2uxL<&;$=w@m>~~{LVu) zIIs|U#O(cIcsLAuoBh*UeH6U;zVAG%EI@t98}v`l(fc`Ch)~?#T>5~uJmA#OMXtP0 z=+FD0qJXJTC<)~QJNEgDpgkKpVxQCf4aIHA-gfr-EsN*;GFZxUjHQ6cE?Znts+$}X zJ0U{A3H3me@I7{J!%Gp|cUG4(geF6Um(PJ@NblU=C!l`yCvQ>gN!ofB`sq9Vk`hbF z-7@fqHO!dH1h|!ZmRBj~#kkTST*N@$i*WTb{xzV^HUo+}NENnjT%BmMavB4y5(^7o zlQ5uOH+Bx;jkGq^u$0EW(GOxzQiXU4_#_sTCV3c9}Tg*@%S0* z!8cNnRRSL&6yjN*01Fgh(QuvKGk5TG0#Sh{Q3?mc$=lGnCEwC)Mo7+~1-+G8F#I7Q zTFTTPnqg+H?d4snTef?Qt*O3OdrROI(yV^75#~&1V9=eo7pJWLs!vN%^7aRAaKCGX zR%PPJT(3q`o0saktXW1eUu(8Iol=o84 zkNIH7Qv4G+OV-&y$-PjyB=CK|LTE3Bl`YC|Dfn`b)T7jabbC-uYn}SOD|&AKk~wbA zs@Z)c=np{8EwP*{ewTURzf@%S`E>S@Lv0UH&)j!}fSAq0(c9ZQI67)h>1~)@B8UeA z9jsZ7%*I43gr#-b;>(0(4RV^x>ZLUahu+9s-iENVsDKpcl@l+;vE>8%cF>oLtS^;# zUoYzmVZCz~)?C@%D<(A(UtQer5p%OR%kJS5jAiNX&Y9^()iIp#m`&g+R1dmC4!Cpb zK}_<(Fg(ezN^Fu%VU7rcbI9{aU;{t#P@C+xV6v4&E|5=IR~w;1uJ_+d7&mFQ0=HZi zNc1$jt;^edA+~x3dri^`J0b4NAwfU_ffcy_9phi?39Jol%uN7}PV^4OCSwU>Hi3*N zK|gte0pAt`86Y7YD2JOZvb)$mszWlIr=hKNL?@^!=9eC1&umwUv#dwgr5_vMe~tK3 z+^xcDL`60%X;%M(hP#!%xnP-lX`z9JFpc2MBCsHvy2aIonoBP^n2rO%YP>d=(k<drad8q{x}4$S4%Kc$HtUswx8DBRWEXsGgAkE8d7Gy8Ke z2oG=WKa?le2G5&$64^vYlwJ~N4y*12$xRrS2p#+5vde|~=`b#WJ6tHQ<4YDRs^ya@giMJcDz*= zSV?sIxsiny~}^Ews-~4zgix!#NzLzS^r7-_owF9YK*_y3i7|zX8b1n4LJY# zQ2iI-duAYd19SV={QNi7Z$qn+<)Nn7mWUIxh(txm%m{3 zf6E1l|9|@CFPQy<%kMpXwcs~az5?xEP4pkQ{1@2$-)~#`KQR3}=>B&TUt#yJHvJEL z{vCw>JEvC|{;MGh{7+~74^aMhpZy(3USavKCJkivk07le0}1uI3>Ele2gIK*g0KJm Ee^%LY!vFvP literal 0 HcmV?d00001 diff --git a/imgs.batch.sh b/imgs.batch.sh index 90a0cf7cf..a764c63d4 100755 --- a/imgs.batch.sh +++ b/imgs.batch.sh @@ -2,7 +2,7 @@ datasource="../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/production_imgs/" -examples="ALN_baoh_021.tif FeMoOx_AntiA_04_1k5x_CN.tif" +examples="ALN_baoh_021.tif" # FeMoOx_AntiA_04_1k5x_CN.tif" for example in $examples; do echo $example diff --git a/imgs.dev.ipynb b/imgs.dev.ipynb index deee2d480..82317d0a6 100755 --- a/imgs.dev.ipynb +++ b/imgs.dev.ipynb @@ -8,7 +8,7 @@ "outputs": [], "source": [ "fnm = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/production_imgs/ALN_baoh_021.tif\"\n", - "fnm = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/production_imgs/FeMoOx_AntiA_04_1k5x_CN.tif\"" + "# fnm = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/production_imgs/FeMoOx_AntiA_04_1k5x_CN.tif\"" ] }, { @@ -18,6 +18,7 @@ "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "from PIL import Image\n", "from PIL.TiffTags import TAGS\n", "# print(TAGS)" @@ -58,14 +59,16 @@ " czi_keys = [34118, 34119]\n", " for czi_key in czi_keys:\n", " if czi_key in fp.tag_v2:\n", + " print(f\"Found czi_key {tfs_key}...\")\n", " utf = fp.tag[czi_key]\n", " print(type(utf))\n", " if len(utf) == 1:\n", " print(utf[0])\n", - " exit(1)\n", + " # exit(1)\n", " tfs_keys = [34682]\n", " for tfs_key in tfs_keys:\n", " if tfs_key in fp.tag_v2:\n", + " print(f\"Found tfs_key {tfs_key}...\")\n", " utf = fp.tag[tfs_key]\n", " print(type(utf))\n", " if len(utf) == 1:\n", diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py new file mode 100644 index 000000000..6fbd61fba --- /dev/null +++ b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py @@ -0,0 +1,130 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Subparser for harmonizing ThermoFisher-specific content in TIFF files.""" + +import mmap +import numpy as np +from typing import Dict +from PIL import Image +from PIL.TiffTags import TAGS + +from pynxtools.dataconverter.readers.em.subparsers.image_tiff import TiffSubParser +from pynxtools.dataconverter.readers.em.subparsers.image_tiff_tfs_cfg import \ + tfs_section_names, tfs_section_details +from pynxtools.dataconverter.readers.em.utils.image_utils import \ + sort_tuple, if_str_represents_float + + +class TfsTiffSubParser(TiffSubParser): + def __init__(self, file_path: str = ""): + super().__init__(file_path) + self.prfx = None + self.tmp: Dict = {} + self.supported_version: Dict = {} + self.version: Dict = {} + self.tags: Dict = {} + self.supported = False + self.check_if_tiff() + self.tfs: Dict = {} + + def check_if_tiff_tfs(self): + """Check if resource behind self.file_path is a TaggedImageFormat file.""" + self.supported = 0 # voting-based + with open(self.file_path, 'rb', 0) as file: + s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) + magic = s.read(4) + if magic == b'II*\x00': # https://en.wikipedia.org/wiki/TIFF + self.supported += 1 + + with Image.open(self.fiele_path, mode="r") as fp: + tfs_keys = [34682] + for tfs_key in tfs_keys: + if tfs_key in fp.tag_v2: + if len(fp.tag[tfs_key]) == 1: + self.supported += 1 # found TFS-specific tag + if self.supported == 2: + self.supported = True + else: + self.supported = False + + def get_metadata(self): + """Extract metadata in TFS specific tags if present.""" + print("Reporting the tags found in this TIFF file...") + # for an overview of tags + # https://www.loc.gov/preservation/digital/formats/content/tiff_tags.shtml + # with Image.open(self.file_path, mode="r") as fp: + # self.tags = {TAGS[key] : fp.tag[key] for key in fp.tag_v2} + # for key, val in self.tags.items(): + # print(f"{key}, {val}") + tfs_section_offsets = {} + with open(self.file_path, 'rb', 0) as fp: + s = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) + for section_name in tfs_section_names: + pos = s.find(bytes(section_name, "utf8")) # != -1 + tfs_section_offsets[section_name] = pos + print(tfs_section_offsets) + + # define search offsets + tpl = [] + for key, value in tfs_section_offsets.items(): + tpl.append((key, value)) + tpl = sort_tuple(tpl) + print(tpl) + + # exemplar parsing of specific TFS section content into a dict + # here for section_name == "[System]": + pos_s = None + pos_e = None + for idx in np.arange(0, len(tpl)): + if tpl[idx][0] != "[System]": + continue + else: + pos_s = tpl[idx][1] + if idx <= len(tpl) - 1: + pos_e = tpl[idx + 1][1] + break + print(f"Search for [System] in between byte offsets {pos_s} and {pos_e}") + if pos_s is None or pos_e is None: + raise ValueError(f"Search for [System] was unsuccessful !") + + # fish metadata of e.g. the system section + for term in tfs_section_details["[System]"]: + s.seek(pos_s, 0) + pos = s.find(bytes(term, "utf8")) + if pos < pos_e: # check if pos_e is None + s.seek(pos, 0) + value = f"{s.readline().strip().decode('utf8').replace(f'{term}=', '')}" + if value != "": + if if_str_represents_float(value) is True: + self.tfs[f"system/{term}"] = np.float64(value) + elif value.isdigit() is True: + self.tfs[f"system/{term}"] = np.int64(value) + else: + self.tfs[f"system/{term}"] = None + else: + pass + print(self.tfs) + + def parse_and_normalize(self): + """Perform actual parsing filling cache self.tmp.""" + if self.supported is True: + print(f"Parsing via ThermoFisher-specific metadata...") + self.get_metadata() + else: + print(f"{self.file_path} is not a ThermoFisher-specific " + f"TIFF file that this parser can process !") diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py new file mode 100644 index 000000000..53797ce87 --- /dev/null +++ b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py @@ -0,0 +1,62 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Configuration of the image_tiff_tfs subparser.""" + + +tfs_section_names = ["[User]", + "[System]", + "[Beam]", + "[EBeam]", + "[GIS]", + "[Scan]", + "[EScan]", + "[Stage]", + "[Image]", + "[Vacuum]", + "[Specimen]", + "[Detectors]", + "[T2]", + "[Accessories]", + "[EBeamDeceleration]", + "[CompoundLensFilter]", + "[PrivateFei]", + "[HiResIllumination]", + "[EasyLift]", + "[HotStageMEMS]", + "[HotStage]", + "[HotStageHVHS]", + "[ColdStage]"] + +tfs_section_details = {"[System]": ["Type", + "Dnumber", + "Software", + "BuildNr", + "Source", + "Column", + "FinalLens", + "Chamber", + "Stage", + "Pump", + "ESEM", + "Aperture", + "Scan", + "Acq", + "EucWD", + "SystemType", + "DisplayWidth", + "DisplayHeight"]} diff --git a/pynxtools/dataconverter/readers/em/subparsers/nxs_imgs.py b/pynxtools/dataconverter/readers/em/subparsers/nxs_imgs.py index 6b2c2f479..495cec07b 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/nxs_imgs.py +++ b/pynxtools/dataconverter/readers/em/subparsers/nxs_imgs.py @@ -20,7 +20,7 @@ import numpy as np # from typing import Dict, Any, List -from pynxtools.dataconverter.readers.em.subparsers.image_tiff import TiffSubParser +from pynxtools.dataconverter.readers.em.subparsers.image_tiff_tfs import TfsTiffSubParser class NxEmImagesSubParser: @@ -38,9 +38,9 @@ def __init__(self, entry_id: int = 1, input_file_name: str = ""): def identify_image_type(self): """Identify if image matches known mime type and has content for which subparser exists.""" # tech partner formats used for measurement - img = TiffSubParser(f"{self.file_path}") + img = TfsTiffSubParser(f"{self.file_path}") if img.supported is True: - return "tiff" + return "tiff_tfs" return None def parse(self, template: dict) -> dict: @@ -52,8 +52,8 @@ def parse(self, template: dict) -> dict: # see also comments for respective nxs_pyxem parser # and its interaction with tech-partner-specific hfive_* subparsers - if image_parser_type == "tiff": - tiff = TiffSubParser(self.file_path) + if image_parser_type == "tiff_tfs": + tiff = TfsTiffSubParser(self.file_path) tiff.parse_and_normalize() self.process_into_template(tiff.tmp, template) else: # none or something unsupported diff --git a/pynxtools/dataconverter/readers/em/utils/image_utils.py b/pynxtools/dataconverter/readers/em/utils/image_utils.py new file mode 100644 index 000000000..342af0bfc --- /dev/null +++ b/pynxtools/dataconverter/readers/em/utils/image_utils.py @@ -0,0 +1,40 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np + + +# https://www.geeksforgeeks.org/python-program-to-sort-a-list-of-tuples-by-second-item/ +def sort_tuple(tup): + # convert the list of tuples to a numpy array with data type (object, int) + arr = np.array(tup, dtype=[('col1', object), ('col2', int)]) + # get the indices that would sort the array based on the second column + indices = np.argsort(arr['col2']) + # use the resulting indices to sort the array + sorted_arr = arr[indices] + # convert the sorted numpy array back to a list of tuples + sorted_tup = [(row['col1'], row['col2']) for row in sorted_arr] + return sorted_tup + + +def if_str_represents_float(s): + try: + float(s) + return str(float(s)) == s + except ValueError: + return False