From 458e1296344a78674eda82d7d71543d23954bd68 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 16 Jan 2024 00:37:51 +0100 Subject: [PATCH] Added simple id management for objects written to HDF5, bugfixes for pdi/InGe, ikz/AlGaO, and ikz/GeSi example with EBSD maps also working --- debug/spctrscpy.batch.sh | 6 +- debug/spctrscpy.dev.ipynb | 68 +++++++++---------- .../readers/em/examples/ebsd_database.py | 3 +- .../readers/em/subparsers/hfive_apex.py | 7 +- .../readers/em/subparsers/nxs_pyxem.py | 65 ++++++++++++++---- 5 files changed, 97 insertions(+), 52 deletions(-) diff --git a/debug/spctrscpy.batch.sh b/debug/spctrscpy.batch.sh index a82b4501b..089ae2028 100755 --- a/debug/spctrscpy.batch.sh +++ b/debug/spctrscpy.batch.sh @@ -1,15 +1,15 @@ #!/bin/bash datasource="../../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/pdi/" -#datasource="../../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/ikz/" +datasource="../../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/ikz/" # apex examples ikz, pdi # examples="ikz/VInP_108_L2.h5 ikz/GeSn_13.h5 pynx/46_ES-LP_L1_brg.bcf pynx/1613_Si_HAADF_610_kx.emd pynx/EELS_map_2_ROI_1_location_4.dm3 pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina" +examples="InGaN_nanowires_spectra.edaxh5" examples="AlGaO.nxs" examples="GeSi.nxs" -examples="VInP_108_L2.h5" -examples="InGaN_nanowires_spectra.edaxh5" +#examples="VInP_108_L2.h5" for example in $examples; do echo $example diff --git a/debug/spctrscpy.dev.ipynb b/debug/spctrscpy.dev.ipynb index 5d1d5327e..8eb7d3d4e 100644 --- a/debug/spctrscpy.dev.ipynb +++ b/debug/spctrscpy.dev.ipynb @@ -16,34 +16,6 @@ "from ase.data import chemical_symbols" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "8e721dee-7b6f-4dd0-b50e-ea8ff05d4682", - "metadata": {}, - "outputs": [], - "source": [ - "xray_lines = {}\n", - "for symbol in chemical_symbols[1:]:\n", - " # print(f\"{symbol}\")\n", - " for name, line in xraydb.xray_lines(symbol).items():\n", - " xray_lines[f\"{symbol}-{name}\"] = line.energy\n", - " # print(f\"{name}, {line.energy} eV\")\n", - "print(len(xray_lines))\n", - "\n", - "def get_xray_line_candidates(e_min=1200., e_max=1250.):\n", - " cand = []\n", - " for key, val in xray_lines.items():\n", - " if val < e_min:\n", - " continue\n", - " if val > e_max:\n", - " continue\n", - " cand.append(key)\n", - " return cand\n", - "\n", - "print(get_xray_line_candidates())" - ] - }, { "cell_type": "code", "execution_count": null, @@ -52,17 +24,17 @@ "outputs": [], "source": [ "src = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy\"\n", - "fnms = [(\"ikz\", \"VInP_108_L2.h5\"),\n", + "fnms = [(\"pdi\", \"InGaN_nanowires_spectra.edaxh5\"),\n", + " (\"ikz\", \"AlGaO.nxs\"),\n", " (\"ikz\", \"GeSi.nxs\"),\n", " (\"ikz\", \"GeSn_13.nxs\"),\n", - " (\"ikz\", \"AlGaO.nxs\"),\n", + " (\"ikz\", \"VInP_108_L2.h5\"),\n", " (\"bruker\", \"pynx/46_ES-LP_L1_brg.bcf\"),\n", " (\"emd\", \"pynx/1613_Si_HAADF_610_kx.emd\"),\n", " (\"digitalmicrograph\", \"pynx/EELS_map_2_ROI_1_location_4.dm3\"),\n", - " (\"oxfordinstruments\", \"pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina\"),\n", - " (\"pdi\", \"InGaN_nanowires_spectra.edaxh5\")]\n", + " (\"oxfordinstruments\", \"pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina\")]\n", "# pyUSID, HSMA\n", - "case = 0 # len(fnms) - 1 # len(fnms) - 1\n", + "case = 2 # len(fnms) - 1 # len(fnms) - 1\n", "fnm = f\"{src}/{fnms[case][0]}/{fnms[case][1]}\"\n", "print(fnm)" ] @@ -84,7 +56,7 @@ "metadata": {}, "outputs": [], "source": [ - "H5Web(\"debug.InGaN_nanowires_spectra.edaxh5.nxs\")" + "H5Web(f\"debug.{fnms[case][1]}.nxs\")" ] }, { @@ -171,6 +143,34 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e721dee-7b6f-4dd0-b50e-ea8ff05d4682", + "metadata": {}, + "outputs": [], + "source": [ + "xray_lines = {}\n", + "for symbol in chemical_symbols[1:]:\n", + " # print(f\"{symbol}\")\n", + " for name, line in xraydb.xray_lines(symbol).items():\n", + " xray_lines[f\"{symbol}-{name}\"] = line.energy\n", + " # print(f\"{name}, {line.energy} eV\")\n", + "print(len(xray_lines))\n", + "\n", + "def get_xray_line_candidates(e_min=1200., e_max=1250.):\n", + " cand = []\n", + " for key, val in xray_lines.items():\n", + " if val < e_min:\n", + " continue\n", + " if val > e_max:\n", + " continue\n", + " cand.append(key)\n", + " return cand\n", + "\n", + "print(get_xray_line_candidates())" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/pynxtools/dataconverter/readers/em/examples/ebsd_database.py b/pynxtools/dataconverter/readers/em/examples/ebsd_database.py index e140b570c..35f14f05a 100644 --- a/pynxtools/dataconverter/readers/em/examples/ebsd_database.py +++ b/pynxtools/dataconverter/readers/em/examples/ebsd_database.py @@ -316,4 +316,5 @@ ASSUME_PHASE_NAME_TO_SPACE_GROUP = {"Silver": 225, "Copper": 225, "Ni (Nickel)": 225, - "Face Centered Cubic": 225} + "Face Centered Cubic": 225, + "Ge (Germanium)": 225} # Ge (Germanium), really? diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py index 9412ae0b6..a7e5041f4 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py @@ -123,8 +123,9 @@ def parse_and_normalize(self): self.parse_and_normalize_group_ebsd_header(h5r, ckey) self.parse_and_normalize_group_ebsd_phases(h5r, ckey) self.parse_and_normalize_group_ebsd_data(h5r, ckey) - self.parse_and_normalize_group_ebsd_complete(ckey) + self.parse_and_normalize_group_ebsd_check(ckey) self.cache_id += 1 + continue # TODO: conceptually the content of the three # above-mentioned groups has and uses for some @@ -270,6 +271,8 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str): space_group = None if phase_name in ASSUME_PHASE_NAME_TO_SPACE_GROUP.keys(): space_group = ASSUME_PHASE_NAME_TO_SPACE_GROUP[phase_name] + else: + raise ValueError(f"{phase_name} is not in ASSUME_PHASE_NAME_TO_SPACE_GROUP !") self.tmp[ckey]["phases"][int(phase_id)]["space_group"] = space_group if len(self.tmp[ckey]["space_group"]) > 0: @@ -361,7 +364,7 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str): # almost two decades of commercialization of the technique now get_scan_point_coords(self.tmp[ckey]) - def parse_and_normalize_group_ebsd_complete(ckey: str): + def parse_and_normalize_group_ebsd_check(self, ckey: str): """Check if all relevant data for EBSD are available, if not clear the cache.""" # TODO::implement check and clearing procedure pass diff --git a/pynxtools/dataconverter/readers/em/subparsers/nxs_pyxem.py b/pynxtools/dataconverter/readers/em/subparsers/nxs_pyxem.py index ee424fab2..87690b4a9 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/nxs_pyxem.py +++ b/pynxtools/dataconverter/readers/em/subparsers/nxs_pyxem.py @@ -59,6 +59,7 @@ from pynxtools.dataconverter.readers.em.subparsers.hfive_ebsd import HdfFiveCommunityReader from pynxtools.dataconverter.readers.em.subparsers.hfive_emsoft import HdfFiveEmSoftReader from pynxtools.dataconverter.readers.em.subparsers.hfive_dreamthreed import HdfFiveDreamThreedReader +from pynxtools.dataconverter.readers.em.concepts.nxs_image_r_set import NxImageRealSpaceSet PROJECTION_VECTORS = [Vector3d.xvector(), Vector3d.yvector(), Vector3d.zvector()] @@ -92,7 +93,11 @@ def __init__(self, entry_id: int = 1, input_file_name: str = ""): else: self.entry_id = 1 self.file_path = input_file_name - self.event_id = 1 + self.id_mgn = {"event": 1, + "event_img": 1, + "event_spc": 1, + "roi": 1, + "eds_img": 1} self.cache = {"is_filled": False} def parse(self, template: dict) -> dict: @@ -313,8 +318,9 @@ def process_roi_overview_ebsd_based(self, def process_roi_overview_eds_based(self, inp: dict, template: dict) -> dict: - trg = f"/ENTRY[entry{self.entry_id}]/measurement/event_data_em_set/EVENT_DATA_EM" \ - f"[event_data_em{self.event_id}]/IMAGE_R_SET[image_r_set1]/DATA[image_twod]" + trg = f"/ENTRY[entry{self.entry_id}]/measurement/event_data_em_set/" \ + f"EVENT_DATA_EM[event_data_em{self.id_mgn['event']}]/" \ + f"IMAGE_R_SET[image_r_set{self.id_mgn['event_img']}]/DATA[image_twod]" template[f"{trg}/@NX_class"] = "NXdata" # TODO::should be autodecorated template[f"{trg}/description"] = inp.tmp["source"] template[f"{trg}/title"] = f"Region-of-interest overview image" @@ -333,8 +339,8 @@ def process_roi_overview_eds_based(self, = {"compress": inp.tmp[f"image_twod/axis_{dim[0]}"].value, "strength": 1} template[f"{trg}/AXISNAME[axis_{dim[0]}]/@long_name"] \ = inp.tmp[f"image_twod/axis_{dim[0]}@long_name"].value - # template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit}" - self.event_id += 1 + self.id_mgn["event_img"] += 1 + self.id_mgn["event"] += 1 return template def process_roi_ebsd_maps(self, inp: dict, template: dict) -> dict: @@ -662,8 +668,9 @@ def process_roi_eds_spectra(self, inp: dict, template: dict) -> dict: for ckey in inp.keys(): if ckey.startswith("eds_spc") and inp[ckey] != {}: trg = f"/ENTRY[entry{self.entry_id}]/measurement/event_data_em_set/" \ - f"EVENT_DATA_EM[event_data_em{self.event_id}]/" \ - f"SPECTRUM_SET[spectrum_set1]/DATA[spectrum_zerod]" + f"EVENT_DATA_EM[event_data_em{self.id_mgn['event']}]/" \ + f"SPECTRUM_SET[spectrum_set{self.id_mgn['event_spc']}]/" \ + f"DATA[spectrum_zerod]" # TODO::check if its a spectrum_zerod !!! template[f"{trg}/@NX_class"] = "NXdata" # TODO::should be autodecorated template[f"{trg}/description"] = inp[ckey].tmp["source"] @@ -673,17 +680,51 @@ def process_roi_eds_spectra(self, inp: dict, template: dict) -> dict: template[f"{trg}/intensity"] \ = {"compress": inp[ckey].tmp["spectrum_zerod/intensity"].value, "strength": 1} - template[f"{trg}/intensity/@long_name"] = f"Signal" + template[f"{trg}/intensity/@long_name"] \ + = inp[ckey].tmp["spectrum_zerod/intensity@long_name"].value # f"Signal" template[f"{trg}/@AXISNAME_indices[axis_energy_indices]"] = np.uint32(0) template[f"{trg}/AXISNAME[axis_energy]"] \ = {"compress": inp[ckey].tmp[f"spectrum_zerod/axis_energy"].value, "strength": 1} template[f"{trg}/AXISNAME[axis_energy]/@long_name"] \ - = inp[ckey].tmp[f"spectrum_zerod/axis_energy@long_name"].value - # template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit}" - # TODO::increment spectrum_set1 - self.event_id += 1 + = inp[ckey].tmp[f"spectrum_zerod/axis_energy@long_name"].value + self.id_mgn["event_spc"] += 1 + self.id_mgn["event"] += 1 return template def process_roi_eds_maps(self, inp: dict, template: dict) -> dict: + for ckey in inp.keys(): + if ckey.startswith("eds_map") and inp[ckey] != {}: + trg = f"/ENTRY[entry{self.entry_id}]/ROI[roi{self.id_mgn['roi']}]/" \ + f"eds/indexing/" + template[f"{trg}/source"] = inp.tmp["source"] + for img in inp.tmp["IMAGE_R_SET"]: + if not isinstance(img, NxImageRealSpaceSet): + continue + trg = f"/ENTRY[entry{self.entry_id}]/ROI[roi{self.id_mgn['roi']}]/eds/" \ + f"indexing/IMAGE_R_SET[image_r_set{self.id_mgn['eds_img']}]/" + template[f"{trg}/description"] \ + = img.tmp["description"] + template[f"{trg}/iupac_line_candidates"] \ + = img.tmp["iupac_line_candidates"] + template[f"{trg}/@NX_class"] = "NXdata" # TODO::should be autodecorated + template[f"{trg}/title"] = f"EDS map" + template[f"{trg}/@signal"] = "intensity" + template[f"{trg}/@axes"] = ["axis_y", "axis_x"] + template[f"{trg}/intensity"] \ + = {"compress": img.tmp["image_twod/intensity"].value, + "strength": 1} + template[f"{trg}/intensity/@long_name"] = f"Signal" + dims = [("x", 0), ("y", 1)] + for dim in dims: + template[f"{trg}/@AXISNAME_indices[axis_{dim[0]}_indices]"] \ + = np.uint32(dim[1]) + template[f"{trg}/AXISNAME[axis_{dim[0]}]"] \ + = {"compress": img.tmp[f"image_twod/axis_{dim[0]}"].value, + "strength": 1} + template[f"{trg}/AXISNAME[axis_{dim[0]}]/@long_name"] \ + = img.tmp[f"image_twod/axis_{dim[0]}"].value + self.id_mgn["eds_img"] += 1 + self.id_mgn["roi"] += 1 + return template