Added simple id management for objects written to HDF5, bugfixes for …

…pdi/InGe, ikz/AlGaO, and ikz/GeSi example with EBSD maps also working

debug/spctrscpy.batch.sh

@@ -1,15 +1,15 @@
 #!/bin/bash
 
 datasource="../../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/pdi/"
-#datasource="../../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/ikz/"
+datasource="../../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/ikz/"
 
 
 # apex examples ikz, pdi
 # examples="ikz/VInP_108_L2.h5 ikz/GeSn_13.h5 pynx/46_ES-LP_L1_brg.bcf pynx/1613_Si_HAADF_610_kx.emd pynx/EELS_map_2_ROI_1_location_4.dm3 pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina"
+examples="InGaN_nanowires_spectra.edaxh5"
 examples="AlGaO.nxs"
 examples="GeSi.nxs"
-examples="VInP_108_L2.h5"
-examples="InGaN_nanowires_spectra.edaxh5"
+#examples="VInP_108_L2.h5"
 
 for example in $examples; do
 	echo $example

debug/spctrscpy.dev.ipynb

@@ -16,34 +16,6 @@
     "from ase.data import chemical_symbols"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8e721dee-7b6f-4dd0-b50e-ea8ff05d4682",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "xray_lines = {}\n",
-    "for symbol in chemical_symbols[1:]:\n",
-    "    # print(f\"{symbol}\")\n",
-    "    for name, line in xraydb.xray_lines(symbol).items():\n",
-    "        xray_lines[f\"{symbol}-{name}\"] = line.energy\n",
-    "        # print(f\"{name}, {line.energy} eV\")\n",
-    "print(len(xray_lines))\n",
-    "\n",
-    "def get_xray_line_candidates(e_min=1200., e_max=1250.):\n",
-    "    cand = []\n",
-    "    for key, val in xray_lines.items():\n",
-    "        if val < e_min:\n",
-    "            continue\n",
-    "        if val > e_max:\n",
-    "            continue\n",
-    "        cand.append(key)\n",
-    "    return cand\n",
-    "\n",
-    "print(get_xray_line_candidates())"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -52,17 +24,17 @@
    "outputs": [],
    "source": [
     "src = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy\"\n",
-    "fnms = [(\"ikz\", \"VInP_108_L2.h5\"),\n",
+    "fnms = [(\"pdi\", \"InGaN_nanowires_spectra.edaxh5\"),\n",
+    "        (\"ikz\", \"AlGaO.nxs\"),\n",
     "        (\"ikz\", \"GeSi.nxs\"),\n",
     "        (\"ikz\", \"GeSn_13.nxs\"),\n",
-    "        (\"ikz\", \"AlGaO.nxs\"),\n",
+    "        (\"ikz\", \"VInP_108_L2.h5\"),\n",
     "        (\"bruker\", \"pynx/46_ES-LP_L1_brg.bcf\"),\n",
     "        (\"emd\", \"pynx/1613_Si_HAADF_610_kx.emd\"),\n",
     "        (\"digitalmicrograph\", \"pynx/EELS_map_2_ROI_1_location_4.dm3\"),\n",
-    "        (\"oxfordinstruments\", \"pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina\"),\n",
-    "        (\"pdi\", \"InGaN_nanowires_spectra.edaxh5\")]\n",
+    "        (\"oxfordinstruments\", \"pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina\")]\n",
     "# pyUSID, HSMA\n",
-    "case = 0 # len(fnms) - 1  # len(fnms) - 1\n",
+    "case = 2 # len(fnms) - 1  # len(fnms) - 1\n",
     "fnm = f\"{src}/{fnms[case][0]}/{fnms[case][1]}\"\n",
     "print(fnm)"
    ]
@@ -84,7 +56,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "H5Web(\"debug.InGaN_nanowires_spectra.edaxh5.nxs\")"
+    "H5Web(f\"debug.{fnms[case][1]}.nxs\")"
    ]
   },
   {
@@ -171,6 +143,34 @@
    "outputs": [],
    "source": []
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e721dee-7b6f-4dd0-b50e-ea8ff05d4682",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "xray_lines = {}\n",
+    "for symbol in chemical_symbols[1:]:\n",
+    "    # print(f\"{symbol}\")\n",
+    "    for name, line in xraydb.xray_lines(symbol).items():\n",
+    "        xray_lines[f\"{symbol}-{name}\"] = line.energy\n",
+    "        # print(f\"{name}, {line.energy} eV\")\n",
+    "print(len(xray_lines))\n",
+    "\n",
+    "def get_xray_line_candidates(e_min=1200., e_max=1250.):\n",
+    "    cand = []\n",
+    "    for key, val in xray_lines.items():\n",
+    "        if val < e_min:\n",
+    "            continue\n",
+    "        if val > e_max:\n",
+    "            continue\n",
+    "        cand.append(key)\n",
+    "    return cand\n",
+    "\n",
+    "print(get_xray_line_candidates())"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

pynxtools/dataconverter/readers/em/examples/ebsd_database.py

@@ -316,4 +316,5 @@
 ASSUME_PHASE_NAME_TO_SPACE_GROUP = {"Silver": 225,
                                     "Copper": 225,
                                     "Ni (Nickel)": 225,
-                                    "Face Centered Cubic": 225}
+                                    "Face Centered Cubic": 225,
+                                    "Ge (Germanium)": 225}  # Ge (Germanium), really?

pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py

@@ -123,8 +123,9 @@ def parse_and_normalize(self):
                                     self.parse_and_normalize_group_ebsd_header(h5r, ckey)
                                     self.parse_and_normalize_group_ebsd_phases(h5r, ckey)
                                     self.parse_and_normalize_group_ebsd_data(h5r, ckey)
-                                    self.parse_and_normalize_group_ebsd_complete(ckey)
+                                    self.parse_and_normalize_group_ebsd_check(ckey)
                                     self.cache_id += 1
+                                continue
 
                                 # TODO: conceptually the content of the three
                                 # above-mentioned groups has and uses for some
@@ -270,6 +271,8 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str):
                 space_group = None
                 if phase_name in ASSUME_PHASE_NAME_TO_SPACE_GROUP.keys():
                     space_group = ASSUME_PHASE_NAME_TO_SPACE_GROUP[phase_name]
+                else:
+                    raise ValueError(f"{phase_name} is not in ASSUME_PHASE_NAME_TO_SPACE_GROUP !")
                 self.tmp[ckey]["phases"][int(phase_id)]["space_group"] = space_group
 
                 if len(self.tmp[ckey]["space_group"]) > 0:
@@ -361,7 +364,7 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str):
         # almost two decades of commercialization of the technique now
         get_scan_point_coords(self.tmp[ckey])
 
-    def parse_and_normalize_group_ebsd_complete(ckey: str):
+    def parse_and_normalize_group_ebsd_check(self, ckey: str):
         """Check if all relevant data for EBSD are available, if not clear the cache."""
         # TODO::implement check and clearing procedure
         pass

pynxtools/dataconverter/readers/em/subparsers/nxs_pyxem.py

@@ -59,6 +59,7 @@
 from pynxtools.dataconverter.readers.em.subparsers.hfive_ebsd import HdfFiveCommunityReader
 from pynxtools.dataconverter.readers.em.subparsers.hfive_emsoft import HdfFiveEmSoftReader
 from pynxtools.dataconverter.readers.em.subparsers.hfive_dreamthreed import HdfFiveDreamThreedReader
+from pynxtools.dataconverter.readers.em.concepts.nxs_image_r_set import NxImageRealSpaceSet
 
 
 PROJECTION_VECTORS = [Vector3d.xvector(), Vector3d.yvector(), Vector3d.zvector()]
@@ -92,7 +93,11 @@ def __init__(self, entry_id: int = 1, input_file_name: str = ""):
         else:
             self.entry_id = 1
         self.file_path = input_file_name
-        self.event_id = 1
+        self.id_mgn = {"event": 1,
+                       "event_img": 1,
+                       "event_spc": 1,
+                       "roi": 1,
+                       "eds_img": 1}
         self.cache = {"is_filled": False}
 
     def parse(self, template: dict) -> dict:
@@ -313,8 +318,9 @@ def process_roi_overview_ebsd_based(self,
     def process_roi_overview_eds_based(self,
                                        inp: dict,
                                        template: dict) -> dict:
-        trg = f"/ENTRY[entry{self.entry_id}]/measurement/event_data_em_set/EVENT_DATA_EM" \
-              f"[event_data_em{self.event_id}]/IMAGE_R_SET[image_r_set1]/DATA[image_twod]"
+        trg = f"/ENTRY[entry{self.entry_id}]/measurement/event_data_em_set/" \
+              f"EVENT_DATA_EM[event_data_em{self.id_mgn['event']}]/" \
+              f"IMAGE_R_SET[image_r_set{self.id_mgn['event_img']}]/DATA[image_twod]"
         template[f"{trg}/@NX_class"] = "NXdata"  # TODO::should be autodecorated
         template[f"{trg}/description"] = inp.tmp["source"]
         template[f"{trg}/title"] = f"Region-of-interest overview image"
@@ -333,8 +339,8 @@ def process_roi_overview_eds_based(self,
                 = {"compress": inp.tmp[f"image_twod/axis_{dim[0]}"].value, "strength": 1}
             template[f"{trg}/AXISNAME[axis_{dim[0]}]/@long_name"] \
                 = inp.tmp[f"image_twod/axis_{dim[0]}@long_name"].value
-            # template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit}"
-        self.event_id += 1
+        self.id_mgn["event_img"] += 1
+        self.id_mgn["event"] += 1
         return template
 
     def process_roi_ebsd_maps(self, inp: dict, template: dict) -> dict:
@@ -662,8 +668,9 @@ def process_roi_eds_spectra(self, inp: dict, template: dict) -> dict:
         for ckey in inp.keys():
             if ckey.startswith("eds_spc") and inp[ckey] != {}:
                 trg = f"/ENTRY[entry{self.entry_id}]/measurement/event_data_em_set/" \
-                      f"EVENT_DATA_EM[event_data_em{self.event_id}]/" \
-                      f"SPECTRUM_SET[spectrum_set1]/DATA[spectrum_zerod]"
+                      f"EVENT_DATA_EM[event_data_em{self.id_mgn['event']}]/" \
+                      f"SPECTRUM_SET[spectrum_set{self.id_mgn['event_spc']}]/" \
+                      f"DATA[spectrum_zerod]"
                 # TODO::check if its a spectrum_zerod !!!
                 template[f"{trg}/@NX_class"] = "NXdata"  # TODO::should be autodecorated
                 template[f"{trg}/description"] = inp[ckey].tmp["source"]
@@ -673,17 +680,51 @@ def process_roi_eds_spectra(self, inp: dict, template: dict) -> dict:
                 template[f"{trg}/intensity"] \
                     = {"compress": inp[ckey].tmp["spectrum_zerod/intensity"].value,
                        "strength": 1}
-                template[f"{trg}/intensity/@long_name"] = f"Signal"
+                template[f"{trg}/intensity/@long_name"] \
+                    = inp[ckey].tmp["spectrum_zerod/intensity@long_name"].value  # f"Signal"
                 template[f"{trg}/@AXISNAME_indices[axis_energy_indices]"] = np.uint32(0)
                 template[f"{trg}/AXISNAME[axis_energy]"] \
                     = {"compress": inp[ckey].tmp[f"spectrum_zerod/axis_energy"].value,
                        "strength": 1}
                 template[f"{trg}/AXISNAME[axis_energy]/@long_name"] \
-                        = inp[ckey].tmp[f"spectrum_zerod/axis_energy@long_name"].value
-                    # template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit}"
-                # TODO::increment spectrum_set1
-                self.event_id += 1
+                    = inp[ckey].tmp[f"spectrum_zerod/axis_energy@long_name"].value
+                self.id_mgn["event_spc"] += 1
+                self.id_mgn["event"] += 1
         return template
 
     def process_roi_eds_maps(self, inp: dict, template: dict) -> dict:
+        for ckey in inp.keys():
+            if ckey.startswith("eds_map") and inp[ckey] != {}:
+                trg = f"/ENTRY[entry{self.entry_id}]/ROI[roi{self.id_mgn['roi']}]/" \
+                      f"eds/indexing/"
+                template[f"{trg}/source"] = inp.tmp["source"]
+                for img in inp.tmp["IMAGE_R_SET"]:
+                    if not isinstance(img, NxImageRealSpaceSet):
+                        continue
+                    trg = f"/ENTRY[entry{self.entry_id}]/ROI[roi{self.id_mgn['roi']}]/eds/" \
+                          f"indexing/IMAGE_R_SET[image_r_set{self.id_mgn['eds_img']}]/"
+                    template[f"{trg}/description"] \
+                        = img.tmp["description"]
+                    template[f"{trg}/iupac_line_candidates"] \
+                        = img.tmp["iupac_line_candidates"]
+                    template[f"{trg}/@NX_class"] = "NXdata"  # TODO::should be autodecorated
+                    template[f"{trg}/title"] = f"EDS map"
+                    template[f"{trg}/@signal"] = "intensity"
+                    template[f"{trg}/@axes"] = ["axis_y", "axis_x"]
+                    template[f"{trg}/intensity"] \
+                        = {"compress": img.tmp["image_twod/intensity"].value,
+                           "strength": 1}
+                    template[f"{trg}/intensity/@long_name"] = f"Signal"
+                    dims = [("x", 0), ("y", 1)]
+                    for dim in dims:
+                        template[f"{trg}/@AXISNAME_indices[axis_{dim[0]}_indices]"] \
+                            = np.uint32(dim[1])
+                        template[f"{trg}/AXISNAME[axis_{dim[0]}]"] \
+                            = {"compress": img.tmp[f"image_twod/axis_{dim[0]}"].value,
+                               "strength": 1}
+                        template[f"{trg}/AXISNAME[axis_{dim[0]}]/@long_name"] \
+                            = img.tmp[f"image_twod/axis_{dim[0]}"].value
+                    self.id_mgn["eds_img"] += 1
+                self.id_mgn["roi"] += 1
+
         return template