From 1459b8c49bc4dbc6e5bc80f881eda54b18708b68 Mon Sep 17 00:00:00 2001 From: "markus.kuehbach" Date: Thu, 26 Oct 2023 11:50:12 +0200 Subject: [PATCH] Added EBSD map downsampling and bugfixing of non working AM example, this completes the implementation of the tech partner HDF5 parsers for EBSD, next step linting and adding EDS reader (for IKZ, etc) --- .../readers/em/subparsers/hfive_apex.py | 18 +-- .../readers/em/subparsers/hfive_bruker.py | 8 +- .../readers/em/subparsers/hfive_ebsd.py | 8 +- .../readers/em/subparsers/hfive_edax.py | 19 ++- .../readers/em/subparsers/hfive_oxford.py | 29 ++-- .../readers/em/subparsers/nxs_hfive.py | 137 ++++++++++++++---- .../readers/em/utils/hfive_utils.py | 24 ++- .../readers/em/utils/hfive_web_constants.py | 3 +- .../readers/em/utils/image_processing.py | 7 +- test.all.sh | 7 +- 10 files changed, 172 insertions(+), 88 deletions(-) diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py index 86df97733..26008e21b 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py @@ -28,14 +28,14 @@ from diffpy.structure import Lattice, Structure from orix import plot from orix.crystal_map import create_coordinate_arrays, CrystalMap, PhaseList -from orix.quaternion import Rotation +from orix.quaternion import Rotation, Orientation from orix.vector import Vector3d import matplotlib.pyplot as plt from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser from pynxtools.dataconverter.readers.em.utils.hfive_utils import \ - read_strings_from_dataset, read_first_scalar, format_euler_parameterization + read_strings_from_dataset from pynxtools.dataconverter.readers.em.examples.ebsd_database import \ ASSUME_PHASE_NAME_TO_SPACE_GROUP @@ -164,6 +164,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str): angles = [fp[f"{sub_grp_name}/Lattice Constant Alpha"][0], fp[f"{sub_grp_name}/Lattice Constant Beta"][0], fp[f"{sub_grp_name}/Lattice Constant Gamma"][0]] + # TODO::available examples support reporting in angstroem and degree self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] \ = np.asarray(a_b_c, np.float32) * 0.1 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] \ @@ -210,22 +211,17 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str): for i in np.arange(0, n_pts): # check shape of internal virtual chunked number array - r = Rotation.from_matrix([np.reshape(dat[i][0], (3, 3))]) - self.tmp[ckey]["euler"][i, :] = r.to_euler(degrees=False) + oris = Orientation.from_matrix([np.reshape(dat[i][0], (3, 3))]) + self.tmp[ckey]["euler"][i, :] = oris.to_euler(degrees=False) self.tmp[ckey]["ci"][i] = dat[i][2] self.tmp[ckey]["phase_id"][i] = dat[i][3] + 1 # APEX seems to define # notIndexed as -1 and the first valid phase id 0 - - + if np.isnan(self.tmp[ckey]["euler"]).any(): + raise ValueError(f"Conversion of om2eu unexpectedly resulted in NaN !") # TODO::convert orientation matrix to Euler angles via om_eu but what are conventions ! # orix based transformation ends up in positive half space and with degrees=False # as radiants but the from_matrix command above might miss one rotation - # inconsistency f32 in file although specification states float - # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"], - # direction='lab2crystal', - # degrees=is_degrees) - # compute explicit hexagon grid cells center of mass pixel positions # TODO::currently assuming s_x and s_y are already the correct center of mass # distances for hexagonal or square tiling of R^2 diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py index bc37da596..4af1cd5e0 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py @@ -151,6 +151,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str): a_b_c = values[0:3] angles = values[3:6] self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] = a_b_c * 0.1 + # TODO::all examples indicate reporting in angstroem self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] = angles # Space Group, no, H5T_NATIVE_INT32, (1, 1), Space group index. @@ -202,15 +203,12 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str): self.tmp[ckey]["euler"] = np.zeros((n_pts[0], 3), np.float32) column_id = 0 for angle in ["phi1", "PHI", "phi2"]: + # TODO::available examples support that Bruker reports Euler triplets in degree self.tmp[ckey]["euler"][:, column_id] \ - = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32) + = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32) / 180. * np.pi column_id += 1 self.tmp[ckey]["euler"] = format_euler_parameterization(self.tmp[ckey]["euler"]) n_pts = n_pts[0] - # inconsistency f32 in file although specification states float - # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"], - # direction='lab2crystal', - # degrees=is_degrees) # index of phase, 0 if not indexed # no normalization needed, also in NXem_ebsd the null model notIndexed is phase_identifier 0 diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py index 1c85eab4c..3a11eddec 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py @@ -152,6 +152,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str): values = np.asarray(fp[f"{sub_grp_name}/LatticeConstants"][:].flatten()) a_b_c = values[0:3] angles = values[3:6] + # TODO::available examples support that community H5EBSD reports lattice constants in angstroem self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] = a_b_c * 0.1 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] = angles @@ -205,15 +206,12 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str): self.tmp[ckey]["euler"] = np.zeros((n_pts[0], 3), np.float32) column_id = 0 for angle in ["phi1", "PHI", "phi2"]: + # TODO::available examples support that community H5EBSD reports Euler triplets in degree self.tmp[ckey]["euler"][:, column_id] \ - = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32) + = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32) / 180. * np.pi column_id += 1 self.tmp[ckey]["euler"] = format_euler_parameterization(self.tmp[ckey]["euler"]) n_pts = n_pts[0] - # inconsistency f32 in file although specification states float - # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"], - # direction='lab2crystal', - # degrees=is_degrees) # index of phase, 0 if not indexed # no normalization needed, also in NXem_ebsd the null model notIndexed is phase_identifier 0 diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py index eaa459cb2..586179a51 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py @@ -36,7 +36,7 @@ import matplotlib.pyplot as plt from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser -from pynxtools.dataconverter.readers.em.utils.hfive_utils import \ +from pynxtools.dataconverter.readers.em.utils.hfive_utils import EULER_SPACE_SYMMETRY, \ read_strings_from_dataset, read_first_scalar, format_euler_parameterization from pynxtools.dataconverter.readers.em.examples.ebsd_database import \ ASSUME_PHASE_NAME_TO_SPACE_GROUP @@ -164,6 +164,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str): angles = [fp[f"{sub_grp_name}/Lattice Constant alpha"][()], fp[f"{sub_grp_name}/Lattice Constant beta"][()], fp[f"{sub_grp_name}/Lattice Constant gamma"][()]] + # TODO::available examples support reporting in angstroem and degree self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] \ = np.asarray(a_b_c, np.float32) * 0.1 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] \ @@ -209,6 +210,12 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str): n_pts = self.tmp[ckey]["n_x"] * self.tmp[ckey]["n_y"] self.tmp[ckey]["euler"] = np.zeros((n_pts, 3), np.float32) + # TODO::available examples support that rumour that in EDAX file sometimes values + # of Euler angle triplets are larger than mathematically possible + # unfortunately there is no confirmation from EDAX what is the reported unit and + # normalization for each software version, TODO::here rad is assumed but then values + # as large as 12.... should not be possible + # TODO::there has to be a mechanism which treats these dirty scan points! self.tmp[ckey]["euler"][:, 0] = np.asarray(fp[f"{grp_name}/Phi1"][:], np.float32) self.tmp[ckey]["euler"][:, 1] = np.asarray(fp[f"{grp_name}/Phi"][:], np.float32) self.tmp[ckey]["euler"][:, 2] = np.asarray(fp[f"{grp_name}/Phi2"][:], np.float32) @@ -217,11 +224,19 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str): # given no official EDAX OimAnalysis spec we cannot define for sure if # phase_id == 0 means just all was indexed with the first/zeroth phase or nothing - # was indexed, TODO::assuming it means all indexed: + # was indexed, TODO::assuming it means all indexed with first phase: if np.all(fp[f"{grp_name}/Phase"][:] == 0): self.tmp[ckey]["phase_id"] = np.zeros(n_pts, np.int32) + 1 else: self.tmp[ckey]["phase_id"] = np.asarray(fp[f"{grp_name}/Phase"][:], np.int32) + # TODO::mark scan points as dirty + # the line below shows an example how this could be achieved + # is_dirty = np.zeros((n_pts,), bool) + # for column_id in [0, 1, 2]: + # is_dirty = is_dirty & np.abs(self.tmp[ckey]["euler"][:, column_id]) > EULER_SPACE_SYMMETRY + # print(f"Found {np.sum(is_dirty)} scan points which are marked now as dirty!") + # self.tmp[ckey]["phase_id"][is_dirty] = 0 + # promoting int8 to int32 no problem self.tmp[ckey]["ci"] = np.asarray(fp[f"{grp_name}/CI"][:], np.float32) # normalize pixel coordinates to physical positions even though the origin can still dangle somewhere diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py index 75740996f..e05d7f4d6 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py @@ -36,7 +36,8 @@ import matplotlib.pyplot as plt from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser -from pynxtools.dataconverter.readers.em.utils.hfive_utils import read_strings_from_dataset +from pynxtools.dataconverter.readers.em.utils.hfive_utils import \ + read_strings_from_dataset, format_euler_parameterization class HdfFiveOxfordReader(HdfFiveBaseParser): @@ -171,17 +172,17 @@ def parse_and_normalize_slice_ebsd_phases(self, fp, ckey: str): = read_strings_from_dataset(fp[f"{sub_grp_name}/Reference"][()]) # Lattice Angles, yes, H5T_NATIVE_FLOAT, (1, 3), Three columns for the alpha, beta and gamma angles in radians - is_degrees = False if read_strings_from_dataset(fp[f"{sub_grp_name}/Lattice Angles"].attrs["Unit"]) == "rad": - is_degrees = False - angles = np.asarray(fp[f"{sub_grp_name}/Lattice Angles"][:].flatten()) / np.pi * 180. + angles = np.asarray(fp[f"{sub_grp_name}/Lattice Angles"][:].flatten()) + else: + raise ValueError(f"Unexpected case that Lattice Angles are not reported in rad !") self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] = angles # Lattice Dimensions, yes, H5T_NATIVE_FLOAT, (1, 3), Three columns for a, b and c dimensions in Angstroms - is_nanometer = False if read_strings_from_dataset(fp[f"{sub_grp_name}/Lattice Dimensions"].attrs["Unit"]) == "angstrom": - is_nanometer = False - a_b_c = np.asarray(fp[f"{sub_grp_name}/Lattice Dimensions"][:].flatten()) * 0.1 + a_b_c = np.asarray(fp[f"{sub_grp_name}/Lattice Dimensions"][:].flatten()) * 0.1 + else: + raise ValueError(f"Unexpected case that Lattice Dimensions are not reported in angstroem !") self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] = a_b_c # Space Group, no, H5T_NATIVE_INT32, (1, 1), Space group index. @@ -216,17 +217,11 @@ def parse_and_normalize_slice_ebsd_data(self, fp, ckey: str): raise ValueError(f"Unable to parse {grp_name}/{req_field} !") # Euler, yes, H5T_NATIVE_FLOAT, (size, 3), Orientation of Crystal (CS2) to Sample-Surface (CS1). - is_degrees = False - is_negative = False if read_strings_from_dataset(fp[f"{grp_name}/Euler"].attrs["Unit"]) == "rad": - is_degrees = False - self.tmp[ckey]["euler"] = np.asarray(fp[f"{grp_name}/Euler"], np.float32) - # TODO::handle possible case of negative Euler angles (examples though do not indicate) - # that AZTec reports negative Euler angles... - # inconsistency f32 in file although specification states float - # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"], - # direction='lab2crystal', - # degrees=is_degrees) + self.tmp[ckey]["euler"] = np.asarray(fp[f"{grp_name}/Euler"], np.float32) + else: + raise ValueError(f"Unexpected case that Euler angle are not reported in rad !") + self.tmp[ckey]["euler"] = format_euler_parameterization(self.tmp[ckey]["euler"]) # Phase, yes, H5T_NATIVE_INT32, (size, 1), Index of phase, 0 if not indexed # no normalization needed, also in NXem_ebsd the null model notIndexed is phase_identifier 0 diff --git a/pynxtools/dataconverter/readers/em/subparsers/nxs_hfive.py b/pynxtools/dataconverter/readers/em/subparsers/nxs_hfive.py index 4bc3fbcf5..ae2444519 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/nxs_hfive.py +++ b/pynxtools/dataconverter/readers/em/subparsers/nxs_hfive.py @@ -47,11 +47,13 @@ from orix.crystal_map import create_coordinate_arrays, CrystalMap, PhaseList from orix.quaternion import Rotation from orix.vector import Vector3d +from scipy.spatial import KDTree import matplotlib.pyplot as plt from pynxtools.dataconverter.readers.em.utils.hfive_utils import read_strings_from_dataset -from pynxtools.dataconverter.readers.em.utils.hfive_web_constants import HFIVE_WEB_MAXIMUM_RGB +from pynxtools.dataconverter.readers.em.utils.hfive_web_constants \ + import HFIVE_WEB_MAXIMUM_ROI, HFIVE_WEB_MAXIMUM_RGB from pynxtools.dataconverter.readers.em.utils.image_processing import thumbnail from pynxtools.dataconverter.readers.em.subparsers.hfive_oxford import HdfFiveOxfordReader @@ -182,8 +184,10 @@ def process_roi_overview_ebsd_based(self, roi_id: str, template: dict) -> dict: print("Parse ROI default plot...") - # prfx = f"/ENTRY[entry{self.entry_id}]/experiment/indexing/region_of_interest/roi{roi_id}" - # prfx = f"/roi{roi_id}" + if np.max((inp["n_x"], inp["n_y"])) > HFIVE_WEB_MAXIMUM_ROI: + raise ValueError(f"Plotting roi_overviews larger than " \ + f"{HFIVE_WEB_MAXIMUM_ROI} is not supported !") + trg = f"/ENTRY[entry{self.entry_id}]/ROI[roi{roi_id}]/ebsd/indexing/DATA[roi]" template[f"{trg}/title"] = f"Region-of-interest overview image" template[f"{trg}/@NX_class"] = f"NXdata" # TODO::writer should decorate automatically! @@ -239,7 +243,78 @@ def process_roi_xmap(self, inp: dict, roi_id: int, template: dict) -> dict: self.xmap = None self.axis_x = None self.axis_y = None - if np.max((inp["n_x"], inp["n_y"])) < HFIVE_WEB_MAXIMUM_RGB: + + print(f"Unique phase_identifier {np.unique(inp['phase_id'])}") + min_phase_id = np.min(np.unique(inp["phase_id"])) + + if np.max((inp["n_x"], inp["n_y"])) > HFIVE_WEB_MAXIMUM_RGB: + # assume center of mass of the scan points + # TODO::check if mapping correct for hexagonal and square grid + aabb = [np.min(inp["scan_point_x"]) - 0.5 * inp["s_x"], + np.max(inp["scan_point_x"]) + 0.5 * inp["s_x"], + np.min(inp["scan_point_y"]) - 0.5 * inp["s_y"], + np.max(inp["scan_point_y"]) + 0.5 * inp["s_y"]] + print(f"{aabb}") + if aabb[1] - aabb[0] >= aabb[3] - aabb[2]: + sqr_step_size = (aabb[1] - aabb[0]) / HFIVE_WEB_MAXIMUM_RGB + nxy = [HFIVE_WEB_MAXIMUM_RGB, + int(np.ceil((aabb[3] - aabb[2]) / sqr_step_size))] + else: + sqr_step_size = (aabb[3] - aabb[2]) / HFIVE_WEB_MAXIMUM_RGB + nxy = [int(np.ceil((aabb[1] - aabb[0]) / sqr_step_size)), + HFIVE_WEB_MAXIMUM_RGB] + print(f"H5Web default plot generation, scaling nxy0 {[inp['n_x'], inp['n_y']]}, nxy {nxy}") + # the above estimate is not exactly correct (may create a slight real space shift) + # of the EBSD map TODO:: regrid the real world axis-aligned bounding box aabb with + # a regular tiling of squares or hexagons + # https://stackoverflow.com/questions/18982650/differences-between-matlab-and-numpy-and-pythons-round-function + # MTex/Matlab round not exactly the same as numpy round but reasonably close + + # scan point positions were normalized by tech partner subparsers such that they + # always build on pixel coordinates calibrated for step size not by giving absolute positions + # in the sample surface frame of reference as this is typically not yet consistently documented + # because we assume in addition that we always start at the top left corner the zeroth/first + # coordinate is always 0., 0. ! + xy = np.column_stack( + (np.tile(np.linspace(0, nxy[0] - 1, num=nxy[0], endpoint=True) * sqr_step_size, nxy[1]), + np.repeat(np.linspace(0, nxy[1] - 1, num=nxy[1], endpoint=True) * sqr_step_size, nxy[0]))) + print(f"xy {xy}, shape {np.shape(xy)}") + tree = KDTree(np.column_stack((inp["scan_point_x"], inp["scan_point_y"]))) + d, idx = tree.query(xy, k=1) + if np.sum(idx == tree.n) > 0: + raise ValueError(f"kdtree query left some query points without a neighbor!") + del d + del tree + pyxem_euler = np.zeros((np.shape(xy)[0], 3), np.float32) + pyxem_euler = np.nan + pyxem_euler = inp["euler"][idx, :] + if np.isnan(pyxem_euler).any() is True: + raise ValueError(f"Downsampling of the EBSD map left pixels without euler!") + phase_new = np.zeros((np.shape(xy)[0],), np.int32) - 2 + phase_new = inp["phase_id"][idx] + if np.sum(phase_new == -2) > 0: + raise ValueError(f"Downsampling of the EBSD map left pixels without euler!") + del xy + + if min_phase_id > 0: + pyxem_phase_id = phase_new - min_phase_id + elif min_phase_id == 0: + pyxem_phase_id = phase_new - 1 + else: + raise ValueError(f"Unable how to deal with unexpected phase_identifier!") + del phase_new + + coordinates, _ = create_coordinate_arrays( + (nxy[1], nxy[0]), (sqr_step_size, sqr_step_size)) + xaxis = coordinates["x"] + yaxis = coordinates["y"] + print(f"coordinates" \ + f"xmi {np.min(xaxis)}, xmx {np.max(xaxis)}, " \ + f"ymi {np.min(yaxis)}, ymx {np.max(yaxis)}") + del coordinates + self.axis_x = np.linspace(0, nxy[0] - 1, num=nxy[0], endpoint=True) * sqr_step_size + self.axis_y = np.linspace(0, nxy[1] - 1, num=nxy[1], endpoint=True) * sqr_step_size + else: # can use the map discretization as is coordinates, _ = create_coordinate_arrays( (inp["n_y"], inp["n_x"]), (inp["s_y"], inp["s_x"])) @@ -250,35 +325,27 @@ def process_roi_xmap(self, inp: dict, roi_id: int, template: dict) -> dict: del coordinates self.axis_x = self.get_named_axis(inp, "x") self.axis_y = self.get_named_axis(inp, "y") - else: - raise ValueError(f"Downsampling for too large EBSD maps is currently not supported !") - # need to regrid to downsample too large maps - # TODO::implement 1NN-based downsampling approach - # build grid - # tree-based 1NN - # proceed as usual - - # TODO::there was one example 093_0060.h5oina - # where HitRate was 75% but no pixel left unidentified ?? - print(f"Unique phase_identifier {np.unique(inp['phase_id'])}") - min_phase_id = np.min(np.unique(inp["phase_id"])) - if min_phase_id > 0: - pyxem_phase_identifier = inp["phase_id"] - min_phase_id - elif min_phase_id == 0: - pyxem_phase_identifier = inp["phase_id"] - 1 - else: - raise ValueError(f"Unable how to deal with unexpected phase_identifier!") + + pyxem_euler = inp["euler"] + # TODO::there was one example 093_0060.h5oina + # where HitRate was 75% but no pixel left unidentified ?? + if min_phase_id > 0: + pyxem_phase_id = inp["phase_id"] - min_phase_id + elif min_phase_id == 0: + pyxem_phase_id = inp["phase_id"] - 1 + else: + raise ValueError(f"Unable how to deal with unexpected phase_identifier!") + # inp["phase_id"] - (np.min(inp["phase_id"]) - (-1)) # for pyxem the non-indexed has to be -1 instead of 0 which is what NeXus uses # -1 always because content of inp["phase_id"] is normalized # to NeXus NXem_ebsd_crystal_structure concept already! - print(f"Unique pyxem_phase_identifier {np.unique(pyxem_phase_identifier)}") - - self.xmap = CrystalMap(rotations=Rotation.from_euler(euler=inp["euler"], + print(f"Unique pyxem_phase_id {np.unique(pyxem_phase_id)}") + self.xmap = CrystalMap(rotations=Rotation.from_euler(euler=pyxem_euler, direction='lab2crystal', degrees=False), x=xaxis, y=yaxis, - phase_id=pyxem_phase_identifier, + phase_id=pyxem_phase_id, phase_list=PhaseList(space_groups=inp["space_group"], structures=inp["phase"]), prop={}, @@ -312,8 +379,24 @@ def process_roi_phases(self, inp: dict, roi_id: int, template: dict) -> dict: # phase_id of pyxem notIndexed is -1 while for NeXus # it is 0 so add + 1 in naming schemes trg = f"{prfx}/EM_EBSD_CRYSTAL_STRUCTURE_MODEL[phase{pyxem_phase_id + 1}]" + + min_phase_id = np.min(np.unique(inp["phase_id"])) + if min_phase_id > 0: + pyx_phase_id = inp["phase_id"] - min_phase_id + elif min_phase_id == 0: + pyx_phase_id = inp["phase_id"] - 1 + else: + raise ValueError(f"Unable how to deal with unexpected phase_identifier!") + del min_phase_id + template[f"{trg}/number_of_scan_points"] \ - = np.uint32(np.sum(self.xmap.phase_id == pyxem_phase_id)) + = np.uint32(np.sum(pyx_phase_id == pyxem_phase_id)) + del pyx_phase_id + # not self.xmap.phase_id because in NeXus the number_of_scan_points is always + # accounting for the original map size and not the potentially downscaled version + # of the map as the purpose of the later one is exclusively to show a plot at all + # because of a technical limitation of H5Web if there would be a tool that + # could show larger RGB plots we would not need to downscale the EBSD map resolution! template[f"{trg}/phase_identifier"] = np.uint32(pyxem_phase_id + 1) template[f"{trg}/phase_name"] \ = f"{inp['phases'][pyxem_phase_id + 1]['phase_name']}" diff --git a/pynxtools/dataconverter/readers/em/utils/hfive_utils.py b/pynxtools/dataconverter/readers/em/utils/hfive_utils.py index 5ae6a9cae..bf1e7af10 100644 --- a/pynxtools/dataconverter/readers/em/utils/hfive_utils.py +++ b/pynxtools/dataconverter/readers/em/utils/hfive_utils.py @@ -37,26 +37,22 @@ DIRTY_FIX_SPACEGROUP = {} +EULER_SPACE_SYMMETRY = [2. * np.pi, np.pi, 2. * np.pi] + def format_euler_parameterization(triplet_set): """Transform degrees to radiant and apply orientation space symmetry""" - is_degrees = False - for column_id in [0, 1, 2]: - # not robust enough as a single crystal close to the cube orientation - # with a very low orientation spread may also have all Euler angle values - # smaller than 2pi - # TODO::therefore the real specs of each tech partner's format is needed! - if np.max(np.abs(triplet_set[:, column_id])) > 2. * np.pi: - is_degrees = True - if is_degrees is True: - for column_id in [0, 1, 2]: - triplet_set[:, column_id] = triplet_set[:, column_id] / 180. * np.pi - - sothree_shift = [2. * np.pi, np.pi, 2. * np.pi] + # it is not robust in general to judge just from the collection of euler angles + # whether they are reported in radiant or degree + # indeed an EBSD map of a slightly deformed single crystal close to e.g. the cube ori + # can have euler angles for each scan point within pi, 2pi respectively + # similarly there was an example in the data 229_2096.oh5 where 3 out of 20.27 mio + # scan points where not reported in radiant but rather using 4pi as a marker to indicate + # there was a problem with the scan point for column_id in [0, 1, 2]: here = np.where(triplet_set[:, column_id] < 0.) if len(here[0]) > 0: triplet_set[here, column_id] \ - = sothree_shift[column_id] + triplet_set[here, column_id] + = EULER_SPACE_SYMMETRY[column_id] + triplet_set[here, column_id] return triplet_set def read_strings_from_dataset(obj): diff --git a/pynxtools/dataconverter/readers/em/utils/hfive_web_constants.py b/pynxtools/dataconverter/readers/em/utils/hfive_web_constants.py index 8f480dbaa..72b4f2519 100644 --- a/pynxtools/dataconverter/readers/em/utils/hfive_web_constants.py +++ b/pynxtools/dataconverter/readers/em/utils/hfive_web_constants.py @@ -17,4 +17,5 @@ # """Constants relevant when working with H5Web.""" -HFIVE_WEB_MAXIMUM_RGB = 2**14 +HFIVE_WEB_MAXIMUM_ROI = 2**14 - 1 +HFIVE_WEB_MAXIMUM_RGB = 2**11 - 1 diff --git a/pynxtools/dataconverter/readers/em/utils/image_processing.py b/pynxtools/dataconverter/readers/em/utils/image_processing.py index 34f98266f..88d2eb5cf 100644 --- a/pynxtools/dataconverter/readers/em/utils/image_processing.py +++ b/pynxtools/dataconverter/readers/em/utils/image_processing.py @@ -39,16 +39,15 @@ def thumbnail(img, size=300): return img if old_width == old_height: - img.thumbnail((size, size), pil.ANTIALIAS) + img.thumbnail((size, size)) elif old_height > old_width: ratio = float(old_width) / float(old_height) new_width = ratio * size - img = img.resize((int(np.floor(new_width)), size), pil.ANTIALIAS) + img = img.resize((int(np.floor(new_width)), size)) elif old_width > old_height: ratio = float(old_height) / float(old_width) new_height = ratio * size - img = img.resize((size, int(np.floor(new_height))), pil.ANTIALIAS) - + img = img.resize((size, int(np.floor(new_height)))) return img diff --git a/test.all.sh b/test.all.sh index c93c93f02..977b7a82f 100755 --- a/test.all.sh +++ b/test.all.sh @@ -9,12 +9,15 @@ # 026_0007.h5 026_0027.h5 026_0029.h5 026_0030.h5 026_0033.h5 026_0039.h5 026_0041.h5 delmic hdf5 have no ebsd data # 173_0056.h5oina has only eds data -# Examples="026_0046.h5oina 026_0049.h5oina 026_0050.h5oina 026_0052.h5oina 066_0013.h5 066_0015.h5 066_0016.h5 066_0023.h5 066_0025.h5 066_0034.h5 078_0004.h5 087_0021.h5 088_0009.h5 093_0045.h5oina 093_0047.h5oina 093_0048.h5oina 093_0051.h5oina 093_0053.h5oina 093_0054.h5oina 093_0055.h5oina 093_0058.h5oina 093_0059.h5oina 093_0060.h5oina 093_0062.h5oina 093_0063.h5oina 101_0040.h5 110_0012.h5 114_0017.h5 116_0008.h5 116_0014.h5 116_0018.h5 116_0019.h5 116_0020.h5 116_0022.h5 116_0037.h5 116_0042.h5 124_0002.h5 124_0036.h5 125_0006.h5 126_0038.h5 130_0003.h5 130_2082.h5 130_2083.h5 130_2084.h5 130_2085.h5 130_2086.h5 130_2087.h5 130_2088.h5 130_2089.h5 130_2090.h5 130_2091.h5 130_2092.h5 130_2093.h5 130_2094.h5 132_0005.h5 144_0043.h5 173_0056.h5oina 173_0057.h5oina 174_0031.h5 207_2081.edaxh5 208_0061.h5oina 212_2095.h5oina 229_2096.oh5 229_2097.oh5" +Examples="026_0046.h5oina 026_0049.h5oina 026_0050.h5oina 026_0052.h5oina 066_0013.h5 066_0015.h5 066_0016.h5 066_0023.h5 066_0025.h5 066_0034.h5 078_0004.h5 087_0021.h5 088_0009.h5 093_0045.h5oina 093_0047.h5oina 093_0048.h5oina 093_0051.h5oina 093_0053.h5oina 093_0054.h5oina 093_0055.h5oina 093_0058.h5oina 093_0059.h5oina 093_0060.h5oina 093_0062.h5oina 093_0063.h5oina 101_0040.h5 110_0012.h5 114_0017.h5 116_0008.h5 116_0014.h5 116_0018.h5 116_0019.h5 116_0020.h5 116_0022.h5 116_0037.h5 116_0042.h5 124_0002.h5 124_0036.h5 125_0006.h5 126_0038.h5 130_0003.h5 130_2082.h5 130_2083.h5 130_2084.h5 130_2085.h5 130_2086.h5 130_2087.h5 130_2088.h5 130_2089.h5 130_2090.h5 130_2091.h5 130_2092.h5 130_2093.h5 130_2094.h5 132_0005.h5 144_0043.h5 173_0056.h5oina 173_0057.h5oina 174_0031.h5 207_2081.edaxh5 208_0061.h5oina 212_2095.h5oina 229_2096.oh5 229_2097.oh5" # Examples="207_2081.edaxh5" # Examples="173_0057.h5oina" # oxford, bruker, britton, edax old noncali, edax old calib, apex -Examples="173_0057.h5oina 130_0003.h5 088_0009.h5 116_0014.h5 229_2097.oh5 207_2081.edaxh5" +# Examples="173_0057.h5oina 130_0003.h5 088_0009.h5 116_0014.h5 229_2097.oh5 207_2081.edaxh5" + +# Examples="229_2096.oh5" # this is the largest EBSD map, a composite +# Examples="229_2097.oh5" for example in $Examples; do echo $example dataconverter --reader em --nxdl NXroot --input-file $example --output debug.$example.nxs 1>stdout.$example.nxs.txt 2>stderr.$example.nxs.txt