diff --git a/imgs.batch.sh b/imgs.batch.sh index ae80412cf..8b6498b58 100755 --- a/imgs.batch.sh +++ b/imgs.batch.sh @@ -1,6 +1,6 @@ #!/bin/bash -datasource="../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_imgs/ikz_robert/" +datasource="../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_imgs/axon/" # comments is detector mode examples="kit/FeMoOx_AntiA_04_1k5x_CN.tif" @@ -9,9 +9,9 @@ examples="ikz_martin/ALN_baoh_021.tif" # T2 examples="ikz_robert/T3_image.tif" examples="ikz_robert/ETD_image.tif" # ETD examples="ikz_martin/NavCam_normal_vis_light_ccd.tif" # NavCam - examples="0c8nA_3deg_003_AplusB_test.tif ALN_baoh_021.tif T3_image.tif ETD_image.tif NavCam_normal_vis_light_ccd.tif" - +examples="axon/20210426T224437.049Raw0.png" #axon +examples="ReductionOfFeOx.zip" for example in $examples; do echo $example diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py b/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py index ba8b8a40b..fbdc311e7 100644 --- a/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py @@ -68,8 +68,8 @@ def __init__(self): self.tmp: Dict = {} for entry in NX_IMAGE_REAL_SPACE_SET_HDF_PATH: if entry.endswith("-field") is True: - self.tmp[entry[0:len(entry)-len("-field")]] = NxObject(eqv_hdf="dataset") + self.tmp[entry[0:len(entry) - len("-field")]] = NxObject(eqv_hdf="dataset") elif entry.endswith("-attribute") is True: - self.tmp[entry[0:len(entry)-len("-attribute")]] = NxObject(eqv_hdf="attribute") + self.tmp[entry[0:len(entry) - len("-attribute")]] = NxObject(eqv_hdf="attribute") else: - self.tmp[entry[0:len(entry)-len("-group")]] = NxObject(eqv_hdf="group") + self.tmp[entry[0:len(entry) - len("-group")]] = NxObject(eqv_hdf="group") diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_object.py b/pynxtools/dataconverter/readers/em/concepts/nxs_object.py index c5d3c9f77..696be9a86 100644 --- a/pynxtools/dataconverter/readers/em/concepts/nxs_object.py +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_object.py @@ -36,7 +36,7 @@ def __init__(self, if (unit is not None) and (unit == ""): raise ValueError(f"Value for argument unit needs to be a non-empty string !") if (dtype is not None) and isinstance(dtype, type) is False: - raise ValueError(f"Value of argument dtype must not be None " \ + raise ValueError(f"Value of argument dtype must not be None " f" and a valid, ideally a numpy datatype !") # self.doc = None # docstring self.name = name # name of the field diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py b/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py index a16851ce1..e1bed0f39 100644 --- a/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py @@ -20,12 +20,12 @@ # pylint: disable=no-member,too-few-public-methods -from typing import Dict +from typing import Dict, List from pynxtools.dataconverter.readers.em.concepts.nxs_object import NxObject -NX_SPECTRUM_SET_HDF_PATH = [] +NX_SPECTRUM_SET_HDF_PATH: List = [] # this one needs an update ! @@ -34,8 +34,8 @@ def __init__(self): self.tmp: Dict = {} for entry in NX_SPECTRUM_SET_HDF_PATH: if entry.endswith("-field") is True: - self.tmp[entry[0:len(entry)-len("-field")]] = NxObject(eqv_hdf="dataset") + self.tmp[entry[0:len(entry) - len("-field")]] = NxObject(eqv_hdf="dataset") elif entry.endswith("-attribute") is True: - self.tmp[entry[0:len(entry)-len("-attribute")]] = NxObject(eqv_hdf="attribute") + self.tmp[entry[0:len(entry) - len("-attribute")]] = NxObject(eqv_hdf="attribute") else: - self.tmp[entry[0:len(entry)-len("-group")]] = NxObject(eqv_hdf="group") + self.tmp[entry[0:len(entry) - len("-group")]] = NxObject(eqv_hdf="group") diff --git a/pynxtools/dataconverter/readers/em/reader.py b/pynxtools/dataconverter/readers/em/reader.py index 971579de4..76dd5fdee 100644 --- a/pynxtools/dataconverter/readers/em/reader.py +++ b/pynxtools/dataconverter/readers/em/reader.py @@ -147,7 +147,8 @@ def read(self, # print("Create NeXus default plottable data...") # em_default_plot_generator(template, 1) - if True is False: + run_block = False + if run_block is True: nxs_plt = NxEmDefaultPlotResolver() # if nxs_mtex is the sub-parser resolved_path = nxs_plt.nxs_mtex_get_nxpath_to_default_plot( diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py index 2275ee159..67a3868df 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py @@ -322,7 +322,7 @@ def parse_and_normalize_eds_spd(self, fp, src: str, ckey: str): for req in reqs: if req not in fp[f"{src}/SPD"].attrs.keys(): # also check for shape raise ValueError(f"Required attribute named {req} not found in {src}/SPD !") - + nyxe = {"y": fp[f"{src}/SPD"].attrs["NumberOfLines"][0], "x": fp[f"{src}/SPD"].attrs["NumberOfPoints"][0], "e": fp[f"{src}/SPD"].attrs["NumberofChannels"][0]} @@ -334,7 +334,7 @@ def parse_and_normalize_eds_spd(self, fp, src: str, ckey: str): # thereby these EDAX energy count arrays are just some payload inside a set of compressed chunks # without some extra logic to resolve the third (energy) dimension reading them can be super inefficient # so let's read chunk-by-chunk to reuse chunk cache, hopefully... - chk_bnds = {"x": [], "y": []} + chk_bnds: Dict = {"x": [], "y": []} chk_info = {"ny": nyxe["y"], "cy": fp[f"{src}/SPD"].chunks[0], "nx": nyxe["x"], "cx": fp[f"{src}/SPD"].chunks[1]} for dim in ["y", "x"]: diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_png_protochips.py b/pynxtools/dataconverter/readers/em/subparsers/image_png_protochips.py index 994702814..49adfd67e 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/image_png_protochips.py +++ b/pynxtools/dataconverter/readers/em/subparsers/image_png_protochips.py @@ -24,7 +24,6 @@ from typing import Dict from PIL import Image from zipfile import ZipFile -from collections import OrderedDict from pynxtools.dataconverter.readers.em.subparsers.image_png_protochips_concepts import \ get_protochips_variadic_concept @@ -34,28 +33,9 @@ import variadic_path_to_specific_path from pynxtools.dataconverter.readers.em.subparsers.image_png_protochips_modifier import \ get_nexus_value -from pynxtools.dataconverter.readers.em.subparsers.image_base import \ - ImgsBaseParser - - -def flatten_xml_to_dict(xml_content) -> dict: - # https://codereview.stackexchange.com/a/21035 - # https://stackoverflow.com/questions/38852822/how-to-flatten-xml-file-in-python - def items(): - for key, value in xml_content.items(): - # nested subtree - if isinstance(value, dict): - for subkey, subvalue in flatten_xml_to_dict(value).items(): - yield '{}.{}'.format(key, subkey), subvalue - # nested list - elif isinstance(value, list): - for num, elem in enumerate(value): - for subkey, subvalue in flatten_xml_to_dict(elem).items(): - yield '{}.[{}].{}'.format(key, num, subkey), subvalue - # everything else (only leafs should remain) - else: - yield key, value - return OrderedDict(items()) +from pynxtools.dataconverter.readers.em.subparsers.image_base import ImgsBaseParser +from pynxtools.dataconverter.readers.em.utils.xml_utils import flatten_xml_to_dict +from pynxtools.dataconverter.readers.shared.shared_utils import get_sha256_of_file_content class ProtochipsPngSetSubParser(ImgsBaseParser): @@ -99,8 +79,8 @@ def check_if_zipped_png_protochips(self): try: nparr = np.array(png) self.png_info[file] = np.shape(nparr) - except: - raise ValueError(f"Loading image data in-place from {self.file_path}:{file} failed !") + except IOError: + print(f"Loading image data in-place from {self.file_path}:{file} failed !") if method == "smart": # knowing where to hunt width and height in PNG metadata # https://dev.exiv2.org/projects/exiv2/wiki/The_Metadata_in_PNG_files magic = fp.read(8) @@ -125,6 +105,53 @@ def check_if_zipped_png_protochips(self): print("All tests passed successfully") self.supported = True + def get_xml_metadata(self, file, fp): + try: + fp.seek(0) + with Image.open(fp) as png: + png.load() + if "MicroscopeControlImage" in png.info.keys(): + meta = flatten_xml_to_dict( + xmltodict.parse(png.info["MicroscopeControlImage"])) + # first phase analyse the collection of Protochips metadata concept instance symbols and reduce to unique concepts + grpnm_lookup = {} + for concept, value in meta.items(): + # not every key is allowed to define a concept + # print(f"{concept}: {value}") + idxs = re.finditer(r".\[[0-9]+\].", concept) + if (sum(1 for _ in idxs) > 0): # is_variadic + markers = [".Name", ".PositionerName"] + for marker in markers: + if concept.endswith(marker): + grpnm_lookup[f"{concept[0:len(concept)-len(marker)]}"] = value + else: + grpnm_lookup[concept] = value + # second phase, evaluate each concept instance symbol wrt to its prefix coming from the unique concept + self.tmp["meta"][file] = {} + for k, v in meta.items(): + grpnms = None + idxs = re.finditer(r".\[[0-9]+\].", k) + if (sum(1 for _ in idxs) > 0): # is variadic + search_argument = k[0:k.rfind("].") + 1] + for parent_grpnm, child_grpnm in grpnm_lookup.items(): + if parent_grpnm.startswith(search_argument): + grpnms = (parent_grpnm, child_grpnm) + break + if grpnms is not None: + if len(grpnms) == 2: + if "PositionerSettings" in k and k.endswith(".PositionerName") is False: + self.tmp["meta"][file][f"{grpnms[0]}.{grpnms[1]}{k[k.rfind('.') + 1:]}"] = v + if k.endswith(".Value"): + self.tmp["meta"][file][f"{grpnms[0]}.{grpnms[1]}"] = v + else: + self.tmp["meta"][file][f"{k}"] = v + # TODO::simplify and check that metadata end up correctly in self.tmp["meta"][file] + except ValueError: + print(f"Flattening XML metadata content {self.file_path}:{file} failed !") + + def get_file_hash(self, file, fp): + self.tmp["meta"][file]["sha256"] = get_sha256_of_file_content(fp) + def parse_and_normalize(self): """Perform actual parsing filling cache self.tmp.""" if self.supported is True: @@ -133,47 +160,13 @@ def parse_and_normalize(self): with ZipFile(self.file_path) as zip_file_hdl: for file in self.png_info.keys(): with zip_file_hdl.open(file) as fp: - try: - with Image.open(fp) as png: - png.load() - if "MicroscopeControlImage" in png.info.keys(): - meta = flatten_xml_to_dict( - xmltodict.parse(png.info["MicroscopeControlImage"])) - # first phase analyse the collection of Protochips metadata concept instance symbols and reduce to unique concepts - self.tmp["meta"][file] = {} - for concept, value in meta.items(): - # not every key is allowed to define a concept - # print(f"{concept}: {value}") - idxs = re.finditer(".\[[0-9]+\].", concept) - if (sum(1 for _ in idxs) > 0): # is_variadic - markers = [".Name", ".PositionerName"] - for marker in markers: - if concept.endswith(marker): - self.tmp["meta"][file][f"{concept[0:len(concept)-len(marker)]}"] = value - else: - self.tmp["meta"][file][concept] = value - # print(f"First phase of metadata parsing {self.file_path}:{file} successful") - # second phase, evaluate each concept instance symbol wrt to its prefix coming from the unique concept - for k, v in meta.items(): - grpnms = None - idxs = re.finditer(".\[[0-9]+\].", k) - if (sum(1 for _ in idxs) > 0): # is variadic - search_argument = k[0:k.rfind("].")+1] - for parent_grpnm, child_grpnm in self.tmp["meta"][file].items(): - if parent_grpnm.startswith(search_argument): - grpnms = (parent_grpnm, child_grpnm) - break - if grpnms is not None: - if len(grpnms) == 2: - if "PositionerSettings" in k and k.endswith(".PositionerName") is False: - print(f"vv: {grpnms[0]}.{grpnms[1]}{k[k.rfind('.') + 1:]}: {v}") - if k.endswith(".Value"): - print(f"vv: {grpnms[0]}.{grpnms[1]}: {v}") - else: - print(f"nv: {k}: {v}") - # TODO::simplify and check that metadata end up correctly in self.tmp["meta"][file] - except: - raise ValueError(f"Flattening XML metadata content {self.file_path}:{file} failed !") + self.get_xml_metadata(file, fp) + self.get_file_hash(file, fp) + # print(f"Debugging self.tmp.file.items {file}") + # for k, v in self.tmp["meta"][file].items(): + # print(f"{k}: {v}") + print(f"{self.file_path} metadata within PNG collection processed " + f"successfully ({len(self.tmp['meta'].keys())} PNGs evaluated).") else: print(f"{self.file_path} is not a Protochips-specific " f"PNG file that this parser can process !") @@ -181,7 +174,7 @@ def parse_and_normalize(self): def process_into_template(self, template: dict) -> dict: if self.supported is True: self.process_event_data_em_metadata(template) - self.process_event_data_em_data(template) + # self.process_event_data_em_data(template) return template def process_event_data_em_metadata(self, template: dict) -> dict: diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_concepts.py b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_concepts.py index 8e7f2d69d..4ada716d4 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_concepts.py +++ b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_concepts.py @@ -27,243 +27,243 @@ # taken for each detector and imaging mode # we then assume that one can work with the joint set of these concepts -TIFF_TFS_PARENT_CONCEPTS = ["Accessories", - "Beam", - "ColdStage", - "CompoundLensFilter", - "Detectors", - "EBeam", - "EBeamDeceleration", - "EScan", - "ETD", - "EasyLift", - "GIS", - "HiResIllumination", - "HotStage", - "HotStageHVHS", - "HotStageMEMS", - "IRBeam", - "Image", - "Nav-Cam", - "PrivateFei", - "Scan", - "Specimen", - "Stage", - "System", - "T1", - "T2", - "T3", - "User", - "Vacuum"] +TIFF_TFS_PARENT_CONCEPTS = ["Accessories", + "Beam", + "ColdStage", + "CompoundLensFilter", + "Detectors", + "EBeam", + "EBeamDeceleration", + "EScan", + "ETD", + "EasyLift", + "GIS", + "HiResIllumination", + "HotStage", + "HotStageHVHS", + "HotStageMEMS", + "IRBeam", + "Image", + "Nav-Cam", + "PrivateFei", + "Scan", + "Specimen", + "Stage", + "System", + "T1", + "T2", + "T3", + "User", + "Vacuum"] TIFF_TFS_ALL_CONCEPTS = ["Accessories/Number", - "Beam/Beam", - "Beam/BeamShiftX", - "Beam/BeamShiftY", - "Beam/FineStageBias", - "Beam/HV", - "Beam/ImageMode", - "Beam/Scan", - "Beam/ScanRotation", - "Beam/Spot", - "Beam/StigmatorX", - "Beam/StigmatorY", - "ColdStage/ActualTemperature", - "ColdStage/Humidity", - "ColdStage/SampleBias", - "ColdStage/TargetTemperature", - "CompoundLensFilter/IsOn", - "CompoundLensFilter/ThresholdEnergy", - "Detectors/Mode", - "Detectors/Name", - "Detectors/Number", - "EasyLift/Rotation", - "EBeam/Acq", - "EBeam/Aperture", - "EBeam/ApertureDiameter", - "EBeam/ATubeVoltage", - "EBeam/BeamCurrent", - "EBeam/BeamMode", - "EBeam/BeamShiftX", - "EBeam/BeamShiftY", - "EBeam/ColumnType", - "EBeam/DynamicFocusIsOn", - "EBeam/DynamicWDIsOn", - "EBeam/EmissionCurrent", - "EBeam/EucWD", - "EBeam/FinalLens", - "EBeam/HFW", - "EBeam/HV", - "EBeam/ImageMode", - "EBeam/LensMode", - "EBeam/LensModeA", - "EBeam/MagnificationCorrection", - "EBeam/PreTilt", - "EBeam/ScanRotation", - "EBeam/SemOpticalMode", - "EBeam/Source", - "EBeam/SourceTiltX", - "EBeam/SourceTiltY", - "EBeam/StageR", - "EBeam/StageTa", - "EBeam/StageTb", - "EBeam/StageX", - "EBeam/StageY", - "EBeam/StageZ", - "EBeam/StigmatorX", - "EBeam/StigmatorY", - "EBeam/TiltCorrectionAngle", - "EBeam/TiltCorrectionIsOn", - "EBeam/UseCase", - "EBeam/VFW", - "EBeam/WD", - "EBeam/WehneltBias", - "EBeamDeceleration/ImmersionRatio", - "EBeamDeceleration/LandingEnergy", - "EBeamDeceleration/ModeOn", - "EBeamDeceleration/StageBias", - "EScan/Dwell", - "EScan/FrameTime", - "EScan/HorFieldsize", - "EScan/InternalScan", - "EScan/LineIntegration", - "EScan/LineTime", - "EScan/Mainslock", - "EScan/PixelHeight", - "EScan/PixelWidth", - "EScan/Scan", - "EScan/ScanInterlacing", - "EScan/VerFieldsize", - "ETD/Brightness", - "ETD/BrightnessDB", - "ETD/Contrast", - "ETD/ContrastDB", - "ETD/Grid", - "ETD/MinimumDwellTime", - "ETD/Mix", - "ETD/Setting", - "ETD/Signal", - "GIS/Number", - "HiResIllumination/BrightFieldIsOn", - "HiResIllumination/BrightFieldValue", - "HiResIllumination/DarkFieldIsOn", - "HiResIllumination/DarkFieldValue", - "HotStage/ActualTemperature", - "HotStage/SampleBias", - "HotStage/ShieldBias", - "HotStage/TargetTemperature", - "HotStageHVHS/ActualTemperature", - "HotStageHVHS/SampleBias", - "HotStageHVHS/ShieldBias", - "HotStageHVHS/TargetTemperature", - "HotStageMEMS/ActualTemperature", - "HotStageMEMS/HeatingCurrent", - "HotStageMEMS/HeatingPower", - "HotStageMEMS/HeatingVoltage", - "HotStageMEMS/SampleBias", - "HotStageMEMS/SampleResistance", - "HotStageMEMS/TargetTemperature", - "Image/Average", - "Image/DigitalBrightness", - "Image/DigitalContrast", - "Image/DigitalGamma", - "Image/DriftCorrected", - "Image/Integrate", - "Image/MagCanvasRealWidth", - "Image/MagnificationMode", - "Image/PostProcessing", - "Image/ResolutionX", - "Image/ResolutionY", - "Image/ScreenMagCanvasRealWidth", - "Image/ScreenMagnificationMode", - "Image/Transformation", - "Image/ZoomFactor", - "Image/ZoomPanX", - "Image/ZoomPanY", - "IRBeam/HFW", - "IRBeam/n", - "IRBeam/ScanRotation", - "IRBeam/SiDepth", - "IRBeam/StageR", - "IRBeam/StageTa", - "IRBeam/StageTb", - "IRBeam/StageX", - "IRBeam/StageY", - "IRBeam/StageZ", - "IRBeam/VFW", - "IRBeam/WD", - "PrivateFei/BitShift", - "PrivateFei/DataBarAvailable", - "PrivateFei/DatabarHeight", - "PrivateFei/DataBarSelected", - "PrivateFei/TimeOfCreation", - "Scan/Average", - "Scan/Dwelltime", - "Scan/FrameTime", - "Scan/HorFieldsize", - "Scan/Integrate", - "Scan/InternalScan", - "Scan/PixelHeight", - "Scan/PixelWidth", - "Scan/VerFieldsize", - "Specimen/SpecimenCurrent", - "Specimen/Temperature", - "Stage/ActiveStage", - "Stage/SpecTilt", - "Stage/StageR", - "Stage/StageT", - "Stage/StageTb", - "Stage/StageX", - "Stage/StageY", - "Stage/StageZ", - "Stage/WorkingDistance", - "System/Acq", - "System/Aperture", - "System/BuildNr", - "System/Chamber", - "System/Column", - "System/DisplayHeight", - "System/DisplayWidth", - "System/Dnumber", - "System/ESEM", - "System/EucWD", - "System/FinalLens", - "System/Pump", - "System/Scan", - "System/Software", - "System/Source", - "System/Stage", - "System/SystemType", - "System/Type", - "T1/Brightness", - "T1/BrightnessDB", - "T1/Contrast", - "T1/ContrastDB", - "T1/MinimumDwellTime", - "T1/Setting", - "T1/Signal", - "T2/Brightness", - "T2/BrightnessDB", - "T2/Contrast", - "T2/ContrastDB", - "T2/MinimumDwellTime", - "T2/Setting", - "T2/Signal", - "T3/Brightness", - "T3/BrightnessDB", - "T3/Contrast", - "T3/ContrastDB", - "T3/MinimumDwellTime", - "T3/Signal", - "User/Date", - "User/Time", - "User/User", - "User/UserText", - "User/UserTextUnicode", - "Vacuum/ChPressure", - "Vacuum/Gas", - "Vacuum/Humidity", - "Vacuum/UserMode"] + "Beam/Beam", + "Beam/BeamShiftX", + "Beam/BeamShiftY", + "Beam/FineStageBias", + "Beam/HV", + "Beam/ImageMode", + "Beam/Scan", + "Beam/ScanRotation", + "Beam/Spot", + "Beam/StigmatorX", + "Beam/StigmatorY", + "ColdStage/ActualTemperature", + "ColdStage/Humidity", + "ColdStage/SampleBias", + "ColdStage/TargetTemperature", + "CompoundLensFilter/IsOn", + "CompoundLensFilter/ThresholdEnergy", + "Detectors/Mode", + "Detectors/Name", + "Detectors/Number", + "EasyLift/Rotation", + "EBeam/Acq", + "EBeam/Aperture", + "EBeam/ApertureDiameter", + "EBeam/ATubeVoltage", + "EBeam/BeamCurrent", + "EBeam/BeamMode", + "EBeam/BeamShiftX", + "EBeam/BeamShiftY", + "EBeam/ColumnType", + "EBeam/DynamicFocusIsOn", + "EBeam/DynamicWDIsOn", + "EBeam/EmissionCurrent", + "EBeam/EucWD", + "EBeam/FinalLens", + "EBeam/HFW", + "EBeam/HV", + "EBeam/ImageMode", + "EBeam/LensMode", + "EBeam/LensModeA", + "EBeam/MagnificationCorrection", + "EBeam/PreTilt", + "EBeam/ScanRotation", + "EBeam/SemOpticalMode", + "EBeam/Source", + "EBeam/SourceTiltX", + "EBeam/SourceTiltY", + "EBeam/StageR", + "EBeam/StageTa", + "EBeam/StageTb", + "EBeam/StageX", + "EBeam/StageY", + "EBeam/StageZ", + "EBeam/StigmatorX", + "EBeam/StigmatorY", + "EBeam/TiltCorrectionAngle", + "EBeam/TiltCorrectionIsOn", + "EBeam/UseCase", + "EBeam/VFW", + "EBeam/WD", + "EBeam/WehneltBias", + "EBeamDeceleration/ImmersionRatio", + "EBeamDeceleration/LandingEnergy", + "EBeamDeceleration/ModeOn", + "EBeamDeceleration/StageBias", + "EScan/Dwell", + "EScan/FrameTime", + "EScan/HorFieldsize", + "EScan/InternalScan", + "EScan/LineIntegration", + "EScan/LineTime", + "EScan/Mainslock", + "EScan/PixelHeight", + "EScan/PixelWidth", + "EScan/Scan", + "EScan/ScanInterlacing", + "EScan/VerFieldsize", + "ETD/Brightness", + "ETD/BrightnessDB", + "ETD/Contrast", + "ETD/ContrastDB", + "ETD/Grid", + "ETD/MinimumDwellTime", + "ETD/Mix", + "ETD/Setting", + "ETD/Signal", + "GIS/Number", + "HiResIllumination/BrightFieldIsOn", + "HiResIllumination/BrightFieldValue", + "HiResIllumination/DarkFieldIsOn", + "HiResIllumination/DarkFieldValue", + "HotStage/ActualTemperature", + "HotStage/SampleBias", + "HotStage/ShieldBias", + "HotStage/TargetTemperature", + "HotStageHVHS/ActualTemperature", + "HotStageHVHS/SampleBias", + "HotStageHVHS/ShieldBias", + "HotStageHVHS/TargetTemperature", + "HotStageMEMS/ActualTemperature", + "HotStageMEMS/HeatingCurrent", + "HotStageMEMS/HeatingPower", + "HotStageMEMS/HeatingVoltage", + "HotStageMEMS/SampleBias", + "HotStageMEMS/SampleResistance", + "HotStageMEMS/TargetTemperature", + "Image/Average", + "Image/DigitalBrightness", + "Image/DigitalContrast", + "Image/DigitalGamma", + "Image/DriftCorrected", + "Image/Integrate", + "Image/MagCanvasRealWidth", + "Image/MagnificationMode", + "Image/PostProcessing", + "Image/ResolutionX", + "Image/ResolutionY", + "Image/ScreenMagCanvasRealWidth", + "Image/ScreenMagnificationMode", + "Image/Transformation", + "Image/ZoomFactor", + "Image/ZoomPanX", + "Image/ZoomPanY", + "IRBeam/HFW", + "IRBeam/n", + "IRBeam/ScanRotation", + "IRBeam/SiDepth", + "IRBeam/StageR", + "IRBeam/StageTa", + "IRBeam/StageTb", + "IRBeam/StageX", + "IRBeam/StageY", + "IRBeam/StageZ", + "IRBeam/VFW", + "IRBeam/WD", + "PrivateFei/BitShift", + "PrivateFei/DataBarAvailable", + "PrivateFei/DatabarHeight", + "PrivateFei/DataBarSelected", + "PrivateFei/TimeOfCreation", + "Scan/Average", + "Scan/Dwelltime", + "Scan/FrameTime", + "Scan/HorFieldsize", + "Scan/Integrate", + "Scan/InternalScan", + "Scan/PixelHeight", + "Scan/PixelWidth", + "Scan/VerFieldsize", + "Specimen/SpecimenCurrent", + "Specimen/Temperature", + "Stage/ActiveStage", + "Stage/SpecTilt", + "Stage/StageR", + "Stage/StageT", + "Stage/StageTb", + "Stage/StageX", + "Stage/StageY", + "Stage/StageZ", + "Stage/WorkingDistance", + "System/Acq", + "System/Aperture", + "System/BuildNr", + "System/Chamber", + "System/Column", + "System/DisplayHeight", + "System/DisplayWidth", + "System/Dnumber", + "System/ESEM", + "System/EucWD", + "System/FinalLens", + "System/Pump", + "System/Scan", + "System/Software", + "System/Source", + "System/Stage", + "System/SystemType", + "System/Type", + "T1/Brightness", + "T1/BrightnessDB", + "T1/Contrast", + "T1/ContrastDB", + "T1/MinimumDwellTime", + "T1/Setting", + "T1/Signal", + "T2/Brightness", + "T2/BrightnessDB", + "T2/Contrast", + "T2/ContrastDB", + "T2/MinimumDwellTime", + "T2/Setting", + "T2/Signal", + "T3/Brightness", + "T3/BrightnessDB", + "T3/Contrast", + "T3/ContrastDB", + "T3/MinimumDwellTime", + "T3/Signal", + "User/Date", + "User/Time", + "User/User", + "User/UserText", + "User/UserTextUnicode", + "Vacuum/ChPressure", + "Vacuum/Gas", + "Vacuum/Humidity", + "Vacuum/UserMode"] # there is more to know and understand than just knowing TFS/FEI uses # the above-mentioned concepts in their taxonomy: diff --git a/pynxtools/dataconverter/readers/em/subparsers/rsciio_bruker.py b/pynxtools/dataconverter/readers/em/subparsers/rsciio_bruker.py index 13c32c7dd..8d4cbef24 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/rsciio_bruker.py +++ b/pynxtools/dataconverter/readers/em/subparsers/rsciio_bruker.py @@ -44,7 +44,7 @@ def check_if_supported(self): # based on this one could then plan how much memory has to be reserved # in the template and stream out accordingly self.supported = True - except: + except IOError: print(f"Loading {self.file_path} using {self.__name__} is not supported !") def parse_and_normalize(self): @@ -72,9 +72,9 @@ def process_into_template(self, template: dict) -> dict: self.process_event_data_em_metadata(template) self.process_event_data_em_data(template) return template - + def process_event_data_em_metadata(self, template: dict) -> dict: return template def process_event_data_em_data(self, template: dict) -> dict: - return template \ No newline at end of file + return template diff --git a/pynxtools/dataconverter/readers/em/utils/xml_utils.py b/pynxtools/dataconverter/readers/em/utils/xml_utils.py new file mode 100644 index 000000000..c7a3223f8 --- /dev/null +++ b/pynxtools/dataconverter/readers/em/utils/xml_utils.py @@ -0,0 +1,41 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Flatten content of an XML tree into a Python dictionary.""" + +from collections import OrderedDict + + +def flatten_xml_to_dict(xml_content) -> dict: + """Flatten content of an XML tree into a Python dictionary.""" + # https://codereview.stackexchange.com/a/21035 + # https://stackoverflow.com/questions/38852822/how-to-flatten-xml-file-in-python + def items(): + for key, value in xml_content.items(): + # nested subtree + if isinstance(value, dict): + for subkey, subvalue in flatten_xml_to_dict(value).items(): + yield '{}.{}'.format(key, subkey), subvalue + # nested list + elif isinstance(value, list): + for num, elem in enumerate(value): + for subkey, subvalue in flatten_xml_to_dict(elem).items(): + yield '{}.[{}].{}'.format(key, num, subkey), subvalue + # everything else (only leafs should remain) + else: + yield key, value + return OrderedDict(items()) diff --git a/pynxtools/dataconverter/readers/shared/shared_utils.py b/pynxtools/dataconverter/readers/shared/shared_utils.py index 629e29a0f..5311bb3a1 100644 --- a/pynxtools/dataconverter/readers/shared/shared_utils.py +++ b/pynxtools/dataconverter/readers/shared/shared_utils.py @@ -23,6 +23,7 @@ # pylint: disable=E1101, R0801 # import git +import hashlib def get_repo_last_commit() -> str: @@ -36,6 +37,16 @@ def get_repo_last_commit() -> str: return "unknown git commit id or unable to parse git reverse head" +def get_sha256_of_file_content(file_hdl) -> str: + """Compute a hashvalue of given file, here SHA256.""" + file_hdl.seek(0) + # Read and update hash string value in blocks of 4K + sha256_hash = hashlib.sha256() + for byte_block in iter(lambda: file_hdl.read(4096), b""): + sha256_hash.update(byte_block) + return str(sha256_hash.hexdigest()) + + class NxObject: # pylint: disable=R0903 """An object in a graph e.g. a field or group in NeXus."""