diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py index 30aa5ffaa..79b6a9375 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py +++ b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py @@ -28,6 +28,10 @@ TiffTfsConcepts, TiffTfsToNeXusCfg, get_fei_parent_concepts, get_fei_childs from pynxtools.dataconverter.readers.em.utils.image_utils import \ sort_ascendingly_by_second_argument, if_str_represents_float +from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ + import variadic_path_to_specific_path +from pynxtools.dataconverter.readers.em.subparsers.image_tiff_tfs_modifier import \ + get_nexus_value class TfsTiffSubParser(TiffSubParser): @@ -141,18 +145,13 @@ def parse_and_normalize(self): f"TIFF file that this parser can process !") def process_into_template(self, template: dict) -> dict: - self.process_event_data_em_metadata(template) - self.process_event_data_em_data(template) + if self.supported is True: + self.process_event_data_em_metadata(template) + self.process_event_data_em_data(template) return template - def process_event_data_em_metadata(self, template: dict) -> dict: - """Add respective event_data_em header.""" - # contextualization to understand how the image relates to the EM session - print(f"Mapping some of the TFS/FEI metadata concepts onto NeXus concepts") - return template - def process_event_data_em_data(self, template: dict) -> dict: - """Add respective heavy image data.""" + """Add respective heavy data.""" # default display of the image(s) representing the data collected in this event print(f"Writing TFS/FEI TIFF image as a onto the respective NeXus concept") # read image in-place @@ -215,8 +214,14 @@ def process_event_data_em_data(self, template: dict) -> dict: template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit[dim]}" return template - def process_event_data_em_state(self, template: dict) -> dict: - """Add em-state as they were during the event_data_em event.""" - # state of the microscope not repeating static/long-valid microscope metadata - print(f"Writing TFS/FEI event_data_em state") + def process_event_data_em_metadata(self, template: dict) -> dict: + """Add respective metadata.""" + # contextualization to understand how the image relates to the EM session + print(f"Mapping some of the TFS/FEI metadata concepts onto NeXus concepts") + identifier = [self.entry_id, self.event_id, 1] + for nx_path, modifier in TiffTfsToNeXusCfg.items(): + if (nx_path != "IGNORE") and (nx_path != "UNCLEAR"): + trg = variadic_path_to_specific_path(nx_path, identifier) + template[trg] = get_nexus_value(modifier, self.tmp["meta"]) + # print(f"nx_path: {nx_path}, trg: {trg}, tfs_concept: {template[trg]}\n") return template diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py index e38999b68..0452cc6ad 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py +++ b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py @@ -195,6 +195,23 @@ "ColdStage/Humidity", "ColdStage/SampleBias"] +# there is more to know and understand than just knowing TFS/FEI uses +# the above-mentioned concepts in their taxonomy: +# take the example of System/Source for which an example file (instance) has the +# value "FEG" +# similar like in NeXus "System/Source" labels a concept for which (assumption!) there +# is a controlled enumeration of symbols possible (as the example shows "FEG" is one such +# allowed symbol of the enumeration. +# The key issue is that the symbols for the leaf (here "FEG") means nothing eventually +# when one has another semantic world-view, like in NOMAD metainfo or NeXus +# (only us) humans understand that what TFS/FEI likely means with the symbol +# "FEG" is exactly the same as what we mean in NeXus when setting emitter_type of +# NXebeam_column to "cold_cathode_field_emitter" +# world with the controlled enumeration value "other" because we do not know +# if FEG means really a filament or a cold_cathode_field_emitter + +TfsToNexusConceptMapping = {"System/Source/FEG": "cold_field_cathode_emitter"} + def get_fei_parent_concepts() -> List: """Get list of unique FEI parent concepts.""" @@ -215,7 +232,8 @@ def get_fei_childs(concept: str) -> List: return list(child_concepts) -TiffTfsToNeXusCfg = {"/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/start_time": {"fun": "ikz_berlin_apreo_iso8601", "terms": ["User/Date", "User/Time"]}, +# "/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/start_time" +TiffTfsToNeXusCfg = {"IGNORE": {"fun": "ikz_berlin_apreo_iso8601", "terms": ["User/Date", "User/Time"]}, "IGNORE": { "fun": "load_from", "terms": "User/User" }, "IGNORE": { "fun": "load_from", "terms": "User/UserText" }, "IGNORE": { "fun": "load_from", "terms": "User/UserTextUnicode" }, @@ -223,8 +241,8 @@ def get_fei_childs(concept: str) -> List: "IGNORE": { "fun": "load_from", "terms": "System/Dnumber" }, "IGNORE": { "fun": "load_from", "terms": "System/Software" }, "/ENTRY[entry*]/measurement/em_lab/FABRICATION[fabrication]/identifier": { "fun": "load_from", "terms": "System/BuildNr" }, - "/ENTRY[entry*]/measurement/em_lab/EBEAM_COLUMN[ebeam_column]/electron_source/emitter_type": { "fun": "ikz_berlin_apreo", "terms": "System/Source" }, - "/ENTRY[entry*]/measurement/em_lab/FABRICATION[fabrication]/vendor": { "fun": "load_from", "terms": "System/Column" }, + "/ENTRY[entry*]/measurement/em_lab/EBEAM_COLUMN[ebeam_column]/electron_source/emitter_type": { "fun": "tfs_to_nexus", "terms": "System/Source" }, + "/ENTRY[entry*]/measurement/em_lab/FABRICATION[fabrication]/vendor": "FEI", "IGNORE": { "fun": "load_from", "terms": "System/FinalLens" }, "IGNORE": { "fun": "load_from", "terms": "System/Chamber" }, "IGNORE": { "fun": "load_from", "terms": "System/Stage" }, diff --git a/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_modifier.py b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_modifier.py new file mode 100644 index 000000000..dddd59bd3 --- /dev/null +++ b/pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_modifier.py @@ -0,0 +1,49 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utilities for working with TFS/FEI-specific concepts.""" + +# pylint: disable=no-member + +from pynxtools.dataconverter.readers.em.subparsers.image_tiff_tfs_cfg import \ + TfsToNexusConceptMapping + + +def get_nexus_value(modifier, metadata: dict): + """Interpret a functional mapping using data from dct via calling modifiers.""" + if isinstance(modifier, dict): + # different commands are available + if set(["fun", "terms"]) == set(modifier.keys()): + if modifier["fun"] == "load_from": + if modifier["terms"] in metadata.keys(): + return metadata[modifier['terms']] + else: + raise ValueError(f"Unable to interpret modififier load_from for argument {modifier['terms']}") + if modifier["fun"] == "tfs_to_nexus": + # print(metadata[modifier['terms']]) + if f"{modifier['terms']}/{metadata[modifier['terms']]}" in TfsToNexusConceptMapping.keys(): + return TfsToNexusConceptMapping[f"{modifier['terms']}/{metadata[modifier['terms']]}"] + else: + raise ValueError(f"Unable to interpret modifier tfs_to_nexus for argument {modifier['terms']}/{metadata[modifier['terms']]}") + else: + print(f"WARNING::Modifier {modifier} is currently not implemented !") + # elif set(["link"]) == set(modifier.keys()), with the jsonmap reader Sherjeel conceptualized "link" + return None + elif isinstance(modifier, str): + return modifier # metadata[modifier] + else: + return None diff --git a/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py b/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py index 6ee855b84..8851e3427 100644 --- a/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py +++ b/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py @@ -78,11 +78,9 @@ def apply_modifier(modifier, dct: dict): return load_from_modifier(modifier["terms"], dct) if modifier["fun"] == "convert_iso8601": return convert_iso8601_modifier(modifier["terms"], dct) - elif set(["link"]) == set(modifier.keys()): - # CURRENTLY NOT IMPLEMENTED - # with the jsonmap reader Sherjeel conceptualized "link" - return None else: + print(f"WARNING::Modifier {modifier} is currently not implemented !") + # elif set(["link"]) == set(modifier.keys()), with the jsonmap reader Sherjeel conceptualized "link" return None if isinstance(modifier, str): return modifier