Made also now the mapping of TFS vocabulary to NeXus work, awaiting n…

…ow feedback from IKZ to implement mapping of further quantities, test NeXus file writes successfully, next step refactor em_spctrscpy parser

pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs.py

@@ -28,6 +28,10 @@
     TiffTfsConcepts, TiffTfsToNeXusCfg, get_fei_parent_concepts, get_fei_childs
 from pynxtools.dataconverter.readers.em.utils.image_utils import \
     sort_ascendingly_by_second_argument, if_str_represents_float
+from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \
+    import variadic_path_to_specific_path
+from pynxtools.dataconverter.readers.em.subparsers.image_tiff_tfs_modifier import \
+    get_nexus_value
 
 
 class TfsTiffSubParser(TiffSubParser):
@@ -141,18 +145,13 @@ def parse_and_normalize(self):
                   f"TIFF file that this parser can process !")
 
     def process_into_template(self, template: dict) -> dict:
-        self.process_event_data_em_metadata(template)
-        self.process_event_data_em_data(template)
+        if self.supported is True:
+            self.process_event_data_em_metadata(template)
+            self.process_event_data_em_data(template)
         return template
 
-    def process_event_data_em_metadata(self, template: dict) -> dict:
-        """Add respective event_data_em header."""
-        # contextualization to understand how the image relates to the EM session
-        print(f"Mapping some of the TFS/FEI metadata concepts onto NeXus concepts")
-        return template
-
     def process_event_data_em_data(self, template: dict) -> dict:
-        """Add respective heavy image data."""
+        """Add respective heavy data."""
         # default display of the image(s) representing the data collected in this event
         print(f"Writing TFS/FEI TIFF image as a onto the respective NeXus concept")
         # read image in-place
@@ -215,8 +214,14 @@ def process_event_data_em_data(self, template: dict) -> dict:
                 template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit[dim]}"
         return template
 
-    def process_event_data_em_state(self, template: dict) -> dict:
-        """Add em-state as they were during the event_data_em event."""
-        # state of the microscope not repeating static/long-valid microscope metadata
-        print(f"Writing TFS/FEI event_data_em state")
+    def process_event_data_em_metadata(self, template: dict) -> dict:
+        """Add respective metadata."""
+        # contextualization to understand how the image relates to the EM session
+        print(f"Mapping some of the TFS/FEI metadata concepts onto NeXus concepts")
+        identifier = [self.entry_id, self.event_id, 1]
+        for nx_path, modifier in TiffTfsToNeXusCfg.items():
+            if (nx_path != "IGNORE") and (nx_path != "UNCLEAR"):
+                trg = variadic_path_to_specific_path(nx_path, identifier)
+                template[trg] = get_nexus_value(modifier, self.tmp["meta"])
+                # print(f"nx_path: {nx_path}, trg: {trg}, tfs_concept: {template[trg]}\n")
         return template

pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_cfg.py

@@ -195,6 +195,23 @@
                    "ColdStage/Humidity",
                    "ColdStage/SampleBias"]
 
+# there is more to know and understand than just knowing TFS/FEI uses
+# the above-mentioned concepts in their taxonomy:
+# take the example of System/Source for which an example file (instance) has the
+# value "FEG"
+# similar like in NeXus "System/Source" labels a concept for which (assumption!) there
+# is a controlled enumeration of symbols possible (as the example shows "FEG" is one such
+# allowed symbol of the enumeration.
+# The key issue is that the symbols for the leaf (here "FEG") means nothing eventually
+# when one has another semantic world-view, like in NOMAD metainfo or NeXus
+# (only us) humans understand that what TFS/FEI likely means with the symbol
+# "FEG" is exactly the same as what we mean in NeXus when setting emitter_type of
+# NXebeam_column to "cold_cathode_field_emitter"
+# world with the controlled enumeration value "other" because we do not know
+# if FEG means really a filament or a cold_cathode_field_emitter
+
+TfsToNexusConceptMapping = {"System/Source/FEG": "cold_field_cathode_emitter"}
+
 
 def get_fei_parent_concepts() -> List:
     """Get list of unique FEI parent concepts."""
@@ -215,16 +232,17 @@ def get_fei_childs(concept: str) -> List:
     return list(child_concepts)
 
 
-TiffTfsToNeXusCfg = {"/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/start_time": {"fun": "ikz_berlin_apreo_iso8601", "terms": ["User/Date", "User/Time"]},
+# "/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/start_time"
+TiffTfsToNeXusCfg = {"IGNORE": {"fun": "ikz_berlin_apreo_iso8601", "terms": ["User/Date", "User/Time"]},
                      "IGNORE": { "fun": "load_from", "terms": "User/User" },	
                      "IGNORE": { "fun": "load_from", "terms": "User/UserText" },
                      "IGNORE": { "fun": "load_from", "terms": "User/UserTextUnicode" },	
                      "IGNORE": { "fun": "load_from", "terms": "System/Type" },
                      "IGNORE": { "fun": "load_from", "terms": "System/Dnumber" },
                      "IGNORE": { "fun": "load_from", "terms": "System/Software" },
                      "/ENTRY[entry*]/measurement/em_lab/FABRICATION[fabrication]/identifier": { "fun": "load_from", "terms": "System/BuildNr" },
-                     "/ENTRY[entry*]/measurement/em_lab/EBEAM_COLUMN[ebeam_column]/electron_source/emitter_type": { "fun": "ikz_berlin_apreo", "terms": "System/Source" },
-                     "/ENTRY[entry*]/measurement/em_lab/FABRICATION[fabrication]/vendor": { "fun": "load_from", "terms": "System/Column" },	
+                     "/ENTRY[entry*]/measurement/em_lab/EBEAM_COLUMN[ebeam_column]/electron_source/emitter_type": { "fun": "tfs_to_nexus", "terms": "System/Source" },
+                     "/ENTRY[entry*]/measurement/em_lab/FABRICATION[fabrication]/vendor": "FEI",
                      "IGNORE": { "fun": "load_from", "terms": "System/FinalLens" },
                      "IGNORE": { "fun": "load_from", "terms": "System/Chamber" },
                      "IGNORE": { "fun": "load_from", "terms": "System/Stage" },

pynxtools/dataconverter/readers/em/subparsers/image_tiff_tfs_modifier.py

@@ -0,0 +1,49 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Utilities for working with TFS/FEI-specific concepts."""
+
+# pylint: disable=no-member
+
+from pynxtools.dataconverter.readers.em.subparsers.image_tiff_tfs_cfg import \
+    TfsToNexusConceptMapping
+
+
+def get_nexus_value(modifier, metadata: dict):
+    """Interpret a functional mapping using data from dct via calling modifiers."""
+    if isinstance(modifier, dict):
+        # different commands are available
+        if set(["fun", "terms"]) == set(modifier.keys()):
+            if modifier["fun"] == "load_from":
+                if modifier["terms"] in metadata.keys():
+                    return metadata[modifier['terms']]
+                else:
+                    raise ValueError(f"Unable to interpret modififier load_from for argument {modifier['terms']}")
+            if modifier["fun"] == "tfs_to_nexus":
+                # print(metadata[modifier['terms']])
+                if f"{modifier['terms']}/{metadata[modifier['terms']]}" in TfsToNexusConceptMapping.keys():
+                    return TfsToNexusConceptMapping[f"{modifier['terms']}/{metadata[modifier['terms']]}"]
+                else:
+                    raise ValueError(f"Unable to interpret modifier tfs_to_nexus for argument {modifier['terms']}/{metadata[modifier['terms']]}")
+        else:
+            print(f"WARNING::Modifier {modifier} is currently not implemented !")
+            # elif set(["link"]) == set(modifier.keys()), with the jsonmap reader Sherjeel conceptualized "link"
+            return None
+    elif isinstance(modifier, str):
+        return modifier  # metadata[modifier]
+    else:
+        return None

pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py

@@ -78,11 +78,9 @@ def apply_modifier(modifier, dct: dict):
                 return load_from_modifier(modifier["terms"], dct)
             if modifier["fun"] == "convert_iso8601":
                 return convert_iso8601_modifier(modifier["terms"], dct)
-        elif set(["link"]) == set(modifier.keys()):
-            # CURRENTLY NOT IMPLEMENTED
-            # with the jsonmap reader Sherjeel conceptualized "link"
-            return None
         else:
+            print(f"WARNING::Modifier {modifier} is currently not implemented !")
+            # elif set(["link"]) == set(modifier.keys()), with the jsonmap reader Sherjeel conceptualized "link"
             return None
     if isinstance(modifier, str):
         return modifier