Added skeleton for collection of image files from the Reduction of Fe…

…Ox example from AXON Studio 10.4.4.1 aka Protochips PNG file collection reader

concept_mapper.ipynb

@@ -0,0 +1,74 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4f240b8f-71d4-4004-ab56-d7480b44d96e",
+   "metadata": {},
+   "source": [
+    "# Generate Python list of tuple with concept mapping to be used for the configuration of tech-partner-specific subparsers."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "869670b4-0780-4bf4-bc08-d802288fa5df",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "fnm = \"image_tiff_tfs_to_nexus.ods\"\n",
+    "fnm = \"image_png_protochips_to_nexus.ods\"\n",
+    "\n",
+    "ods = pd.read_excel(fnm, engine=\"odf\")\n",
+    "ods = ods.fillna(\"\")\n",
+    "# print(ods)\n",
+    "cfg = []\n",
+    "for row_idx in np.arange(1, ods.shape[0]):\n",
+    "    nxpath = ods.iloc[row_idx, 0]\n",
+    "    functor = ods.iloc[row_idx, 1]\n",
+    "    if nxpath != \"\" and ods.iloc[row_idx, 4] != \"\":  # not in [\"IGNORE\", \"UNCLEAR\"]:\n",
+    "        if functor != \"fun\":\n",
+    "            cfg.append((f\"{nxpath}\", f\"{ods.iloc[row_idx, 4]}\"))\n",
+    "        else:\n",
+    "            cfg.append((f\"{nxpath}\", f\"{ods.iloc[row_idx, 2]}\", ods.iloc[row_idx, 4]))  # not fstring because can be a list!\n",
+    "\n",
+    "indent = \"                               \"\n",
+    "for entry in cfg:\n",
+    "    print(f\"{indent}{entry},\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f27812fa-d023-4ed6-a5ee-d417a8705828",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pynxtools/dataconverter/readers/em/subparsers/image_png_protochips.py

@@ -0,0 +1,201 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Subparser for exemplar reading of raw PNG files collected on a TEM with Protochip heating_chip."""
+
+import mmap
+import numpy as np
+from typing import Dict
+from PIL import Image
+from zipfile import ZipFile
+
+from pynxtools.dataconverter.readers.em.subparsers.image_png_protochips_concepts import \
+    get_protochips_variadic_concept
+from pynxtools.dataconverter.readers.em.subparsers.image_png_protochips_cfg import \
+    PNG_PROTOCHIPS_TO_NEXUS_CFG
+from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \
+    import variadic_path_to_specific_path
+from pynxtools.dataconverter.readers.em.subparsers.image_png_protochips_modifier import \
+    get_nexus_value
+from pynxtools.dataconverter.readers.em.subparsers.image_base import \
+    ImgsBaseParser
+
+
+class ProtochipsPngSetSubParser(ImgsBaseParser):
+    def __init__(self, file_path: str = "", entry_id: int = 1):
+        super().__init__(file_path)
+        self.entry_id = entry_id
+        self.event_id = 1
+        self.prfx = None
+        self.tmp: Dict = {"data": None,
+                          "meta": {}}
+        self.supported_version: Dict = {}
+        self.version: Dict = {}
+        self.supported = False
+        self.check_if_zipped_png_protochips()
+
+    def check_if_zipped_png_protochips(self):
+        """Check if resource behind self.file_path is a TaggedImageFormat file."""
+        # all tests have to be passed before the input self.file_path
+        # can at all be processed with this parser
+        # test 1: check if file is a zipfile
+        with open(self.file_path, 'rb', 0) as file:
+            s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)
+            magic = s.read(8)
+            if magic != b'PK\x03\x04\x14\x00\x08\x00':  # https://en.wikipedia.org/wiki/List_of_file_signatures
+                print(f"Test 1 failed, {self.file_path} is not a ZIP archive !")
+                return
+        # test 2: check if there are at all PNG files with iTXt metadata from Protochips in this zip file
+        png_info = {}  # collect all those PNGs to work with and write a tuple of their image dimensions
+        with ZipFile(self.file_path) as zip_file_hdl:
+            for file in zip_file_hdl.namelist():
+                if file.lower().endswith(".png") is True:
+                    with zip_file_hdl.open(file) as fp:
+                        magic = fp.read(8)
+                        if magic == b'\x89PNG\r\n\x1a\n':
+                            method = "smart"  # "lazy"
+                            # get image dimensions
+                            if method == "lazy":  # lazy but paid with the price of reading the image content
+                                fp.seek(0)  # seek back to beginning of file required because fp.read advanced fp implicitly!
+                                with Image.open(fp) as png:
+                                    try:
+                                        nparr = np.array(png)
+                                        png_info[file] = np.shape(nparr)
+                                    except:
+                                        raise ValueError(f"Loading image data in-place from {self.file_path}:{file} failed !")
+                            if method == "smart":  # knowing where to hunt width and height in PNG metadata
+                                # https://dev.exiv2.org/projects/exiv2/wiki/The_Metadata_in_PNG_files
+                                magic = fp.read(8)
+                                png_info[file] = (np.frombuffer(fp.read(4), dtype=">i4"),
+                                                  np.frombuffer(fp.read(4), dtype=">i4"))
+
+        # test 3: check there are some PNGs
+        if len(png_info.keys()) == 0:
+            print("Test 3 failed, there are no PNGs !")
+            return
+        # test 4: check that all PNGs have the same dimensions, TODO::could check for other things here
+        target_dims = None
+        for file_name, tpl in png_info.items():
+            if target_dims is not None:
+                if tpl == target_dims:
+                    continue
+                else:
+                    print("Test 4 failed, not all PNGs have the same dimensions")
+                    return
+            else:
+                target_dims = tpl
+        print("All tests passed successfully")
+        self.supported = True
+
+    def parse_and_normalize(self):
+        """Perform actual parsing filling cache self.tmp."""
+        if self.supported is True:
+            print(f"Parsing via Protochips-specific metadata...")
+            # may need to set self.supported = False on error
+        else:
+            print(f"{self.file_path} is not a Protochips-specific "
+                  f"PNG file that this parser can process !")
+
+    def process_into_template(self, template: dict) -> dict:
+        if self.supported is True:
+            self.process_event_data_em_metadata(template)
+            self.process_event_data_em_data(template)
+        return template
+
+    def process_event_data_em_metadata(self, template: dict) -> dict:
+        """Add respective metadata."""
+        # contextualization to understand how the image relates to the EM session
+        print(f"Mapping some of the Protochips-specific metadata on respective NeXus concept instance")
+        identifier = [self.entry_id, self.event_id, 1]
+        for tpl in PNG_PROTOCHIPS_TO_NEXUS_CFG:
+            if isinstance(tpl, tuple):
+                trg = variadic_path_to_specific_path(tpl[0], identifier)
+                if len(tpl) == 2:
+                    template[trg] = tpl[1]
+                if len(tpl) == 3:
+                    # nxpath, modifier, value to load from and eventually to be modified
+                    retval = get_nexus_value(tpl[1], tpl[2], self.tmp["meta"])
+                    if retval is not None:
+                        template[trg] = retval
+        return template
+
+    def process_event_data_em_data(self, template: dict) -> dict:
+        """Add respective heavy data."""
+        # default display of the image(s) representing the data collected in this event
+        print(f"Writing Protochips PNG image into a respective NeXus concept instance")
+        # read image in-place
+        with Image.open(self.file_path, mode="r") as fp:
+            nparr = np.array(fp)
+            # print(f"type: {type(nparr)}, dtype: {nparr.dtype}, shape: {np.shape(nparr)}")
+            # TODO::discussion points
+            # - how do you know we have an image of real space vs. imaginary space (from the metadata?)
+            # - how do deal with the (ugly) scale bar that is typically stamped into the TIFF image content?
+            # with H5Web and NeXus most of this is obsolete unless there are metadata stamped which are not
+            # available in NeXus or in the respective metadata in the metadata section of the TIFF image
+            # remember H5Web images can be scaled based on the metadata allowing basically the same
+            # explorative viewing using H5Web than what traditionally typical image viewers are meant for
+            image_identifier = 1
+            trg = f"/ENTRY[entry{self.entry_id}]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/" \
+                  f"EVENT_DATA_EM[event_data_em{self.event_id}]/" \
+                  f"IMAGE_R_SET[image_r_set{image_identifier}]/DATA[image]"
+            # TODO::writer should decorate automatically!
+            template[f"{trg}/title"] = f"Image"
+            template[f"{trg}/@NX_class"] = f"NXdata"  # TODO::writer should decorate automatically!
+            template[f"{trg}/@signal"] = "intensity"
+            dims = ["x", "y"]
+            idx = 0
+            for dim in dims:
+                template[f"{trg}/@AXISNAME_indices[axis_{dim}_indices]"] = np.uint32(idx)
+                idx += 1
+            template[f"{trg}/@axes"] = []
+            for dim in dims[::-1]:
+                template[f"{trg}/@axes"].append(f"axis_{dim}")
+            template[f"{trg}/intensity"] = {"compress": np.array(fp), "strength": 1}
+            #  0 is y while 1 is x for 2d, 0 is z, 1 is y, while 2 is x for 3d
+            template[f"{trg}/intensity/@long_name"] = f"Signal"
+
+            sxy = {"x": 1., "y": 1.}
+            scan_unit = {"x": "m", "y": "m"}  # assuming FEI reports SI units
+            # we may face the CCD overview camera for the chamber for which there might not be a calibration!
+            if ("EScan/PixelWidth" in self.tmp["meta"].keys()) and ("EScan/PixelHeight" in self.tmp["meta"].keys()):
+                sxy = {"x": self.tmp["meta"]["EScan/PixelWidth"],
+                       "y": self.tmp["meta"]["EScan/PixelHeight"]}
+                scan_unit = {"x": "px", "y": "px"}
+            nxy = {"x": np.shape(np.array(fp))[1], "y": np.shape(np.array(fp))[0]}
+            # TODO::be careful we assume here a very specific coordinate system
+            # however the TIFF file gives no clue, TIFF just documents in which order
+            # it arranges a bunch of pixels that have stream in into a n-d tiling
+            # e.g. a 2D image
+            # also we have to be careful because TFS just gives us here
+            # typical case of an image without an information without its location
+            # on the physical sample surface, therefore we can only scale
+            # pixel_identifier by physical scaling quantities s_x, s_y
+            # also the dimensions of the image are on us to fish with the image
+            # reading library instead of TFS for consistency checks adding these
+            # to the metadata the reason is that TFS TIFF use the TIFF tagging mechanism
+            # and there is already a proper TIFF tag for the width and height of an
+            # image in number of pixel
+            for dim in dims:
+                template[f"{trg}/AXISNAME[axis_{dim}]"] \
+                    = {"compress": np.asarray(np.linspace(0,
+                                                          nxy[dim] - 1,
+                                                          num=nxy[dim],
+                                                          endpoint=True) * sxy[dim], np.float64), "strength": 1}
+                template[f"{trg}/AXISNAME[axis_{dim}]/@long_name"] \
+                    = f"Coordinate along {dim}-axis ({scan_unit[dim]})"
+                template[f"{trg}/AXISNAME[axis_{dim}]/@units"] = f"{scan_unit[dim]}"
+        return template

pynxtools/dataconverter/readers/em/subparsers/image_png_protochips_cfg.py

@@ -0,0 +1,44 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Configuration of the image_png_protochips subparser."""
+
+
+PNG_PROTOCHIPS_TO_NEXUS_CFG = [('/ENTRY[entry*]/measurement/em_lab/STAGE_LAB[stage_lab]/alias', 'load_from', 'MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[*].Stage.Name'),
+                               ('/ENTRY[entry*]/measurement/em_lab/STAGE_LAB[stage_lab]/design', 'heating_chip'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/tilt_1', 'load_from', 'MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[*].Stage.A'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/tilt_2', 'load_from', 'MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[*].Stage.B'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/position', 'load_from_concatenate', 'MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[*].Stage.X, MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[*].Stage.Y, MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[*].Stage.Z'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/HEATER[heater]/current', 'load_from', 'MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[*].DataValues.AuxiliaryDataValue.[*].HeatingCurrent'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/HEATER[heater]/current/@units', 'A'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/HEATER[heater]/power', 'load_from', 'MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[*].DataValues.AuxiliaryDataValue.[*].HeatingPower'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/HEATER[heater]/power/@units', 'W'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/HEATER[heater]/voltage', 'load_from', 'MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[*].DataValues.AuxiliaryDataValue.[*].HeatingVoltage'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/HEATER[heater]/voltage/@units', 'V'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/SENSOR[sensor2]/value', 'load_from', 'MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[*].DataValues.AuxiliaryDataValue.[*].HolderPressure'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/SENSOR[sensor2]/value/@units', 'torr'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/SENSOR[sensor2]/measurement', 'pressure'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/SENSOR[sensor1]/value', 'load_from', 'MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[*].DataValues.AuxiliaryDataValue.[*].HolderTemperature'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/SENSOR[sensor1]/value/@units', '°C'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage_lab]/SENSOR[sensor1]/measurement', 'temperature'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/EBEAM_COLUMN[ebeam_column]/electron_source/voltage', 'load_from', 'MicroscopeControlImageMetadata.MicroscopeSettings.AcceleratingVoltage'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/EBEAM_COLUMN[ebeam_column]/DEFLECTOR[beam_blanker1]/state', 'load_from', 'MicroscopeControlImageMetadata.MicroscopeSettings.BeamBlankerState'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/OPTICAL_SYSTEM_EM[optical_system_em]/camera_length', 'load_from', 'MicroscopeControlImageMetadata.MicroscopeSettings.CameraLengthValue'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/OPTICAL_SYSTEM_EM[optical_system_em]/magnification', 'load_from', 'MicroscopeControlImageMetadata.MicroscopeSettings.MagnificationValue'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/event_type', 'As tested with AXON 10.4.4.21, 2021-04-26T22:51:28.4539893-05:00 not included in Protochips PNG metadata'),
+                               ('/ENTRY[entry*]/measurement/EVENT_DATA_EM_SET[event_data_em_set]/EVENT_DATA_EM[event_data_em*]/em_lab/DETECTOR[detector*]/mode', 'As tested with AXON 10.4.4.21, 2021-04-26T22:51:28.4539893-05:00 not included in Protochips PNG metadata'),
+                               ('/ENTRY[entry*]/measurement/em_lab/DETECTOR[detector*]/local_name', 'As tested with AXON 10.4.4.21, 2021-04-26T22:51:28.4539893-05:00 not included in Protochips PNG metadata')]