From bd4edec8f69625c0ad1753d8576cc698d7202e74 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 12 Jan 2024 15:40:00 +0100 Subject: [PATCH] Added xraydb to get all X-ray emission lines --- dev-requirements.txt | 11 ++++ .../readers/em/concepts/nxs_concepts.py | 12 ++++- .../em/concepts/nxs_em_eds_indexing.py | 54 +++++++++++++++++++ .../readers/em/concepts/nxs_image_r_set.py | 3 -- .../readers/em/concepts/nxs_spectrum_set.py | 3 -- .../readers/em/subparsers/hfive_apex.py | 9 ++++ pyproject.toml | 7 +-- 7 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 pynxtools/dataconverter/readers/em/concepts/nxs_em_eds_indexing.py diff --git a/dev-requirements.txt b/dev-requirements.txt index dd12c799b..733d5cb2f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -108,6 +108,8 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via pynxtools (pyproject.toml) +greenlet==3.0.3 + # via sqlalchemy h5py==3.10.0 # via # fabio @@ -297,6 +299,7 @@ numpy==1.24.4 # tables # tifffile # xarray + # xraydb # zarr numpy-quaternion==2022.4.3 # via orix @@ -319,6 +322,7 @@ packaging==23.2 # scikit-image # tables # xarray + # xraydb pandas==2.0.3 # via # ifes-apt-tc-data-modeling @@ -352,6 +356,7 @@ platformdirs==4.0.0 # pooch # pylint # requests-cache + # xraydb pluggy==1.3.0 # via pytest ply==3.11 @@ -470,6 +475,7 @@ scipy==1.9.1 # scikit-image # scikit-learn # sparse + # xraydb silx==1.1.2 # via pyfai six==1.16.0 @@ -482,6 +488,8 @@ smmap==5.0.1 # via gitdb sparse==0.14.0 # via hyperspy +sqlalchemy==2.0.25 + # via xraydb stack-data==0.6.3 # via ipython structlog==23.2.0 @@ -555,6 +563,7 @@ typing-extensions==4.8.0 # astroid # cattrs # mypy + # sqlalchemy tzdata==2023.3 # via # pandas @@ -582,6 +591,8 @@ xarray==2023.1.0 # via pynxtools (pyproject.toml) xmltodict==0.13.0 # via pynxtools (pyproject.toml) +xraydb==4.5.4 + # via pynxtools (pyproject.toml) zarr==2.16.1 # via hyperspy zipp==3.17.0 diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_concepts.py b/pynxtools/dataconverter/readers/em/concepts/nxs_concepts.py index 33716e2d2..09e00228b 100644 --- a/pynxtools/dataconverter/readers/em/concepts/nxs_concepts.py +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_concepts.py @@ -17,7 +17,7 @@ # """Implement NeXus-specific groups and fields to document software and versions used.""" -# pylint: disable=no-member +# pylint: disable=no-member,too-few-members from typing import List from pynxtools.dataconverter.readers.em.concepts.concept_mapper \ @@ -59,3 +59,13 @@ def parse(self, template: dict, entry_id: int = 1, cmd_line_args: List = []) -> template["/cs_profiling/@NX_class"] = "NXcs_profiling" template["/cs_profiling/command_line_call"] = cmd_line_args return template + + +class NxConcept(): + """"Define a NeXus concept object to handle paths. + + """ + def __init__(self, hdf_paths: List = []): + # TODO::remove redundant code for instantiating specific NxConcepts like + # NxSpectrum, NxImageRealSpaceSet, NxEmEdsIndexing + pass diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_em_eds_indexing.py b/pynxtools/dataconverter/readers/em/concepts/nxs_em_eds_indexing.py new file mode 100644 index 000000000..0ab4bb3ec --- /dev/null +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_em_eds_indexing.py @@ -0,0 +1,54 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""NXem_eds indexing instance data.""" + +from typing import Dict, List + +from pynxtools.dataconverter.readers.em.concepts.nxs_object import NxObject + + +NX_EM_EDS_INDEXING_HDF_PATH = ["indexing-group", + "indexing/element_names-field", + "indexing/IMAGE_R_SET-group", + "indexing/IMAGE_R_SET/PROCESS-group", + "indexing/IMAGE_R_SET/PROCESS/peaks-field", + "indexing/IMAGE_R_SET/PROCESS/weights-field", + "indexing/PEAK-group", + "indexing/PEAK/ION-group", + "indexing/PEAK/ION/energy-field", + "indexing/PEAK/ION/energy_range-field", + "indexing/PEAK/ION/iupac_line_names-field", + "indexing/PROGRAM-group", + "indexing/summary-group", + "indexing/summary/axis_energy-field", + "indexing/summary/axis_energy@long_name-attribute", + "indexing/summary/intensity-field", + "indexing/summary/intensity@long_name-attribute"] + + +class NxEmEdsIndexing(): + def __init__(self): + self.tmp: Dict = {} + self.tmp["source"] = None + for entry in NX_EM_EDS_INDEXING_HDF_PATH: + if entry.endswith("-field") is True: + self.tmp[entry[0:len(entry) - len("-field")]] = NxObject(eqv_hdf="dataset") + elif entry.endswith("-attribute") is True: + self.tmp[entry[0:len(entry) - len("-attribute")]] = NxObject(eqv_hdf="attribute") + else: + self.tmp[entry[0:len(entry) - len("-group")]] = NxObject(eqv_hdf="group") diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py b/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py index bc8e675a8..4a2ca2e58 100644 --- a/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_image_r_set.py @@ -17,9 +17,6 @@ # """NXem spectrum set (element of a labelled property graph) to store instance data.""" -# pylint: disable=no-member,too-few-public-methods - - from typing import Dict from pynxtools.dataconverter.readers.em.concepts.nxs_object import NxObject diff --git a/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py b/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py index 077e30648..0e09e4ec9 100644 --- a/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py +++ b/pynxtools/dataconverter/readers/em/concepts/nxs_spectrum_set.py @@ -17,9 +17,6 @@ # """NXem spectrum set (element of a labelled property graph) to store instance data.""" -# pylint: disable=no-member,too-few-public-methods - - from typing import Dict, List from pynxtools.dataconverter.readers.em.concepts.nxs_object import NxObject diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py index 5707c49b3..6539417ea 100644 --- a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py +++ b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py @@ -201,6 +201,11 @@ def parse_and_normalize(self): h5r, f"/{grp_nm}/{sub_grp_nm}/{sub_sub_grp_nm}/{area_grp_nm}", ckey) cache_id += 1 + ckey = self.init_named_cache(f"eds_map{cache_id}") + self.parse_and_normalize_eds_rois( + h5r, f"/{grp_nm}/{sub_grp_nm}/{sub_sub_grp_nm}/{area_grp_nm}", ckey) + cache_id += 1 + def parse_and_normalize_group_ebsd_header(self, fp, ckey: str): # no official documentation yet from EDAX/APEX, deeply nested, chunking, virtual ds if f"{self.prfx}/EBSD/ANG/DATA/DATA" not in fp: @@ -581,3 +586,7 @@ def parse_and_normalize_eds_lsd(self, fp, src: str, ckey: str): for key, val in self.tmp[ckey].tmp.items(): if key.startswith("spectrum_oned"): print(f"spectrum_oned, key: {key}, val: {val}") + + def parse_and_normalize_eds_rois(self, fp, src: str, ckey: str): + """Normalize and scale APEX-specific EDS element emission line maps to NeXus.""" + diff --git a/pyproject.toml b/pyproject.toml index 828d199ee..7dd67d11d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,10 @@ dependencies = [ "pandas>=1.3.2", "odfpy>=1.4.1", "ase>=3.19.0", - "flatdict>=4.0.1", - "hyperspy>=1.7.6", - "rosettasciio>=0.2", + "flatdict", + "hyperspy", + "rosettasciio", + "xraydb", "ifes_apt_tc_data_modeling>=0.1", "gitpython>=3.1.24", "pytz>=2021.1",