From 8d8f0eeba6674a877f179b2bdeeac76ea20ffa9c Mon Sep 17 00:00:00 2001 From: Sherjeel Shabih Date: Sat, 2 Dec 2023 12:17:42 +0100 Subject: [PATCH 1/3] Adds a get method to Dataconverter Template --- pynxtools/dataconverter/template.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pynxtools/dataconverter/template.py b/pynxtools/dataconverter/template.py index 28762ea17..a3fa1bedb 100644 --- a/pynxtools/dataconverter/template.py +++ b/pynxtools/dataconverter/template.py @@ -125,6 +125,13 @@ def __contains__(self, k): k in self.required ]) + def get(self, key: str, default=None): + """Proxies the get function to our internal __getitem__""" + try: + return self[key] + except KeyError: + return default + def __getitem__(self, k): """Handles how values are accessed from the Template object.""" # Try setting item in all else throw error. Does not append to default. From 295f72db96f0510935829a3b3813dfb240ce091c Mon Sep 17 00:00:00 2001 From: Sherjeel Shabih Date: Sat, 2 Dec 2023 15:25:40 +0100 Subject: [PATCH 2/3] Adds KeyError for not accepted keys in Template --- pynxtools/dataconverter/template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pynxtools/dataconverter/template.py b/pynxtools/dataconverter/template.py index a3fa1bedb..fa6907d36 100644 --- a/pynxtools/dataconverter/template.py +++ b/pynxtools/dataconverter/template.py @@ -148,7 +148,10 @@ def __getitem__(self, k): return self.required[k] except KeyError: return self.undocumented[k] - return self.get_optionality(k) + if k in ("required", "optional", "recommended", "undocumented"): + return self.get_optionality(k) + raise KeyError("Only paths starting with '/' or one of [optional_parents, " + "lone_groups, required, optional, recommended, undocumented] can be used.") def clear(self): """Clears all data stored in the Template object.""" From e7d5206a6da28529a62d4461280d995d8d14ce7c Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Sun, 3 Dec 2023 10:43:03 +0100 Subject: [PATCH 3/3] Xrd reader integration (#179) * added xrd_parser.py from Pepe's plugin * General skelaton for xrd reader. * RB * removing x-ray utils in from xrd reader. * adding Sherejeel PR review. * Removing some code. * Update pynxtools/dataconverter/template.py from Sheerjeel Co-authored-by: Sherjeel Shabih * Adds get function in Template to proxy __getitem__ * update. * moving to the old code. * removing get_file_obj() * ci * Adding test. * adding test for tranform_to_intended_data. * remove sts. * updating nxdl_path to nxdl_f_path. * CI * fix unit. * update * update readme file. * RB * Including lukus sugestion. * CI/CD * updating READ me file. * Updating link. * dev-requirements.txt * Adding test file for nexus. * Test file. * Updating mpes test according to application definition. --------- Co-authored-by: Sebastian Brueckner Co-authored-by: Sherjeel Shabih --- dev-requirements.txt | 9 +- pynxtools/dataconverter/convert.py | 140 ++++-- pynxtools/dataconverter/helpers.py | 77 ++- pynxtools/dataconverter/readers/xrd/README.md | 40 ++ .../dataconverter/readers/xrd/__init__.py | 15 + pynxtools/dataconverter/readers/xrd/config.py | 117 +++++ pynxtools/dataconverter/readers/xrd/reader.py | 176 +++++++ .../dataconverter/readers/xrd/xrd_helper.py | 293 +++++++++++ .../dataconverter/readers/xrd/xrd_parser.py | 448 +++++++++++++++++ pynxtools/dataconverter/template.py | 21 + pynxtools/dataconverter/writer.py | 25 +- pynxtools/definitions | 2 +- pynxtools/eln_mapper/scheme_eln.py | 1 - pynxtools/nexus/nexus.py | 41 +- .../readers/mpes/Ref_nexus_mpes.log | 228 +++++---- .../readers/xrd/ACZCTS_5-60_181.xrdml | 106 ++++ tests/data/nexus/NXtest2.nxdl.xml | 455 ++++++++++++++++++ tests/dataconverter/test_convert.py | 2 +- tests/dataconverter/test_helpers.py | 23 + tests/nexus/test_nexus.py | 5 +- 20 files changed, 2062 insertions(+), 162 deletions(-) create mode 100644 pynxtools/dataconverter/readers/xrd/README.md create mode 100644 pynxtools/dataconverter/readers/xrd/__init__.py create mode 100644 pynxtools/dataconverter/readers/xrd/config.py create mode 100644 pynxtools/dataconverter/readers/xrd/reader.py create mode 100644 pynxtools/dataconverter/readers/xrd/xrd_helper.py create mode 100644 pynxtools/dataconverter/readers/xrd/xrd_parser.py create mode 100644 tests/data/dataconverter/readers/xrd/ACZCTS_5-60_181.xrdml create mode 100644 tests/data/nexus/NXtest2.nxdl.xml diff --git a/dev-requirements.txt b/dev-requirements.txt index 8dff5778d..f6ac22e2f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,10 +4,6 @@ # # pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml # -appnope==0.1.3 - # via - # ipykernel - # ipython asciitree==0.3.3 # via zarr ase==3.22.1 @@ -48,16 +44,13 @@ comm==0.2.0 contourpy==1.1.1 # via matplotlib coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov + # via pytest-cov cycler==0.12.1 # via matplotlib cython==3.0.6 # via tables dask[array]==2023.5.0 # via - # dask # hyperspy # kikuchipy # orix diff --git a/pynxtools/dataconverter/convert.py b/pynxtools/dataconverter/convert.py index 41e4a4c2b..46c9af7eb 100644 --- a/pynxtools/dataconverter/convert.py +++ b/pynxtools/dataconverter/convert.py @@ -22,7 +22,7 @@ import logging import os import sys -from typing import List, Tuple +from typing import List, Tuple, Optional import xml.etree.ElementTree as ET import click @@ -80,60 +80,146 @@ def get_names_of_all_readers() -> List[str]: return all_readers + plugins -# pylint: disable=too-many-arguments,too-many-locals -def convert(input_file: Tuple[str, ...], - reader: str, - nxdl: str, - output: str, - generate_template: bool = False, - fair: bool = False, - undocumented: bool = False, - **kwargs): - """The conversion routine that takes the input parameters and calls the necessary functions.""" +def get_nxdl_root_and_path(nxdl: str): + """Get xml root element and file path from nxdl name e.g. NXapm. + + Parameters + ---------- + nxdl: str + Name of nxdl file e.g. NXapm from NXapm.nxdl.xml. + + Returns + ------- + ET.root + Root element of nxdl file. + str + Path of nxdl file. + + Raises + ------ + FileNotFoundError + Error if no file with the given nxdl name is found. + """ # Reading in the NXDL and generating a template definitions_path = nexus.get_nexus_definitions_path() if nxdl == "NXtest": - nxdl_path = os.path.join( + nxdl_f_path = os.path.join( f"{os.path.abspath(os.path.dirname(__file__))}/../../", "tests", "data", "dataconverter", "NXtest.nxdl.xml") elif nxdl == "NXroot": - nxdl_path = os.path.join(definitions_path, "base_classes", "NXroot.nxdl.xml") + nxdl_f_path = os.path.join(definitions_path, "base_classes", "NXroot.nxdl.xml") else: - nxdl_path = os.path.join(definitions_path, "contributed_definitions", f"{nxdl}.nxdl.xml") - if not os.path.exists(nxdl_path): - nxdl_path = os.path.join(definitions_path, "applications", f"{nxdl}.nxdl.xml") - if not os.path.exists(nxdl_path): + nxdl_f_path = os.path.join(definitions_path, "contributed_definitions", f"{nxdl}.nxdl.xml") + if not os.path.exists(nxdl_f_path): + nxdl_f_path = os.path.join(definitions_path, "applications", f"{nxdl}.nxdl.xml") + if not os.path.exists(nxdl_f_path): + nxdl_f_path = os.path.join(definitions_path, "base_classes", f"{nxdl}.nxdl.xml") + if not os.path.exists(nxdl_f_path): raise FileNotFoundError(f"The nxdl file, {nxdl}, was not found.") - nxdl_root = ET.parse(nxdl_path).getroot() + return ET.parse(nxdl_f_path).getroot(), nxdl_f_path - if undocumented: - logger.setLevel(UNDOCUMENTED) + +def transfer_data_into_template(input_file, + reader, nxdl_name, + nxdl_root: Optional[ET.Element] = None, + **kwargs): + """Transfer parse and merged data from input experimental file, config file and eln. + + Experimental and eln files will be parsed and finally will be merged into template. + Before returning the template validate the template data. + + Parameters + ---------- + input_file : Union[tuple[str], str] + Tuple of files or file + reader: str + Name of reader such as xps + nxdl_name : str + Root name of nxdl file, e.g. NXmpes from NXmpes.nxdl.xml + nxdl_root : ET.element + Root element of nxdl file, otherwise provide nxdl_name + + Returns + ------- + Template + Template filled with data from raw file and eln file. + + """ + if nxdl_root is None: + nxdl_root, _ = get_nxdl_root_and_path(nxdl=nxdl_name) template = Template() helpers.generate_template_from_nxdl(nxdl_root, template) - if generate_template: - logger.info(template) - return - # Setting up all the input data if isinstance(input_file, str): input_file = (input_file,) + bulletpoint = "\n\u2022 " logger.info("Using %s reader to convert the given files: %s ", reader, bulletpoint.join((" ", *input_file))) data_reader = get_reader(reader) - if not (nxdl in data_reader.supported_nxdls or "*" in data_reader.supported_nxdls): + if not (nxdl_name in data_reader.supported_nxdls or "*" in data_reader.supported_nxdls): raise NotImplementedError("The chosen NXDL isn't supported by the selected reader.") data = data_reader().read( # type: ignore[operator] template=Template(template), file_paths=input_file, - **kwargs, + **kwargs ) helpers.validate_data_dict(template, data, nxdl_root) + return data + + +# pylint: disable=too-many-arguments,too-many-locals +def convert(input_file: Tuple[str, ...], + reader: str, + nxdl: str, + output: str, + generate_template: bool = False, + fair: bool = False, + undocumented: bool = False, + **kwargs): + """The conversion routine that takes the input parameters and calls the necessary functions. + + Parameters + ---------- + input_file : Tuple[str] + Tuple of files or file + reader: str + Name of reader such as xps + nxdl : str + Root name of nxdl file, e.g. NXmpes for NXmpes.nxdl.xml + output : str + Output file name. + generate_template : bool, default False + True if user wants template in logger info. + fair : bool, default False + If True, a warning is given that there are undocumented paths + in the template. + undocumented : bool, default False + If True, an undocumented warning is given. + + Returns + ------- + None. + """ + + nxdl_root, nxdl_f_path = get_nxdl_root_and_path(nxdl) + + if generate_template: + template = Template() + helpers.generate_template_from_nxdl(nxdl_root, template) + logger.info(template) + return + + data = transfer_data_into_template(input_file=input_file, reader=reader, + nxdl_name=nxdl, nxdl_root=nxdl_root, + **kwargs) + if undocumented: + logger.setLevel(UNDOCUMENTED) if fair and data.undocumented.keys(): logger.warning("There are undocumented paths in the template. This is not acceptable!") return @@ -147,7 +233,7 @@ def convert(input_file: Tuple[str, ...], path ) helpers.add_default_root_attributes(data=data, filename=os.path.basename(output)) - Writer(data=data, nxdl_path=nxdl_path, output_path=output).write() + Writer(data=data, nxdl_f_path=nxdl_f_path, output_path=output).write() logger.info("The output file generated: %s", output) diff --git a/pynxtools/dataconverter/helpers.py b/pynxtools/dataconverter/helpers.py index f9da1b300..57d526f4b 100644 --- a/pynxtools/dataconverter/helpers.py +++ b/pynxtools/dataconverter/helpers.py @@ -17,12 +17,13 @@ # """Helper functions commonly used by the convert routine.""" -from typing import List +from typing import List, Optional, Any from typing import Tuple, Callable, Union import re import xml.etree.ElementTree as ET from datetime import datetime, timezone import logging +import json import numpy as np from ase.data import chemical_symbols @@ -650,3 +651,77 @@ def extract_atom_types(formula, mode='hill'): return convert_to_hill(atom_types) return atom_types + + +# pylint: disable=too-many-branches +def transform_to_intended_dt(str_value: Any) -> Optional[Any]: + """Transform string to the intended data type, if not then return str_value. + + E.g '2.5E-2' will be transfor into 2.5E-2 + tested with: '2.4E-23', '28', '45.98', 'test', ['59', '3.00005', '498E-34'], + '23 34 444 5000', None + with result: 2.4e-23, 28, 45.98, test, [5.90000e+01 3.00005e+00 4.98000e-32], + np.array([23 34 444 5000]), None + NOTE: add another arg in this func for giving 'hint' what kind of data like + numpy array or list + Parameters + ---------- + str_value : str + Data from other format that comes as string e.g. string of list. + + Returns + ------- + Union[str, int, float, np.ndarray] + Converted data type + """ + + symbol_list_for_data_seperation = [';', ' '] + transformed: Any = None + + if isinstance(str_value, list): + try: + transformed = np.array(str_value, dtype=np.float64) + return transformed + except ValueError: + pass + + elif isinstance(str_value, np.ndarray): + return str_value + elif isinstance(str_value, str): + try: + transformed = int(str_value) + except ValueError: + try: + transformed = float(str_value) + except ValueError: + if '[' in str_value and ']' in str_value: + transformed = json.loads(str_value) + if transformed is not None: + return transformed + for sym in symbol_list_for_data_seperation: + if sym in str_value: + parts = str_value.split(sym) + modified_parts: List = [] + for part in parts: + part = transform_to_intended_dt(part) + if isinstance(part, (int, float)): + modified_parts.append(part) + else: + return str_value + return transform_to_intended_dt(modified_parts) + + return str_value + + +def nested_dict_to_slash_separated_path(nested_dict: dict, + flattened_dict: dict, + parent_path=''): + """Convert nested dict into slash separeted path upto certain level.""" + sep = '/' + + for key, val in nested_dict.items(): + path = parent_path + sep + key + if isinstance(val, dict): + nested_dict_to_slash_separated_path(val, flattened_dict, path) + else: + flattened_dict[path] = val diff --git a/pynxtools/dataconverter/readers/xrd/README.md b/pynxtools/dataconverter/readers/xrd/README.md new file mode 100644 index 000000000..53c64dfc7 --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/README.md @@ -0,0 +1,40 @@ +# XRD Reader +With the XRD reader, data from X-ray diffraction experiment can be read and written into a NeXus file (h5 type file with extension .nxs) according to NXxrd_pan application definition in [NeXus](https://github.com/FAIRmat-NFDI/nexus_definitions). There are a few different methods of measuring XRD: 1. θ:2θ instruments (e.g. Rigaku H3R), and 2. θ:θ instrument (e.g. PANalytical X’Pert Pro). The goal with this reader is to support both of these methods. + +**NOTE: This reader is still under development. As of now, the reader can only handle files with the extension `.xrdml` , obtained with PANalytical X’Pert Pro version 1.5 (method 2 described above). Currently we are wtoking to include more file types and file versions.** + +## Contact Person in FAIRmat +In principle, you can reach out to any member of Area B of the FAIRmat consortium, but Rubel Mozumder could be more reasonable for the early response. + +## Parsers +Though, in computer science, parser is a process that reads code into smaller parts (called tocken) with relations among tockens in a tree diagram. The process helps compiler to understand the tocken relationship of the source code. + +The XRD reader calls a program or class (called parser) that reads the experimenal input file and re-organises the different physical/experiment concepts or properties in a certain structure which is defined by developer. + +### class pynxtools.dataconverter.readers.xrd.xrd_parser.XRDMLParser + + **inputs:** + file_path: Full path of the input file. + + **Important method:** + get_slash_separated_xrd_dict() -> dict + + This method can be used to check if all the data from the input file have been read or not, it returns the slash separated dict as described. + + +### Other Parsers + **Coming Soon!!** + +### How To +The reader can be run from Jupyter-notebook or Jupyter-lab with the following command: + +```sh + ! dataconverter \ +--reader xrd \ +--nxdl NXxrd_pan \ +--input-file $ \ +--input-file $ \ +--output .nxs +``` + +An example file can be found here in GitLab in [nomad-remote-tools-hub](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-remote-tools-hub/-/tree/develop/docker/xrd) feel free to vist and try out the reader. diff --git a/pynxtools/dataconverter/readers/xrd/__init__.py b/pynxtools/dataconverter/readers/xrd/__init__.py new file mode 100644 index 000000000..d4ec4a8cc --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/__init__.py @@ -0,0 +1,15 @@ +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/pynxtools/dataconverter/readers/xrd/config.py b/pynxtools/dataconverter/readers/xrd/config.py new file mode 100644 index 000000000..4d3757b10 --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/config.py @@ -0,0 +1,117 @@ +"""This is config file that mainly maps nexus definition to data path in raw file.""" + +# pylint: disable=C0301 +xrdml = { + "/ENTRY[entry]/2theta_plot/chi": {"xrdml_1.5": {"value": "", + "@units": "", + "@chi_indices": 0}, + }, + "/ENTRY[entry]/2theta_plot/intensity": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/intensities", + "@units": "counts/s"} + }, + "/ENTRY[entry]/2theta_plot/omega": {"xrdml_1.5": {"value": "", + "@units": "", + "@omega_indices": 1}, + }, + "/ENTRY[entry]/2theta_plot/title": "Intensity Vs. Two Theta (deg.)", + "/ENTRY[entry]/2theta_plot/phi": {"xrdml_1.5": {"value": "", + "@units": "", + "@phi_indices": 0}, + }, + "/ENTRY[entry]/2theta_plot/two_theta": {"xrdml_1.5": {"value": "", + "@units": "deg", + "@two_theta_indices": 0}, + }, + "/ENTRY[entry]/COLLECTION[collection]/beam_attenuation_factors": {"xrdml_1.5": {"value": "/beamAttenuationFactors", + "@units": ""}, + }, + "/ENTRY[entry]/COLLECTION[collection]/omega/start": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/startPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/omega/end": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/endPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/omega/step": {"xrdml_1.5": {"value": "/xrdMeasurements/comment/entry_2/MinimumstepsizeOmega", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/2theta/start": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/startPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/2theta/end": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/endPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/2theta/step": {"xrdml_1.5": {"value": "/xrdMeasurements/comment/entry_2/Minimumstepsize2Theta", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/count_time": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/commonCountingTime", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/commonCountingTime/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/data_file": {"xrdml_1.5": {"value": ""} + }, + "/ENTRY[entry]/COLLECTION[collection]/goniometer_x": {"xrdml_1.5": {"value": "/X", + "@units": ""}, + }, + "/ENTRY[entry]/COLLECTION[collection]/goniometer_y": {"xrdml_1.5": {"value": "/Y", + "@units": ""}, + }, + "/ENTRY[entry]/COLLECTION[collection]/goniometer_z": {"xrdml_1.5": {"value": "/Z", + "@units": ""}, + }, + "/ENTRY[entry]/COLLECTION[collection]/measurement_type": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/measurementType", + "@units": ""}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/DETECTOR[detector]/integration_time": {"xrdml_1.5": {"value": "", + "@units": ""}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/DETECTOR[detector]/integration_time/@units": {"xrdml_1.5": {"value": "", + "@units": ""}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/DETECTOR[detector]/scan_axis": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/scanAxis", + "@units": ""}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/DETECTOR[detector]/scan_mode": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/mode", + "@units": ""}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_one": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/usedWavelength/kAlpha1", + "@units": "/xrdMeasurements/xrdMeasurement/usedWavelength/kAlpha1/unit"}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_two": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/usedWavelength/kAlpha2", + "@units": "/xrdMeasurements/xrdMeasurement/usedWavelength/kAlpha2/unit"}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/kbeta": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/usedWavelength/kBeta", + "@units": "/xrdMeasurements/xrdMeasurement/usedWavelength/kBeta/unit"}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/ratio_k_alphatwo_k_alphaone": {"xrdml_1.5": {"value": "", + "@units": ""} + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_current": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/incidentBeamPath/xRayTube/current", + "@units": "/xrdMeasurements/xrdMeasurement/incidentBeamPath/xRayTube/current/unit"} + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/source_peak_wavelength": {"xrdml_1.5": {"value": "", + "@units": ""} + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_material": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/incidentBeamPath/xRayTube/anodeMaterial", + "@units": ""}, + }, + "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_voltage": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/incidentBeamPath/xRayTube/tension", + "@units": "/xrdMeasurements/xrdMeasurement/incidentBeamPath/xRayTube/tension/unit"} + }, + "/ENTRY[entry]/SAMPLE[sample]/prepared_by": {"xrdml_1.5": {"value": ""} + }, + "/ENTRY[entry]/SAMPLE[sample]/sample_id": {"xrdml_1.5": {"value": ""}, + }, + "/ENTRY[entry]/SAMPLE[sample]/sample_mode": {"xrdml_1.5": {"value": ""}, + }, + "/ENTRY[entry]/SAMPLE[sample]/sample_name": {"xrdml_1.5": {"value": ""}, + }, + "/ENTRY[entry]/definition": "NXxrd_pan", + "/ENTRY[entry]/method": "X-Ray Diffraction (XRD)", + "/ENTRY[entry]/q_plot/intensity": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/intensities", + "@units": "counts/s"}, + }, + "/ENTRY[entry]/q_plot/q": {"xrdml_1.5": {"value": "", + "@units": ""}, + }, + "/@default": "entry", + "/ENTRY[entry]/@default": "2theta_plot", +} diff --git a/pynxtools/dataconverter/readers/xrd/reader.py b/pynxtools/dataconverter/readers/xrd/reader.py new file mode 100644 index 000000000..242498790 --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/reader.py @@ -0,0 +1,176 @@ +"""XRD reader.""" +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Tuple, Any, Dict, Union +import json +from pathlib import Path +import xml.etree.ElementTree as ET + +import yaml + +from pynxtools.dataconverter.helpers import (generate_template_from_nxdl, + validate_data_dict) +from pynxtools.dataconverter.template import Template +from pynxtools.dataconverter.readers.xrd.xrd_parser import parse_and_fill_template +from pynxtools.dataconverter.readers.utils import flatten_and_replace, FlattenSettings +from pynxtools.dataconverter.readers.base.reader import BaseReader + +CONVERT_DICT: Dict[str, str] = { + 'unit': '@units', + 'Instrument': 'INSTRUMENT[instrument]', + 'Source': 'SOURCE[source]', + 'Detector': 'DETECTOR[detector]', + 'Collection': 'COLLECTION[collection]', + 'Sample': 'SAMPLE[sample]', + 'version': '@version', + 'User': 'USER[user]', +} + + +# Global var to collect the root from get_template_from_nxdl_name() +# and use it in the the the varidate_data_dict() +ROOT: ET.Element = None +REPLACE_NESTED: Dict[str, Any] = {} +XRD_FILE_EXTENSIONS = [".xrdml", "xrdml", ".udf", ".raw", ".xye"] + + +def get_template_from_nxdl_name(nxdl_name): + """Generate template from nxdl name. + + Example of nxdl name could be NXxrd_pan. + Parameters + ---------- + nxdl_name : str + Name of nxdl file e.g. NXmpes + + Returns + ------- + Template + Empty template. + + Raises + ------ + ValueError + Error if nxdl file is not found. + """ + nxdl_file = nxdl_name + ".nxdl.xml" + current_path = Path(__file__) + def_path = current_path.parent.parent.parent.parent / 'definitions' + # Check contributed defintions + full_nxdl_path = Path(def_path, 'contributed_definitions', nxdl_file) + root = None + if full_nxdl_path.exists(): + root = ET.parse(full_nxdl_path).getroot() + else: + # Check application definition + full_nxdl_path = Path(def_path, 'applications', nxdl_file) + + if root is None and full_nxdl_path.exists(): + root = ET.parse(full_nxdl_path).getroot() + else: + full_nxdl_path = Path(def_path, 'base_classes', nxdl_file) + + if root is None and full_nxdl_path.exists(): + root = ET.parse(full_nxdl_path).getroot() + elif root is None: + raise ValueError("Need correct NXDL name") + + template = Template() + generate_template_from_nxdl(root=root, template=template) + return template + + +def get_template_from_xrd_reader(nxdl_name, file_paths): + """Get filled template from reader. + + Parameters + ---------- + nxdl_name : str + Name of nxdl definition + file_paths : Tuple[str] + Tuple of path of files. + + Returns + ------- + Template + Template which is a map from NeXus concept path to value. + """ + + template = get_template_from_nxdl_name(nxdl_name) + + data = XRDReader().read(template=template, + file_paths=file_paths) + validate_data_dict(template=template, data=data, nxdl_root=ROOT) + return data + + +# pylint: disable=too-few-public-methods +class XRDReader(BaseReader): + """Reader for XRD.""" + + supported_nxdls = ["NXxrd_pan"] + + def read(self, + template: dict = None, + file_paths: Tuple[str] = None, + objects: Tuple[Any] = None): + """General read menthod to prepare the template.""" + + if not isinstance(file_paths, tuple) and not isinstance(file_paths, list): + file_paths = (file_paths,) + filled_template: Union[Dict, None] = Template() + eln_dict: Union[Dict[str, Any], None] = None + config_dict: Dict = {} + xrd_file: str = "" + xrd_file_ext: str = "" + for file in file_paths: + ext = "".join(Path(file).suffixes) + if ext == '.json': + with open(file, mode="r", encoding="utf-8") as fl_obj: + config_dict = json.load(fl_obj) + elif ext in ['.yaml', '.yml']: + with open(file, mode="r", encoding="utf-8") as fl_obj: + eln_dict = flatten_and_replace( + FlattenSettings( + yaml.safe_load(fl_obj), + CONVERT_DICT, REPLACE_NESTED + ) + ) + elif ext in XRD_FILE_EXTENSIONS: + xrd_file_ext = ext + xrd_file = file + if xrd_file: + parse_and_fill_template(template, xrd_file, config_dict, eln_dict) + else: + raise ValueError(f"Allowed XRD experimental with extenstion from" + f" {XRD_FILE_EXTENSIONS} found {xrd_file_ext}") + + # Get rid of empty concept and cleaning up Template + for key, val in template.items(): + + if val is None: + del template[key] + else: + filled_template[key] = val + if not filled_template.keys(): + raise ValueError("Reader could not read anything! Check for input files and the" + " corresponding extention.") + return filled_template + + +READER = XRDReader diff --git a/pynxtools/dataconverter/readers/xrd/xrd_helper.py b/pynxtools/dataconverter/readers/xrd/xrd_helper.py new file mode 100644 index 000000000..40874be50 --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/xrd_helper.py @@ -0,0 +1,293 @@ +"""XRD helper stuffs.""" + +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +import numpy as np +from pynxtools.dataconverter.helpers import transform_to_intended_dt +from pynxtools.dataconverter.template import Template + + +class KeyValueNotFoundWaring(Warning): + """New Wanrning class""" + + +def get_a_value_or_warn(return_value="", + warning_catagory=KeyValueNotFoundWaring, + message="Key-value not found.", + stack_level=2): + """It returns a value that and rase the warning massage.""" + + warnings.warn(f"\033[1;31m {message}:\033[0m]", warning_catagory, stack_level) + return return_value + + +def check_unit(unit: str): + """Handle conflicted unit. + Some units comes with verdor file that do not follow correct format. + """ + if unit is None: + return unit + unit_map = {'Angstrom': '\u212B', + } + correct_unit = unit_map.get(unit, None) + if correct_unit is None: + return unit + return correct_unit + + +# pylint: disable=too-many-statements +def feed_xrdml_to_template(template, xrd_dict, eln_dict, file_term, config_dict=None): + """Fill template with data from xrdml type file. + + Parameters + ---------- + template : Dict + Template generated from nxdl definition file. + xrd_dict : dict + Just a dict mapping slash separated key to the data. The key is equivalent to the + path directing the location in data file. + eln_dict : dict + That brings the data from user especially using NeXus according to NeXus concept. + file_term : str + Terminological string to describe file ext. and version (e.g. xrdml_1.5) to find proper + dict from config file. + config_dict : Dict + Dictionary from config file that maps NeXus concept to data from different data file + versions. E.g. + { + "/ENTRY[entry]/2theta_plot/chi": {"file_exp": {"value": "", + "@units": ""},}, + "/ENTRY[entry]/2theta_plot/intensity": {"file_exp": {"value": "/detector", + "@units": ""},} + } + """ + + def fill_template_from_config_data(config_dict: dict, template: Template, + xrd_dict: dict, file_term: str) -> None: + """ + Parameters + ---------- + config_dict : dict + Python dict that is nested dict for different file versions. + e.g. + {"/ENTRY[entry]/2theta_plot/chi": {"file_exp": {"value": "", + "@units": ""},}, + "/ENTRY[entry]/2theta_plot/intensity": {"file_exp": {"value": "/detector", + "@units": ""},} + } + template : Template + + Return + ------ + None + """ + for nx_key, val in config_dict.items(): + if isinstance(val, dict): + raw_data_des: dict = val.get(file_term, None) + if raw_data_des is None: + raise ValueError(f"conflict file config file does not have any data map" + f" for file {file_term}") + # the field does not have any value + if not raw_data_des.get('value', None): + continue + # Note: path is the data path in raw file + for val_atr_key, path in raw_data_des.items(): + # data or field val + if val_atr_key == 'value': + template[nx_key] = xrd_dict.get(path, None) + elif path and val_atr_key == '@units': + template[nx_key + '/' + val_atr_key] = check_unit( + xrd_dict.get(path, None)) + # attr e.g. @AXISNAME + elif path and val_atr_key.startswith('@'): + template[nx_key + '/' + val_atr_key] = xrd_dict.get(path, None) + if not isinstance(val, dict) and isinstance(val, str): + template[nx_key] = val + + def two_theta_plot(): + + intesity = transform_to_intended_dt(template.get("/ENTRY[entry]/2theta_plot/intensity", + None)) + if intesity is not None: + intsity_len = np.shape(intesity)[0] + else: + raise ValueError("No intensity is found") + + two_theta_gr = "/ENTRY[entry]/2theta_plot/" + if template.get(f"{two_theta_gr}omega", None) is None: + omega_start = template.get("/ENTRY[entry]/COLLECTION[collection]/omega/start", None) + omega_end = template.get("/ENTRY[entry]/COLLECTION[collection]/omega/end", None) + + template["/ENTRY[entry]/2theta_plot/omega"] = np.linspace(float(omega_start), + float(omega_end), + intsity_len) + + if template.get(f"{two_theta_gr}two_theta", None) is None: + tw_theta_start = template.get("/ENTRY[entry]/COLLECTION[collection]/2theta/start", + None) + tw_theta_end = template.get("/ENTRY[entry]/COLLECTION[collection]/2theta/end", None) + template[f"{two_theta_gr}two_theta"] = np.linspace(float(tw_theta_start), + float(tw_theta_end), + intsity_len) + template[two_theta_gr + "/" + "@axes"] = ["two_theta"] + template[two_theta_gr + "/" + "@signal"] = "intensity" + + def q_plot(): + q_plot_gr = "/ENTRY[entry]/q_plot" + alpha_2 = template.get("/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_two", + None) + alpha_1 = template.get("/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_one", + None) + two_theta: np.ndarray = template.get("/ENTRY[entry]/2theta_plot/two_theta", None) + if two_theta is None: + raise ValueError("Two-theta data is not found") + if isinstance(two_theta, np.ndarray): + theta: np.ndarray = two_theta / 2 + ratio_k = "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/ratio_k_alphatwo_k_alphaone" + if alpha_1 and alpha_2: + ratio = alpha_2 / alpha_1 + template[ratio_k] = ratio + lamda = ratio * alpha_1 + (1 - ratio) * alpha_2 + q_vec = (4 * np.pi / lamda) * np.sin(np.deg2rad(theta)) + template[q_plot_gr + "/" + "q_vec"] = q_vec + template[q_plot_gr + "/" + "@q_vec_indicies"] = 0 + template[q_plot_gr + "/" + "@axes"] = ["q_vec"] + + template[q_plot_gr + "/" + "@signal"] = "intensity" + + def handle_special_fields(): + """Some fields need special treatment.""" + + key = "/ENTRY[entry]/COLLECTION[collection]/goniometer_x" + gonio_x = template.get(key, None) + + template[key] = gonio_x[0] if (isinstance(gonio_x, np.ndarray) + and gonio_x.shape == (1,)) else gonio_x + + key = "/ENTRY[entry]/COLLECTION[collection]/goniometer_y" + gonio_y = template.get(key, None) + + template[key] = gonio_y[0] if (isinstance(gonio_y, np.ndarray) + and gonio_y.shape == (1,)) else gonio_y + + key = "/ENTRY[entry]/COLLECTION[collection]/goniometer_z" + gonio_z = template.get(key, None) + + template[key] = gonio_z[0] if (isinstance(gonio_z, np.ndarray) + and gonio_z.shape == (1,)) else gonio_z + + key = "/ENTRY[entry]/COLLECTION[collection]/count_time" + count_time = template.get(key, None) + + template[key] = count_time[0] if (isinstance(count_time, np.ndarray) + and count_time.shape == (1,)) else count_time + + fill_template_from_config_data(config_dict, template, + xrd_dict, file_term) + two_theta_plot() + q_plot() + handle_special_fields() + + fill_template_from_eln_data(eln_dict, template) + + +# pylint: disable=unused-argument +def feed_udf_to_template(template, xrd_dict, eln_dict, config_dict): + """_summary_ + + Parameters + ---------- + template : _type_ + _description_ + xrd_dict : _type_ + _description_ + eln_dict : _type_ + _description_ + config_dict : _type_ + _description_ + """ + + +def feed_raw_to_template(template, xrd_dict, eln_dict, config_dict): + """_summary_ + + Parameters + ---------- + template : _type_ + _description_ + xrd_dict : _type_ + _description_ + eln_dict : _type_ + _description_ + config_dict : _type_ + _description_ + """ + + +def feed_xye_to_template(template, xrd_dict, eln_dict, config_dict): + """_summary_ + + Parameters + ---------- + template : _type_ + _description_ + xrd_dict : _type_ + _description_ + eln_dict : _type_ + _description_ + config_dict : _type_ + _description_ + """ + + +def fill_template_from_eln_data(eln_data_dict, template): + """Fill out the template from dict that generated from eln yaml file. + Parameters: + ----------- + eln_data_dict : dict[str, Any] + Python dictionary from eln file. + template : dict[str, Any] + Return: + ------- + None + """ + + if eln_data_dict is None: + return + for e_key, e_val in eln_data_dict.items(): + template[e_key] = transform_to_intended_dt(e_val) + + +def fill_nxdata_from_xrdml(template, + xrd_flattend_dict, + dt_nevigator_from_config_file, + data_group_concept + ): + """_summary_ + + Parameters + ---------- + template : _type_ + _description_ + xrd_flattend_dict : _type_ + _description_ + dt_nevigator_from_config_file : _type_ + _description_ + data_group_concept : _type_ + _description_ + """ diff --git a/pynxtools/dataconverter/readers/xrd/xrd_parser.py b/pynxtools/dataconverter/readers/xrd/xrd_parser.py new file mode 100644 index 000000000..9d944cad7 --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/xrd_parser.py @@ -0,0 +1,448 @@ +""" +XRD file parser collection. +""" + +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple, Optional, List + +from pathlib import Path +import warnings +import xml.etree.ElementTree as ET # for XML parsing +from pynxtools.dataconverter.helpers import transform_to_intended_dt, remove_namespace_from_tag +from pynxtools.dataconverter.readers.xrd.xrd_helper import feed_xrdml_to_template + + +def fill_slash_sep_dict_from_nested_dict(parent_path: str, nested_dict: dict, + slash_sep_dict: dict): + """Convert a nested dict into slash separated dict. + + Extend slash_sep_dict by key (slash separated key) from nested dict. + + Parameters + ---------- + parent_path : str + Parent path to be appended at the starting of slash separated key. + nested_dict : dict + Dict nesting other dict. + slash_sep_dict : dict + Plain dict to be extended by key value generated from nested_dict. + """ + for key, val in nested_dict.items(): + slash_sep_path = parent_path + key + if isinstance(val, dict): + fill_slash_sep_dict_from_nested_dict(slash_sep_path, val, slash_sep_dict) + else: + slash_sep_dict[slash_sep_path] = val + + +class IgnoreNodeTextWarning(Warning): + """Special class to warn node text skip.""" + + +class XRDMLParser: + """Parser for xrdml file with the help of other XRD library e.g. panalytical_xml.""" + + def __init__(self, file_path): + """Construct XRDMLParser obj. + + Parameters + ---------- + file_path : str + Path of the file. + """ + # In future it can be utilised later it different versions of file + # self.__version = None + self.__xrd_dict = {} + self.__file_path = file_path + self.xrdml_version: str = "" + self.xml_root = ET.parse(self.__file_path).getroot() + self.find_version() + # Important note for key-val pair separator list: preceding elements have precedence on the + # on the following elements + self.key_val_pair_sprtr = (';', ',') + # Important note for key-val separator list: preceding elements have precedence on the + # on the following elements + self.key_val_sprtr = ('=', ':') + + def find_version(self): + """To find xrdml file version.""" + schema_loc = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation" + # str: 'http://www.xrdml.com/XRDMeasurement/1.5 + version = self.xml_root.get(schema_loc).split(' ')[0] + self.xrdml_version = version.split('/')[-1] + + def get_slash_separated_xrd_dict(self): + """Return a dict with slash separated key and value from xrd file. + + The key is the slash separated string path for nested xml elements. + + Returns + ------- + dict: + Dictionary where key maps xml nested elements by slash separated str. + """ + # To navigate different functions in future according to some parameters + # such as version, and data analysis module from panalytical_xml + self.handle_with_panalytical_module() + return self.__xrd_dict + + def handle_with_panalytical_module(self): + """Handeling XRDml file by parsing xml file and Pnanalytical_xml parser + + Panalytical module extends and constructs some array data from experiment settings + comes with xml file. + """ + self.parse_each_elm(parent_path='/', xml_node=self.xml_root) + nested_data_dict: Dict[str, any] = {} + # Note: To use panalytical lib + # Extract other numerical data e.g. 'hkl', 'Omega', '2Theta', CountTime etc + # using panalytical_xml module + # parsed_data = XRDMLFile(self.__file_path) + # nested_data_dict = parsed_data.scan.ddict + fill_slash_sep_dict_from_nested_dict('/', nested_data_dict, self.__xrd_dict) + + def process_node_text(self, parent_path, node_txt) -> None: + """Processing text of node + + Parameters + ---------- + parent_path : str + Starting str of the key when forming a string key. + node_txt : str + text from node. + + Returns + ------ + None + """ + key_val_pairs = [] + # get key-val pair + for sep in self.key_val_pair_sprtr: + if sep in node_txt: + key_val_pairs.extend(node_txt.split(sep)) + break + # Separate key-val, build full path and + # store them in dict + if key_val_pairs: + for key_val in key_val_pairs: + for k_v_sep in self.key_val_sprtr: + if k_v_sep in key_val: + key, val = key_val.split(k_v_sep) + key = key.replace(' ', '') + self.__xrd_dict['/'.join([parent_path, key])] = val + break + # Handling array data comes as node text + else: + try: + self.__xrd_dict[parent_path] = transform_to_intended_dt(node_txt) + except ValueError: + warnings.warn(f'Element text {node_txt} is ignored from parseing!', + IgnoreNodeTextWarning) + + def parse_each_elm(self, parent_path, xml_node, + multi_childs_tag: str = '', + tag_extensions: Optional[List[int]] = None): + """Check each xml element and send the element to intended function. + + Parameters + ---------- + parent_path : str + Path to be in the starting of the key composing from element e.g. '/'. + xml_node : XML.Element + Any element except process instruction nodes. + multi_childs_tag : str + Tag that is available on several child nodes. + tag_extension : List[int] + List of extension of the child tag if there are several childs having the same + tag. + + Returns + ------ + None + """ + + tag = remove_namespace_from_tag(xml_node.tag) + # Take care of special node of 'entry' tag + if tag == 'entry': + parent_path = self.parse_entry_elm(parent_path, xml_node, + multi_childs_tag, tag_extensions) + else: + parent_path = self.parse_general_elm(parent_path, xml_node, + multi_childs_tag, tag_extensions) + + _, multi_childs_tag = self.has_multi_childs_with_same_tag(xml_node) + # List of tag extensions for child nodes which have the same tag. + tag_extensions = [0] + for child in iter(xml_node): + if child is not None: + self.parse_each_elm(parent_path, child, + multi_childs_tag, tag_extensions) + + def has_multi_childs_with_same_tag(self, parent_node: ET.Element) -> Tuple[bool, str]: + """Check for multiple childs that have the same tag. + + Parameter: + ---------- + parent_node : ET.Element + Parent node that might has multiple childs with the same tag. + + Returns: + -------- + Tuple[bool, str] + (true if multiple childs with the same tag, tag). + """ + tag: str = None + for child in iter(parent_node): + temp_tag = remove_namespace_from_tag(child.tag) + if tag is None: + tag = temp_tag + else: + if tag == temp_tag: + return (True, tag) + + return (False, '') + + def parse_general_elm(self, parent_path, xml_node, + multi_childs_tag, tag_extensions: List[int]): + """Handle general element except entry element. + Parameters + ---------- + parent_path : str + Path to be in the starting of the key composing from element e.g. '/'. + xml_node : XML.Element + Any element except process instruction and entry nodes. + multi_childs_tag : str + Tag that is available on several siblings. + tag_extension : List[int] + List of extension of the shiblings tag if there are several shiblings having + the same tag. + + Returns + ------- + None + """ + + tag = remove_namespace_from_tag(xml_node.tag) + if tag == multi_childs_tag: + new_ext = tag_extensions[-1] + 1 + tag = tag + '_' + str(new_ext) + tag_extensions.append(new_ext) + + if parent_path == '/': + parent_path = parent_path + tag + else: + # New parent path ends with element tag + parent_path = '/'.join([parent_path, tag]) + + node_attr = xml_node.attrib + if node_attr: + for key, val in node_attr.items(): + # Some attr has namespace + key = remove_namespace_from_tag(key) + key = key.replace(' ', '_') + path_extend = '/'.join([parent_path, key]) + self.__xrd_dict[path_extend] = val + + node_txt = xml_node.text + if node_txt: + self.process_node_text(parent_path, node_txt) + + return parent_path + + def parse_entry_elm(self, parent_path: str, xml_node: ET.Element, + multi_childs_tag: str, tag_extensions: List[int]): + """Handle entry element. + + Parameters + ---------- + parent_path : str + Path to be in the starting of the key composing from element e.g. '/'. + xml_node : XML.Element + Any entry node. + multi_childs_tag : str + Tag that is available on several siblings. + tag_extension : List[int] + List of extension of the shiblings tag if there are several shiblings having + the same tag. + + Returns + ------- + str: + Parent path. + """ + + tag = remove_namespace_from_tag(xml_node.tag) + + if tag == multi_childs_tag: + new_ext = tag_extensions[-1] + 1 + tag_extensions.append(new_ext) + tag = tag + '_' + str(new_ext) + + if parent_path == '/': + parent_path = '/' + tag + else: + # Parent path ends with element tag + parent_path = '/'.join([parent_path, tag]) + + node_attr = xml_node.attrib + if node_attr: + for key, val in node_attr.items(): + # Some attributes have namespace + key = remove_namespace_from_tag(key) + path_extend = '/'.join([parent_path, key]) + self.__xrd_dict[path_extend] = val + + # In entry element text must get special care on it + node_txt = xml_node.text + if node_txt: + self.process_node_text(parent_path, node_txt) + + return parent_path + + +class FormatParser: + """A class to identify and parse different file formats.""" + + def __init__(self, file_path): + """Construct FormatParser obj. + + Parameters + ---------- + file_path : str + XRD file to be parsed. + + Returns + ------- + None + """ + self.file_path = file_path + self.file_parser = XRDMLParser(self.file_path) + # termilnological name of file to read config file + self.file_term = 'xrdml_' + self.file_parser.xrdml_version + + def get_file_format(self): + """Identifies the format of a given file. + + Returns: + -------- + str: + The file extension of the file. + """ + file_extension = ''.join(Path(self.file_path).suffixes) + return file_extension + + def parse_xrdml(self): + """Parses a Panalytical XRDML file. + + Returns + ------- + dict + A dictionary containing the parsed XRDML data. + """ + return self.file_parser.get_slash_separated_xrd_dict() + + def parse_panalytical_udf(self): + """Parse the Panalytical .udf file. + + Returns + ------- + None + Placeholder for parsing .udf files. + """ + + def parse_bruker_raw(self): + """Parse the Bruker .raw file. + + Returns + None + """ + + def parse_bruker_xye(self): + """Parse the Bruker .xye file. + + Returns + None + """ + + # pylint: disable=import-outside-toplevel + def parse_and_populate_template(self, template, config_dict, eln_dict): + """Parse xrd file into dict and fill the template. + + Parameters + ---------- + template : Template + NeXus template generated from NeXus application definitions. + xrd_file : str + Name of the xrd file. + config_dict : dict + A dict geenerated from python + eln_dict : dict + A dict generatd from eln yaml file. + Returns: + None + """ + + xrd_dict = self.parse() + if len(config_dict) == 0 and self.file_parser.xrdml_version == '1.5': + from pynxtools.dataconverter.readers.xrd.config import xrdml + config_dict = xrdml + feed_xrdml_to_template(template, xrd_dict, eln_dict, + file_term=self.file_term, config_dict=config_dict) + + def parse(self): + '''Parses the file based on its format. + + Returns: + dict + A dictionary containing the parsed data. + + Raises: + ValueError: If the file format is unsupported. + ''' + file_format = self.get_file_format() + slash_sep_dict = {} + if file_format == ".xrdml": + slash_sep_dict = self.parse_xrdml() + # elif file_format == ".udf": + # return self.parse_panalytical_udf() + # elif file_format == ".raw": + # return self.parse_bruker_raw() + # elif file_format == ".xye": + # return self.parse_bruker_xye() + # else: + # raise ValueError(f"Unsupported file format: {file_format}") + return slash_sep_dict + + +def parse_and_fill_template(template, xrd_file, config_dict, eln_dict): + """Parse xrd file and fill the template with data from that file. + + Parameters + ---------- + template : Template[dict] + Template gnenerated from nxdl definition. + xrd_file : str + Name of the xrd file with extension + config_dict : Dict + Dictionary from config.json or similar file. + eln_dict : Dict + Plain and '/' separated dictionary from yaml for ELN. + """ + + format_parser = FormatParser(xrd_file) + format_parser.parse_and_populate_template(template, config_dict, eln_dict) diff --git a/pynxtools/dataconverter/template.py b/pynxtools/dataconverter/template.py index 28762ea17..a93c921e6 100644 --- a/pynxtools/dataconverter/template.py +++ b/pynxtools/dataconverter/template.py @@ -125,6 +125,27 @@ def __contains__(self, k): k in self.required ]) + def get(self, key, return_value=None): + """Implementing get method for template. + Parameters + ---------- + key : str + Template key + return_value : Any + return : + The value comes with return_value + """ + val = self.optional.get(key, None) + if val is None: + val = self.recommended.get(key, None) + if val is None: + val = self.required.get(key, None) + if val is None: + val = self.undocumented.get(key, None) + if val is None: + return return_value + return val + def __getitem__(self, k): """Handles how values are accessed from the Template object.""" # Try setting item in all else throw error. Does not append to default. diff --git a/pynxtools/dataconverter/writer.py b/pynxtools/dataconverter/writer.py index 6fc52337f..81b3045da 100644 --- a/pynxtools/dataconverter/writer.py +++ b/pynxtools/dataconverter/writer.py @@ -178,25 +178,27 @@ class Writer: Args: data (dict): Dictionary containing the data to convert. - nxdl_path (str): Path to the nxdl file to use during conversion. + nxdl_f_path (str): Path to the nxdl file to use during conversion. output_path (str): Path to the output NeXus file. Attributes: data (dict): Dictionary containing the data to convert. - nxdl_path (str): Path to the nxdl file to use during conversion. + nxdl_f_path (str): Path to the nxdl file to use during conversion. output_path (str): Path to the output NeXus file. output_nexus (h5py.File): The h5py file object to manipulate output file. nxdl_data (dict): Stores xml data from given nxdl file to use during conversion. nxs_namespace (str): The namespace used in the NXDL tags. Helps search for XML children. """ - def __init__(self, data: dict = None, nxdl_path: str = None, output_path: str = None): + def __init__(self, data: dict = None, + nxdl_f_path: str = None, + output_path: str = None): """Constructs the necessary objects required by the Writer class.""" self.data = data - self.nxdl_path = nxdl_path + self.nxdl_f_path = nxdl_f_path self.output_path = output_path self.output_nexus = h5py.File(self.output_path, "w") - self.nxdl_data = ET.parse(self.nxdl_path).getroot() + self.nxdl_data = ET.parse(self.nxdl_f_path).getroot() self.nxs_namespace = get_namespace(self.nxdl_data) def __nxdl_to_attrs(self, path: str = '/') -> dict: @@ -241,8 +243,9 @@ def ensure_and_get_parent_node(self, path: str, undocumented_paths) -> h5py.Grou return grp return self.output_nexus[parent_path_hdf5] - def write(self): - """Writes the NeXus file with previously validated data from the reader with NXDL attrs.""" + def _put_data_into_hdf5(self): + """Store data in hdf5 in in-memory file or file.""" + hdf5_links_for_later = [] def add_units_key(dataset, path): @@ -277,6 +280,7 @@ def add_units_key(dataset, path): except Exception as exc: raise IOError(f"Unknown error occured writing the path: {path} " f"with the following message: {str(exc)}") from exc + for links in hdf5_links_for_later: dataset = handle_dicts_entries(*links) if dataset is None: @@ -306,4 +310,9 @@ def add_units_key(dataset, path): raise IOError(f"Unknown error occured writing the path: {path} " f"with the following message: {str(exc)}") from exc - self.output_nexus.close() + def write(self): + """Writes the NeXus file with previously validated data from the reader with NXDL attrs.""" + try: + self._put_data_into_hdf5() + finally: + self.output_nexus.close() diff --git a/pynxtools/definitions b/pynxtools/definitions index 1a694807a..999837671 160000 --- a/pynxtools/definitions +++ b/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 1a694807aaea98cea34240ee60300692a4fb5dc9 +Subproject commit 999837671373b962fed932829becd42acb7482f6 diff --git a/pynxtools/eln_mapper/scheme_eln.py b/pynxtools/eln_mapper/scheme_eln.py index 277658c28..1152bbd08 100644 --- a/pynxtools/eln_mapper/scheme_eln.py +++ b/pynxtools/eln_mapper/scheme_eln.py @@ -276,7 +276,6 @@ def generate_scheme_eln(nexus_def: str, eln_file_name: str = None) -> None: recursive_dict: Dict[str, Any] = {} get_eln_recursive_dict(recursive_dict, nxdl_file) - # print('recursive_dict', recursive_dict) with open(out_file, mode='w', encoding='utf-8') as out_f: yaml.dump(recursive_dict, sort_keys=False, stream=out_f) diff --git a/pynxtools/nexus/nexus.py b/pynxtools/nexus/nexus.py index ae6a794eb..ef5f64cd5 100644 --- a/pynxtools/nexus/nexus.py +++ b/pynxtools/nexus/nexus.py @@ -564,8 +564,11 @@ def hdf_node_to_self_concept_path(hdf_info, logger): class HandleNexus: """documentation""" + + # pylint: disable=too-many-instance-attributes def __init__(self, logger, nexus_file, - d_inq_nd=None, c_inq_nd=None): + d_inq_nd=None, c_inq_nd=None, + is_in_memory_file=False): self.logger = logger local_dir = os.path.abspath(os.path.dirname(__file__)) @@ -573,6 +576,7 @@ def __init__(self, logger, nexus_file, os.path.join(local_dir, '../../tests/data/nexus/201805_WSe2_arpes.nxs') self.parser = None self.in_file = None + self.is_hdf5_file_obj = is_in_memory_file self.d_inq_nd = d_inq_nd self.c_inq_nd = c_inq_nd # Aggregating hdf path corresponds to concept query node @@ -639,19 +643,28 @@ def full_visit(self, root, hdf_node, name, func): def process_nexus_master_file(self, parser): """Process a nexus master file by processing all its nodes and their attributes""" self.parser = parser - self.in_file = h5py.File( - self.input_file_name[0] - if isinstance(self.input_file_name, list) - else self.input_file_name, 'r' - ) - self.full_visit(self.in_file, self.in_file, '', self.visit_node) - if self.d_inq_nd is None and self.c_inq_nd is None: - get_default_plotable(self.in_file, self.logger) - # To log the provided concept and concepts founded - if self.c_inq_nd is not None: - for hdf_path in self.hdf_path_list_for_c_inq_nd: - self.logger.info(hdf_path) - self.in_file.close() + try: + if not self.is_hdf5_file_obj: + self.in_file = h5py.File( + self.input_file_name[0] + if isinstance(self.input_file_name, list) + else self.input_file_name, 'r' + ) + else: + self.in_file = self.input_file_name + + self.full_visit(self.in_file, self.in_file, '', self.visit_node) + + if self.d_inq_nd is None and self.c_inq_nd is None: + get_default_plotable(self.in_file, self.logger) + # To log the provided concept and concepts founded + if self.c_inq_nd is not None: + for hdf_path in self.hdf_path_list_for_c_inq_nd: + self.logger.info(hdf_path) + finally: + # To test if hdf_file is open print(self.in_file.id.valid) + self.in_file.close() + # To test if hdf_file is open print(self.in_file.id.valid) @click.command() diff --git a/tests/data/dataconverter/readers/mpes/Ref_nexus_mpes.log b/tests/data/dataconverter/readers/mpes/Ref_nexus_mpes.log index b9241804a..d4a58e2ee 100644 --- a/tests/data/dataconverter/readers/mpes/Ref_nexus_mpes.log +++ b/tests/data/dataconverter/readers/mpes/Ref_nexus_mpes.log @@ -1362,14 +1362,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_A@NX_class) DEBUG - value: NXlens_em @@ -1398,8 +1398,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_A/voltage@units) DEBUG - value: V @@ -1419,14 +1421,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_B@NX_class) DEBUG - value: NXlens_em @@ -1455,8 +1457,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_B/voltage@units) DEBUG - value: V @@ -1476,14 +1480,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_C@NX_class) DEBUG - value: NXlens_em @@ -1512,8 +1516,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_C/voltage@units) DEBUG - value: V @@ -1533,14 +1539,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_D@NX_class) DEBUG - value: NXlens_em @@ -1569,8 +1575,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_D/voltage@units) DEBUG - value: V @@ -1590,14 +1598,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_E@NX_class) DEBUG - value: NXlens_em @@ -1626,8 +1634,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_E/voltage@units) DEBUG - value: V @@ -1647,14 +1657,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_F@NX_class) DEBUG - value: NXlens_em @@ -1683,8 +1693,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_F/voltage@units) DEBUG - value: V @@ -1704,14 +1716,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_Foc@NX_class) DEBUG - value: NXlens_em @@ -1740,8 +1752,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_Foc/voltage@units) DEBUG - value: V @@ -1761,14 +1775,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_G@NX_class) DEBUG - value: NXlens_em @@ -1797,8 +1811,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_G/voltage@units) DEBUG - value: V @@ -1818,14 +1834,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_H@NX_class) DEBUG - value: NXlens_em @@ -1854,8 +1870,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_H/voltage@units) DEBUG - value: V @@ -1875,14 +1893,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_I@NX_class) DEBUG - value: NXlens_em @@ -1911,8 +1929,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_I/voltage@units) DEBUG - value: V @@ -1932,14 +1952,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_UCA@NX_class) DEBUG - value: NXlens_em @@ -1968,8 +1988,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_UCA/voltage@units) DEBUG - value: V @@ -1989,14 +2011,14 @@ DEBUG - DEBUG - documentation (NXlens_em.nxdl.xml:): DEBUG - - Description of an electro-magnetic lens or a compound lens. + Base class for an electro-magnetic lens or a compound lens. - For NXtransformations the origin of the coordinate system is placed - in the center of the lens - (its polepiece, pinhole, or another point of reference). - The origin should be specified in the NXtransformations. + For :ref:`NXtransformations` the origin of the coordinate system is placed + in the center of the lens (its polepiece, pinhole, or another + point of reference). The origin should be specified in the :ref:`NXtransformations`. - For details of electro-magnetic lenses in the literature see e.g. `L. Reimer `_ + For details of electro-magnetic lenses in the literature + see e.g. `L. Reimer `_ DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_UFA@NX_class) DEBUG - value: NXlens_em @@ -2025,8 +2047,10 @@ NXlens_em.nxdl.xml:/voltage DEBUG - <> DEBUG - documentation (NXlens_em.nxdl.xml:/voltage): DEBUG - - Excitation voltage of the lens. For dipoles it is a single number. For higher - orders, it is an array. + Excitation voltage of the lens. + + For dipoles it is a single number. + For higher order multipoles, it is an array. DEBUG - ===== ATTRS (//entry/instrument/electronanalyser/collectioncolumn/lens_UFA/voltage@units) DEBUG - value: V @@ -3669,7 +3693,13 @@ DEBUG - Use a0, a1, ..., an for the coefficients, corresponding to the values in the coefficients field. - Use x0, x1, ..., xn for the variables. + Use x0, x1, ..., xn for the nth position in the `original_axis` field. + If there is the symbol attribute specified for the `original_axis` this may be used instead of x. + If you want to use the whole axis use `x`. + Alternate axis can also be available as specified by the `input_SYMBOL` field. + The data should then be referred here by the `SYMBOL` name, e.g., for a field + name `input_my_field` it should be referred here by `my_field` or `my_field0` if + you want to read the zeroth element of the array. The formula should be numpy compliant. @@ -3734,6 +3764,7 @@ DEBUG - <> DEBUG - documentation (NXcalibration.nxdl.xml:/offset): DEBUG - For linear calibration. Offset parameter. + This is should yield the relation `calibrated_axis` = `scaling` * `original_axis` + `offset`. DEBUG - ===== FIELD (//entry/process/kx_calibration/scaling): DEBUG - value: 0.01046958495673419 @@ -3744,6 +3775,7 @@ DEBUG - <> DEBUG - documentation (NXcalibration.nxdl.xml:/scaling): DEBUG - For linear calibration. Scaling parameter. + This is should yield the relation `calibrated_axis` = `scaling` * `original_axis` + `offset`. DEBUG - ===== GROUP (//entry/process/ky_calibration [NXmpes::/NXentry/NXprocess/NXcalibration]): DEBUG - classpath: ['NXentry', 'NXprocess', 'NXcalibration'] @@ -3796,6 +3828,7 @@ DEBUG - <> DEBUG - documentation (NXcalibration.nxdl.xml:/offset): DEBUG - For linear calibration. Offset parameter. + This is should yield the relation `calibrated_axis` = `scaling` * `original_axis` + `offset`. DEBUG - ===== FIELD (//entry/process/ky_calibration/scaling): DEBUG - value: 0.01046958495673419 @@ -3806,6 +3839,7 @@ DEBUG - <> DEBUG - documentation (NXcalibration.nxdl.xml:/scaling): DEBUG - For linear calibration. Scaling parameter. + This is should yield the relation `calibrated_axis` = `scaling` * `original_axis` + `offset`. DEBUG - ===== GROUP (//entry/process/registration [NXmpes::/NXentry/NXprocess/NXregistration]): DEBUG - classpath: ['NXentry', 'NXprocess', 'NXregistration'] diff --git a/tests/data/dataconverter/readers/xrd/ACZCTS_5-60_181.xrdml b/tests/data/dataconverter/readers/xrd/ACZCTS_5-60_181.xrdml new file mode 100644 index 000000000..5af61b718 --- /dev/null +++ b/tests/data/dataconverter/readers/xrd/ACZCTS_5-60_181.xrdml @@ -0,0 +1,106 @@ + + + + Configuration=XYZ Stage, Owner=User-1, Creation date=02-Nov-17 2:08:05 PM + Goniometer=PW3050/60 (Theta/Theta); Minimum step size 2Theta:0.001; Minimum step size Omega:0.001 + Sample stage=Programmable x,y,z stage; Minimum step size X:0.01; Minimum step size Y:0.01; Minimum step size Z:0.001 + Diffractometer system=XPERT-PRO + Measurement program=D:\user\Pepe\program_files\BB_10-80_18min.xrdmp, Identifier={5202C7B3-EFFD-43D7-83CA-1A77018B086F} + Batch program=D:\user\Pepe\program_files\batch_5samples.xrdmp, Identifier={49B9A751-97B5-49F0-9CAA-E07AE5E71B6A} + + + + + + + + + PHD Lower Level = 4.02 (keV), PHD Upper Level = 11.26 (keV) + + + 1.5405980 + 1.5444260 + 1.3922500 + 0.5000 + + + 240.00 + + 40 + 40 + Cu + + 12.0 + 0.4 + 6.0 + + + + 0.0400 + + + 11.60 + + + 140.00 + 0.38 + + + + 240.00 + + 8.00 + + + 0.0400 + + + Ni + 0.020 + + + + 25.0 + 70.0 + + Scanning + 3.347 + + + +
+ 2018-06-12T18:34:49+02:00 + 2018-06-12T18:53:34+02:00 + + cnu_xlab + + + Data Collector + XPERT-PRO + 0000000011035964 + +
+ + + 10.00656514 + 79.99097181 + + + 5.00328257 + 39.99548591 + + + 0.00 + + + 50.00 + + + 7.868 + + 49.470 + 2086 2053 2118 2024 2127 2128 2115 2093 2063 1985 2038 2118 2107 2136 2080 2150 2036 2039 2073 2025 2069 2028 2070 1975 1978 2068 2116 2075 2000 2082 1944 2037 1994 2061 2035 2132 2046 2133 1981 2066 1986 2008 2028 2052 2065 2136 2104 2096 1935 2029 2036 2023 1933 2023 2040 2080 1964 2003 1960 1940 2079 2102 1959 2058 1970 2002 1974 1955 1998 2025 2060 1908 2014 2037 1925 2028 1944 2052 2012 2002 2018 1977 2063 1963 2073 1963 1964 1904 1971 1979 1975 2012 2008 1988 1946 1990 1894 1990 1897 1979 1934 2009 1986 1978 1915 1915 2010 1953 1891 1978 1983 1997 1984 1959 2032 1990 1949 1959 1925 1978 1963 1879 2055 1883 1974 1949 1929 1987 2000 1893 1926 1961 1978 1920 1965 1962 1919 1930 2007 1970 1960 1904 2005 1946 1893 1950 1908 1906 1983 1885 1928 1882 1835 1916 1951 2040 1986 1905 1970 1889 1916 2011 1973 1898 1954 1935 1839 1977 1876 1834 1850 1844 1928 2021 1913 1858 1860 1800 1907 1844 1850 1856 1924 1896 1902 1910 1904 1881 1916 2005 1825 1881 1902 1986 1936 1860 1928 1935 1858 1918 1873 1854 1954 1807 1871 1778 1847 1898 1836 1861 1855 1907 1804 1907 1861 1833 1904 1835 1899 1862 1826 1881 1905 1876 1969 1962 1869 1888 1808 1770 1888 1865 1794 1852 1834 1851 1840 1846 1872 1850 1829 1818 1817 1869 1760 1888 1895 1866 1896 1807 1867 1834 1743 1835 1866 1817 1816 1821 1781 1828 1790 1836 1841 1774 1831 1825 1804 1795 1776 1888 1805 1771 1837 1857 1777 1758 1801 1864 1796 1808 1801 1798 1786 1758 1815 1865 1809 1767 1790 1785 1719 1869 1762 1819 1739 1855 1746 1796 1804 1748 1745 1787 1835 1826 1753 1823 1867 1833 1777 1761 1778 1806 1826 1833 1740 1735 1792 1788 1738 1704 1711 1767 1841 1749 1791 1850 1794 1735 1761 1863 1800 1673 1684 1738 1651 1779 1686 1700 1767 1737 1722 1805 1812 1746 1720 1647 1744 1746 1724 1739 1734 1697 1780 1684 1672 1699 1667 1685 1777 1647 1695 1761 1750 1700 1724 1719 1678 1726 1727 1653 1766 1753 1753 1729 1816 1708 1720 1661 1686 1812 1733 1823 1696 1716 1663 1727 1665 1731 1756 1698 1692 1726 1676 1751 1761 1628 1774 1719 1565 1690 1709 1667 1660 1663 1682 1759 1685 1673 1653 1724 1648 1745 1698 1615 1708 1691 1619 1694 1603 1620 1647 1663 1673 1646 1616 1627 1618 1615 1624 1630 1625 1629 1705 1657 1661 1633 1613 1611 1673 1643 1653 1576 1696 1649 1582 1601 1565 1696 1637 1592 1639 1579 1665 1561 1640 1716 1640 1632 1674 1712 1723 1647 1688 1666 1728 1706 1780 1731 1737 1801 1785 1910 1889 1967 1925 1833 1872 1855 1773 1799 1784 1684 1751 1765 1688 1633 1661 1699 1747 1668 1631 1607 1620 1690 1590 1594 1633 1665 1666 1698 1652 1642 1628 1662 1647 1621 1646 1674 1669 1696 1640 1717 1634 1611 1637 1670 1653 1583 1606 1620 1690 1528 1627 1596 1689 1625 1576 1612 1543 1716 1575 1690 1571 1586 1556 1637 1611 1608 1586 1670 1579 1646 1570 1695 1602 1518 1491 1679 1755 1685 1729 1572 1564 1592 1655 1661 1609 1621 1652 1676 1612 1631 1617 1571 1584 1690 1624 1660 1667 1635 1606 1670 1555 1604 1709 1646 1652 1613 1663 1588 1648 1625 1548 1654 1639 1542 1495 1639 1628 1602 1615 1609 1541 1721 1580 1640 1567 1604 1685 1610 1619 1677 1753 1667 1723 1697 1782 1777 1682 1737 1746 1654 1730 1687 1592 1552 1590 1576 1606 1608 1541 1589 1489 1596 1494 1598 1591 1610 1553 1527 1602 1626 1664 1549 1542 1580 1587 1544 1562 1563 1533 1482 1508 1602 1555 1557 1585 1507 1469 1582 1552 1575 1575 1528 1515 1518 1539 1560 1615 1491 1523 1537 1474 1537 1571 1521 1523 1560 1522 1488 1466 1627 1528 1637 1553 1580 1619 1505 1472 1552 1443 1558 1501 1542 1589 1574 1532 1566 1446 1503 1555 1563 1467 1492 1476 1570 1540 1562 1515 1480 1491 1483 1509 1455 1549 1502 1543 1531 1546 1516 1447 1527 1480 1592 1550 1496 1496 1474 1431 1468 1537 1519 1526 1507 1525 1500 1494 1503 1504 1522 1541 1584 1522 1466 1498 1435 1533 1520 1584 1415 1490 1406 1520 1481 1548 1440 1471 1559 1511 1571 1518 1544 1440 1410 1451 1516 1437 1549 1459 1493 1510 1533 1487 1419 1475 1452 1500 1415 1438 1474 1493 1436 1449 1422 1441 1404 1407 1422 1413 1477 1408 1419 1475 1425 1414 1501 1335 1431 1434 1487 1430 1496 1440 1489 1420 1435 1537 1352 1456 1438 1460 1455 1443 1533 1434 1506 1417 1441 1440 1429 1483 1400 1475 1512 1424 1454 1485 1401 1513 1449 1403 1393 1417 1432 1438 1391 1393 1380 1376 1473 1430 1387 1405 1434 1376 1441 1394 1407 1389 1363 1396 1452 1420 1396 1360 1412 1432 1427 1375 1424 1367 1372 1407 1523 1410 1372 1383 1417 1449 1330 1343 1407 1377 1416 1448 1429 1498 1464 1412 1390 1402 1313 1351 1434 1360 1410 1412 1370 1343 1359 1402 1381 1352 1419 1361 1417 1388 1388 1359 1396 1366 1369 1368 1338 1333 1406 1336 1351 1366 1369 1364 1365 1358 1346 1372 1340 1251 1314 1363 1472 1391 1344 1398 1394 1363 1410 1312 1430 1450 1381 1410 1371 1371 1378 1370 1316 1326 1355 1257 1303 1326 1388 1322 1368 1324 1350 1403 1413 1423 1403 1400 1414 1363 1403 1318 1324 1309 1360 1325 1376 1402 1391 1419 1323 1338 1477 1372 1430 1425 1427 1426 1414 1478 1515 1547 1635 1621 1653 1795 1726 1706 1611 1538 1541 1466 1505 1430 1449 1358 1377 1375 1350 1315 1356 1312 1279 1285 1303 1285 1300 1385 1283 1302 1413 1344 1327 1288 1288 1317 1321 1322 1377 1346 1385 1331 1419 1350 1271 1405 1356 1318 1283 1328 1303 1358 1352 1333 1310 1267 1276 1287 1318 1304 1277 1321 1274 1294 1292 1333 1235 1233 1249 1299 1334 1313 1266 1321 1278 1293 1267 1257 1247 1381 1313 1342 1268 1300 1239 1286 1328 1319 1300 1333 1300 1285 1355 1202 1251 1340 1255 1306 1300 1270 1277 1292 1182 1265 1332 1291 1211 1296 1366 1242 1257 1323 1296 1255 1241 1312 1310 1206 1198 1234 1269 1322 1272 1341 1339 1195 1247 1284 1278 1323 1287 1242 1251 1225 1273 1271 1231 1266 1208 1246 1289 1229 1221 1224 1266 1248 1256 1273 1227 1254 1246 1207 1296 1292 1236 1245 1275 1252 1216 1250 1241 1218 1246 1249 1253 1291 1250 1269 1266 1186 1229 1283 1243 1297 1224 1201 1200 1192 1246 1211 1275 1297 1274 1202 1239 1243 1223 1211 1250 1300 1267 1232 1233 1256 1213 1168 1234 1349 1331 1229 1280 1344 1281 1302 1265 1316 1297 1229 1284 1241 1275 1268 1292 1315 1323 1357 1363 1362 1387 1428 1480 1582 1459 1478 1425 1303 1284 1233 1312 1269 1249 1218 1215 1258 1183 1238 1224 1276 1238 1216 1148 1141 1144 1158 1159 1140 1203 1234 1224 1269 1147 1167 1192 1225 1160 1175 1169 1266 1204 1175 1227 1173 1211 1202 1143 1201 1170 1171 1255 1246 1222 1211 1132 1184 1160 1163 1158 1107 1197 1202 1143 1243 1137 1212 1182 1156 1166 1213 1232 1217 1229 1170 1203 1172 1175 1246 1214 1185 1185 1280 1136 1201 1170 1210 1164 1180 1205 1272 1156 1104 1202 1187 1190 1162 1207 1186 1184 1289 1187 1165 1156 1144 1125 1250 1222 1215 1224 1204 1247 1176 1213 1226 1270 1237 1336 1323 1295 1384 1445 1468 1431 1459 1497 1540 1510 1544 1524 1612 1554 1417 1518 1441 1484 1376 1415 1396 1443 1360 1359 1317 1347 1352 1378 1335 1360 1331 1303 1429 1285 1338 1341 1417 1356 1437 1353 1380 1410 1412 1360 1425 1516 1457 1554 1527 1489 1613 1672 1661 1901 1849 1830 2014 2074 2072 2293 2524 2763 2892 3235 3594 4081 4820 5534 6332 7671 9236 11473 14125 17900 23015 29549 33330 33088 27079 20836 18840 18681 17777 14397 9708 6420 4517 3701 3046 2821 2422 2290 2084 1966 1863 1790 1692 1678 1594 1564 1529 1448 1395 1441 1498 1455 1417 1441 1394 1403 1384 1394 1352 1303 1333 1309 1321 1279 1271 1318 1254 1327 1285 1258 1198 1261 1278 1294 1196 1197 1174 1202 1253 1179 1211 1214 1233 1231 1283 1273 1231 1340 1337 1360 1363 1362 1396 1431 1401 1420 1445 1465 1394 1373 1325 1298 1284 1235 1245 1276 1199 1192 1156 1157 1237 1179 1148 1088 1159 1145 1172 1160 1075 1136 1159 1134 1135 1158 1143 1194 1114 1122 1081 1131 1077 1119 1091 1061 1091 1113 1159 1037 1109 1065 1105 1117 1073 1134 1090 1073 1101 1094 1043 1039 1121 1075 1125 1082 1023 1045 1042 1133 1115 1070 1014 1019 1069 998 1116 983 1030 1053 1024 1022 1063 1058 1018 1089 1034 1040 1041 1099 1002 1066 1008 1048 978 1110 1041 1032 1064 1032 1047 1086 1049 1076 1088 1003 1041 1056 1099 1101 1084 1074 1110 1095 1106 1058 1145 1141 1113 1142 1177 1087 1096 1122 1108 1097 1076 1036 1041 1020 1045 1116 1056 1040 1030 1053 1030 1022 1052 1030 954 1044 1052 1013 1018 1035 1013 1010 1043 989 1009 1027 994 997 1043 962 1013 1033 1044 971 988 989 988 991 1044 1000 967 1005 1016 1037 1011 991 957 965 1031 1053 992 996 1044 969 953 986 1021 986 989 972 939 1001 1043 974 981 1020 1019 957 967 1002 1016 969 949 988 912 983 975 952 990 1015 927 1035 996 1043 933 992 1016 1005 989 920 1060 1003 955 969 1014 961 985 1030 932 987 1021 1034 1019 998 1031 1051 954 1008 946 977 980 993 973 1004 1081 1070 1027 1053 1010 1072 1120 1079 1051 1087 1064 1098 1081 1102 1189 1120 1224 1377 1318 1410 1504 1582 1717 1620 1629 1555 1488 1500 1486 1385 1416 1364 1329 1200 1227 1226 1149 1208 1182 1181 1083 1164 1187 1191 1186 1178 1154 1166 1200 1227 1249 1292 1205 1329 1246 1279 1353 1384 1390 1364 1287 1281 1338 1325 1273 1220 1303 1157 1210 1195 1148 1170 1176 1259 1163 1127 1185 1174 1123 1107 1146 1196 1065 1077 1128 1119 1200 1164 1144 1166 1248 1154 1154 1115 1122 1112 1136 1086 1155 1273 1124 1190 1151 1071 1154 1074 1167 1134 1114 1082 1077 1105 1022 1070 1051 1053 1015 980 1021 1005 997 1009 1021 952 1012 1061 1032 1008 1006 1019 1006 947 965 994 1056 907 970 981 955 955 993 1011 1002 951 1013 1017 976 984 889 984 964 970 984 931 933 933 932 916 945 946 944 930 921 924 907 917 907 1024 953 904 927 937 921 950 948 975 869 926 861 946 915 886 891 923 895 879 825 899 878 919 896 879 860 832 928 911 864 882 921 888 892 879 846 963 919 952 893 910 913 884 954 824 884 879 891 860 879 917 887 928 857 933 857 927 881 875 844 873 910 881 840 887 854 899 860 909 857 885 835 854 887 896 932 893 896 865 879 899 878 887 844 843 864 850 800 869 905 857 894 897 928 873 841 822 886 849 828 854 922 882 855 825 817 858 901 863 840 894 886 902 862 842 877 854 845 870 823 864 892 890 866 854 860 917 871 888 863 900 860 794 866 912 935 884 827 914 891 924 964 899 892 890 884 907 851 876 924 944 996 974 1058 1011 1002 1055 1089 1091 1147 1209 1167 1237 1157 1164 1107 1082 1046 1075 1041 1022 982 979 956 953 881 831 861 889 894 834 901 873 839 878 895 891 954 916 958 959 908 969 913 950 835 887 910 902 948 909 875 877 905 856 900 926 872 843 854 829 836 857 904 800 817 899 894 871 778 834 821 873 878 799 877 853 847 847 874 902 901 864 845 835 897 850 833 810 804 850 876 897 905 887 860 858 899 887 834 889 863 893 830 914 925 927 909 885 965 940 947 907 911 916 873 885 847 923 846 836 850 856 804 784 847 862 824 844 881 837 813 818 848 887 887 853 808 804 853 840 862 919 837 820 808 905 838 861 857 817 800 841 871 880 874 839 889 855 844 875 874 872 848 881 896 831 822 833 840 876 858 847 854 841 841 930 977 950 939 940 956 932 952 959 1014 992 967 984 973 950 984 1008 997 994 991 1006 1012 1063 1010 1091 1115 1160 1105 1179 1198 1202 1222 1282 1314 1311 1389 1377 1405 1312 1393 1377 1416 1385 1349 1347 1410 1391 1327 1442 1382 1349 1386 1397 1445 1386 1474 1458 1450 1414 1420 1448 1469 1521 1501 1487 1537 1531 1456 1594 1529 1635 1574 1574 1611 1675 1568 1634 1589 1642 1717 1679 1708 1737 1659 1745 1763 1717 1730 1847 1782 1803 1864 1952 2040 2017 2101 2098 2067 2177 2236 2309 2344 2535 2521 2625 2722 2983 2906 3164 3413 3643 3771 4111 4374 4700 5170 5606 5920 6519 6981 7807 8535 9324 10317 11260 12671 13820 15930 17757 19828 22311 25843 29316 33738 38285 43955 50089 56440 62996 69428 74307 77322 77919 76144 73310 68798 65125 61819 58419 55442 51345 46303 41361 35801 31152 26874 23004 19856 17349 15284 13441 12150 10675 9276 8485 7737 6932 6383 5863 5307 4915 4516 4192 4020 3706 3556 3398 3187 3064 3015 2861 2714 2625 2473 2409 2405 2286 2247 2259 2141 2098 2051 2054 1956 1979 1973 1864 1824 1795 1720 1800 1753 1688 1682 1568 1642 1570 1530 1523 1570 1583 1501 1501 1572 1478 1499 1405 1369 1423 1472 1457 1392 1361 1440 1305 1326 1352 1329 1302 1309 1278 1278 1295 1284 1252 1222 1234 1274 1212 1236 1183 1202 1161 1198 1265 1141 1123 1131 1172 1237 1158 1115 1172 1226 1122 1127 1172 1135 1089 1077 1100 1073 1033 1042 1064 1088 1121 1092 1095 1104 1061 1058 1049 1057 1040 1007 1050 1014 1042 1026 979 1082 1040 963 953 965 999 1017 1014 1025 907 935 1023 983 982 990 959 1025 939 897 917 925 998 984 948 964 936 1010 921 922 883 982 930 872 912 955 922 924 937 894 982 964 934 911 908 928 981 956 985 959 972 990 946 962 1004 967 1025 1085 1038 1068 1072 1065 1095 1119 1100 1065 1160 1138 1098 1134 1032 1072 1072 1024 1033 1015 1016 996 945 937 909 961 984 903 908 955 880 893 866 874 903 877 892 865 942 895 904 892 900 887 914 865 896 841 876 916 940 936 934 987 912 945 959 947 980 979 997 1050 1052 1105 1066 1156 1137 1118 1094 1225 1177 1199 1180 1166 1105 1174 1144 1095 1112 1066 1063 1089 1029 1004 1026 919 991 944 916 893 879 903 949 953 879 934 917 878 875 872 918 923 910 897 866 808 857 827 816 837 814 820 768 802 777 731 818 759 782 808 815 797 724 792 745 759 714 787 777 749 758 865 767 707 810 772 737 739 748 727 764 787 753 761 763 766 766 765 730 828 789 862 801 816 795 797 832 888 888 821 772 802 848 831 795 883 789 794 828 761 784 797 810 764 785 774 738 728 736 745 773 733 753 731 742 675 736 741 721 728 726 731 726 733 761 741 734 711 762 716 702 726 742 749 688 726 758 775 728 746 709 694 687 694 639 724 727 777 671 693 747 667 711 743 738 741 730 723 756 713 721 680 675 718 708 704 745 728 738 675 686 738 708 758 711 702 756 793 749 717 792 693 756 763 775 821 840 877 932 934 1009 1090 1302 1464 1448 1383 1347 1208 1126 1163 1175 1169 1176 1209 1098 1067 1056 1049 977 964 917 1034 963 1071 1236 1316 1477 1628 1928 2090 2115 1943 1715 1525 1332 1346 1373 1375 1474 1490 1406 1238 1076 994 885 851 813 804 834 780 796 713 717 689 698 803 719 735 653 718 690 617 692 702 687 671 684 742 695 720 714 716 688 701 725 681 725 642 709 713 600 680 659 648 601 642 657 682 629 689 638 685 601 635 662 665 669 666 645 682 641 663 644 672 698 669 639 636 637 697 677 603 616 669 648 619 663 621 608 678 672 663 640 667 655 683 638 631 623 626 655 598 611 622 589 642 626 636 642 668 579 639 588 667 638 685 649 609 605 671 625 608 630 622 633 627 630 629 677 647 652 623 646 622 629 621 583 682 656 630 662 659 611 596 593 590 604 585 568 676 657 605 643 638 657 598 607 597 645 578 620 590 587 635 590 644 625 601 596 656 648 612 635 653 663 646 631 631 684 646 706 664 695 714 652 657 705 702 600 691 693 708 648 694 638 651 669 643 640 705 651 728 638 611 610 578 697 729 662 686 667 690 640 615 643 638 599 665 609 635 619 614 595 582 623 612 578 614 617 587 608 586 596 564 608 600 625 597 554 592 622 602 588 624 581 589 664 610 586 659 583 598 605 607 587 584 610 600 589 593 587 605 598 581 576 580 607 615 593 641 560 585 614 619 594 593 562 598 585 551 563 605 581 644 568 578 613 616 573 609 634 601 600 593 607 606 610 596 595 624 649 631 614 642 588 592 630 664 623 610 587 598 571 580 582 595 581 604 589 564 544 557 581 593 560 608 570 573 583 602 587 574 576 586 609 588 609 593 550 588 574 577 595 576 548 557 559 553 583 514 514 602 506 564 564 619 581 578 595 580 581 524 579 596 540 588 588 540 543 557 586 578 544 573 574 581 539 572 573 607 579 567 547 540 589 619 584 542 551 548 548 571 591 574 518 525 534 534 542 547 573 548 549 521 572 560 510 569 550 589 558 547 584 552 564 548 538 591 539 585 564 598 557 566 583 551 532 581 593 497 553 621 590 565 589 593 580 570 515 565 531 553 542 583 576 545 583 530 534 558 580 548 553 580 582 541 564 541 547 549 532 566 535 536 543 560 536 550 566 530 455 525 530 550 578 533 585 576 562 525 521 542 522 575 548 529 603 522 515 510 522 571 582 521 548 577 575 588 570 589 593 604 577 599 584 612 579 615 621 577 625 579 574 551 546 592 578 558 602 567 566 559 564 551 559 534 553 613 568 498 536 504 531 544 493 563 518 527 538 539 551 534 553 529 602 533 530 542 528 541 582 586 541 512 602 540 504 507 557 516 552 527 548 539 512 504 506 539 529 527 530 552 522 516 540 510 544 550 521 556 578 517 594 520 576 490 582 563 543 588 603 570 558 561 580 521 533 542 604 512 573 549 551 548 542 546 545 536 541 572 523 600 580 577 570 559 575 561 607 591 545 601 625 571 639 647 612 617 655 690 756 864 895 957 1087 1232 1266 1418 1328 1237 1165 1057 1027 1008 944 929 1025 1043 1038 989 977 851 897 739 761 670 684 671 660 615 601 607 600 575 602 594 608 576 584 619 577 564 630 665 667 687 741 719 758 768 786 873 915 859 870 846 791 737 729 713 667 752 747 739 689 704 704 678 604 643 588 583 520 546 567 578 577 570 579 607 616 597 575 572 587 595 596 624 568 598 580 598 600 624 624 596 520 572 602 560 529 490 567 612 596 545 578 558 522 543 574 578 591 522 488 528 524 555 509 554 525 525 495 533 550 487 511 518 508 547 528 523 498 547 512 497 542 545 549 503 523 538 559 524 559 523 541 553 543 535 525 489 564 587 541 530 575 501 533 535 530 518 495 565 566 544 530 540 510 548 527 530 508 523 527 562 516 529 473 543 553 498 516 489 517 572 557 509 527 533 510 586 546 544 527 583 567 552 503 563 548 517 534 531 555 542 563 553 555 587 553 559 579 625 600 602 624 574 623 694 643 733 749 841 893 1000 974 1011 925 848 760 725 720 702 706 741 810 784 828 823 761 732 678 668 670 593 634 647 621 577 675 640 644 660 671 633 634 608 601 631 627 642 624 566 620 617 632 594 651 643 657 648 620 585 593 609 667 537 591 590 589 609 642 607 591 591 555 544 622 571 607 585 554 594 595 612 585 584 572 632 595 535 547 559 570 562 542 537 532 567 541 592 568 567 586 527 555 591 480 541 506 573 544 584 572 512 562 536 560 533 521 574 478 466 550 508 505 513 525 497 541 543 530 575 558 522 497 519 488 574 571 476 516 537 479 514 503 482 512 512 489 557 505 476 512 537 493 476 509 507 505 517 539 522 495 473 509 518 502 501 528 511 555 525 512 502 528 483 516 522 501 521 496 545 503 519 505 504 509 480 536 536 508 518 551 508 497 538 509 497 501 525 499 532 484 508 504 518 497 513 531 492 477 530 527 496 553 536 513 486 465 519 458 446 490 467 497 482 499 506 494 531 502 509 487 480 482 485 485 508 468 475 489 473 508 504 503 494 486 490 483 474 474 528 482 504 497 491 477 504 498 488 484 505 472 502 426 514 487 506 540 519 488 486 524 457 473 484 466 501 469 498 503 528 430 467 497 481 509 505 525 500 457 495 493 520 492 459 482 512 487 485 467 452 474 481 431 491 466 492 505 515 512 467 477 489 565 476 490 482 488 468 482 507 460 494 511 514 470 401 436 498 462 462 471 480 496 447 494 497 449 497 438 465 466 460 468 472 483 468 465 460 496 494 460 479 472 477 476 484 487 470 473 461 439 466 471 492 472 463 438 409 477 507 444 478 440 467 449 446 461 483 456 494 462 462 510 459 527 474 469 477 480 467 464 444 505 435 459 463 528 525 509 521 510 486 564 536 479 514 480 498 499 501 508 436 485 464 522 519 500 524 512 462 459 470 492 446 477 450 468 469 465 466 426 475 472 472 489 449 446 460 471 462 455 468 440 484 497 431 471 445 492 500 465 464 462 466 449 524 482 471 427 489 486 497 461 424 462 482 417 436 494 470 451 483 446 446 460 491 470 465 496 449 494 494 451 447 477 451 464 463 443 430 477 508 416 476 448 445 486 471 449 478 447 473 470 423 476 457 452 459 454 552 449 468 467 444 441 435 419 436 476 433 424 509 416 459 434 459 462 491 457 466 456 448 460 468 448 439 437 449 425 447 443 474 450 491 417 444 405 467 454 461 459 466 477 440 448 480 468 422 404 464 418 493 464 453 472 460 501 461 447 462 466 432 453 419 412 450 427 415 470 450 454 458 413 411 477 476 470 465 497 437 437 498 472 492 435 494 458 453 473 459 443 436 467 438 392 419 445 448 435 436 453 406 502 486 503 427 432 482 425 451 433 432 446 461 421 451 413 479 446 419 457 429 458 450 449 434 433 491 481 450 465 428 447 465 464 458 426 416 484 479 435 438 425 454 431 483 455 460 451 462 441 443 460 443 430 444 457 444 432 471 436 414 474 452 437 462 457 406 459 465 449 450 452 441 442 456 479 412 440 474 424 435 432 429 429 449 450 456 439 456 464 431 458 460 409 434 443 451 442 438 426 476 448 409 435 445 415 422 434 398 479 465 477 425 442 460 441 423 447 405 477 406 443 414 440 474 450 461 440 440 479 418 474 511 465 472 460 458 454 472 466 513 520 498 511 501 576 509 532 473 479 487 517 499 452 493 487 474 513 444 493 478 485 484 472 484 483 461 497 494 460 424 467 460 425 448 473 413 430 426 440 417 463 444 445 421 473 423 421 449 433 485 441 455 424 460 460 399 424 455 446 455 437 444 406 453 443 456 429 436 459 443 453 438 469 452 432 420 462 403 471 504 465 440 493 478 469 454 475 446 446 466 526 481 436 422 446 508 456 429 471 425 450 405 431 446 469 460 459 455 425 486 465 510 438 405 430 460 445 423 413 434 443 430 439 417 394 471 472 458 449 445 428 431 457 406 458 452 434 431 432 426 446 445 478 439 399 439 434 418 381 423 441 470 431 421 459 412 427 440 463 392 419 431 407 437 457 431 407 412 430 423 439 459 454 419 451 430 465 452 445 422 435 404 441 416 448 420 442 464 419 431 431 439 443 439 457 423 467 420 413 395 421 386 456 418 434 438 424 424 442 446 448 426 425 436 464 433 419 464 416 440 418 440 444 399 433 456 504 446 477 459 451 461 451 426 437 472 434 449 445 453 430 425 428 450 453 461 481 423 442 453 433 445 430 403 413 454 439 414 465 397 416 430 423 404 379 419 424 413 428 400 408 429 475 431 425 399 453 409 353 421 417 423 402 425 401 439 383 405 412 417 410 409 420 425 462 430 423 489 435 420 422 420 412 430 393 420 414 390 409 454 402 422 392 419 385 433 428 438 401 419 441 402 405 417 444 438 417 395 428 380 450 415 403 437 425 406 409 444 405 398 387 427 387 396 379 436 422 428 441 422 416 403 438 443 419 425 438 428 397 427 428 408 424 428 415 429 374 474 439 449 427 415 428 419 407 432 419 431 455 439 446 426 438 449 446 473 430 419 451 417 404 469 453 491 457 456 472 481 482 460 492 488 480 464 507 471 508 511 530 534 531 538 553 569 545 546 578 597 602 588 606 592 650 655 658 642 687 679 727 676 676 706 753 753 737 759 716 696 754 790 771 778 797 807 757 734 805 724 691 764 721 767 794 746 678 754 723 717 662 691 675 634 616 655 638 651 633 676 614 537 578 565 569 537 606 554 533 565 520 568 535 461 517 482 538 473 487 529 490 474 461 440 451 478 455 419 438 434 420 445 460 430 425 440 488 455 457 487 423 422 464 425 448 441 429 405 435 457 436 455 446 400 412 440 426 498 413 428 393 423 401 412 379 435 449 494 475 423 444 440 487 488 526 522 487 506 532 497 467 465 472 470 455 469 409 484 503 469 437 459 483 461 467 481 491 473 517 441 419 458 469 480 460 473 426 466 487 491 506 511 543 602 662 651 651 563 558 506 503 507 461 427 474 462 512 463 522 498 481 532 534 493 472 454 456 426 446 440 452 394 410 412 434 423 416 361 378 390 416 391 390 391 384 366 394 401 360 393 424 408 346 384 404 381 397 383 385 398 415 392 390 388 405 395 383 399 397 385 398 386 371 386 386 378 356 369 392 351 411 383 468 365 416 392 374 420 380 395 370 403 349 378 371 410 380 357 387 393 374 405 398 388 398 376 384 401 356 381 400 389 384 395 344 397 406 370 402 400 433 430 376 369 403 399 384 382 379 389 430 370 402 372 406 424 374 401 424 466 365 412 415 403 409 412 414 384 393 388 390 407 398 425 415 414 424 401 369 422 384 392 396 373 355 380 421 359 356 408 443 369 428 423 389 370 372 367 371 400 372 402 382 363 357 360 428 391 357 379 382 392 365 380 366 392 385 366 389 430 376 402 389 390 413 363 375 414 358 382 392 363 393 423 395 439 412 373 364 392 402 351 379 424 410 370 356 378 372 369 369 405 393 419 367 404 383 387 387 376 375 390 343 382 364 411 362 372 388 345 351 392 396 388 349 372 391 396 388 347 375 350 379 375 381 373 397 379 392 342 373 378 325 400 352 388 349 364 389 345 358 382 360 379 355 359 348 369 412 374 348 429 361 399 353 391 368 385 357 376 358 355 383 354 346 351 373 402 364 361 435 375 405 391 423 381 359 351 378 362 406 368 335 404 334 384 380 356 359 365 397 398 348 342 396 365 357 354 365 388 365 342 404 378 372 333 344 362 397 393 346 333 384 387 365 357 394 347 + + + + diff --git a/tests/data/nexus/NXtest2.nxdl.xml b/tests/data/nexus/NXtest2.nxdl.xml new file mode 100644 index 000000000..7b33b2165 --- /dev/null +++ b/tests/data/nexus/NXtest2.nxdl.xml @@ -0,0 +1,455 @@ + + + + + + + Characterization of a sample during a session on an electron microscope. + + + + + + + + Metadata and numerical data of the microscope and the lab in which it stands. + + + + + + Given name of the microscope at the hosting institution. This is an alias. + Examples could be NionHermes, Titan, JEOL, Gemini, etc. + + + + + Location of the lab or place where the instrument is installed. + Using GEOREF is preferred. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If the lens is described at least one of the fields + voltage, current, or value should be defined. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Description of the type of the detector. + + Electron microscopes have typically multiple detectors. + Different technologies are in use like CCD, scintillator, + direct electron, CMOS, or image plate to name but a few. + + + + Instrument-specific alias/name + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A container for storing a set of NXevent_data_em instances. + + + + + \ No newline at end of file diff --git a/tests/dataconverter/test_convert.py b/tests/dataconverter/test_convert.py index e0b17c5ad..a317c1470 100644 --- a/tests/dataconverter/test_convert.py +++ b/tests/dataconverter/test_convert.py @@ -112,7 +112,7 @@ def test_cli(caplog, cli_inputs): def test_links_and_virtual_datasets(tmp_path): """A test for the convert CLI to check whether a Dataset object is created, -when the template contains links.""" + when the template contains links.""" move_xarray_file_to_tmp(tmp_path) dirpath = os.path.join(os.path.dirname(__file__), diff --git a/tests/dataconverter/test_helpers.py b/tests/dataconverter/test_helpers.py index 07a396fd6..421f8ce9b 100644 --- a/tests/dataconverter/test_helpers.py +++ b/tests/dataconverter/test_helpers.py @@ -80,6 +80,29 @@ def listify_template(data_dict: Template): return listified_template +@pytest.mark.parametrize("input_data, expected_output", [ + ('2.4E-23', 2.4e-23), + ('28', 28), + ('45.98', 45.98), + ('test', 'test'), + (['59', '3.00005', '498E-36'], np.array([59.0, 3.00005, 4.98e-34])), + ('23 34 444 5000', np.array([23., 34., 444., 5000.])), + ('xrd experiment', 'xrd experiment'), + (None, None), +]) +def test_transform_to_intended_dt(input_data, expected_output): + """Transform to possible numerical method.""" + result = helpers.transform_to_intended_dt(input_data) + + # Use pytest.approx for comparing floating-point numbers + if isinstance(expected_output, np.ndarray): + np.testing.assert_allclose(result, expected_output, rtol=1e-3) + elif isinstance(expected_output, float): + assert result == pytest.approx(expected_output, rel=1e-5) + else: + assert result == expected_output + + @pytest.fixture(name="nxdl_root") def fixture_nxdl_root(): """pytest fixture to load the same NXDL file for all tests.""" diff --git a/tests/nexus/test_nexus.py b/tests/nexus/test_nexus.py index aec3feb17..d69b0fae2 100644 --- a/tests/nexus/test_nexus.py +++ b/tests/nexus/test_nexus.py @@ -79,9 +79,6 @@ def test_nexus(tmp_path): # # didn't work with filecmp library # log = os.path.join(local_dir, '../data/nexus_test_data/nexus_test.log') # ref = os.path.join(local_dir, '../data/nexus_test_data/Ref_nexus_test.log') - # print(filecmp.cmp(log, ref, shallow=False)) - - # print('Testing of nexus.py is SUCCESSFUL.') def test_get_node_at_nxdl_path(): @@ -102,7 +99,7 @@ def test_get_node_at_nxdl_path(): nxdl_file_path = os.path.join( local_dir, - "../../pynxtools/definitions/contributed_definitions/NXem.nxdl.xml" + "../data/nexus/NXtest2.nxdl.xml" ) elem = ET.parse(nxdl_file_path).getroot() node = nexus.get_node_at_nxdl_path(