From 0270cf6de2f4dc5a0946f0a9a4f290c6d2ec84f0 Mon Sep 17 00:00:00 2001 From: Rubel Date: Wed, 6 Sep 2023 23:32:58 +0200 Subject: [PATCH] General skelaton for xrd reader. --- .../dataconverter/readers/xrd/__init__.py | 15 +++ pynxtools/dataconverter/readers/xrd/reader.py | 100 ++++++++++++++++++ .../dataconverter/readers/xrd/xrd_helper.py | 15 +++ .../dataconverter/readers}/xrd/xrd_parser.py | 26 ++++- 4 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 pynxtools/dataconverter/readers/xrd/__init__.py create mode 100644 pynxtools/dataconverter/readers/xrd/reader.py create mode 100644 pynxtools/dataconverter/readers/xrd/xrd_helper.py rename {src/nomad_measurements => pynxtools/dataconverter/readers}/xrd/xrd_parser.py (87%) diff --git a/pynxtools/dataconverter/readers/xrd/__init__.py b/pynxtools/dataconverter/readers/xrd/__init__.py new file mode 100644 index 000000000..c9157de8d --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/__init__.py @@ -0,0 +1,15 @@ +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/pynxtools/dataconverter/readers/xrd/reader.py b/pynxtools/dataconverter/readers/xrd/reader.py new file mode 100644 index 000000000..4b27911aa --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/reader.py @@ -0,0 +1,100 @@ +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Tuple, Any, Dict, Union +import json +from pynxtools.dataconverter.readers.xrd.xrd_parser import parse_and_convert_file +from pynxtools.dataconverter.readers.utils import flatten_and_replace, FlattenSettings +import yaml +from pynxtools.dataconverter.template import Template +from pynxtools.dataconverter.readers.base.reader import BaseReader + + +CONVERT_DICT = { + 'Instrument': 'INSTRUMENT[instrument]', + 'Software': 'SOFTWARE[software]', + 'Hardware': 'Hardware[hardware]', + 'Analyser': 'ELECTRONANALYSER[electronanalyser]', + 'Beam': 'BEAM[beam]', + 'unit': '@units', + 'version': '@version', + 'Sample': 'SAMPLE[sample]', + 'User': 'USER[user]', + 'Data': 'DATA[data]', + 'Source': 'SOURCE[source]', + 'Environment': 'ENVIRONMENT[environment]', + 'Sample_bias': 'SAMPLE_BIAS[sample_bias]' +} + +REPLACE_NESTED: Dict[str, str] = {} + + +class STMReader(BaseReader): + """ Reader for XPS. + """ + + supported_nxdls = ["NXroot"] + + def read(self, + template: dict = None, + file_paths: Tuple[str] = None, + objects: Tuple[Any] = None): + """ + General read menthod to prepare the template. + """ + # has_sxm_file: bool = False + # sxm_file: str = "" + # has_dat_file: bool = False + # dat_file: str = "" + filled_template: Union[Dict, None] = Template() + # config_dict: Union[Dict[str, Any], None] = None + eln_dict: Union[Dict[str, Any], None] = None + config_dict: Dict = {} + + data_file: str = "" + for file in file_paths: + ext = file.rsplit('.', 1)[-1] + if ext == 'json': + with open(file, mode="r", encoding="utf-8") as fl_obj: + config_dict = json.load(fl_obj) + elif ext in ['yaml', 'yml']: + with open(file, mode="r", encoding="utf-8") as fl_obj: + eln_dict = flatten_and_replace( + FlattenSettings( + yaml.safe_load(fl_obj), + CONVERT_DICT, + REPLACE_NESTED + ) + ) + else: + xrd_dict = parse_and_convert_file(file) + # TODO combine nyaml, json, and xrd data here + + # Get rid of empty concept and cleaning up Template + for key, val in template.items(): + + if val is None: + del template[key] + else: + filled_template[key] = val + if not filled_template.keys(): + raise ValueError("Reader could not read anything! Check for input files and the" + " corresponding extention.") + return filled_template + + +READER = STMReader diff --git a/pynxtools/dataconverter/readers/xrd/xrd_helper.py b/pynxtools/dataconverter/readers/xrd/xrd_helper.py new file mode 100644 index 000000000..c9157de8d --- /dev/null +++ b/pynxtools/dataconverter/readers/xrd/xrd_helper.py @@ -0,0 +1,15 @@ +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/src/nomad_measurements/xrd/xrd_parser.py b/pynxtools/dataconverter/readers/xrd/xrd_parser.py similarity index 87% rename from src/nomad_measurements/xrd/xrd_parser.py rename to pynxtools/dataconverter/readers/xrd/xrd_parser.py index 75e6f72dc..1540f2f69 100644 --- a/src/nomad_measurements/xrd/xrd_parser.py +++ b/pynxtools/dataconverter/readers/xrd/xrd_parser.py @@ -1,3 +1,23 @@ +""" +XRD file parser collection. +TODO: Extend the module level doc. +""" +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re # for regular expressions import os # for file path operations import xml.etree.ElementTree as ET # for XML parsing @@ -14,7 +34,7 @@ def __init__(self, file_path): def read_file(self): '''Reads the content of a file from the given file path. - + Returns: str: The content of the file. ''' @@ -45,7 +65,7 @@ def parse_metadata(self): root = ET.fromstring(content) ns_version = root.tag.split("}")[0].strip("{") - ns = {'xrd': ns_version} + ns = {'xrd': ns_version} xrd_measurement = root.find("xrd:xrdMeasurement", ns) @@ -103,7 +123,7 @@ def __init__(self, file_path): def identify_format(self): '''Identifies the format of a given file. - + Returns: str: The file extension of the file. '''