From 3a0cb9d3fb9b9209c84acccf16a67684634e548b Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Sun, 31 Dec 2023 20:13:34 +0100 Subject: [PATCH 01/15] Starting the refactoring of the apm reader, added support for ATO, ENV, FAU/Erlangen's pyccapt calibration and ranging, and FAU/Erlangen's Matlab Atom probe Toolbox fig ranging definitions, currently deactivated ELN and configuration parsing also in the io_case initial check of the apm reader, these functionalities need to use the updated and refactored ifes_apt_tc_data_modeling library, which however as of 2023/12/31 has not yet been placed on pypi (but committed into the repo, that's why this commit works with a local py3.11.5 where the sitepackage ifes_apt_tc_data_modeling was modified manually to avoid frequent publishing of dysfunctional ifes version and test them here first, next steps: i) run the reader against all 150+ test cases, ii) fix bugs of these examples, iii) move changes to ifes library and publish that on pypi, iv) update pyproject toml, v) reactivate ELN and config parsing, vi) update apm definitions to use the refactored apm base classes, vii) integrate ruff changes, viii) commit on pynxtools master --- pynxtools/dataconverter/readers/apm/reader.py | 13 +- .../readers/apm/utils/apm_define_io_cases.py | 67 ++++--- .../utils/apm_load_deployment_specifics.py | 16 +- .../readers/apm/utils/apm_load_generic_eln.py | 18 +- .../readers/apm/utils/apm_load_ranging.py | 180 +++++++++--------- .../apm/utils/apm_load_reconstruction.py | 154 +++++++++++---- .../apm/utils/apm_parse_composition_table.py | 1 - 7 files changed, 266 insertions(+), 183 deletions(-) diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 2e946257f..9208e3977 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -17,30 +17,23 @@ # """Generic parser for loading atom probe microscopy data into NXapm.""" -# pylint: disable=no-member +# pylint: disable=no-member,too-few-public-methods from typing import Tuple, Any from pynxtools.dataconverter.readers.base.reader import BaseReader - from pynxtools.dataconverter.readers.apm.utils.apm_define_io_cases \ import ApmUseCaseSelector - from pynxtools.dataconverter.readers.apm.utils.apm_load_deployment_specifics \ import NxApmNomadOasisConfigurationParser - from pynxtools.dataconverter.readers.apm.utils.apm_load_generic_eln \ import NxApmNomadOasisElnSchemaParser - from pynxtools.dataconverter.readers.apm.utils.apm_load_reconstruction \ import ApmReconstructionParser - from pynxtools.dataconverter.readers.apm.utils.apm_load_ranging \ import ApmRangingDefinitionsParser - from pynxtools.dataconverter.readers.apm.utils.apm_create_nx_default_plots \ import apm_default_plot_generator - from pynxtools.dataconverter.readers.apm.utils.apm_generate_synthetic_data \ import ApmCreateExampleData @@ -69,8 +62,6 @@ class ApmReader(BaseReader): """ - # pylint: disable=too-few-public-methods - # Whitelist for the NXDLs that the reader supports and can process supported_nxdls = ["NXapm"] @@ -98,6 +89,7 @@ def read(self, assert case.is_valid is True, \ "Such a combination of input-file(s, if any) is not supported !" + """ print("Parse (meta)data coming from an ELN...") if len(case.eln) == 1: nx_apm_eln = NxApmNomadOasisElnSchemaParser(case.eln[0], entry_id) @@ -111,6 +103,7 @@ def read(self, nx_apm_cfg = NxApmNomadOasisConfigurationParser(case.cfg[0], entry_id) nx_apm_cfg.report(template) # having and or using a deployment-specific configuration is optional + """ print("Parse (numerical) data and metadata from ranging definitions file...") if len(case.reconstruction) == 1: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py index 26a73a1e9..6fd7986e7 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py @@ -20,6 +20,9 @@ # pylint: disable=no-member,duplicate-code from typing import Tuple, Dict, List +VALID_FILE_NAME_SUFFIX_RECON = [".apt", ".pos", ".epos", ".ato", ".csv", ".h5"] +VALID_FILE_NAME_SUFFIX_RANGE = [".rng", ".rrng", ".env", ".fig.txt", "range_.h5"] +VALID_FILE_NAME_SUFFIX_CONFIG = [".yaml", ".yml"] class ApmUseCaseSelector: # pylint: disable=too-few-public-methods @@ -41,50 +44,58 @@ def __init__(self, file_paths: Tuple[str] = None): self.reconstruction: List[str] = [] self.ranging: List[str] = [] self.is_valid = False - self.supported_mime_types = [ - "pos", "epos", "apt", "rrng", "rng", "txt", "yaml", "yml"] - for mime_type in self.supported_mime_types: - self.case[mime_type] = [] - - self.sort_files_by_mime_type(file_paths) + self.supported_file_name_suffixes = VALID_FILE_NAME_SUFFIX_RECON \ + + VALID_FILE_NAME_SUFFIX_RANGE + VALID_FILE_NAME_SUFFIX_CONFIG + print(f"self.supported_file_name_suffixes: {self.supported_file_name_suffixes}") + self.sort_files_by_file_name_suffix(file_paths) self.check_validity_of_file_combinations() - def sort_files_by_mime_type(self, file_paths: Tuple[str] = None): - """Sort all input-files based on their mimetype to prepare validity check.""" - for file_name in file_paths: - index = file_name.lower().rfind(".") - if index >= 0: - suffix = file_name.lower()[index + 1::] - if suffix in self.supported_mime_types: - if file_name not in self.case[suffix]: - self.case[suffix].append(file_name) + def sort_files_by_file_name_suffix(self, file_paths: Tuple[str] = None): + """Sort all input-files based on their name suffix to prepare validity check.""" + for suffix in self.supported_file_name_suffixes: + self.case[suffix] = [] + for fpath in file_paths: + for suffix in self.supported_file_name_suffixes: + if suffix not in [".h5", "range_.h5"]: + if (fpath.lower().endswith(suffix)) and (fpath not in self.case[suffix]): + self.case[suffix].append(fpath) + else: + if fpath.lower().endswith("range_.h5") is True: + self.case["range_.h5"].append(fpath) + elif fpath.lower().endswith(".h5") is True: + self.case[".h5"].append(fpath) + else: + continue + # HDF5 files need special treatment, this already shows that magic numbers + # should better have been used or signatures to avoid having to have as + # complicated content checks as we had to implement e.g. for the em reader def check_validity_of_file_combinations(self): """Check if this combination of types of files is supported.""" - recon_input = 0 # reconstruction relevant file e.g. POS, ePOS, APT - range_input = 0 # ranging definition file, e.g. RNG, RRNG + recon_input = 0 # reconstruction relevant file e.g. POS, ePOS, APT, ATO, CSV + range_input = 0 # ranging definition file, e.g. RNG, RRNG, ENV, FIG.TXT other_input = 0 # generic ELN or OASIS-specific configurations - for mime_type, value in self.case.items(): - if mime_type in ["pos", "epos", "apt"]: + for suffix, value in self.case.items(): + if suffix in VALID_FILE_NAME_SUFFIX_RECON: recon_input += len(value) - elif mime_type in ["rrng", "rng", "txt"]: + elif suffix in VALID_FILE_NAME_SUFFIX_RANGE: range_input += len(value) - elif mime_type in ["yaml", "yml"]: + elif suffix in VALID_FILE_NAME_SUFFIX_CONFIG: other_input += len(value) else: continue - if (recon_input == 1) and (range_input == 1) and (1 <= other_input <= 2): + if (recon_input == 1) and (range_input == 1): # and (1 <= other_input <= 2): self.is_valid = True self.reconstruction: List[str] = [] self.ranging: List[str] = [] - for mime_type in ["pos", "epos", "apt"]: - self.reconstruction += self.case[mime_type] - for mime_type in ["rrng", "rng", "txt"]: - self.ranging += self.case[mime_type] + for suffix in VALID_FILE_NAME_SUFFIX_RECON: + self.reconstruction += self.case[suffix] + for suffix in VALID_FILE_NAME_SUFFIX_RANGE: + self.ranging += self.case[suffix] yml: List[str] = [] - for mime_type in ["yaml", "yml"]: - yml += self.case[mime_type] + for suffix in VALID_FILE_NAME_SUFFIX_CONFIG: + yml += self.case[suffix] for entry in yml: if entry.endswith(".oasis.specific.yaml") \ or entry.endswith(".oasis.specific.yml"): diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py index 87dc05950..22f26c6f6 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py @@ -20,12 +20,10 @@ # pylint: disable=no-member import flatdict as fd - import yaml from pynxtools.dataconverter.readers.apm.map_concepts.apm_deployment_specifics_to_nx_map \ import NxApmDeploymentSpecificInput - from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ import apply_modifier, variadic_path_to_specific_path @@ -33,17 +31,17 @@ class NxApmNomadOasisConfigurationParser: # pylint: disable=too-few-public-methods """Parse deployment specific configuration.""" - def __init__(self, file_name: str, entry_id: int): - print(f"Extracting data from deployment specific configuration file: {file_name}") - if (file_name.rsplit('/', 1)[-1].endswith(".oasis.specific.yaml") - or file_name.endswith(".oasis.specific.yml")) and entry_id > 0: + def __init__(self, file_path: str, entry_id: int): + print(f"Extracting data from deployment specific configuration file: {file_path}") + if (file_path.rsplit('/', 1)[-1].endswith(".oasis.specific.yaml") + or file_path.endswith(".oasis.specific.yml")) and entry_id > 0: self.entry_id = entry_id - self.file_name = file_name - with open(self.file_name, "r", encoding="utf-8") as stream: + self.file_path = file_path + with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") else: self.entry_id = 1 - self.file_name = "" + self.file_path = "" self.yml = {} def report(self, template: dict) -> dict: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py index ed36eec23..869d3d967 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py @@ -20,17 +20,13 @@ # pylint: disable=no-member,duplicate-code,too-many-nested-blocks import flatdict as fd - import yaml from ase.data import chemical_symbols - from pynxtools.dataconverter.readers.apm.map_concepts.apm_eln_to_nx_map \ import NxApmElnInput, NxUserFromListOfDict - from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ import variadic_path_to_specific_path, apply_modifier - from pynxtools.dataconverter.readers.apm.utils.apm_parse_composition_table \ import parse_composition_table @@ -53,17 +49,17 @@ class NxApmNomadOasisElnSchemaParser: # pylint: disable=too-few-public-methods during the verification of the template dictionary. """ - def __init__(self, file_name: str, entry_id: int): - print(f"Extracting data from ELN file: {file_name}") - if (file_name.rsplit('/', 1)[-1].startswith("eln_data") - or file_name.startswith("eln_data")) and entry_id > 0: + def __init__(self, file_path: str, entry_id: int): + print(f"Extracting data from ELN file: {file_path}") + if (file_path.rsplit('/', 1)[-1].startswith("eln_data") + or file_path.startswith("eln_data")) and entry_id > 0: self.entry_id = entry_id - self.file_name = file_name - with open(self.file_name, "r", encoding="utf-8") as stream: + self.file_path = file_path + with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") else: self.entry_id = 1 - self.file_name = "" + self.file_path = "" self.yml = {} def parse_sample_composition(self, template: dict) -> dict: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py index c2e3ca91f..49979eca0 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py @@ -15,33 +15,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # + """Wrapping multiple parsers for vendor files with ranging definition files.""" # pylint: disable=no-member from typing import Dict, Any - import numpy as np from ase.data import chemical_symbols -# ase encodes the zeroth entry as the unknown element X to have -# atom_numbers all starting with 1 up to len(chemical_symbols) - 1 - from ifes_apt_tc_data_modeling.utils.utils \ import create_isotope_vector, isotope_vector_to_nuclid_list, \ isotope_vector_to_human_readable_name - from ifes_apt_tc_data_modeling.utils.definitions \ - import MAX_NUMBER_OF_ATOMS_PER_ION, MQ_EPSILON - + import MAX_NUMBER_OF_ATOMS_PER_ION, MQ_EPSILON, MAX_NUMBER_OF_ION_SPECIES +from ifes_apt_tc_data_modeling.env.env_reader import ReadEnvFileFormat +from ifes_apt_tc_data_modeling.fig.fig_reader import ReadFigTxtFileFormat +from ifes_apt_tc_data_modeling.pyccapt.pyccapt_reader import ReadPyccaptRangingFileFormat from ifes_apt_tc_data_modeling.rng.rng_reader import ReadRngFileFormat - from ifes_apt_tc_data_modeling.rrng.rrng_reader import ReadRrngFileFormat - -from ifes_apt_tc_data_modeling.fig.fig_reader import ReadFigTxtFileFormat - from pynxtools.dataconverter.readers.apm.utils.apm_versioning \ import NX_APM_EXEC_NAME, NX_APM_EXEC_VERSION +from pynxtools.dataconverter.readers.apm.utils.apm_define_io_cases \ + import VALID_FILE_NAME_SUFFIX_RANGE def add_unknown_iontype(template: dict, entry_id: int) -> dict: @@ -59,7 +55,6 @@ def add_unknown_iontype(template: dict, entry_id: int) -> dict: nuclid_list = isotope_vector_to_nuclid_list(ivec) template[f"{trg}nuclid_list"] = np.asarray(nuclid_list, np.uint16) template[f"{trg}name"] = isotope_vector_to_human_readable_name(ivec, 0) - return template @@ -69,16 +64,15 @@ def add_standardize_molecular_ions(ion_lst: list, template: dict, entry_id: int) trg = f"/ENTRY[entry{entry_id}]/atom_probe/ranging/peak_identification/" for ion in ion_lst: path = f"{trg}ION[ion{ion_id}]/" - template[f"{path}isotope_vector"] = np.reshape( - np.asarray(ion.isotope_vector.typed_value, np.uint16), + np.asarray(ion.isotope_vector.values, np.uint16), (1, MAX_NUMBER_OF_ATOMS_PER_ION)) - template[f"{path}charge_state"] = np.int8(ion.charge_state.typed_value) + template[f"{path}charge_state"] = np.int8(ion.charge_state.values) template[f"{path}mass_to_charge_range"] \ - = np.array(ion.ranges.typed_value, np.float32) + = np.array(ion.ranges.values, np.float32) template[f"{path}mass_to_charge_range/@units"] = "Da" # ion.ranges.unit - template[f"{path}nuclid_list"] = ion.nuclid_list.typed_value - template[f"{path}name"] = ion.name.typed_value + template[f"{path}nuclid_list"] = ion.nuclid_list.values + template[f"{path}name"] = ion.name.values path = f"{trg}ION[ion{ion_id}]/charge_state_model/" template[f"{path}min_abundance"] \ @@ -107,82 +101,98 @@ def add_standardize_molecular_ions(ion_lst: list, template: dict, entry_id: int) = {"compress": np.array(ion.charge_state_model["min_half_life_vector"], np.float64), "strength": 1} template[f"{path}min_half_life_vector/@units"] = "s" - ion_id += 1 trg = f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" template[f"{trg}number_of_ion_types"] = np.uint32(ion_id) - return template +# modify the template to take into account ranging +# ranging is currently not resolved recursively because +# ranging(NXprocess) is a group which has a minOccurs=1, \er +# maxOccurs="unbounded" set of possible named +# NXion members, same case for more than one operator +# ion indices are on the interval [0, 256) -def extract_data_from_rng_file(file_name: str, template: dict, entry_id: int) -> dict: - """Add those required information which an RNG file has.""" - # modify the template to take into account ranging - # ranging is currently not resolved recursively because - # ranging(NXprocess) is a group which has a minOccurs=1, \ - # maxOccurs="unbounded" set of possible named - # NXion members, same case for more than one operator - print(f"Extracting data from RNG file: {file_name}") - rangefile = ReadRngFileFormat(file_name) - - # ion indices are on the interval [0, 256) - assert len(rangefile.rng["molecular_ions"]) <= np.iinfo(np.uint8).max + 1, \ - "Current implementation does not support more than 256 ion types" - add_standardize_molecular_ions( - rangefile.rng["molecular_ions"], template, entry_id) +def extract_data_from_env_file(file_path: str, template: dict, entry_id: int) -> dict: + """Add those required information which a ENV file has.""" + print(f"Extracting data from ENV file: {file_path}") + rangefile = ReadEnvFileFormat(file_path) + if len(rangefile.env["molecular_ions"]) > np.iinfo(np.uint8).max + 1: + raise ValueError(f"Current implementation does not support " + f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") + add_standardize_molecular_ions( + rangefile.env["molecular_ions"], template, entry_id) return template -def extract_data_from_rrng_file(file_name: str, template: dict, entry_id) -> dict: - """Add those required information which an RRNG file has.""" - # modify the template to take into account ranging - # ranging is currently not resolved recursively because - # ranging(NXprocess) is a group which has a minOccurs=1, \er - # maxOccurs="unbounded" set of possible named - # NXion members, same case for more than one operator - print(f"Extracting data from RRNG file: {file_name}") - rangefile = ReadRrngFileFormat(file_name) - - # ion indices are on the interval [0, 256) - assert len(rangefile.rrng["molecular_ions"]) <= np.iinfo(np.uint8).max + 1, \ - "Current implementation does not support more than 256 ion types" +def extract_data_from_fig_txt_file(file_path: str, template: dict, entry_id: int) -> dict: + """Add those required information which a FIG.TXT file has.""" + print(f"Extracting data from FIG.TXT file: {file_path}") + rangefile = ReadFigTxtFileFormat(file_path) + if len(rangefile.fig["molecular_ions"]) > np.iinfo(np.uint8).max + 1: + raise ValueError(f"Current implementation does not support " + f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") add_standardize_molecular_ions( - rangefile.rrng["molecular_ions"], template, entry_id) - + rangefile.fig["molecular_ions"], template, entry_id) return template -def extract_data_from_fig_txt_file(file_name: str, template: dict, entry_id) -> dict: - """Add those required information which an transcoded Matlab figure TXT file has.""" - print(f"Extracting data from FIG.TXT file: {file_name}") - rangefile = ReadFigTxtFileFormat(file_name) +def extract_data_from_pyccapt_file(file_path: str, template: dict, entry_id: int) -> dict: + """Add those required information which a pyccapt/ranging HDF5 file has.""" + print(f"Extracting data from pyccapt/ranging HDF5 file: {file_path}") + rangefile = ReadPyccaptRangingFileFormat(file_path) + if len(rangefile.rng["molecular_ions"]) > np.iinfo(np.uint8).max + 1: + raise ValueError(f"Current implementation does not support " + f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") - # ion indices are on the interval [0, 256) - assert len(rangefile.fig["molecular_ions"]) <= np.iinfo(np.uint8).max + 1, \ - "Current implementation does not support more than 256 ion types" + add_standardize_molecular_ions( + rangefile.rng["molecular_ions"], template, entry_id) + return template + + +def extract_data_from_rng_file(file_path: str, template: dict, entry_id: int) -> dict: + """Add those required information which an RNG file has.""" + print(f"Extracting data from RNG file: {file_path}") + rangefile = ReadRngFileFormat(file_path) + if len(rangefile.rng["molecular_ions"]) > np.iinfo(np.uint8).max + 1: + raise ValueError(f"Current implementation does not support " + f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") add_standardize_molecular_ions( - rangefile.fig["molecular_ions"], template, entry_id) + rangefile.rng["molecular_ions"], template, entry_id) + return template + +def extract_data_from_rrng_file(file_path: str, template: dict, entry_id) -> dict: + """Add those required information which an RRNG file has.""" + print(f"Extracting data from RRNG file: {file_path}") + rangefile = ReadRrngFileFormat(file_path) + if len(rangefile.rrng["molecular_ions"]) > np.iinfo(np.uint8).max + 1: + raise ValueError(f"Current implementation does not support more " + f"than {MAX_NUMBER_OF_ION_SPECIES} ion types") + + add_standardize_molecular_ions( + rangefile.rrng["molecular_ions"], template, entry_id) return template class ApmRangingDefinitionsParser: # pylint: disable=too-few-public-methods """Wrapper for multiple parsers for vendor specific files.""" - def __init__(self, file_name: str, entry_id: int): - self.meta: Dict[str, Any] = {} - self.meta["file_format"] = "none" - self.meta["file_name"] = file_name - self.meta["entry_id"] = entry_id - index = file_name.lower().rfind(".") - if index >= 0: - mime_type = file_name.lower()[index + 1::] - self.meta["file_format"] = mime_type + def __init__(self, file_path: str, entry_id: int): + self.meta: Dict[str, Any] = {"file_format": None, + "file_path": file_path, + "entry_id": entry_id} + for suffix in VALID_FILE_NAME_SUFFIX_RANGE: + if file_path.lower().endswith(suffix) is True: + self.meta["file_format"] = suffix + break + if self.meta["file_format"] is None: + raise ValueError(f"{file_path} is not a supported ranging definitions file!") def update_atom_types_ranging_definitions_based(self, template: dict) -> dict: """Update the atom_types list in the specimen based on ranging defs.""" @@ -225,7 +235,8 @@ def report(self, template: dict) -> dict: """ # resolve the next two program references more informatively trg = f"/ENTRY[entry{self.meta['entry_id']}]/atom_probe/ranging/" - template[f"{trg}maximum_number_of_atoms_per_molecular_ion"] = np.uint32(32) + template[f"{trg}maximum_number_of_atoms_per_molecular_ion"] \ + = np.uint32(MAX_NUMBER_OF_ATOMS_PER_ION) # mass_to_charge_distribution will be filled by default plot # background_quantification data are not available in RNG/RRNG files @@ -238,22 +249,22 @@ def report(self, template: dict) -> dict: add_unknown_iontype(template, self.meta["entry_id"]) - if self.meta["file_name"] != "" and self.meta["file_format"] != "none": - if self.meta["file_format"] == "rng": - extract_data_from_rng_file( - self.meta["file_name"], - template, - self.meta["entry_id"]) - elif self.meta["file_format"] == "rrng": - extract_data_from_rrng_file( - self.meta["file_name"], - template, - self.meta["entry_id"]) - elif self.meta["file_format"] == "txt": - extract_data_from_fig_txt_file( - self.meta["file_name"], - template, - self.meta["entry_id"]) + if self.meta["file_path"] != "" and self.meta["file_format"] is not None: + if self.meta["file_format"] == ".env": + extract_data_from_env_file(self.meta["file_path"], + template, self.meta["entry_id"]) + elif self.meta["file_format"] == ".fig.txt": + extract_data_from_fig_txt_file(self.meta["file_path"], + template, self.meta["entry_id"]) + elif self.meta["file_format"] == "range_.h5": + extract_data_from_pyccapt_file(self.meta["file_path"], + template, self.meta["entry_id"]) + elif self.meta["file_format"] == ".rng": + extract_data_from_rng_file(self.meta["file_path"], + template, self.meta["entry_id"]) + elif self.meta["file_format"] == ".rrng": + extract_data_from_rrng_file(self.meta["file_path"], + template, self.meta["entry_id"]) else: trg = f"/ENTRY[entry{self.meta['entry_id']}]/atom_probe/ranging/" template[f"{trg}number_of_ion_types"] = 1 @@ -262,5 +273,4 @@ def report(self, template: dict) -> dict: template[f"{trg}number_of_ion_types"] = 1 self.update_atom_types_ranging_definitions_based(template) - return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py index 36e6eb20f..43b75dec2 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py @@ -19,52 +19,57 @@ # pylint: disable=no-member +from typing import Dict, Any import numpy as np from ifes_apt_tc_data_modeling.apt.apt6_reader import ReadAptFileFormat - -from ifes_apt_tc_data_modeling.pos.pos_reader import ReadPosFileFormat - from ifes_apt_tc_data_modeling.epos.epos_reader import ReadEposFileFormat +from ifes_apt_tc_data_modeling.pos.pos_reader import ReadPosFileFormat +from ifes_apt_tc_data_modeling.ato.ato_reader import ReadAtoFileFormat +from ifes_apt_tc_data_modeling.csv.csv_reader import ReadCsvFileFormat +from ifes_apt_tc_data_modeling.pyccapt.pyccapt_reader \ + import ReadPyccaptCalibrationFileFormat +from pynxtools.dataconverter.readers.apm.utils.apm_define_io_cases \ + import VALID_FILE_NAME_SUFFIX_RECON -def extract_data_from_pos_file(file_name: str, prefix: str, template: dict) -> dict: +def extract_data_from_pos_file(file_path: str, prefix: str, template: dict) -> dict: """Add those required information which a POS file has.""" - print(f"Extracting data from POS file: {file_name}") - posfile = ReadPosFileFormat(file_name) + print(f"Extracting data from POS file: {file_path}") + posfile = ReadPosFileFormat(file_path) trg = f"{prefix}reconstruction/" xyz = posfile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.typed_value, np.float32), "strength": 1} + = {"compress": np.array(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = posfile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.typed_value, np.float32), "strength": 1} + = {"compress": np.array(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z return template -def extract_data_from_epos_file(file_name: str, prefix: str, template: dict) -> dict: +def extract_data_from_epos_file(file_path: str, prefix: str, template: dict) -> dict: """Add those required information which an ePOS file has.""" - print(f"Extracting data from EPOS file: {file_name}") - eposfile = ReadEposFileFormat(file_name) + print(f"Extracting data from EPOS file: {file_path}") + eposfile = ReadEposFileFormat(file_path) trg = f"{prefix}reconstruction/" xyz = eposfile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.typed_value, np.float32), "strength": 1} + = {"compress": np.array(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = eposfile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.typed_value, np.float32), "strength": 1} + = {"compress": np.array(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z @@ -119,29 +124,27 @@ def extract_data_from_epos_file(file_name: str, prefix: str, template: dict) -> # = ions_per_pulse.unit # del ions_per_pulse # --> - return template -def extract_data_from_apt_file(file_name: str, prefix: str, template: dict) -> dict: +def extract_data_from_apt_file(file_path: str, prefix: str, template: dict) -> dict: """Add those required information which a APT file has.""" - print(f"Extracting data from APT file: {file_name}") - aptfile = ReadAptFileFormat(file_name) + print(f"Extracting data from APT file: {file_path}") + aptfile = ReadAptFileFormat(file_path) trg = f"{prefix}reconstruction/" xyz = aptfile.get_named_quantity("Position") template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.typed_value, np.float32), "strength": 1} + = {"compress": np.array(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = aptfile.get_named_quantity("Mass") template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.typed_value, np.float32), "strength": 1} + = {"compress": np.array(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z - # all less explored optional branches in an APT6 file can also already # be accessed via the aptfile.get_named_quantity function # but it needs to be checked if this returns reasonable values @@ -150,18 +153,82 @@ def extract_data_from_apt_file(file_name: str, prefix: str, template: dict) -> d return template +def extract_data_from_ato_file(file_path: str, prefix: str, template: dict) -> dict: + """Add those required information which a ATO file has.""" + print(f"Extracting data from ATO file: {file_path}") + atofile = ReadAtoFileFormat(file_path) + + trg = f"{prefix}reconstruction/" + xyz = atofile.get_reconstructed_positions() + template[f"{trg}reconstructed_positions"] \ + = {"compress": np.array(xyz.values, np.float32), "strength": 1} + template[f"{trg}reconstructed_positions/@units"] = xyz.unit + del xyz + + trg = f"{prefix}mass_to_charge_conversion/" + m_z = atofile.get_mass_to_charge_state_ratio() + template[f"{trg}mass_to_charge"] \ + = {"compress": np.array(m_z.values, np.float32), "strength": 1} + template[f"{trg}mass_to_charge/@units"] = m_z.unit + del m_z + return template + + +def extract_data_from_csv_file(file_path: str, prefix: str, template: dict) -> dict: + """Add those required information which a CSV file has.""" + print(f"Extracting data from CSV file: {file_path}") + csvfile = ReadCsvFileFormat(file_path) + + trg = f"{prefix}reconstruction/" + xyz = csvfile.get_reconstructed_positions() + template[f"{trg}reconstructed_positions"] \ + = {"compress": np.array(xyz.values, np.float32), "strength": 1} + template[f"{trg}reconstructed_positions/@units"] = xyz.unit + del xyz + + trg = f"{prefix}mass_to_charge_conversion/" + m_z = csvfile.get_mass_to_charge_state_ratio() + template[f"{trg}mass_to_charge"] \ + = {"compress": np.array(m_z.values, np.float32), "strength": 1} + template[f"{trg}mass_to_charge/@units"] = m_z.unit + del m_z + return template + + +def extract_data_from_pyc_file(file_path: str, prefix: str, template: dict) -> dict: + """Add those required information which a pyccapt/calibration HDF5 file has.""" + print(f"Extracting data from pyccapt/calibration HDF5 file: {file_path}") + pycfile = ReadPyccaptCalibrationFileFormat(file_path) + + trg = f"{prefix}reconstruction/" + xyz = pycfile.get_reconstructed_positions() + template[f"{trg}reconstructed_positions"] \ + = {"compress": np.array(xyz.values, np.float32), "strength": 1} + template[f"{trg}reconstructed_positions/@units"] = xyz.unit + del xyz + + trg = f"{prefix}mass_to_charge_conversion/" + m_z = pycfile.get_mass_to_charge_state_ratio() + template[f"{trg}mass_to_charge"] \ + = {"compress": np.array(m_z.values, np.float32), "strength": 1} + template[f"{trg}mass_to_charge/@units"] = m_z.unit + del m_z + return template + + class ApmReconstructionParser: # pylint: disable=too-few-public-methods """Wrapper for multiple parsers for vendor specific files.""" - def __init__(self, file_name: str, entry_id: int): - self.file_format = "none" - self.file_name = file_name - index = file_name.lower().rfind(".") - if index >= 0: - mime_type = file_name.lower()[index + 1::] - if mime_type in ["pos", "epos", "apt"]: - self.file_format = mime_type - self.entry_id = entry_id + def __init__(self, file_path: str, entry_id: int): + self.meta: Dict[str, Any] = {"file_format": None, + "file_path": file_path, + "entry_id": entry_id} + for suffix in VALID_FILE_NAME_SUFFIX_RECON: + if file_path.lower().endswith(suffix) is True: + self.meta["file_format"] = suffix + break + if self.meta["file_format"] is None: + raise ValueError(f"{file_path} is not a supported reconstruction file!") def report(self, template: dict) -> dict: """Copy data from self into template the appdef instance. @@ -169,15 +236,24 @@ def report(self, template: dict) -> dict: Paths in template are prefixed by prefix and have to be compliant with the application definition. """ - prfx = f"/ENTRY[entry{self.entry_id}]/atom_probe/" - if self.file_name != "" and self.file_format != "none": - if self.file_format == "pos": - extract_data_from_pos_file( - self.file_name, prfx, template) - if self.file_format == "epos": - extract_data_from_epos_file( - self.file_name, prfx, template) - if self.file_format == "apt": + prfx = f"/ENTRY[entry{self.meta['entry_id']}]/atom_probe/" + if self.meta["file_path"] != "" and self.meta["file_format"] is not None: + if self.meta["file_format"] == ".apt": extract_data_from_apt_file( - self.file_name, prfx, template) + self.meta["file_path"], prfx, template) + if self.meta["file_format"] == ".epos": + extract_data_from_epos_file( + self.meta["file_path"], prfx, template) + if self.meta["file_format"] == ".pos": + extract_data_from_pos_file( + self.meta["file_path"], prfx, template) + if self.meta["file_format"] == ".ato": + extract_data_from_ato_file( + self.meta["file_path"], prfx, template) + if self.meta["file_format"] == ".csv": + extract_data_from_csv_file( + self.meta["file_path"], prfx, template) + if self.meta["file_format"] == ".h5": + extract_data_from_pyc_file( + self.meta["file_path"], prfx, template) return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_parse_composition_table.py b/pynxtools/dataconverter/readers/apm/utils/apm_parse_composition_table.py index cf8f2bc56..4db3614c6 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_parse_composition_table.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_parse_composition_table.py @@ -20,7 +20,6 @@ # pylint: disable=no-member,too-many-branches import re - import numpy as np from ase.data import chemical_symbols From 60f52c75e661687c47a47a03ce95540257200876 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 2 Jan 2024 02:12:29 +0100 Subject: [PATCH 02/15] First round of fixes detected and solved when working through the large 176 dataset example case --- pynxtools/dataconverter/readers/apm/reader.py | 2 +- .../readers/apm/utils/apm_load_ranging.py | 73 ++++++++++++------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 9208e3977..44033615b 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -63,7 +63,7 @@ class ApmReader(BaseReader): """ # Whitelist for the NXDLs that the reader supports and can process - supported_nxdls = ["NXapm"] + supported_nxdls = ["NXapm", "NXroot"] def read(self, template: dict = None, diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py index 49979eca0..af38ff5e9 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py @@ -74,33 +74,52 @@ def add_standardize_molecular_ions(ion_lst: list, template: dict, entry_id: int) template[f"{path}nuclid_list"] = ion.nuclid_list.values template[f"{path}name"] = ion.name.values - path = f"{trg}ION[ion{ion_id}]/charge_state_model/" - template[f"{path}min_abundance"] \ - = np.float64(ion.charge_state_model["min_abundance"]) - template[f"{path}min_abundance_product"] \ - = np.float64(ion.charge_state_model["min_abundance_product"]) - template[f"{path}min_half_life"] \ - = np.float64(ion.charge_state_model["min_half_life"]) - template[f"{path}min_half_life/@units"] = "s" - template[f"{path}sacrifice_isotopic_uniqueness"] \ - = np.uint8(ion.charge_state_model["sacrifice_isotopic_uniqueness"]) - template[f"{path}isotope_matrix"] \ - = {"compress": np.array(ion.charge_state_model["isotope_matrix"], - np.uint16), "strength": 1} - template[f"{path}charge_state_vector"] \ - = {"compress": np.array(ion.charge_state_model["charge_state_vector"], - np.int8), "strength": 1} - template[f"{path}mass_vector"] \ - = {"compress": np.array(ion.charge_state_model["mass_vector"], - np.float64), "strength": 1} - template[f"{path}mass_vector/@units"] = "u" - template[f"{path}natural_abundance_product_vector"] \ - = {"compress": np.array(ion.charge_state_model["nat_abun_prod_vector"], - np.float64), "strength": 1} - template[f"{path}min_half_life_vector"] \ - = {"compress": np.array(ion.charge_state_model["min_half_life_vector"], - np.float64), "strength": 1} - template[f"{path}min_half_life_vector/@units"] = "s" + if ion.charge_state_model["n_cand"] > 0: + path = f"{trg}ION[ion{ion_id}]/charge_state_model/" + template[f"{path}min_abundance"] \ + = np.float64(ion.charge_state_model["min_abundance"]) + template[f"{path}min_abundance_product"] \ + = np.float64(ion.charge_state_model["min_abundance_product"]) + template[f"{path}min_half_life"] \ + = np.float64(ion.charge_state_model["min_half_life"]) + template[f"{path}min_half_life/@units"] = "s" + template[f"{path}sacrifice_isotopic_uniqueness"] \ + = np.uint8(ion.charge_state_model["sacrifice_isotopic_uniqueness"]) + if ion.charge_state_model["n_cand"] == 1: + template[f"{path}isotope_matrix"] \ + = np.asarray(ion.charge_state_model["isotope_matrix"], np.uint16) + template[f"{path}charge_state_vector"] \ + = np.int8(ion.charge_state_model["charge_state_vector"]) + template[f"{path}mass_vector"] \ + = np.float64(ion.charge_state_model["mass_vector"]) + template[f"{path}mass_vector/@units"] \ + = "u" + template[f"{path}natural_abundance_product_vector"] \ + = np.float64(ion.charge_state_model["nat_abun_prod_vector"]) + template[f"{path}min_half_life_vector"] \ + = np.float64(ion.charge_state_model["min_half_life_vector"]) + template[f"{path}min_half_life_vector/@units"] \ + = "s" + else: + template[f"{path}isotope_matrix"] \ + = {"compress": np.asarray(ion.charge_state_model["isotope_matrix"], + np.uint16), "strength": 1} + template[f"{path}charge_state_vector"] \ + = {"compress": np.asarray(ion.charge_state_model["charge_state_vector"], + np.int8), "strength": 1} + template[f"{path}mass_vector"] \ + = {"compress": np.asarray(ion.charge_state_model["mass_vector"], + np.float64), "strength": 1} + template[f"{path}mass_vector/@units"] \ + = "u" + template[f"{path}natural_abundance_product_vector"] \ + = {"compress": np.asarray(ion.charge_state_model["nat_abun_prod_vector"], + np.float64), "strength": 1} + template[f"{path}min_half_life_vector"] \ + = {"compress": np.asarray(ion.charge_state_model["min_half_life_vector"], + np.float64), "strength": 1} + template[f"{path}min_half_life_vector/@units"] \ + = "s" ion_id += 1 trg = f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" From f15e1cfbff02b563f1c1827e162d641e3cd73708 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 2 Jan 2024 16:51:21 +0100 Subject: [PATCH 03/15] Fixed steps i), ii), all 176 examples processed successfully, PSE coverage 58 elements, 5.9 billions ions in total, 77 projects from research of all major groups worldwide, next steps: iii) move to ifes, vi) nxdef refactor, v) eln, cfg parsing, RUN ALL and check proper annotation, styling, linting ruff --- pynxtools/dataconverter/readers/apm/reader.py | 2 ++ .../readers/apm/utils/apm_define_io_cases.py | 27 ++++++++++++------- .../readers/apm/utils/apm_load_ranging.py | 22 +++++++-------- .../apm/utils/apm_load_reconstruction.py | 2 +- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 44033615b..2b72228f0 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -119,8 +119,10 @@ def read(self, print("No input-file defined for ranging definitions!") return {} + """ print("Create NeXus default plottable data...") apm_default_plot_generator(template, n_entries) + """ # print("Reporting state of template before passing to HDF5 writing...") # for keyword in template.keys(): diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py index 6fd7986e7..36c494718 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py @@ -59,11 +59,14 @@ def sort_files_by_file_name_suffix(self, file_paths: Tuple[str] = None): if suffix not in [".h5", "range_.h5"]: if (fpath.lower().endswith(suffix)) and (fpath not in self.case[suffix]): self.case[suffix].append(fpath) + break else: - if fpath.lower().endswith("range_.h5") is True: + if fpath.lower().endswith("range_.h5") == True: self.case["range_.h5"].append(fpath) - elif fpath.lower().endswith(".h5") is True: + break + elif fpath.lower().endswith(".h5") == True: self.case[".h5"].append(fpath) + break else: continue # HDF5 files need special treatment, this already shows that magic numbers @@ -76,14 +79,20 @@ def check_validity_of_file_combinations(self): range_input = 0 # ranging definition file, e.g. RNG, RRNG, ENV, FIG.TXT other_input = 0 # generic ELN or OASIS-specific configurations for suffix, value in self.case.items(): - if suffix in VALID_FILE_NAME_SUFFIX_RECON: - recon_input += len(value) - elif suffix in VALID_FILE_NAME_SUFFIX_RANGE: - range_input += len(value) - elif suffix in VALID_FILE_NAME_SUFFIX_CONFIG: - other_input += len(value) + if suffix not in [".h5", "range_.h5"]: + if suffix in VALID_FILE_NAME_SUFFIX_RECON: + recon_input += len(value) + elif suffix in VALID_FILE_NAME_SUFFIX_RANGE: + range_input += len(value) + elif suffix in VALID_FILE_NAME_SUFFIX_CONFIG: + other_input += len(value) + else: + continue else: - continue + if suffix == "range_.h5": + range_input += len(value) + if suffix == ".h5": + recon_input += len(value) if (recon_input == 1) and (range_input == 1): # and (1 <= other_input <= 2): self.is_valid = True diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py index af38ff5e9..4681d2eaf 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py @@ -39,6 +39,11 @@ from pynxtools.dataconverter.readers.apm.utils.apm_define_io_cases \ import VALID_FILE_NAME_SUFFIX_RANGE +WARNING_TOO_MANY_DEFINITIONS \ + = f"WARNING::Range file contains more than {MAX_NUMBER_OF_ION_SPECIES} " \ + f"WARNING::definitions! This is often a signature of duplicates or " \ + f"WARNING::contradicting definitions." + def add_unknown_iontype(template: dict, entry_id: int) -> dict: """Add default unknown iontype.""" @@ -139,8 +144,7 @@ def extract_data_from_env_file(file_path: str, template: dict, entry_id: int) -> print(f"Extracting data from ENV file: {file_path}") rangefile = ReadEnvFileFormat(file_path) if len(rangefile.env["molecular_ions"]) > np.iinfo(np.uint8).max + 1: - raise ValueError(f"Current implementation does not support " - f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") + print(WARNING_TOO_MANY_DEFINITIONS) add_standardize_molecular_ions( rangefile.env["molecular_ions"], template, entry_id) @@ -152,8 +156,7 @@ def extract_data_from_fig_txt_file(file_path: str, template: dict, entry_id: int print(f"Extracting data from FIG.TXT file: {file_path}") rangefile = ReadFigTxtFileFormat(file_path) if len(rangefile.fig["molecular_ions"]) > np.iinfo(np.uint8).max + 1: - raise ValueError(f"Current implementation does not support " - f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") + print(WARNING_TOO_MANY_DEFINITIONS) add_standardize_molecular_ions( rangefile.fig["molecular_ions"], template, entry_id) @@ -165,8 +168,7 @@ def extract_data_from_pyccapt_file(file_path: str, template: dict, entry_id: int print(f"Extracting data from pyccapt/ranging HDF5 file: {file_path}") rangefile = ReadPyccaptRangingFileFormat(file_path) if len(rangefile.rng["molecular_ions"]) > np.iinfo(np.uint8).max + 1: - raise ValueError(f"Current implementation does not support " - f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") + print(WARNING_TOO_MANY_DEFINITIONS) add_standardize_molecular_ions( rangefile.rng["molecular_ions"], template, entry_id) @@ -178,8 +180,7 @@ def extract_data_from_rng_file(file_path: str, template: dict, entry_id: int) -> print(f"Extracting data from RNG file: {file_path}") rangefile = ReadRngFileFormat(file_path) if len(rangefile.rng["molecular_ions"]) > np.iinfo(np.uint8).max + 1: - raise ValueError(f"Current implementation does not support " - f"more than {MAX_NUMBER_OF_ION_SPECIES} ion types") + print(WARNING_TOO_MANY_DEFINITIONS) add_standardize_molecular_ions( rangefile.rng["molecular_ions"], template, entry_id) @@ -191,8 +192,7 @@ def extract_data_from_rrng_file(file_path: str, template: dict, entry_id) -> dic print(f"Extracting data from RRNG file: {file_path}") rangefile = ReadRrngFileFormat(file_path) if len(rangefile.rrng["molecular_ions"]) > np.iinfo(np.uint8).max + 1: - raise ValueError(f"Current implementation does not support more " - f"than {MAX_NUMBER_OF_ION_SPECIES} ion types") + print(WARNING_TOO_MANY_DEFINITIONS) add_standardize_molecular_ions( rangefile.rrng["molecular_ions"], template, entry_id) @@ -207,7 +207,7 @@ def __init__(self, file_path: str, entry_id: int): "file_path": file_path, "entry_id": entry_id} for suffix in VALID_FILE_NAME_SUFFIX_RANGE: - if file_path.lower().endswith(suffix) is True: + if file_path.lower().endswith(suffix) == True: self.meta["file_format"] = suffix break if self.meta["file_format"] is None: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py index 43b75dec2..319045102 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py @@ -224,7 +224,7 @@ def __init__(self, file_path: str, entry_id: int): "file_path": file_path, "entry_id": entry_id} for suffix in VALID_FILE_NAME_SUFFIX_RECON: - if file_path.lower().endswith(suffix) is True: + if file_path.lower().endswith(suffix) == True: self.meta["file_format"] = suffix break if self.meta["file_format"] is None: From d8be28b8497ea6765fb3babecf39cfd954d12313 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 2 Jan 2024 17:00:23 +0100 Subject: [PATCH 04/15] Deactivate synthesizer because that needs refactoring and because we have now plenty of real world examples to show and run these tools with --- pynxtools/dataconverter/readers/apm/reader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 2b72228f0..97ceabccc 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -75,6 +75,11 @@ def read(self, n_entries = 1 entry_id = 1 if len(file_paths) == 1: + """ + # TODO::better make this an option rather than hijack and demand a + # specifically named file to trigger the synthesizer + # the synthesize functionality is currently deactivated, we have enough + # example datasets and the synthesizer is in need for a refactoring. if file_paths[0].startswith("synthesize"): synthesis_id = int(file_paths[0].replace("synthesize", "")) print(f"synthesis_id {synthesis_id}") @@ -83,6 +88,7 @@ def read(self, print("Create one synthetic entry in one NeXus file...") synthetic = ApmCreateExampleData(synthesis_id) synthetic.synthesize(template) + """ else: # eln_data, and ideally recon and ranging definitions from technology partner file print("Parse ELN and technology partner file(s)...") case = ApmUseCaseSelector(file_paths) From d9350ec46a98e630bc7f0e851f1b71d35db13239 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Mon, 8 Jan 2024 11:23:06 +0100 Subject: [PATCH 05/15] Switched nxdefs from working fairmat 615ff37cbafd2ca017fb61c119c0f5c0cf052a34 to not tested developmental apm_refactoring --- pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynxtools/definitions b/pynxtools/definitions index 999837671..34fc49b46 160000 --- a/pynxtools/definitions +++ b/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 999837671373b962fed932829becd42acb7482f6 +Subproject commit 34fc49b46221a94dcce242c56b976759542c2ced From 5ec47ff06bed01837da38836c9f97fc368e63438 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Mon, 8 Jan 2024 16:11:30 +0100 Subject: [PATCH 06/15] Reactivated NOMAD OASIS deployment specific parsing --- debug/apm.oasis.specific.yaml | 1 + debug/batch_queue.03.sh | 177 ++++++++++++++++++ debug/batch_queue.04.sh | 4 + debug/eln_data_apm.yaml | 118 ++++++++++++ ...o_nx_map.py => apm_oasis_cfg_to_nx_map.py} | 19 +- pynxtools/dataconverter/readers/apm/reader.py | 11 +- .../apm/utils/apm_create_nx_default_plots.py | 20 +- .../readers/apm/utils/apm_define_io_cases.py | 3 +- .../apm/utils/apm_generate_synthetic_data.py | 4 +- .../utils/apm_load_deployment_specifics.py | 30 ++- .../readers/apm/utils/apm_load_ranging.py | 6 +- .../apm/utils/apm_load_reconstruction.py | 24 +-- pyproject.toml | 3 + 13 files changed, 372 insertions(+), 48 deletions(-) create mode 100644 debug/apm.oasis.specific.yaml create mode 100755 debug/batch_queue.03.sh create mode 100755 debug/batch_queue.04.sh create mode 100644 debug/eln_data_apm.yaml rename pynxtools/dataconverter/readers/apm/map_concepts/{apm_deployment_specifics_to_nx_map.py => apm_oasis_cfg_to_nx_map.py} (77%) diff --git a/debug/apm.oasis.specific.yaml b/debug/apm.oasis.specific.yaml new file mode 100644 index 000000000..e75a3eb92 --- /dev/null +++ b/debug/apm.oasis.specific.yaml @@ -0,0 +1 @@ +operation_mode: apt diff --git a/debug/batch_queue.03.sh b/debug/batch_queue.03.sh new file mode 100755 index 000000000..d29dd6070 --- /dev/null +++ b/debug/batch_queue.03.sh @@ -0,0 +1,177 @@ +#!/bin/bash + +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_60075-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/FWHM-range.rrng" --output=debug.aus_sydney_bilal_0.nxs 1>stdout.debug.aus_sydney_bilal_0.txt 2>stderr.debug.aus_sydney_bilal_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R04_23087-v02.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R4_23087.RRNG" --output=debug.aus_sydney_breen_0.nxs 1>stdout.debug.aus_sydney_breen_0.txt 2>stderr.debug.aus_sydney_breen_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_53705-v03.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_53705-v03.rng" --output=debug.aus_sydney_breen_1.nxs 1>stdout.debug.aus_sydney_breen_1.txt 2>stderr.debug.aus_sydney_breen_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_55377-v02.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_55377-v02.rng" --output=debug.aus_sydney_breen_2.nxs 1>stdout.debug.aus_sydney_breen_2.txt 2>stderr.debug.aus_sydney_breen_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_58152-v02.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_58152-v02.RRNG" --output=debug.aus_sydney_breen_3.nxs 1>stdout.debug.aus_sydney_breen_3.txt 2>stderr.debug.aus_sydney_breen_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_59377-v01.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_59377.rng" --output=debug.aus_sydney_breen_4.nxs 1>stdout.debug.aus_sydney_breen_4.txt 2>stderr.debug.aus_sydney_breen_4.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_60310-v02.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_60310-v02.rng" --output=debug.aus_sydney_breen_5.nxs 1>stdout.debug.aus_sydney_breen_5.txt 2>stderr.debug.aus_sydney_breen_5.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/DN5.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/DN5.rrng" --output=debug.aus_sydney_cairney_0.nxs 1>stdout.debug.aus_sydney_cairney_0.txt 2>stderr.debug.aus_sydney_cairney_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/A718 GB R04_11350-V01.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R04_11350-v01.rng.fig.txt" --output=debug.aus_sydney_felfer_0.nxs 1>stdout.debug.aus_sydney_felfer_0.txt 2>stderr.debug.aus_sydney_felfer_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Al-Cu-Sn.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Al-Cu-Sn.rng" --output=debug.aus_sydney_fontaine_alcusn_0.nxs 1>stdout.debug.aus_sydney_fontaine_alcusn_0.txt 2>stderr.debug.aus_sydney_fontaine_alcusn_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Human-tooth-enamel-A.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Human-tooth-enamel.rrng" --output=debug.aus_sydney_fontaine_tooth_0.nxs 1>stdout.debug.aus_sydney_fontaine_tooth_0.txt 2>stderr.debug.aus_sydney_fontaine_tooth_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Human-tooth-enamel-B.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Human-tooth-enamel.rrng" --output=debug.aus_sydney_fontaine_tooth_1.nxs 1>stdout.debug.aus_sydney_fontaine_tooth_1.txt 2>stderr.debug.aus_sydney_fontaine_tooth_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/A.rrng" --output=debug.aus_sydney_lafontaine_0.nxs 1>stdout.debug.aus_sydney_lafontaine_0.txt 2>stderr.debug.aus_sydney_lafontaine_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/B.rrng" --output=debug.aus_sydney_lafontaine_1.nxs 1>stdout.debug.aus_sydney_lafontaine_1.txt 2>stderr.debug.aus_sydney_lafontaine_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/C.rrng" --output=debug.aus_sydney_lafontaine_2.nxs 1>stdout.debug.aus_sydney_lafontaine_2.txt 2>stderr.debug.aus_sydney_lafontaine_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/D.rrng" --output=debug.aus_sydney_lafontaine_3.nxs 1>stdout.debug.aus_sydney_lafontaine_3.txt 2>stderr.debug.aus_sydney_lafontaine_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/E.rrng" --output=debug.aus_sydney_lafontaine_4.nxs 1>stdout.debug.aus_sydney_lafontaine_4.txt 2>stderr.debug.aus_sydney_lafontaine_4.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/F.rrng" --output=debug.aus_sydney_lafontaine_5.nxs 1>stdout.debug.aus_sydney_lafontaine_5.txt 2>stderr.debug.aus_sydney_lafontaine_5.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/H.rrng" --output=debug.aus_sydney_lafontaine_6.nxs 1>stdout.debug.aus_sydney_lafontaine_6.txt 2>stderr.debug.aus_sydney_lafontaine_6.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/I.rrng" --output=debug.aus_sydney_lafontaine_7.nxs 1>stdout.debug.aus_sydney_lafontaine_7.txt 2>stderr.debug.aus_sydney_lafontaine_7.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ranging.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/G- Peak Ranges.rrng" --output=debug.aus_sydney_lafontaine_8.nxs 1>stdout.debug.aus_sydney_lafontaine_8.txt 2>stderr.debug.aus_sydney_lafontaine_8.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-C.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-C.rrng" --output=debug.aus_sydney_lafontaine_9.nxs 1>stdout.debug.aus_sydney_lafontaine_9.txt 2>stderr.debug.aus_sydney_lafontaine_9.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-H.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-H.rrng" --output=debug.aus_sydney_lafontaine_10.nxs 1>stdout.debug.aus_sydney_lafontaine_10.txt 2>stderr.debug.aus_sydney_lafontaine_10.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-A_1.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-A_1.RNG" --output=debug.aus_sydney_lafontaine_11.nxs 1>stdout.debug.aus_sydney_lafontaine_11.txt 2>stderr.debug.aus_sydney_lafontaine_11.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-B_1.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/81-B_1.rrng" --output=debug.aus_sydney_lafontaine_12.nxs 1>stdout.debug.aus_sydney_lafontaine_12.txt 2>stderr.debug.aus_sydney_lafontaine_12.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-B_2.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-B_2.rrng" --output=debug.aus_sydney_lafontaine_13.nxs 1>stdout.debug.aus_sydney_lafontaine_13.txt 2>stderr.debug.aus_sydney_lafontaine_13.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87D_1.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87D_1.rng" --output=debug.aus_sydney_lafontaine_14.nxs 1>stdout.debug.aus_sydney_lafontaine_14.txt 2>stderr.debug.aus_sydney_lafontaine_14.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87D_2.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87D_2.rrng" --output=debug.aus_sydney_lafontaine_15.nxs 1>stdout.debug.aus_sydney_lafontaine_15.txt 2>stderr.debug.aus_sydney_lafontaine_15.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87D_3.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87D_3.rrng" --output=debug.aus_sydney_lafontaine_16.nxs 1>stdout.debug.aus_sydney_lafontaine_16.txt 2>stderr.debug.aus_sydney_lafontaine_16.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E_1.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E_1.rrng" --output=debug.aus_sydney_lafontaine_17.nxs 1>stdout.debug.aus_sydney_lafontaine_17.txt 2>stderr.debug.aus_sydney_lafontaine_17.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E_2.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E_2.rrng" --output=debug.aus_sydney_lafontaine_18.nxs 1>stdout.debug.aus_sydney_lafontaine_18.txt 2>stderr.debug.aus_sydney_lafontaine_18.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F-1.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F.rrng" --output=debug.aus_sydney_lafontaine_19.nxs 1>stdout.debug.aus_sydney_lafontaine_19.txt 2>stderr.debug.aus_sydney_lafontaine_19.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F_2.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F.rrng" --output=debug.aus_sydney_lafontaine_20.nxs 1>stdout.debug.aus_sydney_lafontaine_20.txt 2>stderr.debug.aus_sydney_lafontaine_20.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-G_1.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-G_1 Peak Ranges.rrng" --output=debug.aus_sydney_lafontaine_21.nxs 1>stdout.debug.aus_sydney_lafontaine_21.txt 2>stderr.debug.aus_sydney_lafontaine_21.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-G_2.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-G_2 Peak Ranges.rrng" --output=debug.aus_sydney_lafontaine_22.nxs 1>stdout.debug.aus_sydney_lafontaine_22.txt 2>stderr.debug.aus_sydney_lafontaine_22.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-I-1.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-I-1.rrng" --output=debug.aus_sydney_lafontaine_23.nxs 1>stdout.debug.aus_sydney_lafontaine_23.txt 2>stderr.debug.aus_sydney_lafontaine_23.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-I-2.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-I-2.rrng" --output=debug.aus_sydney_lafontaine_24.nxs 1>stdout.debug.aus_sydney_lafontaine_24.txt 2>stderr.debug.aus_sydney_lafontaine_24.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-A'.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-A'.rrng" --output=debug.aus_sydney_lafontaine_25.nxs 1>stdout.debug.aus_sydney_lafontaine_25.txt 2>stderr.debug.aus_sydney_lafontaine_25.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-B'.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-B'.rrng" --output=debug.aus_sydney_lafontaine_26.nxs 1>stdout.debug.aus_sydney_lafontaine_26.txt 2>stderr.debug.aus_sydney_lafontaine_26.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E'.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E'.rng" --output=debug.aus_sydney_lafontaine_27.nxs 1>stdout.debug.aus_sydney_lafontaine_27.txt 2>stderr.debug.aus_sydney_lafontaine_27.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F'.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F'.RNG" --output=debug.aus_sydney_lafontaine_28.nxs 1>stdout.debug.aus_sydney_lafontaine_28.txt 2>stderr.debug.aus_sydney_lafontaine_28.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-G'.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-G'-Peak Ranges.rrng" --output=debug.aus_sydney_lafontaine_29.nxs 1>stdout.debug.aus_sydney_lafontaine_29.txt 2>stderr.debug.aus_sydney_lafontaine_29.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E'.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-E'.rrng" --output=debug.aus_sydney_lafontaine_30.nxs 1>stdout.debug.aus_sydney_lafontaine_30.txt 2>stderr.debug.aus_sydney_lafontaine_30.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F'.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-F'.rrng" --output=debug.aus_sydney_lafontaine_31.nxs 1>stdout.debug.aus_sydney_lafontaine_31.txt 2>stderr.debug.aus_sydney_lafontaine_31.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-H'.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-H'.rrng" --output=debug.aus_sydney_lafontaine_32.nxs 1>stdout.debug.aus_sydney_lafontaine_32.txt 2>stderr.debug.aus_sydney_lafontaine_32.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R87-I'.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/87-I'.rrng" --output=debug.aus_sydney_lafontaine_33.nxs 1>stdout.debug.aus_sydney_lafontaine_33.txt 2>stderr.debug.aus_sydney_lafontaine_33.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R04_22071.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R04_22071.RRNG" --output=debug.aus_sydney_rielli_primig_0.nxs 1>stdout.debug.aus_sydney_rielli_primig_0.txt 2>stderr.debug.aus_sydney_rielli_primig_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_08680-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_08680.rrng" --output=debug.aut_leoben_leitner_0.nxs 1>stdout.debug.aut_leoben_leitner_0.txt 2>stderr.debug.aut_leoben_leitner_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_07575-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_07575-v02.rrng" --output=debug.aut_leoben_mendez_martin_0.nxs 1>stdout.debug.aut_leoben_mendez_martin_0.txt 2>stderr.debug.aut_leoben_mendez_martin_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/finfet_system.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/finfet.rrng" --output=debug.bel_leuven_imec_0.nxs 1>stdout.debug.bel_leuven_imec_0.txt 2>stderr.debug.bel_leuven_imec_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/multilayer_system.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/multilayer.rrng" --output=debug.bel_leuven_imec_1.nxs 1>stdout.debug.bel_leuven_imec_1.txt 2>stderr.debug.bel_leuven_imec_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R47_00667-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/RRNG-FeMnC.RRNG" --output=debug.can_hamilton_morisset_0.nxs 1>stdout.debug.can_hamilton_morisset_0.txt 2>stderr.debug.can_hamilton_morisset_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26731-v03.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26731.rrng" --output=debug.gbr_oxford_chen_0.nxs 1>stdout.debug.gbr_oxford_chen_0.txt 2>stderr.debug.gbr_oxford_chen_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25438-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25438.rrng" --output=debug.gbr_oxford_chen_1.nxs 1>stdout.debug.gbr_oxford_chen_1.txt 2>stderr.debug.gbr_oxford_chen_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26717-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26717.rrng" --output=debug.gbr_oxford_chen_2.nxs 1>stdout.debug.gbr_oxford_chen_2.txt 2>stderr.debug.gbr_oxford_chen_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25556-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25556.rrng" --output=debug.gbr_oxford_chen_3.nxs 1>stdout.debug.gbr_oxford_chen_3.txt 2>stderr.debug.gbr_oxford_chen_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05028-v07.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05028-v07.rrng" --output=debug.gbr_oxford_chen_4.nxs 1>stdout.debug.gbr_oxford_chen_4.txt 2>stderr.debug.gbr_oxford_chen_4.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25977-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25977.rrng" --output=debug.gbr_oxford_chen_5.nxs 1>stdout.debug.gbr_oxford_chen_5.txt 2>stderr.debug.gbr_oxford_chen_5.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05025-v04.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05025.rrng" --output=debug.gbr_oxford_chen_6.nxs 1>stdout.debug.gbr_oxford_chen_6.txt 2>stderr.debug.gbr_oxford_chen_6.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26702-v03.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26702.rrng" --output=debug.gbr_oxford_chen_7.nxs 1>stdout.debug.gbr_oxford_chen_7.txt 2>stderr.debug.gbr_oxford_chen_7.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26697-v03.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26697.rrng" --output=debug.gbr_oxford_chen_8.nxs 1>stdout.debug.gbr_oxford_chen_8.txt 2>stderr.debug.gbr_oxford_chen_8.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25949-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_25949.rrng" --output=debug.gbr_oxford_chen_9.nxs 1>stdout.debug.gbr_oxford_chen_9.txt 2>stderr.debug.gbr_oxford_chen_9.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_09806-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_09806-v02.rrng" --output=debug.gbr_oxford_davis_193.nxs 1>stdout.debug.gbr_oxford_davis_193.txt 2>stderr.debug.gbr_oxford_davis_193.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_09837-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_09837-v01.rrng" --output=debug.gbr_oxford_davis_285.nxs 1>stdout.debug.gbr_oxford_davis_285.txt 2>stderr.debug.gbr_oxford_davis_285.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_22266-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_22266.RRNG" --output=debug.gbr_oxford_dffit_0.nxs 1>stdout.debug.gbr_oxford_dffit_0.txt 2>stderr.debug.gbr_oxford_dffit_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_17798-v01-crop.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_17798-v01.RRNG" --output=debug.gbr_oxford_dffit_1.nxs 1>stdout.debug.gbr_oxford_dffit_1.txt 2>stderr.debug.gbr_oxford_dffit_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_24453-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_24436_R14_24453.rrng" --output=debug.gbr_oxford_gascell_0.nxs 1>stdout.debug.gbr_oxford_gascell_0.txt 2>stderr.debug.gbr_oxford_gascell_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_24546-v12.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_24546_Mg880Ge_PostO2.rrng" --output=debug.gbr_oxford_gascell_1.nxs 1>stdout.debug.gbr_oxford_gascell_1.txt 2>stderr.debug.gbr_oxford_gascell_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_24902-v07.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_24902-v07_PostH2O.rrng" --output=debug.gbr_oxford_gascell_2.nxs 1>stdout.debug.gbr_oxford_gascell_2.txt 2>stderr.debug.gbr_oxford_gascell_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26444-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26444-v01.RRNG" --output=debug.gbr_oxford_gascell_7.nxs 1>stdout.debug.gbr_oxford_gascell_7.txt 2>stderr.debug.gbr_oxford_gascell_7.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26445-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_26445-v01.rrng" --output=debug.gbr_oxford_gascell_8.nxs 1>stdout.debug.gbr_oxford_gascell_8.txt 2>stderr.debug.gbr_oxford_gascell_8.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05028-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05028-v02.rrng" --output=debug.gbr_oxford_haley_1.nxs 1>stdout.debug.gbr_oxford_haley_1.txt 2>stderr.debug.gbr_oxford_haley_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_21676-v04.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_21676 range.rrng" --output=debug.gbr_oxford_haley_2.nxs 1>stdout.debug.gbr_oxford_haley_2.txt 2>stderr.debug.gbr_oxford_haley_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05068-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R34_05068.rrng" --output=debug.gbr_oxford_haley_3.nxs 1>stdout.debug.gbr_oxford_haley_3.txt 2>stderr.debug.gbr_oxford_haley_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_19943-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R14_19943.rrng" --output=debug.gbr_oxford_haley_4.nxs 1>stdout.debug.gbr_oxford_haley_4.txt 2>stderr.debug.gbr_oxford_haley_4.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_07490-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_07490-v02.cluster.rrng" --output=debug.gbr_oxford_jenkins_0.nxs 1>stdout.debug.gbr_oxford_jenkins_0.txt 2>stderr.debug.gbr_oxford_jenkins_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/23314.POS" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5083_23314_MJ_decomp_Dcopy_YS.RRNG" --output=debug.gbr_oxford_shi_0.nxs 1>stdout.debug.gbr_oxford_shi_0.txt 2>stderr.debug.gbr_oxford_shi_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R68_06299-v05.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/MgGdZn.rrng" --output=debug.ger_aachen_mouhib_0.nxs 1>stdout.debug.ger_aachen_mouhib_0.txt 2>stderr.debug.ger_aachen_mouhib_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R68_06337-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/MgGdZn.rrng" --output=debug.ger_aachen_mouhib_1.nxs 1>stdout.debug.ger_aachen_mouhib_1.txt 2>stderr.debug.ger_aachen_mouhib_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R68_06796-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/MgGdZn.rrng" --output=debug.ger_aachen_mouhib_2.nxs 1>stdout.debug.ger_aachen_mouhib_2.txt 2>stderr.debug.ger_aachen_mouhib_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R68_06994-v04.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/MgGd.RRNG" --output=debug.ger_aachen_mouhib_3.nxs 1>stdout.debug.ger_aachen_mouhib_3.txt 2>stderr.debug.ger_aachen_mouhib_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_600C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_600C.rrng" --output=debug.ger_aachen_saelker_0.nxs 1>stdout.debug.ger_aachen_saelker_0.txt 2>stderr.debug.ger_aachen_saelker_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_flake_900C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_flake_900C.rrng" --output=debug.ger_aachen_saelker_1.nxs 1>stdout.debug.ger_aachen_saelker_1.txt 2>stderr.debug.ger_aachen_saelker_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_700C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_700C.rrng" --output=debug.ger_aachen_saelker_2.nxs 1>stdout.debug.ger_aachen_saelker_2.txt 2>stderr.debug.ger_aachen_saelker_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_asdep.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_asdep.rrng" --output=debug.ger_aachen_saelker_3.nxs 1>stdout.debug.ger_aachen_saelker_3.txt 2>stderr.debug.ger_aachen_saelker_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/TiAlN_film_cross-section_1200C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/TiAlN_film_cross-section_1200C.rrng" --output=debug.ger_aachen_saelker_4.nxs 1>stdout.debug.ger_aachen_saelker_4.txt 2>stderr.debug.ger_aachen_saelker_4.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_800C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_800C.rrng" --output=debug.ger_aachen_saelker_5.nxs 1>stdout.debug.ger_aachen_saelker_5.txt 2>stderr.debug.ger_aachen_saelker_5.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_flake_1100C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_flake_1100C.rrng" --output=debug.ger_aachen_saelker_6.nxs 1>stdout.debug.ger_aachen_saelker_6.txt 2>stderr.debug.ger_aachen_saelker_6.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/TiAlN_film_cross-section_850C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/TiAlN_film_cross-section_850C.rrng" --output=debug.ger_aachen_saelker_7.nxs 1>stdout.debug.ger_aachen_saelker_7.txt 2>stderr.debug.ger_aachen_saelker_7.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_800C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_plan-view_800C.rrng" --output=debug.ger_aachen_saelker_8.nxs 1>stdout.debug.ger_aachen_saelker_8.txt 2>stderr.debug.ger_aachen_saelker_8.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_600C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_600C.rrng" --output=debug.ger_aachen_saelker_9.nxs 1>stdout.debug.ger_aachen_saelker_9.txt 2>stderr.debug.ger_aachen_saelker_9.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/TiAlN_film_cross-section_asdep.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/TiAlN_film_cross-section_asdep.rrng" --output=debug.ger_aachen_saelker_10.nxs 1>stdout.debug.ger_aachen_saelker_10.txt 2>stderr.debug.ger_aachen_saelker_10.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_900C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_900C.rrng" --output=debug.ger_aachen_saelker_11.nxs 1>stdout.debug.ger_aachen_saelker_11.txt 2>stderr.debug.ger_aachen_saelker_11.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_asdep.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_asdep.rrng" --output=debug.ger_aachen_saelker_12.nxs 1>stdout.debug.ger_aachen_saelker_12.txt 2>stderr.debug.ger_aachen_saelker_12.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_700C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_700C.rrng" --output=debug.ger_aachen_saelker_13.nxs 1>stdout.debug.ger_aachen_saelker_13.txt 2>stderr.debug.ger_aachen_saelker_13.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_flake_asdep.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_flake_asdep.rrng" --output=debug.ger_aachen_saelker_14.nxs 1>stdout.debug.ger_aachen_saelker_14.txt 2>stderr.debug.ger_aachen_saelker_14.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_1000C.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/VAlN_film_cross-section_1000C.rrng" --output=debug.ger_aachen_saelker_15.nxs 1>stdout.debug.ger_aachen_saelker_15.txt 2>stderr.debug.ger_aachen_saelker_15.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/8fc2f3c8-213d-46d6-84da-255cb3dee017.apt" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R68_06755 Top Level ROI.RRNG" --output=debug.ger_aachen_seehaus_0.nxs 1>stdout.debug.ger_aachen_seehaus_0.txt 2>stderr.debug.ger_aachen_seehaus_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/d3ecc995-071d-4374-be52-3d489b5a0f6a.apt" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R68_07086 Top Level ROI.RRNG" --output=debug.ger_aachen_seehaus_1.nxs 1>stdout.debug.ger_aachen_seehaus_1.txt 2>stderr.debug.ger_aachen_seehaus_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5006_29110_Top_Level_ROI.apt" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5006_29110_Top_Level_ROI.RRNG" --output=debug.ger_duesseldorf_antonov_0.nxs 1>stdout.debug.ger_duesseldorf_antonov_0.txt 2>stderr.debug.ger_duesseldorf_antonov_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R87_02935-v03.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R87_02935-v03.rrng" --output=debug.ger_duesseldorf_antonov_1.nxs 1>stdout.debug.ger_duesseldorf_antonov_1.txt 2>stderr.debug.ger_duesseldorf_antonov_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_27632-v07.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_27632-v07.rrng" --output=debug.ger_duesseldorf_bajaj_0.nxs 1>stdout.debug.ger_duesseldorf_bajaj_0.txt 2>stderr.debug.ger_duesseldorf_bajaj_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_30139-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_30139-v02.rrng" --output=debug.ger_duesseldorf_bajaj_1.nxs 1>stdout.debug.ger_duesseldorf_bajaj_1.txt 2>stderr.debug.ger_duesseldorf_bajaj_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_33455-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_33455-v01.rrng" --output=debug.ger_duesseldorf_bajaj_2.nxs 1>stdout.debug.ger_duesseldorf_bajaj_2.txt 2>stderr.debug.ger_duesseldorf_bajaj_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R96_33145-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R96_33145-v01.rrng" --output=debug.ger_duesseldorf_balachandran_0.nxs 1>stdout.debug.ger_duesseldorf_balachandran_0.txt 2>stderr.debug.ger_duesseldorf_balachandran_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5076_33297-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5076_33297-v01.rrng" --output=debug.ger_duesseldorf_balachandran_1.nxs 1>stdout.debug.ger_duesseldorf_balachandran_1.txt 2>stderr.debug.ger_duesseldorf_balachandran_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5076_33303-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5076_33303-v01.rrng" --output=debug.ger_duesseldorf_balachandran_2.nxs 1>stdout.debug.ger_duesseldorf_balachandran_2.txt 2>stderr.debug.ger_duesseldorf_balachandran_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_35459-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/range_file_RNG.RNG" --output=debug.ger_duesseldorf_balachandran_3.nxs 1>stdout.debug.ger_duesseldorf_balachandran_3.txt 2>stderr.debug.ger_duesseldorf_balachandran_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06656-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06656-v02.rrng" --output=debug.ger_duesseldorf_cautaerts_25.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_25.txt 2>stderr.debug.ger_duesseldorf_cautaerts_25.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06656-v03.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06656-v03.rrng" --output=debug.ger_duesseldorf_cautaerts_27.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_27.txt 2>stderr.debug.ger_duesseldorf_cautaerts_27.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R56_02293-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/rng_R56_02293-v01.rng" --output=debug.ger_duesseldorf_cautaerts_58.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_58.txt 2>stderr.debug.ger_duesseldorf_cautaerts_58.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_10355-cluster.cluster.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_10355-cluster.cluster.rrng" --output=debug.ger_duesseldorf_cautaerts_95.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_95.txt 2>stderr.debug.ger_duesseldorf_cautaerts_95.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_10355-v02.cluster.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_10355-v02.cluster.rrng" --output=debug.ger_duesseldorf_cautaerts_106.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_106.txt 2>stderr.debug.ger_duesseldorf_cautaerts_106.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06667-v03.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/rangeOptimized.rrng" --output=debug.ger_duesseldorf_cautaerts_289.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_289.txt 2>stderr.debug.ger_duesseldorf_cautaerts_289.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06667-v06.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06667-v06.rng" --output=debug.ger_duesseldorf_cautaerts_305.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_305.txt 2>stderr.debug.ger_duesseldorf_cautaerts_305.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06696-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06696-v01.rrng" --output=debug.ger_duesseldorf_cautaerts_371.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_371.txt 2>stderr.debug.ger_duesseldorf_cautaerts_371.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06697-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06697-v01.rrng" --output=debug.ger_duesseldorf_cautaerts_396.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_396.txt 2>stderr.debug.ger_duesseldorf_cautaerts_396.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06703-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06703-v01.rrng" --output=debug.ger_duesseldorf_cautaerts_462.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_462.txt 2>stderr.debug.ger_duesseldorf_cautaerts_462.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06703-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06703-v02.rrng" --output=debug.ger_duesseldorf_cautaerts_472.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_472.txt 2>stderr.debug.ger_duesseldorf_cautaerts_472.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06703-v03.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R33_06703-v03.rrng" --output=debug.ger_duesseldorf_cautaerts_477.nxs 1>stdout.debug.ger_duesseldorf_cautaerts_477.txt 2>stderr.debug.ger_duesseldorf_cautaerts_477.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R18_15386-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/allicumgag.rrng" --output=debug.ger_duesseldorf_gault_0.nxs 1>stdout.debug.ger_duesseldorf_gault_0.txt 2>stderr.debug.ger_duesseldorf_gault_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_21659-v01.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_21659-v01.rng" --output=debug.ger_duesseldorf_kwiatkowski_0.nxs 1>stdout.debug.ger_duesseldorf_kwiatkowski_0.txt 2>stderr.debug.ger_duesseldorf_kwiatkowski_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealed CoCrNi_100.csv" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealing_M2C_100.RRNG" --output=debug.ger_duesseldorf_li_0.nxs 1>stdout.debug.ger_duesseldorf_li_0.txt 2>stderr.debug.ger_duesseldorf_li_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealed CoCrNi_111.csv" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealing_M2C_111.RRNG" --output=debug.ger_duesseldorf_li_1.nxs 1>stdout.debug.ger_duesseldorf_li_1.txt 2>stderr.debug.ger_duesseldorf_li_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealed_CoCrNi_111_random_dataset.csv" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealing_M2C_111.RRNG" --output=debug.ger_duesseldorf_li_2.nxs 1>stdout.debug.ger_duesseldorf_li_2.txt 2>stderr.debug.ger_duesseldorf_li_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealed_CoCrNi_100_random_dataset.csv" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Annealing_M2C_100.RRNG" --output=debug.ger_duesseldorf_li_3.nxs 1>stdout.debug.ger_duesseldorf_li_3.txt 2>stderr.debug.ger_duesseldorf_li_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_37592-v05.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/37592.rrng" --output=debug.ger_duesseldorf_mayweg_0.nxs 1>stdout.debug.ger_duesseldorf_mayweg_0.txt 2>stderr.debug.ger_duesseldorf_mayweg_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_41455-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_41455-v02.rrng" --output=debug.ger_duesseldorf_mayweg_1.nxs 1>stdout.debug.ger_duesseldorf_mayweg_1.txt 2>stderr.debug.ger_duesseldorf_mayweg_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_41894-v05.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_41894-v05.rrng" --output=debug.ger_duesseldorf_mayweg_2.nxs 1>stdout.debug.ger_duesseldorf_mayweg_2.txt 2>stderr.debug.ger_duesseldorf_mayweg_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_42390-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_42390-v02.RNG" --output=debug.ger_duesseldorf_mayweg_3.nxs 1>stdout.debug.ger_duesseldorf_mayweg_3.txt 2>stderr.debug.ger_duesseldorf_mayweg_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/D1_High_Hc_R5076_52126.apt" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/D1_High_Hc_R5076_52126.RRNG" --output=debug.ger_duesseldorf_polin_0.nxs 1>stdout.debug.ger_duesseldorf_polin_0.txt 2>stderr.debug.ger_duesseldorf_polin_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ed47f8ee-c770-4289-8a10-c24f70229d88.POS" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5096_52749_recent_range_file_1.RRNG" --output=debug.ger_duesseldorf_saksena_2.nxs 1>stdout.debug.ger_duesseldorf_saksena_2.txt 2>stderr.debug.ger_duesseldorf_saksena_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_20169-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_20169-v01.rrng" --output=debug.ger_duesseldorf_stephenson_0.nxs 1>stdout.debug.ger_duesseldorf_stephenson_0.txt 2>stderr.debug.ger_duesseldorf_stephenson_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5076_36088-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5076_36088-v01.rrng" --output=debug.ger_duesseldorf_zhao_0.nxs 1>stdout.debug.ger_duesseldorf_zhao_0.txt 2>stderr.debug.ger_duesseldorf_zhao_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_24909-0.5h.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76_24909-0.5h.rng" --output=debug.ger_duesseldorf_zhao_8.nxs 1>stdout.debug.ger_duesseldorf_zhao_8.txt 2>stderr.debug.ger_duesseldorf_zhao_8.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26865-PA.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26865-PA.RNG" --output=debug.ger_duesseldorf_zhao_9.nxs 1>stdout.debug.ger_duesseldorf_zhao_9.txt 2>stderr.debug.ger_duesseldorf_zhao_9.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26737-PA.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26737-PA.RNG" --output=debug.ger_duesseldorf_zhao_10.nxs 1>stdout.debug.ger_duesseldorf_zhao_10.txt 2>stderr.debug.ger_duesseldorf_zhao_10.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26120-OA.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26120-OA.RNG" --output=debug.ger_duesseldorf_zhao_11.nxs 1>stdout.debug.ger_duesseldorf_zhao_11.txt 2>stderr.debug.ger_duesseldorf_zhao_11.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26110-30min.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-26110-0.5h.RNG" --output=debug.ger_duesseldorf_zhao_12.nxs 1>stdout.debug.ger_duesseldorf_zhao_12.txt 2>stderr.debug.ger_duesseldorf_zhao_12.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-25946-2h.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-25946-2h.RNG" --output=debug.ger_duesseldorf_zhao_13.nxs 1>stdout.debug.ger_duesseldorf_zhao_13.txt 2>stderr.debug.ger_duesseldorf_zhao_13.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-25139-2h.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-25139-2h.RNG" --output=debug.ger_duesseldorf_zhao_14.nxs 1>stdout.debug.ger_duesseldorf_zhao_14.txt 2>stderr.debug.ger_duesseldorf_zhao_14.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-24909-30min.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R76-24909-30min.rng" --output=debug.ger_duesseldorf_zhao_15.nxs 1>stdout.debug.ger_duesseldorf_zhao_15.txt 2>stderr.debug.ger_duesseldorf_zhao_15.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R25_33096-v02.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R25_33096-v02.rrng" --output=debug.ger_duesseldorf_zhao_16.nxs 1>stdout.debug.ger_duesseldorf_zhao_16.txt 2>stderr.debug.ger_duesseldorf_zhao_16.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/AluminiumNEW.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/AluminiumNEW_MassSpec_ranged.fig.txt" --output=debug.ger_erlangen_felfer_0.nxs 1>stdout.debug.ger_erlangen_felfer_0.txt 2>stderr.debug.ger_erlangen_felfer_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Superalloy.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Superalloy_MassSpec_ranged.fig.txt" --output=debug.ger_erlangen_felfer_1.nxs 1>stdout.debug.ger_erlangen_felfer_1.txt 2>stderr.debug.ger_erlangen_felfer_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R56_01769-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R56_01769.rng.fig.txt" --output=debug.ger_erlangen_felfer_2.nxs 1>stdout.debug.ger_erlangen_felfer_2.txt 2>stderr.debug.ger_erlangen_felfer_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_08680-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R21_08680.rrng" --output=debug.ger_erlangen_heller_0.nxs 1>stdout.debug.ger_erlangen_heller_0.txt 2>stderr.debug.ger_erlangen_heller_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R56_03446-v01.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/3446_AP06.RRNG" --output=debug.ger_erlangen_heller_1.nxs 1>stdout.debug.ger_erlangen_heller_1.txt 2>stderr.debug.ger_erlangen_heller_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/1748_Al.h5" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/1748_Al_range_.h5" --output=debug.ger_erlangen_monajem_0.nxs 1>stdout.debug.ger_erlangen_monajem_0.txt 2>stderr.debug.ger_erlangen_monajem_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R52_05169-v04.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/range_file.rrng" --output=debug.ger_karlsruhe_pantenburg_3.nxs 1>stdout.debug.ger_karlsruhe_pantenburg_3.txt 2>stderr.debug.ger_karlsruhe_pantenburg_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R36_03240-v01.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R36_03240-v01.rng" --output=debug.ger_saarbruecken_pauly_0.nxs 1>stdout.debug.ger_saarbruecken_pauly_0.txt 2>stderr.debug.ger_saarbruecken_pauly_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R36_03244-v01.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R36_03244-v01.rng" --output=debug.ger_saarbruecken_pauly_1.nxs 1>stdout.debug.ger_saarbruecken_pauly_1.txt 2>stderr.debug.ger_saarbruecken_pauly_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/060720a-1.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ZrCuAl-1.rng" --output=debug.jap_tsukuba_hono_0.nxs 1>stdout.debug.jap_tsukuba_hono_0.txt 2>stderr.debug.jap_tsukuba_hono_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO_pole.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO.env" --output=debug.nor_trondheim_hatzeglou_0.nxs 1>stdout.debug.nor_trondheim_hatzeglou_0.txt 2>stderr.debug.nor_trondheim_hatzeglou_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/SiGe.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/SiGe.RRNG" --output=debug.nor_trondheim_hatzeglou_1.nxs 1>stdout.debug.nor_trondheim_hatzeglou_1.txt 2>stderr.debug.nor_trondheim_hatzeglou_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5106_01246-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5106_01246-v02.rrng" --output=debug.nor_trondheim_sohail_2.nxs 1>stdout.debug.nor_trondheim_sohail_2.txt 2>stderr.debug.nor_trondheim_sohail_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5106_01263-v05.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5106_01263-v03range file.rrng" --output=debug.nor_trondheim_sohail_3.nxs 1>stdout.debug.nor_trondheim_sohail_3.txt 2>stderr.debug.nor_trondheim_sohail_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Autoclave exposed SL CrN-NbN coated Zr cladding_coating-substrate interface (Fig10 - R52_16437).pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Range file for coating-substrate interface (Fig10).rrng" --output=debug.swe_gothenburg_fazi_0.nxs 1>stdout.debug.swe_gothenburg_fazi_0.txt 2>stderr.debug.swe_gothenburg_fazi_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/As-deposited SL CrN-NbN coated Zr cladding_coating-substrate interface (Fig10 - R52_16433).pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Range file for coating-substrate interface (Fig10).rrng" --output=debug.swe_gothenburg_fazi_1.nxs 1>stdout.debug.swe_gothenburg_fazi_1.txt 2>stderr.debug.swe_gothenburg_fazi_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Autoclave exposed NL CrN-NbN coated Zr cladding_coating bulk (Fig8 - R52_16422).pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Range file for CrNNbN coating (Fig8).rrng" --output=debug.swe_gothenburg_fazi_2.nxs 1>stdout.debug.swe_gothenburg_fazi_2.txt 2>stderr.debug.swe_gothenburg_fazi_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/As-deposited NL CrN-NbN coated Zr cladding_coating bulk (Fig8 -R52_16413).pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Range file for CrNNbN coating (Fig8).rrng" --output=debug.swe_gothenburg_fazi_3.nxs 1>stdout.debug.swe_gothenburg_fazi_3.txt 2>stderr.debug.swe_gothenburg_fazi_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Autoclave exposed SL CrN-NbN coated Zr cladding_coating bulk (Fig8 -R52_15858).pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Range file for CrNNbN coating (Fig8).rrng" --output=debug.swe_gothenburg_fazi_4.nxs 1>stdout.debug.swe_gothenburg_fazi_4.txt 2>stderr.debug.swe_gothenburg_fazi_4.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/As-deposited SL CrN-NbN coated Zr cladding_coating bulk (Fig8 - R52_15840).pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Range file for CrNNbN coating (Fig8).rrng" --output=debug.swe_gothenburg_fazi_5.nxs 1>stdout.debug.swe_gothenburg_fazi_5.txt 2>stderr.debug.swe_gothenburg_fazi_5.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5038_00333-v02.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/rng_5pj.rrng" --output=debug.usa_denton_smith_0.nxs 1>stdout.debug.usa_denton_smith_0.txt 2>stderr.debug.usa_denton_smith_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R5038_00556-v04.epos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/rng_5pj.rrng" --output=debug.usa_denton_smith_1.nxs 1>stdout.debug.usa_denton_smith_1.txt 2>stderr.debug.usa_denton_smith_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Si.apt" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Si.RNG" --output=debug.usa_denton_smith_2.nxs 1>stdout.debug.usa_denton_smith_2.txt 2>stderr.debug.usa_denton_smith_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R13_03871-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ORNL format CuNi Range File.rng" --output=debug.usa_harvard_barroo_0.nxs 1>stdout.debug.usa_harvard_barroo_0.txt 2>stderr.debug.usa_harvard_barroo_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R13_03871-v01.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ORNL format CuNi Range File.rng" --output=debug.usa_harvard_barroo_1.nxs 1>stdout.debug.usa_harvard_barroo_1.txt 2>stderr.debug.usa_harvard_barroo_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R94_10497.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R94_10497.rrng" --output=debug.usa_knoxville_levine_0.nxs 1>stdout.debug.usa_knoxville_levine_0.txt 2>stderr.debug.usa_knoxville_levine_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_06365-v02.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_06365-v02.rrng" --output=debug.usa_portland_wang_0.nxs 1>stdout.debug.usa_portland_wang_0.txt 2>stderr.debug.usa_portland_wang_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_11381-v02.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ceria_11553.rrng" --output=debug.usa_richland_kruska_0.nxs 1>stdout.debug.usa_richland_kruska_0.txt 2>stderr.debug.usa_richland_kruska_0.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_11553-v01.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ceria_11553.rrng" --output=debug.usa_richland_kruska_1.nxs 1>stdout.debug.usa_richland_kruska_1.txt 2>stderr.debug.usa_richland_kruska_1.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_11556-v01.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ceria_11553.rrng" --output=debug.usa_richland_kruska_2.nxs 1>stdout.debug.usa_richland_kruska_2.txt 2>stderr.debug.usa_richland_kruska_2.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_11554-v01.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ceria_11553.rrng" --output=debug.usa_richland_kruska_3.nxs 1>stdout.debug.usa_richland_kruska_3.txt 2>stderr.debug.usa_richland_kruska_3.txt +dataconverter --reader apm --nxdl NXroot --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/R31_11378-v01.ato" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/Ceria_11553.rrng" --output=debug.usa_richland_kruska_4.nxs 1>stdout.debug.usa_richland_kruska_4.txt 2>stderr.debug.usa_richland_kruska_4.txt diff --git a/debug/batch_queue.04.sh b/debug/batch_queue.04.sh new file mode 100755 index 000000000..1f07fe91c --- /dev/null +++ b/debug/batch_queue.04.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +dataconverter --reader apm --nxdl NXapm --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO_pole.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO.env" --input-file="apm.oasis.specific.yaml" --output=debug.nor_trondheim_hatzeglou_0.nxs +# 1>stdout.debug.nor_trondheim_hatzeglou_0.txt 2>stderr.debug.nor_trondheim_hatzeglou_0.txt diff --git a/debug/eln_data_apm.yaml b/debug/eln_data_apm.yaml new file mode 100644 index 000000000..ddd67ebcf --- /dev/null +++ b/debug/eln_data_apm.yaml @@ -0,0 +1,118 @@ +atom_probe: + analysis_chamber_pressure: + unit: torr + value: 2.0e-10 + control_software_program: IVAS + control_software_program__attr_version: 3.6.8 + fabrication_identifier: '12' + fabrication_model: LEAP3000 + fabrication_vendor: Cameca + field_of_view: + unit: nm + value: 20. + flight_path_length: + unit: m + value: 1.2 + instrument_name: LEAP + ion_detector_model: n/a + ion_detector_name: n/a + ion_detector_serial_number: n/a + ion_detector_type: mcp_dld + local_electrode_name: L1 + location: Denton + pulser: + laser_source: + - name: laser1 + power: + unit: nW + value: 24.0 + pulse_energy: + unit: pJ + value: 24.0 + wavelength: + unit: nm + value: 355.0 + - name: laser2 + power: + unit: nW + value: 12.0 + pulse_energy: + unit: pJ + value: 12.0 + wavelength: + unit: nm + value: 254.0 + pulse_fraction: 0.8 + pulse_frequency: + unit: kHz + value: 250.0 + pulse_mode: laser_and_voltage + reflectron_applied: true + specimen_monitoring_detection_rate: 0.8 + specimen_monitoring_initial_radius: + unit: nm + value: 12.0 + specimen_monitoring_shank_angle: + unit: ° + value: 5.0 + stage_lab_base_temperature: + unit: K + value: 20.0 + status: success +entry: + experiment_description: '

Normal

+ +

Bold

+ +

Italics

' + experiment_identifier: Si test + start_time: '2023-06-11T11:20:00+00:00' + end_time: '2023-06-11T11:20:00+00:00' + run_number: '2121' + operation_mode: apt +ranging: + program: IVAS + program__attr_version: 3.6.8 +reconstruction: + crystallographic_calibration: n/a + parameter: kf = 1.8, icf = 3.3 + program: IVAS + program__attr_version: 3.6.8 + protocol_name: bas +sample: + composition: + - Mo + - Al 12 +- 3 + - B 50 ppm +- 12 + - C 3.6 + grain_diameter: + unit: µm + value: 200.0 + grain_diameter_error: + unit: µm + value: 50.0 + heat_treatment_quenching_rate: + unit: K / s + value: 150.0 + heat_treatment_quenching_rate_error: + unit: K / s + value: 10.0 + heat_treatment_temperature: + unit: K + value: 600.0 + heat_treatment_temperature_error: + unit: K + value: 20.0 +specimen: + alias: Si + description: '

normal

+ +

bold

+ +

italics

' + is_polycrystalline: true + name: usa_denton_smith_si + preparation_date: '2023-06-11T12:51:00+00:00' +user: +- {} +- {} diff --git a/pynxtools/dataconverter/readers/apm/map_concepts/apm_deployment_specifics_to_nx_map.py b/pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py similarity index 77% rename from pynxtools/dataconverter/readers/apm/map_concepts/apm_deployment_specifics_to_nx_map.py rename to pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py index d4cdf84f6..3f14d9414 100644 --- a/pynxtools/dataconverter/readers/apm/map_concepts/apm_deployment_specifics_to_nx_map.py +++ b/pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Dict mapping values for a specific deployed config of NOMAD OASIS + ELN + apm reader.""" +"""Dict mapping values for a specifically configured NOMAD OASIS.""" # pylint: disable=no-member,line-too-long @@ -40,13 +40,18 @@ # such that it executes after reading generic ELN data (eventually available entries) # in the template get overwritten +import datetime as dt + from pynxtools.dataconverter.readers.apm.utils.apm_versioning \ import NX_APM_ADEF_NAME, NX_APM_ADEF_VERSION, NX_APM_EXEC_NAME, NX_APM_EXEC_VERSION -NxApmDeploymentSpecificInput \ - = {"/ENTRY[entry*]/@version": f"{NX_APM_ADEF_VERSION}", - "/ENTRY[entry*]/definition": f"{NX_APM_ADEF_NAME}", - "/ENTRY[entry*]/PROGRAM[program1]/program": f"{NX_APM_EXEC_NAME}", - "/ENTRY[entry*]/PROGRAM[program1]/program/@version": f"{NX_APM_EXEC_VERSION}", - "/ENTRY[entry*]/atom_probe/location": {"fun": "load_from", "terms": "location"}} +APM_OASIS_TO_NEXUS_CFG \ + = [("/ENTRY[entry*]/@version", f"{NX_APM_ADEF_VERSION}"), + ("/ENTRY[entry*]/definition", f"{NX_APM_ADEF_NAME}"), + ("/ENTRY[entry*]/operation_mode", "load_from", "operation_mode"), + ("/ENTRY[entry*]/start_time", f"{dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z')}")] + +# ("/ENTRY[entry*]/PROGRAM[program1]/program", f"{NX_APM_EXEC_NAME}"), +# ("/ENTRY[entry*]/PROGRAM[program1]/program/@version", f"{NX_APM_EXEC_VERSION}"), +# ("/ENTRY[entry*]/atom_probe/location", "load_from", "location")] diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 97ceabccc..6e1b6b068 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -92,8 +92,9 @@ def read(self, else: # eln_data, and ideally recon and ranging definitions from technology partner file print("Parse ELN and technology partner file(s)...") case = ApmUseCaseSelector(file_paths) - assert case.is_valid is True, \ - "Such a combination of input-file(s, if any) is not supported !" + if case.is_valid == False: + print("Such a combination of input-file(s, if any) is not supported !") + return {} """ print("Parse (meta)data coming from an ELN...") @@ -103,13 +104,13 @@ def read(self, else: print("No input file defined for eln data !") return {} + """ print("Parse (meta)data coming from a configuration that specific OASIS...") if len(case.cfg) == 1: nx_apm_cfg = NxApmNomadOasisConfigurationParser(case.cfg[0], entry_id) nx_apm_cfg.report(template) # having and or using a deployment-specific configuration is optional - """ print("Parse (numerical) data and metadata from ranging definitions file...") if len(case.reconstruction) == 1: @@ -131,8 +132,8 @@ def read(self, """ # print("Reporting state of template before passing to HDF5 writing...") - # for keyword in template.keys(): - # print(keyword) + for keyword in template.keys(): + print(keyword) # print(template[keyword]) print("Forward instantiated template to the NXS writer...") diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py b/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py index a0eb9846e..825ebbc6e 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py @@ -82,17 +82,17 @@ def create_default_plot_reconstruction(template: dict, entry_id: int) -> dict: # mind that histogram does not follow Cartesian conventions so a transpose # might be necessary, for now we implement the transpose in the appdef template[f"{trg}DATA[data_counts]"] \ - = {"compress": np.array(hist3d[0], np.uint32), "strength": 1} + = {"compress": np.asarray(hist3d[0], np.uint32), "strength": 1} template[f"{trg}AXISNAME[axis_x]"] \ - = {"compress": np.array(hist3d[1][0][1::], np.float32), "strength": 1} + = {"compress": np.asarray(hist3d[1][0][1::], np.float32), "strength": 1} template[f"{trg}AXISNAME[axis_x]/@units"] = "nm" template[f"{trg}AXISNAME[axis_x]/@long_name"] = "x (nm)" template[f"{trg}AXISNAME[axis_y]"] \ - = {"compress": np.array(hist3d[1][1][1::], np.float32), "strength": 1} + = {"compress": np.asarray(hist3d[1][1][1::], np.float32), "strength": 1} template[f"{trg}AXISNAME[axis_y]/@units"] = "nm" template[f"{trg}AXISNAME[axis_y]/@long_name"] = "y (nm)" template[f"{trg}AXISNAME[axis_z]"] \ - = {"compress": np.array(hist3d[1][2][1::], np.float32), "strength": 1} + = {"compress": np.asarray(hist3d[1][2][1::], np.float32), "strength": 1} template[f"{trg}AXISNAME[axis_z]/@units"] = "nm" template[f"{trg}AXISNAME[axis_z]/@long_name"] = "z (nm)" print("Default plot 3D discretized reconstruction at 1 nm binning.") @@ -130,9 +130,9 @@ def create_default_plot_mass_spectrum(template: dict, entry_id: int) -> dict: template[f"{trg}PROGRAM[program1]/program/@version"] = get_repo_last_commit() template[f"{trg}range_increment"] = mqincr - template[f"{trg}range_increment/@units"] = "Da" - template[f"{trg}range_minmax"] = np.array([mqmin, mqmax], np.float32) - template[f"{trg}range_minmax/@units"] = "Da" + template[f"{trg}range_increment/@units"] = "u" + template[f"{trg}range_minmax"] = np.asarray([mqmin, mqmax], np.float32) + template[f"{trg}range_minmax/@units"] = "u" trg = f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" \ f"mass_to_charge_distribution/mass_spectrum/" @@ -141,11 +141,11 @@ def create_default_plot_mass_spectrum(template: dict, entry_id: int) -> dict: template[f"{trg}@axes"] = "axis_mass_to_charge" template[f"{trg}@AXISNAME_indices[axis_mass_to_charge]"] = np.uint32(0) template[f"{trg}DATA[data_counts]"] \ - = {"compress": np.array(hist1d[0], np.uint32), "strength": 1} + = {"compress": np.asarray(hist1d[0], np.uint32), "strength": 1} template[f"{trg}DATA[data_counts]/@long_name"] = "Counts (1)" template[f"{trg}AXISNAME[axis_mass_to_charge]"] \ - = {"compress": np.array(hist1d[1][1::], np.float32), "strength": 1} - template[f"{trg}AXISNAME[axis_mass_to_charge]/@units"] = "Da" + = {"compress": np.asarray(hist1d[1][1::], np.float32), "strength": 1} + template[f"{trg}AXISNAME[axis_mass_to_charge]/@units"] = "u" template[f"{trg}AXISNAME[axis_mass_to_charge]/@long_name"] \ = "Mass-to-charge-state ratio (Da)" print("Plot mass spectrum at 0.01 Da binning was created.") diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py index 36c494718..f637eac1d 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py @@ -93,8 +93,9 @@ def check_validity_of_file_combinations(self): range_input += len(value) if suffix == ".h5": recon_input += len(value) + print(f"{recon_input}, {range_input}, {other_input}") - if (recon_input == 1) and (range_input == 1): # and (1 <= other_input <= 2): + if (recon_input == 1) and (range_input == 1) and (1 <= other_input <= 2): self.is_valid = True self.reconstruction: List[str] = [] self.ranging: List[str] = [] diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_generate_synthetic_data.py b/pynxtools/dataconverter/readers/apm/utils/apm_generate_synthetic_data.py index c34d30f7b..5fb4656a1 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_generate_synthetic_data.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_generate_synthetic_data.py @@ -274,7 +274,7 @@ def composition_to_ranging_definitions(self, template: dict) -> dict: template[f"{path}charge_state"] = np.int8(tpl[1]) template[f"{path}mass_to_charge_range"] = np.reshape( np.asarray([tpl[2], tpl[2] + MQ_EPSILON], np.float32), (1, 2)) - template[f"{path}mass_to_charge_range/@units"] = "Da" + template[f"{path}mass_to_charge_range/@units"] = "u" nuclid_list = np.zeros([2, 32], np.uint16) nuclid_list = isotope_vector_to_nuclid_list(ivec) template[f"{path}nuclid_list"] = np.asarray(nuclid_list, np.uint16) @@ -552,6 +552,6 @@ def synthesize(self, template: dict) -> dict: trg = f"{prefix}mass_to_charge_conversion/" template[f"{trg}mass_to_charge"] \ = {"compress": np.asarray(self.m_z, np.float32), "strength": 1} - template[f"{trg}mass_to_charge/@units"] = "Da" + template[f"{trg}mass_to_charge/@units"] = "u" return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py index 22f26c6f6..b7839d419 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py @@ -22,8 +22,8 @@ import flatdict as fd import yaml -from pynxtools.dataconverter.readers.apm.map_concepts.apm_deployment_specifics_to_nx_map \ - import NxApmDeploymentSpecificInput +from pynxtools.dataconverter.readers.apm.map_concepts.apm_oasis_cfg_to_nx_map \ + import APM_OASIS_TO_NEXUS_CFG from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ import apply_modifier, variadic_path_to_specific_path @@ -39,6 +39,7 @@ def __init__(self, file_path: str, entry_id: int): self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") + print(self.yml) else: self.entry_id = 1 self.file_path = "" @@ -46,10 +47,23 @@ def __init__(self, file_path: str, entry_id: int): def report(self, template: dict) -> dict: """Copy data from configuration applying mapping functors.""" - for nx_path, modifier in NxApmDeploymentSpecificInput.items(): - if nx_path not in ("IGNORE", "UNCLEAR"): - trg = variadic_path_to_specific_path(nx_path, [self.entry_id, 1]) - res = apply_modifier(modifier, self.yml) - if res is not None: - template[trg] = res + for tpl in APM_OASIS_TO_NEXUS_CFG: + identifier = [self.entry_id] + if isinstance(tpl, tuple): + if tpl[0] not in ("IGNORE", "UNCLEAR"): + trg = variadic_path_to_specific_path(tpl[0], identifier) + print(f"processing tpl {tpl} ... trg {trg}") + # print(f"Target {trg} after variadic name resolution identifier {identifier}") + if len(tpl) == 2: + # nxpath, value to use directly + template[trg] = tpl[1] + if len(tpl) == 3: + # nxpath, modifier, value, modifier (function) evaluates value to use + if tpl[1] == "load_from": + if tpl[2] in self.yml.keys(): + template[trg] = self.yml[tpl[2]] + else: + raise ValueError(f"tpl2 {tpl[2]} not in self.yml.keys()!") + else: + raise ValueError(f"tpl1 {tpl[1]} is not load_from!") return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py index 4681d2eaf..a40340830 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py @@ -56,7 +56,7 @@ def add_unknown_iontype(template: dict, entry_id: int) -> dict: template[f"{trg}charge_state"] = np.int8(0) template[f"{trg}mass_to_charge_range"] \ = np.reshape(np.asarray([0.0, MQ_EPSILON], np.float32), (1, 2)) - template[f"{trg}mass_to_charge_range/@units"] = "Da" + template[f"{trg}mass_to_charge_range/@units"] = "u" nuclid_list = isotope_vector_to_nuclid_list(ivec) template[f"{trg}nuclid_list"] = np.asarray(nuclid_list, np.uint16) template[f"{trg}name"] = isotope_vector_to_human_readable_name(ivec, 0) @@ -74,8 +74,8 @@ def add_standardize_molecular_ions(ion_lst: list, template: dict, entry_id: int) (1, MAX_NUMBER_OF_ATOMS_PER_ION)) template[f"{path}charge_state"] = np.int8(ion.charge_state.values) template[f"{path}mass_to_charge_range"] \ - = np.array(ion.ranges.values, np.float32) - template[f"{path}mass_to_charge_range/@units"] = "Da" # ion.ranges.unit + = np.asarray(ion.ranges.values, np.float32) + template[f"{path}mass_to_charge_range/@units"] = "u" # ion.ranges.unit template[f"{path}nuclid_list"] = ion.nuclid_list.values template[f"{path}name"] = ion.name.values diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py index 319045102..8584db1e6 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py @@ -41,14 +41,14 @@ def extract_data_from_pos_file(file_path: str, prefix: str, template: dict) -> d trg = f"{prefix}reconstruction/" xyz = posfile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.values, np.float32), "strength": 1} + = {"compress": np.asarray(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = posfile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.values, np.float32), "strength": 1} + = {"compress": np.asarray(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z return template @@ -62,14 +62,14 @@ def extract_data_from_epos_file(file_path: str, prefix: str, template: dict) -> trg = f"{prefix}reconstruction/" xyz = eposfile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.values, np.float32), "strength": 1} + = {"compress": np.asarray(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = eposfile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.values, np.float32), "strength": 1} + = {"compress": np.asarray(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z @@ -135,14 +135,14 @@ def extract_data_from_apt_file(file_path: str, prefix: str, template: dict) -> d trg = f"{prefix}reconstruction/" xyz = aptfile.get_named_quantity("Position") template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.values, np.float32), "strength": 1} + = {"compress": np.asarray(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = aptfile.get_named_quantity("Mass") template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.values, np.float32), "strength": 1} + = {"compress": np.asarray(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z # all less explored optional branches in an APT6 file can also already @@ -161,14 +161,14 @@ def extract_data_from_ato_file(file_path: str, prefix: str, template: dict) -> d trg = f"{prefix}reconstruction/" xyz = atofile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.values, np.float32), "strength": 1} + = {"compress": np.asarray(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = atofile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.values, np.float32), "strength": 1} + = {"compress": np.asarray(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z return template @@ -182,14 +182,14 @@ def extract_data_from_csv_file(file_path: str, prefix: str, template: dict) -> d trg = f"{prefix}reconstruction/" xyz = csvfile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.values, np.float32), "strength": 1} + = {"compress": np.asarray(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = csvfile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.values, np.float32), "strength": 1} + = {"compress": np.asarray(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z return template @@ -203,14 +203,14 @@ def extract_data_from_pyc_file(file_path: str, prefix: str, template: dict) -> d trg = f"{prefix}reconstruction/" xyz = pycfile.get_reconstructed_positions() template[f"{trg}reconstructed_positions"] \ - = {"compress": np.array(xyz.values, np.float32), "strength": 1} + = {"compress": np.asarray(xyz.values, np.float32), "strength": 1} template[f"{trg}reconstructed_positions/@units"] = xyz.unit del xyz trg = f"{prefix}mass_to_charge_conversion/" m_z = pycfile.get_mass_to_charge_state_ratio() template[f"{trg}mass_to_charge"] \ - = {"compress": np.array(m_z.values, np.float32), "strength": 1} + = {"compress": np.asarray(m_z.values, np.float32), "strength": 1} template[f"{trg}mass_to_charge/@units"] = m_z.unit del m_z return template diff --git a/pyproject.toml b/pyproject.toml index 828d199ee..99b9b173a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ dev = [ "types-pytz", "types-requests", "pip-tools", + "jupyterlab", + "jupyterlab_h5web", + "nyaml" ] [project.scripts] From 65c17400b88bf053fca7d64594f411683edd5545 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Mon, 8 Jan 2024 17:59:19 +0100 Subject: [PATCH 07/15] Reactivated default plots for m_z, xyz and path to default plot --- pynxtools/dataconverter/readers/apm/reader.py | 8 +- .../apm/utils/apm_create_nx_default_plots.py | 254 ++++++++---------- .../readers/apm/utils/apm_versioning.py | 6 +- .../readers/shared/shared_utils.py | 25 ++ 4 files changed, 147 insertions(+), 146 deletions(-) diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 6e1b6b068..8c781eb4e 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -126,14 +126,12 @@ def read(self, print("No input-file defined for ranging definitions!") return {} - """ print("Create NeXus default plottable data...") - apm_default_plot_generator(template, n_entries) - """ + apm_default_plot_generator(template, entry_id) # print("Reporting state of template before passing to HDF5 writing...") - for keyword in template.keys(): - print(keyword) + # for keyword in template.keys(): + # print(keyword) # print(template[keyword]) print("Forward instantiated template to the NXS writer...") diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py b/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py index 825ebbc6e..2187a2759 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py @@ -21,8 +21,18 @@ import numpy as np -from pynxtools.dataconverter.readers.shared.shared_utils \ - import get_repo_last_commit +from pynxtools.dataconverter.readers.shared.shared_utils import \ + get_repo_last_commit, decorate_path_to_default_plot +from pynxtools.dataconverter.readers.apm.utils.apm_versioning import \ + NX_APM_EXEC_NAME, NX_APM_EXEC_VERSION, \ + MASS_SPECTRUM_DEFAULT_BINNING, NAIVE_GRID_DEFAULT_VOXEL_SIZE + + +def iedge(imi, imx, resolution): + """Generate linearly space support position.""" + return np.linspace(imi, imx, + num=int(np.ceil((imx - imi) / resolution)) + 1, + endpoint=True) def create_default_plot_reconstruction(template: dict, entry_id: int) -> dict: @@ -31,73 +41,62 @@ def create_default_plot_reconstruction(template: dict, entry_id: int) -> dict: xyz = template[f"{trg}reconstructed_positions"]["compress"] print(f"\tEnter histogram computation, np.shape(xyz) {np.shape(xyz)}") - resolution = 1.0 # cubic voxel edge length in nm - bounds = np.zeros([3, 2], np.float32) # in nm - for i in np.arange(0, 3): - bounds[i, 0] = np.min(xyz[:, i]) - bounds[i, 1] = np.max(xyz[:, i]) + resolution = NAIVE_GRID_DEFAULT_VOXEL_SIZE # cubic voxel edge length in nm # make the bounding box a quadric prism - imi = np.floor(bounds[0, 0]) - resolution - imx = np.ceil(bounds[0, 1]) + resolution - xedges = np.linspace(imi, imx, num=int(np.ceil((imx - imi) / resolution)) + 1, - endpoint=True) - # this partitioning is not general enough, imi and imx should be left and right - # bounds respectively - imi = np.floor(bounds[1, 0]) - resolution - imx = np.ceil(bounds[1, 1]) + resolution - yedges = np.linspace(imi, imx, num=int(np.ceil((imx - imi) / resolution)) + 1, - endpoint=True) - imi = np.floor(bounds[2, 0]) - resolution - imx = np.ceil(bounds[2, 1]) + resolution - zedges = np.linspace(imi, imx, - num=int(np.ceil((imx - imi) / resolution)) + 1, - endpoint=True) + aabb = {"x": [0., 0.], + "y": [0., 0.], + "z": [0., 0.], + "xedge": None, + "yedge": None, + "zedge": None} + col = 0 + for dim in ["x", "y", "z"]: + aabb[f"{dim}"] = np.asarray((np.min(xyz[:, col]), np.max(xyz[:, col]))) + imi = np.floor(aabb[f"{dim}"][0]) - NAIVE_GRID_DEFAULT_VOXEL_SIZE + imx = np.ceil(aabb[f"{dim}"][1]) + NAIVE_GRID_DEFAULT_VOXEL_SIZE + aabb[f"{dim}edge"] = iedge(imi, imx, NAIVE_GRID_DEFAULT_VOXEL_SIZE) + col += 1 hist3d = np.histogramdd((xyz[:, 0], xyz[:, 1], xyz[:, 2]), - bins=(xedges, yedges, zedges)) + bins=(aabb["xedge"], aabb["yedge"], aabb["zedge"])) del xyz - assert isinstance(hist3d[0], np.ndarray), \ - "Hist3d computation from the reconstruction failed!" - assert len(np.shape(hist3d[0])) == 3, \ - "Hist3d computation from the reconstruction failed!" - for i in np.arange(0, 3): - assert np.shape(hist3d[0])[i] > 0, \ - "Dimensions " + str(i) + " has no length!" - + if isinstance(hist3d[0], np.ndarray) == False: + raise ValueError("Hist3d computation from the reconstruction failed!") + if len(np.shape(hist3d[0])) != 3: + raise ValueError("Hist3d computation from the reconstruction failed!") + for idx in [0, 1, 2]: + if np.shape(hist3d[0])[idx] == 0: + raise ValueError(f"Dimensions {idx} has no length!") + + trg = f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/naive_discretization/" + template[f"{trg}PROGRAM[program1]/program"] = NX_APM_EXEC_NAME + template[f"{trg}PROGRAM[program1]/program/@version"] = NX_APM_EXEC_VERSION trg = f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/" \ - f"naive_point_cloud_density_map/" - template[f"{trg}PROGRAM[program1]/program"] = "nomad-parser-nexus/apm/reader.py" - template[f"{trg}PROGRAM[program1]/program/@version"] = get_repo_last_commit() - - trg = f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/" \ - f"naive_point_cloud_density_map/DATA[data]/" + f"naive_discretization/DATA[data]/" template[f"{trg}title"] = "Discretized reconstruction space" # template[f"{trg}@long_name"] = "Discretized reconstruction space" - template[f"{trg}@signal"] = "data_counts" - template[f"{trg}@axes"] = ["axis_x", "axis_y", "axis_z"] - template[f"{trg}@AXISNAME_indices[axis_x]"] = np.uint32(0) - template[f"{trg}@AXISNAME_indices[axis_y]"] = np.uint32(1) - template[f"{trg}@AXISNAME_indices[axis_z]"] = np.uint32(2) + template[f"{trg}@signal"] = "intensity" + col = 0 + dims = ["x", "y", "z"] + axes = [] + for dim in dims: + axes.append(f"axis_{dim}") + template[f"{trg}@AXISNAME_indices[axis_{dim}]"] = np.uint32(col) + col += 1 + template[f"{trg}@axes"] = axes # mind that histogram does not follow Cartesian conventions so a transpose # might be necessary, for now we implement the transpose in the appdef - template[f"{trg}DATA[data_counts]"] \ + template[f"{trg}intensity"] \ = {"compress": np.asarray(hist3d[0], np.uint32), "strength": 1} - template[f"{trg}AXISNAME[axis_x]"] \ - = {"compress": np.asarray(hist3d[1][0][1::], np.float32), "strength": 1} - template[f"{trg}AXISNAME[axis_x]/@units"] = "nm" - template[f"{trg}AXISNAME[axis_x]/@long_name"] = "x (nm)" - template[f"{trg}AXISNAME[axis_y]"] \ - = {"compress": np.asarray(hist3d[1][1][1::], np.float32), "strength": 1} - template[f"{trg}AXISNAME[axis_y]/@units"] = "nm" - template[f"{trg}AXISNAME[axis_y]/@long_name"] = "y (nm)" - template[f"{trg}AXISNAME[axis_z]"] \ - = {"compress": np.asarray(hist3d[1][2][1::], np.float32), "strength": 1} - template[f"{trg}AXISNAME[axis_z]/@units"] = "nm" - template[f"{trg}AXISNAME[axis_z]/@long_name"] = "z (nm)" - print("Default plot 3D discretized reconstruction at 1 nm binning.") - del hist3d - + col = 0 + for dim in dims: + template[f"{trg}AXISNAME[axis_{dim}]"] \ + = {"compress": np.asarray(hist3d[1][col][1::], np.float32), "strength": 1} + template[f"{trg}AXISNAME[axis_{dim}]/@units"] = "nm" + template[f"{trg}AXISNAME[axis_{dim}]/@long_name"] = f"{dim} (nm)" + col += 1 + print(f"Default plot naive discretization 3D {NAIVE_GRID_DEFAULT_VOXEL_SIZE} nm^3.") return template @@ -107,9 +106,10 @@ def create_default_plot_mass_spectrum(template: dict, entry_id: int) -> dict: m_z = template[f"{trg}mass_to_charge"]["compress"] print(f"\tEnter mass spectrum computation, np.shape(m_z) {np.shape(m_z)}") - mqmin = 0.0 # in Da, do not plot unphysical values < 0.0 + # the next three in u + mqmin = 0.0 + mqincr = MASS_SPECTRUM_DEFAULT_BINNING mqmax = np.ceil(np.max(m_z[:])) - mqincr = 0.01 # in Da by default hist1d = np.histogram( m_z[:], @@ -117,103 +117,77 @@ def create_default_plot_mass_spectrum(template: dict, entry_id: int) -> dict: num=int(np.ceil((mqmax - mqmin) / mqincr)) + 1, endpoint=True)) del m_z - assert isinstance(hist1d[0], np.ndarray), \ - "Hist1d computation from the mass spectrum failed!" - assert len(np.shape(hist1d[0])) == 1, \ - "Hist1d computation from the mass spectrum failed!" - for i in np.arange(0, 1): - assert np.shape(hist1d[0])[i] > 0, \ - "Dimensions " + str(i) + " has no length!" + if isinstance(hist1d[0], np.ndarray) == False: + raise ValueError("Hist1d computation from the mass spectrum failed!") + if len(np.shape(hist1d[0])) != 1: + raise ValueError("Hist1d computation from the mass spectrum failed!") + for idx in np.arange(0, 1): + if np.shape(hist1d[0])[idx] == 0: + raise ValueError(f"Dimensions {idx} has no length!") trg = f"/ENTRY[entry{entry_id}]/atom_probe/ranging/mass_to_charge_distribution/" - template[f"{trg}PROGRAM[program1]/program"] = "nomad-parser-nexus/apm/reader.py" - template[f"{trg}PROGRAM[program1]/program/@version"] = get_repo_last_commit() - - template[f"{trg}range_increment"] = mqincr - template[f"{trg}range_increment/@units"] = "u" - template[f"{trg}range_minmax"] = np.asarray([mqmin, mqmax], np.float32) - template[f"{trg}range_minmax/@units"] = "u" + template[f"{trg}PROGRAM[program1]/program"] = NX_APM_EXEC_NAME + template[f"{trg}PROGRAM[program1]/program/@version"] = NX_APM_EXEC_VERSION + template[f"{trg}min_incr_max"] = np.asarray([mqmin, mqincr, mqmax], np.float32) + template[f"{trg}min_incr_max/@units"] = "u" trg = f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" \ f"mass_to_charge_distribution/mass_spectrum/" - template[f"{trg}title"] = "Mass spectrum (0.01 Da binning)" - template[f"{trg}@signal"] = "data_counts" + template[f"{trg}title"] = f"Mass spectrum ({MASS_SPECTRUM_DEFAULT_BINNING} u binning)" + template[f"{trg}@signal"] = "intensity" template[f"{trg}@axes"] = "axis_mass_to_charge" template[f"{trg}@AXISNAME_indices[axis_mass_to_charge]"] = np.uint32(0) - template[f"{trg}DATA[data_counts]"] \ + template[f"{trg}DATA[intensity]"] \ = {"compress": np.asarray(hist1d[0], np.uint32), "strength": 1} - template[f"{trg}DATA[data_counts]/@long_name"] = "Counts (1)" + template[f"{trg}DATA[intensity]/@long_name"] = "Intensity (1)" # Counts (1)" template[f"{trg}AXISNAME[axis_mass_to_charge]"] \ = {"compress": np.asarray(hist1d[1][1::], np.float32), "strength": 1} + del hist1d template[f"{trg}AXISNAME[axis_mass_to_charge]/@units"] = "u" template[f"{trg}AXISNAME[axis_mass_to_charge]/@long_name"] \ - = "Mass-to-charge-state ratio (Da)" - print("Plot mass spectrum at 0.01 Da binning was created.") - del hist1d - + = "Mass-to-charge-state-ratio (u)" + print(f"Plot mass spectrum at {MASS_SPECTRUM_DEFAULT_BINNING} u binning was created.") return template -def apm_default_plot_generator(template: dict, n_entries: int) -> dict: +def apm_default_plot_generator(template: dict, entry_id: int) -> dict: """Copy data from self into template the appdef instance.""" print("Create default plots on-the-fly...") - # now the reader implements what is effectively the task of a normalizer step - # adding plot (discretized representation of the dataset), for now the default plot - # adding plot mass-to-charge-state ratio histogram, - # termed mass spectrum in APM community - - # NEW ISSUE: add path to default plottable data - - # check if reconstructed ion positions have been stored - for entry_id in np.arange(1, n_entries + 1): - trg = f"/ENTRY[entry{entry_id}]/atom_probe/mass_to_charge_conversion/" - has_valid_m_z = False - path = f"{trg}mass_to_charge" - if isinstance(template[f"{trg}mass_to_charge"], dict): - if "compress" in template[path].keys(): - if isinstance(template[path]["compress"], np.ndarray): - has_valid_m_z = True - - trg = f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/" - has_valid_xyz = False - path = f"{trg}reconstructed_positions" - if isinstance(template[path], dict): - if "compress" in template[path].keys(): - if isinstance(template[path]["compress"], np.ndarray): - has_valid_xyz = True - - has_default_data = has_valid_m_z or has_valid_xyz - assert has_default_data is True, \ - "Having no recon or mass-to-charge data is inacceptable at the moment!" - + # default plot is histogram of mass-to-charge-state-ratio values (aka mass spectrum) + # naively discretized 3D reconstruction as a fallback + + has_valid_m_z = False + trg = f"/ENTRY[entry{entry_id}]/atom_probe/mass_to_charge_conversion/mass_to_charge" + if isinstance(template[trg], dict): + if "compress" in template[trg].keys(): + if isinstance(template[trg]["compress"], np.ndarray): + has_valid_m_z = True + has_valid_xyz = False + trg = f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/reconstructed_positions" + if isinstance(template[trg], dict): + if "compress" in template[trg].keys(): + if isinstance(template[trg]["compress"], np.ndarray): + has_valid_xyz = True + print(f"m_z, xyz: {has_valid_m_z}, {has_valid_xyz}") + + if (has_valid_m_z == False) and (has_valid_xyz == False): # NEW ISSUE: fall-back solution to plot something else, however # currently POS, EPOS and APT provide always xyz, and m_z data - - # generate default plottable and add path - template["/@default"] = f"entry{entry_id}" - trg = f"/ENTRY[entry{entry_id}]/" - template[f"{trg}@default"] = "atom_probe" - - if has_valid_m_z is True: - create_default_plot_mass_spectrum(template, entry_id) - # mass_spectrum main default... - trg += "atom_probe/" - template[f"{trg}@default"] = "ranging" - trg += "ranging/" - template[f"{trg}@default"] = "mass_to_charge_distribution" - trg += "mass_to_charge_distribution/" - template[f"{trg}@default"] = "mass_spectrum" - - if has_valid_xyz is True: - # ... discretized naive tomographic reconstruction as fallback... - create_default_plot_reconstruction(template, entry_id) - # generate path to the default plottable - if has_valid_m_z is False: - trg += "atom_probe/" - template[f"{trg}@default"] = "reconstruction" - trg += "reconstruction/" - template[f"{trg}@default"] = "naive_point_cloud_density_map" - trg += "naive_point_cloud_density_map/" - template[f"{trg}@default"] = "data" - + return template + + # generate default plottable and add path + if has_valid_m_z == True: + create_default_plot_mass_spectrum(template, entry_id) + decorate_path_to_default_plot( + template, + f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" \ + f"mass_to_charge_distribution/mass_spectrum") + + if has_valid_xyz == True: + create_default_plot_reconstruction(template, entry_id) + if has_valid_m_z == False: + decorate_path_to_default_plot( + template, + f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/" \ + f"naive_discretization/DATA[data]") return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_versioning.py b/pynxtools/dataconverter/readers/apm/utils/apm_versioning.py index a7e9a39a9..10cf3b638 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_versioning.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_versioning.py @@ -27,5 +27,9 @@ NX_APM_ADEF_VERSION = "nexus-fairmat-proposal successor of " \ "9636feecb79bb32b828b1a9804269573256d7696" # based on https://fairmat-experimental.github.io/nexus-fairmat-proposal -NX_APM_EXEC_NAME = "dataconverter/readers/apm/reader.py" +NX_APM_EXEC_NAME = "pynxtools/dataconverter/readers/apm/reader.py" NX_APM_EXEC_VERSION = get_repo_last_commit() + +# numerics +MASS_SPECTRUM_DEFAULT_BINNING = 0.01 # u +NAIVE_GRID_DEFAULT_VOXEL_SIZE = 1. # nm diff --git a/pynxtools/dataconverter/readers/shared/shared_utils.py b/pynxtools/dataconverter/readers/shared/shared_utils.py index 5311bb3a1..9bca903d1 100644 --- a/pynxtools/dataconverter/readers/shared/shared_utils.py +++ b/pynxtools/dataconverter/readers/shared/shared_utils.py @@ -24,6 +24,7 @@ # import git import hashlib +import numpy as np def get_repo_last_commit() -> str: @@ -96,3 +97,27 @@ def print(self): # test = NxObject(name="test", unit="baud", dtype=np.uint32, value=32000) # test.print() + + +def decorate_path_to_default_plot(template: dict, nxpath: str) -> dict: + """Write @default attribute to point to the default plot.""" + # an example for nxpath + # "/ENTRY[entry1]/atom_probe/ranging/mass_to_charge_distribution/mass_spectrum" + # if nxpath in template.keys(): + print(f"nxpath: {nxpath}") + path = nxpath.split("/") + print(f"path: {path}") + trg = f"/" + for idx in np.arange(0, len(path) - 1): + print(f"trg: {trg}") + symbol_s = path[idx + 1].find("[") + symbol_e = path[idx + 1].find("]") + if (symbol_s >= 0) and (symbol_e > symbol_s): + template[f"{trg}@default"] \ + = f"{path[idx + 1][symbol_s + 1:symbol_e]}" + trg += f"{path[idx + 1][symbol_s + 1:symbol_e]}/" + else: + template[f"{trg}@default"] = f"{path[idx + 1]}" + trg += f"{path[idx + 1]}/" + print(f"trg: {trg}") + return template From 30645d8ac1e864ed1d0599faec7d481eaaa86a46 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Mon, 8 Jan 2024 19:47:44 +0100 Subject: [PATCH 08/15] Refactoring the exemplar NOMAD-OASIS-custom-ELN-schema-to-NXapm mapping, incomplete --- .../apm/map_concepts/apm_eln_to_nx_map.py | 109 ------------------ .../map_concepts/apm_example_eln_to_nx_map.py | 106 +++++++++++++++++ .../readers/apm/utils/apm_load_generic_eln.py | 2 +- 3 files changed, 107 insertions(+), 110 deletions(-) delete mode 100644 pynxtools/dataconverter/readers/apm/map_concepts/apm_eln_to_nx_map.py create mode 100644 pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py diff --git a/pynxtools/dataconverter/readers/apm/map_concepts/apm_eln_to_nx_map.py b/pynxtools/dataconverter/readers/apm/map_concepts/apm_eln_to_nx_map.py deleted file mode 100644 index 76c763f47..000000000 --- a/pynxtools/dataconverter/readers/apm/map_concepts/apm_eln_to_nx_map.py +++ /dev/null @@ -1,109 +0,0 @@ -# -# Copyright The NOMAD Authors. -# -# This file is part of NOMAD. See https://nomad-lab.eu for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Dict mapping custom schema instances from eln_data.yaml file on concepts in NXapm.""" - -NxApmElnInput = {"IGNORE": {"fun": "load_from_dict_list", "terms": "em_lab/detector"}, - "IGNORE": {"fun": "load_from", "terms": "em_lab/ebeam_column/aberration_correction/applied"}, - "IGNORE": {"fun": "load_from_dict_list", "terms": "em_lab/ebeam_column/aperture_em"}, - "/ENTRY[entry*]/PROGRAM[program2]/program": {"fun": "load_from", "terms": "atom_probe/control_software_program"}, - "/ENTRY[entry*]/PROGRAM[program2]/program/@version": {"fun": "load_from", "terms": "atom_probe/control_software_program__attr_version"}, - "/ENTRY[entry*]/experiment_identifier": {"fun": "load_from", "terms": "entry/experiment_identifier"}, - "/ENTRY[entry*]/start_time": {"fun": "load_from", "terms": "entry/start_time"}, - "/ENTRY[entry*]/end_time": {"fun": "load_from", "terms": "entry/end_time"}, - "/ENTRY[entry*]/run_number": {"fun": "load_from", "terms": "entry/run_number"}, - "/ENTRY[entry*]/operation_mode": {"fun": "load_from", "terms": "entry/operation_mode"}, - "/ENTRY[entry*]/experiment_description": {"fun": "load_from", "terms": "entry/experiment_description"}, - "IGNORE": {"fun": "load_from", "terms": "sample/alias"}, - "/ENTRY[entry*]/sample/grain_diameter": {"fun": "load_from", "terms": "sample/grain_diameter/value"}, - "/ENTRY[entry*]/sample/grain_diameter/@units": {"fun": "load_from", "terms": "sample/grain_diameter/unit"}, - "/ENTRY[entry*]/sample/grain_diameter_error": {"fun": "load_from", "terms": "sample/grain_diameter_error/value"}, - "/ENTRY[entry*]/sample/grain_diameter_error/@units": {"fun": "load_from", "terms": "sample/grain_diameter_error/unit"}, - "/ENTRY[entry*]/sample/heat_treatment_quenching_rate": {"fun": "load_from", "terms": "sample/heat_treatment_quenching_rate/value"}, - "/ENTRY[entry*]/sample/heat_treatment_quenching_rate/@units": {"fun": "load_from", "terms": "sample/heat_treatment_quenching_rate/unit"}, - "/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error": {"fun": "load_from", "terms": "sample/heat_treatment_quenching_rate_error/value"}, - "/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error/@units": {"fun": "load_from", "terms": "sample/heat_treatment_quenching_rate_error/unit"}, - "/ENTRY[entry*]/sample/heat_treatment_temperature": {"fun": "load_from", "terms": "sample/heat_treatment_temperature/value"}, - "/ENTRY[entry*]/sample/heat_treatment_temperature/@units": {"fun": "load_from", "terms": "sample/heat_treatment_temperature/unit"}, - "/ENTRY[entry*]/sample/heat_treatment_temperature_error": {"fun": "load_from", "terms": "sample/heat_treatment_temperature_error/value"}, - "/ENTRY[entry*]/sample/heat_treatment_temperature_error/@units": {"fun": "load_from", "terms": "sample/heat_treatment_temperature_error/unit"}, - "/ENTRY[entry*]/specimen/name": {"fun": "load_from", "terms": "specimen/name"}, - "/ENTRY[entry*]/specimen/preparation_date": {"fun": "load_from", "terms": "specimen/preparation_date"}, - "IGNORE": {"fun": "load_from", "terms": "specimen/sample_history"}, - "/ENTRY[entry*]/specimen/alias": {"fun": "load_from", "terms": "specimen/alias"}, - "/ENTRY[entry*]/specimen/is_polycrystalline": {"fun": "load_from", "terms": "specimen/is_polycrystalline"}, - "/ENTRY[entry*]/specimen/description": {"fun": "load_from", "terms": "specimen/description"}, - "/ENTRY[entry*]/atom_probe/FABRICATION[fabrication]/identifier": {"fun": "load_from", "terms": "atom_probe/fabrication_identifier"}, - "/ENTRY[entry*]/atom_probe/FABRICATION[fabrication]/model": {"fun": "load_from", "terms": "atom_probe/fabrication_model"}, - "/ENTRY[entry*]/atom_probe/FABRICATION[fabrication]/vendor": {"fun": "load_from", "terms": "atom_probe/fabrication_vendor"}, - "/ENTRY[entry*]/atom_probe/analysis_chamber/pressure": {"fun": "load_from", "terms": "atom_probe/analysis_chamber_pressure/value"}, - "/ENTRY[entry*]/atom_probe/analysis_chamber/pressure/@units": {"fun": "load_from", "terms": "atom_probe/analysis_chamber_pressure/unit"}, - "/ENTRY[entry*]/atom_probe/control_software/PROGRAM[program1]/program": {"fun": "load_from", "terms": "atom_probe/control_software_program"}, - "/ENTRY[entry*]/atom_probe/control_software/PROGRAM[program1]/program/@version": {"fun": "load_from", "terms": "atom_probe/control_software_program__attr_version"}, - "/ENTRY[entry*]/atom_probe/field_of_view": {"fun": "load_from", "terms": "atom_probe/field_of_view/value"}, - "/ENTRY[entry*]/atom_probe/field_of_view/@units": {"fun": "load_from", "terms": "atom_probe/field_of_view/unit"}, - "/ENTRY[entry*]/atom_probe/flight_path_length": {"fun": "load_from", "terms": "atom_probe/flight_path_length/value"}, - "/ENTRY[entry*]/atom_probe/flight_path_length/@units": {"fun": "load_from", "terms": "atom_probe/flight_path_length/unit"}, - "/ENTRY[entry*]/atom_probe/instrument_name": {"fun": "load_from", "terms": "atom_probe/instrument_name"}, - "/ENTRY[entry*]/atom_probe/ion_detector/model": {"fun": "load_from", "terms": "atom_probe/ion_detector_model"}, - "/ENTRY[entry*]/atom_probe/ion_detector/name": {"fun": "load_from", "terms": "atom_probe/ion_detector_name"}, - "/ENTRY[entry*]/atom_probe/ion_detector/serial_number": {"fun": "load_from", "terms": "atom_probe/ion_detector_serial_number"}, - "/ENTRY[entry*]/atom_probe/ion_detector/type": {"fun": "load_from", "terms": "atom_probe/ion_detector_type"}, - "/ENTRY[entry*]/atom_probe/local_electrode/name": {"fun": "load_from", "terms": "atom_probe/local_electrode_name"}, - "/ENTRY[entry*]/atom_probe/location": {"fun": "load_from", "terms": "atom_probe/location"}, - "/ENTRY[entry*]/atom_probe/REFLECTRON[reflectron]/applied": {"fun": "load_from", "terms": "atom_probe/reflectron_applied"}, - "/ENTRY[entry*]/atom_probe/stage_lab/base_temperature": {"fun": "load_from", "terms": "atom_probe/stage_lab_base_temperature/value"}, - "/ENTRY[entry*]/atom_probe/stage_lab/base_temperature/@units": {"fun": "load_from", "terms": "atom_probe/stage_lab_base_temperature/unit"}, - "/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate": {"fun": "load_from", "terms": "atom_probe/specimen_monitoring_detection_rate/value"}, - "/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate/@units": {"fun": "load_from", "terms": "atom_probe/specimen_monitoring_detection_rate/unit"}, - "/ENTRY[entry*]/atom_probe/specimen_monitoring/initial_radius": {"fun": "load_from", "terms": "atom_probe/specimen_monitoring_initial_radius/value"}, - "/ENTRY[entry*]/atom_probe/specimen_monitoring/initial_radius/@units": {"fun": "load_from", "terms": "atom_probe/specimen_monitoring_initial_radius/unit"}, - "/ENTRY[entry*]/atom_probe/specimen_monitoring/shank_angle": {"fun": "load_from", "terms": "atom_probe/specimen_monitoring_shank_angle/value"}, - "/ENTRY[entry*]/atom_probe/specimen_monitoring/shank_angle/@units": {"fun": "load_from", "terms": "atom_probe/specimen_monitoring_shank_angle/unit"}, - "/ENTRY[entry*]/atom_probe/status": {"fun": "load_from", "terms": "atom_probe/status"}, - "/ENTRY[entry*]/atom_probe/pulser/pulse_fraction": {"fun": "load_from", "terms": "atom_probe/pulser/pulse_fraction"}, - "/ENTRY[entry*]/atom_probe/pulser/pulse_frequency": {"fun": "load_from", "terms": "atom_probe/pulser/pulse_frequency/value"}, - "/ENTRY[entry*]/atom_probe/pulser/pulse_frequency/@units": {"fun": "load_from", "terms": "atom_probe/pulser/pulse_frequency/unit"}, - "/ENTRY[entry*]/atom_probe/pulser/pulse_mode": {"fun": "load_from", "terms": "atom_probe/pulser/pulse_mode"}, - "/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program": {"fun": "load_from", "terms": "atom_probe/ranging/program"}, - "/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program/@version": {"fun": "load_from", "terms": "atom_probe/ranging/program__attr_version"}, - "/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program": {"fun": "load_from", "terms": "atom_probe/reconstruction/program"}, - "/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program/@version": {"fun": "load_from", "terms": "atom_probe/reconstruction/program__attr_version"}, - "/ENTRY[entry*]/atom_probe/reconstruction/crystallographic_calibration": {"fun": "load_from", "terms": "atom_probe/reconstruction/crystallographic_calibration"}, - "/ENTRY[entry*]/atom_probe/reconstruction/parameter": {"fun": "load_from", "terms": "atom_probe/reconstruction/parameter"}, - "/ENTRY[entry*]/atom_probe/reconstruction/protocol_name": {"fun": "load_from", "terms": "atom_probe/reconstruction/protocol_name"}} - -# NeXus concept specific mapping tables which require special treatment as the current -# NOMAD OASIS custom schema implementation delivers them as a list of dictionaries instead -# of a directly flattenable list of keyword, value pairs - -NxUserFromListOfDict = {"/ENTRY[entry*]/USER[user*]/name": {"fun": "load_from", "terms": "name"}, - "/ENTRY[entry*]/USER[user*]/affiliation": {"fun": "load_from", "terms": "affiliation"}, - "/ENTRY[entry*]/USER[user*]/address": {"fun": "load_from", "terms": "address"}, - "/ENTRY[entry*]/USER[user*]/email": {"fun": "load_from", "terms": "email"}, - "/ENTRY[entry*]/USER[user*]/orcid": {"fun": "load_from", "terms": "orcid"}, - "/ENTRY[entry*]/USER[user*]/orcid_platform": {"fun": "load_from", "terms": "orcid_platform"}, - "/ENTRY[entry*]/USER[user*]/telephone_number": {"fun": "load_from", "terms": "telephone_number"}, - "/ENTRY[entry*]/USER[user*]/role": {"fun": "load_from", "terms": "role"}, - "/ENTRY[entry*]/USER[user*]/social_media_name": {"fun": "load_from", "terms": "social_media_name"}, - "/ENTRY[entry*]/USER[user*]/social_media_platform": {"fun": "load_from", "terms": "social_media_platform"}} - -# LEAP6000 can use up to two lasers and voltage pulsing (both at the same time?) -NxPulserFromListOfDict = {"/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/name": {"fun": "load_from", "terms": "name"}, - "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/power": {"fun": "load_from", "terms": "power"}, - "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/pulse_energy": {"fun": "load_from", "terms": "pulse_energy"}, - "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/wavelength": {"fun": "load_from", "terms": "wavelength"}} diff --git a/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py b/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py new file mode 100644 index 000000000..291c8c49e --- /dev/null +++ b/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py @@ -0,0 +1,106 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Dict mapping custom schema instances from eln_data.yaml file on concepts in NXapm.""" + +# "/ENTRY[entry*]/PROGRAM[program2]/program": "load_from", "atom_probe/control_software_program"), +# "/ENTRY[entry*]/PROGRAM[program2]/program/@version": "load_from", "atom_probe/control_software_program__attr_version"), +# ("/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate", "load_from", "atom_probe/specimen_monitoring_detection_rate/value"), +# "/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate/@units", "load_from", "atom_probe/specimen_monitoring_detection_rate/unit"), +# "/ENTRY[entry*]/atom_probe/control_software/PROGRAM[program1]/program", "load_from", "atom_probe/control_software_program"), +# "/ENTRY[entry*]/atom_probe/control_software/PROGRAM[program1]/program/@version", "load_from", "atom_probe/control_software_program__attr_version"), + + +APM_EXAMPLE_TO_NEXUS \ + = [("/ENTRY[entry*]/experiment_identifier", "load_from", "entry/experiment_identifier"), + ("/ENTRY[entry*]/start_time", "load_from", "entry/start_time"), + ("/ENTRY[entry*]/end_time", "load_from", "entry/end_time"), + ("/ENTRY[entry*]/run_number", "load_from", "entry/run_number"), + ("ENTRY[entry*]/operation_mode", "load_from", "entry/operation_mode"), + ("/ENTRY[entry*]/experiment_description", "load_from", "entry/experiment_description"), + ("/ENTRY[entry*]/sample/method", "load_from", "sample/method"), + ("/ENTRY[entry*]/sample/alias", "load_from", "sample/alias"), + ("/ENTRY[entry*]/sample/grain_diameter", "load_from", "sample/grain_diameter/value"), + ("/ENTRY[entry*]/sample/grain_diameter/@units", "load_from", "sample/grain_diameter/unit"), + ("/ENTRY[entry*]/sample/grain_diameter_error", "load_from", "sample/grain_diameter_error/value"), + ("/ENTRY[entry*]/sample/grain_diameter_error/@units", "load_from", "sample/grain_diameter_error/unit"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate", "load_from", "sample/heat_treatment_quenching_rate/value"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate/@units", "load_from", "sample/heat_treatment_quenching_rate/unit"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error", "load_from", "sample/heat_treatment_quenching_rate_error/value"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error/@units", "load_from", "sample/heat_treatment_quenching_rate_error/unit"), + ("/ENTRY[entry*]/sample/heat_treatment_temperature", "load_from", "sample/heat_treatment_temperature/value"), + ("/ENTRY[entry*]/sample/heat_treatment_temperature/@units", "load_from", "sample/heat_treatment_temperature/unit"), + ("/ENTRY[entry*]/sample/heat_treatment_temperature_error", "load_from", "sample/heat_treatment_temperature_error/value"), + ("/ENTRY[entry*]/sample/heat_treatment_temperature_error/@units", "load_from", "sample/heat_treatment_temperature_error/unit"), + ("/ENTRY[entry*]/sample/description", "load_from", "sample/description"), + ("/ENTRY[entry*]/specimen/alias", "load_from", "specimen/alias"), + ("/ENTRY[entry*]/specimen/preparation_date", "load_from", "specimen/preparation_date"), + ("/ENTRY[entry*]/specimen/description", "load_from", "specimen/description"), + ("/ENTRY[entry*]/specimen/is_polycrystalline", "load_from", "specimen/is_polycrystalline"), + ("/ENTRY[entry*]/specimen/is_amorphous", "load_from", "specimen/is_amorphous"), + ("/ENTRY[entry*]/specimen/initial_radius", "load_from", "specimen/initial_radius/value"), + ("/ENTRY[entry*]/specimen/initial_radius/@units", "load_from", "specimen/initial_radius/unit"), + ("/ENTRY[entry*]/specimen/shank_angle", "load_from", "specimen/shank_angle/value"), + ("/ENTRY[entry*]/specimen/shank_angle/@units", "load_from", "specimen/shank_angle/unit"), + ("/ENTRY[entry*]/measurement/instrument/instrument_name", "load_from", "atom_probe/instrument_name"), + ("/ENTRY[entry*]/measurement/instrument/location", "load_from", "atom_probe/location"), + ("/ENTRY[entry*]/measurement/instrument/FABRICATION[fabrication]/vendor", "load_from", "atom_probe/fabrication_vendor"), + ("/ENTRY[entry*]/measurement/instrument/FABRICATION[fabrication]/model", "load_from", "atom_probe/fabrication_model"), + ("/ENTRY[entry*]/measurement/instrument/FABRICATION[fabrication]/identifier", "load_from", "atom_probe/fabrication_identifier"), + ("/ENTRY[entry*]/measurement/instrument/reflectron/status", "load_from", "atom_probe/reflectron_status"), + ("/ENTRY[entry*]/measurement/instrument/local_electrode/name", "load_from", "atom_probe/local_electrode_name"), + ("/ENTRY[entry*]/measurement/instrument/pulser/pulse_mode", "load_from", "atom_probe/pulser/pulse_mode"), + ("/ENTRY[entry*]/measurement/instrument/analysis_chamber/flight_path", "load_from", "atom_probe/flight_path_length/value"), + ("/ENTRY[entry*]/measurement/instrument/analysis_chamber/flight_path/@units", "load_from", "atom_probe/flight_path_length/unit"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/pulse_frequency", "load_from", "atom_probe/pulser/pulse_frequency/value"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/pulse_frequency/@units", "load_from", "atom_probe/pulser/pulse_frequency/unit"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/pulse_fraction", "load_from", "atom_probe/pulser/pulse_fraction"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/average_pressure", "load_from", "atom_probe/analysis_chamber_pressure/value"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/average_pressure/@units", "load_from", "atom_probe/analysis_chamber_pressure/unit"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/average_temperature", "load_from", "atom_probe/stage_lab_base_temperature/value"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/average_temperature/@units", "load_from", "atom_probe/stage_lab_base_temperature/unit"), + ("/ENTRY[entry*]/atom_probe/status", "load_from", "atom_probe/status"), + ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program", "load_from", "atom_probe/ranging/program"), + ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program/@version", "load_from", "atom_probe/ranging/program__attr_version"), + ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program", "load_from", "atom_probe/reconstruction/program"), + ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program/@version", "load_from", "atom_probe/reconstruction/program__attr_version"), + ("/ENTRY[entry*]/atom_probe/reconstruction/protocol_name", "load_from", "atom_probe/reconstruction/protocol_name"), + ("/ENTRY[entry*]/atom_probe/reconstruction/crystallographic_calibration", "load_from", "atom_probe/reconstruction/crystallographic_calibration"), + ("/ENTRY[entry*]/atom_probe/reconstruction/parameter", "load_from", "atom_probe/reconstruction/parameter"), + ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view", "load_from", "atom_probe/reconstruction/field_of_view/value"), + ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view/@units", "load_from", "atom_probe/reconstruction/field_of_view/unit")] + +# NeXus concept specific mapping tables which require special treatment as the current +# NOMAD OASIS custom schema implementation delivers them as a list of dictionaries instead +# of a directly flattenable list of keyword, value pairs + +NxUserFromListOfDict = {"/ENTRY[entry*]/USER[user*]/name", "load_from", "name"), + "/ENTRY[entry*]/USER[user*]/affiliation", "load_from", "affiliation"), + "/ENTRY[entry*]/USER[user*]/address", "load_from", "address"), + "/ENTRY[entry*]/USER[user*]/email", "load_from", "email"), + "/ENTRY[entry*]/USER[user*]/orcid", "load_from", "orcid"), + "/ENTRY[entry*]/USER[user*]/orcid_platform", "load_from", "orcid_platform"), + "/ENTRY[entry*]/USER[user*]/telephone_number", "load_from", "telephone_number"), + "/ENTRY[entry*]/USER[user*]/role", "load_from", "role"), + "/ENTRY[entry*]/USER[user*]/social_media_name", "load_from", "social_media_name"), + "/ENTRY[entry*]/USER[user*]/social_media_platform", "load_from", "social_media_platform"}} + +# LEAP6000 can use up to two lasers and voltage pulsing (both at the same time?) +NxPulserFromListOfDict = {"/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/name", "load_from", "name"), + "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/power", "load_from", "power"), + "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/pulse_energy", "load_from", "pulse_energy"), + "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/wavelength", "load_from", "wavelength"}} diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py index 869d3d967..d3f2034c4 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py @@ -23,7 +23,7 @@ import yaml from ase.data import chemical_symbols -from pynxtools.dataconverter.readers.apm.map_concepts.apm_eln_to_nx_map \ +from pynxtools.dataconverter.readers.apm.map_concepts.apm_example_eln_to_nx_map \ import NxApmElnInput, NxUserFromListOfDict from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ import variadic_path_to_specific_path, apply_modifier From b5ec396abc50ed8edfe914961090fbb4fef29b0e Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 9 Jan 2024 11:59:20 +0100 Subject: [PATCH 09/15] Added support for adding references to proprietary/Cameca/non-Cameca workflow-related files in the NeXus file so that NOMAD is available could use them further, also added feedback of best practice metadata to report from T. Blum et al. from the geoscience community, next steps: i) fix eln parsing, ii) add 176case study specific citation code, iii) customize nomad schema, iv) runthrough --- debug/eln_data_apm.yaml | 76 +++---- dev-requirements.txt | 200 +++++++++++++++++- .../map_concepts/apm_example_eln_to_nx_map.py | 93 +++++--- 3 files changed, 290 insertions(+), 79 deletions(-) diff --git a/debug/eln_data_apm.yaml b/debug/eln_data_apm.yaml index ddd67ebcf..90d0ea28f 100644 --- a/debug/eln_data_apm.yaml +++ b/debug/eln_data_apm.yaml @@ -1,23 +1,8 @@ atom_probe: - analysis_chamber_pressure: - unit: torr - value: 2.0e-10 - control_software_program: IVAS - control_software_program__attr_version: 3.6.8 fabrication_identifier: '12' fabrication_model: LEAP3000 fabrication_vendor: Cameca - field_of_view: - unit: nm - value: 20. - flight_path_length: - unit: m - value: 1.2 instrument_name: LEAP - ion_detector_model: n/a - ion_detector_name: n/a - ion_detector_serial_number: n/a - ion_detector_type: mcp_dld local_electrode_name: L1 location: Denton pulser: @@ -47,17 +32,18 @@ atom_probe: unit: kHz value: 250.0 pulse_mode: laser_and_voltage - reflectron_applied: true - specimen_monitoring_detection_rate: 0.8 - specimen_monitoring_initial_radius: - unit: nm - value: 12.0 - specimen_monitoring_shank_angle: - unit: ° - value: 5.0 - stage_lab_base_temperature: + reflectron_status: used + evaporation_control: detection_rate + target_detection_rate: 0.01 + chamber_pressure: + unit: torr + value: 2.0e-10 + base_temperature: unit: K value: 20.0 + nominal_flight_path: + unit: m + value: 1.2 status: success entry: experiment_description: '

Normal

@@ -65,54 +51,72 @@ entry:

Bold

Italics

' - experiment_identifier: Si test start_time: '2023-06-11T11:20:00+00:00' end_time: '2023-06-11T11:20:00+00:00' run_number: '2121' operation_mode: apt +workflow: + raw_dat_file: str.str + hit_dat_file: hits.hits + recon_cfg_file: hits.root + recon_res_file: recon.apt + range_dat_file: recon.rrng ranging: program: IVAS - program__attr_version: 3.6.8 + program_version: 3.6.8 reconstruction: + program: IVAS + program_version: 3.6.8 crystallographic_calibration: n/a parameter: kf = 1.8, icf = 3.3 - program: IVAS - program__attr_version: 3.6.8 protocol_name: bas + field_of_view: + unit: nm + value: 20. sample: composition: - Mo - Al 12 +- 3 - B 50 ppm +- 12 - C 3.6 + method: measurement + alias: Si grain_diameter: unit: µm value: 200.0 grain_diameter_error: unit: µm value: 50.0 - heat_treatment_quenching_rate: - unit: K / s - value: 150.0 - heat_treatment_quenching_rate_error: - unit: K / s - value: 10.0 heat_treatment_temperature: unit: K value: 600.0 heat_treatment_temperature_error: unit: K value: 20.0 + heat_treatment_quenching_rate: + unit: K / s + value: 150.0 + heat_treatment_quenching_rate_error: + unit: K / s + value: 10.0 + description: '

nothing

' specimen: - alias: Si + method: measurement + alias: usa_denton_smith_si description: '

normal

bold

italics

' is_polycrystalline: true - name: usa_denton_smith_si + is_amorphous: false preparation_date: '2023-06-11T12:51:00+00:00' + initial_radius: + unit: nm + value: 12.0 + shank_angle: + unit: ° + value: 5.0 user: - {} - {} diff --git a/dev-requirements.txt b/dev-requirements.txt index dd12c799b..2a07cd3af 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,6 +4,14 @@ # # pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml # +anyio==4.2.0 + # via jupyter-server +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration asciitree==0.3.3 # via zarr ase==3.22.1 @@ -16,12 +24,22 @@ astroid==3.0.1 # via pylint asttokens==2.4.1 # via stack-data +async-lru==2.0.4 + # via jupyterlab attrs==23.1.0 # via # cattrs + # jsonschema + # referencing # requests-cache +babel==2.14.0 + # via jupyterlab-server backcall==0.2.0 # via ipython +beautifulsoup4==4.12.2 + # via nbconvert +bleach==6.1.0 + # via nbconvert blosc2==2.0.0 # via tables build==1.0.3 @@ -30,11 +48,14 @@ cattrs==23.2.2 # via requests-cache certifi==2023.11.17 # via requests +cffi==1.16.0 + # via argon2-cffi-bindings charset-normalizer==3.3.2 # via requests click==8.1.7 # via # dask + # nyaml # pip-tools # pynxtools (pyproject.toml) cloudpickle==3.0.0 @@ -66,7 +87,9 @@ decorator==5.1.1 # ipyparallel # ipython defusedxml==0.7.1 - # via odfpy + # via + # nbconvert + # odfpy diffpy-structure==3.1.0 # via # diffsims @@ -84,6 +107,7 @@ entrypoints==0.4 # via ipyparallel exceptiongroup==1.2.0 # via + # anyio # cattrs # pytest executing==2.0.1 @@ -94,10 +118,14 @@ fabio==2023.10.0 # silx fasteners==0.19 # via zarr +fastjsonschema==2.19.1 + # via nbformat flatdict==4.0.1 # via pynxtools (pyproject.toml) fonttools==4.45.1 # via matplotlib +fqdn==1.5.1 + # via jsonschema fsspec==2023.10.0 # via # dask @@ -108,12 +136,16 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via pynxtools (pyproject.toml) +h5grove==1.3.0 + # via jupyterlab-h5web h5py==3.10.0 # via # fabio + # h5grove # hdf5plugin # hyperspy # ifes-apt-tc-data-modeling + # jupyterlab-h5web # kikuchipy # nionswift # orix @@ -129,7 +161,10 @@ hyperspy==1.7.6 # pynxtools (pyproject.toml) # pyxem idna==3.6 - # via requests + # via + # anyio + # jsonschema + # requests ifes-apt-tc-data-modeling==0.1 # via pynxtools (pyproject.toml) imageio==2.27.0 @@ -147,7 +182,9 @@ importlib-metadata==6.8.0 iniconfig==2.0.0 # via pytest ipykernel==6.27.1 - # via ipyparallel + # via + # ipyparallel + # jupyterlab ipyparallel==8.6.1 # via hyperspy ipython==8.12.3 @@ -155,22 +192,68 @@ ipython==8.12.3 # hyperspy # ipykernel # ipyparallel +isoduration==20.11.0 + # via jsonschema isort==5.12.0 # via pylint jedi==0.19.1 # via ipython jinja2==3.1.2 - # via hyperspy + # via + # hyperspy + # jupyter-server + # jupyterlab + # jupyterlab-server + # nbconvert joblib==1.3.2 # via scikit-learn +json5==0.9.14 + # via jupyterlab-server +jsonpointer==2.4 + # via jsonschema +jsonschema[format-nongpl]==4.20.0 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema jupyter-client==8.6.0 # via # ipykernel # ipyparallel + # jupyter-server + # nbclient jupyter-core==5.5.0 # via # ipykernel # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.9.0 + # via jupyter-server +jupyter-lsp==2.2.1 + # via jupyterlab +jupyter-server==2.12.2 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-h5web + # jupyterlab-server + # notebook-shim +jupyter-server-terminals==0.5.1 + # via jupyter-server +jupyterlab==4.0.10 + # via pynxtools (pyproject.toml) +jupyterlab-h5web==11.0.0 + # via pynxtools (pyproject.toml) +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.25.2 + # via jupyterlab kikuchipy==0.9.0 # via pynxtools (pyproject.toml) kiwisolver==1.4.5 @@ -186,9 +269,13 @@ lmfit==1.2.2 locket==1.0.0 # via partd lxml==4.9.3 - # via fabio + # via + # fabio + # nyaml markupsafe==2.1.3 - # via jinja2 + # via + # jinja2 + # nbconvert matplotlib==3.7.4 # via # ase @@ -210,6 +297,8 @@ mccabe==0.7.0 # via pylint mergedeep==1.3.4 # via pynxtools (pyproject.toml) +mistune==3.0.2 + # via nbconvert mpmath==1.3.0 # via sympy msgpack==1.0.7 @@ -220,6 +309,15 @@ mypy-extensions==1.0.0 # via mypy natsort==8.4.0 # via hyperspy +nbclient==0.9.0 + # via nbconvert +nbconvert==7.14.0 + # via jupyter-server +nbformat==5.9.2 + # via + # jupyter-server + # nbclient + # nbconvert nest-asyncio==1.5.8 # via ipykernel networkx==3.1 @@ -242,6 +340,8 @@ nionutils==0.4.6 # nionswift # nionswift-io # nionui +notebook-shim==0.2.3 + # via jupyterlab numba==0.58.1 # via # diffsims @@ -265,6 +365,7 @@ numpy==1.24.4 # dask # diffsims # fabio + # h5grove # h5py # hyperspy # ifes-apt-tc-data-modeling @@ -300,6 +401,8 @@ numpy==1.24.4 # zarr numpy-quaternion==2022.4.3 # via orix +nyaml==0.0.6 + # via pynxtools (pyproject.toml) odfpy==1.4.1 # via pynxtools (pyproject.toml) orix==0.11.1 @@ -307,13 +410,21 @@ orix==0.11.1 # diffsims # kikuchipy # pyxem +orjson==3.9.10 + # via h5grove +overrides==7.4.0 + # via jupyter-server packaging==23.2 # via # build # dask # hyperspy # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server # matplotlib + # nbconvert # pooch # pytest # scikit-image @@ -324,6 +435,8 @@ pandas==2.0.3 # ifes-apt-tc-data-modeling # pynxtools (pyproject.toml) # xarray +pandocfilters==1.5.0 + # via nbconvert parso==0.8.3 # via jedi partd==1.4.1 @@ -362,6 +475,8 @@ pooch==1.8.0 # orix prettytable==3.9.0 # via hyperspy +prometheus-client==0.19.0 + # via jupyter-server prompt-toolkit==3.0.41 # via ipython psutil==5.9.6 @@ -371,7 +486,9 @@ psutil==5.9.6 # ipyparallel # pyxem ptyprocess==0.7.0 - # via pexpect + # via + # pexpect + # terminado pure-eval==0.2.2 # via stack-data py-cpuinfo==9.0.0 @@ -380,10 +497,14 @@ pycifrw==4.4.6 # via diffpy-structure pycodestyle==2.11.1 # via pynxtools (pyproject.toml) +pycparser==2.21 + # via cffi pyfai==2023.9.0 # via pyxem pygments==2.17.2 - # via ipython + # via + # ipython + # nbconvert pylint==3.0.2 # via pynxtools (pyproject.toml) pyparsing==3.1.1 @@ -403,12 +524,15 @@ python-box==6.1.0 # via rosettasciio python-dateutil==2.8.2 # via + # arrow # hyperspy # ipyparallel # jupyter-client # matplotlib # pandas # rosettasciio +python-json-logger==2.0.7 + # via jupyter-events pytz==2023.3.post1 # via # nionswift @@ -424,7 +548,9 @@ pyyaml==6.0.1 # via # dask # hyperspy + # jupyter-events # kikuchipy + # nyaml # pynxtools (pyproject.toml) # rosettasciio pyzmq==25.1.1 @@ -432,18 +558,37 @@ pyzmq==25.1.1 # ipykernel # ipyparallel # jupyter-client + # jupyter-server radioactivedecay==0.4.21 # via ifes-apt-tc-data-modeling +referencing==0.32.1 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events requests==2.31.0 # via # hyperspy + # jupyterlab-server # pooch # pynxtools (pyproject.toml) # requests-cache requests-cache==1.1.1 # via pynxtools (pyproject.toml) +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events rosettasciio==0.2 # via pynxtools (pyproject.toml) +rpds-py==0.16.2 + # via + # jsonschema + # referencing scikit-image==0.20.0 # via # hyperspy @@ -470,16 +615,24 @@ scipy==1.9.1 # scikit-image # scikit-learn # sparse +send2trash==1.8.2 + # via jupyter-server silx==1.1.2 # via pyfai six==1.16.0 # via # asttokens + # bleach # diffpy-structure # python-dateutil + # rfc3339-validator # url-normalize smmap==5.0.1 # via gitdb +sniffio==1.3.0 + # via anyio +soupsieve==2.5 + # via beautifulsoup4 sparse==0.14.0 # via hyperspy stack-data==0.6.3 @@ -492,16 +645,24 @@ sympy==1.12 # radioactivedecay tables==3.8.0 # via ifes-apt-tc-data-modeling +terminado==0.18.0 + # via + # jupyter-server + # jupyter-server-terminals threadpoolctl==3.2.0 # via scikit-learn tifffile==2023.7.10 # via + # h5grove # hyperspy # scikit-image +tinycss2==1.2.1 + # via nbconvert tomli==2.0.1 # via # build # coverage + # jupyterlab # mypy # pip-tools # pylint @@ -519,6 +680,9 @@ tornado==6.3.3 # ipykernel # ipyparallel # jupyter-client + # jupyter-server + # jupyterlab + # terminado tqdm==4.66.1 # via # diffsims @@ -535,7 +699,13 @@ traitlets==5.14.0 # ipython # jupyter-client # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab # matplotlib-inline + # nbclient + # nbconvert + # nbformat traits==6.4.3 # via # hyperspy @@ -544,6 +714,8 @@ transforms3d==0.4.1 # via # diffsims # pyxem +types-python-dateutil==2.8.19.20240106 + # via arrow types-pytz==2023.3.1.1 # via pynxtools (pyproject.toml) types-pyyaml==6.0.12.12 @@ -552,7 +724,9 @@ types-requests==2.31.0.10 # via pynxtools (pyproject.toml) typing-extensions==4.8.0 # via + # anyio # astroid + # async-lru # cattrs # mypy tzdata==2023.3 @@ -565,6 +739,8 @@ tzlocal==4.3 # pynxtools (pyproject.toml) uncertainties==3.1.7 # via lmfit +uri-template==1.3.0 + # via jsonschema url-normalize==1.4.3 # via requests-cache urllib3==2.1.0 @@ -576,6 +752,14 @@ wcwidth==0.2.12 # via # prettytable # prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.7.0 + # via jupyter-server wheel==0.42.0 # via pip-tools xarray==2023.1.0 diff --git a/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py b/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py index 291c8c49e..a47e5fd30 100644 --- a/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py +++ b/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py @@ -21,16 +21,13 @@ # "/ENTRY[entry*]/PROGRAM[program2]/program/@version": "load_from", "atom_probe/control_software_program__attr_version"), # ("/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate", "load_from", "atom_probe/specimen_monitoring_detection_rate/value"), # "/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate/@units", "load_from", "atom_probe/specimen_monitoring_detection_rate/unit"), -# "/ENTRY[entry*]/atom_probe/control_software/PROGRAM[program1]/program", "load_from", "atom_probe/control_software_program"), -# "/ENTRY[entry*]/atom_probe/control_software/PROGRAM[program1]/program/@version", "load_from", "atom_probe/control_software_program__attr_version"), -APM_EXAMPLE_TO_NEXUS \ - = [("/ENTRY[entry*]/experiment_identifier", "load_from", "entry/experiment_identifier"), +APM_EXAMPLE_ENTRIES_TO_NEXUS \ + = [("/ENTRY[entry*]/run_number", "load_from", "entry/run_number"), ("/ENTRY[entry*]/start_time", "load_from", "entry/start_time"), ("/ENTRY[entry*]/end_time", "load_from", "entry/end_time"), - ("/ENTRY[entry*]/run_number", "load_from", "entry/run_number"), - ("ENTRY[entry*]/operation_mode", "load_from", "entry/operation_mode"), + ("/ENTRY[entry*]/operation_mode", "load_from", "entry/operation_mode"), ("/ENTRY[entry*]/experiment_description", "load_from", "entry/experiment_description"), ("/ENTRY[entry*]/sample/method", "load_from", "sample/method"), ("/ENTRY[entry*]/sample/alias", "load_from", "sample/alias"), @@ -38,15 +35,16 @@ ("/ENTRY[entry*]/sample/grain_diameter/@units", "load_from", "sample/grain_diameter/unit"), ("/ENTRY[entry*]/sample/grain_diameter_error", "load_from", "sample/grain_diameter_error/value"), ("/ENTRY[entry*]/sample/grain_diameter_error/@units", "load_from", "sample/grain_diameter_error/unit"), - ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate", "load_from", "sample/heat_treatment_quenching_rate/value"), - ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate/@units", "load_from", "sample/heat_treatment_quenching_rate/unit"), - ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error", "load_from", "sample/heat_treatment_quenching_rate_error/value"), - ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error/@units", "load_from", "sample/heat_treatment_quenching_rate_error/unit"), ("/ENTRY[entry*]/sample/heat_treatment_temperature", "load_from", "sample/heat_treatment_temperature/value"), ("/ENTRY[entry*]/sample/heat_treatment_temperature/@units", "load_from", "sample/heat_treatment_temperature/unit"), ("/ENTRY[entry*]/sample/heat_treatment_temperature_error", "load_from", "sample/heat_treatment_temperature_error/value"), ("/ENTRY[entry*]/sample/heat_treatment_temperature_error/@units", "load_from", "sample/heat_treatment_temperature_error/unit"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate", "load_from", "sample/heat_treatment_quenching_rate/value"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate/@units", "load_from", "sample/heat_treatment_quenching_rate/unit"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error", "load_from", "sample/heat_treatment_quenching_rate_error/value"), + ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error/@units", "load_from", "sample/heat_treatment_quenching_rate_error/unit"), ("/ENTRY[entry*]/sample/description", "load_from", "sample/description"), + ("/ENTRY[entry*]/specimen/method", "load_from", "specimen/method"), ("/ENTRY[entry*]/specimen/alias", "load_from", "specimen/alias"), ("/ENTRY[entry*]/specimen/preparation_date", "load_from", "specimen/preparation_date"), ("/ENTRY[entry*]/specimen/description", "load_from", "specimen/description"), @@ -64,43 +62,68 @@ ("/ENTRY[entry*]/measurement/instrument/reflectron/status", "load_from", "atom_probe/reflectron_status"), ("/ENTRY[entry*]/measurement/instrument/local_electrode/name", "load_from", "atom_probe/local_electrode_name"), ("/ENTRY[entry*]/measurement/instrument/pulser/pulse_mode", "load_from", "atom_probe/pulser/pulse_mode"), - ("/ENTRY[entry*]/measurement/instrument/analysis_chamber/flight_path", "load_from", "atom_probe/flight_path_length/value"), - ("/ENTRY[entry*]/measurement/instrument/analysis_chamber/flight_path/@units", "load_from", "atom_probe/flight_path_length/unit"), + ("/ENTRY[entry*]/measurement/instrument/analysis_chamber/flight_path", "load_from", "atom_probe/nominal_flight_path/value"), + ("/ENTRY[entry*]/measurement/instrument/analysis_chamber/flight_path/@units", "load_from", "atom_probe/nominal_flight_path/unit"), + ("/ENTRY[entry*]/measurement/instrument/status", "load_from", "atom_probe/status"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/control/evaporation_control", "load_from", "atom_probe/evaporation_control"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/control/target_detection_rate", "load_from", "atom_probe/target_detection_rate"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/control/target_detection_rate/@units", "ions/pulse"), ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/pulse_frequency", "load_from", "atom_probe/pulser/pulse_frequency/value"), ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/pulse_frequency/@units", "load_from", "atom_probe/pulser/pulse_frequency/unit"), ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/pulse_fraction", "load_from", "atom_probe/pulser/pulse_fraction"), - ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/average_pressure", "load_from", "atom_probe/analysis_chamber_pressure/value"), - ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/average_pressure/@units", "load_from", "atom_probe/analysis_chamber_pressure/unit"), - ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/average_temperature", "load_from", "atom_probe/stage_lab_base_temperature/value"), - ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/average_temperature/@units", "load_from", "atom_probe/stage_lab_base_temperature/unit"), - ("/ENTRY[entry*]/atom_probe/status", "load_from", "atom_probe/status"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/chamber_pressure", "load_from", "atom_probe/chamber_pressure/value"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/chamber_pressure/@units", "load_from", "atom_probe/chamber_pressure/unit"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/base_temperature", "load_from", "atom_probe/base_temperature/value"), + ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/base_temperature/@units", "load_from", "atom_probe/base_temperature/unit"), ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program", "load_from", "atom_probe/ranging/program"), - ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program/@version", "load_from", "atom_probe/ranging/program__attr_version"), + ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program/@version", "load_from", "atom_probe/ranging/program_version"), ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program", "load_from", "atom_probe/reconstruction/program"), - ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program/@version", "load_from", "atom_probe/reconstruction/program__attr_version"), + ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program/@version", "load_from", "atom_probe/reconstruction/program_version"), ("/ENTRY[entry*]/atom_probe/reconstruction/protocol_name", "load_from", "atom_probe/reconstruction/protocol_name"), ("/ENTRY[entry*]/atom_probe/reconstruction/crystallographic_calibration", "load_from", "atom_probe/reconstruction/crystallographic_calibration"), ("/ENTRY[entry*]/atom_probe/reconstruction/parameter", "load_from", "atom_probe/reconstruction/parameter"), ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view", "load_from", "atom_probe/reconstruction/field_of_view/value"), - ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view/@units", "load_from", "atom_probe/reconstruction/field_of_view/unit")] + ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view/@units", "load_from", "atom_probe/reconstruction/field_of_view/unit"), + ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/type", "file"), + ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/path", "load_from", "workflow/raw_dat_file"), + ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/checksum", "sha256", "workflow/raw_dat_file"), + ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/algorithm", "SHA256"), + ("/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]/type", "file"), + ("/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]/path", "load_from", "workflow/hit_dat_file"), + ("/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]/checksum", "sha256", "workflow/hit_dat_file"), + ("/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]/algorithm", "SHA256"), + ("/ENTRY[entry*]/atom_probe/reconstruction/config/type", "file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/config/path", "load_from", "workflow/recon_cfg_file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/config/checksum", "sha256", "workflow/recon_cfg_file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/config/algorithm", "SHA256"), + ("/ENTRY[entry*]/atom_probe/reconstruction/results/type", "file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/results/path", "load_from", "workflow/recon_res_file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/results/checksum", "sha256", "workflow/recon_res_file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/results/algorithm", "SHA256"), + ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/type", "file"), + ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/path", "load_from", "workflow/range_dat_file"), + ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/checksum", "sha256", "workflow/range_dat_file"), + ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/algorithm", "SHA256")] # NeXus concept specific mapping tables which require special treatment as the current # NOMAD OASIS custom schema implementation delivers them as a list of dictionaries instead -# of a directly flattenable list of keyword, value pairs +# of a directly flattenable list of key, value pairs -NxUserFromListOfDict = {"/ENTRY[entry*]/USER[user*]/name", "load_from", "name"), - "/ENTRY[entry*]/USER[user*]/affiliation", "load_from", "affiliation"), - "/ENTRY[entry*]/USER[user*]/address", "load_from", "address"), - "/ENTRY[entry*]/USER[user*]/email", "load_from", "email"), - "/ENTRY[entry*]/USER[user*]/orcid", "load_from", "orcid"), - "/ENTRY[entry*]/USER[user*]/orcid_platform", "load_from", "orcid_platform"), - "/ENTRY[entry*]/USER[user*]/telephone_number", "load_from", "telephone_number"), - "/ENTRY[entry*]/USER[user*]/role", "load_from", "role"), - "/ENTRY[entry*]/USER[user*]/social_media_name", "load_from", "social_media_name"), - "/ENTRY[entry*]/USER[user*]/social_media_platform", "load_from", "social_media_platform"}} +APM_EXAMPLE_USER_TO_NEXUS \ + = [("/ENTRY[entry*]/USER[user*]/name", "load_from", "name"), + ("/ENTRY[entry*]/USER[user*]/affiliation", "load_from", "affiliation"), + ("/ENTRY[entry*]/USER[user*]/address", "load_from", "address"), + ("/ENTRY[entry*]/USER[user*]/email", "load_from", "email"), + ("/ENTRY[entry*]/USER[user*]/orcid", "load_from", "orcid"), + ("/ENTRY[entry*]/USER[user*]/orcid_platform", "load_from", "orcid_platform"), + ("/ENTRY[entry*]/USER[user*]/telephone_number", "load_from", "telephone_number"), + ("/ENTRY[entry*]/USER[user*]/role", "load_from", "role"), + ("/ENTRY[entry*]/USER[user*]/social_media_name", "load_from", "social_media_name"), + ("/ENTRY[entry*]/USER[user*]/social_media_platform", "load_from", "social_media_platform")] # LEAP6000 can use up to two lasers and voltage pulsing (both at the same time?) -NxPulserFromListOfDict = {"/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/name", "load_from", "name"), - "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/power", "load_from", "power"), - "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/pulse_energy", "load_from", "pulse_energy"), - "/ENTRY[entry*]/atom_probe/pulser/SOURCE[source*]/wavelength", "load_from", "wavelength"}} +# ("/ENTRY[entry*]/measurement/instrument/pulser/SOURCE[source*]/name", "load_from", "atom_probe/pulser/source_name"), +# ("/ENTRY[entry*]/measurement/instrument/pulser/SOURCE[source*]/wavelength", "load_from", "atom_probe/pulser/source_wavelength/value"), +# ("/ENTRY[entry*]/measurement/instrument/pulser/SOURCE[source*]/wavelength/@units", "load_from", "atom_probe/pulser/source_wavelength/unit"), +# ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/SOURCE[source*]/pulse_energy", "load_from", "atom_probe/pulser/source_pulse_energy/value"), +# ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/pulser/SOURCE[source*]/pulse_energy/@units", "load_from", "atom_probe/pulser/source_pulse_energy/unit"), From 56a42ac75bc109e1466145599f1f738e1c511376 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 10 Jan 2024 14:38:49 +0100 Subject: [PATCH 10/15] Added support for being able to incorporate D. R. Diercks et al. APT mass spectrum database which is currently hosted on an AirTable (but no longer maintained), added ELN and I/O logic parsing to populate NXserialized artifacts which can be used for implementing a visualization of a information provenance (workflow) for NOMAD OASIS, added ELN parsing, fix i) except for user --- debug/batch_queue.04.sh | 3 +- debug/eln_data_apm.yaml | 4 +- debug/hits.hits | 0 debug/root.root | 0 debug/str.str | 0 .../map_concepts/apm_example_eln_to_nx_map.py | 44 ++++--- pynxtools/dataconverter/readers/apm/reader.py | 11 +- .../readers/apm/utils/apm_define_io_cases.py | 56 ++++++++- .../utils/apm_load_deployment_specifics.py | 11 +- .../readers/apm/utils/apm_load_generic_eln.py | 108 ++++++++++++------ .../shared/map_concepts/mapping_functors.py | 4 + .../readers/shared/shared_utils.py | 6 +- 12 files changed, 171 insertions(+), 76 deletions(-) create mode 100644 debug/hits.hits create mode 100644 debug/root.root create mode 100644 debug/str.str diff --git a/debug/batch_queue.04.sh b/debug/batch_queue.04.sh index 1f07fe91c..1d3cfa858 100755 --- a/debug/batch_queue.04.sh +++ b/debug/batch_queue.04.sh @@ -1,4 +1,3 @@ #!/bin/bash -dataconverter --reader apm --nxdl NXapm --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO_pole.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO.env" --input-file="apm.oasis.specific.yaml" --output=debug.nor_trondheim_hatzeglou_0.nxs -# 1>stdout.debug.nor_trondheim_hatzeglou_0.txt 2>stderr.debug.nor_trondheim_hatzeglou_0.txt +dataconverter --reader apm --nxdl NXapm --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO_pole.pos" --input-file="/home/kaiobach/Research/paper_paper_paper/joss_nomad_apt/bb_analysis/analysis/harvest_examples/production/ErMnO.env" --input-file="apm.oasis.specific.yaml" --input-file="eln_data_apm.yaml" --output=debug.nor_trondheim_hatzeglou_0.nxs 1>stdout.debug.nor_trondheim_hatzeglou_0.txt 2>stderr.debug.nor_trondheim_hatzeglou_0.txt diff --git a/debug/eln_data_apm.yaml b/debug/eln_data_apm.yaml index 90d0ea28f..2df755758 100644 --- a/debug/eln_data_apm.yaml +++ b/debug/eln_data_apm.yaml @@ -59,8 +59,8 @@ workflow: raw_dat_file: str.str hit_dat_file: hits.hits recon_cfg_file: hits.root - recon_res_file: recon.apt - range_dat_file: recon.rrng + # recon_res_file: recon.apt + # range_dat_file: recon.rrng ranging: program: IVAS program_version: 3.6.8 diff --git a/debug/hits.hits b/debug/hits.hits new file mode 100644 index 000000000..e69de29bb diff --git a/debug/root.root b/debug/root.root new file mode 100644 index 000000000..e69de29bb diff --git a/debug/str.str b/debug/str.str new file mode 100644 index 000000000..e69de29bb diff --git a/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py b/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py index a47e5fd30..3addf9fd0 100644 --- a/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py +++ b/pynxtools/dataconverter/readers/apm/map_concepts/apm_example_eln_to_nx_map.py @@ -17,19 +17,14 @@ # """Dict mapping custom schema instances from eln_data.yaml file on concepts in NXapm.""" -# "/ENTRY[entry*]/PROGRAM[program2]/program": "load_from", "atom_probe/control_software_program"), -# "/ENTRY[entry*]/PROGRAM[program2]/program/@version": "load_from", "atom_probe/control_software_program__attr_version"), -# ("/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate", "load_from", "atom_probe/specimen_monitoring_detection_rate/value"), -# "/ENTRY[entry*]/atom_probe/specimen_monitoring/detection_rate/@units", "load_from", "atom_probe/specimen_monitoring_detection_rate/unit"), - -APM_EXAMPLE_ENTRIES_TO_NEXUS \ +APM_EXAMPLE_OTHER_TO_NEXUS \ = [("/ENTRY[entry*]/run_number", "load_from", "entry/run_number"), ("/ENTRY[entry*]/start_time", "load_from", "entry/start_time"), ("/ENTRY[entry*]/end_time", "load_from", "entry/end_time"), ("/ENTRY[entry*]/operation_mode", "load_from", "entry/operation_mode"), ("/ENTRY[entry*]/experiment_description", "load_from", "entry/experiment_description"), - ("/ENTRY[entry*]/sample/method", "load_from", "sample/method"), + ("/ENTRY[entry*]/sample/method", "experiment"), ("/ENTRY[entry*]/sample/alias", "load_from", "sample/alias"), ("/ENTRY[entry*]/sample/grain_diameter", "load_from", "sample/grain_diameter/value"), ("/ENTRY[entry*]/sample/grain_diameter/@units", "load_from", "sample/grain_diameter/unit"), @@ -44,7 +39,7 @@ ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error", "load_from", "sample/heat_treatment_quenching_rate_error/value"), ("/ENTRY[entry*]/sample/heat_treatment_quenching_rate_error/@units", "load_from", "sample/heat_treatment_quenching_rate_error/unit"), ("/ENTRY[entry*]/sample/description", "load_from", "sample/description"), - ("/ENTRY[entry*]/specimen/method", "load_from", "specimen/method"), + ("/ENTRY[entry*]/specimen/method", "experiment"), ("/ENTRY[entry*]/specimen/alias", "load_from", "specimen/alias"), ("/ENTRY[entry*]/specimen/preparation_date", "load_from", "specimen/preparation_date"), ("/ENTRY[entry*]/specimen/description", "load_from", "specimen/description"), @@ -75,16 +70,27 @@ ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/analysis_chamber/chamber_pressure/@units", "load_from", "atom_probe/chamber_pressure/unit"), ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/base_temperature", "load_from", "atom_probe/base_temperature/value"), ("/ENTRY[entry*]/measurement/event_data_apm_set/EVENT_DATA_APM[event_data_apm]/instrument/stage_lab/base_temperature/@units", "load_from", "atom_probe/base_temperature/unit"), - ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program", "load_from", "atom_probe/ranging/program"), - ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program/@version", "load_from", "atom_probe/ranging/program_version"), - ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program", "load_from", "atom_probe/reconstruction/program"), - ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program/@version", "load_from", "atom_probe/reconstruction/program_version"), - ("/ENTRY[entry*]/atom_probe/reconstruction/protocol_name", "load_from", "atom_probe/reconstruction/protocol_name"), - ("/ENTRY[entry*]/atom_probe/reconstruction/crystallographic_calibration", "load_from", "atom_probe/reconstruction/crystallographic_calibration"), - ("/ENTRY[entry*]/atom_probe/reconstruction/parameter", "load_from", "atom_probe/reconstruction/parameter"), - ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view", "load_from", "atom_probe/reconstruction/field_of_view/value"), - ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view/@units", "load_from", "atom_probe/reconstruction/field_of_view/unit"), - ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/type", "file"), + ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program", "load_from", "ranging/program"), + ("/ENTRY[entry*]/atom_probe/ranging/PROGRAM[program1]/program/@version", "load_from", "ranging/program_version"), + ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program", "load_from", "reconstruction/program"), + ("/ENTRY[entry*]/atom_probe/reconstruction/PROGRAM[program1]/program/@version", "load_from", "reconstruction/program_version"), + ("/ENTRY[entry*]/atom_probe/reconstruction/protocol_name", "load_from", "reconstruction/protocol_name"), + ("/ENTRY[entry*]/atom_probe/reconstruction/crystallographic_calibration", "load_from", "reconstruction/crystallographic_calibration"), + ("/ENTRY[entry*]/atom_probe/reconstruction/parameter", "load_from", "reconstruction/parameter"), + ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view", "load_from", "reconstruction/field_of_view/value"), + ("/ENTRY[entry*]/atom_probe/reconstruction/field_of_view/@units", "load_from", "reconstruction/field_of_view/unit"), + ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/path", "load_from", "workflow/raw_dat_file"), + ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/checksum", "sha256", "workflow/raw_dat_file"), + ("/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]/path", "load_from", "workflow/hit_dat_file"), + ("/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]/checksum", "sha256", "workflow/hit_dat_file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/config/path", "load_from", "workflow/recon_cfg_file"), + ("/ENTRY[entry*]/atom_probe/reconstruction/config/checksum", "sha256", "workflow/recon_cfg_file")] + +""" +# depending on the case some of these are populated by the I/O logic resolver +# some of them will be loaded by the ELN parser +APM_EXAMPLE_WORKFLOW_TO_NEXUS \ + = [("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/type", "file"), ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/path", "load_from", "workflow/raw_dat_file"), ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/checksum", "sha256", "workflow/raw_dat_file"), ("/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]/algorithm", "SHA256"), @@ -104,11 +110,13 @@ ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/path", "load_from", "workflow/range_dat_file"), ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/checksum", "sha256", "workflow/range_dat_file"), ("/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]/algorithm", "SHA256")] +""" # NeXus concept specific mapping tables which require special treatment as the current # NOMAD OASIS custom schema implementation delivers them as a list of dictionaries instead # of a directly flattenable list of key, value pairs + APM_EXAMPLE_USER_TO_NEXUS \ = [("/ENTRY[entry*]/USER[user*]/name", "load_from", "name"), ("/ENTRY[entry*]/USER[user*]/affiliation", "load_from", "affiliation"), diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 8c781eb4e..8fc739fa0 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -34,8 +34,8 @@ import ApmRangingDefinitionsParser from pynxtools.dataconverter.readers.apm.utils.apm_create_nx_default_plots \ import apm_default_plot_generator -from pynxtools.dataconverter.readers.apm.utils.apm_generate_synthetic_data \ - import ApmCreateExampleData +# from pynxtools.dataconverter.readers.apm.utils.apm_generate_synthetic_data \ +# import ApmCreateExampleData # this apm parser combines multiple sub-parsers # so we need the following input: @@ -95,8 +95,8 @@ def read(self, if case.is_valid == False: print("Such a combination of input-file(s, if any) is not supported !") return {} + case.report_workflow(template, entry_id) - """ print("Parse (meta)data coming from an ELN...") if len(case.eln) == 1: nx_apm_eln = NxApmNomadOasisElnSchemaParser(case.eln[0], entry_id) @@ -104,7 +104,6 @@ def read(self, else: print("No input file defined for eln data !") return {} - """ print("Parse (meta)data coming from a configuration that specific OASIS...") if len(case.cfg) == 1: @@ -131,8 +130,8 @@ def read(self, # print("Reporting state of template before passing to HDF5 writing...") # for keyword in template.keys(): - # print(keyword) - # print(template[keyword]) + # print(f"keyword: {keyword}, template[keyword]: {template[keyword]}") + # exit(1) print("Forward instantiated template to the NXS writer...") return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py index f637eac1d..26d7796fe 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py @@ -15,17 +15,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Utility class to analyze which vendor/community files are passed to em reader.""" +"""Utility class to analyze which vendor/community files are passed to apm reader.""" # pylint: disable=no-member,duplicate-code from typing import Tuple, Dict, List + +from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ + import variadic_path_to_specific_path +from pynxtools.dataconverter.readers.shared.shared_utils import get_sha256_of_file_content + VALID_FILE_NAME_SUFFIX_RECON = [".apt", ".pos", ".epos", ".ato", ".csv", ".h5"] VALID_FILE_NAME_SUFFIX_RANGE = [".rng", ".rrng", ".env", ".fig.txt", "range_.h5"] VALID_FILE_NAME_SUFFIX_CONFIG = [".yaml", ".yml"] -class ApmUseCaseSelector: # pylint: disable=too-few-public-methods +class ApmUseCaseSelector: """Decision maker about what needs to be parsed given arbitrary input. Users might invoke this dataconverter with arbitrary input, no input, or @@ -93,7 +98,7 @@ def check_validity_of_file_combinations(self): range_input += len(value) if suffix == ".h5": recon_input += len(value) - print(f"{recon_input}, {range_input}, {other_input}") + # print(f"{recon_input}, {range_input}, {other_input}") if (recon_input == 1) and (range_input == 1) and (1 <= other_input <= 2): self.is_valid = True @@ -112,3 +117,48 @@ def check_validity_of_file_combinations(self): self.cfg += [entry] else: self.eln += [entry] + print(f"recon_results: {self.reconstruction}\n" \ + f"range_results: {self.ranging}\n" \ + f"OASIS ELN: {self.eln}\n" \ + f"OASIS local config: {self.cfg}\n") + + def report_workflow(self, template: dict, entry_id: int) -> dict: + """Initialize the reporting of the workflow.""" + steps = ["/ENTRY[entry*]/atom_probe/raw_data/SERIALIZED[serialized]", + "/ENTRY[entry*]/atom_probe/hit_finding/SERIALIZED[serialized]", + "/ENTRY[entry*]/atom_probe/reconstruction/config", + "/ENTRY[entry*]/atom_probe/reconstruction/results", + "/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]"] + defaults = [("type", "file"), + ("path", ""), + ("checksum", ""), + ("algorithm", "SHA256")] + identifier = [entry_id] + # populate workflow first with default steps to communicate in the NeXus file + # which usually recommended files have not been provided for an NXentry + # keep in mind that already in 2013 D. Larson et al. documented clearly + # in their book which files one should ideally document to enable as best as + # possible the repeating of an analysis until the reconstruction and ranging + # when using IVAS/APSuite ! + for step in steps: + trg = variadic_path_to_specific_path(step, identifier) + for dflt in defaults: + template[f"{trg}/{dflt[0]}"] = f"{dflt[1]}" + # populate automatically input-files used + # rely on assumption made in check_validity_of_file_combination + for fpath in self.reconstruction: + prfx = variadic_path_to_specific_path( + "/ENTRY[entry*]/atom_probe/reconstruction/results", identifier) + with open(fpath, "rb") as fp: + template[f"{prfx}/path"] = f"{fpath}" + template[f"{prfx}/checksum"] = get_sha256_of_file_content(fp) + for fpath in self.ranging: + prfx = variadic_path_to_specific_path( + "/ENTRY[entry*]/atom_probe/ranging/SERIALIZED[serialized]", identifier) + with open(fpath, "rb") as fp: + template[f"{prfx}/path"] = f"{fpath}" + template[f"{prfx}/checksum"] = get_sha256_of_file_content(fp) + # FAU/Erlangen's pyccapt control and calibration file have not functional + # distinction which makes it non-trivial to decide if a given HDF5 qualifies + # as control or calibration file TODO::for this reason it is currently ignored + return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py index b7839d419..befbe0a48 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py @@ -31,7 +31,7 @@ class NxApmNomadOasisConfigurationParser: # pylint: disable=too-few-public-methods """Parse deployment specific configuration.""" - def __init__(self, file_path: str, entry_id: int): + def __init__(self, file_path: str, entry_id: int, verbose: bool = False): print(f"Extracting data from deployment specific configuration file: {file_path}") if (file_path.rsplit('/', 1)[-1].endswith(".oasis.specific.yaml") or file_path.endswith(".oasis.specific.yml")) and entry_id > 0: @@ -39,7 +39,9 @@ def __init__(self, file_path: str, entry_id: int): self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") - print(self.yml) + if verbose == True: + for key, val in self.yml.items(): + print(f"key: {key}, val: {val}") else: self.entry_id = 1 self.file_path = "" @@ -49,11 +51,10 @@ def report(self, template: dict) -> dict: """Copy data from configuration applying mapping functors.""" for tpl in APM_OASIS_TO_NEXUS_CFG: identifier = [self.entry_id] - if isinstance(tpl, tuple): + if isinstance(tpl, tuple) and len(tpl) >= 2: if tpl[0] not in ("IGNORE", "UNCLEAR"): trg = variadic_path_to_specific_path(tpl[0], identifier) - print(f"processing tpl {tpl} ... trg {trg}") - # print(f"Target {trg} after variadic name resolution identifier {identifier}") + # print(f"processing tpl {tpl} ... trg {trg}") if len(tpl) == 2: # nxpath, value to use directly template[trg] = tpl[1] diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py index d3f2034c4..15875b2d9 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py @@ -23,12 +23,13 @@ import yaml from ase.data import chemical_symbols -from pynxtools.dataconverter.readers.apm.map_concepts.apm_example_eln_to_nx_map \ - import NxApmElnInput, NxUserFromListOfDict +from pynxtools.dataconverter.readers.apm.map_concepts.apm_example_eln_to_nx_map import \ + APM_EXAMPLE_OTHER_TO_NEXUS, APM_EXAMPLE_USER_TO_NEXUS from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ import variadic_path_to_specific_path, apply_modifier from pynxtools.dataconverter.readers.apm.utils.apm_parse_composition_table \ import parse_composition_table +from pynxtools.dataconverter.readers.shared.shared_utils import get_sha256_of_file_content class NxApmNomadOasisElnSchemaParser: # pylint: disable=too-few-public-methods @@ -44,12 +45,12 @@ class NxApmNomadOasisElnSchemaParser: # pylint: disable=too-few-public-methods in an instance of NXapm The functionalities in this ELN YAML parser do not check if the - instantiated template yields an instance which is compliant NXapm. + instantiated template yields an instance which is compliant with NXapm. Instead, this task is handled by the generic part of the dataconverter during the verification of the template dictionary. """ - def __init__(self, file_path: str, entry_id: int): + def __init__(self, file_path: str, entry_id: int, verbose: bool = False): print(f"Extracting data from ELN file: {file_path}") if (file_path.rsplit('/', 1)[-1].startswith("eln_data") or file_path.startswith("eln_data")) and entry_id > 0: @@ -57,6 +58,9 @@ def __init__(self, file_path: str, entry_id: int): self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") + if verbose == True: + for key, val in self.yml.items(): + print(f"key: {key}, value: {val}") else: self.entry_id = 1 self.file_path = "" @@ -111,61 +115,91 @@ def parse_user_section(self, template: dict) -> dict: # identifier to get instance NeXus path from variadic NeXus path # try to find all quantities on the left-hand side of the mapping # table and check if we can find these - for nx_path, modifier in NxUserFromListOfDict.items(): - if nx_path not in ("IGNORE", "UNCLEAR"): - trg = variadic_path_to_specific_path(nx_path, identifier) - res = apply_modifier(modifier, user_dict) - if res is not None: - template[trg] = res + for tpl in APM_EXAMPLE_USER_TO_NEXUS: + if isinstance(tpl, tuple) and len(tpl) >= 2: + if len(tpl) == 3: + if tpl[1] == "load_from": + trg = variadic_path_to_specific_path( + tpl[0], identifier) + # res = apply_modifier(modifier, user_dict) + # res is not None + template[trg] = user_dict[tpl[2]] user_id += 1 return template def parse_laser_pulser_details(self, template: dict) -> dict: - """Copy data in pulser section.""" + """Copy data into the (laser)/source section of the pulser.""" # additional laser-specific details only relevant when the laser was used - src = "atom_probe/pulser/pulse_mode" - if src in self.yml.keys(): - if self.yml[src] == "voltage": + if "atom_probe/pulser/pulse_mode" in self.yml.keys(): + if self.yml["atom_probe/pulser/pulse_mode"] == "voltage": return template - else: - return template + src = "atom_probe/pulser/laser_source" if src in self.yml.keys(): if isinstance(self.yml[src], list): if all(isinstance(entry, dict) for entry in self.yml[src]) is True: laser_id = 1 # custom schema delivers a list of dictionaries... - trg = f"/ENTRY[entry{self.entry_id}]/atom_probe/pulser" \ - f"/SOURCE[source{laser_id}]" - for laser_dict in self.yml[src]: - if "name" in laser_dict.keys(): - template[f"{trg}/name"] = laser_dict["name"] - quantities = ["power", "pulse_energy", "wavelength"] - for quant in quantities: - if isinstance(laser_dict[quant], dict): - if ("value" in laser_dict[quant].keys()) \ - and ("unit" in laser_dict[quant].keys()): - template[f"{trg}/{quant}"] \ - = laser_dict[quant]["value"] - template[f"{trg}/{quant}/@units"] \ - = laser_dict[quant]["unit"] + for ldct in self.yml[src]: + trg_sta = f"/ENTRY[entry{self.entry_id}]/measurement/" \ + f"pulser/SOURCE[source{laser_id}]" + if "name" in ldct: + template[f"{trg_sta}/name"] = ldct["name"] + quantities = ["wavelength"] + for qnt in quantities: + if ("value" in ldct[qnt]) and ("unit" in ldct[qnt]): + template[f"{trg_sta}/{qnt}"] = ldct[qnt]["value"] + template[f"{trg_sta}/{qnt}/@units"] = ldct[qnt]["unit"] + + trg_dyn = f"/ENTRY[entry{self.entry_id}]/measurement/" \ + f"event_data_apm_set/EVENT_DATA_APM[event_data_apm]/" \ + f"instrument/pulser/SOURCE[source{laser_id}]" + quantities = ["power", "pulse_energy"] + for qnt in quantities: + if isinstance(ldct[qnt], dict): + if ("value" in ldct[qnt]) and ("unit" in ldct[qnt]): + template[f"{trg_dyn}/{qnt}"] = ldct[qnt]["value"] + template[f"{trg_dyn}/{qnt}/@units"] = ldct[qnt]["unit"] laser_id += 1 + return template + print("WARNING: pulse_mode != voltage but no laser details specified!") return template def parse_other_sections(self, template: dict) -> dict: """Copy data from custom schema into template.""" - for nx_path, modifier in NxApmElnInput.items(): - if nx_path not in ("IGNORE", "UNCLEAR"): - trg = variadic_path_to_specific_path(nx_path, [self.entry_id, 1]) - res = apply_modifier(modifier, self.yml) - if res is not None: - template[trg] = res + identifier = [self.entry_id] + for tpl in APM_EXAMPLE_OTHER_TO_NEXUS: + if isinstance(tpl, tuple) and len(tpl) >= 2: + if tpl[1] not in ("ignore"): + trg = variadic_path_to_specific_path(tpl[0], identifier) + # print(f"processing tpl {tpl} ... trg {trg}") + if len(tpl) == 2: + template[trg] = tpl[1] + if len(tpl) == 3: + # nxpath, modifier, value, modifier (function) evaluates value to use + if tpl[1] == "load_from": + if tpl[2] in self.yml.keys(): + template[trg] = self.yml[tpl[2]] + else: + raise ValueError(f"tpl2 {tpl[2]} not in self.yml.keys()!") + elif tpl[1] == "sha256": + if tpl[2] in self.yml.keys(): + with open(self.yml[tpl[2]], "rb") as fp: + template[trg] = get_sha256_of_file_content(fp) + else: + raise ValueError(f"tpl2 {tpl[2]} not in self.yml.keys()!") + else: + raise ValueError(f"tpl1 {tpl[1]} is an modifier (function)!") return template + def parse_workflow(self, template: dict) -> dict: + """Parse workflow-relevant details""" + + def report(self, template: dict) -> dict: """Copy data from self into template the appdef instance.""" self.parse_sample_composition(template) - self.parse_user_section(template) + # self.parse_user_section(template) self.parse_laser_pulser_details(template) self.parse_other_sections(template) return template diff --git a/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py b/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py index e79791939..b9acd74fe 100644 --- a/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py +++ b/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py @@ -111,4 +111,8 @@ def variadic_path_to_specific_path(path, instance_identifier: list): idx += 1 nx_specific_path += f"{tmp[-1]}" return nx_specific_path + else: + raise ValueError(f"Insufficient len(identifier) to resolve variadic path\n" \ + f"path: {path}" \ + f"instance_identifier: {instance_identifier}") return None diff --git a/pynxtools/dataconverter/readers/shared/shared_utils.py b/pynxtools/dataconverter/readers/shared/shared_utils.py index 9bca903d1..c4e6c42e1 100644 --- a/pynxtools/dataconverter/readers/shared/shared_utils.py +++ b/pynxtools/dataconverter/readers/shared/shared_utils.py @@ -104,9 +104,9 @@ def decorate_path_to_default_plot(template: dict, nxpath: str) -> dict: # an example for nxpath # "/ENTRY[entry1]/atom_probe/ranging/mass_to_charge_distribution/mass_spectrum" # if nxpath in template.keys(): - print(f"nxpath: {nxpath}") + # print(f"nxpath: {nxpath}") path = nxpath.split("/") - print(f"path: {path}") + # print(f"path: {path}") trg = f"/" for idx in np.arange(0, len(path) - 1): print(f"trg: {trg}") @@ -119,5 +119,5 @@ def decorate_path_to_default_plot(template: dict, nxpath: str) -> dict: else: template[f"{trg}@default"] = f"{path[idx + 1]}" trg += f"{path[idx + 1]}/" - print(f"trg: {trg}") + # print(f"trg: {trg}") return template From 9c362c853138bf84a2c4dd8d7c8cbcbdbc0dc7fb Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 10 Jan 2024 20:30:14 +0100 Subject: [PATCH 11/15] User parsing also fixed, therewith i) completed --- debug/eln_data_apm.yaml | 7 +++- .../readers/apm/utils/apm_load_generic_eln.py | 38 +++++++++---------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/debug/eln_data_apm.yaml b/debug/eln_data_apm.yaml index 2df755758..2147f5fa1 100644 --- a/debug/eln_data_apm.yaml +++ b/debug/eln_data_apm.yaml @@ -58,7 +58,7 @@ entry: workflow: raw_dat_file: str.str hit_dat_file: hits.hits - recon_cfg_file: hits.root + recon_cfg_file: root.root # recon_res_file: recon.apt # range_dat_file: recon.rrng ranging: @@ -118,5 +118,8 @@ specimen: unit: ° value: 5.0 user: -- {} +- name: MarkusK + orcid: '0000' + orcid_platform: orcid +- name: AlexR - {} diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py index 15875b2d9..34d86afe7 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py @@ -101,24 +101,28 @@ def parse_sample_composition(self, template: dict) -> dict: ion_id += 1 return template - def parse_user_section(self, template: dict) -> dict: + def parse_user(self, template: dict) -> dict: """Copy data from user section into template.""" src = "user" - if src in self.yml.keys(): + if src in self.yml: if isinstance(self.yml[src], list): if all(isinstance(entry, dict) for entry in self.yml[src]) is True: user_id = 1 # custom schema delivers a list of dictionaries... for user_dict in self.yml[src]: - # ... for each of them inspect for fields mappable on NeXus + if user_dict == {}: + continue identifier = [self.entry_id, user_id] - # identifier to get instance NeXus path from variadic NeXus path - # try to find all quantities on the left-hand side of the mapping - # table and check if we can find these - for tpl in APM_EXAMPLE_USER_TO_NEXUS: - if isinstance(tpl, tuple) and len(tpl) >= 2: - if len(tpl) == 3: - if tpl[1] == "load_from": + for key in user_dict: + for tpl in APM_EXAMPLE_USER_TO_NEXUS: + # this currently N_user * M_nxpaths complexity implementation should + # be replaced with a faster ideally N_user * log(M_nxpaths) + # implementation also because the mapping table is defined + # trg <- modifier, src, instead of src, modifier -> trg + # but the first formulation is easier to read as typically + # the nxpath is longer than the src path + if isinstance(tpl, tuple) and (len(tpl) == 3): + if (tpl[1] == "load_from") and (tpl[2] == key): trg = variadic_path_to_specific_path( tpl[0], identifier) # res = apply_modifier(modifier, user_dict) @@ -127,7 +131,7 @@ def parse_user_section(self, template: dict) -> dict: user_id += 1 return template - def parse_laser_pulser_details(self, template: dict) -> dict: + def parse_pulser_source(self, template: dict) -> dict: """Copy data into the (laser)/source section of the pulser.""" # additional laser-specific details only relevant when the laser was used if "atom_probe/pulser/pulse_mode" in self.yml.keys(): @@ -165,7 +169,7 @@ def parse_laser_pulser_details(self, template: dict) -> dict: print("WARNING: pulse_mode != voltage but no laser details specified!") return template - def parse_other_sections(self, template: dict) -> dict: + def parse_other(self, template: dict) -> dict: """Copy data from custom schema into template.""" identifier = [self.entry_id] for tpl in APM_EXAMPLE_OTHER_TO_NEXUS: @@ -192,14 +196,10 @@ def parse_other_sections(self, template: dict) -> dict: raise ValueError(f"tpl1 {tpl[1]} is an modifier (function)!") return template - def parse_workflow(self, template: dict) -> dict: - """Parse workflow-relevant details""" - - def report(self, template: dict) -> dict: """Copy data from self into template the appdef instance.""" self.parse_sample_composition(template) - # self.parse_user_section(template) - self.parse_laser_pulser_details(template) - self.parse_other_sections(template) + self.parse_user(template) + self.parse_pulser_source(template) + self.parse_other(template) return template From d784ecd4105da03439808e3bf73b6ac54cd9e740 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 10 Jan 2024 21:06:54 +0100 Subject: [PATCH 12/15] Added citations for 176case study and fixed NX_CLASS decoration bug for charge_state_model --- apm.dev.ipynb | 59 +++++++++++++++++++ debug/apm.oasis.specific.yaml | 6 +- .../map_concepts/apm_oasis_cfg_to_nx_map.py | 13 ++-- .../utils/apm_load_deployment_specifics.py | 40 +++++++++---- .../readers/apm/utils/apm_load_generic_eln.py | 10 +--- .../readers/apm/utils/apm_load_ranging.py | 2 +- 6 files changed, 102 insertions(+), 28 deletions(-) create mode 100644 apm.dev.ipynb diff --git a/apm.dev.ipynb b/apm.dev.ipynb new file mode 100644 index 000000000..93626caa6 --- /dev/null +++ b/apm.dev.ipynb @@ -0,0 +1,59 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "6da1aea0-545b-446b-a3d1-1574af72f6c6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "from jupyterlab_h5web import H5Web" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3889c47f-11c4-4bf3-97de-04fc52f0798d", + "metadata": {}, + "outputs": [], + "source": [ + "src = f\"{os.getcwd()}\"\n", + "print(src)\n", + "fnm = [\"debug.nor_trondheim_hatzeglou_0.nxs\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aec5b20a-20ed-4fe3-8097-a53bcc9dded0", + "metadata": {}, + "outputs": [], + "source": [ + "H5Web(f\"{src}/debug/{fnm[0]}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/debug/apm.oasis.specific.yaml b/debug/apm.oasis.specific.yaml index e75a3eb92..fc3cc7a8f 100644 --- a/debug/apm.oasis.specific.yaml +++ b/debug/apm.oasis.specific.yaml @@ -1 +1,5 @@ -operation_mode: apt +citation: +- doi: https://doi.org/repo + description: DOI to the respective entry in a data repository. +- doi: https://doi.org/paper + description: DOI to the respective publication. diff --git a/pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py b/pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py index 3f14d9414..de57b2193 100644 --- a/pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py +++ b/pynxtools/dataconverter/readers/apm/map_concepts/apm_oasis_cfg_to_nx_map.py @@ -17,7 +17,7 @@ # """Dict mapping values for a specifically configured NOMAD OASIS.""" -# pylint: disable=no-member,line-too-long +# pylint: disable=line-too-long # currently by virtue of design NOMAD OASIS specific examples show how different tools and # services can be specifically coupled and implemented so that they work together @@ -43,15 +43,16 @@ import datetime as dt from pynxtools.dataconverter.readers.apm.utils.apm_versioning \ - import NX_APM_ADEF_NAME, NX_APM_ADEF_VERSION, NX_APM_EXEC_NAME, NX_APM_EXEC_VERSION + import NX_APM_ADEF_NAME, NX_APM_ADEF_VERSION APM_OASIS_TO_NEXUS_CFG \ = [("/ENTRY[entry*]/@version", f"{NX_APM_ADEF_VERSION}"), ("/ENTRY[entry*]/definition", f"{NX_APM_ADEF_NAME}"), - ("/ENTRY[entry*]/operation_mode", "load_from", "operation_mode"), + ("/ENTRY[entry*]/operation_mode", "ignore", "operation_mode"), ("/ENTRY[entry*]/start_time", f"{dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z')}")] -# ("/ENTRY[entry*]/PROGRAM[program1]/program", f"{NX_APM_EXEC_NAME}"), -# ("/ENTRY[entry*]/PROGRAM[program1]/program/@version", f"{NX_APM_EXEC_VERSION}"), -# ("/ENTRY[entry*]/atom_probe/location", "load_from", "location")] + +APM_PARAPROBE_EXAMPLE_TO_NEXUS_CFG \ + = [("/ENTRY[entry*]/CITE[cite*]/doi", "load_from", "doi"), + ("/ENTRY[entry*]/CITE[cite*]/description", "load_from", "description")] diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py index befbe0a48..78cb8a763 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py @@ -23,9 +23,9 @@ import yaml from pynxtools.dataconverter.readers.apm.map_concepts.apm_oasis_cfg_to_nx_map \ - import APM_OASIS_TO_NEXUS_CFG + import APM_OASIS_TO_NEXUS_CFG, APM_PARAPROBE_EXAMPLE_TO_NEXUS_CFG from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ - import apply_modifier, variadic_path_to_specific_path + import variadic_path_to_specific_path class NxApmNomadOasisConfigurationParser: # pylint: disable=too-few-public-methods @@ -49,22 +49,38 @@ def __init__(self, file_path: str, entry_id: int, verbose: bool = False): def report(self, template: dict) -> dict: """Copy data from configuration applying mapping functors.""" + identifier = [self.entry_id] for tpl in APM_OASIS_TO_NEXUS_CFG: - identifier = [self.entry_id] if isinstance(tpl, tuple) and len(tpl) >= 2: - if tpl[0] not in ("IGNORE", "UNCLEAR"): + if tpl[1] != "ignore": trg = variadic_path_to_specific_path(tpl[0], identifier) - # print(f"processing tpl {tpl} ... trg {trg}") if len(tpl) == 2: # nxpath, value to use directly template[trg] = tpl[1] if len(tpl) == 3: # nxpath, modifier, value, modifier (function) evaluates value to use - if tpl[1] == "load_from": - if tpl[2] in self.yml.keys(): - template[trg] = self.yml[tpl[2]] - else: - raise ValueError(f"tpl2 {tpl[2]} not in self.yml.keys()!") - else: - raise ValueError(f"tpl1 {tpl[1]} is not load_from!") + if (tpl[1] == "load_from") and (tpl[2] in self.yml): + template[trg] = self.yml[tpl[2]] + + # related to joint paper https://arxiv.org/abs/2205.13510 and NOMAD OASIS + src = "citation" + if src in self.yml: + if isinstance(self.yml[src], list): + if all(isinstance(entry, dict) for entry in self.yml[src]) is True: + ref_id = 1 + # custom schema delivers a list of dictionaries... + for cite_dict in self.yml[src]: + if cite_dict == {}: + continue + identifier = [self.entry_id, ref_id] + for key in cite_dict: + for tpl in APM_PARAPROBE_EXAMPLE_TO_NEXUS_CFG: + if isinstance(tpl, tuple) and (len(tpl) == 3): + if (tpl[1] == "load_from") and (key == tpl[2]): + trg = variadic_path_to_specific_path( + tpl[0], identifier) + # res = apply_modifier(modifier, user_dict) + # res is not None + template[trg] = cite_dict[tpl[2]] + ref_id += 1 return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py index 34d86afe7..e99954902 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py @@ -115,14 +115,8 @@ def parse_user(self, template: dict) -> dict: identifier = [self.entry_id, user_id] for key in user_dict: for tpl in APM_EXAMPLE_USER_TO_NEXUS: - # this currently N_user * M_nxpaths complexity implementation should - # be replaced with a faster ideally N_user * log(M_nxpaths) - # implementation also because the mapping table is defined - # trg <- modifier, src, instead of src, modifier -> trg - # but the first formulation is easier to read as typically - # the nxpath is longer than the src path if isinstance(tpl, tuple) and (len(tpl) == 3): - if (tpl[1] == "load_from") and (tpl[2] == key): + if (tpl[1] == "load_from") and (key == tpl[2]): trg = variadic_path_to_specific_path( tpl[0], identifier) # res = apply_modifier(modifier, user_dict) @@ -146,7 +140,7 @@ def parse_pulser_source(self, template: dict) -> dict: # custom schema delivers a list of dictionaries... for ldct in self.yml[src]: trg_sta = f"/ENTRY[entry{self.entry_id}]/measurement/" \ - f"pulser/SOURCE[source{laser_id}]" + f"instrument/pulser/SOURCE[source{laser_id}]" if "name" in ldct: template[f"{trg_sta}/name"] = ldct["name"] quantities = ["wavelength"] diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py index a40340830..8a20e6104 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py @@ -80,7 +80,7 @@ def add_standardize_molecular_ions(ion_lst: list, template: dict, entry_id: int) template[f"{path}name"] = ion.name.values if ion.charge_state_model["n_cand"] > 0: - path = f"{trg}ION[ion{ion_id}]/charge_state_model/" + path = f"{trg}ION[ion{ion_id}]/charge_state_analysis/" template[f"{path}min_abundance"] \ = np.float64(ion.charge_state_model["min_abundance"]) template[f"{path}min_abundance_product"] \ From 0572bfa9df6f513325024185ed1f1c1901d0758c Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 10 Jan 2024 21:29:48 +0100 Subject: [PATCH 13/15] pycodestyle, pylint, mypy, ready for runthroughs and nomad integration --- apm.dev.ipynb | 8 +++++ pynxtools/dataconverter/readers/apm/reader.py | 5 ++- .../apm/utils/apm_create_nx_default_plots.py | 35 +++++++++---------- .../readers/apm/utils/apm_define_io_cases.py | 14 ++++---- .../utils/apm_load_deployment_specifics.py | 4 +-- .../readers/apm/utils/apm_load_generic_eln.py | 11 +++--- .../readers/apm/utils/apm_load_ranging.py | 12 +++---- .../apm/utils/apm_load_reconstruction.py | 2 +- .../shared/map_concepts/mapping_functors.py | 4 +-- .../readers/shared/shared_utils.py | 9 ++--- 10 files changed, 52 insertions(+), 52 deletions(-) diff --git a/apm.dev.ipynb b/apm.dev.ipynb index 93626caa6..70e5155b5 100644 --- a/apm.dev.ipynb +++ b/apm.dev.ipynb @@ -33,6 +33,14 @@ "source": [ "H5Web(f\"{src}/debug/{fnm[0]}\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77198c3d-7830-49a6-8e75-c8569ab84e76", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/pynxtools/dataconverter/readers/apm/reader.py b/pynxtools/dataconverter/readers/apm/reader.py index 8fc739fa0..84c615732 100644 --- a/pynxtools/dataconverter/readers/apm/reader.py +++ b/pynxtools/dataconverter/readers/apm/reader.py @@ -17,7 +17,7 @@ # """Generic parser for loading atom probe microscopy data into NXapm.""" -# pylint: disable=no-member,too-few-public-methods +# pylint: disable=no-member,too-few-public-methods,pointless-string-statement from typing import Tuple, Any @@ -72,7 +72,6 @@ def read(self, """Read data from given file, return filled template dictionary apm.""" template.clear() - n_entries = 1 entry_id = 1 if len(file_paths) == 1: """ @@ -92,7 +91,7 @@ def read(self, else: # eln_data, and ideally recon and ranging definitions from technology partner file print("Parse ELN and technology partner file(s)...") case = ApmUseCaseSelector(file_paths) - if case.is_valid == False: + if case.is_valid is False: print("Such a combination of input-file(s, if any) is not supported !") return {} case.report_workflow(template, entry_id) diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py b/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py index 2187a2759..5b1c2cfa7 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_create_nx_default_plots.py @@ -22,7 +22,7 @@ import numpy as np from pynxtools.dataconverter.readers.shared.shared_utils import \ - get_repo_last_commit, decorate_path_to_default_plot + decorate_path_to_default_plot from pynxtools.dataconverter.readers.apm.utils.apm_versioning import \ NX_APM_EXEC_NAME, NX_APM_EXEC_VERSION, \ MASS_SPECTRUM_DEFAULT_BINNING, NAIVE_GRID_DEFAULT_VOXEL_SIZE @@ -41,17 +41,16 @@ def create_default_plot_reconstruction(template: dict, entry_id: int) -> dict: xyz = template[f"{trg}reconstructed_positions"]["compress"] print(f"\tEnter histogram computation, np.shape(xyz) {np.shape(xyz)}") - resolution = NAIVE_GRID_DEFAULT_VOXEL_SIZE # cubic voxel edge length in nm - # make the bounding box a quadric prism - aabb = {"x": [0., 0.], - "y": [0., 0.], - "z": [0., 0.], - "xedge": None, - "yedge": None, - "zedge": None} + # make the bounding box a quadric prism, discretized using cubic voxel edge in nm + aabb: dict = {"x": [0., 0.], + "y": [0., 0.], + "z": [0., 0.], + "xedge": None, + "yedge": None, + "zedge": None} col = 0 for dim in ["x", "y", "z"]: - aabb[f"{dim}"] = np.asarray((np.min(xyz[:, col]), np.max(xyz[:, col]))) + aabb[f"{dim}"] = [np.min(xyz[:, col]), np.max(xyz[:, col])] imi = np.floor(aabb[f"{dim}"][0]) - NAIVE_GRID_DEFAULT_VOXEL_SIZE imx = np.ceil(aabb[f"{dim}"][1]) + NAIVE_GRID_DEFAULT_VOXEL_SIZE aabb[f"{dim}edge"] = iedge(imi, imx, NAIVE_GRID_DEFAULT_VOXEL_SIZE) @@ -60,7 +59,7 @@ def create_default_plot_reconstruction(template: dict, entry_id: int) -> dict: hist3d = np.histogramdd((xyz[:, 0], xyz[:, 1], xyz[:, 2]), bins=(aabb["xedge"], aabb["yedge"], aabb["zedge"])) del xyz - if isinstance(hist3d[0], np.ndarray) == False: + if isinstance(hist3d[0], np.ndarray) is False: raise ValueError("Hist3d computation from the reconstruction failed!") if len(np.shape(hist3d[0])) != 3: raise ValueError("Hist3d computation from the reconstruction failed!") @@ -117,7 +116,7 @@ def create_default_plot_mass_spectrum(template: dict, entry_id: int) -> dict: num=int(np.ceil((mqmax - mqmin) / mqincr)) + 1, endpoint=True)) del m_z - if isinstance(hist1d[0], np.ndarray) == False: + if isinstance(hist1d[0], np.ndarray) is False: raise ValueError("Hist1d computation from the mass spectrum failed!") if len(np.shape(hist1d[0])) != 1: raise ValueError("Hist1d computation from the mass spectrum failed!") @@ -170,24 +169,24 @@ def apm_default_plot_generator(template: dict, entry_id: int) -> dict: has_valid_xyz = True print(f"m_z, xyz: {has_valid_m_z}, {has_valid_xyz}") - if (has_valid_m_z == False) and (has_valid_xyz == False): + if (has_valid_m_z is False) and (has_valid_xyz is False): # NEW ISSUE: fall-back solution to plot something else, however # currently POS, EPOS and APT provide always xyz, and m_z data return template # generate default plottable and add path - if has_valid_m_z == True: + if has_valid_m_z is True: create_default_plot_mass_spectrum(template, entry_id) decorate_path_to_default_plot( template, - f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" \ + f"/ENTRY[entry{entry_id}]/atom_probe/ranging/" f"mass_to_charge_distribution/mass_spectrum") - if has_valid_xyz == True: + if has_valid_xyz is True: create_default_plot_reconstruction(template, entry_id) - if has_valid_m_z == False: + if has_valid_m_z is False: decorate_path_to_default_plot( template, - f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/" \ + f"/ENTRY[entry{entry_id}]/atom_probe/reconstruction/" f"naive_discretization/DATA[data]") return template diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py index 26d7796fe..2fced0e60 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_define_io_cases.py @@ -17,7 +17,7 @@ # """Utility class to analyze which vendor/community files are passed to apm reader.""" -# pylint: disable=no-member,duplicate-code +# pylint: disable=no-member,duplicate-code,too-many-branches from typing import Tuple, Dict, List @@ -66,14 +66,12 @@ def sort_files_by_file_name_suffix(self, file_paths: Tuple[str] = None): self.case[suffix].append(fpath) break else: - if fpath.lower().endswith("range_.h5") == True: + if fpath.lower().endswith("range_.h5") is True: self.case["range_.h5"].append(fpath) break - elif fpath.lower().endswith(".h5") == True: + if fpath.lower().endswith(".h5") is True: self.case[".h5"].append(fpath) break - else: - continue # HDF5 files need special treatment, this already shows that magic numbers # should better have been used or signatures to avoid having to have as # complicated content checks as we had to implement e.g. for the em reader @@ -117,9 +115,9 @@ def check_validity_of_file_combinations(self): self.cfg += [entry] else: self.eln += [entry] - print(f"recon_results: {self.reconstruction}\n" \ - f"range_results: {self.ranging}\n" \ - f"OASIS ELN: {self.eln}\n" \ + print(f"recon_results: {self.reconstruction}\n" + f"range_results: {self.ranging}\n" + f"OASIS ELN: {self.eln}\n" f"OASIS local config: {self.cfg}\n") def report_workflow(self, template: dict, entry_id: int) -> dict: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py index 78cb8a763..80ea0aa06 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_deployment_specifics.py @@ -17,7 +17,7 @@ # """Load deployment-specific quantities.""" -# pylint: disable=no-member +# pylint: disable=no-member,too-many-branches,too-many-nested-blocks import flatdict as fd import yaml @@ -39,7 +39,7 @@ def __init__(self, file_path: str, entry_id: int, verbose: bool = False): self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") - if verbose == True: + if verbose is True: for key, val in self.yml.items(): print(f"key: {key}, val: {val}") else: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py index e99954902..3864b0994 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_generic_eln.py @@ -23,13 +23,14 @@ import yaml from ase.data import chemical_symbols -from pynxtools.dataconverter.readers.apm.map_concepts.apm_example_eln_to_nx_map import \ - APM_EXAMPLE_OTHER_TO_NEXUS, APM_EXAMPLE_USER_TO_NEXUS +from pynxtools.dataconverter.readers.apm.map_concepts.apm_example_eln_to_nx_map \ + import APM_EXAMPLE_OTHER_TO_NEXUS, APM_EXAMPLE_USER_TO_NEXUS from pynxtools.dataconverter.readers.shared.map_concepts.mapping_functors \ - import variadic_path_to_specific_path, apply_modifier + import variadic_path_to_specific_path from pynxtools.dataconverter.readers.apm.utils.apm_parse_composition_table \ import parse_composition_table -from pynxtools.dataconverter.readers.shared.shared_utils import get_sha256_of_file_content +from pynxtools.dataconverter.readers.shared.shared_utils \ + import get_sha256_of_file_content class NxApmNomadOasisElnSchemaParser: # pylint: disable=too-few-public-methods @@ -58,7 +59,7 @@ def __init__(self, file_path: str, entry_id: int, verbose: bool = False): self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") - if verbose == True: + if verbose is True: for key, val in self.yml.items(): print(f"key: {key}, value: {val}") else: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py index 8a20e6104..1fce52424 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_ranging.py @@ -108,21 +108,21 @@ def add_standardize_molecular_ions(ion_lst: list, template: dict, entry_id: int) else: template[f"{path}isotope_matrix"] \ = {"compress": np.asarray(ion.charge_state_model["isotope_matrix"], - np.uint16), "strength": 1} + np.uint16), "strength": 1} template[f"{path}charge_state_vector"] \ = {"compress": np.asarray(ion.charge_state_model["charge_state_vector"], - np.int8), "strength": 1} + np.int8), "strength": 1} template[f"{path}mass_vector"] \ = {"compress": np.asarray(ion.charge_state_model["mass_vector"], - np.float64), "strength": 1} + np.float64), "strength": 1} template[f"{path}mass_vector/@units"] \ = "u" template[f"{path}natural_abundance_product_vector"] \ = {"compress": np.asarray(ion.charge_state_model["nat_abun_prod_vector"], - np.float64), "strength": 1} + np.float64), "strength": 1} template[f"{path}min_half_life_vector"] \ = {"compress": np.asarray(ion.charge_state_model["min_half_life_vector"], - np.float64), "strength": 1} + np.float64), "strength": 1} template[f"{path}min_half_life_vector/@units"] \ = "s" ion_id += 1 @@ -207,7 +207,7 @@ def __init__(self, file_path: str, entry_id: int): "file_path": file_path, "entry_id": entry_id} for suffix in VALID_FILE_NAME_SUFFIX_RANGE: - if file_path.lower().endswith(suffix) == True: + if file_path.lower().endswith(suffix) is True: self.meta["file_format"] = suffix break if self.meta["file_format"] is None: diff --git a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py index 8584db1e6..e887911b0 100644 --- a/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py +++ b/pynxtools/dataconverter/readers/apm/utils/apm_load_reconstruction.py @@ -224,7 +224,7 @@ def __init__(self, file_path: str, entry_id: int): "file_path": file_path, "entry_id": entry_id} for suffix in VALID_FILE_NAME_SUFFIX_RECON: - if file_path.lower().endswith(suffix) == True: + if file_path.lower().endswith(suffix) is True: self.meta["file_format"] = suffix break if self.meta["file_format"] is None: diff --git a/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py b/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py index b9acd74fe..358e5f279 100644 --- a/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py +++ b/pynxtools/dataconverter/readers/shared/map_concepts/mapping_functors.py @@ -112,7 +112,7 @@ def variadic_path_to_specific_path(path, instance_identifier: list): nx_specific_path += f"{tmp[-1]}" return nx_specific_path else: - raise ValueError(f"Insufficient len(identifier) to resolve variadic path\n" \ - f"path: {path}" \ + raise ValueError(f"Insufficient len(identifier) to resolve variadic path\n" + f"path: {path}" f"instance_identifier: {instance_identifier}") return None diff --git a/pynxtools/dataconverter/readers/shared/shared_utils.py b/pynxtools/dataconverter/readers/shared/shared_utils.py index c4e6c42e1..8e610da1f 100644 --- a/pynxtools/dataconverter/readers/shared/shared_utils.py +++ b/pynxtools/dataconverter/readers/shared/shared_utils.py @@ -103,21 +103,16 @@ def decorate_path_to_default_plot(template: dict, nxpath: str) -> dict: """Write @default attribute to point to the default plot.""" # an example for nxpath # "/ENTRY[entry1]/atom_probe/ranging/mass_to_charge_distribution/mass_spectrum" - # if nxpath in template.keys(): - # print(f"nxpath: {nxpath}") path = nxpath.split("/") - # print(f"path: {path}") - trg = f"/" + trg = "/" for idx in np.arange(0, len(path) - 1): - print(f"trg: {trg}") symbol_s = path[idx + 1].find("[") symbol_e = path[idx + 1].find("]") - if (symbol_s >= 0) and (symbol_e > symbol_s): + if 0 <= symbol_s < symbol_e: template[f"{trg}@default"] \ = f"{path[idx + 1][symbol_s + 1:symbol_e]}" trg += f"{path[idx + 1][symbol_s + 1:symbol_e]}/" else: template[f"{trg}@default"] = f"{path[idx + 1]}" trg += f"{path[idx + 1]}/" - # print(f"trg: {trg}") return template From 4aba063e98ed173142ea9740341e93f0103d25f2 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 10 Jan 2024 22:06:04 +0100 Subject: [PATCH 14/15] Synced nxdefs with FAIRmat-NFDI/nexus_definitions/apm_refactoring branch 5c7d197 --- pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynxtools/definitions b/pynxtools/definitions index 34fc49b46..5c7d19750 160000 --- a/pynxtools/definitions +++ b/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 34fc49b46221a94dcce242c56b976759542c2ced +Subproject commit 5c7d197505c425cad1a097d5b783be4c29100e4d From a681f25be8b635c959838b2b3cc1aba9255a8fd4 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 23 Jan 2024 10:36:14 +0100 Subject: [PATCH 15/15] Hooked in the correct NeXus definitions branch to use for the em_refactoring feature branch --- apm.dev.ipynb | 30 +++++++++++++++++++++++++----- pynxtools/definitions | 2 +- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/apm.dev.ipynb b/apm.dev.ipynb index 70e5155b5..f16c24419 100644 --- a/apm.dev.ipynb +++ b/apm.dev.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "6da1aea0-545b-446b-a3d1-1574af72f6c6", "metadata": {}, "outputs": [], @@ -14,10 +14,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "3889c47f-11c4-4bf3-97de-04fc52f0798d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/kaiobach/Research/hu_hu_hu/sprint17/pynx-apm/pynxtools\n" + ] + } + ], "source": [ "src = f\"{os.getcwd()}\"\n", "print(src)\n", @@ -26,10 +34,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "aec5b20a-20ed-4fe3-8097-a53bcc9dded0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/x-hdf5": "/home/kaiobach/Research/hu_hu_hu/sprint17/pynx-apm/pynxtools/debug/debug.nor_trondheim_hatzeglou_0.nxs", + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "H5Web(f\"{src}/debug/{fnm[0]}\")" ] diff --git a/pynxtools/definitions b/pynxtools/definitions index 5c7d19750..1072a9a53 160000 --- a/pynxtools/definitions +++ b/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 5c7d197505c425cad1a097d5b783be4c29100e4d +Subproject commit 1072a9a53a060442f208b5dd8f72e9f43feef7df