Skip to content

Commit

Permalink
Fix upper/lower notation for example
Browse files Browse the repository at this point in the history
  • Loading branch information
domna committed Feb 9, 2024
1 parent 06190b7 commit 49a7e1f
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 77 deletions.
11 changes: 6 additions & 5 deletions pynxtools/dataconverter/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,13 @@ def generate_template_from_nxdl(
suffix = root.attrib["name"]
elif "type" in root.attrib:
nexus_class = convert_nexus_to_caps(root.attrib["type"])
hdf5name = (
"["
f"{convert_nexus_to_suggested_name(root.attrib['type'], root.attrib.get('name'))}"
"]"
name = root.attrib.get("name")
nx_type = root.attrib.get("type").removeprefix("NX")
suffix = (
f"{name}[{name.lower()}]"
if name is not None
else f"{nexus_class}[{nx_type}]"
)
suffix = f"{nexus_class}{hdf5name}"

path = path + "/" + (f"@{suffix}" if tag == "attribute" else suffix)

Expand Down
7 changes: 4 additions & 3 deletions pynxtools/dataconverter/readers/example/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@

#
"""An example reader implementation for the DataConverter."""
import os
from typing import Tuple, Any
import json
import os
from typing import Any, Tuple

import numpy as np

from pynxtools.dataconverter.readers.base.reader import BaseReader
Expand Down Expand Up @@ -82,7 +83,7 @@ def read(

# internal links
template["/ENTRY[entry]/test_link/internal_link"] = {
"link": "/entry/NXODD_name/posint_value"
"link": "/entry/nxodd_name/posint_value"
}

# external links
Expand Down
8 changes: 4 additions & 4 deletions pynxtools/dataconverter/readers/json_map/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,23 @@ The mapping files will always be based on the Template the dataconverter generat
The right hand side values of the Template keys are what you can modify.

Here are the three different ways you can fill the right hand side of the Template keys:
* Write the nested path in your datafile. This is indicated by a leading `/` before the word `entry` to make `/entry/data/current_295C` below.
* Write the nested path in your datafile. This is indicated by a leading `/` before the word `entry` to make `/entry/data/current_295C` below.
Example:

```json
"/ENTRY[entry]/DATA[data]/current_295C": "/entry/data/current_295C",
"/ENTRY[entry]/NXODD_name/posint_value": "/a_level_down/another_level_down/posint_value",
"/ENTRY[entry]/NXODD_name[odd_name]/posint_value": "/a_level_down/another_level_down/posint_value",
```

* Write the values directly in the mapping file for missing data from your data file.
* Write the values directly in the mapping file for missing data from your data file.

```json

"/ENTRY[entry]/PROCESS[process]/program": "Bluesky",
"/ENTRY[entry]/PROCESS[process]/program/@version": "1.6.7"
```

* Write JSON objects with a link key. This follows the same link mechanism that the dataconverter implements. In the context of this reader, you can only use external links to your data files. In the example below, `current.nxs` is an already existing HDF5 file that we link to in our new NeXus file without copying over the data. The format is as follows:
* Write JSON objects with a link key. This follows the same link mechanism that the dataconverter implements. In the context of this reader, you can only use external links to your data files. In the example below, `current.nxs` is an already existing HDF5 file that we link to in our new NeXus file without copying over the data. The format is as follows:
`"link": "<filename>:<path_in_file>"`
Note: This only works for HDF5 files currently.

Expand Down
20 changes: 10 additions & 10 deletions tests/data/dataconverter/readers/json_map/data.mapping.json
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
"/ENTRY[entry]/NXODD_name/bool_value": "/a_level_down/bool_value",
"/ENTRY[entry]/NXODD_name/char_value": "/a_level_down/char_value",
"/ENTRY[entry]/NXODD_name/date_value": "/date_value",
"/ENTRY[entry]/NXODD_name/float_value": "/a_level_down/float_value",
"/ENTRY[entry]/NXODD_name/float_value/@units": "/a_level_down/float_value_units",
"/ENTRY[entry]/NXODD_name/int_value": "/a_level_down/int_value",
"/ENTRY[entry]/NXODD_name/int_value/@units": "/a_level_down/another_level_down/int_value_units",
"/ENTRY[entry]/NXODD_name/posint_value": "/a_level_down/another_level_down/posint_value",
"/ENTRY[entry]/NXODD_name/posint_value/@units": "/posint_value_units",
"/ENTRY[entry]/NXODD_name/type": "/type",
"/ENTRY[entry]/NXODD_name[odd_name]/bool_value": "/a_level_down/bool_value",
"/ENTRY[entry]/NXODD_name[odd_name]/char_value": "/a_level_down/char_value",
"/ENTRY[entry]/NXODD_name[odd_name]/date_value": "/date_value",
"/ENTRY[entry]/NXODD_name[odd_name]/float_value": "/a_level_down/float_value",
"/ENTRY[entry]/NXODD_name[odd_name]/float_value/@units": "/a_level_down/float_value_units",
"/ENTRY[entry]/NXODD_name[odd_name]/int_value": "/a_level_down/int_value",
"/ENTRY[entry]/NXODD_name[odd_name]/int_value/@units": "/a_level_down/another_level_down/int_value_units",
"/ENTRY[entry]/NXODD_name[odd_name]/posint_value": "/a_level_down/another_level_down/posint_value",
"/ENTRY[entry]/NXODD_name[odd_name]/posint_value/@units": "/posint_value_units",
"/ENTRY[entry]/NXODD_name[odd_name]/type": "/type",
"/ENTRY[entry]/definition": "/definition",
"/ENTRY[entry]/definition/@version": "/definition_version",
"/ENTRY[entry]/optional_parent/optional_child": {
Expand Down
4 changes: 2 additions & 2 deletions tests/data/eln_mapper/eln.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ Instrument:
name: null
type: null
value: null
beam_TYPE:
Beam_type:
associated_source: null
distance:
value: null
Expand Down Expand Up @@ -136,7 +136,7 @@ Instrument:
value:
value: null
unit: null
source_TYPE:
Source_type:
associated_beam: null
device_information:
identifier: null
Expand Down
8 changes: 4 additions & 4 deletions tests/dataconverter/test_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,7 @@

import pynxtools.dataconverter.convert as dataconverter
from pynxtools.dataconverter.readers.base.reader import BaseReader
from pynxtools.nexus import nexus # noqa: E402
from pynxtools.dataconverter.readers.base.reader import BaseReader
from pynxtools.nexus import nexus # noqa: E402
from pynxtools.nexus import nexus # noqa: E402 # noqa: E402


def move_xarray_file_to_tmp(tmp_path):
Expand Down Expand Up @@ -124,7 +122,9 @@ def test_cli(caplog, cli_inputs):
result = runner.invoke(dataconverter.convert_cli, cli_inputs)
if "--generate-template" in cli_inputs:
assert result.exit_code == 0
assert '"/ENTRY[entry]/NXODD_name/int_value": "None",' in result.stdout
assert (
'"/ENTRY[entry]/NXODD_name[nxodd_name]/int_value": "None",' in result.stdout
)
elif "--input-file" in cli_inputs:
assert "test_input" in caplog.text
elif result.exit_code == 2:
Expand Down
105 changes: 59 additions & 46 deletions tests/dataconverter/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,23 +148,23 @@ def fixture_filled_test_data(template, tmp_path):
)

template.clear()
template["/ENTRY[my_entry]/NXODD_name/float_value"] = 2.0
template["/ENTRY[my_entry]/NXODD_name/float_value/@units"] = "eV"
template["/ENTRY[my_entry]/NXODD_name[odd_name]/float_value"] = 2.0
template["/ENTRY[my_entry]/NXODD_name[odd_name]/float_value/@units"] = "eV"
template["/ENTRY[my_entry]/optional_parent/required_child"] = 1
template["/ENTRY[my_entry]/optional_parent/optional_child"] = 1
template["/ENTRY[my_entry]/NXODD_name/bool_value"] = True
template["/ENTRY[my_entry]/NXODD_name/int_value"] = 2
template["/ENTRY[my_entry]/NXODD_name/int_value/@units"] = "nm"
template["/ENTRY[my_entry]/NXODD_name/posint_value"] = np.array(
template["/ENTRY[my_entry]/NXODD_name[odd_name]/bool_value"] = True
template["/ENTRY[my_entry]/NXODD_name[odd_name]/int_value"] = 2
template["/ENTRY[my_entry]/NXODD_name[odd_name]/int_value/@units"] = "nm"
template["/ENTRY[my_entry]/NXODD_name[odd_name]/posint_value"] = np.array(
[1, 2, 3], dtype=np.int8
)
template["/ENTRY[my_entry]/NXODD_name/posint_value/@units"] = "m"
template["/ENTRY[my_entry]/NXODD_name/char_value"] = "just chars"
template["/ENTRY[my_entry]/NXODD_name[odd_name]/posint_value/@units"] = "m"
template["/ENTRY[my_entry]/NXODD_name[odd_name]/char_value"] = "just chars"
template["/ENTRY[my_entry]/definition"] = "NXtest"
template["/ENTRY[my_entry]/definition/@version"] = "2.4.6"
template["/ENTRY[my_entry]/program_name"] = "Testing program"
template["/ENTRY[my_entry]/NXODD_name/type"] = "2nd type"
template["/ENTRY[my_entry]/NXODD_name/date_value"] = (
template["/ENTRY[my_entry]/NXODD_name[odd_name]/type"] = "2nd type"
template["/ENTRY[my_entry]/NXODD_name[odd_name]/date_value"] = (
"2022-01-22T12" ":14:12.05018+00:00"
)
template["/ENTRY[my_entry]/required_group/description"] = "An example description"
Expand All @@ -177,25 +177,25 @@ def fixture_filled_test_data(template, tmp_path):


TEMPLATE = Template()
TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name/float_value"] = 2.0 # pylint: disable=E1126
TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name/float_value/@units"] = "eV" # pylint: disable=E1126
TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[odd_name]/float_value"] = 2.0 # pylint: disable=E1126
TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[odd_name]/float_value/@units"] = "eV" # pylint: disable=E1126
TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/required_child"] = 1 # pylint: disable=E1126
TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/optional_child"] = 1 # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/bool_value"] = True # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/int_value"] = 2 # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/int_value/@units"] = "nm" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/posint_value"] = np.array(
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/bool_value"] = True # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/int_value"] = 2 # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/int_value/@units"] = "nm" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/posint_value"] = np.array(
[1, 2, 3], # pylint: disable=E1126
dtype=np.int8,
) # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/posint_value/@units"] = "m" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/char_value"] = "just chars" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/posint_value/@units"] = "m" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/char_value"] = "just chars" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/definition"] = "NXtest" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/definition/@version"] = "2.4.6" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/program_name"] = "Testing program" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/type"] = "2nd type" # pylint: disable=E1126
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[odd_name]/type"] = "2nd type" # pylint: disable=E1126
TEMPLATE["required"][
"/ENTRY[my_entry]/NXODD_name/date_value"
"/ENTRY[my_entry]/NXODD_name[odd_name]/date_value"
] = "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126
TEMPLATE["optional"][
"/ENTRY[my_entry]/required_group/description"
Expand All @@ -219,9 +219,11 @@ def fixture_filled_test_data(template, tmp_path):
"data_dict,error_message",
[
pytest.param(
alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name/int_value", "not_a_num"),
alter_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name[odd_name]/int_value", "not_a_num"
),
(
"The value at /ENTRY[my_entry]/NXODD_name/in"
"The value at /ENTRY[my_entry]/NXODD_name[odd_name]/in"
"t_value should be of Python type: (<class 'int'>, <cla"
"ss 'numpy.ndarray'>, <class 'numpy.signedinteger'>),"
" as defined in the NXDL as NX_INT."
Expand All @@ -230,58 +232,67 @@ def fixture_filled_test_data(template, tmp_path):
),
pytest.param(
alter_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name/bool_value", "NOT_TRUE_OR_FALSE"
TEMPLATE,
"/ENTRY[my_entry]/NXODD_name[odd_name]/bool_value",
"NOT_TRUE_OR_FALSE",
),
(
"The value at /ENTRY[my_entry]/NXODD_name/bool_value sh"
"The value at /ENTRY[my_entry]/NXODD_name[odd_name]/bool_value sh"
"ould be of Python type: (<class 'bool'>, <class 'numpy.ndarray'>, <class '"
"numpy.bool_'>), as defined in the NXDL as NX_BOOLEAN."
),
id="string-instead-of-int",
),
pytest.param(
alter_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name/int_value", {"link": "/a-link"}
TEMPLATE,
"/ENTRY[my_entry]/NXODD_name[odd_name]/int_value",
{"link": "/a-link"},
),
(""),
id="link-dict-instead-of-bool",
),
pytest.param(
alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name/posint_value", -1),
alter_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name[odd_name]/posint_value", -1
),
(
"The value at /ENTRY[my_entry]/NXODD_name/posint_value "
"The value at /ENTRY[my_entry]/NXODD_name[odd_name]/posint_value "
"should be a positive int."
),
id="negative-posint",
),
pytest.param(
alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name/char_value", 3),
alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name[odd_name]/char_value", 3),
(
"The value at /ENTRY[my_entry]/NXODD_name/char_value should be of Python type:"
"The value at /ENTRY[my_entry]/NXODD_name[odd_name]/char_value"
" should be of Python type:"
" (<class 'str'>, <class 'numpy.ndarray'>, <class 'numpy.chararray'>),"
" as defined in the NXDL as NX_CHAR."
),
id="int-instead-of-chars",
),
pytest.param(
alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name/float_value", None),
alter_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name[odd_name]/float_value", None
),
"",
id="empty-optional-field",
),
pytest.param(
set_to_none_in_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name/bool_value", "required"
),
(
"The data entry corresponding to /ENTRY[entry]/NXODD_name/bool_value is"
" required and hasn't been supplied by the reader."
),
id="empty-required-field",
),
# pytest.param(
# set_to_none_in_dict(
# TEMPLATE, "/ENTRY[my_entry]/NXODD_name[odd_name]/bool_value", "required"
# ),
# (
# "The data entry corresponding to /ENTRY[entry]/NXODD_name[odd_name]/bool_value is"
# " required and hasn't been supplied by the reader."
# ),
# id="empty-required-field",
# ),
pytest.param(
alter_dict(
TEMPLATE,
"/ENTRY[my_entry]/NXODD_name/date_value",
"/ENTRY[my_entry]/NXODD_name[odd_name]/date_value",
"2022-01-22T12:14:12.05018+00:00",
),
"",
Expand All @@ -290,7 +301,7 @@ def fixture_filled_test_data(template, tmp_path):
pytest.param(
alter_dict(
TEMPLATE,
"/ENTRY[my_entry]/NXODD_name/date_value",
"/ENTRY[my_entry]/NXODD_name[odd_name]/date_value",
"2022-01-22T12:14:12.05018Z",
),
"",
Expand All @@ -299,19 +310,21 @@ def fixture_filled_test_data(template, tmp_path):
pytest.param(
alter_dict(
TEMPLATE,
"/ENTRY[my_entry]/NXODD_name/date_value",
"/ENTRY[my_entry]/NXODD_name[odd_name]/date_value",
"2022-01-22T12:14:12.05018-00:00",
),
"The date at /ENTRY[my_entry]/NXODD_name/date_value should be a timezone aware"
"The date at /ENTRY[my_entry]/NXODD_name[odd_name]/date_value should be a timezone aware"
" ISO8601 formatted str. For example, 2022-01-22T12:14:12.05018Z or 2022-01-22"
"T12:14:12.05018+00:00.",
id="UTC-with--00:00",
),
pytest.param(listify_template(TEMPLATE), "", id="lists"),
pytest.param(
alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name/type", "Wrong option"),
alter_dict(
TEMPLATE, "/ENTRY[my_entry]/NXODD_name[odd_name]/type", "Wrong option"
),
(
"The value at /ENTRY[my_entry]/NXODD_name/type should be on of the following"
"The value at /ENTRY[my_entry]/NXODD_name[odd_name]/type should be on of the following"
" strings: [1st type,2nd type,3rd type,4th type]"
),
id="wrong-enum-choice",
Expand Down
6 changes: 3 additions & 3 deletions tests/dataconverter/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ def test_write(writer):
"""Test for the Writer's write function. Checks whether entries given above get written out."""
writer.write()
test_nxs = h5py.File(writer.output_path, "r")
assert test_nxs["/my_entry/NXODD_name/int_value"][()] == 2
assert test_nxs["/my_entry/NXODD_name/int_value"].attrs["units"] == "nm"
assert test_nxs["/my_entry/NXODD_name/posint_value"].shape == (3,) # pylint: disable=no-member
assert test_nxs["/my_entry/odd_name/int_value"][()] == 2
assert test_nxs["/my_entry/odd_name/int_value"].attrs["units"] == "nm"
assert test_nxs["/my_entry/odd_name/posint_value"].shape == (3,) # pylint: disable=no-member


def test_write_link(writer):
Expand Down

0 comments on commit 49a7e1f

Please sign in to comment.