Skip to content

Commit

Permalink
Merge pull request #159 from FAIRmat-NFDI/adds-test-group-optionality
Browse files Browse the repository at this point in the history 
Adds test for group optionality
  • Loading branch information
@sherjeelshabih
sherjeelshabih authored Sep 15, 2023
2 parents debd5cb + 8718b81 commit d49f5af
Show file tree
Hide file tree
Showing 7 changed files with 88 additions and 45 deletions.
22 changes: 16 additions & 6 deletions pynxtools/dataconverter/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ def path_in_data_dict(nxdl_path: str, data: dict) -> Tuple[bool, str]:
for key in data.keys():
if nxdl_path == convert_data_converter_dict_to_nxdl_path(key):
return True, key
return False, ""
return False, None


def check_for_optional_parent(path: str, nxdl_root: ET.Element) -> str:
Expand Down Expand Up @@ -380,6 +380,8 @@ def all_required_children_are_set(optional_parent_path, data, nxdl_root):
"""Walks over optional parent's children and makes sure all required ones are set"""
optional_parent_path = convert_data_converter_dict_to_nxdl_path(optional_parent_path)
for key in data:
if key in data["lone_groups"]:
continue
nxdl_key = convert_data_converter_dict_to_nxdl_path(key)
if nxdl_key[0:nxdl_key.rfind("/")] == optional_parent_path \
and is_node_required(nxdl_key, nxdl_root) \
Expand Down Expand Up @@ -438,17 +440,26 @@ def does_group_exist(path_to_group, data):
return False


def ensure_all_required_fields_exist(template, data):
def ensure_all_required_fields_exist(template, data, nxdl_root):
"""Checks whether all the required fields are in the returned data object."""
for path in template["required"]:
entry_name = get_name_from_data_dict_entry(path[path.rindex('/') + 1:])
if entry_name == "@units":
continue
nxdl_path = convert_data_converter_dict_to_nxdl_path(path)
is_path_in_data_dict, renamed_path = path_in_data_dict(nxdl_path, data)
if path in template["lone_groups"] and does_group_exist(path, data):
continue

renamed_path = path if renamed_path is None else renamed_path
if path in template["lone_groups"]:
opt_parent = check_for_optional_parent(path, nxdl_root)
if opt_parent != "<<NOT_FOUND>>":
if does_group_exist(opt_parent, data) and not does_group_exist(renamed_path, data):
raise ValueError(f"The required group, {path}, hasn't been supplied"
f" while its optional parent, {path}, is supplied.")
continue
if not does_group_exist(renamed_path, data):
raise ValueError(f"The required group, {path}, hasn't been supplied.")
continue
if not is_path_in_data_dict or data[renamed_path] is None:
raise ValueError(f"The data entry corresponding to {path} is required "
f"and hasn't been supplied by the reader.")
Expand Down Expand Up @@ -489,11 +500,10 @@ def validate_data_dict(template, data, nxdl_root: ET.Element):
nxdl_path_to_elm: dict = {}

# Make sure all required fields exist.
ensure_all_required_fields_exist(template, data)
ensure_all_required_fields_exist(template, data, nxdl_root)
try_undocumented(data, nxdl_root)

for path in data.get_documented().keys():
# print(f"{path}")
if data[path] is not None:
entry_name = get_name_from_data_dict_entry(path[path.rindex('/') + 1:])
nxdl_path = convert_data_converter_dict_to_nxdl_path(path)
Expand Down
7 changes: 6 additions & 1 deletion pynxtools/dataconverter/readers/example/reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ def read(self,
for k in template.keys():
# The entries in the template dict should correspond with what the dataconverter
# outputs with --generate-template for a provided NXDL file
if k.startswith("/ENTRY[entry]/required_group"):
if k.startswith("/ENTRY[entry]/required_group") \
or k == "/ENTRY[entry]/optional_parent/req_group_in_opt_group":
continue

field_name = k[k.rfind("/") + 1:]
Expand All @@ -61,6 +62,10 @@ def read(self,
if f"{field_name}_units" in data.keys() and f"{k}/@units" in template.keys():
template[f"{k}/@units"] = data[f"{field_name}_units"]

template["required"]["/ENTRY[entry]/optional_parent/required_child"] = 1
template["optional"][("/ENTRY[entry]/optional_parent/"
"req_group_in_opt_group/DATA[data]")] = [0, 1]

# Add non template key
template["/ENTRY[entry]/does/not/exist"] = "None"
template["/ENTRY[entry]/required_group/description"] = "A test description"
Expand Down
3 changes: 0 additions & 3 deletions pynxtools/dataconverter/readers/json_map/reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@

from pynxtools.dataconverter.readers.base.reader import BaseReader
from pynxtools.dataconverter.template import Template
from pynxtools.dataconverter.helpers import ensure_all_required_fields_exist
from pynxtools.dataconverter import hdfdict


Expand Down Expand Up @@ -158,8 +157,6 @@ def read(self,

fill_undocumented(mapping, new_template, data)

ensure_all_required_fields_exist(template, new_template)

return new_template


Expand Down
3 changes: 3 additions & 0 deletions tests/data/dataconverter/NXtest.nxdl.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@
<field name="optional_child" optional="true" type="NX_INT">
<doc>A dummy entry to test optional parent check for required child.</doc>
</field>
<group type="NXdata" name="req_group_in_opt_group">
<doc>This is a required group in an optional group.</doc>
</group>
</group>
</group>
</definition>
3 changes: 2 additions & 1 deletion tests/data/dataconverter/readers/json_map/data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,6 @@
"type": "2nd type",
"date_value": "2022-01-22T12:14:12.05018+00:00",
"required_child": 1,
"optional_child": 1
"optional_child": 1,
"random_data": [0, 1]
}
3 changes: 2 additions & 1 deletion tests/data/dataconverter/readers/json_map/data.mapping.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@
"/ENTRY[entry]/optional_parent/required_child": "/required_child",
"/ENTRY[entry]/program_name": "Example for listing exact data in the map file: Nexus Parser",
"/ENTRY[entry]/required_group/description": "An example description",
"/ENTRY[entry]/required_group2/description": "An example description"
"/ENTRY[entry]/required_group2/description": "An example description",
"/ENTRY[entry]/optional_parent/req_group_in_opt_group/DATA[data]": "/random_data"
}
92 changes: 59 additions & 33 deletions tests/dataconverter/test_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@
from pynxtools.dataconverter.template import Template


def remove_optional_parent(data_dict: Template):
"""Completely removes the optional group from the test Template."""
internal_dict = Template(data_dict)
del internal_dict["/ENTRY[my_entry]/optional_parent/required_child"]
del internal_dict["/ENTRY[my_entry]/optional_parent/optional_child"]
del internal_dict["/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]"]

return internal_dict


def alter_dict(data_dict: Template, key: str, value: object):
"""Helper function to alter a single entry in dict for parametrize."""
if data_dict is not None:
Expand Down Expand Up @@ -101,31 +111,31 @@ def fixture_filled_test_data(template, tmp_path):
tmp_path)

template.clear()
template["optional"]["/ENTRY[my_entry]/NXODD_name/float_value"] = 2.0
template["optional"]["/ENTRY[my_entry]/NXODD_name/float_value/@units"] = "nm"
template["optional"]["/ENTRY[my_entry]/optional_parent/required_child"] = 1
template["optional"]["/ENTRY[my_entry]/optional_parent/optional_child"] = 1
template["required"]["/ENTRY[my_entry]/NXODD_name/bool_value"] = True
template["required"]["/ENTRY[my_entry]/NXODD_name/int_value"] = 2
template["required"]["/ENTRY[my_entry]/NXODD_name/int_value/@units"] = "eV"
template["required"]["/ENTRY[my_entry]/NXODD_name/posint_value"] = np.array([1, 2, 3],
dtype=np.int8)
template["required"]["/ENTRY[my_entry]/NXODD_name/posint_value/@units"] = "kg"
template["required"]["/ENTRY[my_entry]/NXODD_name/char_value"] = "just chars"
template["required"]["/ENTRY[my_entry]/definition"] = "NXtest"
template["required"]["/ENTRY[my_entry]/definition/@version"] = "2.4.6"
template["required"]["/ENTRY[my_entry]/program_name"] = "Testing program"
template["required"]["/ENTRY[my_entry]/NXODD_name/type"] = "2nd type"
template["required"]["/ENTRY[my_entry]/NXODD_name/date_value"] = ("2022-01-22T12"
":14:12.05018+00:00")
template["optional"]["/ENTRY[my_entry]/required_group/description"] = "An example description"
template["optional"]["/ENTRY[my_entry]/required_group2/description"] = "An example description"
template["undocumented"]["/ENTRY[my_entry]/does/not/exist"] = "random"
template["undocumented"]["/ENTRY[my_entry]/links/ext_link"] = {"link":
f"{tmp_path}/"
f"xarray_saved_small_cali"
f"bration.h5:/axes/ax3"
}
template["/ENTRY[my_entry]/NXODD_name/float_value"] = 2.0
template["/ENTRY[my_entry]/NXODD_name/float_value/@units"] = "nm"
template["/ENTRY[my_entry]/optional_parent/required_child"] = 1
template["/ENTRY[my_entry]/optional_parent/optional_child"] = 1
template["/ENTRY[my_entry]/NXODD_name/bool_value"] = True
template["/ENTRY[my_entry]/NXODD_name/int_value"] = 2
template["/ENTRY[my_entry]/NXODD_name/int_value/@units"] = "eV"
template["/ENTRY[my_entry]/NXODD_name/posint_value"] = np.array([1, 2, 3],
dtype=np.int8)
template["/ENTRY[my_entry]/NXODD_name/posint_value/@units"] = "kg"
template["/ENTRY[my_entry]/NXODD_name/char_value"] = "just chars"
template["/ENTRY[my_entry]/definition"] = "NXtest"
template["/ENTRY[my_entry]/definition/@version"] = "2.4.6"
template["/ENTRY[my_entry]/program_name"] = "Testing program"
template["/ENTRY[my_entry]/NXODD_name/type"] = "2nd type"
template["/ENTRY[my_entry]/NXODD_name/date_value"] = ("2022-01-22T12"
":14:12.05018+00:00")
template["/ENTRY[my_entry]/required_group/description"] = "An example description"
template["/ENTRY[my_entry]/required_group2/description"] = "An example description"
template["/ENTRY[my_entry]/does/not/exist"] = "random"
template["/ENTRY[my_entry]/links/ext_link"] = {"link":
f"{tmp_path}/"
f"xarray_saved_small_cali"
f"bration.h5:/axes/ax3"
}
yield template


Expand All @@ -148,7 +158,10 @@ def fixture_filled_test_data(template, tmp_path):
TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name/date_value"] = "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126
TEMPLATE["optional"]["/ENTRY[my_entry]/required_group/description"] = "An example description"
TEMPLATE["optional"]["/ENTRY[my_entry]/required_group2/description"] = "An example description"
# TEMPLATE["optional_parents"].append("/ENTRY[entry]/optional_parent")
TEMPLATE["required"]["/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]"] = 1
TEMPLATE["lone_groups"] = ['/ENTRY[entry]/required_group',
'/ENTRY[entry]/required_group2',
'/ENTRY[entry]/optional_parent/req_group_in_opt_group']


@pytest.mark.parametrize("data_dict,error_message", [
Expand Down Expand Up @@ -241,13 +254,11 @@ def fixture_filled_test_data(template, tmp_path):
id="valid-data-dict"),
pytest.param(
remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group/description"),
("The data entry corresponding to /ENTRY[entry]/required_group "
"is required and hasn't been supplied by the reader."),
"The required group, /ENTRY[entry]/required_group, hasn't been supplied.",
id="missing-empty-yet-required-group"),
pytest.param(
remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group2/description"),
("The data entry corresponding to /ENTRY[entry]/required_group2 "
"is required and hasn't been supplied by the reader."),
"The required group, /ENTRY[entry]/required_group2, hasn't been supplied.",
id="missing-empty-yet-required-group2"),
pytest.param(
alter_dict(
Expand All @@ -258,6 +269,21 @@ def fixture_filled_test_data(template, tmp_path):
(""),
id="allow-required-and-empty-group"
),
pytest.param(
remove_from_dict(TEMPLATE,
"/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]",
"required"
),
("The required group, /ENTRY[entry]/optional_parent/req_group_in_opt_group, hasn't been "
"supplied while its optional parent, /ENTRY[entry]/optional_parent/"
"req_group_in_opt_group, is supplied."),
id="req-group-in-opt-parent-removed"
),
pytest.param(
remove_optional_parent(TEMPLATE),
(""),
id="opt-group-completely-removed"
),
])
def test_validate_data_dict(data_dict, error_message, template, nxdl_root, request):
"""Unit test for the data validation routine"""
Expand All @@ -269,12 +295,12 @@ def test_validate_data_dict(data_dict, error_message, template, nxdl_root, reque
"no-child-provided-optional-parent",
"int-instead-of-chars",
"link-dict-instead-of-bool",
"allow-required-and-empty-group"):
"allow-required-and-empty-group",
"opt-group-completely-removed"):
helpers.validate_data_dict(template, data_dict, nxdl_root)
else:
with pytest.raises(Exception) as execinfo:
helpers.validate_data_dict(template, data_dict, nxdl_root)

assert (error_message) == str(execinfo.value)


Expand All @@ -285,7 +311,7 @@ def test_validate_data_dict(data_dict, error_message, template, nxdl_root, reque
id="path-exists-in-dict"),
pytest.param(
"/RANDOM/does/not/@exist",
(False, ""),
(False, None),
id="path-does-not-exist-in-dict")
])
def test_path_in_data_dict(nxdl_path, expected, template):
Expand Down

0 comments on commit d49f5af

Please sign in to comment.