From 63049b6e64ad8f6a6866849cfdfc9cae73a5ae7f Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 14:39:53 +0200 Subject: [PATCH 1/7] Regenerate requirements.txt file. --- dev-requirements.txt | 84 +++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 27205fe2c..9ff9529e3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,9 +4,9 @@ anytree==2.12.1 # via pynxtools (pyproject.toml) ase==3.23.0 # via pynxtools (pyproject.toml) -babel==2.15.0 +babel==2.16.0 # via mkdocs-material -certifi==2024.7.4 +certifi==2024.8.30 # via requests cfgv==3.4.0 # via pre-commit @@ -22,17 +22,15 @@ click-default-group==1.2.4 # via pynxtools (pyproject.toml) colorama==0.4.6 # via mkdocs-material -contourpy==1.2.1 +contourpy==1.3.0 # via matplotlib -coverage==7.5.4 +coverage==7.6.1 # via pytest-cov cycler==0.12.1 # via matplotlib distlib==0.3.8 # via virtualenv -exceptiongroup==1.2.1 - # via pytest -filelock==3.15.4 +filelock==3.16.0 # via virtualenv fonttools==4.53.1 # via matplotlib @@ -40,11 +38,11 @@ ghp-import==2.1.0 # via mkdocs h5py==3.11.0 # via pynxtools (pyproject.toml) -identify==2.5.36 +identify==2.6.0 # via pre-commit -idna==3.7 +idna==3.8 # via requests -importlib-metadata==8.0.0 +importlib-metadata==8.5.0 # via pynxtools (pyproject.toml) iniconfig==2.0.0 # via pytest @@ -53,28 +51,31 @@ jinja2==3.1.4 # mkdocs # mkdocs-macros-plugin # mkdocs-material -kiwisolver==1.4.5 +kiwisolver==1.4.7 # via matplotlib -lxml==5.2.2 +lxml==5.3.0 # via pynxtools (pyproject.toml) -markdown==3.6 +markdown==3.7 # via + # markdown-include # mkdocs # mkdocs-click # mkdocs-material # pymdown-extensions +markdown-include==0.8.1 + # via pynxtools (pyproject.toml) markupsafe==2.1.5 # via # jinja2 # mkdocs -matplotlib==3.9.1 +matplotlib==3.9.2 # via ase mergedeep==1.3.4 # via # pynxtools (pyproject.toml) # mkdocs # mkdocs-get-deps -mkdocs==1.6.0 +mkdocs==1.6.1 # via # pynxtools (pyproject.toml) # mkdocs-macros-plugin @@ -85,19 +86,19 @@ mkdocs-get-deps==0.2.0 # via mkdocs mkdocs-macros-plugin==1.0.5 # via pynxtools (pyproject.toml) -mkdocs-material==9.5.28 +mkdocs-material==9.5.34 # via pynxtools (pyproject.toml) mkdocs-material-extensions==1.3.1 # via # pynxtools (pyproject.toml) # mkdocs-material -mypy==1.10.1 +mypy==1.11.2 # via pynxtools (pyproject.toml) mypy-extensions==1.0.0 # via mypy nodeenv==1.9.1 # via pre-commit -numpy==2.0.0 +numpy==2.1.1 # via # pynxtools (pyproject.toml) # ase @@ -113,7 +114,7 @@ packaging==24.1 # mkdocs # pytest # xarray -paginate==0.5.6 +paginate==0.5.7 # via mkdocs-material pandas==2.2.2 # via @@ -123,21 +124,21 @@ pathspec==0.12.1 # via mkdocs pillow==10.4.0 # via matplotlib -platformdirs==4.2.2 +platformdirs==4.3.2 # via # mkdocs-get-deps # virtualenv pluggy==1.5.0 # via pytest -pre-commit==3.7.1 +pre-commit==3.8.0 # via pynxtools (pyproject.toml) pygments==2.18.0 # via mkdocs-material -pymdown-extensions==10.8.1 +pymdown-extensions==10.9 # via mkdocs-material -pyparsing==3.1.2 +pyparsing==3.1.4 # via matplotlib -pytest==8.2.2 +pytest==8.3.3 # via # pynxtools (pyproject.toml) # pytest-cov @@ -152,9 +153,9 @@ python-dateutil==2.9.0.post0 # matplotlib # mkdocs-macros-plugin # pandas -pytz==2024.1 +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # pynxtools (pyproject.toml) # mkdocs @@ -165,48 +166,43 @@ pyyaml==6.0.1 # pyyaml-env-tag pyyaml-env-tag==0.1 # via mkdocs -regex==2024.5.15 +regex==2024.9.11 # via mkdocs-material requests==2.32.3 # via mkdocs-material -ruff==0.4.8 +ruff==0.5.5 # via pynxtools (pyproject.toml) -scipy==1.14.0 +scipy==1.14.1 # via ase six==1.16.0 # via # anytree # python-dateutil -structlog==24.2.0 +structlog==24.4.0 # via pynxtools (pyproject.toml) termcolor==2.4.0 # via mkdocs-macros-plugin -tomli==2.0.1 - # via - # coverage - # mypy - # pytest -types-pytz==2024.1.0.20240417 +types-pytz==2024.2.0.20240913 # via pynxtools (pyproject.toml) -types-pyyaml==6.0.12.20240311 +types-pyyaml==6.0.12.20240808 # via pynxtools (pyproject.toml) -types-requests==2.32.0.20240622 +types-requests==2.32.0.20240907 # via pynxtools (pyproject.toml) typing-extensions==4.12.2 # via mypy tzdata==2024.1 # via pandas -urllib3==2.2.2 +urllib3==2.2.3 # via # requests # types-requests -uv==0.2.21 +uv==0.4.9 # via pynxtools (pyproject.toml) -virtualenv==20.26.3 +virtualenv==20.26.4 # via pre-commit -watchdog==4.0.1 +watchdog==5.0.2 # via mkdocs -xarray==2024.6.0 +xarray==2024.9.0 # via pynxtools (pyproject.toml) -zipp==3.19.2 +zipp==3.20.1 # via importlib-metadata From 88a3325956ecc14ce4b92c01544f060477114bea Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 14:46:13 +0200 Subject: [PATCH 2/7] update pyproject and dev-req. file. --- dev-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 9ff9529e3..2035d46d1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -98,7 +98,7 @@ mypy-extensions==1.0.0 # via mypy nodeenv==1.9.1 # via pre-commit -numpy==2.1.1 +numpy==1.26.4 # via # pynxtools (pyproject.toml) # ase diff --git a/pyproject.toml b/pyproject.toml index 8dc2e354a..b2b22876d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "h5py>=3.6.0", "xarray>=0.20.2", "PyYAML>=6.0", - "numpy>=1.21.2", + 'numpy>=1.22.4,<2.0.0', "pandas>=1.3.2", "ase>=3.19.0", "mergedeep", From 1a11e95309981388e2f308cedc6ac914acdc65af Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 14:50:46 +0200 Subject: [PATCH 3/7] Ruff. --- src/pynxtools/dataconverter/debug.py | 40 +++++++++++++++++++ src/pynxtools/dataconverter/helpers.py | 3 +- .../dataconverter/readers/example/reader.py | 1 - 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 src/pynxtools/dataconverter/debug.py diff --git a/src/pynxtools/dataconverter/debug.py b/src/pynxtools/dataconverter/debug.py new file mode 100644 index 000000000..373e00f88 --- /dev/null +++ b/src/pynxtools/dataconverter/debug.py @@ -0,0 +1,40 @@ +import importlib +import os + +from click.testing import CliRunner + +from pynxtools.dataconverter.verify import verify + +# TODO: use importlib for absolute path of pynxtools +# +# # module_path = importlib + + +current_path = os.getcwd() + +if __name__ == "__main__": + # test_file1 = ( + # current_path + "/tests/data/test_data_for_validator/SiO2onSi.ellips.nxs" + # ) + # test_file2 = ( + # current_path + "/tests/data/test_data_for_validator/201805_WSe2_arpes.nxs" + # ) + # runner = CliRunner() + # result = runner.invoke(verify, [test_file2]) + # output = result.output + # print(result.output) + + from pynxtools.dataconverter import helpers + from pynxtools.dataconverter.template import Template + + nxdl_root = None + # nxdl_name = "NXtest_extended" + nxdl_name = "NXhdf5_validator_2" + # nxdl_name = "NXspm" + if nxdl_root is None: + nxdl_root, _ = helpers.get_nxdl_root_and_path(nxdl=nxdl_name) + + template = Template() + helpers.generate_template_from_nxdl(nxdl_root, template) + + print(template) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 27f051e22..027d51fbb 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -41,6 +41,7 @@ from pynxtools.definitions.dev_tools.utils.nxdl_utils import ( get_required_string as nexus_get_required_string, ) +from numpy.char import chararray logger = logging.getLogger("pynxtools") @@ -582,7 +583,7 @@ def is_value_valid_element_of_enum(value, elist) -> Tuple[bool, list]: "ISO8601": (str,), "NX_BINARY": (bytes, bytearray, np.byte, np.ubyte, np.ndarray), "NX_BOOLEAN": (bool, np.ndarray, np.bool_), - "NX_CHAR": (str, np.ndarray, np.chararray), + "NX_CHAR": (str, np.ndarray, chararray), "NX_DATE_TIME": (str,), "NX_FLOAT": (float, np.ndarray, np.floating), "NX_INT": (int, np.ndarray, np.signedinteger), diff --git a/src/pynxtools/dataconverter/readers/example/reader.py b/src/pynxtools/dataconverter/readers/example/reader.py index 34d0aae1c..3e3fd09af 100644 --- a/src/pynxtools/dataconverter/readers/example/reader.py +++ b/src/pynxtools/dataconverter/readers/example/reader.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# """An example reader implementation for the DataConverter.""" import json From c679bbe1ef504e9ce16d0d52b29ebc4507b17b6a Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 15:01:10 +0200 Subject: [PATCH 4/7] Ruff --- src/pynxtools/dataconverter/helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 027d51fbb..6b64f9939 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -41,7 +41,6 @@ from pynxtools.definitions.dev_tools.utils.nxdl_utils import ( get_required_string as nexus_get_required_string, ) -from numpy.char import chararray logger = logging.getLogger("pynxtools") @@ -583,7 +582,7 @@ def is_value_valid_element_of_enum(value, elist) -> Tuple[bool, list]: "ISO8601": (str,), "NX_BINARY": (bytes, bytearray, np.byte, np.ubyte, np.ndarray), "NX_BOOLEAN": (bool, np.ndarray, np.bool_), - "NX_CHAR": (str, np.ndarray, chararray), + "NX_CHAR": (str, np.ndarray, np.char.chararray), "NX_DATE_TIME": (str,), "NX_FLOAT": (float, np.ndarray, np.floating), "NX_INT": (int, np.ndarray, np.signedinteger), From 13a0d448f561bef18906a22500a115f7caf8f617 Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 15:13:42 +0200 Subject: [PATCH 5/7] Mypy --- src/pynxtools/dataconverter/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 6b64f9939..438b8c60d 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -647,7 +647,7 @@ def convert_str_to_bool_safe(value): return None -def is_valid_data_field(value, nxdl_type, path) -> bool: +def is_valid_data_field(value, nxdl_type, path): """Checks whether a given value is valid according to what is defined in the NXDL. This function will also try to convert typical types, for example int to float, From b3383e13bd28c6768b671d2fb4b854d0df6b0348 Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 15:16:08 +0200 Subject: [PATCH 6/7] Mypx. --- src/pynxtools/dataconverter/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 438b8c60d..e9b35dffb 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -648,6 +648,8 @@ def convert_str_to_bool_safe(value): def is_valid_data_field(value, nxdl_type, path): + # todo: Check this funciton and wtire test for it. It seems the funciton is not + # working as expected. """Checks whether a given value is valid according to what is defined in the NXDL. This function will also try to convert typical types, for example int to float, From ef0e20557c7cadd7bce112088c2bd01330e372ae Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 13 Sep 2024 15:21:23 +0200 Subject: [PATCH 7/7] remove debug.py file which is unintended. --- src/pynxtools/dataconverter/debug.py | 40 ---------------------------- 1 file changed, 40 deletions(-) delete mode 100644 src/pynxtools/dataconverter/debug.py diff --git a/src/pynxtools/dataconverter/debug.py b/src/pynxtools/dataconverter/debug.py deleted file mode 100644 index 373e00f88..000000000 --- a/src/pynxtools/dataconverter/debug.py +++ /dev/null @@ -1,40 +0,0 @@ -import importlib -import os - -from click.testing import CliRunner - -from pynxtools.dataconverter.verify import verify - -# TODO: use importlib for absolute path of pynxtools -# -# # module_path = importlib - - -current_path = os.getcwd() - -if __name__ == "__main__": - # test_file1 = ( - # current_path + "/tests/data/test_data_for_validator/SiO2onSi.ellips.nxs" - # ) - # test_file2 = ( - # current_path + "/tests/data/test_data_for_validator/201805_WSe2_arpes.nxs" - # ) - # runner = CliRunner() - # result = runner.invoke(verify, [test_file2]) - # output = result.output - # print(result.output) - - from pynxtools.dataconverter import helpers - from pynxtools.dataconverter.template import Template - - nxdl_root = None - # nxdl_name = "NXtest_extended" - nxdl_name = "NXhdf5_validator_2" - # nxdl_name = "NXspm" - if nxdl_root is None: - nxdl_root, _ = helpers.get_nxdl_root_and_path(nxdl=nxdl_name) - - template = Template() - helpers.generate_template_from_nxdl(nxdl_root, template) - - print(template)