Merge branch 'master' into mpesworkshop_167_new

FAIRmat-NFDI · Dec 19, 2023 · 43fe11f · 43fe11f
2 parents 0bdb8fe + 8bd900e
commit 43fe11f
Show file tree

Hide file tree

Showing 152 changed files with 12,592 additions and 13,320 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml
@@ -0,0 +1,33 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["bug"]
+projects: ["FAIRmat-NFDI/pynxtools"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. [email protected]
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+      value: "A bug happened!"
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -22,12 +22,12 @@ jobs:
       - name: Install dev requirements
         run: |
           python -m pip install -r dev-requirements.txt
-      - name: pycodestyle
+      - name: ruff
         run: |
-          python -m pycodestyle --ignore=E501,E701,E731,W503 pynxtools tests --exclude=pynxtools/definitions
-      - name: pylint
+          ruff pynxtools tests
+      - name: ruff formatting
         run: |
-          python -m pylint pynxtools tests --ignore definitions
+          ruff format --check pynxtools tests
       - name: mypy
         run: |
-          python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional --exclude=pynxtools/definitions pynxtools tests
+          mypy pynxtools tests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.7
+    hooks:
+      # Run the linter.
+      - id: ruff
+      # Run the formatter.
+      - id: ruff-format
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -2,33 +2,14 @@
   "editor.rulers": [90],
   "editor.renderWhitespace": "all",
   "editor.tabSize": 4,
-  "[javascript]": {
-    "editor.tabSize": 2
-  },
   "files.trimTrailingWhitespace": true,
-  "python.linting.pycodestylePath": "pycodestyle",
-  "python.linting.pycodestyleEnabled": true,
-  "python.linting.pycodestyleArgs": ["--ignore=E501,E701,E731,W503"],
-  "python.linting.mypyEnabled": true,
-  "python.linting.mypyArgs": [
-    "--ignore-missing-imports",
-    "--follow-imports=silent",
-    "--no-strict-optional",
-    "--show-error-codes"
-  ],
-  "python.linting.pylintEnabled": true,
-  "python.linting.enabled": true,
-  "python.linting.pylintArgs": [
-    "--load-plugins=nomad.metainfo.pylint_plugin",
-    "pynxtools",
-    "tests"
-  ],
   "[python]": {
-    "editor.defaultFormatter": "ms-python.black-formatter",
-    "editor.formatOnSave": false,
+    "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
-      "source.organizeImports": false
-    }
+      "source.fixAll": true,
+      "source.organizeImports": true
+    },
+    "editor.defaultFormatter": "charliermarsh.ruff"
   },
   "python.testing.pytestArgs": ["tests"],
   "python.testing.unittestEnabled": false,

diff --git a/README.md b/README.md
@@ -1,3 +1,4 @@
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 ![](https://github.com/FAIRmat-NFDI/pynxtools/actions/workflows/pytest.yml/badge.svg)
 ![](https://github.com/FAIRmat-NFDI/pynxtools/actions/workflows/pylint.yml/badge.svg)
 ![](https://github.com/FAIRmat-NFDI/pynxtools/actions/workflows/publish.yml/badge.svg)
@@ -40,7 +41,6 @@ data into the NeXus standard and visualising the files content.
 
 # Command line tools
 
-- [**nyaml2nxdl**](https://github.com/FAIRmat-NFDI/pynxtools/blob/master/pynxtools/nyaml2nxdl/README.md): Converts, easy to read, YAML [NeXus schemas](https://nexusformat.org) into NeXus XML definition language (NXDL).
 - [**dataconverter**](https://github.com/FAIRmat-NFDI/pynxtools/blob/master/pynxtools/dataconverter/README.md): Creates compliant instances of NeXus/HDF5 files to [NeXus schemas](https://nexusformat.org).
 - [**read_nexus**](https://github.com/FAIRmat-NFDI/pynxtools/blob/master/pynxtools/nexus/README.md): Outputs a debug log for a given NeXus file.
 
@@ -62,6 +62,14 @@ python -m pip install -e .
 python -m pip install -e ".[dev]"
 ```
 
+There is also a [pre-commit hook](https://pre-commit.com/#intro) available
+which formats the code and checks the linting before actually commiting.
+It can be installed with
+```shell
+pre-commit install
+```
+from the root of this repository.
+
 ## Test this software
 
 Especially relevant for developers, there exists a basic test framework written in
@@ -96,3 +104,8 @@ on how to build on this work, or to get your parser included into NOMAD, you can
 ### Does this software require NOMAD or NOMAD OASIS ?
 
 No. The data files produced here can be uploaded to Nomad. Therefore, this acts like the framework to design schemas and instances of data within the NeXus universe.
+
+# Troubleshooting
+
+Please check this [guide](TROUBLESHOOTING.md) for any issues you face with the tool. If you don't find a solution there, please make a new [Github Issue](https://github.com/FAIRmat-NFDI/pynxtools/issues/new?template=bug.yaml).
+
diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md
@@ -0,0 +1,38 @@
+# Troubleshooting Guide
+
+### If you don't find a solution here, please make a new [Github Issue](https://github.com/FAIRmat-NFDI/pynxtools/issues/new?template=bug.yaml).
+
+<br/>
+
+## Module import error
+
+```python
+ImportError: cannot import name 'get_nexus_version' from 'pynxtools' (unknown location)
+```
+
+This is, unfortunately, expected behavior.  The Python import mechanism will first look for a folder with the package name in the current working directory. If it finds a folder named `pynxtools`, it will try to import from there. In this case we have the cloned repo folder with the same name, `pynxtools`. Python tries to import from this folder but the code resides in `pynxtools/pynxtools`.
+
+### Solution
+
+If you are working in a directory containing the repo folder, `pynxtools`, rename this repo folder to `pynxtools2` so your current directory looks like this:
+
+```
+.
+├── pynxtools2                    # Renamed repo folder
+│   ├── pynxtools          # Actual module code is in here
+│   ├── tests
+│   └── ...
+└── my_code_imports_pynxtools.py
+```
+
+Then **reinstall the package** and your import will work.
+
+### Steps to reproduce
+
+```
+git clone --recurse-submodules https://github.com/FAIRmat-NFDI/pynxtools.git
+cd pynxtools
+pip install -e .
+cd ..
+```
+
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -4,6 +4,10 @@
 #
 #    pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml
 #
+appnope==0.1.3
+    # via
+    #   ipykernel
+    #   ipython
 asciitree==0.3.3
     # via zarr
 ase==3.22.1
@@ -12,8 +16,6 @@ ase==3.22.1
     #   pynxtools (pyproject.toml)
 asteval==0.9.31
     # via lmfit
-astroid==3.0.1
-    # via pylint
 asttokens==2.4.1
     # via stack-data
 attrs==23.1.0
@@ -30,6 +32,8 @@ cattrs==23.2.2
     # via requests-cache
 certifi==2023.11.17
     # via requests
+cfgv==3.4.0
+    # via pre-commit
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
@@ -44,13 +48,16 @@ comm==0.2.0
 contourpy==1.1.1
     # via matplotlib
 coverage[toml]==7.3.2
-    # via pytest-cov
+    # via
+    #   coverage
+    #   pytest-cov
 cycler==0.12.1
     # via matplotlib
 cython==3.0.6
     # via tables
 dask[array]==2023.5.0
     # via
+    #   dask
     #   hyperspy
     #   kikuchipy
     #   orix
@@ -71,9 +78,9 @@ diffsims==0.5.2
     #   kikuchipy
     #   pyxem
 dill==0.3.7
-    # via
-    #   hyperspy
-    #   pylint
+    # via hyperspy
+distlib==0.3.7
+    # via virtualenv
 entrypoints==0.4
     # via ipyparallel
 exceptiongroup==1.2.0
@@ -88,6 +95,8 @@ fabio==2023.10.0
     #   silx
 fasteners==0.19
     # via zarr
+filelock==3.13.1
+    # via virtualenv
 flatdict==4.0.1
     # via pynxtools (pyproject.toml)
 fonttools==4.45.1
@@ -122,6 +131,8 @@ hyperspy==1.7.6
     #   kikuchipy
     #   pynxtools (pyproject.toml)
     #   pyxem
+identify==2.5.33
+    # via pre-commit
 idna==3.6
     # via requests
 ifes-apt-tc-data-modeling==0.1
@@ -138,6 +149,7 @@ importlib-metadata==6.8.0
     # via
     #   dask
     #   hyperspy
+    #   pynxtools (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
 ipykernel==6.27.1
@@ -149,8 +161,6 @@ ipython==8.12.3
     #   hyperspy
     #   ipykernel
     #   ipyparallel
-isort==5.12.0
-    # via pylint
 jedi==0.19.1
     # via ipython
 jinja2==3.1.2
@@ -200,8 +210,6 @@ matplotlib-inline==0.1.6
     #   ipython
 matplotlib-scalebar==0.8.1
     # via orix
-mccabe==0.7.0
-    # via pylint
 mergedeep==1.3.4
     # via pynxtools (pyproject.toml)
 mpmath==1.3.0
@@ -236,6 +244,8 @@ nionutils==0.4.6
     #   nionswift
     #   nionswift-io
     #   nionui
+nodeenv==1.8.0
+    # via pre-commit
 numba==0.58.1
     # via
     #   diffsims
@@ -339,8 +349,8 @@ platformdirs==4.0.0
     # via
     #   jupyter-core
     #   pooch
-    #   pylint
     #   requests-cache
+    #   virtualenv
 pluggy==1.3.0
     # via pytest
 ply==3.11
@@ -349,6 +359,8 @@ pooch==1.8.0
     # via
     #   kikuchipy
     #   orix
+pre-commit==3.5.0
+    # via pynxtools (pyproject.toml)
 prettytable==3.9.0
     # via hyperspy
 prompt-toolkit==3.0.41
@@ -367,14 +379,10 @@ py-cpuinfo==9.0.0
     # via tables
 pycifrw==4.4.6
     # via diffpy-structure
-pycodestyle==2.11.1
-    # via pynxtools (pyproject.toml)
 pyfai==2023.9.0
     # via pyxem
 pygments==2.17.2
     # via ipython
-pylint==3.0.2
-    # via pynxtools (pyproject.toml)
 pyparsing==3.1.1
     # via matplotlib
 pyproject-hooks==1.0.0
@@ -411,6 +419,7 @@ pyyaml==6.0.1
     #   dask
     #   hyperspy
     #   kikuchipy
+    #   pre-commit
     #   pynxtools (pyproject.toml)
 pyzmq==25.1.1
     # via
@@ -427,6 +436,8 @@ requests==2.31.0
     #   requests-cache
 requests-cache==1.1.1
     # via pynxtools (pyproject.toml)
+ruff==0.1.7
+    # via pynxtools (pyproject.toml)
 scikit-image==0.20.0
     # via
     #   hyperspy
@@ -487,11 +498,8 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pip-tools
-    #   pylint
     #   pyproject-hooks
     #   pytest
-tomlkit==0.12.3
-    # via pylint
 toolz==0.12.0
     # via
     #   dask
@@ -535,7 +543,6 @@ types-requests==2.31.0.10
     # via pynxtools (pyproject.toml)
 typing-extensions==4.8.0
     # via
-    #   astroid
     #   cattrs
     #   mypy
 tzdata==2023.3
@@ -555,6 +562,8 @@ urllib3==2.1.0
     #   requests
     #   requests-cache
     #   types-requests
+virtualenv==20.25.0
+    # via pre-commit
 wcwidth==0.2.12
     # via
     #   prettytable