Manual merging in of fairmat 8620c4e into the em_refactoring branch t…

…o prepare with the resolving of possible merge conflicts
FAIRmat-NFDI · Dec 4, 2023 · 7e1cfd9 · 7e1cfd9
1 parent 96fb1c4
commit 7e1cfd9
Show file tree

Hide file tree

Showing 95 changed files with 6,832 additions and 1,744 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -19,6 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          fetch-depth: 0
           submodules: recursive
       - name: Set up Python
         uses: actions/setup-python@v4

diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           git submodule sync --recursive

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -13,33 +13,31 @@ jobs:
   pytest:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python_version: ["3.8", "3.9", "3.10"]
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
         with:
+          fetch-depth: 0
           submodules: recursive
       - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python_version }}
       - name: Install dependencies
         run: |
-          git submodule sync --recursive
-          git submodule update --init --recursive --jobs=4
           python -m pip install --upgrade pip
           python -m pip install coverage coveralls
       - name: Install package
         run: |
-          python -m pip install --no-deps .
-      - name: Install dev requirements
-        run: |
-          python -m pip install -r dev-requirements.txt
+          pip install ".[dev]"
       - name: Test with pytest
         run: |
           coverage run -m pytest -sv --show-capture=no tests
       - name: Submit to coveralls
+        continue-on-error: true
         env:
             GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |

diff --git a/.gitignore b/.gitignore
@@ -88,6 +88,8 @@ cover/
 
 # Django stuff:
 *.log
+!tests/data/dataconverter/readers/mpes/Ref_nexus_mpes.log
+!tests/data/nexus/Ref_nexus_test.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -203,7 +205,7 @@ build/
 nexusparser.egg-info/PKG-INFO
 .python-version
 
-# reader specific custom settings
+# em-reader-specific custom settings
 *.h5
 *.oh5
 *.edaxh5

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -23,6 +23,13 @@
     "pynxtools",
     "tests"
   ],
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.formatOnSave": false,
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": false
+    }
+  },
   "python.testing.pytestArgs": ["tests"],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,6 @@
 recursive-include pynxtools/definitions/base_classes/ *.xml
 recursive-include pynxtools/definitions/applications/ *.xml
 recursive-include pynxtools/definitions/contributed_definitions/ *.xml
-include pynxtools/definitions/ *.xsd
+include pynxtools/definitions/*.xsd
+include pynxtools/nexus-version.txt
+include pynxtools/definitions/NXDL_VERSION
diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ It allows to develop ontologies and to create ontological instances based on the
 
 # Installation
 
-It is recommended to use python 3.8 with a dedicated virtual environment for this package.
+It is recommended to use python 3.10 with a dedicated virtual environment for this package.
 Learn how to manage [python versions](https://github.com/pyenv/pyenv) and
 [virtual environments](https://realpython.com/python-virtual-environments-a-primer/).
 
@@ -71,6 +71,17 @@ Especially relevant for developers, there exists a basic test framework written
 python -m pytest -sv tests
 ```
 
+## Run examples
+
+A number of examples exist which document how the tools can be used. For a standalone
+usage convenient jupyter notebooks are available for each tool. To use them jupyter
+and related tools have to be installed in the development environment as follows:
+
+```shell
+python -m pip install jupyter
+python -m pip install jupyterlab
+python -m pip install jupyterlab_h5web
+```
 # Questions, suggestions?
 
 To ask further questions, to make suggestions how we can improve these tools, to get advice

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -23,6 +23,8 @@ attrs==22.1.0
     #   requests-cache
 backcall==0.2.0
     # via ipython
+blosc2==2.3.2
+    # via tables
 build==0.10.0
     # via pip-tools
 cattrs==22.2.0
@@ -113,7 +115,7 @@ hyperspy==1.7.5
     #   pyxem
 idna==3.4
     # via requests
-ifes-apt-tc-data-modeling==0.0.9
+ifes-apt-tc-data-modeling==0.1
     # via pynxtools (pyproject.toml)
 imageio==2.22.1
     # via
@@ -192,14 +194,20 @@ matplotlib-scalebar==0.8.1
     # via orix
 mccabe==0.7.0
     # via pylint
+mergedeep==1.3.4
+    # via pynxtools (pyproject.toml)
 mpmath==1.2.1
     # via sympy
+msgpack==1.0.7
+    # via blosc2
 mypy==0.982
     # via pynxtools (pyproject.toml)
 mypy-extensions==0.4.3
     # via mypy
 natsort==8.2.0
     # via hyperspy
+ndindex==1.7
+    # via blosc2
 nest-asyncio==1.5.6
     # via
     #   ipykernel
@@ -238,9 +246,11 @@ numexpr==2.8.3
     # via
     #   hyperspy
     #   pyfai
+    #   tables
 numpy==1.21.6
     # via
     #   ase
+    #   blosc2
     #   dask
     #   diffsims
     #   fabio
@@ -271,6 +281,7 @@ numpy==1.21.6
     #   scipy
     #   silx
     #   sparse
+    #   tables
     #   tifffile
     #   xarray
     #   zarr
@@ -293,6 +304,7 @@ packaging==21.3
     #   pooch
     #   pytest
     #   scikit-image
+    #   tables
 pandas==1.3.5
     # via
     #   ifes-apt-tc-data-modeling
@@ -341,6 +353,10 @@ ptyprocess==0.7.0
     # via pexpect
 py==1.11.0
     # via pytest
+py-cpuinfo==9.0.0
+    # via
+    #   blosc2
+    #   tables
 pycifrw==4.4.5
     # via diffpy-structure
 pycodestyle==2.9.1
@@ -448,6 +464,8 @@ sympy==1.10.1
     # via
     #   hyperspy
     #   radioactivedecay
+tables==3.9.2
+    # via ifes-apt-tc-data-modeling
 threadpoolctl==3.1.0
     # via scikit-learn
 tifffile==2021.11.2

diff --git a/examples/apm/Write.NXapm.Example.1.ipynb b/examples/apm/Write.NXapm.Example.1.ipynb
@@ -195,6 +195,20 @@
     "### **Step 3c**: Convert the files in the example into an NXapm-compliant NeXus/HDF5 file."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<div class=\"alert alert-block alert-info\">\n",
+    "The deployment_specific YAML file can be used to simplify a specific ELN, e.g. to avoid having to type<br>\n",
+    "repetitive information. A typical use case is an OASIS in a laboratory where there is a fixed number of<br>\n",
+    "instruments for which many details stay the same, these can be carried over via a *.oasis.specific.yaml file.<br>\n",
+    "It is the responsibility of the OASIS maintainer to document and version these specific configurations.<br>\n",
+    "We would be happy to learn how this functionality is useful and learn about your feedback to improve<br>\n",
+    "this feature to make working with NeXus and ELNs more convenient.<br>\n",
+    "</div>"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -204,35 +218,38 @@
    "outputs": [],
    "source": [
     "#parser-nexus/tests/data/tools/dataconverter/readers/em_om/\n",
+    "import numpy as np\n",
     "eln_data_file_name = [\"eln_data_apm.yaml\"]\n",
+    "deployment_specific = [\"apm.oasis.specific.yaml\"]\n",
     "input_recon_file_name = [\"Si.apt\",\n",
     "                         \"Si.epos\",\n",
     "                         \"Si.pos\",\n",
-    "                         \"R31_06365-v02.pos\",\n",
-    "                         \"R18_58152-v02.epos\",\n",
-    "                         \"70_50_50.apt\"]\n",
+    "#                         \"R31_06365-v02.pos\",\n",
+    "#                         \"R18_58152-v02.epos\",\n",
+    "#                         \"70_50_50.apt\"]\n",
     "#                         \"R56_01769-v01.pos\"]\n",
     "input_range_file_name = [\"Si.RRNG\",\n",
     "                         \"Si.RNG\",\n",
     "                         \"Si.RNG\",\n",
-    "                         \"R31_06365-v02.rrng\",\n",
-    "                         \"R31_06365-v02.rrng\",\n",
-    "                         \"R31_06365-v02.rrng\"]\n",
+    "#                         \"R31_06365-v02.rrng\",\n",
+    "#                         \"R31_06365-v02.rrng\",\n",
+    "#                         \"R31_06365-v02.rrng\"]\n",
     "#                         \"R56_01769.rng.fig.txt\"]\n",
     "output_file_name = [\"apm.case1.nxs\",\n",
     "                    \"apm.case2.nxs\",\n",
     "                    \"apm.case3.nxs\",\n",
     "                    \"apm.case4.nxs\",\n",
     "                    \"apm.case5.nxs\",\n",
     "                    \"apm.case6.nxs\"]\n",
-    "for case_id in [0]:\n",
+    "for case_id in np.arange(0, 3):\n",
     "    ELN = eln_data_file_name[0]\n",
+    "    OASIS = deployment_specific[0]\n",
     "    INPUT_RECON = input_recon_file_name[case_id]\n",
     "    INPUT_RANGE = input_range_file_name[case_id]\n",
     "    OUTPUT = output_file_name[case_id]\n",
     "\n",
-    "    ! dataconverter --reader apm --nxdl NXapm --input-file $ELN --input-file \\\n",
-    "    $INPUT_RECON --input-file $INPUT_RANGE --output $OUTPUT"
+    "    ! dataconverter --reader apm --nxdl NXapm --input-file $ELN --input-file $OASIS \\\n",
+    "    --input-file $INPUT_RECON --input-file $INPUT_RANGE --output $OUTPUT"
    ]
   },
   {
@@ -258,7 +275,7 @@
    "outputs": [],
    "source": [
     "# H5Web(OUTPUT)\n",
-    "H5Web(\"apm.case1.nxs\")"
+    "H5Web(\"apm.case3.nxs\")"
    ]
   },
   {
@@ -417,7 +434,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<div class=\"alert alert-block alert-danger\">\n",
+    "<div class=\"alert alert-block alert-warning\">\n",
     "Currently, this functionality requires a Python environment with a newer version of the ase library than the one<br>\n",
     "which is used by the installation of pynxtools (which is currently ase==3.19.0). Instead, ase>=3.22.1 should be used.<br>\n",
     "The issue with the specific functionalities used in the *create_reconstructed_positions* function is that when using<br>\n",
@@ -448,12 +465,14 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<div class=\"alert alert-block alert-danger\">\n",
-    "This functionality uses recent features of ase which demands an environment that is currently not supported<br>\n",
+    "<div class=\"alert alert-block alert-warning\">\n",
+    "This functionality uses recent features of ase which demands an environment that is not necessarily supported<br>\n",
     "by NOMAD OASIS. As the here exemplified settings for this example are configured to represent an environment<br>\n",
-    "matching close to NOMAD users who are interested in this developer functionality should do the following:<br>\n",
+    "matching one which is close to NOMAD, users who are interested in this dev functionality should do the following:<br>\n",
     "Run this example in a standalone environment where ase is upgraded to the latest version and then use<br>\n",
     "the generated NeXus files either as is or upload them to NOMAD OASIS.<br>\n",
+    "If the above-mentioned cell detects e.g. that a recent version of ase was installed<br>\n",
+    "(e.g. >3.22.x) then the code in the following cell can be executed without issues.<br>\n",
     "</div>"
    ]
   },
@@ -465,7 +484,7 @@
    },
    "outputs": [],
    "source": [
-    "# ! dataconverter --reader apm --nxdl NXapm --input-file synthesize1 --output apm.case0.nxs"
+    "! dataconverter --reader apm --nxdl NXapm --input-file synthesize1 --output apm.case0.nxs"
    ]
   },
   {
@@ -496,7 +515,7 @@
    "metadata": {},
    "source": [
     "### Contact person for the apm reader and related examples in FAIRmat:\n",
-    "Markus Kühbach, 2023/05<br>\n",
+    "Markus Kühbach, 2023/08/31<br>\n",
     "\n",
     "### Funding\n",
     "<a href=\"https://www.fairmat-nfdi.eu/fairmat\">FAIRmat</a> is a consortium on research data management which is part of the German NFDI.<br>\n",
@@ -527,7 +546,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.16"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

diff --git a/examples/apm/apm.oasis.specific.yaml b/examples/apm/apm.oasis.specific.yaml
@@ -0,0 +1 @@
+location: Leoben