diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..b222fc5cf --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,37 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 4e8335c97..0434aa247 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -3,7 +3,7 @@ name: python on: [push] jobs: - build: + linting: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -31,7 +31,3 @@ jobs: - name: mypy run: | python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional --exclude=nexusutils/definitions nexusutils tests - - name: Test with pytest - if: ${{ always() }} - run: | - python -m pytest -sv --show-capture=no tests diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 000000000..6b9dfdf65 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Pytest + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + matrix: + python_version: ["3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + - name: Install dependencies + run: | + git submodule sync --recursive + git submodule update --init --recursive --jobs=4 + python -m pip install --upgrade pip + - name: Install package + run: | + python -m pip install --no-deps . + - name: Install dev requirements + run: | + python -m pip install -r dev-requirements.txt + - name: Test with pytest + run: | + python -m pytest -sv --show-capture=no tests diff --git a/README.md b/README.md index d6e6ca389..9c797b2d1 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,12 @@ data into the NeXus standard and then aloow the visualisation of the hierarchica of the generated NeXus file in JupyterLab. ## Getting started + You should create a virtual environment. We tested on Ubuntu with Python 3.7 and Python 3.8. We recommend to use Python 3.8 but acknowledge that NOMAD uses still Python 3.7. If you don't have Python 3.8 installed on your computer, follow these commands: + ``` sudo add-apt-repository ppa:deadsnakes/ppa sudo apt install python3.8 python3-dev libpython3.8-dev python3.8-venv @@ -39,7 +41,7 @@ mkdir cd pip install virtualenv virtualenv --python=python3.8 .py38 -source .pyenv/bin/activate +source .py38/bin/activate pip install pip-tools ``` @@ -48,6 +50,7 @@ version specifier with 3.7, keep in mind though that installing python-numpy is required also. ## Does this software require NOMAD or NOMAD OASIS ? + No because the tools here allow you to parse data from generic descriptions that can work with different electronic lab notebooks and files from different technology partners into a NeXus/HDF5-based format. Using NOMAD OASIS though @@ -61,12 +64,24 @@ Go into the cloned directory and directly run the parser from there: Now you have to decide your role. Are you a user or a developer? ### I want to use nexusutils as a user + If you are a user you should install nexusutils as a standalone tool: + ``` pip install nexusutils --extra-index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple ``` +### I want to use the latest development version of nexusutils + +If you are a user and want to install the latest nexusutils development version without +cloning the repository you can it with pip via git: + +``` +pip install git+https://github.com/nomad-coe/nomad-parser-nexus.git +``` + ### I am a developer and want to explore or contribute + Install the package with its dependencies: ``` @@ -80,6 +95,7 @@ python -m pip install -e .[dev] ``` ## Test this software + Especially relevant for developers, there exists a basic test framework written in [pytest](https://docs.pytest.org/en/stable/) which can be used as follows: @@ -90,6 +106,7 @@ python -m pytest -sv tests ## Individual modules of this software and their functions ### **nyaml2nxdl** + This module is a tool which automatically converts YAML files containing specifications for NeXus base classes and application definitions, i.e. data schemes, into specifically formatted XML files required by NeXus as it is maintained by the NeXus International Advisory Committee. @@ -119,8 +136,8 @@ Options: Further documentation: [README.md](nyaml2nxdl/README.md) - ### **dataconverter** + This module is a tool which enables users to create instances of NeXus/HDF5 files. The instances are specific for the experiment and technique. Each instance supports a combination of specifically-versioned software components: @@ -166,12 +183,13 @@ Options: Further documentation: [README.md](dataconverter/README.md) -Inspect also the specific README.md files of the respective parser(s) which you wish +Inspect also the specific README.md files of the respective parser(s) which you wish to use or contribute developing to. This will help you to learn how these parser(s) work, which input data they expect, which output they return, and which limitations they have and which assumptions are made. ### **read_nexus** + This utility outputs a debug log for a given NeXus file by annotating the data and metadata entries with the schema definitions from the respective NeXus base classes and application definitions to which the file refers to. @@ -180,11 +198,12 @@ and application definitions to which the file refers to. user@box:~$ read_nexus ``` -*The environmental variable called "NEXUS_DEF_PATH" can be set to +_The environmental variable called "NEXUS_DEF_PATH" can be set to a directory, which contains the NeXus definitions as XML files. If this environmental -variable is not defined, the module will use the definitions in its bundle.* +variable is not defined, the module will use the definitions in its bundle._ An environmental variable can be set as follows: + ``` export 'NEXUS_DEF_PATH'= ``` @@ -192,6 +211,7 @@ export 'NEXUS_DEF_PATH'= ## Next steps Our documentation provides several resources that might be interesting for developers of NOMAD (Oasis): + - [How to write a parser](https://nomad-lab.eu/prod/rae/docs/parser.html). Provides a more detailed tutorial on how to write a parser for NOMAD. - [Introduction to the NOMAD Metainfo](https://nomad-lab.eu/prod/rae/docs/metainfo.html). @@ -200,10 +220,11 @@ Our documentation provides several resources that might be interesting for devel in the scientific literature https://doi.org/10.1038/s41524-017-0048-5. ## Questions, suggestions? + To ask further questions, to make suggestions how we can improve these tools, to get advice on how to build on this work, or to get your parser included into NOMAD, you can: + - Open an issue on the [nexus-parser GitHub project](https://github.com/nomad-coe/nomad-parser-nexus/issues) - Use our forums at [matsci.org](https://matsci.org/c/nomad/32) - Write to [support@nomad-lab.eu](mailto:support@nomad-lab.eu) - Contact directly the lead developers of the individual parsers. - diff --git a/nexusutils/nexus/nexus.py b/nexusutils/nexus/nexus.py index 177e45fde..1d0dddec0 100644 --- a/nexusutils/nexus/nexus.py +++ b/nexusutils/nexus/nexus.py @@ -802,7 +802,7 @@ def helper_get_inherited_nodes(hdf_info, elist, pind, attr): return hdf_path, hdf_node, hdf_class_path, elist, pind, attr, html_name -#@lru_cache(maxsize=None) +# @lru_cache(maxsize=None) def get_inherited_nodes(nxdl_path: str = None, nx_name: str = None, elem: ET.Element = None, hdf_node=None, attr=False): diff --git a/pyproject.toml b/pyproject.toml index a3852218f..9ebb58851 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=64.0.1", "setuptools-scm[toml]>=6.2"] build-backend = "setuptools.build_meta" [project] name = "nexusutils" -version = "0.0.1" +dynamic = ["version"] authors = [ { name = "The NOMAD Authors" }, ] @@ -72,3 +72,7 @@ nexusutils = ["definitions/**/*.xml", "definitions/**/*.xsd"] [tool.setuptools.packages.find] exclude = ["nexusutils/definitions*"] + +[tool.setuptools_scm] +version_scheme = "guess-next-dev" +local_scheme = "node-and-date" \ No newline at end of file