Merge pull request #335 from FAIRmat-NFDI/small-test-fixes

Small changes to plugin tests

.github/workflows/plugin_test.yaml

@@ -1,7 +1,7 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python 3.9 and run the tests of all supported plugins.
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: test plugin
+name: test plugins
 
 on:
   push:
@@ -11,104 +11,61 @@ on:
 
 jobs:
   pytest:
+    name: pytest (${{ matrix.plugin }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
+        include:
+          # - plugin: pynxtools-apm
+          #   branch: main
+          #   tests_to_run: tests/.
+          # - plugin: pynxtools-ellips
+          #   branch: main
+          #   tests_to_run: tests/.
+          # - plugin: pynxtools-em
+          #   branch: main
+          #   tests_to_run: tests/.
+          # - plugin: pynxtools-mpes
+          #   branch: main
+          #   tests_to_run: tests/.
+          - plugin: pynxtools-stm
+            branch: main
+            tests_to_run: tests/.
+          - plugin: pynxtools-xps
+            branch: main
+            tests_to_run: tests/.
+          # - plugin: pynxtools-xrd
+          #   branch: update-tests
+          #   tests_to_run: tests/.
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Set up Python ${{ matrix.python_version }}
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python_version }}
+          python-version: 3.9
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install coverage coveralls
       - name: Install package
         run: |
           pip install ".[dev]"
-      # Test for STM Plugin
-      - name: clone STM repo
+      - name: Clone ${{ matrix.plugin }} repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          repository: FAIRmat-NFDI/pynxtools-stm
-          path: pynxtools-stm
-      - name: Run STM tests
+          repository: FAIRmat-NFDI/${{ matrix.plugin }}
+          path: ${{ matrix.plugin }}
+          ref: ${{ matrix.branch }}
+      - name: Install ${{ matrix.plugin }}
         run: |
-          cd pynxtools-stm
-          git submodule sync --recursive
-          git submodule update --init --recursive --jobs=4
-          pip install .
-          pytest tests/test_reader.py::test_stm_reader
-      # # Test for MPES Plugin
-      # - name: clone MPES repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-mpes
-      #     path: pynxtools-mpes
-      # - name: Run MPES tests
-      #   run: |
-      #     cd pynxtools-mpes
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
-      # Test for XPS Plugin
-      - name: clone XPS repo
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          repository: FAIRmat-NFDI/pynxtools-xps
-          path: pynxtools-xps
-      - name: Run XPS tests
+          cd ${{ matrix.plugin }} 
+          pip install .      
+      - name: Run ${{ matrix.plugin }} tests
         run: |
-          cd pynxtools-xps
-          git checkout main
-          pip install .
-          pytest tests/.
-      # # Test for ellipsometry Plugin
-      # - name: clone ellipsometry repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-ellips
-      #     path: pynxtools-ellips
-      # - name: Run ellipsometry tests
-      #   run: |
-      #     cd pynxtools-ellips
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
-      # # Test for EM Plugin
-      # - name: clone EM repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-em
-      #     path: pynxtools-em
-      # - name: Run EM tests
-      #   run: |
-      #     cd pynxtools-em
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
-      # # Test for APM Plugin
-      # - name: clone APM repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-apm
-      #     path: pynxtools-apm
-      # - name: Run apm tests
-      #   run: |
-      #     cd pynxtools-apm
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
+          cd ${{ matrix.plugin }}
+          pytest ${{ matrix.tests_to_run }}

docs/how-tos/using-pynxtools-test-framework.md

@@ -12,7 +12,6 @@ It is very simple to write a test to verify the plugin integration with `pynxtoo
 
 import os
 
-from pynxtools_foo.reader import FOOReader
 import pytest
 from pynxtools.testing.nexus_conversion import ReaderTest
 
@@ -21,22 +20,22 @@ module_dir = os.path.dirname(os.path.abspath(__file__))
 
 
 @pytest.mark.parametrize(
-    "nxdl,reader,files_or_dir",
+    "nxdl,reader_name,files_or_dir",
     [
-        ("NXfoo", FOOReader, f"{module_dir}/../test/data/test_data_dir_1"),
-        ("NXfoo", FOOReader, f"{module_dir}/../test/data/test_data_dir_2")
+        ("NXfoo", "foo", f"{module_dir}/../test/data/test_data_dir_1"),
+        ("NXfoo", "foo", f"{module_dir}/../test/data/test_data_dir_2")
     ],
 )
-def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
+def test_foo_reader(nxdl, reader_name, files_or_dir, tmp_path, caplog):
     """Test for the FooReader or foo reader plugin.
 
     Parameters
     ----------
     nxdl : str
         Name of the NXDL application definition that is to be tested by
         this reader plugin (e.g. NXfoo), without the file ending .nxdl.xml.
-    reader : class
-        Name of the class of the reader (e.g.READERFoo)
+    reader_name : str
+        Name of the class of the reader (e.g. "foo")
     files_or_dir : class
         Name of the class of the reader.
     tmp_path : pytest.fixture
@@ -46,11 +45,17 @@ def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
         Pytest fixture variable, used to capture the log messages during the test.
     """
     # test plugin reader
-    test = ReaderTest(nxdl, reader, files_or_dir, tmp_path, caplog)
+    test = ReaderTest(nxdl, reader_name, files_or_dir, tmp_path, caplog)
     test.convert_to_nexus()
     # Use `ignore_undocumented` to skip undocumented fields
     # test.convert_to_nexus(ignore_undocumented=True)
     test.check_reproducibility_of_nexus()
 ```
 
-Alongside the test data in the `test/data`, it is also possible to add other type of test data inside the test directory of the plugin. It is also possible to pass the boolean `ignore_undocumented` to `test.convert_to_nexus`. If true, any undocumented keys are ignored in the verification and it is simply checked if the required fields are properly set.
+Alongside the test data in `test/data`, it is also possible to add other types of test data inside the test directory of the plugin.
+
+You can also pass additional parameters to `test.convert_to_nexus`:
+
+- `log_level` (str): Can be either "ERROR" (by default) or "warning". This parameter determines the level at which the caplog is set during testing. If it is "WARNING", the test will also fail if any warnings are reported by the reader.
+
+- `ignore_undocumented` (boolean): If true, the test skipts the verification of undocumented keys. Otherwise, a warning massages for undocumented keys is raised

pynxtools/testing/nexus_conversion.py

@@ -1,14 +1,13 @@
 """Generic test for reader plugins."""
 
+from typing import Literal
+
 import logging
 import os
 from glob import glob
 
-from pynxtools.dataconverter.helpers import (
-    generate_template_from_nxdl,
-    get_nxdl_root_and_path,
-)
-from pynxtools.dataconverter.template import Template
+from pynxtools.dataconverter.helpers import get_nxdl_root_and_path
+from pynxtools.dataconverter.convert import get_reader, transfer_data_into_template
 from pynxtools.dataconverter.validation import validate_dict_against
 from pynxtools.dataconverter.writer import Writer
 from pynxtools.nexus import nexus
@@ -33,15 +32,15 @@ def get_log_file(nxs_file, log_file, tmp_path):
 class ReaderTest:
     """Generic test for reader plugins."""
 
-    def __init__(self, nxdl, reader, files_or_dir, tmp_path, caplog) -> None:
+    def __init__(self, nxdl, reader_name, files_or_dir, tmp_path, caplog) -> None:
         """Initialize the test object.
 
         Parameters
         ----------
         nxdl : str
             Name of the NXDL application definition that is to be tested by this reader plugin (e.g. NXsts, NXmpes, etc).
-        reader : class
-            The reader class (e.g. STMReader, MPESReader) to be tested.
+        reader_name : str
+            The name of the reader class (e.g. stm, mpes, xps, ...) to be tested.
         files_or_dir : str
             List of input files or full path string to the example data directory that contains all the files
             required for running the data conversion through the reader.
@@ -55,18 +54,22 @@ def __init__(self, nxdl, reader, files_or_dir, tmp_path, caplog) -> None:
         """
 
         self.nxdl = nxdl
-        self.reader = reader
+        self.reader_name = reader_name
+        self.reader = get_reader(self.reader_name)
         self.files_or_dir = files_or_dir
         self.ref_nexus_file = ""
         self.tmp_path = tmp_path
         self.caplog = caplog
         self.created_nexus = f"{tmp_path}/{os.sep}/output.nxs"
 
-    def convert_to_nexus(self, ignore_undocumented=False):
+    def convert_to_nexus(
+        self,
+        caplog_level: Literal["ERROR", "WARNING"] = "ERROR",
+        ignore_undocumented: bool = False,
+    ):
         """
         Test the example data for the reader plugin.
         """
-
         assert hasattr(
             self.reader, "supported_nxdls"
         ), f"Reader{self.reader} must have supported_nxdls attribute"
@@ -85,24 +88,26 @@ def convert_to_nexus(self, ignore_undocumented=False):
             self.nxdl in self.reader.supported_nxdls
         ), f"Reader does not support {self.nxdl} NXDL."
 
-        root, nxdl_file = get_nxdl_root_and_path(self.nxdl)
+        nxdl_root, nxdl_file = get_nxdl_root_and_path(self.nxdl)
         assert os.path.exists(nxdl_file), f"NXDL file {nxdl_file} not found"
-        template = Template()
-        generate_template_from_nxdl(root, template)
 
-        read_data = self.reader().read(
-            template=Template(template), file_paths=tuple(input_files)
+        read_data = transfer_data_into_template(
+            input_file=input_files,
+            reader=self.reader_name,
+            nxdl_name=self.nxdl,
+            nxdl_root=nxdl_root,
+            skip_verify=True,
         )
 
-        assert isinstance(read_data, Template)
-        with self.caplog.at_level("ERROR", "WARNING"):
+        # Clear the log of `transfer_data_into_template`
+        self.caplog.clear()
+
+        with self.caplog.at_level(caplog_level):
             _ = validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )
-        assert not self.caplog.records, "Validation is not successful. Check logs."
-        for record in self.caplog.records:
-            if record.levelname == "ERROR":
-                assert False, record.message
+        assert self.caplog.text == ""
+
         Writer(read_data, nxdl_file, self.created_nexus).write()
 
     def check_reproducibility_of_nexus(self):