From 7f6cf6c95af95104bdcd35e91bc0c5acf1927def Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 14:09:24 +0200
Subject: [PATCH 01/19] small changes to plugin tests

---
 .github/workflows/plugin_test.yaml    |  2 +-
 pynxtools/testing/nexus_conversion.py | 19 ++++++++++---------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml
index dd92a0e33..4b5f1b5b8 100644
--- a/.github/workflows/plugin_test.yaml
+++ b/.github/workflows/plugin_test.yaml
@@ -70,7 +70,7 @@ jobs:
       - name: Run XPS tests
         run: |
           cd pynxtools-xps
-          git checkout main
+          git checkout update-tests
           pip install .
           pytest tests/.
       # # Test for ellipsometry Plugin
diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 774449737..c2969ae8e 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -7,6 +7,7 @@
 from pynxtools.dataconverter.helpers import (
     generate_template_from_nxdl,
     get_nxdl_root_and_path,
+    write_nexus_def_to_entry,
 )
 from pynxtools.dataconverter.template import Template
 from pynxtools.dataconverter.validation import validate_dict_against
@@ -62,7 +63,7 @@ def __init__(self, nxdl, reader, files_or_dir, tmp_path, caplog) -> None:
         self.caplog = caplog
         self.created_nexus = f"{tmp_path}/{os.sep}/output.nxs"
 
-    def convert_to_nexus(self, ignore_undocumented=False):
+    def convert_to_nexus(self, ignore_undocumented: bool = False):
         """
         Test the example data for the reader plugin.
         """
@@ -94,15 +95,15 @@ def convert_to_nexus(self, ignore_undocumented=False):
             template=Template(template), file_paths=tuple(input_files)
         )
 
+        entry_names = read_data.get_all_entry_names()  # type: ignore[attr-defined]
+        for entry_name in entry_names:
+            write_nexus_def_to_entry(read_data, entry_name, self.nxdl)
+
         assert isinstance(read_data, Template)
-        with self.caplog.at_level("ERROR", "WARNING"):
-            _ = validate_dict_against(
-                self.nxdl, read_data, ignore_undocumented=ignore_undocumented
-            )
-        assert not self.caplog.records, "Validation is not successful. Check logs."
-        for record in self.caplog.records:
-            if record.levelname == "ERROR":
-                assert False, record.message
+
+        with self.caplog.at_level(logging.WARNING):
+            assert validate_dict_against(self.nxdl, read_data, ignore_undocumented=True)
+        assert self.caplog.text == ""
         Writer(read_data, nxdl_file, self.created_nexus).write()
 
     def check_reproducibility_of_nexus(self):

From 28bd1e4490b9a144e139befabf5acfcc84d47712 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 14:56:12 +0200
Subject: [PATCH 02/19] reuse transfer_data_into_template in test

---
 pynxtools/testing/nexus_conversion.py | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index c2969ae8e..05b14bf8a 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -4,12 +4,8 @@
 import os
 from glob import glob
 
-from pynxtools.dataconverter.helpers import (
-    generate_template_from_nxdl,
-    get_nxdl_root_and_path,
-    write_nexus_def_to_entry,
-)
-from pynxtools.dataconverter.template import Template
+from pynxtools.dataconverter.helpers import get_nxdl_root_and_path
+from pynxtools.dataconverter.convert import transfer_data_into_template
 from pynxtools.dataconverter.validation import validate_dict_against
 from pynxtools.dataconverter.writer import Writer
 from pynxtools.nexus import nexus
@@ -86,21 +82,17 @@ def convert_to_nexus(self, ignore_undocumented: bool = False):
             self.nxdl in self.reader.supported_nxdls
         ), f"Reader does not support {self.nxdl} NXDL."
 
-        root, nxdl_file = get_nxdl_root_and_path(self.nxdl)
+        nxdl_root, nxdl_file = get_nxdl_root_and_path(self.nxdl)
         assert os.path.exists(nxdl_file), f"NXDL file {nxdl_file} not found"
-        template = Template()
-        generate_template_from_nxdl(root, template)
 
-        read_data = self.reader().read(
-            template=Template(template), file_paths=tuple(input_files)
+        read_data = transfer_data_into_template(
+            input_file=input_files,
+            reader=self.reader,
+            nxdl_name=self.nxdl,
+            nxdl_root=nxdl_root,
+            skip_verify=True,
         )
 
-        entry_names = read_data.get_all_entry_names()  # type: ignore[attr-defined]
-        for entry_name in entry_names:
-            write_nexus_def_to_entry(read_data, entry_name, self.nxdl)
-
-        assert isinstance(read_data, Template)
-
         with self.caplog.at_level(logging.WARNING):
             assert validate_dict_against(self.nxdl, read_data, ignore_undocumented=True)
         assert self.caplog.text == ""

From 82ea26c96104bcf90cc07f8762c1958e7ea80f20 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 15:06:40 +0200
Subject: [PATCH 03/19] use reader name instead of class

---
 docs/how-tos/using-pynxtools-test-framework.md | 9 ++++-----
 pynxtools/testing/nexus_conversion.py          | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/how-tos/using-pynxtools-test-framework.md b/docs/how-tos/using-pynxtools-test-framework.md
index 978bc4c6d..c38821f49 100644
--- a/docs/how-tos/using-pynxtools-test-framework.md
+++ b/docs/how-tos/using-pynxtools-test-framework.md
@@ -12,7 +12,6 @@ It is very simple to write a test to verify the plugin integration with `pynxtoo
 
 import os
 
-from pynxtools_foo.reader import FOOReader
 import pytest
 from pynxtools.testing.nexus_conversion import ReaderTest
 
@@ -23,8 +22,8 @@ module_dir = os.path.dirname(os.path.abspath(__file__))
 @pytest.mark.parametrize(
     "nxdl,reader,files_or_dir",
     [
-        ("NXfoo", FOOReader, f"{module_dir}/../test/data/test_data_dir_1"),
-        ("NXfoo", FOOReader, f"{module_dir}/../test/data/test_data_dir_2")
+        ("NXfoo", "foo", f"{module_dir}/../test/data/test_data_dir_1"),
+        ("NXfoo", "foo", f"{module_dir}/../test/data/test_data_dir_2")
     ],
 )
 def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
@@ -35,8 +34,8 @@ def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
     nxdl : str
         Name of the NXDL application definition that is to be tested by
         this reader plugin (e.g. NXfoo), without the file ending .nxdl.xml.
-    reader : class
-        Name of the class of the reader (e.g.READERFoo)
+    reader : str
+        Name of the class of the reader (e.g. "foo")
     files_or_dir : class
         Name of the class of the reader.
     tmp_path : pytest.fixture
diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 05b14bf8a..8314a1839 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -38,7 +38,7 @@ def __init__(self, nxdl, reader, files_or_dir, tmp_path, caplog) -> None:
         nxdl : str
             Name of the NXDL application definition that is to be tested by this reader plugin (e.g. NXsts, NXmpes, etc).
         reader : class
-            The reader class (e.g. STMReader, MPESReader) to be tested.
+            The name of the reader class (e.g. stm, mpes, xps, ...) to be tested.
         files_or_dir : str
             List of input files or full path string to the example data directory that contains all the files
             required for running the data conversion through the reader.

From 856e6ad8377f681920663ec80da2320e72babd0f Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 15:16:55 +0200
Subject: [PATCH 04/19] use reader name for testing input

---
 pynxtools/testing/nexus_conversion.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 8314a1839..47b11dd23 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -5,7 +5,7 @@
 from glob import glob
 
 from pynxtools.dataconverter.helpers import get_nxdl_root_and_path
-from pynxtools.dataconverter.convert import transfer_data_into_template
+from pynxtools.dataconverter.convert import get_reader, transfer_data_into_template
 from pynxtools.dataconverter.validation import validate_dict_against
 from pynxtools.dataconverter.writer import Writer
 from pynxtools.nexus import nexus
@@ -30,14 +30,14 @@ def get_log_file(nxs_file, log_file, tmp_path):
 class ReaderTest:
     """Generic test for reader plugins."""
 
-    def __init__(self, nxdl, reader, files_or_dir, tmp_path, caplog) -> None:
+    def __init__(self, nxdl, reader_name, files_or_dir, tmp_path, caplog) -> None:
         """Initialize the test object.
 
         Parameters
         ----------
         nxdl : str
             Name of the NXDL application definition that is to be tested by this reader plugin (e.g. NXsts, NXmpes, etc).
-        reader : class
+        reader_name : str
             The name of the reader class (e.g. stm, mpes, xps, ...) to be tested.
         files_or_dir : str
             List of input files or full path string to the example data directory that contains all the files
@@ -52,7 +52,8 @@ def __init__(self, nxdl, reader, files_or_dir, tmp_path, caplog) -> None:
         """
 
         self.nxdl = nxdl
-        self.reader = reader
+        self.reader_name = reader
+        self.reader = get_reader(self.reader_name)
         self.files_or_dir = files_or_dir
         self.ref_nexus_file = ""
         self.tmp_path = tmp_path
@@ -87,7 +88,7 @@ def convert_to_nexus(self, ignore_undocumented: bool = False):
 
         read_data = transfer_data_into_template(
             input_file=input_files,
-            reader=self.reader,
+            reader=self.reader_name,
             nxdl_name=self.nxdl,
             nxdl_root=nxdl_root,
             skip_verify=True,

From 7e8f43b9c21f1cb1f60b59bd771fc8dfd1c5af61 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 15:23:16 +0200
Subject: [PATCH 05/19] update testing docs

typo fix
---
 docs/how-tos/using-pynxtools-test-framework.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/how-tos/using-pynxtools-test-framework.md b/docs/how-tos/using-pynxtools-test-framework.md
index c38821f49..db8921686 100644
--- a/docs/how-tos/using-pynxtools-test-framework.md
+++ b/docs/how-tos/using-pynxtools-test-framework.md
@@ -20,13 +20,13 @@ module_dir = os.path.dirname(os.path.abspath(__file__))
 
 
 @pytest.mark.parametrize(
-    "nxdl,reader,files_or_dir",
+    "nxdl,reader_name,files_or_dir",
     [
         ("NXfoo", "foo", f"{module_dir}/../test/data/test_data_dir_1"),
         ("NXfoo", "foo", f"{module_dir}/../test/data/test_data_dir_2")
     ],
 )
-def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
+def test_foo_reader(nxdl, reader_name, files_or_dir, tmp_path, caplog):
     """Test for the FooReader or foo reader plugin.
 
     Parameters
@@ -34,7 +34,7 @@ def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
     nxdl : str
         Name of the NXDL application definition that is to be tested by
         this reader plugin (e.g. NXfoo), without the file ending .nxdl.xml.
-    reader : str
+    reader_name : str
         Name of the class of the reader (e.g. "foo")
     files_or_dir : class
         Name of the class of the reader.
@@ -45,7 +45,7 @@ def test_foo_reader(nxdl, reader, files_or_dir, tmp_path, caplog):
         Pytest fixture variable, used to capture the log messages during the test.
     """
     # test plugin reader
-    test = ReaderTest(nxdl, reader, files_or_dir, tmp_path, caplog)
+    test = ReaderTest(nxdl, reader_name, files_or_dir, tmp_path, caplog)
     test.convert_to_nexus()
     # Use `ignore_undocumented` to skip undocumented fields
     # test.convert_to_nexus(ignore_undocumented=True)

From 3ea0c051a55f2d232a6b1db433e143c0a2485ff5 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 15:36:02 +0200
Subject: [PATCH 06/19] use reader name instead of class in testing

---
 pynxtools/testing/nexus_conversion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 47b11dd23..483727de9 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -52,7 +52,7 @@ def __init__(self, nxdl, reader_name, files_or_dir, tmp_path, caplog) -> None:
         """
 
         self.nxdl = nxdl
-        self.reader_name = reader
+        self.reader_name = reader_name
         self.reader = get_reader(self.reader_name)
         self.files_or_dir = files_or_dir
         self.ref_nexus_file = ""

From 19b6af8ed09fb8171e90f7cd93523b33fc4da974 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 15:55:58 +0200
Subject: [PATCH 07/19] clear caplog in testing

---
 pynxtools/testing/nexus_conversion.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 483727de9..1ba8ffa53 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -94,6 +94,9 @@ def convert_to_nexus(self, ignore_undocumented: bool = False):
             skip_verify=True,
         )
 
+        # Clear the log of `transfer_data_into_template`
+        self.caplog.clear()
+
         with self.caplog.at_level(logging.WARNING):
             assert validate_dict_against(self.nxdl, read_data, ignore_undocumented=True)
         assert self.caplog.text == ""

From cf9342e7469fa33248e313baff93fe11481fe2d7 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Tue, 28 May 2024 17:08:01 +0200
Subject: [PATCH 08/19] use different stm branch

---
 .github/workflows/plugin_test.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml
index 4b5f1b5b8..1531321a4 100644
--- a/.github/workflows/plugin_test.yaml
+++ b/.github/workflows/plugin_test.yaml
@@ -43,6 +43,7 @@ jobs:
       - name: Run STM tests
         run: |
           cd pynxtools-stm
+          git checkout use-new-testing
           git submodule sync --recursive
           git submodule update --init --recursive --jobs=4
           pip install .

From 2af7a5649622cc21d2366a53562d4e3ac3979b6c Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 10:35:58 +0200
Subject: [PATCH 09/19] allow different levels of error logging for plugin
 tests

---
 docs/how-tos/using-pynxtools-test-framework.md |  8 +++++++-
 pynxtools/testing/nexus_conversion.py          | 17 ++++++++++++++---
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/docs/how-tos/using-pynxtools-test-framework.md b/docs/how-tos/using-pynxtools-test-framework.md
index db8921686..5f899abe5 100644
--- a/docs/how-tos/using-pynxtools-test-framework.md
+++ b/docs/how-tos/using-pynxtools-test-framework.md
@@ -52,4 +52,10 @@ def test_foo_reader(nxdl, reader_name, files_or_dir, tmp_path, caplog):
     test.check_reproducibility_of_nexus()
 ```
 
-Alongside the test data in the `test/data`, it is also possible to add other type of test data inside the test directory of the plugin. It is also possible to pass the boolean `ignore_undocumented` to `test.convert_to_nexus`. If true, any undocumented keys are ignored in the verification and it is simply checked if the required fields are properly set.
+Alongside the test data in `test/data`, it is also possible to add other types of test data inside the test directory of the plugin.
+
+You can also pass additional parameters to `test.convert_to_nexus`:
+
+- `log_level` (str): Can be either "error" (by default) or "warning". This parameter determines the level at which the caplog is set during testing. If it is "warning", the test will also fail if any warnings are reported by the reader.
+
+- `ignore_undocumented` (boolean): If true, any undocumented keys are ignored in the verification and it is simply checked if the required fields are properly set.
diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 1ba8ffa53..29edf2851 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -1,5 +1,7 @@
 """Generic test for reader plugins."""
 
+from typing import Literal
+
 import logging
 import os
 from glob import glob
@@ -60,7 +62,11 @@ def __init__(self, nxdl, reader_name, files_or_dir, tmp_path, caplog) -> None:
         self.caplog = caplog
         self.created_nexus = f"{tmp_path}/{os.sep}/output.nxs"
 
-    def convert_to_nexus(self, ignore_undocumented: bool = False):
+    def convert_to_nexus(
+        self,
+        log_level: Literal["error", "warning"] = "error",
+        ignore_undocumented: bool = False,
+    ):
         """
         Test the example data for the reader plugin.
         """
@@ -97,8 +103,13 @@ def convert_to_nexus(self, ignore_undocumented: bool = False):
         # Clear the log of `transfer_data_into_template`
         self.caplog.clear()
 
-        with self.caplog.at_level(logging.WARNING):
-            assert validate_dict_against(self.nxdl, read_data, ignore_undocumented=True)
+        caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
+        caplog_level = caplog_levels.get(log_level, logging.ERROR)
+
+        with self.caplog.at_level(caplog_level):
+            assert validate_dict_against(
+                self.nxdl, read_data, ignore_undocumented=ignore_undocumented
+            )
         assert self.caplog.text == ""
         Writer(read_data, nxdl_file, self.created_nexus).write()
 

From 483e858ae022ce87e8cd0ac82d811a615da1f6e1 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 11:30:32 +0200
Subject: [PATCH 10/19] use matrix for testing plugins

---
 .github/workflows/plugin_test.yaml | 118 +++++++++--------------------
 1 file changed, 37 insertions(+), 81 deletions(-)

diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml
index 1531321a4..66b55e380 100644
--- a/.github/workflows/plugin_test.yaml
+++ b/.github/workflows/plugin_test.yaml
@@ -1,7 +1,7 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python 3.9 and run the tests of all supported plugins.
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: test plugin
+name: test plugins
 
 on:
   push:
@@ -11,21 +11,42 @@ on:
 
 jobs:
   pytest:
+    name: pytest (${{ matrix.plugin }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
+        include:
+          # - plugin: pynxtools-apm
+          #   branch: main
+          #   tests_to_run: tests/.
+          # - plugin: pynxtools-ellips
+          #   branch: main
+          #   tests_to_run: tests/.
+          # - plugin: pynxtools-em
+          #   branch: main
+          #   tests_to_run: tests/.
+          # - plugin: pynxtools-mpes
+          #   branch: main
+          #   tests_to_run: tests/.
+          - plugin: pynxtools-stm
+            branch: use-new-testing
+            tests_to_run: tests/test_reader.py::test_stm_reader
+          - plugin: pynxtools-xps
+            branch: update-tests
+            tests_to_run: tests/.
+          # - plugin: pynxtools-xrd
+          #   branch: update-tests
+          #   tests_to_run: tests/.
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Set up Python ${{ matrix.python_version }}
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python_version }}
+          python-version: 3.9
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -33,83 +54,18 @@ jobs:
       - name: Install package
         run: |
           pip install ".[dev]"
-      # Test for STM Plugin
-      - name: clone STM repo
+      - name: Clone ${{ matrix.plugin }} repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          repository: FAIRmat-NFDI/pynxtools-stm
-          path: pynxtools-stm
-      - name: Run STM tests
+          repository: FAIRmat-NFDI/${{ matrix.plugin }}
+          path: ${{ matrix.plugin }}
+          ref: ${{ matrix.branch }}
+      - name: Install ${{ matrix.plugin }}
         run: |
-          cd pynxtools-stm
-          git checkout use-new-testing
-          git submodule sync --recursive
-          git submodule update --init --recursive --jobs=4
-          pip install .
-          pytest tests/test_reader.py::test_stm_reader
-      # # Test for MPES Plugin
-      # - name: clone MPES repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-mpes
-      #     path: pynxtools-mpes
-      # - name: Run MPES tests
-      #   run: |
-      #     cd pynxtools-mpes
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
-      # Test for XPS Plugin
-      - name: clone XPS repo
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          repository: FAIRmat-NFDI/pynxtools-xps
-          path: pynxtools-xps
-      - name: Run XPS tests
+          cd ${{ matrix.plugin }} 
+          pip install .      
+      - name: Run ${{ matrix.plugin }} tests
         run: |
-          cd pynxtools-xps
-          git checkout update-tests
-          pip install .
-          pytest tests/.
-      # # Test for ellipsometry Plugin
-      # - name: clone ellipsometry repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-ellips
-      #     path: pynxtools-ellips
-      # - name: Run ellipsometry tests
-      #   run: |
-      #     cd pynxtools-ellips
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
-      # # Test for EM Plugin
-      # - name: clone EM repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-em
-      #     path: pynxtools-em
-      # - name: Run EM tests
-      #   run: |
-      #     cd pynxtools-em
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
-      # # Test for APM Plugin
-      # - name: clone APM repo
-      #   uses: actions/checkout@v4
-      #   with:
-      #     fetch-depth: 0
-      #     repository: FAIRmat-NFDI/pynxtools-apm
-      #     path: pynxtools-apm
-      # - name: Run apm tests
-      #   run: |
-      #     cd pynxtools-apm
-      #     git checkout main
-      #     pip install .
-      #     pytest tests/.
\ No newline at end of file
+          cd ${{ matrix.plugin }}
+          pytest ${{ matrix.tests_to_run }}
\ No newline at end of file

From dba6689e5116cff8c38ec387587c857bb3a4f0a1 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 12:20:25 +0200
Subject: [PATCH 11/19] print caplog level

---
 pynxtools/testing/nexus_conversion.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 29edf2851..ebe863272 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -107,6 +107,7 @@ def convert_to_nexus(
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
         with self.caplog.at_level(caplog_level):
+            print(self.caplog.level)
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )

From f14e52a806f090bf7555ee245b7ae39c34444ff4 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 12:21:22 +0200
Subject: [PATCH 12/19] print caplog level

---
 pynxtools/testing/nexus_conversion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index ebe863272..03cf57a50 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -107,7 +107,7 @@ def convert_to_nexus(
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
         with self.caplog.at_level(caplog_level):
-            print(self.caplog.level)
+            print(f"Caplog level: {self.caplog.level}")
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )

From f79c058eac701a57d729163f4f25060d08ea28c2 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 12:39:19 +0200
Subject: [PATCH 13/19] add some temporary debugging code

---
 pynxtools/testing/nexus_conversion.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 03cf57a50..d9beca5ab 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -71,6 +71,16 @@ def convert_to_nexus(
         Test the example data for the reader plugin.
         """
 
+        def print_caplog_level():
+            log_records = self.caplog.record_tuples
+
+            if log_records:
+                current_level = max(record[1] for record in log_records)
+            else:
+                current_level = "NOTSET"  # Default level if no logs captured
+
+            print(current_level)
+
         assert hasattr(
             self.reader, "supported_nxdls"
         ), f"Reader{self.reader} must have supported_nxdls attribute"
@@ -106,12 +116,17 @@ def convert_to_nexus(
         caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
+        print(f"Before setting caplog level: {print_caplog_level()}")
+
         with self.caplog.at_level(caplog_level):
-            print(f"Caplog level: {self.caplog.level}")
+            print(f"Inside context manager: {print_caplog_level()}")
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )
         assert self.caplog.text == ""
+
+        print(f"After setting caplog level: {print_caplog_level()}")
+
         Writer(read_data, nxdl_file, self.created_nexus).write()
 
     def check_reproducibility_of_nexus(self):

From d049fdcf4605231468421e7aaa5ad38269816da4 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 12:46:45 +0200
Subject: [PATCH 14/19] debugging of logging levels

---
 pynxtools/testing/nexus_conversion.py | 28 +++++++++++++--------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index d9beca5ab..97db8691c 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -70,17 +70,6 @@ def convert_to_nexus(
         """
         Test the example data for the reader plugin.
         """
-
-        def print_caplog_level():
-            log_records = self.caplog.record_tuples
-
-            if log_records:
-                current_level = max(record[1] for record in log_records)
-            else:
-                current_level = "NOTSET"  # Default level if no logs captured
-
-            print(current_level)
-
         assert hasattr(
             self.reader, "supported_nxdls"
         ), f"Reader{self.reader} must have supported_nxdls attribute"
@@ -116,16 +105,25 @@ def print_caplog_level():
         caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
-        print(f"Before setting caplog level: {print_caplog_level()}")
+        print(f"Before setting caplog level: {logging.getLevelName(caplog_level)}")
 
         with self.caplog.at_level(caplog_level):
-            print(f"Inside context manager: {print_caplog_level()}")
+            # Print inside
+            print(
+                f"Inside context manager - caplog level: {logging.getLevelName(caplog_level)}"
+            )
+
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )
-        assert self.caplog.text == ""
 
-        print(f"After setting caplog level: {print_caplog_level()}")
+            # Get the minimal log level of captured log records
+            min_log_level = min(record[1] for record in self.caplog.record_tuples)
+            print(f"Minimal log level captured: {logging.getLevelName(min_log_level)}")
+
+        print(f"After setting caplog level: {logging.getLevelName(caplog_level)}")
+
+        assert self.caplog.text == ""
 
         Writer(read_data, nxdl_file, self.created_nexus).write()
 

From c1be902a0b94b5fcb8ae63873ee6c8b6e773c5b6 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 12:51:07 +0200
Subject: [PATCH 15/19] remove print statements

---
 pynxtools/testing/nexus_conversion.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 97db8691c..dd75e1d9c 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -105,24 +105,10 @@ def convert_to_nexus(
         caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
-        print(f"Before setting caplog level: {logging.getLevelName(caplog_level)}")
-
         with self.caplog.at_level(caplog_level):
-            # Print inside
-            print(
-                f"Inside context manager - caplog level: {logging.getLevelName(caplog_level)}"
-            )
-
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )
-
-            # Get the minimal log level of captured log records
-            min_log_level = min(record[1] for record in self.caplog.record_tuples)
-            print(f"Minimal log level captured: {logging.getLevelName(min_log_level)}")
-
-        print(f"After setting caplog level: {logging.getLevelName(caplog_level)}")
-
         assert self.caplog.text == ""
 
         Writer(read_data, nxdl_file, self.created_nexus).write()

From 4985aa907d7886cf34deb0cd9a6ba0bb79dcc6c9 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 13:06:07 +0200
Subject: [PATCH 16/19] explicitly use logger object

---
 pynxtools/testing/nexus_conversion.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index dd75e1d9c..90e8862f4 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -70,6 +70,9 @@ def convert_to_nexus(
         """
         Test the example data for the reader plugin.
         """
+        logger = logging.getLogger(__name__)
+        logger.handlers.clear()
+
         assert hasattr(
             self.reader, "supported_nxdls"
         ), f"Reader{self.reader} must have supported_nxdls attribute"
@@ -105,7 +108,7 @@ def convert_to_nexus(
         caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
-        with self.caplog.at_level(caplog_level):
+        with self.caplog.at_level(caplog_level, logger=logger):
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )

From fb648df8289996a6dce89a09f0c894908f2d96de Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 13:25:27 +0200
Subject: [PATCH 17/19] revert to previous state

---
 pynxtools/testing/nexus_conversion.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index 90e8862f4..dd75e1d9c 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -70,9 +70,6 @@ def convert_to_nexus(
         """
         Test the example data for the reader plugin.
         """
-        logger = logging.getLogger(__name__)
-        logger.handlers.clear()
-
         assert hasattr(
             self.reader, "supported_nxdls"
         ), f"Reader{self.reader} must have supported_nxdls attribute"
@@ -108,7 +105,7 @@ def convert_to_nexus(
         caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
         caplog_level = caplog_levels.get(log_level, logging.ERROR)
 
-        with self.caplog.at_level(caplog_level, logger=logger):
+        with self.caplog.at_level(caplog_level):
             assert validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )

From 5b06b8d60f17c4221333d055ba5633435a72c87b Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 13:37:07 +0200
Subject: [PATCH 18/19] use string for caplog levels

---
 pynxtools/testing/nexus_conversion.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/pynxtools/testing/nexus_conversion.py b/pynxtools/testing/nexus_conversion.py
index dd75e1d9c..e8e3fec00 100644
--- a/pynxtools/testing/nexus_conversion.py
+++ b/pynxtools/testing/nexus_conversion.py
@@ -64,7 +64,7 @@ def __init__(self, nxdl, reader_name, files_or_dir, tmp_path, caplog) -> None:
 
     def convert_to_nexus(
         self,
-        log_level: Literal["error", "warning"] = "error",
+        caplog_level: Literal["ERROR", "WARNING"] = "ERROR",
         ignore_undocumented: bool = False,
     ):
         """
@@ -102,11 +102,8 @@ def convert_to_nexus(
         # Clear the log of `transfer_data_into_template`
         self.caplog.clear()
 
-        caplog_levels = {"warning": logging.WARNING, "error": logging.ERROR}
-        caplog_level = caplog_levels.get(log_level, logging.ERROR)
-
         with self.caplog.at_level(caplog_level):
-            assert validate_dict_against(
+            _ = validate_dict_against(
                 self.nxdl, read_data, ignore_undocumented=ignore_undocumented
             )
         assert self.caplog.text == ""

From b40ce1d51acf526712683d85a9ba0fa6cbd5f7d8 Mon Sep 17 00:00:00 2001
From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com>
Date: Wed, 29 May 2024 17:07:32 +0200
Subject: [PATCH 19/19] use main branches for xps and stm plugin tests

---
 .github/workflows/plugin_test.yaml             | 6 +++---
 docs/how-tos/using-pynxtools-test-framework.md | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml
index 66b55e380..ebc90c7f9 100644
--- a/.github/workflows/plugin_test.yaml
+++ b/.github/workflows/plugin_test.yaml
@@ -30,10 +30,10 @@ jobs:
           #   branch: main
           #   tests_to_run: tests/.
           - plugin: pynxtools-stm
-            branch: use-new-testing
-            tests_to_run: tests/test_reader.py::test_stm_reader
+            branch: main
+            tests_to_run: tests/.
           - plugin: pynxtools-xps
-            branch: update-tests
+            branch: main
             tests_to_run: tests/.
           # - plugin: pynxtools-xrd
           #   branch: update-tests
diff --git a/docs/how-tos/using-pynxtools-test-framework.md b/docs/how-tos/using-pynxtools-test-framework.md
index 5f899abe5..ff3ac15cf 100644
--- a/docs/how-tos/using-pynxtools-test-framework.md
+++ b/docs/how-tos/using-pynxtools-test-framework.md
@@ -56,6 +56,6 @@ Alongside the test data in `test/data`, it is also possible to add other types o
 
 You can also pass additional parameters to `test.convert_to_nexus`:
 
-- `log_level` (str): Can be either "error" (by default) or "warning". This parameter determines the level at which the caplog is set during testing. If it is "warning", the test will also fail if any warnings are reported by the reader.
+- `log_level` (str): Can be either "ERROR" (by default) or "warning". This parameter determines the level at which the caplog is set during testing. If it is "WARNING", the test will also fail if any warnings are reported by the reader.
 
-- `ignore_undocumented` (boolean): If true, any undocumented keys are ignored in the verification and it is simply checked if the required fields are properly set.
+- `ignore_undocumented` (boolean): If true, the test skipts the verification of undocumented keys. Otherwise, a warning massages for undocumented keys is raised
\ No newline at end of file