Adds test for example warnings and adds --undocumented flag (#155)

* Adds test for example warnings * Remove unecesary cli tools from test_readers * Linting * Adds a flag to suppress undocumented fields (Fixes #144) * Supress too-many-locals * Fixes mpes config file
FAIRmat-NFDI · Sep 15, 2023 · debd5cb · debd5cb
1 parent 43d952f
commit debd5cb
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 13 deletions.
diff --git a/pynxtools/dataconverter/convert.py b/pynxtools/dataconverter/convert.py
@@ -36,6 +36,7 @@
 
 
 logger = logging.getLogger(__name__)  # pylint: disable=C0103
+UNDOCUMENTED = 9
 logger.setLevel(logging.INFO)
 logger.addHandler(logging.StreamHandler(sys.stdout))
 
@@ -63,13 +64,14 @@ def get_names_of_all_readers() -> List[str]:
     return all_readers
 
 
-# pylint: disable=too-many-arguments
+# pylint: disable=too-many-arguments,too-many-locals
 def convert(input_file: Tuple[str],
             reader: str,
             nxdl: str,
             output: str,
             generate_template: bool = False,
             fair: bool = False,
+            undocumented: bool = False,
             **kwargs):
     """The conversion routine that takes the input parameters and calls the necessary functions."""
     # Reading in the NXDL and generating a template
@@ -89,6 +91,9 @@ def convert(input_file: Tuple[str],
 
     nxdl_root = ET.parse(nxdl_path).getroot()
 
+    if undocumented:
+        logger.setLevel(UNDOCUMENTED)
+
     template = Template()
     helpers.generate_template_from_nxdl(nxdl_root, template)
     if generate_template:
@@ -120,7 +125,11 @@ def convert(input_file: Tuple[str],
     for path in data.undocumented.keys():
         if "/@default" in path:
             continue
-        logger.warning("The path, %s, is being written but has no documentation.", path)
+        logger.log(
+            UNDOCUMENTED,
+            "The path, %s, is being written but has no documentation.",
+            path
+        )
 
     Writer(data=data, nxdl_path=nxdl_path, output_path=output).write()
 
@@ -170,20 +179,28 @@ def parse_params_file(params_file):
     is_flag=True,
     default=False,
     help='Let the converter know to be stricter in checking the documentation.'
-)  # pylint: disable=too-many-arguments
+)
 @click.option(
     '--params-file',
     type=click.File('r'),
     default=None,
     help='Allows to pass a .yaml file with all the parameters the converter supports.'
 )
+@click.option(
+    '--undocumented',
+    is_flag=True,
+    default=False,
+    help='Shows a log output for all fields which are undocumented'
+)
+# pylint: disable=too-many-arguments
 def convert_cli(input_file: Tuple[str],
                 reader: str,
                 nxdl: str,
                 output: str,
                 generate_template: bool,
                 fair: bool,
-                params_file: str):
+                params_file: str,
+                undocumented: bool):
     """The CLI entrypoint for the convert function"""
     if params_file:
         try:
@@ -199,7 +216,7 @@ def convert_cli(input_file: Tuple[str],
             sys.tracebacklimit = 0
             raise IOError("\nError: Please supply an NXDL file with the option:"
                           " --nxdl <path to NXDL>")
-        convert(input_file, reader, nxdl, output, generate_template, fair)
+        convert(input_file, reader, nxdl, output, generate_template, fair, undocumented)
 
 
 if __name__ == '__main__':

diff --git a/tests/data/dataconverter/readers/mpes/config_file.json b/tests/data/dataconverter/readers/mpes/config_file.json
@@ -339,12 +339,12 @@
   "/ENTRY[entry]/DATA[data]/@signal": "data",
   "/ENTRY[entry]/DATA[data]/data": "@data:data",
   "/ENTRY[entry]/DATA[data]/data/@units": "counts",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[kx]": "@data:kx.data",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[kx]/@units": "@data:kx.unit",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[ky]": "@data:ky.data",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[ky]/@units": "@data:ky.unit",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[energy]": "@data:energy.data",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[energy]/@units": "@data:energy.unit",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[delay]": "@data:delay.data",
-  "/ENTRY[entry]/DATA[data]/VARIABLE[delay]/@units": "@data:delay.unit"
+  "/ENTRY[entry]/DATA[data]/AXISNAME[kx]": "@data:kx.data",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[kx]/@units": "@data:kx.unit",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[ky]": "@data:ky.data",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[ky]/@units": "@data:ky.unit",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[energy]": "@data:energy.data",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[energy]/@units": "@data:energy.unit",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[delay]": "@data:delay.data",
+  "/ENTRY[entry]/DATA[data]/AXISNAME[delay]/@units": "@data:delay.unit"
 }
diff --git a/tests/dataconverter/test_readers.py b/tests/dataconverter/test_readers.py
@@ -102,3 +102,32 @@ def test_has_correct_read_func(reader):
 
             assert isinstance(read_data, Template)
             assert validate_data_dict(template, read_data, root)
+
+
+@pytest.mark.parametrize("reader_name,nxdl,undocumented_keys", [
+    ('mpes', 'NXmpes', ['/@default'])
+])
+def test_shows_correct_warnings(reader_name, nxdl, undocumented_keys):
+    """
+    Checks whether the read function generates the correct warnings.
+    """
+    def_dir = os.path.join(os.getcwd(), "pynxtools", "definitions")
+    dataconverter_data_dir = os.path.join("tests", "data", "dataconverter")
+
+    input_files = sorted(
+        glob.glob(os.path.join(dataconverter_data_dir, "readers", reader_name, "*"))
+    )
+    nxdl_file = os.path.join(
+        def_dir, "contributed_definitions", f"{nxdl}.nxdl.xml"
+    )
+
+    root = ET.parse(nxdl_file).getroot()
+    template = Template()
+    generate_template_from_nxdl(root, template)
+
+    read_data = get_reader(reader_name)().read(
+        template=Template(template), file_paths=tuple(input_files)
+    )
+
+    assert validate_data_dict(template, read_data, root)
+    assert list(read_data.undocumented.keys()) == undocumented_keys