From 7cab1e1963dec9ffc2ccd55727dd776da431781b Mon Sep 17 00:00:00 2001 From: Daniel Woste Date: Mon, 3 Jul 2023 21:54:29 +0200 Subject: [PATCH] Initial JSON Parser implementation (#65) * Initial JSON Parser implementation (80% done) * Uses JSON parser if file has json extension * test and docs * doc fixes * Linter problems fixed * Updated font * Added testcases for the :ref:`json_parser` and updated docs. * Updated `parse_testsuite()` and `parse_testcase()` to read dict-keys from `tr_json_mapping` config value. * Bugfix and complex json test --------- Co-authored-by: Duodu Randy --- docs/Makefile | 2 +- docs/changelog.rst | 5 +- docs/conf.py | 1 - docs/configuration.rst | 80 ++++++- docs/examples/index.rst | 33 ++- docs/filter.rst | 2 + docs/index.rst | 1 + docs/parsers.rst | 52 +++++ docs/ub_theme/conf.py | 2 +- .../test_reports/directives/test_case.py | 6 +- .../test_reports/directives/test_common.py | 16 +- .../test_reports/directives/test_env.py | 4 +- .../test_reports/directives/test_report.py | 3 +- .../test_reports/directives/test_suite.py | 2 +- sphinxcontrib/test_reports/jsonparser.py | 97 ++++++++ sphinxcontrib/test_reports/junitparser.py | 4 + sphinxcontrib/test_reports/test_reports.py | 52 ++++- tests/conftest.py | 14 +- tests/doc_test/json_parser/Makefile | 20 ++ tests/doc_test/json_parser/conf.py | 192 +++++++++++++++ tests/doc_test/json_parser/index.rst | 9 + tests/doc_test/json_parser/make.bat | 36 +++ tests/doc_test/json_parser_complex/Makefile | 20 ++ tests/doc_test/json_parser_complex/conf.py | 219 ++++++++++++++++++ tests/doc_test/json_parser_complex/index.rst | 9 + tests/doc_test/json_parser_complex/make.bat | 36 +++ tests/doc_test/utils/json_complex_data.json | 53 +++++ tests/doc_test/utils/json_data.json | 49 ++++ tests/test_json_parser.py | 111 +++++++++ 29 files changed, 1094 insertions(+), 36 deletions(-) create mode 100644 docs/parsers.rst create mode 100644 sphinxcontrib/test_reports/jsonparser.py create mode 100644 tests/doc_test/json_parser/Makefile create mode 100644 tests/doc_test/json_parser/conf.py create mode 100644 tests/doc_test/json_parser/index.rst create mode 100644 tests/doc_test/json_parser/make.bat create mode 100644 tests/doc_test/json_parser_complex/Makefile create mode 100644 tests/doc_test/json_parser_complex/conf.py create mode 100644 tests/doc_test/json_parser_complex/index.rst create mode 100644 tests/doc_test/json_parser_complex/make.bat create mode 100644 tests/doc_test/utils/json_complex_data.json create mode 100644 tests/doc_test/utils/json_data.json create mode 100644 tests/test_json_parser.py diff --git a/docs/Makefile b/docs/Makefile index 28a4ee2..0cb8ae4 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,7 +6,7 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = sphinx-test-reports SOURCEDIR = . -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/changelog.rst b/docs/changelog.rst index eb6dcc4..8f0b5d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,9 +7,12 @@ Changelog ----- :Released: under development +* Added testcases for the :ref:`json_parser` and updated docs. * Updated project documentation to use the Useblocks theme customization. -* Improvement: Template file encoding coud be configured. See :ref:`tr_import_encoding`. +* Improvement: Template file encoding could be configured. See :ref:`tr_import_encoding`. `#60 `_ +* Improvement: Supporting JSON files containing test results: :ref:`json_parser`. +* Improvement: Implemented :ref:`tr_json_mapping` config option for JSON mapping. 1.0.2 ----- diff --git a/docs/conf.py b/docs/conf.py index ddf9019..70dffdb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -111,7 +111,6 @@ other_options = { "repo_url": "https://github.com/useblocks/sphinx-test-reports", "repo_name": "sphinx-test-reports", - "repo_type": "github", } html_theme_options.update(other_options) html_theme_options["features"].extend(["navigation.tabs", "navigation.tabs.sticky"]) diff --git a/docs/configuration.rst b/docs/configuration.rst index 11fce68..1af3fe3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -109,9 +109,87 @@ Default: **5** .. _tr_import_encoding: tr_import_encoding -__________________ +------------------ .. versionadded:: 1.0.3 Defines the encoding for imported files, e.g. in custom templates. Default: **utf8** + +.. _tr_json_mapping: + +tr_json_mapping +--------------- +.. versionadded:: 1.0.3 + +Takes a mapping configuration, which defines how to map the JSON structure to the internal structure used by +``Sphinx-Test-Reports``. + +``tr_json_mapping`` is a dictionary, where the first key is a name for the configuration. +The name is currently just a placeholder and the first config is used for all JSON imports. + +Two mappings must be configured as dictionary, one for ``testsuite`` and one for the nested ``testcase``. + +The key of this dictionary elements is the **internal** name and fix. + +The value is a tuple, containing a **selector list** and a **default value**, if the selector does not find any data. + +The **selector** is a list, where each entry is representing one level of the data structure. +If the entry is a string, it is used as a key for a dict. If it is a integer number, it is taken as position +of a list. + +**JSON example** + +.. code-block:: python + + { + "level_1": { + "level_2": [ + {"value": "Hello!"} + {"value": "Bye Bye!"} + ] + } + } + +Given the above JSON example, the following "selector" will address the value ``Bye Bye!``:: + + ["level_1", "level_2", 1, "value"] + + +**Example config** + +This example contains **all** internal elements and a mapping as example. +For ``testsuite`` the value ``testcases`` defines the location of nested testcases. + +An example of a JSON file, which supports the below configuration, can be seen in :ref:`json_example`. + +.. code-block:: python + + tr_json_mapping = { + "json_config_1": { + "testsuite": { + "name": (["name"], "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (["testcase"], "unknown"), + }, + "testcase": { + "name": (["name"], "unknown"), + "classname": (["classname"], "unknown"), + "file": (["file"], "unknown"), + "line": (["line"], "unknown"), + "time": (["time"], "unknown"), + "result": (["result"], "unknown"), + "type": (["type"], "unknown"), + "text": (["text"], "unknown"), + "message": (["message"], "unknown"), + "system-out": (["system-out"], "unknown"), + } + } + } + + diff --git a/docs/examples/index.rst b/docs/examples/index.rst index c7494b1..0b2e827 100644 --- a/docs/examples/index.rst +++ b/docs/examples/index.rst @@ -1,7 +1,32 @@ Examples ======== -.. contents:: Contents - :local: + +.. _json_example: + +JSON data example +----------------- +Data: + +.. literalinclude:: ../../tests/doc_test/utils/json_data.json + +.. code-block:: rst + + .. test-file:: My JSON Test file + :file: ../tests/doc_test/utils/json_data.json + :id: JSON_001 + :auto_suites: + :auto_cases: + + +.. test-file:: My JSON Test file + :file: ../tests/doc_test/utils/json_data.json + :id: JSON_001 + :auto_suites: + :auto_cases: + + + + Importing --------- @@ -132,5 +157,5 @@ Test framework related examples .. toctree:: :maxdepth: 1 - pytest.rst - casperjs.rst + pytest + casperjs diff --git a/docs/filter.rst b/docs/filter.rst index b7af787..449a815 100644 --- a/docs/filter.rst +++ b/docs/filter.rst @@ -1,3 +1,5 @@ +:hide-navigation: + .. _filter: Filtering Test Data diff --git a/docs/index.rst b/docs/index.rst index d5abc5d..eba24ab 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -75,6 +75,7 @@ Content install directives/index configuration + parsers filter functions examples/index diff --git a/docs/parsers.rst b/docs/parsers.rst new file mode 100644 index 0000000..4577994 --- /dev/null +++ b/docs/parsers.rst @@ -0,0 +1,52 @@ +:hide-navigation: + +Parsers +======= + +``Sphinx-Test-Reports`` provides different parsers for test result files. +One for **XML-files**, following the **JUnit** format, and one for +generic **JSON-files**, which are not following any standard. + +The needed parser is automatically selected by the file extensions ``xml`` or ``json``. + +Junit Parser +------------ +This parser reads **xml** files and handles the content as **JUnit** data. +Other XML formats are not supported. + +As **JUnit** format is not really standardized, different test framework produce slightly different JUnit xml files. +``Sphinx-Test-Reports`` supports the "dialects" of: + +* pytest +* GoogleTest +* CasperJS + +There is a high chance, that other tet frameworks are supported as well, as long as they provide a Junit file. + +.. _json_parser: + +JSON Parser +----------- + +The JSON parser can read any JSON file. But as the data structure is not following any standard, the user need to +provide a mapping between the used JSON structure and the internal structure used by ``Sphinx-Test-Reports``. + +Even if this means some more configuration effort, the benefit is a completely customizable data structure. + +For an example please take a look into :ref:`json_example`. + +For mapping configuration please see :ref:`tr_json_mapping`. + + +Technical details +----------------- +Each parser is realized by a class, which needs to provide specific functions. +Also the internal object representation is the same. So each parser must map the external format to the internal +representation. + +Only the parser cares about the format. Other internal functions (like directives) just work with the common +data representation and rely on it. +So parsers are not allowed to rename or even extend this internal representation. +If this is needed, all available parsers need to be updated as well. + + diff --git a/docs/ub_theme/conf.py b/docs/ub_theme/conf.py index 6d8d8d3..7e76ffc 100644 --- a/docs/ub_theme/conf.py +++ b/docs/ub_theme/conf.py @@ -2,7 +2,7 @@ "icon": { "repo": "fontawesome/brands/github", }, - "font": {"code": "JetBrains Mono", "text": "Urbanist"}, + "font": {"code": "JetBrains Mono", "text": "Recursive"}, "globaltoc_collapse": True, "features": [ "navigation.top", diff --git a/sphinxcontrib/test_reports/directives/test_case.py b/sphinxcontrib/test_reports/directives/test_case.py index de0aaa5..ab8b1c9 100644 --- a/sphinxcontrib/test_reports/directives/test_case.py +++ b/sphinxcontrib/test_reports/directives/test_case.py @@ -44,13 +44,13 @@ def run(self, nested=False, suite_count=-1, case_count=-1): # access n-th nested suite here self.results = self.results[0]["testsuites"][suite_count] - suite_name = self.options.get("suite", None) + suite_name = self.options.get("suite") if suite_name is None: raise TestReportInvalidOption("Suite not given!") - case_full_name = self.options.get("case", None) - class_name = self.options.get("classname", None) + case_full_name = self.options.get("case") + class_name = self.options.get("classname") if case_full_name is None and class_name is None: raise TestReportInvalidOption("Case or classname not given!") diff --git a/sphinxcontrib/test_reports/directives/test_common.py b/sphinxcontrib/test_reports/directives/test_common.py index 9f5e9c9..68c4af3 100644 --- a/sphinxcontrib/test_reports/directives/test_common.py +++ b/sphinxcontrib/test_reports/directives/test_common.py @@ -8,7 +8,9 @@ from sphinx.util import logging from sphinx_needs.api import make_hashed_id -from sphinxcontrib.test_reports.exceptions import SphinxError, TestReportFileNotSetException +from sphinxcontrib.test_reports.exceptions import ( + SphinxError, TestReportFileNotSetException) +from sphinxcontrib.test_reports.jsonparser import JsonParser from sphinxcontrib.test_reports.junitparser import JUnitParser # fmt: on @@ -65,7 +67,11 @@ def load_test_file(self): return None if self.test_file not in self.app.testreport_data.keys(): - parser = JUnitParser(self.test_file) + if os.path.splitext(self.test_file)[1] == ".json": + mapping = list(self.app.config.tr_json_mapping.values())[0] + parser = JsonParser(self.test_file, json_mapping=mapping) + else: + parser = JUnitParser(self.test_file) self.app.testreport_data[self.test_file] = parser.parse() self.results = self.app.testreport_data[self.test_file] @@ -89,17 +95,17 @@ def prepare_basic_options(self): ), ) else: - self.test_id = self.options.get("id", None) + self.test_id = self.options.get("id") if self.test_id is None: raise SphinxError("ID must be set for test-report.") - self.test_file = self.options.get("file", None) + self.test_file = self.options.get("file") self.test_file_given = self.test_file[:] self.test_links = self.options.get("links", "") self.test_tags = self.options.get("tags", "") - self.test_status = self.options.get("status", None) + self.test_status = self.options.get("status") self.collapse = str(self.options.get("collapse", "")) diff --git a/sphinxcontrib/test_reports/directives/test_env.py b/sphinxcontrib/test_reports/directives/test_env.py index 6ab0024..de05467 100644 --- a/sphinxcontrib/test_reports/directives/test_env.py +++ b/sphinxcontrib/test_reports/directives/test_env.py @@ -39,8 +39,8 @@ class EnvReportDirective(Directive): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.data_option = self.options.get("data", None) - self.environments = self.options.get("env", None) + self.data_option = self.options.get("data") + self.environments = self.options.get("env") if self.environments is not None: self.req_env_list_cpy = self.environments.split(",") diff --git a/sphinxcontrib/test_reports/directives/test_report.py b/sphinxcontrib/test_reports/directives/test_report.py index 42e9a59..53c8065 100644 --- a/sphinxcontrib/test_reports/directives/test_report.py +++ b/sphinxcontrib/test_reports/directives/test_report.py @@ -4,7 +4,8 @@ from docutils import nodes from docutils.parsers.rst import directives -from sphinxcontrib.test_reports.directives.test_common import TestCommonDirective +from sphinxcontrib.test_reports.directives.test_common import \ + TestCommonDirective from sphinxcontrib.test_reports.exceptions import InvalidConfigurationError # fmt: on diff --git a/sphinxcontrib/test_reports/directives/test_suite.py b/sphinxcontrib/test_reports/directives/test_suite.py index 9d462c4..0d30da7 100644 --- a/sphinxcontrib/test_reports/directives/test_suite.py +++ b/sphinxcontrib/test_reports/directives/test_suite.py @@ -46,7 +46,7 @@ def run(self, nested=False, count=-1): # access n-th nested suite here self.results = self.results[0]["testsuite_nested"] - suite_name = self.options.get("suite", None) + suite_name = self.options.get("suite") if suite_name is None: raise TestReportInvalidOption("Suite not given!") diff --git a/sphinxcontrib/test_reports/jsonparser.py b/sphinxcontrib/test_reports/jsonparser.py new file mode 100644 index 0000000..fb6d20c --- /dev/null +++ b/sphinxcontrib/test_reports/jsonparser.py @@ -0,0 +1,97 @@ +""" +A parser for test results in JSON files. + +Must be configured via config, as there is no known standard for Test data in JSON files. + +API must be in sync with the JUnit parser in ``junitparser.py``. +""" +import json +import operator +import os +from functools import reduce +from typing import Any, Dict, List + + +def dict_get(root, items, default=None): + """ + Access a nested object in root by item sequence. + + Usage:: + data = {"nested": {"a_list": [{"finally": "target_data"}]}} + value = dict_get(data, ["nested", "a_list", 0, "finally"], "Not_found") + + """ + try: + value = reduce(operator.getitem, items, root) + except (KeyError, IndexError, TypeError): + return default + return value + + +class JsonParser: + def __init__(self, json_path, *args, **kwargs): + self.json_path = json_path + + if not os.path.exists(self.json_path): + raise JsonFileMissing(f"The given file does not exist: {self.json_path}") + + self.json_data = [] + with open(self.json_path) as jfile: + self.json_data = json.load(jfile) + + self.json_mapping = kwargs.get("json_mapping", {}) + + def validate(self): + # For JSON we validate nothing here. + # But to be compatible with the API, we need to return True + return True + + def parse(self) -> List[Dict[str, Any]]: + """ + Creates a common python list of object, no matter what information are + supported by the parsed json file for test results junit(). + + :return: list of test suites as dictionaries + """ + + def parse_testcase(json_dict) -> Dict[str, Any]: + tc_mapping = self.json_mapping.get("testcase") + tc_dict = { + k: dict_get(json_dict, v[0], v[1]) for k, v in tc_mapping.items() + } + return tc_dict + + def parse_testsuite(json_dict) -> Dict[str, Any]: + ts_mapping = self.json_mapping.get("testsuite") + ts_dict = { + k: dict_get(json_dict, v[0], v[1]) + for k, v in ts_mapping.items() + if k != "testcases" + } + ts_dict.update({"testcases": [], "testsuite_nested": []}) + + testcases = dict_get( + json_dict, ts_mapping["testcases"][0], ts_mapping["testcases"][1] + ) + for tc in testcases: + new_testcase = parse_testcase(tc) + ts_dict["testcases"].append(new_testcase) + + return ts_dict + + # main flow starts here + + result_data = [] + + for testsuite_data in self.json_data: + complete_testsuite = parse_testsuite(testsuite_data) + result_data.append(complete_testsuite) + + return result_data + + def docutils_table(self): + pass + + +class JsonFileMissing(BaseException): + pass diff --git a/sphinxcontrib/test_reports/junitparser.py b/sphinxcontrib/test_reports/junitparser.py index 0fe10cf..7430f6a 100644 --- a/sphinxcontrib/test_reports/junitparser.py +++ b/sphinxcontrib/test_reports/junitparser.py @@ -1,3 +1,7 @@ +""" +JUnit XML parser +""" + import os from lxml import etree, objectify diff --git a/sphinxcontrib/test_reports/test_reports.py b/sphinxcontrib/test_reports/test_reports.py index c9d74ce..25621c9 100644 --- a/sphinxcontrib/test_reports/test_reports.py +++ b/sphinxcontrib/test_reports/test_reports.py @@ -4,14 +4,21 @@ import sphinx # from docutils import nodes from pkg_resources import parse_version -from sphinx_needs.api import add_dynamic_function, add_extra_option, add_need_type - -from sphinxcontrib.test_reports.directives.test_case import TestCase, TestCaseDirective -from sphinxcontrib.test_reports.directives.test_env import EnvReport, EnvReportDirective -from sphinxcontrib.test_reports.directives.test_file import TestFile, TestFileDirective -from sphinxcontrib.test_reports.directives.test_report import TestReport, TestReportDirective -from sphinxcontrib.test_reports.directives.test_results import TestResults, TestResultsDirective -from sphinxcontrib.test_reports.directives.test_suite import TestSuite, TestSuiteDirective +from sphinx_needs.api import (add_dynamic_function, add_extra_option, + add_need_type) + +from sphinxcontrib.test_reports.directives.test_case import (TestCase, + TestCaseDirective) +from sphinxcontrib.test_reports.directives.test_env import (EnvReport, + EnvReportDirective) +from sphinxcontrib.test_reports.directives.test_file import (TestFile, + TestFileDirective) +from sphinxcontrib.test_reports.directives.test_report import ( + TestReport, TestReportDirective) +from sphinxcontrib.test_reports.directives.test_results import ( + TestResults, TestResultsDirective) +from sphinxcontrib.test_reports.directives.test_suite import ( + TestSuite, TestSuiteDirective) from sphinxcontrib.test_reports.environment import install_styles_static_files from sphinxcontrib.test_reports.functions import tr_link @@ -64,6 +71,35 @@ def setup(app): app.add_config_value("tr_case_id_length", 5, "html") app.add_config_value("tr_import_encoding", "utf8", "html") + json_mapping = { + "json_config": { + "testsuite": { + "name": (["name"], "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (["testcase"], "unknown"), + }, + "testcase": { + "name": (["name"], "unknown"), + "classname": (["classname"], "unknown"), + "file": (["file"], "unknown"), + "line": (["line"], "unknown"), + "time": (["time"], "unknown"), + "result": (["result"], "unknown"), + "type": (["type"], "unknown"), + "text": (["text"], "unknown"), + "message": (["message"], "unknown"), + "system-out": (["system-out"], "unknown"), + }, + } + } + + app.add_config_value("tr_json_mapping", json_mapping, "html", types=[dict]) + # nodes app.add_node(TestResults) app.add_node(TestFile) diff --git a/tests/conftest.py b/tests/conftest.py index 981eaab..d9fd8d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,19 +28,19 @@ def test_app(make_app, request): shutil.copytree(util_files, sphinx_test_tempdir / "utils") # copy test srcdir to test temporary directory sphinx_test_tempdir - srcdir = builder_params.get("srcdir", None) + srcdir = builder_params.get("srcdir") src_dir = copy_srcdir_to_tmpdir(srcdir, sphinx_test_tempdir) # return sphinx.testing fixture make_app and new srcdir which in sphinx_test_tempdir app = make_app( buildername=builder_params.get("buildername", "html"), srcdir=src_dir, - freshenv=builder_params.get("freshenv", None), - confoverrides=builder_params.get("confoverrides", None), - status=builder_params.get("status", None), - warning=builder_params.get("warning", None), - tags=builder_params.get("tags", None), - docutilsconf=builder_params.get("docutilsconf", None), + freshenv=builder_params.get("freshenv"), + confoverrides=builder_params.get("confoverrides"), + status=builder_params.get("status"), + warning=builder_params.get("warning"), + tags=builder_params.get("tags"), + docutilsconf=builder_params.get("docutilsconf"), parallel=builder_params.get("parallel", 0), ) diff --git a/tests/doc_test/json_parser/Makefile b/tests/doc_test/json_parser/Makefile new file mode 100644 index 0000000..47330b8 --- /dev/null +++ b/tests/doc_test/json_parser/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = needstestdocs +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/tests/doc_test/json_parser/conf.py b/tests/doc_test/json_parser/conf.py new file mode 100644 index 0000000..49826de --- /dev/null +++ b/tests/doc_test/json_parser/conf.py @@ -0,0 +1,192 @@ +# +# testreport test docs documentation build configuration file, created by +# sphinx-quickstart on Tue Mar 28 11:37:14 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("../../sphinxcontrib")) + +# -- General configuration ------------------------------------------------ + +# If your documentation testreport a minimal Sphinx version, state it here. +# +# testreport_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +extensions = ["sphinx_needs", "sphinxcontrib.test_reports"] + +needs_types = [ + { + "directive": "story", + "title": "User Story", + "prefix": "US_", + "color": "#BFD8D2", + "style": "node", + }, + { + "directive": "spec", + "title": "Specification", + "prefix": "SP_", + "color": "#FEDCD2", + "style": "node", + }, + { + "directive": "impl", + "title": "Implementation", + "prefix": "IM_", + "color": "#DF744A", + "style": "node", + }, + { + "directive": "test", + "title": "Test Case", + "prefix": "TC_", + "color": "#DCB239", + "style": "node", + }, +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "test-report test docs" +copyright = "2017, team useblocks" +author = "team useblocks" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = "1.0" +# The full version, including alpha/beta/rc tags. +release = "1.0" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ["_static"] + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "testreporttestdocsdoc" + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "testreporttestdocs.tex", + "testreport test docs Documentation", + "team useblocks", + "manual", + ), +] + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + master_doc, + "testreporttestdocs", + "testreport test docs Documentation", + [author], + 1, + ) +] + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "testreporttestdocs", + "testreport test docs Documentation", + author, + "testreporttestdocs", + "One line description of project.", + "Miscellaneous", + ), +] diff --git a/tests/doc_test/json_parser/index.rst b/tests/doc_test/json_parser/index.rst new file mode 100644 index 0000000..057469b --- /dev/null +++ b/tests/doc_test/json_parser/index.rst @@ -0,0 +1,9 @@ +JSON Parser Test +================ + + +.. test-file:: JSON Parser Test File + :file: ../utils/json_data.json + :id: PARSER_JSON_001 + :auto_suites: + :auto_cases: diff --git a/tests/doc_test/json_parser/make.bat b/tests/doc_test/json_parser/make.bat new file mode 100644 index 0000000..489ed7d --- /dev/null +++ b/tests/doc_test/json_parser/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=needstestdocs + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/tests/doc_test/json_parser_complex/Makefile b/tests/doc_test/json_parser_complex/Makefile new file mode 100644 index 0000000..47330b8 --- /dev/null +++ b/tests/doc_test/json_parser_complex/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = needstestdocs +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/tests/doc_test/json_parser_complex/conf.py b/tests/doc_test/json_parser_complex/conf.py new file mode 100644 index 0000000..bb0d11d --- /dev/null +++ b/tests/doc_test/json_parser_complex/conf.py @@ -0,0 +1,219 @@ +# +# testreport test docs documentation build configuration file, created by +# sphinx-quickstart on Tue Mar 28 11:37:14 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("../../sphinxcontrib")) + +# -- General configuration ------------------------------------------------ + +# If your documentation testreport a minimal Sphinx version, state it here. +# +# testreport_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +extensions = ["sphinx_needs", "sphinxcontrib.test_reports"] + +tr_json_mapping = { + "json_config_1": { + "testsuite": { + "name": (["internals", "name"], "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (["testcase", "nested"], "unknown"), + }, + "testcase": { + "name": (["name"], "unknown"), + "classname": (["classname"], "unknown"), + "file": (["file"], "unknown"), + "line": (["line"], "unknown"), + "time": (["time"], "unknown"), + "result": (["result"], "unknown"), + "type": (["type"], "unknown"), + "text": (["text"], "unknown"), + "message": (["message"], "unknown"), + "system-out": (["system-out"], "unknown"), + }, + } +} + +needs_types = [ + { + "directive": "story", + "title": "User Story", + "prefix": "US_", + "color": "#BFD8D2", + "style": "node", + }, + { + "directive": "spec", + "title": "Specification", + "prefix": "SP_", + "color": "#FEDCD2", + "style": "node", + }, + { + "directive": "impl", + "title": "Implementation", + "prefix": "IM_", + "color": "#DF744A", + "style": "node", + }, + { + "directive": "test", + "title": "Test Case", + "prefix": "TC_", + "color": "#DCB239", + "style": "node", + }, +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "test-report test docs" +copyright = "2017, team useblocks" +author = "team useblocks" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = "1.0" +# The full version, including alpha/beta/rc tags. +release = "1.0" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ["_static"] + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "testreporttestdocsdoc" + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "testreporttestdocs.tex", + "testreport test docs Documentation", + "team useblocks", + "manual", + ), +] + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + master_doc, + "testreporttestdocs", + "testreport test docs Documentation", + [author], + 1, + ) +] + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "testreporttestdocs", + "testreport test docs Documentation", + author, + "testreporttestdocs", + "One line description of project.", + "Miscellaneous", + ), +] diff --git a/tests/doc_test/json_parser_complex/index.rst b/tests/doc_test/json_parser_complex/index.rst new file mode 100644 index 0000000..eb641d9 --- /dev/null +++ b/tests/doc_test/json_parser_complex/index.rst @@ -0,0 +1,9 @@ +JSON Parser Complex Test +======================== + + +.. test-file:: JSON Parser Test File + :file: ../utils/json_complex_data.json + :id: PARSER_JSON_001 + :auto_suites: + :auto_cases: diff --git a/tests/doc_test/json_parser_complex/make.bat b/tests/doc_test/json_parser_complex/make.bat new file mode 100644 index 0000000..489ed7d --- /dev/null +++ b/tests/doc_test/json_parser_complex/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=needstestdocs + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/tests/doc_test/utils/json_complex_data.json b/tests/doc_test/utils/json_complex_data.json new file mode 100644 index 0000000..5722774 --- /dev/null +++ b/tests/doc_test/utils/json_complex_data.json @@ -0,0 +1,53 @@ +[ + { + "internals": { + "name": "test suite 1" + }, + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": { + "nested": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } + } +] \ No newline at end of file diff --git a/tests/doc_test/utils/json_data.json b/tests/doc_test/utils/json_data.json new file mode 100644 index 0000000..7a4fa50 --- /dev/null +++ b/tests/doc_test/utils/json_data.json @@ -0,0 +1,49 @@ +[ + { + "name": "test suite 1", + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": [ + { + "name": "test case 1", + "classname": "class name 1" , + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } +] \ No newline at end of file diff --git a/tests/test_json_parser.py b/tests/test_json_parser.py new file mode 100644 index 0000000..2636f91 --- /dev/null +++ b/tests/test_json_parser.py @@ -0,0 +1,111 @@ +import hashlib +import os +from pathlib import Path + +import pytest + +json_path = os.path.join(os.path.dirname(__file__), "doc_test/utils", "json_data.json") + + +def test_init_json_parser(): + from sphinxcontrib.test_reports.jsonparser import JsonParser + + parser = JsonParser(json_path) + assert parser is not None + + +def test_parse_json_data(): + from sphinxcontrib.test_reports.jsonparser import JsonParser + + json_mapping = { + "json_config": { + "testsuite": { + "name": (["name"], "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (["testcase"], "unknown"), + }, + "testcase": { + "name": (["name"], "unknown"), + "classname": (["classname"], "unknown"), + "file": (["file"], "unknown"), + "line": (["line"], "unknown"), + "time": (["time"], "unknown"), + "result": (["result"], "unknown"), + "type": (["type"], "unknown"), + "text": (["text"], "unknown"), + "message": (["message"], "unknown"), + "system-out": (["system-out"], "unknown"), + }, + } + } + + mapping = list(json_mapping.values())[0] + parser = JsonParser(json_path, json_mapping=mapping) + results = parser.parse() + assert results is not None + assert len(results) == 1 + assert len(results[0]["testcases"]) == 3 + assert results[0]["testcases"][1]["result"] == "passed" + + +@pytest.mark.parametrize( + "test_app", + [{"buildername": "html", "srcdir": "doc_test/json_parser"}], + indirect=True, +) +def test_json_parser_build_html(test_app): + app = test_app + app.build() + html = Path(app.outdir, "index.html").read_text() + # Check TestFile need object + test_id = "PARSER_JSON_001" + assert 'Test-File' in html + assert "JSON Parser Test" in html + assert "PARSER_JSON_001" in html + assert html.count('Test-File') == 1 + + # Check TestSuite need object(s) + suite_name = "test suite 1" + hash_s = hashlib.sha1(suite_name.encode("UTF-8")).hexdigest() + suite_id = f"{test_id}_{hash_s.upper()[: app.config.tr_suite_id_length]}" + + assert 'Test-Suite' in html + assert suite_name in html + assert suite_id in html + assert html.count('Test-Suite') == 1 + + # Check TestCase need object(s) + case_name = "test case 2" + case_classname = "class name 2" + hash_c = hashlib.sha1( + case_classname.encode("UTF-8") + case_name.encode("UTF-8") + ).hexdigest() + case_id = f"{suite_id}_{hash_c.upper()[: app.config.tr_case_id_length]}" + + assert 'Test-Case' in html + assert case_name in html + assert case_id in html + # Check number of testcases + assert html.count('Test-Case') == 3 + + +@pytest.mark.parametrize( + "test_app", + [{"buildername": "html", "srcdir": "doc_test/json_parser_complex"}], + indirect=True, +) +def test_json_complex_parser_build_html(test_app): + app = test_app + app.build() + html = Path(app.outdir, "index.html").read_text() + + assert 'Test-File' in html + assert "JSON Parser Test" in html + assert "PARSER_JSON_001" in html + + assert "test case 2" in html