diff --git a/callback_plugins/custom_junit.py b/callback_plugins/custom_junit.py new file mode 100644 index 00000000..70bf5ebb --- /dev/null +++ b/callback_plugins/custom_junit.py @@ -0,0 +1,108 @@ +from ansible.plugins.callback.junit import CallbackModule as JunitCallbackModule +from ansible.plugins.callback.junit import HostData + +import os +import time +import re +from ansible.utils._junit_xml import TestCase, TestError, TestFailure, TestSuite, TestSuites + + +class CallbackModule(JunitCallbackModule): + """ + Custom callback that overrides the default JUnit callback + """ + CALLBACK_NAME = 'custom_junit' + + def __init__(self): + super(CallbackModule, self).__init__() + + # Custom environment variable handling + # Update this to parse these values from the config file, as well as the env. + self._output_dir = os.path.expanduser("~/.ansible.log") + self._test_case_prefix = os.getenv('JUNIT_TEST_CASE_PREFIX', 'TEST') + self._fail_on_ignore = 'true' # this is needed because we use "ignore_errors" on the playbooks so that all the tests are run + self._include_setup_tasks_in_report = os.getenv('JUNIT_INCLUDE_SETUP_TASKS_IN_REPORT', 'False').lower() + self._hide_task_arguments = os.getenv('JUNIT_HIDE_TASK_ARGUMENTS', 'True').lower() + self._task_class = False + + print("The output_dir is: %s" % self._output_dir) + # Ensure the output directory exists + if not os.path.exists(self._output_dir): + print("Creating output dir: %s" % (self._output_dir)) + os.makedirs(self._output_dir) + + def _finish_task(self, status, result): + """ record the results of a task for a single host """ + task_uuid = result._task._uuid + if hasattr(result, '_host'): + host_uuid = result._host._uuid + host_name = result._host.name + else: + host_uuid = 'include' + host_name = 'include' + + task_data = self._task_data[task_uuid] + + if self._fail_on_change == 'true' and status == 'ok' and result._result.get('changed', False): + status = 'failed' + + # ignore failure if expected and toggle result if asked for + if status == 'failed' and 'EXPECTED FAILURE' in task_data.name: + status = 'ok' + elif 'TOGGLE RESULT' in task_data.name: + if status == 'failed': + status = 'ok' + elif status == 'ok': + status = 'failed' + + if self._test_case_prefix in task_data.name: + task_data.add_host(HostData(host_uuid, host_name, status, result)) + + # Debugging + if task_data.name.startswith(self._test_case_prefix): + print(f"This task ({task_data.name}) starts with the test_prefix({self._test_case_prefix})") + if self._test_case_prefix in task_data.name: + print(f"This task ({task_data.name}) should be reported because it contains test_prefix({self._test_case_prefix})") + if status == 'failed': + print(f"This task ({task_data.name}) failed, but may not be reported") + + def mutate_task_name(self, task_name): + # Debugging + if not self._test_case_prefix in task_name: + print("task_name (%s) does not contain prefix (%s)" % (task_name, self._test_case_prefix)) + new_name = task_name + new_name = new_name.split("\n")[0] # only use the first line, so we can include IDs and additional description + # this covers when a task is included, but the including task is the one that is the test + new_name = new_name.split(":")[-1] # only provide the last part of the name when the role name is included + + if len(self._test_case_prefix) > 0: + # this one may not be needed... + new_name = new_name.split(self._test_case_prefix)[-1] # remove the test prefix and everything before it + + new_name = new_name.lower() + new_name = re.sub(r'\W', ' ', new_name) # replace all non-alphanumeric characters (except _) with a space + new_name = re.sub(r'(^\W*|\W*$)', '', new_name) # trim any trailing or leading non-alphanumeric characters + new_name = re.sub(r' +', '_', new_name) # replace any number of spaces with _ + + return new_name + + def _build_test_case(self, task_data, host_data): + """ + This is used in generate_report. The task_data and host data will get passed. + """ + # Use the original task name to define the final name + + print("%s\t(task_name, pre-_build_test_case)" % task_data.name) + tc = super()._build_test_case(task_data, host_data) + print("%s\t(tc.name, post-_build_test_case)" % tc.name) + tc.name = self.mutate_task_name(tc.name) + + print("%s\t(tc.name, post-mutate_task_name)" % tc.name) + + # These can be able to omit with a config option + # These two control whether testcases contain the system_out and + # system_err elements that show STDOUT and STDERR + tc.system_out = None + tc.system_err = None + tc.classname = "openstack-observability" + return tc diff --git a/ci/ansible.cfg b/ci/ansible.cfg index 9ac39d6e..7bb50a3f 100644 --- a/ci/ansible.cfg +++ b/ci/ansible.cfg @@ -1,5 +1,5 @@ [defaults] -callbacks_enabled = custom_logger +callbacks_enabled = custom_junit, custom_logger callback_plugins = ../callback_plugins # additional paths to search for roles roles_path = ../roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.ansible/roles:../../ci-framework/roles diff --git a/ci/github/test_logger.yml b/ci/github/test_logger.yml index dd99f4e2..03d45665 100644 --- a/ci/github/test_logger.yml +++ b/ci/github/test_logger.yml @@ -10,8 +10,8 @@ debug: msg="You can place the test ID anywhere in the task name." - name: | - Check how a multiline string task name is - handles for RHOSO-03 + [TEST] Check how a multiline string task name is + handled for RHOSO-03 debug: msg="You can use a multiline string for the task name." - name: try a test where there is no ID, just the RHOSO prefix @@ -23,22 +23,29 @@ - name: What happens when the ID is at the end RHELOSP-043 debug: msg="You can put the test ID at the end of the task name." - - name: This is a negative test + - name: "[TEST] This is a negative test" debug: msg="This task will not be reported by the custom_logger" - - name: Check what happens with multiple matches RHELOSP-054 RHOSO-056 + - name: | + [TEST] Check what happens with multiple matches + RHELOSP-054 RHOSO-056 debug: msg="If there are two test IDs, only the first one is reported." - - name: Check what happens with lowercase rhoso-066 + - name: | + [TEST] Check what happens with lowercase + rhoso-066 debug: msg="The test ID must be uppercase" - - name: Check that failed tests are also represented RHOSO-078 + - name: | + [TEST] Check that failed tests are also represented + RHOSO-078 when: true fail: msg: "If the task fails, the status will be reported since there's a test ID." - - name: "Set the name based on a var input" + - name: "[TEST] Set the name based on a var input" set_fact: testid: "RHOSO-1234" + - name: "run test with variable name - {{ testid }}" debug: msg="test" diff --git a/ci/github/test_logger_expected_xml b/ci/github/test_logger_expected_xml new file mode 100644 index 00000000..46fb2764 --- /dev/null +++ b/ci/github/test_logger_expected_xml @@ -0,0 +1,16 @@ + + + + + + + + + { + "changed": false, + "msg": "If the task fails, the status will be reported since there's a test ID." +} + + + + diff --git a/ci/report_result.yml b/ci/report_result.yml index 5f9f8bd9..e1731aba 100644 --- a/ci/report_result.yml +++ b/ci/report_result.yml @@ -1,5 +1,5 @@ --- -- name: Create the output files +- name: "Create the output files" hosts: - controller vars_files: @@ -11,7 +11,7 @@ state: directory mode: "0755" -- name: Run the log file collection +- name: "Run the log file collection" hosts: - controller - compute @@ -19,14 +19,35 @@ vars_files: - vars/common.yml tasks: - - name: Collect the custom_logger results + - name: "Find the XML file" + ansible.builtin.shell: + cmd: | + find ~/.ansible.log/ -name *.xml + register: xml_file_list + ignore_errors: true + + - name: "Show the XML files" + ansible.builtin.debug: + var: xml_file_list.stdout_lines + ignore_errors: true + + - name: "Collect the XML files, renaming them for the host that they are collected from" + delegate_to: controller + ansible.builtin.copy: + remote_src: true + src: "{{ item }}" + dest: "{{ logs_dir }}/{{ item | basename }} " + with_items: "{{ xml_file_list.stdout_lines }}" + when: xml_file_list.stdout_lines | length != 0 + + - name: "Collect the custom_logger results" delegate_to: controller ansible.builtin.copy: remote_src: true src: "{{ ansible_env.HOME }}/test_run_result.out" dest: "{{ logs_dir }}/test_run_result.out" - - name: Collect the results summary + - name: "Collect the results summary" delegate_to: controller ansible.builtin.copy: remote_src: true