diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 84522452..3278cd2d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,37 +1,16 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.155.1/containers/ubuntu/.devcontainer/base.Dockerfile -ARG VARIANT="2.0.0-p4" -FROM checkmk/check-mk-enterprise:${VARIANT} +ARG VARIANT +# FROM checkmk/check-mk-enterprise:${VARIANT} +FROM robotmk-cmk-python3:${VARIANT} ARG PIP ENV CMK_PASSWORD="cmk" -# install python3 on the container -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install wget build-essential libreadline-gplv2-dev libncursesw5-dev \ - libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ - && cd /tmp && wget https://www.python.org/ftp/python/3.9.4/Python-3.9.4.tgz \ - && tar xzf Python-3.9.4.tgz \ - && cd Python-3.9.4 \ - && ./configure \ - && make build_all \ - && make install - -# install python modules to run the Robotmk plugin in this container -RUN pip3 install robotframework pyyaml mergedeep python-dateutil ipdb - -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq tree htop vim git telnet file lsyncd - # Creates the OMD site, executes post-start hook and halts before site start COPY docker-entrypoint.d /docker-entrypoint.d RUN /docker-entrypoint.sh /bin/true -# ADD requirements.txt /tmp/requirements.txt -# RUN PATH="/omd/sites/cmk/bin:${PATH}" \ -# OMD_ROOT="/omd/sites/cmk" \ -# /omd/sites/cmk/bin/${PIP} install -r /tmp/requirements.txt - # make the agent dir writeable from the CMK site (to link RF example tests) RUN chgrp -R cmk /usr/lib/check_mk_agent && chmod g+w /usr/lib/check_mk_agent diff --git a/.devcontainer/Dockerfile_cmk_python b/.devcontainer/Dockerfile_cmk_python new file mode 100644 index 00000000..5cab59a3 --- /dev/null +++ b/.devcontainer/Dockerfile_cmk_python @@ -0,0 +1,19 @@ +ARG VARIANT +FROM checkmk/check-mk-enterprise:$VARIANT + +# install python3 on the container +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install wget build-essential libreadline-gplv2-dev libncursesw5-dev \ + libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ + && cd /tmp && wget https://www.python.org/ftp/python/3.9.4/Python-3.9.4.tgz \ + && tar xzf Python-3.9.4.tgz \ + && cd Python-3.9.4 \ + && ./configure \ + && make build_all \ + && make install + +# install python modules to run the Robotmk plugin in this container +RUN pip3 install robotframework pyyaml mergedeep python-dateutil ipdb + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends jq tree htop vim git telnet file lsyncd diff --git a/.devcontainer/build-devcontainer.env b/.devcontainer/build-devcontainer.env new file mode 100644 index 00000000..52b70a7e --- /dev/null +++ b/.devcontainer/build-devcontainer.env @@ -0,0 +1,2 @@ +CMKVERSIONS="1.6.0p25 +2.0.0p5" \ No newline at end of file diff --git a/.devcontainer/build-devcontainer.sh b/.devcontainer/build-devcontainer.sh new file mode 100644 index 00000000..8c2c3c46 --- /dev/null +++ b/.devcontainer/build-devcontainer.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# This script should be executed at the very beginning to craft Docker images based on +# the original Checkmk 1/2 Docker images which also contain Python 3.9 and Robotframework. +# +# 1) Edit build-devcontainer.env and change the variable CMKVERSIONS to your needs. +# It should only contain CMK versions you want to test/develop on. +# 2) Start build-devcontainer.sh. It will check if the CMK Docker images are already +# available locally. If not, it asks for credentials to download the +# image from the CMK download page. +# 3) After the image tgz has been downloaded, it will be imported into Docker. +# (approx. 5 minutes) +# 4) In the last step, the script will build an image based on the CMK version, including +# Python3 and robotframework. (approx. 10 minutes) +# $ docker images | grep mk +# robotmk-cmk-python3 2.0.0p5 1d96bebf47a6 27 seconds ago 2.18GB +# robotmk-cmk-python3 1.6.0p25 599e8beeb9c7 10 minutes ago 1.93GB + + +# Name of the resulting images +IMAGE=robotmk-cmk-python3 +# load Checkmk versions +. build-devcontainer.env + +for VERSION in $CMKVERSIONS; do + docker images | egrep "checkmk/check-mk-enterprise.*$VERSION" 2>&1 > /dev/null + if [ $? -gt 0 ]; then + echo "Docker image checkmk/check-mk-enterprise.*$VERSION is not available locally." + read -p "Download this image? " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + + read -p "Username: " user + DOWNLOAD_FOLDER=$(mktemp -d) + URL=https://download.checkmk.com/checkmk/$VERSION + TGZ=check-mk-enterprise-docker-$VERSION.tar.gz + TGZ_FILE=${DOWNLOAD_FOLDER}/${TGZ} + echo "+ Downloading docker image $VERSION to $DOWNLOAD_FOLDER ..." + wget -P $DOWNLOAD_FOLDER --user $user ${URL}/${TGZ} --ask-password + if [ -f $TGZ_FILE ]; then + echo "+ Importing image $TGZ_FILE ..." + docker load -i $TGZ_FILE + else + echo "ERROR: $TGZ_FILE not found!" + fi + else + continue + fi + fi + echo "----" + echo "Docker image checkmk/check-mk-enterprise.*$VERSION is ready to use" + echo "----" + echo "Building now the image robotmk-cmk-python3:$VERSION from Dockerfile_cmk_python ..." + docker build -t robotmk-cmk-python3:$VERSION -f Dockerfile_cmk_python --build-arg VARIANT=$VERSION . +done \ No newline at end of file diff --git a/.devcontainer/create_dummyhost.sh b/.devcontainer/create_dummyhost.sh index 37172219..34729d7b 100755 --- a/.devcontainer/create_dummyhost.sh +++ b/.devcontainer/create_dummyhost.sh @@ -13,9 +13,9 @@ SITE=cmk echo "Creating a dummy host via webapi.py ... " # win10simdows -#curl -k "http://$HOST/$SITE/check_mk/webapi.py?action=add_host&_username=automation&_secret=$SECRET&request_format=python&output_format=python" -d "request={'hostname': 'win10simdows', 'folder': '', 'attributes': {'ipaddress': '192.168.116.8'}, 'create_folders': '1'}" +curl -k "http://$HOST/$SITE/check_mk/webapi.py?action=add_host&_username=automation&_secret=$SECRET&request_format=python&output_format=python" -d "request={'hostname': 'win10simdows', 'folder': '', 'attributes': {'ipaddress': '192.168.116.8'}, 'create_folders': '1'}" # localhost -curl -k "http://$HOST/$SITE/check_mk/webapi.py?action=add_host&_username=automation&_secret=$SECRET&request_format=python&output_format=python" -d "request={'hostname': 'localhost', 'folder': '', 'attributes': {'ipaddress': '127.0.0.1'}, 'create_folders': '1'}" +# curl -k "http://$HOST/$SITE/check_mk/webapi.py?action=add_host&_username=automation&_secret=$SECRET&request_format=python&output_format=python" -d "request={'hostname': 'localhost', 'folder': '', 'attributes': {'ipaddress': '127.0.0.1'}, 'create_folders': '1'}" echo "Discovering ... " cmk -IIv echo "Reloading CMK config ... " diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 03b40065..82151c70 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,10 +3,8 @@ "build": { "dockerfile": "Dockerfile", "args": { - // "VARIANT": "2.0.0p3", - // "VARIANT": "2.0.0p4", - "VARIANT": "2.0.0p5", - "PIP": "pip3" + "VARIANT": "1.6.0p25", + "PIP": "pip" }, }, "containerEnv": { @@ -17,7 +15,8 @@ "settings": { "terminal.integrated.shell.linux": "/bin/bash", // Update any extensions in Docker, how dare you - "extensions.autoUpdate": false + "extensions.autoUpdate": false, + "python.pythonPath": "/opt/omd/sites/cmk/bin/python" }, "extensions": [ "ms-python.python", @@ -26,14 +25,14 @@ ], "forwardPorts": [5000], - "postCreateCommand": ".devcontainer/postCreateCommand.sh", + "postCreateCommand": "/workspaces/robotmk/.devcontainer/postCreateCommand.sh", "remoteUser": "cmk", "remoteEnv": { - "PATH": "/omd/sites/cmk/bin:/omd/sites/cmk/local/lib/python3/bin/:${containerEnv:PATH}", + "PATH": "/omd/sites/cmk/bin:/omd/sites/cmk/local/lib/python/bin/:${containerEnv:PATH}", "OMD_ROOT": "/omd/sites/cmk", "OMD_SITE": "cmk", "CMK_SITE_ID": "cmk", "WORKSPACE": "${containerWorkspaceFolder}" - } + }, } \ No newline at end of file diff --git a/.devcontainer/devcontainer_v1.json b/.devcontainer/devcontainer_v1.json index 84e8fb3e..c2013c02 100644 --- a/.devcontainer/devcontainer_v1.json +++ b/.devcontainer/devcontainer_v1.json @@ -3,7 +3,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "VARIANT": "1.6.0p24", + "VARIANT": "1.6.0p25", "PIP": "pip" }, }, @@ -16,7 +16,7 @@ "terminal.integrated.shell.linux": "/bin/bash", // Update any extensions in Docker, how dare you "extensions.autoUpdate": false, - "python.pythonPath": "/omd/sites/cmk/bin/python2" + "python.pythonPath": "/omd/sites/cmk/bin/python" }, "extensions": [ "ms-python.python", diff --git a/.devcontainer/docker-entrypoint.d/post-create/install_cmk_agent.sh b/.devcontainer/docker-entrypoint.d/post-create/install_cmk_agent.sh index dcb0a729..288ab033 100644 --- a/.devcontainer/docker-entrypoint.d/post-create/install_cmk_agent.sh +++ b/.devcontainer/docker-entrypoint.d/post-create/install_cmk_agent.sh @@ -5,6 +5,8 @@ # As agent installers are only available after the first login into the site, # we do not have access to them. Instead, a recent deb gets installed. Will work # for most needs... +# As soon as the first installer has been baken by the bakery, the agent will +# anyhow have a version from the CMK server. echo "Installing the Checkmk agent..." DEB=$(realpath $(dirname $0))/cmk_agent.deb diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 7eb32d23..9f2c84a0 100755 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -1,7 +1,10 @@ #!/bin/bash -# This step ties the workspace files with the Devcontainer. -# V1 sites use lsyncd instead of symlinks because the bakery/rpmbuild needs real files instead of links. -.devcontainer/linkfiles.sh +# This step ties the workspace files with the Devcontainer. lsyncd is used to synchronize files. +/workspaces/robotmk/.devcontainer/linkfiles.sh + +# Password for the automation user +echo "secret" > /opt/omd/sites/cmk/var/check_mk/web/automation/automation.secret + # Fire up the site omd start \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0ba62c46..d4db14f8 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ dist/* mkpackage.pyc *.mkp *.swp -nohup.out +nohup.out \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 6b89ec03..467e361c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,22 @@ // allow to inspect also foreign code "justMyCode": false }, + { + "name": "devc V2.x bakery localhost", + "type": "python", + "request": "launch", + "program": "/omd/sites/cmk/bin/cmk", + "args": [ + "-A", + "-v", + "-f", + "localhost" + ], + "console": "integratedTerminal", + "python": "/omd/sites/cmk/bin/python3", + // allow to inspect also foreign code + "justMyCode": false + }, { "name": "devc V2.x bakery win10simdows", "type": "python", diff --git a/.vscode/settings.json b/.vscode/settings.json index dc5d2e5e..2a9d32f2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "python.pythonPath": "/omd/sites/cmk/bin/python3", + "python.pythonPath": "/omd/sites/cmk/bin/python", "python.linting.flake8Enabled": true, "python.linting.enabled": true, "python.testing.pytestArgs": [ @@ -10,10 +10,12 @@ "python.testing.pytestEnabled": true, "python.languageServer": "Pylance", "python.autoComplete.extraPaths": [ - "/omd/sites/cmk/lib/python3" + // "/omd/sites/cmk/lib/python3" + "/omd/sites/cmk/lib/python" ], "files.eol": "\n", "python.analysis.extraPaths": [ - "/omd/sites/cmk/lib/python3" + // "/omd/sites/cmk/lib/python3" + "/omd/sites/cmk/lib/python" ] -} \ No newline at end of file +} diff --git a/.vscode/tasks-chooser.json b/.vscode/tasks-chooser.json index c2230ae6..33125fb9 100644 --- a/.vscode/tasks-chooser.json +++ b/.vscode/tasks-chooser.json @@ -18,6 +18,10 @@ { "displayName": "▶︎ Set devcontainer to CMK v2", "command": "bash .devcontainer/set_devcontainer_version.sh 2" + }, + { + "displayName": "▶︎ Create dummyhost", + "command": "bash /workspaces/robotmk/.devcontainer/create_dummyhost.sh" } ], "baseItem": { diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 599ed295..ace9f305 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,8 +1,8 @@ { - "displayName": "▶︎ Set devcontainer to CMK v2", - "command": "cp .devcontainer/devcontainer{_v2,}.json", + "displayName": "▶︎ Create dummyhost", + "command": "bash /workspaces/robotmk/.devcontainer/create_dummyhost.sh", "version": "2.0.0", "type": "shell", "problemMatcher": [], - "chooserIndex": 2 + "chooserIndex": 3 } \ No newline at end of file diff --git a/agents_plugins/robotmk-runner.py b/agents_plugins/robotmk-runner.py index e84f0949..79bd7aaa 100755 --- a/agents_plugins/robotmk-runner.py +++ b/agents_plugins/robotmk-runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# (c) 2020 Simon Meggle +# (c) 2021 Simon Meggle # This file is part of Robotmk, a module for the integration of Robot # framework test results into Checkmk. @@ -20,7 +20,7 @@ try: # Import the main Robotmk functions from the same directory (Windows) - from robotmk import robotmk, RMKPlugin, RMKrunner, test_for_modules + from robotmk import * except ImportError: # If the import fails, try to import robotmk form the parent directory (Linux) # This is the case when the runner gets scheduled asynchronously on Linux where @@ -35,7 +35,7 @@ def main(): RMKPlugin.get_args() rmk = RMKrunner() cmdline_suites='all' # TBD: start suites from cmdline - rmk.start_suites(cmdline_suites) + rmk.run_suites(cmdline_suites) rmk.loginfo("... Quitting Runner, bye. ---") # It is important to write at least one byte to the agent so that it can save this # as a state with a cache_time. diff --git a/agents_plugins/robotmk.py b/agents_plugins/robotmk.py index ed62b3de..e3683bc7 100755 --- a/agents_plugins/robotmk.py +++ b/agents_plugins/robotmk.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# (c) 2020 Simon Meggle +# (c) 2021 Simon Meggle # This file is part of Robotmk, a module for the integration of Robot # framework test results into Checkmk. @@ -42,6 +42,10 @@ import platform import xml.etree.ElementTree as ET from enum import Enum +from abc import ABC, abstractmethod +import glob + +from robot.rebot import rebot local_tz = datetime.utcnow().astimezone().tzinfo @@ -109,9 +113,8 @@ def __suites_from_robotdirs(self): def lsuites(self): return self.cfg_dict['suites'].keys() - @property - def suite_objs(self): - return [RMKSuite(id, self) for id in self.cfg_dict['suites']] + def suite_objs(self, logger): + return [RMKSuite(id, self, logger) for id in self.cfg_dict['suites']] @property def global_dict(self): @@ -292,9 +295,10 @@ def read_state_from_file(self): return data def write_state_to_file(self, data=None): + """Writes the given data structure into the statefile. + Datetime objects are converted to ISO strings.""" if data is None: - data = self._state - # statefile always contains ISO datetimes + data = self._state data = {k: (v.isoformat() if type(v) is datetime else v) for (k, v) in data.items()} try: @@ -357,18 +361,21 @@ def get_statevar(self, name): # fn() # return inner - @property - def now(self): - # return datetime.now(timezone.utc) + def get_now_as_dt(self): return datetime.now(local_tz) + def get_now_as_epoch(self): + return int(self.get_now_as_dt().timestamp()) + class RMKSuite(RMKState): logmark = '~' - def __init__(self, id, config): + def __init__(self, id, config, logger): self.id = id self.config = config + self.logger = logger + self._timestamp = self.get_now_as_epoch() super().__init__() self.set_statevars([ @@ -378,7 +385,30 @@ def __init__(self, id, config): ('path', self.suite_dict['path']), ('tag', self.suite_dict.get('tag', None)), ]) - pass + + self.suite_dict.setdefault('robot_params', {}) + self.suite_dict['robot_params'].update({ + 'outputdir': self.global_dict['outputdir'], + 'console': 'NONE' + }) + # Ref: TgWQvr + if self.source == "local": + self._source_strategy = SourceStrategyFS(path=self.path) + elif self.source == "git": + self._source_strategy == SourceStrategyGit(path=self.path) + elif self.source == "robocorp": + self._source_strategy == SourceStrategyRobocorp(path=self.path) + else: + # TODO: catch this + pass + + if self.python == "os": + self._env_strategy = EnvStrategyOS(self) + elif self.python == "rcc": + self._env_strategy = EnvStrategyRCC(self) + else: + # TODO: catch this + pass def clear_statevars(self): data = {k: None for k in 'start_time end_time runtime rc xml htmllog'.split()} @@ -387,55 +417,35 @@ def clear_statevars(self): def __str__(self): return self.id - def update_filenames(self): - now = int(time()) - suite_filename = "robotframework_%s_%s" % (self.id, str(now)) - self.suite_dict.update({ - 'outputdir': self.global_dict['outputdir'], - 'output': f'{suite_filename}_output.xml', - 'log': f'{suite_filename}_log.html', - 'console': 'NONE', - # Make the Robotmk Library Keywords accessible from the .robot file - wherever it is - 'pythonpath': str(Path(self.global_dict['agent_data_dir']).joinpath('plugins')), - # 'report': f'{suite_filename}_report.html', - }) + def output_filename(self, timestamp, attempt=None): + """Create output file name. If attempt is given, it gets appended to the file name.""" + if attempt is None: + suite_filename = "robotframework_%s_%s" % (self.id, timestamp) + else: + suite_filename = "robotframework_%s_%s_attempt-%d" % (self.id, timestamp, attempt) + return suite_filename + + def update_output_filenames(self, attempt=None): + """Parametrize the output files""" + output_prefix = self.output_filename(str(self.timestamp), attempt) + self.suite_dict['robot_params'].update({ + 'output': "%s_output.xml" % output_prefix, + 'log': "%s_log.html" % output_prefix + }) + def clear_filenames(self): '''Reset the log file names if Robot Framework exited with RC > 250 The files presumed to exist do not in this case. ''' - self.outfile_htmllog = None - # self.outfile_htmlreport = None - self.outfile_xml = None + self.output = None + self.log = None - def robotize_variables(self): - # Preformat Variables to meet the Robot API requirement - # --variable name:value --variable name2:value2 - # => ['name:value', 'name2:value2'] (list of dicts to list of k:v) - if 'variable' in self.suite_dict: - self.suite_dict['variable'] = list( - map( - lambda x: f'{x[0]}:{x[1]}', - self.suite_dict['variable'].items() - )) - def run(self): - self.robotize_variables() - self.update_filenames() - self.write_statevars([ - ('id', self.id), - ('start_time', self.now), - ('cache_time', self.get_suite_or_global('cache_time')) - ]) - rc = robot.run( - self.path, - **self.robot_args) - self.set_statevars([ - ('htmllog', self.outfile_htmllog), - ('xml', self.outfile_xml), - ('end_time', self.now), - ('rc', rc), ]) - self.rc = rc + def run_strategy(self): + # Ref: TgWQvr + # start the suite either with OS Python/RCC/Docker + rc = self._env_strategy.run() return rc def get_suite_or_global(self, name, default=None): @@ -453,49 +463,146 @@ def path(self): # ).joinpath(self.robotpath) ).joinpath(self.suite_dict['path']) + @property + def outputdir(self): + return self.suite_dict['robot_params']['outputdir'] + @outputdir.setter + def outputdir(self, directory): + self.suite_dict['robot_params']['outputdir'] = directory + + @property + def output(self): + return self.suite_dict['robot_params']['output'] + @output.setter + def output(self, file): + self.suite_dict['robot_params']['output'] = file + + @property + def log(self): + return self.suite_dict['robot_params']['log'] + @log.setter + def log(self, file): + self.suite_dict['robot_params']['log'] = file + @property def runtime(self): return (self._state['end_time'] - self._state['start_time']).total_seconds() @property - def suite_dict(self): - return self.config.cfg_dict['suites'][self.id] + def python(self): + """Defines which Python interpreter to use (OS/RCC)""" + return self.suite_dict.get('python', 'os') @property - def robot_args(self): - '''We should pass an arg dict to Robot Framework which is cleaned by - any Robotmk keys. - ''' - robotmk_keys = 'cache_time execution_interval path tag piggybackhost'.split() - return {k: v for (k, v) in self.suite_dict.items() if k not in robotmk_keys} + def source(self): + return self.suite_dict.get('source', 'local') + + @property + def max_executions(self): + return self.suite_dict.get('failed_handling',{}).get('max_executions',1) + + @property + def rerun_selection(self): + return self.suite_dict.get('failed_handling', {}).get('rerun_selection',[]) + + @property + def suite_dict(self): + return self.config.cfg_dict['suites'][self.id] @property def global_dict(self): return self.config.cfg_dict['global'] - @property - def outfile_xml(self): - if not self.suite_dict['output'] is None: - return str(Path(self.global_dict['outputdir']).joinpath( - self.suite_dict['output'])) - else: - return None + # Suite timestamp for filenames @property - def outfile_htmllog(self): - if not self.suite_dict['log'] is None: - return str(Path(self.global_dict['outputdir']).joinpath( - self.suite_dict['log'])) - else: - return None + def timestamp(self): + return self._timestamp + @timestamp.setter + def timestamp(self, t): + self._timestamp = t + +# Ref: TgWQvr +class EnvStrategy(): + """Strategy interface which Python environment to use""" + def __init__(self): + pass + @abstractmethod + def run(self, suite: RMKSuite): + pass + +class EnvStrategyOS(EnvStrategy): + """Use the System Python environment""" + def __init__(self, suite: RMKSuite): + self._suite = suite + super().__init__() - @outfile_xml.setter - def outfile_xml(self, text): - self.suite_dict['output'] = None + def __str__(self): + return("OS Python") + + + def prepare_rf_args(self): + # Format the variables to meet the Robot API requirement + # --variable name:value --variable name2:value2 + # => ['name:value', 'name2:value2'] (list of dicts to list of k:v) + variables = self._suite.suite_dict.get('robot_params').get('variable') + if variables and type(variables) is not list: + variables = list( + map( + lambda x: f'{x[0]}:{x[1]}', + variables.items() + )) + self._suite.suite_dict['robot_params']['variable'] = variables + pass + + def run(self): + """Runs the Robot suite with the OS Python and RF API""" + self.prepare_rf_args() + rc = robot.run( + self._suite.path, + **self._suite.suite_dict.get('robot_params')) + return rc + +class EnvStrategyRCC(EnvStrategy): + """Use rcc to create a dedicated environment for the test""" + def __init__(self, suite: RMKSuite): + self._suite = suite + + def __str__(self): + return("RCC Env Python") + + def run(self, suite: RMKSuite) -> int: + pass + + +class SourceStrategy(): + """Strategy interface where to get the test source code from""" + def __init__(self, path): + self.path = path + pass + +class SourceStrategyFS(SourceStrategy): + """Read the test source from local filesystem""" + def __init__(self, path): + super().__init__(path) + pass + +class SourceStrategyGit(SourceStrategy): + """Clone the test source code from git""" + def __init__(self, path): + super().__init__(path) + pass + +class SourceStrategyRobocorp(SourceStrategy): + """Load a Robocorp Robot""" + def __init__(self, path): + super().__init__(path) + pass + + + + - @outfile_htmllog.setter - def outfile_htmllog(self, text): - self.suite_dict['log'] = None class RMKPlugin(): @@ -730,13 +837,7 @@ def update_runner_statevars(self): self._state['end_time'] - self._state['start_time']).total_seconds() runtime_suites = sum([suite.runtime for suite in self.suites]) runtime_robotmk = runtime_total - runtime_suites - # suites_nonfatal = [(s.id, s.runtime) - # for s in self.suites if s.rc < 252] - # suites_fatal = [(s.id, s.runtime) for s in self.suites if s.rc >= 252] - # suites = { - # 'suites_nonfatal': suites_nonfatal, - # 'suites_fatal': suites_fatal, - # } + self.set_statevars([ ('runtime_total', runtime_total), ('runtime_suites', runtime_suites), @@ -747,9 +848,9 @@ def update_runner_statevars(self): if self.execution_mode == 'agent_serial': self.set_statevars([('cache_time', self.config.global_dict['cache_time']), ( 'execution_interval', self.config.global_dict['execution_interval'])]) - elif self.execution_mode == 'agent_parallel': - self.set_statevars([('cache_time', self.config.suite_dict['cache_time']), ( - 'execution_interval', self.config.suite_dict['execution_interval'])]) + # elif self.execution_mode == 'agent_parallel': + # self.set_statevars([('cache_time', self.config.suite_dict['cache_time']), ( + # 'execution_interval', self.config.suite_dict['execution_interval'])]) elif self.execution_mode == 'external': if self.selective_run: self.set_statevars( @@ -769,12 +870,13 @@ def global_dict(self): def suites_dict(self): return self.config.cfg_dict['suites'] - def start_suites(self, suites_cmdline): + def run_suites(self, suites_cmdline): + """Executes all suites of robotmk.yml/robotdir""" self.update_suites2start(suites_cmdline) - self.suites = self.config.suite_objs + self.suites = self.config.suite_objs(self.logger) self.loginfo( ' => Suites to start: %s' % ', '.join([s.id for s in self.suites])) - self.write_statevars(('start_time', self.now)) + self.write_statevars(('start_time', self.get_now_as_dt())) for suite in self.suites: id = suite.id self.loginfo( @@ -787,10 +889,11 @@ def start_suites(self, suites_cmdline): # if he know about it -> if there is a valid entry in the config. suite.error = error # continue - self.logdebug(f'Starting suite') - rc = suite.run() - self.loginfo( - f'Suite finished with RC {rc}') + self.logdebug(f'Strategy: ' + str(suite._env_strategy) ) + + rc = self.run_suite(suite) + + if rc > 250: self.logerror( 'RC > 250 = Robot exited with fatal error. There are no logs written.') @@ -799,18 +902,92 @@ def start_suites(self, suites_cmdline): suite.clear_filenames() self.loginfo(f'Writing suite statefile {suite.statefile_path}') suite.write_state_to_file() - self.set_statevars(('end_time', self.now)) + self.set_statevars(('end_time', self.get_now_as_dt())) self.update_runner_statevars() self.write_state_to_file() -# rcontroller -# https://stackoverflow.com/a/2251026/14845044 + def merge_results(self, suite): + # output files without attempt suffix + suite.update_output_filenames() + outputfiles = self.glob_suite_outputfiles(suite) + self.logdebug("Merging the results of the following result files into %s: " % suite.output) + filenames = [Path(f).name for f in outputfiles] + for f in filenames: + self.logdebug(" - %s" % f) + rebot( + *outputfiles, + outputdir=suite.outputdir, + output=suite.output, + log=suite.log, + report=None, + merge=True + ) + + def run_suite(self, suite): + """Execute a single suite, including retries""" + suite.write_statevars([ + ('id', suite.id), + ('start_time', suite.get_now_as_dt()), + ('cache_time', suite.get_suite_or_global('cache_time')) + ]) + max_exec = suite.max_executions + for attempt in range(1, max_exec+1): + if max_exec > 1: + self.loginfo(f" > Starting attempt {attempt}/{max_exec}...") + else: + self.loginfo(f" > Starting suite...") + # output files with attempt suffix + suite.update_output_filenames(attempt) + # The execution + rc = suite.run_strategy() + self.loginfo(f" < RC: {rc}") + + if rc == 0: + if attempt == 1: + # Suite passed on the first try; exit the loop + break + else: + # Suite passed on a retry => MERGE + self.merge_results(suite) + break + else: + if max_exec == 1: + # Suite FAILED on the first and only try; exit the loop + break + else: + # Suite FAILED and... + if attempt < max_exec: + # ...chance for next try! + # save the current output XML and use it for the rerun + failed_xml = Path(suite.outputdir).joinpath(suite.output) + suite.suite_dict['robot_params'].update({'rerunfailed': str(failed_xml)}) + # Attempt 2ff can be filtered, add the parameters to the Robot cmdline + suite.suite_dict['robot_params'].update(suite.rerun_selection) + self.loginfo(f"Re-testing the failed ones in {failed_xml}") + else: + # ...GAME OVER! => MERGE + self.loginfo("Even the last attempt was unsuccessful!") + self.merge_results(suite) + + suite.set_statevars([ + ('htmllog', str(Path(suite.outputdir).joinpath(suite.log))), + ('xml', str(Path(suite.outputdir).joinpath(suite.output))), + ('end_time', self.get_now_as_dt()), + ('attempts', attempt), + ('max_executions', max_exec), + ('rc', rc)]) + self.loginfo( + f'Final suite RC: {rc}') + return rc + def glob_suite_outputfiles(self, suite): + """Returns a list of XML output files of all execution attempts""" + output_filename = suite.output_filename(suite.timestamp) + outputfiles = [file for file in glob.glob(str(Path(suite.outputdir).joinpath("%s_attempt*_output.xml" % output_filename)))] + return outputfiles class RMKCtrl(RMKState, RMKPlugin): - # TODO: Cleanup the XML, remove images (#79) - header = '<<>>' logmark = '=' @@ -859,7 +1036,7 @@ def schedule_runner(self): if self.execution_mode == 'agent_serial': execution_interval = timedelta( seconds=self.config.global_dict['execution_interval']) - if never_ran or (self.now > start_time + execution_interval): + if never_ran or (self.get_now_as_dt() > start_time + execution_interval): if never_ran: self.loginfo( "Execution interval (%ds) for Runner is elapsed since last start." % (execution_interval.seconds)) @@ -881,7 +1058,7 @@ def schedule_runner(self): else: # Idle... secs_to_execute = ( - start_time + execution_interval - self.now).seconds + start_time + execution_interval - self.get_now_as_dt()).seconds self.loginfo("Nothing to do. Next Runner execution in %ds (interval=%ds)" % ( secs_to_execute, execution_interval.seconds)) @@ -951,7 +1128,7 @@ def check_suite_statefiles(self, encoding): states = defaultdict(list) self.loginfo("%d Suites to check: %s" % (len(self.suites_dict.keys()), ', '.join(self.suites_dict.keys()))) - for suite in self.config.suite_objs: + for suite in self.config.suite_objs(self.logger): # if (piggyback)host is set, results gets assigned to other CMK host host = suite.suite_dict.get('piggybackhost', 'localhost') self.logdebug("Reading statefile of suite '%s': %s" % ( @@ -1087,6 +1264,7 @@ def test_for_modules(): import yaml global robot import robot + import robot.rebot global mergedeep import mergedeep global parser @@ -1095,11 +1273,9 @@ def test_for_modules(): print('<<>>') print( f'FATAL ERROR!: Robotmk cannot start because of a missing Python3 module (Error was: {str(e)})') + print('Please execute: pip3 install robotframework pyyaml mergedeep python-dateutil') exit(1) -# rmain - - def main(): test_for_modules() RMKPlugin.get_args() diff --git a/bakery/v1/robotmk.py b/bakery/v1/robotmk.py index 2aaa0ced..3b1d0be5 100644 --- a/bakery/v1/robotmk.py +++ b/bakery/v1/robotmk.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- encoding: utf-8; py-indent-offset: 4 -*- -# (c) 2020 Simon Meggle +# (c) 2021 Simon Meggle # This file is part of Robotmk # https://robotmk.org @@ -18,6 +18,8 @@ # to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, # Boston, MA 02110-1301 USA. +ROBOTMK_VERSION = 'v1.2-beta' + import cmk.utils.paths import os import yaml @@ -26,6 +28,21 @@ from cmk.utils.exceptions import MKGeneralException +DEFAULTS = { + 'windows': { + 'newline': "\r\n", + }, + 'linux': { + 'newline': "\n", + }, + 'posix': { + 'newline': "\n", + }, + 'noarch': { + 'cache_time': 900, + } +} + def bake_robotmk(opsys, conf, conf_dir, plugins_dir): # ALWAYS (!) make a deepcopy of the conf dict. Even if you do not change # anything on it, there are strange changes ocurring while building the @@ -36,29 +53,19 @@ def bake_robotmk(opsys, conf, conf_dir, plugins_dir): raise MKGeneralException( "Error in bakery plugin 'robotmk': Robotmk is only supported on Windows and Linux." ) - config = RMKConfigAdapter(myconf, opsys, execution_mode) + config = RMK(myconf, opsys, execution_mode) - # Robotmk RUNNER plugin (async, OS specific) + # Robotmk RUNNER plugin + # executed async, OS-specific if execution_mode == "agent_serial": if opsys == "windows": # async mode in Windows: write configfile in INI-style, will be converted # during installation to YML - # There the Robotmk terminology needs explanation: - # - The plugin "cache time" is in fact the "execution interval". - # - The plugin "timeout" is the max time the plugin is allowed to run - # before going stale = "cache time". - - with Path(conf_dir, "check_mk.ini.plugins.%s" % - config.os_rmk_runner).open("w") as out: - out.write(u" execution %s = async\r\n" % - config.os_rmk_runner) - out.write(u" cache_age %s = %d\r\n" % - (config.os_rmk_runner, - config.global_dict['execution_interval'])) + with Path(conf_dir, "check_mk.ini.plugins.robotmk-runner.py").open("w") as out: + out.write(u" execution robotmk-runner.py = async\r\n") + out.write(u" cache_age robotmk-runner.py = %d\r\n" % config.global_dict['execution_interval']) # Kill the plugin before the next async execution will start - out.write( - u" timeout %s = %d\r\n" % - (config.os_rmk_runner, config.global_dict['cache_time'])) + out.write(u" timeout robotmk-runner.py = %d\r\n" % config.global_dict['cache_time']) out.write(u"\r\n") plugins_dir_async = plugins_dir elif opsys == "linux": @@ -72,177 +79,195 @@ def bake_robotmk(opsys, conf, conf_dir, plugins_dir): ("robotmk", "Robotmk is supported on Windows and Linux only")) src = str( - Path( - cmk.utils.paths.local_agents_dir).joinpath('plugins').joinpath( - RMKConfigAdapter._DEFAULTS['linux']['rmk_runner'])) - dest = str(Path(plugins_dir_async).joinpath(config.os_rmk_runner)) + Path(cmk.utils.paths.local_agents_dir).joinpath('plugins/robotmk-runner.py')) + dest = str(Path(plugins_dir_async).joinpath('robotmk-runner.py')) shutil.copy2(src, dest) + elif execution_mode == "external": + # In CMK1 and external mode, the custom package "robotmk" must be deployed. + pass # II) Robotmk Controller plugin + # executed sync, regular plugin src = str( - Path(cmk.utils.paths.local_agents_dir).joinpath('plugins').joinpath( - RMKConfigAdapter._DEFAULTS['linux']['rmk_ctrl'])) - dest = str(Path(plugins_dir).joinpath(config.os_rmk_ctrl)) + Path(cmk.utils.paths.local_agents_dir).joinpath('plugins/robotmk.py')) + dest = str(Path(plugins_dir).joinpath('robotmk.py')) shutil.copy2(src, dest) - # I)I) Generate YML config file + # III) Generate robotmk.YML config file with open(conf_dir + "/robotmk.yml", "w") as robotmk_yml: - robotmk_yml.write(header) - yaml.safe_dump(config.cfg_dict, - robotmk_yml, - line_break=config.os_newline, - encoding='utf-8', - allow_unicode=True, - sort_keys=True) - - -class RMKConfigAdapter(): - _DEFAULTS = { - 'windows': { - 'newline': "\r\n", - 'robotdir': "C:\\ProgramData\\checkmk\\agent\\robot", - 'rmk_ctrl': 'robotmk.py', - 'rmk_runner': 'robotmk-runner.py' - }, - 'linux': { - 'newline': "\n", - 'robotdir': "/usr/lib/check_mk_agent/robot", - 'rmk_ctrl': 'robotmk.py', - 'rmk_runner': 'robotmk-runner.py' - }, - 'posix': { - 'newline': "\n", - 'robotdir': "/usr/lib/check_mk_agent/robot", - 'rmk_ctrl': 'robotmk.py', - 'rmk_runner': 'robotmk-runner.py' - }, - 'noarch': { - 'cache_time': 900, - } - } + yml_lines = get_yml_lines(config) + for line in yml_lines: + robotmk_yml.write(line + config.os_newline) + pass + +def get_yml_lines(config): + header = "# This file is part of Robotmk, a module for the integration of Robot\n" +\ + "# framework test results into Checkmk.\n" +\ + "#\n" +\ + "# https://robotmk.org\n" +\ + "# https://github.com/simonmeggle/robotmk\n" +\ + "# https://robotframework.org/\n" +\ + "# ROBOTMK VERSION: %s\n" % ROBOTMK_VERSION + headerlist = header.split('\n') + # PyYAML is very picky with Dict subclasses; add a representer to dump the data. + # https://github.com/yaml/pyyaml/issues/142#issuecomment-732556045 + yaml.add_representer( + DictNoNone, + lambda dumper, data: dumper.represent_mapping('tag:yaml.org,2002:map', data.items()) + ) + # Unicode representer, see https://stackoverflow.com/a/62207530 + yaml.add_representer( + unicode, + lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data) + ) + + bodylist = yaml.dump( + config.cfg_dict, + default_flow_style=False, + allow_unicode=True, + encoding='utf-8', + sort_keys=True).split('\n') + return headerlist + bodylist + + +# This dict only adds the new key only if +# * the key already exists +# * the value is a boolean in fact +# * the value contains something meaningful +# This prevents that empty dicts are set as values. +class DictNoNone(dict): + def __setitem__(self, key, value): + if (key in self or type(value) is bool) or bool(value): + dict.__setitem__(self, key, value) + +# This class is common with CMK 1/2 +class RMKSuite(): + def __init__(self, suite_tuple): + self.suite_tuple = suite_tuple + + @property + def suite2dict(self): + suite_dict = DictNoNone() + suite_dict['path']= self.path + suite_dict['tag']= self.tag + # Ref whYeq7 + suite_dict['piggybackhost']= self.piggybackhost + # Ref FF3Vph + suite_dict['robot_params'] = self.robot_params + # Ref au4uPB + suite_dict['failed_handling'] = self.failed_handling + return suite_dict + + # Ref a01uK3 + @property + def path(self): + return self.suite_tuple[0] + + # Ref yJE5bu + @property + def tag(self): + return self.suite_tuple[1].get('tag', None) + + # Ref whYeq7 + @property + def piggybackhost(self): + return self.suite_tuple[2].get('piggybackhost', None) + + # Ref FF3Vph + @property + def robot_params(self): + params = copy.deepcopy(self.suite_tuple[3].get('robot_params', {})) + # Variables: transform the var 'list of tuples' into a dict. + variables_dict = {} + for (k1, v1) in params.items(): + if k1 == 'variable': + for t in v1: + variables_dict.update({t[0]: t[1]}) + params.update(self.dict_if_set('variable', variables_dict)) + return params + + # Ref au4uPB + @property + def failed_handling(self): + failed_handling = copy.deepcopy(self.suite_tuple[4].get('failed_handling', {})) + ret = {} + if failed_handling: + ret.update({'max_executions': failed_handling[0]}) + ret.update(self.dict_if_set('rerun_selection', failed_handling[1])) + return ret + + @property + def suiteid(self): + '''Create a unique ID from the Robot path (dir/.robot file) and the tag. + with underscores for everything but letters, numbers and dot.''' + if bool(self.tag): + tag_suffix = "_%s" % self.tag + else: + tag_suffix = "" + composite = "%s%s" % (self.path, tag_suffix) + outstr = re.sub('[^A-Za-z0-9\.]', '_', composite) + # make underscores unique + return re.sub('_+', '_', outstr).lower() + + @staticmethod + # Return a dict with key:value only if value is set + def dict_if_set(key, value): + if bool(value): + return {key: value} + else: + return {} + +# This class is common with CMK 1/2 +class RMK(): def __init__(self, conf, opsys, execution_mode): - self.opsys = opsys - self.os_newline = self.get_os_default('newline') - self.os_rmk_ctrl = self.get_os_default('rmk_ctrl') - self.os_rmk_runner = self.get_os_default('rmk_runner') + self.os_newline = get_os_default('newline', opsys) + + self.execution_mode = conf['execution_mode'][0] + mode_conf = conf['execution_mode'][1] self.cfg_dict = { - 'global': {}, - 'suites': {}, + 'global': DictNoNone(), + 'suites': DictNoNone(), } + # handy dict shortcuts global_dict = self.cfg_dict['global'] suites_dict = self.cfg_dict['suites'] - global_dict['agent_output_encoding'] = conf['agent_output_encoding'] - global_dict['transmit_html'] = conf['transmit_html'] - global_dict['logging'] = conf['logging'] - global_dict['log_rotation'] = int(conf['log_rotation']) - - robotdir = conf.get('robotdir', None).get('robotdir', None) - global_dict.update(self.dict_if_set( - 'robotdir', - robotdir, - self._DEFAULTS[opsys]['robotdir'])) - - global_dict['execution_mode'] = execution_mode - - # mode_conf: - # >> suite tuples, cache, execution - mode_conf = conf['execution_mode'][1] - # mode specific settings - # Because of the WATO structure ("function follows form") we do not have - # keys here, but only a list of Tuples. Depending on the mode, fields - # are hidden, the indizes my vary! - # serial parallel external idx - # global - # cache_time x x 1 - # execution_interval x 0 - # suite - # cache_time x x - # execution_interval x - - if execution_mode in ['agent_serial']: + global_dict['execution_mode'] = self.execution_mode + global_dict['agent_output_encoding'] = conf['agent_output_encoding'] + global_dict['transmit_html'] = conf['transmit_html'] + global_dict['logging'] = conf['logging'] + global_dict['log_rotation'] = conf['log_rotation'] + # WATO makes robotdir a nested dict with duplicate key. Form follows function :-/ + global_dict['robotdir'] = conf.get('robotdir', {}).get('robotdir', None) + + if self.execution_mode == 'agent_serial': global_dict['cache_time'] = mode_conf[1] global_dict['execution_interval'] = mode_conf[2] - elif execution_mode in ['agent_parallel']: - # set nothing, done in suites! - pass - elif execution_mode in ['external']: + self.execution_interval = mode_conf[2] + elif self.execution_mode == 'external': # For now, we assume that the external mode is meant to execute all # suites exactly as configured. Hence, we can use the global cache time. - global_dict['cache_time'] = mode_conf[1] - # suite_tuple: - # >> path, tag, piggyback, robot_params{}, cachetime, execution_int + global_dict['cache_time'] = mode_conf[1] + if 'suites' in mode_conf[0]: + # each suite suite_tuple: + # 0) path, Ref a01uK3 + # 1) tag, Ref yJE5bu + # 2) piggybackhost, Ref whYeq7 + # 3) robot_params{}, Ref FF3Vph + # 4) failed_handling, Ref au4uPB for suite_tuple in mode_conf[0]['suites']: - #### PATH - path = suite_tuple[0] - - #### TAG - tag = suite_tuple[1].get('tag', None) - suiteid = make_suiteid(path, tag) - suitedict = { - 'path': path, - } - suitedict.update(self.dict_if_set('tag', tag)) - - #### PIGGYBACK - piggybackhost = suite_tuple[2].get('piggybackhost', {}) - suitedict.update(self.dict_if_set('piggybackhost', piggybackhost)) - - #### ROBOT PARAMS - robot_param_dict = suite_tuple[3].get('robot_params', {}) - # Variables: transform the var 'list of tuples' into a dict. - vardict = {} - for (k1, v1) in robot_param_dict.iteritems(): - if k1 == 'variable': - for t in v1: - vardict.update({t[0]: t[1]}) - robot_param_dict.update(self.dict_if_set('variable', vardict)) - suitedict.update(robot_param_dict) - - # CACHE & EXECUTION TIME - timing_dict = {} - if execution_mode == 'agent_parallel': - timing_dict.update({'cache_time': suite_tuple[4]}) - timing_dict.update({'execution_interval': suite_tuple[5]}) - # if execution_mode == 'external': - # timing_dict.update( - # {'cache_time': global_dict['cache_time']}) - suitedict.update(timing_dict) - if suiteid in self.cfg_dict['suites']: + suite = RMKSuite(suite_tuple) + if suite.suiteid in self.cfg_dict['suites']: raise MKGeneralException( - "Error in bakery plugin 'robotmk': Suite with ID %s is not unique. Please use tags to solve this problem." - ) - self.cfg_dict['suites'].update({suiteid: suitedict}) + "Error in bakery plugin 'robotmk': Suite with ID %s is not unique. Please use tags to solve this problem." % suite.suiteid + ) - # If the value is set, return a dict with the key. - # If not, return a dict with key and default value. - # If no default value, return an empty dict. - @staticmethod - def dict_if_set(key, value, default=None): - if bool(value): - return {key: value} - else: - if bool(default): - return {key: default} - else: - return {} - - def get_os_default(self, setting): - '''Read a setting from the DEFAULTS hash. If no OS setting is found, try noarch. - Args: - setting (str): Setting name - Returns: - str: The setting value - ''' - value = self._DEFAULTS[self.opsys].get(setting, None) - if value is None: - value = self._DEFAULTS['noarch'].get(setting, None) - if value is None: - raise MKGeneralException( - "Error in bakery plugin 'robotmk': Cannot determine OS.") - return value + self.cfg_dict['suites'].update({ + suite.suiteid: suite.suite2dict}) + + pass @property def global_dict(self): @@ -252,6 +277,23 @@ def global_dict(self): def suites_dict(self): return self.cfg_dict['suites'] +def get_os_default(setting, opsys): + '''Read a setting from the DEFAULTS hash. If no OS setting is found, try noarch. + Args: + setting (str): Setting name + Returns: + str: The setting value + ''' + value = DEFAULTS[opsys].get(setting, None) + if value is None: + value = DEFAULTS['noarch'].get(setting, None) + if value is None: + raise MKGeneralException( + "Error in bakery plugin 'robotmk': Cannot find setting '%s' for OS %s." % (setting, opsys)) + return value + + + def make_suiteid(robotpath, tag): '''Create a unique ID from the Robot path (dir/.robot file) and the tag. @@ -271,9 +313,4 @@ def make_suiteid(robotpath, tag): "os": ["linux", "windows"], } -header = """# This file is part of Robotmk, a module for the integration of Robot -# framework test results into Checkmk. -# -# https://robotmk.org -# https://github.com/simonmeggle/robotmk -# https://robotframework.org/#tools\n\n""" + diff --git a/bakery/v2/robotmk.py b/bakery/v2/robotmk.py index 48822e7c..a7d4acf4 100644 --- a/bakery/v2/robotmk.py +++ b/bakery/v2/robotmk.py @@ -50,8 +50,7 @@ def __setitem__(self, key, value): if (key in self or type(value) is bool) or bool(value): dict.__setitem__(self, key, value) - - +# This class is common with CMK 1/2 class RMKSuite(): def __init__(self, suite_tuple): self.suite_tuple = suite_tuple @@ -61,33 +60,51 @@ def suite2dict(self): suite_dict = DictNoNone() suite_dict['path']= self.path suite_dict['tag']= self.tag + # Ref whYeq7 suite_dict['piggybackhost']= self.piggybackhost - suite_dict.update(self.robot_param_dict) + # Ref FF3Vph + suite_dict['robot_params'] = self.robot_params + # Ref au4uPB + suite_dict['failed_handling'] = self.failed_handling return suite_dict + # Ref a01uK3 @property def path(self): return self.suite_tuple[0] + # Ref yJE5bu @property def tag(self): return self.suite_tuple[1].get('tag', None) + # Ref whYeq7 @property def piggybackhost(self): return self.suite_tuple[2].get('piggybackhost', None) + # Ref FF3Vph @property - def robot_param_dict(self): - robot_params = copy.deepcopy(self.suite_tuple[3].get('robot_params', {})) + def robot_params(self): + params = copy.deepcopy(self.suite_tuple[3].get('robot_params', {})) # Variables: transform the var 'list of tuples' into a dict. - vardict = {} - for (k1, v1) in robot_params.items(): + variables_dict = {} + for (k1, v1) in params.items(): if k1 == 'variable': for t in v1: - vardict.update({t[0]: t[1]}) - robot_params.update(self.dict_if_set('variable', vardict)) - return robot_params + variables_dict.update({t[0]: t[1]}) + params.update(self.dict_if_set('variable', variables_dict)) + return params + + # Ref au4uPB + @property + def failed_handling(self): + failed_handling = copy.deepcopy(self.suite_tuple[4].get('failed_handling', {})) + ret = {} + if failed_handling: + ret.update({'max_executions': failed_handling[0]}) + ret.update(self.dict_if_set('rerun_selection', failed_handling[1])) + return ret @property def suiteid(self): @@ -110,6 +127,7 @@ def dict_if_set(key, value): else: return {} +# This class is common with CMK 1/2 class RMK(): def __init__(self, conf): self.execution_mode = conf['execution_mode'][0] @@ -127,7 +145,8 @@ def __init__(self, conf): global_dict['logging'] = conf['logging'] global_dict['log_rotation'] = conf['log_rotation'] # WATO makes robotdir a nested dict with duplicate key. Form follows function :-/ - global_dict['robotdir'] = conf.get('robotdir').get('robotdir') + global_dict['robotdir'] = conf.get('robotdir', {}).get('robotdir', None) + if self.execution_mode == 'agent_serial': global_dict['cache_time'] = mode_conf[1] global_dict['execution_interval'] = mode_conf[2] @@ -135,10 +154,15 @@ def __init__(self, conf): elif self.execution_mode == 'external': # For now, we assume that the external mode is meant to execute all # suites exactly as configured. Hence, we can use the global cache time. - global_dict['cache_time'] = mode_conf[1] + global_dict['cache_time'] = mode_conf[1] + if 'suites' in mode_conf[0]: # each suite suite_tuple: - # >> path, tag, piggyback, robot_params{} + # 0) path, Ref a01uK3 + # 1) tag, Ref yJE5bu + # 2) piggybackhost, Ref whYeq7 + # 3) robot_params{}, Ref FF3Vph + # 4) failed_handling, Ref au4uPB for suite_tuple in mode_conf[0]['suites']: suite = RMKSuite(suite_tuple) if suite.suiteid in self.cfg_dict['suites']: @@ -150,13 +174,21 @@ def __init__(self, conf): suite.suiteid: suite.suite2dict}) pass + @property + def global_dict(self): + return self.cfg_dict['global'] + + @property + def suites_dict(self): + return self.cfg_dict['suites'] + + def controller_plugin(self, opsys: OS) -> Plugin: return Plugin( base_os=opsys, source=Path('robotmk.py'), ) - - + def runner_plugin(self, opsys: OS) -> Plugin: # TODO: when external mode: # => bin! @@ -181,9 +213,9 @@ def runner_plugin(self, opsys: OS) -> Plugin: "Error: Execution mode %s is not supported." % self.execution_mode ) - def yml(self, opsys: OS, robotmk) -> PluginConfig: + def yml(self, opsys: OS, config) -> PluginConfig: return PluginConfig(base_os=opsys, - lines=_get_yml_lines(robotmk), + lines=_get_yml_lines(config), target=Path('robotmk.yml'), include_header=True) @@ -198,24 +230,18 @@ def bin_files(self, opsys: OS): )) return files - @property - def global_dict(self): - return self.cfg_dict['global'] - @property - def suites_dict(self): - return self.cfg_dict['suites'] def get_robotmk_files(conf) -> FileGenerator: # ALWAYS (!) make a deepcopy of the conf dict. Even if you do not change # anything on it, there are strange changes ocurring while building the # packages of OS. A deepcopy solves this completely. - robotmk = RMK(copy.deepcopy(conf)) + config = RMK(copy.deepcopy(conf)) for base_os in [OS.LINUX, OS.WINDOWS]: - controller_plugin = robotmk.controller_plugin(base_os) - runner_plugin = robotmk.runner_plugin(base_os) - robotmk_yml = robotmk.yml(base_os, robotmk) - bin_files = robotmk.bin_files(base_os) + controller_plugin = config.controller_plugin(base_os) + runner_plugin = config.runner_plugin(base_os) + robotmk_yml = config.yml(base_os, config) + bin_files = config.bin_files(base_os) yield controller_plugin # in external mode, the runner is only in bin if bool(runner_plugin): yield runner_plugin @@ -223,7 +249,7 @@ def get_robotmk_files(conf) -> FileGenerator: for file in bin_files: yield file -def _get_yml_lines(robotmk) -> List[str]: +def _get_yml_lines(config) -> List[str]: header = "# This file is part of Robotmk, a module for the integration of Robot\n" +\ "# framework test results into Checkmk.\n" +\ @@ -240,7 +266,7 @@ def _get_yml_lines(robotmk) -> List[str]: lambda dumper, data: dumper.represent_mapping('tag:yaml.org,2002:map', data.items()) ) bodylist = yaml.dump( - robotmk.cfg_dict, + config.cfg_dict, default_flow_style=False, allow_unicode=True, sort_keys=True).split('\n') diff --git a/rf_tests/fail1of3/suite.robot b/rf_tests/fail1of3/suite.robot new file mode 100644 index 00000000..551bc415 --- /dev/null +++ b/rf_tests/fail1of3/suite.robot @@ -0,0 +1,11 @@ +*** Test Cases *** + +Test One Fails OMG + Log I am failing right now... + Fail OMG, failed. + +Test Two Works + Log I am working properly. + +Test Three Works + Log I am also working properly. \ No newline at end of file diff --git a/web_plugins/wato/robotmk_wato_params_bakery.py b/web_plugins/wato/robotmk_wato_params_bakery.py index ca95c21e..a6381519 100644 --- a/web_plugins/wato/robotmk_wato_params_bakery.py +++ b/web_plugins/wato/robotmk_wato_params_bakery.py @@ -19,7 +19,7 @@ from cmk.gui.i18n import _ from cmk.gui.valuespec import (DropdownChoice, Dictionary, ListOf, TextAscii, - Tuple, CascadingDropdown) + Tuple, CascadingDropdown, Integer) from cmk.gui.plugins.wato import (CheckParameterRulespecWithItem, rulespec_registry, @@ -61,8 +61,8 @@ # Use cases for this mode: same as 'agent_serial' - in addition, this mode makes sense on test clients which have the CPU/Mem resources for parallel test execution.""" helptext_execution_mode_agent_parallel = "This is only a placeholder for the parallel execution of RF suites. Please choose another mode." helptext_execution_mode_external = """ - The Checkmk agent starts the Robotmk controller as a synchronous check plugin in the agent check interval.
- Rule dependency: The rule Deploy custom files with agent (package robotmk-external) places the runner within the agent's bin directory. + The Checkmk agent starts the Robotmk controller as a synchronous check plugin in the agent check interval.

+ Important note for Checkmk 1.6: The rule Deploy custom files with agent (package robotmk-external) must be used to place the runner within the agent's bin directory (there is no other way in Checkmk 1 to deploy files to that folder).
From there, you can start the runner with any external tool (e.g. systemd timer/cron/task scheduler).

If no suites are specified, the runner will execute all suites listed in robotmk.yml.
If no suites are defined at all, the runner will execute all suites found in the Robot suites directory.

@@ -75,7 +75,7 @@ agent_config_global_suites_execution_interval_agent_serial = Age( title=_("Runner execution interval"), help= - _("Interval the Checkmk agent will execute the runner plugin asynchronously.
" + _("This configures the interval in which the Checkmk agent will execute the runner plugin asynchronously.
" "The default is 15min but strongly depends on the maximum probable runtime of all test suites.
Choose an interval which is a good comprimise between frequency and execution runtime headroom.
" ), minvalue=1, @@ -90,7 +90,8 @@ _("Suite state files are updated by the runner after each execution (Runner execution interval).
" "The controller monitors the age of those files and expects them to be not older than the global cache time.
" "Each suite with a state file older than its result cache time will be reported as 'stale'.
" - "For obvious reasons, the cache time must always be set higher than the runner execution interval." + "For obvious reasons, the cache time must always be set higher than the runner execution interval, including reruns of failed tests/subsuites (if configured).
" + "(Do not confuse it with the cache time which Checkmk uses for the agent plugin configuration.)" ), minvalue=1, maxvalue=65535, @@ -102,7 +103,7 @@ _("Suite state files are updated every time when the runner has executed the suites.
" "The controller monitors the age of those files and expects them to be not older than the global cache time or the suite cache time (if set).
" "Each suite with a state file older than its cache time will be reported as 'stale'.
" - "For obvious reasons, this cache time must always be set higher than the execution interval." + "For obvious reasons, this cache time must always be set higher than the execution interval, including reruns of failed tests/subsuites (if configured)." ), minvalue=1, maxvalue=65535, @@ -113,7 +114,7 @@ agent_config_suite_suites_cache_time_agent_parallel = Age( title=_("Suite cache time"), help= - _("Sets the suite specific cache time. (Must be higher than the suite execution interval)" + _("Sets the suite specific cache time. (Must be higher than the suite execution interval, including reruns of failed tests/subsuites)" ), minvalue=1, maxvalue=65535, @@ -123,7 +124,7 @@ agent_config_suite_suites_cache_time_external = Age( title=_("Suite cache time"), help= - _("Sets suite specific cache times for individual execution intervals" + _("Sets suite specific cache times for individual execution intervals, including reruns of failed tests/subsuites" ), minvalue=1, maxvalue=65535, @@ -188,6 +189,67 @@ size=50, ) +# TEST SELECTION DICT ELEMENTS ================================================= +# To be used in test selection and rerunfailed +# Ref: 7uBbn2 +dict_el_suite_selection = ( + "suite", + ListOfStrings( + title=_("Select suites (--suite)"), + help= + _("Select suites by name.
When this option is used with" + " --test, --include or --exclude, only tests in" + " matching suites and also matching other filtering" + " criteria are selected.
" + " Name can be a simple pattern similarly as with --test and it can contain parent" + " name separated with a dot.
" + " For example, X.Y selects suite Y only if its parent is X.
" + ), + size=40, + ) +) +dict_el_test_selection = ( + "test", + ListOfStrings( + title=_("Select test (--test)"), + help= + _("Select tests by name or by long name containing also" + " parent suite name like Parent.Test.
Name is case" + " and space insensitive and it can also be a simple" + " pattern where * matches anything, ? matches any" + " single character, and [chars] matches one character" + " in brackets.
"), + size=40, + ) +) +dict_el_test_include = ( + "include", + ListOfStrings( + title=_("Include tests by tag (--include)"), + help= + _("Select tests by tag. (About tagging test cases)
Similarly as name with --test," + "tag is case and space insensitive and it is possible" + "to use patterns with *, ? and [] as wildcards.
" + "Tags and patterns can also be combined together with" + "AND, OR, and NOT operators.
" + "Examples:
foo
bar*
fooANDbar*
" + ), + size=40, + ) +) + +dict_el_test_exclude = ( + "exclude", + ListOfStrings( + title=_("Exclude tests by tag (--exclude)"), + help= + _("Select test cases not to run by tag. (About tagging test cases)
These tests are" + " not run even if included with --include.
Tags are" + " matched using same rules as with --include.
"), + size=40, + ) +) + agent_config_testsuites_robotframework_params_dict = Dictionary( help= _("The options here allow to specify the most common cmdline parameters for Robot Framework. (All command line options)" @@ -203,54 +265,11 @@ allow_empty=False, size=50, )), - ("suite", - ListOfStrings( - title=_("Select suites (--suite)"), - help= - _("Select suites by name.
When this option is used with" - " --test, --include or --exclude, only tests in" - " matching suites and also matching other filtering" - " criteria are selected.
" - " Name can be a simple pattern similarly as with --test and it can contain parent" - " name separated with a dot.
" - " For example, X.Y selects suite Y only if its parent is X.
" - ), - size=40, - )), - ("test", - ListOfStrings( - title=_("Select test (--test)"), - help= - _("Select tests by name or by long name containing also" - " parent suite name like Parent.Test.
Name is case" - " and space insensitive and it can also be a simple" - " pattern where * matches anything, ? matches any" - " single character, and [chars] matches one character" - " in brackets.
"), - size=40, - )), - ("include", - ListOfStrings( - title=_("Include tests by tag (--include)"), - help= - _("Select tests by tag. (About tagging test cases)
Similarly as name with --test," - "tag is case and space insensitive and it is possible" - "to use patterns with *, ? and [] as wildcards.
" - "Tags and patterns can also be combined together with" - "AND, OR, and NOT operators.
" - "Examples:
foo
bar*
fooANDbar*
" - ), - size=40, - )), - ("exclude", - ListOfStrings( - title=_("Exclude tests by tag (--exclude)"), - help= - _("Select test cases not to run by tag. (About tagging test cases)
These tests are" - " not run even if included with --include.
Tags are" - " matched using same rules as with --include.
"), - size=40, - )), + # Ref: 7uBbn2 + dict_el_suite_selection, + dict_el_test_selection, + dict_el_test_include, + dict_el_test_exclude, ("critical", ListOfStrings( title=_("Critical test tag (--critical)"), @@ -318,6 +337,45 @@ ("robot_params", agent_config_testsuites_robotframework_params_dict), ]) +agent_config_testsuites_max_executions_selection_dict = Dictionary( + help=_(""" + With the following options it is possible to further filter the list of tests/suites to re-run. (Documentation: Re-executing failed test cases) + """), + elements=[ + # Ref: 7uBbn2 + dict_el_suite_selection, + dict_el_test_selection, + dict_el_test_include, + dict_el_test_exclude, + ] +) + + + +agent_config_testsuites_max_executions_tuple = Tuple( + help=_(""" + Robotmk can immediately re-execute failed tests n-times before submitting the result to the agent. After the n-th iteration, the total suite result contains the most current result of each test case.
+ Use this only as a last resort, for example when applications behave unreliable. Also take into account that the re-execution of failed tests/suites requires additional headroom for the result cache time. + """), + elements=[ + Integer( + title=_("Maximum executions"), + help=_("The maximum number of iterations (including the first attempt)"), + minvalue=1, + default_value=1 + ), + agent_config_testsuites_max_executions_selection_dict + ]) + + + +agent_config_testsuites_max_executions_container = Dictionary( + title=_("Handling of failed tests/suites"), + elements=[ + ("failed_handling", agent_config_testsuites_max_executions_tuple), + ]) + + # Make the help text of SuitList dependent on the type of execution def gen_agent_config_dict_listof_testsuites(mode): @@ -331,7 +389,7 @@ def gen_agent_config_dict_listof_testsuites(mode): ListOf( gen_testsuite_tuple(mode), help=_(""" - Click on 'Add test suite' to add the suites to the execution list and to specify additional parameters, piggyback host and execution order.
+ Click on 'Add test suite' to specify the suites to be executed, including additional parameters, piggyback host and execution order. This is the recommended way.
If you do not add any suite here, the Robotmk plugin will add every .robot file/every directory within the Robot suites directory to the execution list - without any further parametrization.
""" ), add_label=_("Add test suite"), @@ -346,17 +404,7 @@ def gen_testsuite_tuple(mode): agent_config_testsuites_tag, agent_config_testsuites_piggybackhost, agent_config_testsuites_robotframework_params_container, - # timing settings (there aren't any - set globally) - ]) - if mode == 'agent_parallel': - return Tuple(elements=[ - agent_config_testsuites_path, - agent_config_testsuites_tag, - agent_config_testsuites_piggybackhost, - agent_config_testsuites_robotframework_params_container, - # timing settings - agent_config_suite_suites_cache_time_agent_parallel, - agent_config_suite_suites_execution_interval_agent_parallel, + agent_config_testsuites_max_executions_container, ]) if mode == 'external': return Tuple(elements=[ @@ -364,8 +412,7 @@ def gen_testsuite_tuple(mode): agent_config_testsuites_tag, agent_config_testsuites_piggybackhost, agent_config_testsuites_robotframework_params_container, - # timing settings - # agent_config_suite_suites_cache_time_external, + agent_config_testsuites_max_executions_container, ]) @@ -400,7 +447,7 @@ def gen_testsuite_tuple(mode): dropdown_robotmk_log_rotation = CascadingDropdown( title=_("Number of days to keep Robot XML/HTML log files on the host"), help=_( - "This settings helps to keep the test host clean by deleting the log files after a certain amount of days. Log files are:
" + "This setting helps to keep the test host clean by deleting the log files after a certain amount of days. Log files are:
" "robotframework-$SUITENAME-$timestamp-output.xml
" "robotframework-$SUITENAME-$timestamp-log.html
"), choices=[