From e93beb333db5f22bf4d2c01b1e46d86911569052 Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Wed, 3 Aug 2022 00:35:46 +0200 Subject: [PATCH 1/9] Agent Plugin: skips suite execution if DISABLED file present. (closes #192) Added bulb --- CHANGELOG.md | 24 ++++++++++++- agents_plugins/robotmk.py | 34 +++++++++++++++++-- .../wato/robotmk_wato_params_bakery.py | 1 + 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 450eee94..726313b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +* Documentation: Added [FAQs](./FAQ.md) with design decicions +* Check: Added the possibility to temporarily skip the execution of suites by placing a `DISABLED` (case sensitive) file in the suite's root folder. (#192) +Robotmk will silently ignore this suite as long the DISABLED file is there. +(Thanks Marcus for this suggestion - this is the perfect alternative to commenting out the suite in `robotmk.yml` manually). +You can also optionally write a reason ("*Need a short break.*") into the file; it will get logged in Robotmk's log: + +``` +2022-08-02 22:25:28,449 RMKrunner [48887] INFO: #################### +2022-08-02 22:25:28,575 RMKrunner [48887] INFO: => Suites to start: sleep +2022-08-02 22:25:28,583 RMKrunner [48887] INFO: ~~~~ Suite ID: sleep ~~~~ +2022-08-02 22:25:28,591 RMKrunner [48887] WARNING: Suite 'sleep' is skipped because of the 'DISABLED' flagfile in its suite folder. Reason: Need a short break. +2022-08-02 22:25:28,591 RMKrunner [48887] WARNING: (Be aware that the services in Checkmk will become stale soon.) +2022-08-02 22:25:28,595 RMKrunner [48887] INFO: ... Quitting Runner, bye. --- +``` + ### Changed -* Bakery rule: Lowered the log retention threshold (#172) +* Bakery rule: Lowered the log retention threshold (#172) * Check: Fixed a bug which lead to metric names without specials chars like umlauts => metric names can slightly change, sorry. (#197) ### Fixed @@ -20,6 +37,11 @@ before the next execution. (#203) * Check: Creates host dir for HTML logs; updated inline help for HTML log icons (#201) * Check: Allow only 1 runtime threshold by setting the other one to 0 (#198) +### Removed + +### Deprecated + + ## 1.2.9 - 2022-03-25 ### Fixed diff --git a/agents_plugins/robotmk.py b/agents_plugins/robotmk.py index 8e73c2c0..9e69d506 100755 --- a/agents_plugins/robotmk.py +++ b/agents_plugins/robotmk.py @@ -509,6 +509,27 @@ def get_suite_or_global(self, name, default=None): except: return default + @property + def is_disabled_by_flagfile(self): + # if disabled flag file exists, return True + return self.path.joinpath('DISABLED').exists() + + @property + def get_disabled_reason(self): + # If disabled flag file exists, return the content. + # Otherwise return default message. + if self.is_disabled_by_flagfile: + try: + with open(self.path.joinpath('DISABLED'), 'r') as f: + reason = f.read() + if len(reason) > 0: + return "Reason: " + reason + else: + return "" + + except: + return "" + @property def path(self): '''The absolute path to the Robot test (directory or .robot file), @@ -948,7 +969,9 @@ def update_runner_statevars(self): - external mode (a scheduled task starts the runner with suite args)''' runtime_total = ( self._state['end_time'] - self._state['start_time']).total_seconds() - runtime_suites = sum([suite.runtime for suite in self.suites]) + # only count runtimes of suites which ran indeeed. Suites which were skipped + # with a DISABLED file are ignored. + runtime_suites = sum([suite.runtime for suite in self.suites if suite.is_disabled_by_flagfile == False]) runtime_robotmk = runtime_total - runtime_suites self.set_statevars([ @@ -1006,10 +1029,17 @@ def run_suites(self, suites_cmdline): suite.error = error # continue self.logdebug(f'Strategy: ' + str(suite._env_strategy) ) + + # search for a DEBUG file in the suite directory and skip the suite if found + if suite.is_disabled_by_flagfile: + reason = suite.get_disabled_reason.strip() + self.logwarn(f"Suite '{id}' is skipped because of the 'DISABLED' flagfile in its suite folder. {reason}") + self.logwarn("(Be aware that the services in Checkmk will become stale soon.)") + continue + # Robot Framework, the stage is yours! rc = self.run_suite(suite) - if rc > 250: self.logerror( 'RC > 250 = Robot exited with fatal error. There are no logs written.') diff --git a/web_plugins/wato/robotmk_wato_params_bakery.py b/web_plugins/wato/robotmk_wato_params_bakery.py index 7d4b529b..ca52c1d4 100644 --- a/web_plugins/wato/robotmk_wato_params_bakery.py +++ b/web_plugins/wato/robotmk_wato_params_bakery.py @@ -348,6 +348,7 @@ def outputdir(self, val): help= _("Name of the .robot file or directory containing .robot files, relative to the robot suites directory
" "It is highly recommended to organize Robot suites in directories and to specify the directories here without leading/trailing (back)slashes.
" + "💡 If a suite needs to be skipped temporarily, place a file DISABLED in the robot suites directory. Robotmk will silently omit the execution, detected services will be displayed as outdated/stale, but will not be alerted." ), allow_empty=False, size=50, From 2832271ab729bcc358a0a2f1ebcdfcbc98d6a489 Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Wed, 3 Aug 2022 00:38:26 +0200 Subject: [PATCH 2/9] devcontainer: removed lsyncd, added less --- .devcontainer/Dockerfile_cmk_python | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile_cmk_python b/.devcontainer/Dockerfile_cmk_python index 5cab59a3..9ac0b807 100644 --- a/.devcontainer/Dockerfile_cmk_python +++ b/.devcontainer/Dockerfile_cmk_python @@ -16,4 +16,4 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ RUN pip3 install robotframework pyyaml mergedeep python-dateutil ipdb RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq tree htop vim git telnet file lsyncd + && apt-get -y install --no-install-recommends jq tree htop vim git telnet file less From 11f18c895ae09b12444ef8912ba162dcb7d49e32 Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Wed, 3 Aug 2022 00:39:02 +0200 Subject: [PATCH 3/9] Documentation: Debug a agent plugin in Docker --- DEVELOPMENT.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index fb5ed28c..7099cf15 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -165,7 +165,7 @@ VS code displays by default only the files of the workspace (`/workspaces/robotm You can now add breakpoints to the scripts in this folder to debug them. Also, only then the code completion (classes, functions, ...) works properly, because it works in the same Python context as Checkmk. -### Test a new Linux agent in the container +### Install a new Linux agent in the container The Debian packages that the Bakery generates can be installed and tested directly in the container: @@ -193,6 +193,17 @@ systemd not found on this system Reloading xinetd ``` +### Debugging Robotmk agent plugins + +Debugging `robotmk.py` and `robotmk-runner.py` from the devcontainer is not straightforward, because from the context of the `cmk` user, you are not able to access the Robotmk YML and log files. But it is possible (and very useful), if just do the following steps: + +- Jump into the decontainer: `docker ps` => `docker exec -it bash` +- Change permissions on + - the logs: `chmod -R o+w /var/log/robotmk` + - the Robotmk YML file: `chmod o+r /etc/check_mk/robotmk.yml` + +After that, set a breakpoint in VS Code in `agents_plugins/robotmk-runner.py`. Also select the debugging configuration to "devc V2.x robotmk-runner" and press F5 to start debugging. + ### Write a changelog Robotmk's [CHANGELOG.md](CHANGELOG.md) is based on [](https://keepachangelog.com/). From 753122d33c9539e13112fb5c54035065e7265f92 Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Wed, 3 Aug 2022 00:39:37 +0200 Subject: [PATCH 4/9] Test Suites: update --- rf_tests/robot | 1 + rf_tests/sleep/DISABLED | 1 + 2 files changed, 2 insertions(+) create mode 120000 rf_tests/robot create mode 100644 rf_tests/sleep/DISABLED diff --git a/rf_tests/robot b/rf_tests/robot new file mode 120000 index 00000000..7c6b0c8d --- /dev/null +++ b/rf_tests/robot @@ -0,0 +1 @@ +/usr/lib/check_mk_agent/robot \ No newline at end of file diff --git a/rf_tests/sleep/DISABLED b/rf_tests/sleep/DISABLED new file mode 100644 index 00000000..e8dfb061 --- /dev/null +++ b/rf_tests/sleep/DISABLED @@ -0,0 +1 @@ +Need a short break. From 07fa8135b6a74bd53292dd66c678afe55d28afcd Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Wed, 3 Aug 2022 00:47:01 +0200 Subject: [PATCH 5/9] Documentation: added Github CLI instructions. --- DEVELOPMENT.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7099cf15..f735b877 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -34,6 +34,12 @@ Authentication to GitHub is required if you want to release/unrelease. Install the CLI tools from https://github.com/cli/cli. +After installation, authenticate to github. It will ask you about your authentication method, protocol, and present an authentication code which you should open in the opened Github web page. + + gh auth login + +After that you are fully authenticated in the CLI to GitHub. + --- ## Development environment setup From 2e2191bb6bd4b6da065b4f48b9fa46c3e99b7cef Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Tue, 23 Aug 2022 12:30:41 +0200 Subject: [PATCH 6/9] Add V2 workspace --- .vscode/v2/robotmk.code-workspace | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .vscode/v2/robotmk.code-workspace diff --git a/.vscode/v2/robotmk.code-workspace b/.vscode/v2/robotmk.code-workspace new file mode 100644 index 00000000..e136977b --- /dev/null +++ b/.vscode/v2/robotmk.code-workspace @@ -0,0 +1,19 @@ +{ + "folders": [ + { + "path": "../.." + }, + { + "path": "../../../../opt/omd/sites/cmk" + }, + { + "path": "../../../../usr/bin" + }, + { + "path": "../../../../usr/lib/check_mk_agent/plugins" + } + ], + "settings": { + "python.languageServer": "Pylance" + } +} \ No newline at end of file From aca8c46cfd7fc3ed178a114710608e7f65650a99 Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Tue, 23 Aug 2022 12:32:10 +0200 Subject: [PATCH 7/9] Fixed bug in build.sh (missing bakery file) --- .devcontainer/build.sh | 28 +++++++++++++++++++++------- CHANGELOG.md | 2 +- DEVELOPMENT.md | 10 ++++++++++ package/v2/pkginfo | 5 +---- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/.devcontainer/build.sh b/.devcontainer/build.sh index d28b2e44..998ebb99 100755 --- a/.devcontainer/build.sh +++ b/.devcontainer/build.sh @@ -10,6 +10,17 @@ set -e # After the MKP has been built, the script check if it runs within a Github # Workflow. If so, it sets the artifact name as output variable. +if [ -z $WORKSPACE ]; then + echo "ERROR: WORKSPACE environment variable not set. Exiting." + exit 1 +fi + +if [ -z $OMD_SITE ]; then + echo "ERROR: You do not seem to be on a OMD site (variable OMD_SITE not set). Exiting." + exit 1 +fi + +set -u # CMK Major version MVERSION="$(cat $OMD_ROOT/.version_meta/version | cut -d '.' -f1)" NAME="robotmk" @@ -18,18 +29,18 @@ PACKAGEFILE=$OMD_ROOT/var/check_mk/packages/$NAME # get the current tag (Release) or commit hash (Artifact) export RMK_VERSION=$(git describe --exact-match --tags 2> /dev/null || git rev-parse --short HEAD) - +echo "▹ Removing old packages..." rm -f $OMD_ROOT/var/check_mk/packages/* echo "---------------------------------------------" -echo "* Merging the common package info with version $MVERSION specific..." +echo "▹ Merging the common package info with version $MVERSION specific..." jq -s '.[0] * .[1]' $WORKSPACE/package/pkginfo_common $WORKSPACE/package/v$MVERSION/pkginfo | jq '. + {version:env.RMK_VERSION}' > $PACKAGEFILE echo "---------------------------------------------" echo "$PACKAGEFILE:" cat $PACKAGEFILE echo "---------------------------------------------" -echo "* Building MKP '$NAME' on $RMK_VERSION for CMK version $MVERSION..." -set -x +echo "▹ Building MKP '$NAME' on $RMK_VERSION for CMK version $MVERSION..." +# set -x ls -la $PACKAGEFILE mkp -v pack $NAME FILE=$(ls -rt1 *.mkp | tail -1) @@ -38,9 +49,12 @@ FILE=$(ls -rt1 *.mkp | tail -1) NEWFILENAME=$NAME.$RMK_VERSION-cmk$MVERSION.mkp mv $FILE $NEWFILENAME echo "---------------------------------------------" +echo "📦 Package:" +echo "$NEWFILENAME" + -if [ -n "$GITHUB_WORKSPACE" ]; then - echo "* Set Outputs for GitHub Workflow steps" +if [ -n "${GITHUB_WORKSPACE-}" ]; then + echo "▹ Set Outputs for GitHub Workflow steps" echo "::set-output name=pkgfile::$NEWFILENAME" # echo "::set-output name=pkgname::${NAME}" VERSION=$(jq -r '.version' $PACKAGEFILE) @@ -48,7 +62,7 @@ if [ -n "$GITHUB_WORKSPACE" ]; then # echo "::set-output name=cmkmversion::$MVERSION" echo "::set-output name=artifactname::$NEWFILENAME" else - echo "* (No GitHub Workflow detected)" + echo "(No GitHub Workflow detected)" fi echo "END OF build.sh" echo "---------------------------------------------" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 726313b7..cbcb6f80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* Documentation: Added [FAQs](./FAQ.md) with design decicions +* Documentation: Added [FAQs](./FAQ.md) with design decisions * Check: Added the possibility to temporarily skip the execution of suites by placing a `DISABLED` (case sensitive) file in the suite's root folder. (#192) Robotmk will silently ignore this suite as long the DISABLED file is there. (Thanks Marcus for this suggestion - this is the perfect alternative to commenting out the suite in `robotmk.yml` manually). diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f735b877..ac7fec0b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -210,6 +210,16 @@ Debugging `robotmk.py` and `robotmk-runner.py` from the devcontainer is not stra After that, set a breakpoint in VS Code in `agents_plugins/robotmk-runner.py`. Also select the debugging configuration to "devc V2.x robotmk-runner" and press F5 to start debugging. +### Building a MKP inside of the Container + +To test how the Github workflow will build MKPs for Robotmk, you can run the build script within the container locally. + +In VS Code, +- run command "Tasks chooser: Select Task" => "Build Robotmk MKP" +- run command "Tasks: run task" to execute the build script + +- + ### Write a changelog Robotmk's [CHANGELOG.md](CHANGELOG.md) is based on [](https://keepachangelog.com/). diff --git a/package/v2/pkginfo b/package/v2/pkginfo index bea4ae37..93641f0a 100644 --- a/package/v2/pkginfo +++ b/package/v2/pkginfo @@ -2,9 +2,6 @@ "version.min_required": "2.0.0", "files": { "agent_based": ["robotmk.py"], - "agents": ["plugins/robotmk-runner.py", "plugins/robotmk.py"], - "lib": [ - "check_mk/base/cee/plugins/bakery/robotmk.py" - ] + "agents": ["plugins/robotmk-runner.py", "plugins/robotmk.py", "bakery/robotmk.py"] } } \ No newline at end of file From 2e7bd13e97a560c97d39fe37e291986e20cb966e Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Tue, 23 Aug 2022 15:21:34 +0200 Subject: [PATCH 8/9] CHANGELOG: v1.2.10-beta --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbcb6f80..e849290a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 1.2.10-beta - 2022-08-23 ### Added From 4914a6695059ad718b0751b6343aa3954ed058fa Mon Sep 17 00:00:00 2001 From: Simon Meggle Date: Tue, 23 Aug 2022 15:21:35 +0200 Subject: [PATCH 9/9] Version bump: v1.2.10-beta --- agents_plugins/robotmk.py | 2 +- bakery/v1/robotmk.py | 2 +- bakery/v2/robotmk.py | 2 +- checks/v1/robotmk | 2 +- checks/v2/robotmk.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agents_plugins/robotmk.py b/agents_plugins/robotmk.py index 9e69d506..b29ff066 100755 --- a/agents_plugins/robotmk.py +++ b/agents_plugins/robotmk.py @@ -49,7 +49,7 @@ local_tz = datetime.utcnow().astimezone().tzinfo -ROBOTMK_VERSION = 'v1.2.9' +ROBOTMK_VERSION = 'v1.2.10-beta' class RMKConfig(): _PRESERVED_WORDS = [ diff --git a/bakery/v1/robotmk.py b/bakery/v1/robotmk.py index b00aac25..8ca7cfcb 100644 --- a/bakery/v1/robotmk.py +++ b/bakery/v1/robotmk.py @@ -18,7 +18,7 @@ # to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, # Boston, MA 02110-1301 USA. -ROBOTMK_VERSION = 'v1.2.9' +ROBOTMK_VERSION = 'v1.2.10-beta' import cmk.utils.paths import os diff --git a/bakery/v2/robotmk.py b/bakery/v2/robotmk.py index fb553224..c4591794 100644 --- a/bakery/v2/robotmk.py +++ b/bakery/v2/robotmk.py @@ -18,7 +18,7 @@ # to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, # Boston, MA 02110-1301 USA. -ROBOTMK_VERSION = 'v1.2.9' +ROBOTMK_VERSION = 'v1.2.10-beta' from typing import Iterable, TypedDict, List from pathlib import Path diff --git a/checks/v1/robotmk b/checks/v1/robotmk index a3bbb33a..0696aa27 100644 --- a/checks/v1/robotmk +++ b/checks/v1/robotmk @@ -34,7 +34,7 @@ from collections import namedtuple iam = "robotmk" # DO NOT DELETE inventory_robotmk_rules = [] -ROBOTMK_VERSION = 'v1.2.9' +ROBOTMK_VERSION = 'v1.2.10-beta' DEFAULT_SVC_PREFIX = 'Robot Framework E2E $SUITEID$SPACE-$SPACE' HTML_LOG_DIR = "%s/%s" % (os.environ['OMD_ROOT'], 'var/robotmk') diff --git a/checks/v2/robotmk.py b/checks/v2/robotmk.py index 8d0a9ddc..cc4fc040 100644 --- a/checks/v2/robotmk.py +++ b/checks/v2/robotmk.py @@ -35,7 +35,7 @@ from cmk.base.plugins.agent_based.agent_based_api.v1 import * from cmk.utils.exceptions import MKGeneralException -ROBOTMK_VERSION = 'v1.2.9' +ROBOTMK_VERSION = 'v1.2.10-beta' DEFAULT_SVC_PREFIX = 'Robot Framework E2E $SUITEID$SPACE-$SPACE' HTML_LOG_DIR = "%s/%s" % (os.environ['OMD_ROOT'], 'var/robotmk')