From 3740edaf2d16505157755526feec2d2177c8e4f8 Mon Sep 17 00:00:00 2001 From: Li Xi Date: Mon, 25 Sep 2023 19:49:44 +0800 Subject: [PATCH] Update the dashboard of grafana Signed-off-by: Li Xi --- .gitignore | 1 + Makefile.am | 77 ++- autogen.sh | 2 +- .../cluster_status.json.template | 34 +- .../lustre_client.json.template | 286 ++++++++- barreleye/lustre.m4 | 2 +- bash_completion/Makefile.am | 7 +- configure.ac | 39 +- coral.spec.in | 4 +- detect-distro.sh | 3 + ...{barreleye.toml => barreleye.conf.example} | 0 pybarrele/barrele.py | 15 + pybarrele/barrele_agent.py | 110 +++- pybarrele/barrele_collectd.py | 92 ++- pybarrele/barrele_constant.py | 2 + pybarrele/barrele_instance.py | 142 +++-- pybarrele/barrele_server.py | 12 +- pybuild/Makefile.am | 5 +- pybuild/build_barrele.py | 579 ++++++++++++++++-- pybuild/build_common.py | 41 +- pybuild/coral.py | 7 +- pybuild/coral_build.py | 238 ++++++- pycheck/Makefile.am | 19 +- pycoral/cmd_general.py | 2 +- pycoral/install_common.py | 176 +++++- pycoral/lustre_version.py | 30 + pycoral/ssh_host.py | 4 + pycoral/utils.py | 2 + 28 files changed, 1694 insertions(+), 237 deletions(-) rename example_configs/{barreleye.toml => barreleye.conf.example} (100%) diff --git a/.gitignore b/.gitignore index f64a08f..cd11943 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ stamp-h1 *.checked *.pyc *.c_checked +*.h_checked *.o # diff --git a/Makefile.am b/Makefile.am index e8f2637..8cef3bb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,12 @@ SUBDIRS = pybuild -SUBDIRS += barreleye pycheck bash_completion . +DIST_SUBDIRS = pybuild + +if ENABLE_BARRELE +SUBDIRS += barreleye +endif #ENABLE_BARRELE +SUBDIRS += pycheck bash_completion . +DIST_SUBDIRS += barreleye pycheck bash_completion . + build_dir = `pwd`/build rpmbuild_opt = @@ -23,7 +30,7 @@ EXTRA_DIST = \ coral.spec \ detect-distro.sh \ doc \ - example_configs/barreleye.toml \ + example_configs/barreleye.conf.example \ pybarrele/*.py \ pybuild/*.py \ pycoral/*.py \ @@ -48,6 +55,7 @@ clean-local: rm -f $$exe.spec; \ done rm -fr coral_build_* + rm -fr debian/.debhelper all: all-am $(CHECKS) @@ -57,46 +65,89 @@ mrproper: maintainer-clean rm -f Makefile.in aclocal.m4 configure rm -f compile depcomp install-sh missing -BARRELEYE_RPM = build/RPMS/$(target_cpu)/coral-barreleye-$(PACKAGE_VERSION)-$(CORAL_RELEASE).$(DISTRO_SHORT)*.$(target_cpu).rpm +CORAL_PACKAGES = -CORAL_RPMS = if ENABLE_BARRELE rpmbuild_opt += --with barrele -CORAL_RPMS += $(BARRELEYE_RPM) +CORAL_PACKAGES += barreleye else rpmbuild_opt += --without barrele endif -EXES = barrele +EXES = PYINSTALLER_CMD=PYTHONPATH=$(PIP3_PACKAGE_PATH) pyinstaller -F --clean + +if ENABLE_BARRELE +EXES += barrele dist/barrele: mkdir -p dist $(PYINSTALLER_CMD) barrele +endif + -EXE_BINS = $(addprefix dist/, $(EXES)) +EXE_BINS = @DOLLAR@(addprefix dist/, $(EXES)) exes: $(EXE_BINS) EXTRA_OPTION = -rpms: coral.spec dist +ISO_EXTRA = + + +if MAKE_RPMS +rpms: coral.spec dist Makefile mkdir -p $(build_dir)/BUILD $(build_dir)/SPECS $(build_dir)/SRPMS $(build_dir)/RPMS \ && rpmbuild $(rpmbuild_opt) --define="_topdir $(build_dir)" \ --define="_prefix $(prefix)" \ $(EXTRA_OPTION) -tb $(distdir).tar.gz \ && echo "RPMs successfully generated in $(build_dir)/RPMS" - -ISO_EXTRA = -coral-$(PACKAGE_VERSION).$(DISTRO_SHORT).$(target_cpu).iso: $(ISO_EXTRA) rpms +ISO_EXTRA += rpms +endif #MAKE_RPMS + +if MAKE_DEBS +debs: Makefile + VER=$$(echo @VERSION@ | tr '_' '-'); \ + changelog_version=$$(sed -ne '1s/^coral (\(.*\)).*$$/\1/p' debian/changelog); \ + if [ "$$changelog_version" != "$$VER" ]; then \ + echo -e "coral ($$VER) unstable; urgency=low\n\n * Automated changelog.\n\n -- Coral Packaging Team $$(date -R)\n" > debian/changelog; \ + fi; \ + rm -fr debs; \ + dpkg-buildpackage -us -uc -I.git || { \ + rc=$${PIPESTATUS[0]}; \ + [ $${rc} -gt 1 ] && exit $${rc}; \ + exit 0; \ + }; \ + mkdir -p debs && \ + mv ../coral-*_$${VER}_*.deb \ + ../coral_$${VER}_*.buildinfo \ + ../coral_$${VER}.tar.gz \ + ../coral_$${VER}.dsc \ + ../coral_$${VER}_*.changes \ + debs/ +ISO_EXTRA += debs +endif #MAKE_DEBS + +coral-$(PACKAGE_VERSION).$(DISTRO_SHORT).$(target_cpu).iso: $(ISO_EXTRA) rm $(ISO_PATH) -fr rm -f coral-*.iso rm -f coral-*_SHA256SUM cp -a $(ISO_CACHE_PATH) $(ISO_PATH) ./coral release_info save $(DISTRO_SHORT) $(target_cpu) coral_release_info.yaml cp coral_release_info.yaml $(ISO_PATH) - cp $(CORAL_RPMS) $(PACKAGE_PATH) - createrepo $(PACKAGE_PATH) + mkdir -p $(PACKAGE_PATH) + if [ "$(DISTRO_SHORT)" = "ubuntu2204" ]; then \ + DEB_PACKAGE_VERSION=$$(echo $(PACKAGE_VERSION) | tr '_' '-'); \ + for CORAL_PACKAGE in $(CORAL_PACKAGES); do \ + cp debs/coral-$${CORAL_PACKAGE}_$${DEB_PACKAGE_VERSION}_*.deb $(PACKAGE_PATH); \ + done; \ + fi + if [ "$(DISTRO_SHORT)" = "el7" -o "$(DISTRO_SHORT)" = "el8" ]; then \ + for CORAL_PACKAGE in $(CORAL_PACKAGES); do \ + cp build/RPMS/$(target_cpu)/coral-$${CORAL_PACKAGE}-$(PACKAGE_VERSION)-$(CORAL_RELEASE).$(DISTRO_SHORT)*.$(target_cpu).rpm $(PACKAGE_PATH); \ + done; \ + createrepo $(PACKAGE_PATH); \ + fi @if [ "$(DISTRO_SHORT)" = "el8" ]; then \ repo2module -s stable $(PACKAGE_PATH) $(PACKAGE_PATH)/modules.yaml; \ modifyrepo --mdtype=modules $(PACKAGE_PATH)/modules.yaml $(PACKAGE_PATH)/repodata; \ diff --git a/autogen.sh b/autogen.sh index bbf2366..f76b99f 100755 --- a/autogen.sh +++ b/autogen.sh @@ -18,7 +18,7 @@ EOF done } -check_for_application lex yacc autoheader aclocal automake autoconf +check_for_application autoheader aclocal automake autoconf # Actually we don't need the pkg-config executable, but we need the M4 macros. # We check for `pkg-config' here and hope that M4 macros will then be diff --git a/barreleye/grafana_dashboards/cluster_status.json.template b/barreleye/grafana_dashboards/cluster_status.json.template index 73a5d9b..fdcc6e6 100644 --- a/barreleye/grafana_dashboards/cluster_status.json.template +++ b/barreleye/grafana_dashboards/cluster_status.json.template @@ -24,7 +24,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -293,7 +293,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -564,7 +564,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -835,7 +835,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -1106,7 +1106,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -1377,7 +1377,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -1648,7 +1648,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -1919,7 +1919,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -2190,7 +2190,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -2461,7 +2461,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -2732,7 +2732,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -3003,7 +3003,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -3274,7 +3274,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -3545,7 +3545,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -3816,7 +3816,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -4087,7 +4087,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, @@ -4358,7 +4358,7 @@ "ok": "rgba(50, 128, 45, 0.9)", "warn": "rgba(237, 129, 40, 0.9)" }, - "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critial(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", + "description": "#### _Warning(yellow)_\n######   if one or more of the following conditions are true:\n * Idle CPU is less than 20%.\n * Load is higher than 5\n * Free memory is less than 1000 MiB;\n * Free space of \"/\" is less than 10 GiB;\n#### _Critical(red)_\n ######   if one or more of the following conditions are true:\n + Idle CPU is less than 5%.\n + Load is higher than 10\n + Free space of \"/\" is less than 1 GiB;\n + Free memory is less than 100 MiB;\n\n----\n######   The system load is defined as the number of runnable tasks in the run-queue and is provided by many operating systems as a one minute average.", "displayName": "", "flipCard": false, "flipTime": 5, diff --git a/barreleye/grafana_dashboards/lustre_client.json.template b/barreleye/grafana_dashboards/lustre_client.json.template index e5e725c..0e9adc8 100644 --- a/barreleye/grafana_dashboards/lustre_client.json.template +++ b/barreleye/grafana_dashboards/lustre_client.json.template @@ -16,7 +16,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1692325875474, + "iteration": 1693407689985, "links": [], "panels": [ { @@ -41,7 +41,7 @@ "y": 0 }, "hiddenSeries": false, - "id": 205, + "id": 207, "legend": { "avg": false, "current": false, @@ -118,7 +118,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -127,7 +127,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -234,7 +234,123 @@ "label": null, "logBase": 1, "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$BARRELEYE_DATASOURCE_NAME", + "description": "The write operation rate on this client.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 205, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Write Operation Rate", + "groupBy": [], + "measurement": "client_stats_write_bytes_samples", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "client_uuid", + "operator": "=~", + "value": "/^$client_uuid$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", "show": true }, { @@ -243,7 +359,125 @@ "logBase": 1, "max": null, "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$BARRELEYE_DATASOURCE_NAME", + "description": "The read operation rate on this client.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 208, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Read Operation Rate", + "groupBy": [], + "measurement": "client_stats_read_bytes_samples", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\" FROM \"client_stats_read_bytes_samples\" WHERE (\"client_uuid\" =~ /^$client_uuid$/) AND $timeFilter", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "client_uuid", + "operator": "=~", + "value": "/^$client_uuid$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Read Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } ], "yaxis": { @@ -280,7 +514,7 @@ "h": 7, "w": 24, "x": 0, - "y": 8 + "y": 16 }, "id": 90, "interval": null, @@ -552,6 +786,7 @@ }, { "aggregation": "Last", + "alias": "Page Mkwrite", "decimals": 2, "displayAliasType": "Warning / Critical", "displayType": "Regular", @@ -617,7 +852,7 @@ ], "orderByTime": "ASC", "policy": "default", - "query": "SELECT \"value\" * 1000 FROM \"client_stats_seek_mean\" WHERE (\"client_uuid\" =~ /^$client_uuid$/) AND $timeFilter", + "query": "SELECT \"value\" * 1000 FROM \"client_stats_seek_mean\" WHERE (\"client_uuid\" =~ /^$client_uuid $/) AND $timeFilter", "rawQuery": true, "refId": "H", "resultFormat": "time_series", @@ -1605,7 +1840,7 @@ "value": "lustre0" }, "datasource": "$BARRELEYE_DATASOURCE_NAME", - "definition": "SHOW TAG VALUES FROM \"client_stats_open_max\" WITH KEY = fs_name", + "definition": "SHOW TAG VALUES FROM \"client_stats_statfs_max\" WITH KEY = fs_name", "error": null, "hide": 0, "includeAll": false, @@ -1613,7 +1848,7 @@ "multi": false, "name": "fs_name", "options": [], - "query": "SHOW TAG VALUES FROM \"client_stats_open_max\" WITH KEY = fs_name", + "query": "SHOW TAG VALUES FROM \"client_stats_statfs_max\" WITH KEY = fs_name", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1628,11 +1863,38 @@ "allValue": null, "current": { "selected": false, + "text": "autotest-el7-vm313", + "value": "autotest-el7-vm313" + }, + "datasource": "$BARRELEYE_DATASOURCE_NAME", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Client", + "multi": false, + "name": "client_host", + "options": [], + "query": "SHOW TAG VALUES FROM \"client_stats_statfs_max\" WITH KEY = fqdn WHERE \"fs_name\" = '$fs_name'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, "text": "ffff9cca7941f000", "value": "ffff9cca7941f000" }, "datasource": "$BARRELEYE_DATASOURCE_NAME", - "definition": "SHOW TAG VALUES FROM \"client_stats_open_max\" WITH KEY = client_uuid WHERE \"fs_name\" = '$fs_name'", + "definition": "", "error": null, "hide": 0, "includeAll": false, @@ -1640,7 +1902,7 @@ "multi": false, "name": "client_uuid", "options": [], - "query": "SHOW TAG VALUES FROM \"client_stats_open_max\" WITH KEY = client_uuid WHERE \"fs_name\" = '$fs_name'", + "query": "SHOW TAG VALUES FROM \"client_stats_statfs_max\" WITH KEY = client_uuid WHERE \"fs_name\" = '$fs_name' AND \"fqdn\" = '$client_host'", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1675,6 +1937,6 @@ }, "timezone": "browser", "title": "Lustre Client", - "uid": "Bj-VYMRSz", - "version": 14 + "uid": "0ToQbnkSk", + "version": 31 } \ No newline at end of file diff --git a/barreleye/lustre.m4 b/barreleye/lustre.m4 index 109505e..935f525 100644 --- a/barreleye/lustre.m4 +++ b/barreleye/lustre.m4 @@ -613,7 +613,7 @@ define(`CLIENT_STATS_ITEM_FOUR_BYTES', `ELEMENT($1, item, `NAME($1 + 1, client_stats_$2, 1) PATTERN($1 + 1, `$2 +([[:digit:]]+) samples \[$3\] ([[:digit:]]+) ([[:digit:]]+) ([[:digit:]]+)', 0) -CLIENT_STATS_FIELD($1 + 1, 1, $2_samples, number, gauge) +CLIENT_STATS_FIELD($1 + 1, 1, $2_samples, number, derive) CLIENT_STATS_FIELD($1 + 1, 2, $2_min, number, gauge) CLIENT_STATS_FIELD($1 + 1, 3, $2_max, number, gauge) CLIENT_STATS_FIELD($1 + 1, 4, $2_sum, number, derive) diff --git a/bash_completion/Makefile.am b/bash_completion/Makefile.am index 9b3ca95..d0c1fc0 100644 --- a/bash_completion/Makefile.am +++ b/bash_completion/Makefile.am @@ -1,12 +1,13 @@ # Somehow rpmbuild does not include pip3's library to Python3's sys.path # which will cause missing module. So add it explicitly here. PIP3_PACKAGE_PATH = /usr/local/lib/python3.6/site-packages:/usr/local/lib64/python3.6/site-packages - +COMPLETIONS = +if ENABLE_BARRELE +COMPLETIONS += barrele barrele: PYTHONPATH=$(PIP3_PACKAGE_PATH) ../barrele -- --completion > barrele.tmp || exit 1; \ mv barrele.tmp barrele - -COMPLETIONS = barrele +endif clean-local: rm -f $(COMPLETIONS) diff --git a/configure.ac b/configure.ac index 934cffe..2dd1af4 100644 --- a/configure.ac +++ b/configure.ac @@ -1,9 +1,12 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT([coral], [m4_esyscmd(./coral version show)], [Coralfs], [coral]) +AC_INIT([coral],[m4_esyscmd(./coral version show)],[Coralfs],[coral]) AC_CONFIG_SRCDIR(./pycoral/ssh_host.py) -AC_CANONICAL_SYSTEM +AC_CANONICAL_TARGET AC_CONFIG_HEADERS(config.h) +dnl To avoid warning about $(wildcard ...) +AC_SUBST([DOLLAR],[$]) + AM_INIT_AUTOMAKE([tar-pax dist-bzip2 foreign subdir-objects]) AM_EXTRA_RECURSIVE_TARGETS([check_clean]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -19,10 +22,7 @@ AC_PROG_MAKE_SET AM_PROG_CC_C_O AM_CONDITIONAL(COMPILER_IS_GCC, test "x$GCC" = "xyes") -AC_DISABLE_STATIC -AC_PROG_LIBTOOL -AC_PROG_LEX -AC_PROG_YACC +LT_INIT([disable-static]) PKG_PROG_PKG_CONFIG # @@ -84,20 +84,29 @@ AC_DEFUN([MB_ARG_CANON_PATH], [ __MB_ARG_CANON_PATH([$1], m4_translit([$1], [-.], [__]), [$2]) ]) -dnl We may be able to use older versions, but I have not verified that -PKG_CHECK_MODULES([ext2fs], [ext2fs >= 1.42.7 com_err >= 1.42.7]) PKG_CHECK_MODULES([json_c], [json-c >= 0.11]) # -------- check for distro version -------- AC_MSG_CHECKING([for distro version]) DISTRO=$(sh detect-distro.sh) DISTRO_NAME=$(echo $DISTRO | awk -F '-' '{print $1}') -if [[[ "$DISTRO_NAME" != "rhel" ]]]; then +if [[[ "$DISTRO_NAME" = "rhel" ]]]; then + DISTRO_RELEASE=$(echo $DISTRO | awk -F 'rhel-' '{print $2}' | awk -F '.' '{print $1}') + DISTRO_SHORT=el$DISTRO_RELEASE + make_rpms="yes" +elif [[[ "$DISTRO_NAME" = "ubuntu" ]]]; then + DISTRO_RELEASE=$(echo $DISTRO | awk -F 'ubuntu-' '{print $2}' | awk -F '.' '{print $1}') + if [[[ "$DISTRO_RELEASE" != "22" ]]]; then + AC_MSG_ERROR([$DISTRO_RELEASE of Ubuntu is not a supported distro.]) + fi + DISTRO_SHORT=ubuntu2204 + make_debs="yes" +else AC_MSG_ERROR([$DISTRO_NAME is not a supported distro.]) fi -DISTRO_RELEASE=$(echo $DISTRO | awk -F 'rhel-' '{print $2}' | awk -F '.' '{print $1}') -DISTRO_SHORT=el$DISTRO_RELEASE AC_MSG_RESULT([$DISTRO_SHORT]) +AM_CONDITIONAL(MAKE_DEBS, test "$make_debs" = "yes") +AM_CONDITIONAL(MAKE_RPMS, test "$make_rpms" = "yes") AC_SUBST(DISTRO_SHORT) # ------- check for target_cpu -------- @@ -112,7 +121,7 @@ fi # -------- check whether enable zfs support -------- AC_MSG_CHECKING([whether enable zfs support]) AC_ARG_ENABLE([zfs], - AC_HELP_STRING([--enable-zfs], + AS_HELP_STRING([--enable-zfs], [enable zfs support]), [], [enable_zfs="no"]) AC_MSG_RESULT([$enable_zfs]) @@ -179,7 +188,7 @@ AS_IF([test "x$enable_zfs" = xyes ], [ # -------- whether enable Barreleye plugin -------- AC_MSG_CHECKING([whether enable Barreleye plugin]) AC_ARG_ENABLE([barrele], - AC_HELP_STRING([--disable-barrele], + AS_HELP_STRING([--disable-barrele], [disable Barreleye plugin]), [], [enable_barrele="yes"]) AC_MSG_RESULT([$enable_barrele]) @@ -190,10 +199,8 @@ AM_CONDITIONAL(ENABLE_BARRELE, test "$enable_barrele" = "yes") # -------- check for ISO cache dir -------- AC_MSG_CHECKING([for ISO cache dir]) ISO_CACHE_PATH="/var/log/coral/build_cache/release/iso_cache" -AS_IF([test "$enable_devel" != "no"], - [ISO_CACHE_PATH="/var/log/coral/build_cache/devel/iso_cache"]) AC_ARG_WITH([iso-cache], - AC_HELP_STRING([--with-iso-cache=path], + AS_HELP_STRING([--with-iso-cache=path], [set path of ISO cache dir]), [MB_ARG_CANON_PATH([iso-cache], [ISO_CACHE_PATH])], []) diff --git a/coral.spec.in b/coral.spec.in index 603e34b..60a70de 100644 --- a/coral.spec.in +++ b/coral.spec.in @@ -64,8 +64,8 @@ mkdir -p $RPM_BUILD_ROOT%{_datadir}/bash-completion/completions %if %{with barrele} cp dist/barrele $RPM_BUILD_ROOT%{_bindir} -cp example_configs/barreleye.toml \ - $RPM_BUILD_ROOT%{_sysconfdir}/coral/barreleye.conf.example +cp example_configs/barreleye.conf.example \ + $RPM_BUILD_ROOT%{_sysconfdir}/coral install -g 0 -o 0 -m 0644 bash_completion/barrele \ $RPM_BUILD_ROOT%{_datadir}/bash-completion/completions mkdir -p $RPM_BUILD_ROOT%{_sharedstatedir}/coral/barrele/xmls diff --git a/detect-distro.sh b/detect-distro.sh index 8bbdd36..ecff91e 100755 --- a/detect-distro.sh +++ b/detect-distro.sh @@ -30,6 +30,9 @@ if which lsb_release >/dev/null 2>&1; then "Fedora") name="fc" ;; + "Ubuntu") + name="ubuntu" + ;; *) fatal 1 "I don't know what distro name $name and version $version is.\nEither update autodetect_distro() or use the --distro argument." ;; diff --git a/example_configs/barreleye.toml b/example_configs/barreleye.conf.example similarity index 100% rename from example_configs/barreleye.toml rename to example_configs/barreleye.conf.example diff --git a/pybarrele/barrele.py b/pybarrele/barrele.py index 1f66f4a..b8823bc 100644 --- a/pybarrele/barrele.py +++ b/pybarrele/barrele.py @@ -803,6 +803,21 @@ def stop(self, host): ret = barreleye_instance.bei_stop_agents(log, hostnames) cmd_general.cmd_exit(log, ret) + def install(self): + """ + Install the Barreleye agent on the local host. + + This command can be run on a host that has not been configured + as agent in barreleye.conf. It is useful when installing standalone + agents on hosts with distro of Ubuntu. + """ + log, barreleye_instance = init_env(self._bac_config_fpath, + self._bac_logdir, + self._bac_log_to_file, + self._bac_iso) + ret = barreleye_instance.bei_install_agent_locally(log) + cmd_general.cmd_exit(log, ret) + class BarreleCommand(): """ diff --git a/pybarrele/barrele_agent.py b/pybarrele/barrele_agent.py index 49515ca..e191b3b 100644 --- a/pybarrele/barrele_agent.py +++ b/pybarrele/barrele_agent.py @@ -19,7 +19,7 @@ def __init__(self, host, barreleye_server, enable_disk=False, enable_lustre_oss=True, enable_lustre_mds=True, enable_lustre_client=False, enable_infiniband=False): - # Barreleye server with thye of BarreleServer + # Barreleye server with type of BarreleServer self.bea_barreleye_server = barreleye_server # Host to run commands. self.bea_host = host @@ -75,7 +75,9 @@ def _bea_sanity_check(self, log): return -1 distro = self.bea_host.sh_distro(log) - if distro not in [ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8]: + if distro not in (ssh_host.DISTRO_RHEL7, + ssh_host.DISTRO_RHEL8, + ssh_host.DISTRO_UBUNTU2204): log.cl_error("host [%s] has unsupported distro [%s]", self.bea_host.sh_hostname, distro) return -1 @@ -112,7 +114,7 @@ def _bea_sanity_check(self, log): return -1 return 0 - def _bea_check_lustre_version(self, log, lustre_fallback_version): + def _bea_check_lustre_version_rpm(self, log, lustre_fallback_version): """ Check the Lustre version according to the installed RPMs """ @@ -165,6 +167,81 @@ def _bea_check_lustre_version(self, log, lustre_fallback_version): self.bea_lustre_version = version return 0 + def _bea_check_lustre_version_deb(self, log, lustre_fallback_version): + """ + Check the Lustre version according to the installed debs + """ + # pylint: disable=too-many-return-statements,too-many-branches + command = ("apt list --installed | grep lustre-client-modules") + retval = self.bea_host.sh_run(log, command) + if (retval.cr_exit_status == 1 and retval.cr_stdout == ""): + log.cl_info("Lustre deb is not installed on host [%s], " + "using default [%s]", + self.bea_host.sh_hostname, + lustre_fallback_version.lv_name) + self.bea_lustre_version = lustre_fallback_version + return 0 + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + self.bea_host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + deb_lines = retval.cr_stdout.splitlines() + if len(deb_lines) != 1: + log.cl_error("multiple lines outputed by command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + self.bea_host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + deb_line = deb_lines[0] + fields = deb_line.split() + if len(fields) != 4: + log.cl_error("unexpected field number outputed by command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + self.bea_host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + version = fields[1] + lversion = lustre_version.match_lustre_version_from_deb(log, version) + if lversion is None: + log.cl_error("failed to detect Lustre version on host [%s]", + self.bea_host.sh_hostname) + return -1 + + log.cl_info("Lustre version [%s] detected on host [%s]", + lversion.lv_name, + self.bea_host.sh_hostname) + self.bea_lustre_version = lversion + return 0 + + def _bea_check_lustre_version(self, log, lustre_fallback_version): + """ + Check the Lustre version according to the installed RPMs or debs + """ + host = self.bea_host + distro = host.sh_distro(log) + + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + return self._bea_check_lustre_version_rpm(log, lustre_fallback_version) + if distro in (ssh_host.DISTRO_UBUNTU2204): + return self._bea_check_lustre_version_deb(log, lustre_fallback_version) + + log.cl_error("distro [%s] of host [%s] is not supported", + distro, host.sh_hostname) + return -1 + def _bea_generate_collectd_config(self, log, barreleye_instance, collectd_test=False): """ @@ -227,14 +304,6 @@ def bea_generate_configs(self, log, barreleye_instance): "usage") return -1 self.bea_collectd_config_for_production = collectd_config - - # Check that needed collectd RPMs are installed - for rpm_type in self.bea_needed_collectd_rpm_types: - if rpm_type not in barreleye_instance.bei_collectd_rpm_type_dict: - log.cl_error("needed Collectd RPM [%s] of agent [%s] does not " - "exist", - rpm_type, self.bea_host.sh_hostname) - return -1 return 0 def _bea_influxdb_measurement_check(self, log, measurement_name, tags): @@ -354,6 +423,19 @@ def bea_collectd_send_config(self, log, barreleye_instance, Dump and send the collectd.conf to the agent host """ host = self.bea_host + local_host = barreleye_instance.bei_local_host + command = "mkdir -p %s" % barreleye_instance.bei_workspace + retval = local_host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + local_host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + fpath = barreleye_instance.bei_workspace + "/" if test_config: fpath += barrele_collectd.COLLECTD_CONFIG_TEST_FNAME @@ -365,7 +447,11 @@ def bea_collectd_send_config(self, log, barreleye_instance, collectd_config.cdc_dump(fpath) - etc_path = "/etc/collectd.conf" + distro = self.bea_host.sh_distro(log) + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + etc_path = "/etc/collectd.conf" + elif distro in (ssh_host.DISTRO_UBUNTU2204): + etc_path = "/etc/collectd/collectd.conf" ret = host.sh_send_file(log, fpath, etc_path) if ret: log.cl_error("failed to send file [%s] on local host [%s] to " diff --git a/pybarrele/barrele_collectd.py b/pybarrele/barrele_collectd.py index 906ef38..c621ba1 100644 --- a/pybarrele/barrele_collectd.py +++ b/pybarrele/barrele_collectd.py @@ -1298,24 +1298,106 @@ def cdc_plugin_infiniband(self): barreleye_agent.bea_needed_collectd_rpm_types.append(rpm_name) -def collectd_rpm_type_from_name(log, name): +def collectd_package_type_from_name(log, name): """ - Return Collectd type from full RPM name or RPM fname - The Collectd RPM types. The RPM type is the minimum string + Return Collectd type from full RPM/deb name or RPM/deb fname + The Collectd RPM/deb types. The RPM type is the minimum string that yum could understand and find the RPM. For example: libcollectdclient-5.11.0...rpm has a type of libcollectdclient; collectd-5.11.0...rpm has a type of collectd; collectd-disk-5.11.0...rpm has a type of collectd-disk. + + Example debs: + collectd_5.12.0.brl3_amd64.deb + collectd-core_5.12.0.brl3_amd64.deb + collectd-dev_5.12.0.brl3_all.deb + collectd-utils_5.12.0.brl3_amd64.deb + libcollectdclient1_5.12.0.brl3_amd64.deb + libcollectdclient-dev_5.12.0.brl3_amd64.deb """ if ((not name.startswith("collectd")) and (not name.startswith("libcollectdclient"))): return None - collectd_pattern = (r"^(?P\S+)-(\d+)\.(\d+).+") + collectd_pattern = (r"^(?P\S+)[-_](\d+)\.(\d+).+") collectd_regular = re.compile(collectd_pattern) match = collectd_regular.match(name) if match is None: log.cl_error("name [%s] starts with [collectd] but does not match " - "the RPM pattern", name) + "the package pattern", name) return None return match.group("type") + + +def get_collectd_package_type_dict(log, host, packages_dir): + """ + Return a dict. Key is the RPM/deb type, value is the file name. + + The RPM type is the minimum string that yum could understand and + find the RPM. + + For example: + libcollectdclient-5.11.0...rpm has a type of libcollectdclient; + collectd-5.11.0...rpm has a type of collectd; + collectd-disk-5.11.0...rpm has a type of collectd-disk. + """ + fnames = host.sh_get_dir_fnames(log, packages_dir) + if fnames is None: + log.cl_error("failed to get fnames under dir [%s] on " + "host [%s]", packages_dir, host.sh_hostname) + return None + collectd_package_type_dict = {} + for fname in fnames: + if ((not fname.startswith("collectd")) and + (not fname.startswith("libcollectdclient"))): + continue + package_type = collectd_package_type_from_name(log, fname) + if package_type is None: + log.cl_error("failed to get the package type from name [%s]", + fname) + return None + if package_type in collectd_package_type_dict: + log.cl_error("both Collectd packages [%s] and [%s] matches " + "type [%s]", fname, + collectd_package_type_dict[package_type], + package_type) + return None + + collectd_package_type_dict[package_type] = fname + log.cl_debug("Collectd package [%s] is found under dir [%s] on local " + "host [%s]", package_type, packages_dir, + host.sh_hostname) + return collectd_package_type_dict + + +def collectd_debs_install(log, host, packages_dir): + """ + Install all the Collectd debs under the package dir + """ + # Remove the existing collectd configuration to avoid failure of starting + # collectd service when installing Collectd deb file. + command = ("rm -f /etc/collectd/collectd.conf") + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + command = ("dpkg -i %s/collectd*.deb %s/libcollectdclient*.deb" % + (packages_dir, packages_dir)) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + return 0 diff --git a/pybarrele/barrele_constant.py b/pybarrele/barrele_constant.py index e41738e..9a7a03c 100644 --- a/pybarrele/barrele_constant.py +++ b/pybarrele/barrele_constant.py @@ -30,6 +30,8 @@ # RPMs needed to download for Barreleye server and agent BARRELE_DOWNLOAD_DEPENDENT_RPMS = (BARRELE_SERVER_DOWNLOAD_DEPENDENT_RPMS + BARRELE_AGENT_DEPENDENT_RPMS) +# Debs needed to download for Barreleye server and agent (Ubuntu) +BARRELE_DOWNLOAD_DEPENDENT_DEBS = ["librrd8"] # RPMS needed by Barreleye servers. BARRELE_SERVER_DEPENDENT_RPMS = ["influxdb", "grafana"] BARRELE_SERVER_DEPENDENT_RPMS += BARRELE_DOWNLOAD_DEPENDENT_RPMS diff --git a/pybarrele/barrele_instance.py b/pybarrele/barrele_instance.py index 46701e7..dd4352d 100644 --- a/pybarrele/barrele_instance.py +++ b/pybarrele/barrele_instance.py @@ -70,67 +70,39 @@ def __init__(self, workspace, config, config_fpath, log_to_file, self.bei_iso_dir = constant.CORAL_ISO_DIR # The server of barreleye self.bei_barreleye_server = barreleye_server - # The Collectd RPM types. The RPM type is the minimum string - # that yum could understand and find the RPM. - # For example: - # libcollectdclient-5.11.0...rpm has a type of libcollectdclient; - # collectd-5.11.0...rpm has a type of collectd; - # collectd-disk-5.11.0...rpm has a type of collectd-disk. - # - # Key is RPM type. Value is RPM fname. - self.bei_collectd_rpm_type_dict = None # ISO file path self.bei_iso_fpath = iso_fpath - def _bei_get_collectd_rpm_types(self, log): + def _bei_get_collectd_package_type_dict(self, log): """ - Get Collectd RPMs from ISO dir on local host + Return a dict. Key is the RPM/deb type, value is the file name. """ packages_dir = self.bei_iso_dir + "/" + constant.BUILD_PACKAGES - fnames = self.bei_local_host.sh_get_dir_fnames(log, packages_dir) - if fnames is None: - log.cl_error("failed to get fnames under dir [%s] on local " - "host [%s]", self.bei_iso_dir, - self.bei_local_host.sh_hostname) - return -1 - self.bei_collectd_rpm_type_dict = {} - for fname in fnames: - if ((not fname.startswith("collectd")) and - (not fname.startswith("libcollectdclient"))): - continue - rpm_type = \ - barrele_collectd.collectd_rpm_type_from_name(log, fname) - if rpm_type is None: - log.cl_error("failed to get the RPM type from name [%s]", - fname) - return -1 - if rpm_type in self.bei_collectd_rpm_type_dict: - log.cl_error("both Collectd RPMs [%s] and [%s] matches " - "type [%s]", fname, - self.bei_collectd_rpm_type_dict[rpm_type], - rpm_type) - return -1 - - self.bei_collectd_rpm_type_dict[rpm_type] = fname - log.cl_debug("Collectd RPM [%s] is found under dir [%s] on local " - "host [%s]", rpm_type, self.bei_iso_dir, - self.bei_local_host.sh_hostname) - return 0 + return barrele_collectd.get_collectd_package_type_dict(log, + self.bei_local_host, + packages_dir) def _bei_cluster_install_rpms(self, log): """ Install RPMs on the cluster """ - ret = self._bei_get_collectd_rpm_types(log) - if ret: + rpm_type_dict = self._bei_get_collectd_package_type_dict(log) + if rpm_type_dict is None: log.cl_error("failed to get Collectd RPM types") return -1 + for rpm_type in (barrele_collectd.LIBCOLLECTDCLIENT_TYPE_NAME, + barrele_collectd.COLLECTD_TYPE_NAME): + if rpm_type not in rpm_type_dict: + log.cl_error("failed to find Collectd RPM [%s]", + rpm_type) + return -1 + for agent in self.bei_agent_dict.values(): ret = agent.bea_generate_configs(log, self) if ret: - log.cl_error("failed to detect the Lustre version on host [%s]", + log.cl_error("failed to generate Barreleye agent configs on host [%s]", agent.bea_host.sh_hostname) return -1 @@ -269,6 +241,90 @@ def bei_start_agents(self, log, hostnames): return -1 return 0 + def bei_install_agent_locally(self, log): + """ + Install Barreleye agent on local host. + """ + distro = self.bei_local_host.sh_distro(log) + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + log.cl_error("please install agent on host [%s] by using [barrele cluster install] " + "command", + self.bei_local_host.sh_hostname) + return -1 + if distro not in (ssh_host.DISTRO_UBUNTU2204): + log.cl_error("distro [%s] of host [%s] is not supported", + distro, self.bei_local_host.sh_hostname) + return -1 + + iso = self.bei_iso_fpath + if iso is not None: + ret = install_common.sync_iso_dir(log, self.bei_workspace, + self.bei_local_host, iso, + self.bei_iso_dir) + if ret: + log.cl_error("failed to sync ISO files from [%s] to dir [%s] " + "on local host [%s]", + iso, self.bei_iso_dir, + self.bei_local_host.sh_hostname) + return -1 + + # Install Barreleye deb file. + package_dir = self.bei_iso_dir + "/" + constant.BUILD_PACKAGES + command = ("dpkg -i %s/coral-*.deb" % (package_dir)) + retval = self.bei_local_host.sh_run(log, command, timeout=None) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + self.bei_local_host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + package_type_dict = self._bei_get_collectd_package_type_dict(log) + if package_type_dict is None: + log.cl_error("failed to get Collectd package types") + return -1 + + for deb_type in ("collectd-core", "collectd-utils", + "collectd", "libcollectdclient1"): + if deb_type not in package_type_dict: + log.cl_error("failed to find Collectd deb [%s]", + deb_type) + return -1 + + packages_dir = self.bei_iso_dir + "/" + constant.BUILD_PACKAGES + ret = barrele_collectd.collectd_debs_install(log, self.bei_local_host, + packages_dir) + if ret: + log.cl_error("failed to install Collectd debs on local host [%s]", + self.bei_local_host.sh_hostname) + return -1 + + agent = barrele_agent.BarreleAgent(self.bei_local_host, + self.bei_barreleye_server, + enable_disk=False, + enable_lustre_oss=False, + enable_lustre_mds=False, + enable_lustre_client=True, + enable_infiniband=False) + ret = agent.bea_generate_configs(log, self) + if ret: + log.cl_error("failed to generate configs for Barreleye agent [%s]", + agent.bea_host.sh_hostname) + return -1 + + ret = agent.bea_config_agent(log, self) + if ret: + log.cl_error("failed to configure Barreleye agent [%s]", + agent.bea_host.sh_hostname) + return -1 + + log.cl_info("Barreleye agent has been installed successfully on host [%s]", + agent.bea_host.sh_hostname) + return 0 + def parse_server_config(log, config, config_fpath, host_dict): """ diff --git a/pybarrele/barrele_server.py b/pybarrele/barrele_server.py index 8a61606..3061c0f 100644 --- a/pybarrele/barrele_server.py +++ b/pybarrele/barrele_server.py @@ -992,7 +992,7 @@ def _bes_grafana_user_add(self, log, name, email_address, login, return -1 return 0 - def _bes_grafana_user_info(self, log, name): + def _bes_grafana_user_info(self, log, login): """ Add viewer user """ @@ -1001,19 +1001,19 @@ def _bes_grafana_user_info(self, log, name): "Accept": "application/json"} url = self.bes_grafana_admin_url("/api/users/lookup?loginOrEmail=%s" % - (slugify(name))) + (slugify(login))) try: response = requests.get(url, headers=headers) except: - log.cl_error("not able to get users through [%s]: %s", + log.cl_error("failed to get user info through [%s]: %s", url, traceback.format_exc()) return -1, None if response.status_code == HTTPStatus.OK: return 1, response.json() if response.status_code == HTTPStatus.NOT_FOUND: return 0, None - log.cl_error("got status [%d] when getting user info from Grafana", - response.status_code) + log.cl_error("got status [%d] when getting user info through [%s]", + response.status_code, url) return -1, None def _bes_grafana_user_recreate(self, log, name, email_address, login, @@ -1022,7 +1022,7 @@ def _bes_grafana_user_recreate(self, log, name, email_address, login, If user doesn't exist, add the user. If user exists, remove it first. """ - ret, json_info = self._bes_grafana_user_info(log, "viewer") + ret, json_info = self._bes_grafana_user_info(log, login) if ret < 0: return -1 if ret == 1: diff --git a/pybuild/Makefile.am b/pybuild/Makefile.am index 5cfb248..978402b 100644 --- a/pybuild/Makefile.am +++ b/pybuild/Makefile.am @@ -9,11 +9,12 @@ AM_CFLAGS = -Wall -Werror -g $(json_c_CFLAGS) $(json_c_LIBS) \ PIP3_PACKAGE_PATH = /usr/local/lib/python3.6/site-packages:/usr/local/lib64/python3.6/site-packages CORAL_CMD=PYTHONPATH=$(PIP3_PACKAGE_PATH) ../coral +../pycoral/version.py: build_version.py + $(CORAL_CMD) version save ../pycoral/version.py + SOURCES = ../pycoral/version.py CHECKS = -../pycoral/version.py: build_version.py - $(CORAL_CMD) version save ../pycoral/version.py clean-local: rm -fr $(SOURCES) $(CHECKS) diff --git a/pybuild/build_barrele.py b/pybuild/build_barrele.py index f0b2596..0a54f11 100644 --- a/pybuild/build_barrele.py +++ b/pybuild/build_barrele.py @@ -1,6 +1,7 @@ """ Library for building Barreleye """ +# pylint: disable=too-many-lines import os import stat from pycoral import constant @@ -13,12 +14,15 @@ PACAKGE_URL_DICT = {} # The URL of Collectd tarball COLLECTD_URL = ("https://github.com/LiXi-storage/collectd/releases/download/" - "collectd-5.12.0.brl2/collectd-5.12.0.brl2.tar.bz2") + "collectd-5.12.0.brl3/collectd-5.12.0.brl3.tar.bz2") # The sha1sum of Collectd tarball. Need to update together with # COLLECTD_URL -COLLECTD_SHA1SUM = "9fb8be9d7c0bf7c84b93ef5bf441d393b081e7d9" -PACAKGE_URL_DICT["collectd"] = COLLECTD_URL +COLLECTD_SHA1SUM = "7469694df09576b9e5460a0a6ee8d429af962bda" +PACAKGE_URL_DICT["collectd"] = COLLECTD_URL# The RPM names of Collectd to check +# The deb names of Collectd to check +COLLECTD_DEB_NAMES = ["collectd", "collectd-core", + "collectd-utils", "libcollectdclient1"] # The RPM names of Collectd to check COLLECTD_RPM_NAMES = ["collectd", "collectd-disk", "collectd-filedata", "collectd-sensors", "collectd-ssh", @@ -102,8 +106,90 @@ ["postgresql-devel", "python-devel"]) COLLECTD_BUILD_DEPENDENT_RHEL8_RPMS = (BARRELEYE_BUILD_DEPENDENT_COMMON_RPMS + ["libpq-devel", "python36-devel"]) +COLLECTD_BUILD_DEPENDENT_UBUNTU2204_DEBS = ["bison", + "default-jdk", + "default-libmysqlclient-dev", + "flex", + "intel-cmt-cat", + "javahelper", + "libatasmart-dev", + "libcap-dev", + "libcurl4-gnutls-dev", + "libcurl4-gnutls-dev", + "libcurl4-gnutls-dev", + "libdbi-dev", + "libdpdk-dev", + "libesmtp-dev", + "libganglia1-dev", + "libgcrypt20-dev", + "libglib2.0-dev", + "libgps-dev", + "libgrpc++-dev", + "libhiredis-dev", + "libi2c-dev", + "libip4tc-dev", + "libip6tc-dev", + "libiptc-dev", + "libiptc-dev", + "libldap2-dev", + "liblua5.3-dev", + "libmemcached-dev", + "libmicrohttpd-dev", + "libmnl-dev", + "libmodbus-dev", + "libmongoc-dev", + "libmosquitto-dev", + "libnotify-dev", + "libntirpc-dev", + "libopenipmi-dev", + "liboping-dev", + "libpcap0.8-dev", + "libpcap-dev", + "libperl-dev", + "libpq-dev", + "libprotobuf-c-dev", + "libprotobuf-dev", + "libqpid-proton11-dev", + "librabbitmq-dev", + "librdkafka-dev", + "libriemann-client-dev", + "librrd-dev", + "libsensors-dev", + "libslurm-dev", + "libsnmp-dev", + "libssh2-1-dev", + "libudev-dev", + "libupsclient-dev", + "libupsclient-dev", + "libvarnishapi-dev", + "libvirt-dev", + "libxen-dev", + "libxml2-dev", + "libyajl-dev", + "libzmq3-dev", + "perl", + "protobuf-c-compiler", + "protobuf-compiler", + "protobuf-compiler-grpc", + "python3-dev", + "riemann-c-client", + "uthash-dev"] +BARRELE_BUILD_DEPENDENT_UBUNTU2204_DEBS = (COLLECTD_BUILD_DEPENDENT_UBUNTU2204_DEBS + + ["bzip2", + "libattr1-dev", + "libext2fs-dev", + "pylint", + "python3-slugify", + "python3-pip", + "genisoimage"]) + BARRELEYE_BUILD_DEPENDENT_PIPS = ["requests", "python-slugify"] +# The URL of Collectd debian tarball +COLLECTD_DEBIAN_URL = "http://deb.debian.org/debian/pool/main/c/collectd/collectd_5.12.0-14.debian.tar.xz" +# The sha1sum of Collectd debian tarball. Need to update together with +# COLLECTD_DEBIAN_URL +COLLECTD_DEBIAN_SHA1SUM = "b78277bee7e55d5a58da9d75a8d1eab9de3ef733" def get_collectd_rpm_suffix(distro_number, target_cpu, collectd_version_release): @@ -184,11 +270,11 @@ def get_and_clean_collectd_rpms(log, host, packages_dir, return collectd_fnames -def remove_collectd_rpms(log, host, packages_dir): +def remove_collectd_packages(log, host, packages_dir): """ - Remove old Collectd RPMs + Remove old Collectd RPMs or debs """ - patterns = ["collectd-*", "libcollectdclient-*"] + patterns = ["collectd*", "libcollectdclient*"] for pattern in patterns: command = "rm -f %s/%s" % (packages_dir, pattern) retval = host.sh_run(log, command) @@ -336,10 +422,10 @@ def build_collectd_rpms(log, host, target_cpu, packages_dir, return 0 -def collectd_build_and_check(log, host, target_cpu, packages_dir, - collectd_src_dir, collectd_version, - collectd_version_release, - tarball_fpath, extra_package_fnames): +def collectd_build_and_check_rhel(log, host, target_cpu, packages_dir, + collectd_src_dir, collectd_version, + collectd_version_release, + tarball_fpath, extra_package_fnames): """ Check and build Collectd RPMs """ @@ -356,7 +442,7 @@ def collectd_build_and_check(log, host, target_cpu, packages_dir, elif distro == ssh_host.DISTRO_RHEL8: distro_number = "8" else: - log.cl_error("build on distro [%s] is not supported yet", distro) + log.cl_error("build Barreleye on distro [%s] is not supported yet", distro) return -1 ret = check_collectd_rpms_integrity(log, existing_rpm_fnames, @@ -377,7 +463,7 @@ def collectd_build_and_check(log, host, target_cpu, packages_dir, return 0 log.cl_debug("building Collectd RPMs") - ret = remove_collectd_rpms(log, host, packages_dir) + ret = remove_collectd_packages(log, host, packages_dir) if ret: log.cl_error("failed to remove old Collectd RPMs") return -1 @@ -419,7 +505,349 @@ def collectd_build_and_check(log, host, target_cpu, packages_dir, return 0 -def build_collectd_tarball(log, workspace, host, target_cpu, packages_dir, +def get_collectd_deb_suffix(log, target_cpu, + collectd_version): + """ + Return the suffix of Collectd debs. + The suffix starts from "_", e.g. + "_5.12.0.brl3_amd64.deb" + """ + if target_cpu == "x86_64": + deb_target_cpu = "amd64" + else: + log.cl_error("unsupported target CPU [%s]", target_cpu) + return None + return ("_%s_%s.deb" % + (collectd_version, deb_target_cpu)) + + +def get_collectd_dev_deb_suffix(collectd_version): + """ + Return the suffix of Collectd debs. + The suffix starts from "_", e.g. + "_5.12.0.brl3_all.deb" + """ + return ("_%s_all.deb" % + (collectd_version)) + + +def check_collectd_debs_integrity(log, deb_fnames, target_cpu, + collectd_version, quiet=True): + """ + Check whether the existing Collectd debs are complete. + """ + suffix = get_collectd_deb_suffix(log, target_cpu, + collectd_version) + if suffix is None: + return -1 + for collect_deb_name in COLLECTD_DEB_NAMES: + collect_deb_full = collect_deb_name + suffix + if collect_deb_full not in deb_fnames: + if not quiet: + log.cl_error("Deb [%s] does not exist", + collect_deb_full) + else: + log.cl_debug("Deb [%s] does not exist", + collect_deb_full) + return -1 + return 0 + + +def get_and_clean_collectd_debs(log, host, packages_dir, + deb_fnames, target_cpu, + collectd_version, + expect_clean=False): + """ + Return a list of Collectd debs under a directory. + If there are other version of Collectd debs, remove them. + """ + # pylint: disable=too-many-locals + suffix = get_collectd_deb_suffix(log, target_cpu, + collectd_version) + if suffix is None: + return None + dev_suffix = get_collectd_dev_deb_suffix(collectd_version) + + prefixes = ["collectd", "libcollectdclient"] + collectd_fnames = [] + for deb_fname in deb_fnames: + found = False + for prefix in prefixes: + if deb_fname.startswith(prefix): + found = True + if not found: + continue + + if not deb_fname.endswith(suffix) and not deb_fname.endswith(dev_suffix): + if expect_clean: + log.cl_error("Collectd deb [%s] has different suffix, " + "expected [%s]", + deb_fname, suffix) + return None + + log.cl_info("Collectd deb [%s] has different suffix, " + "expected [%s], removing", + deb_fname, suffix) + fpath = packages_dir + "/" + deb_fname + command = ("rm -f %s" % (fpath)) + retval = host.sh_run(command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return None + continue + collectd_fnames.append(deb_fname) + return collectd_fnames + + +def collectd_tarball_fpath2version(log, tarball_fpath): + """ + Return the Collectd version by parse the tarball fpath. + """ + collectd_tarball_fname = os.path.basename(tarball_fpath) + suffix = ".tar.bz2" + if not collectd_tarball_fname.endswith(suffix): + log.cl_error("tarball [%s] does not end with suffix [%s]", + collectd_tarball_fname, suffix) + return None + + prefix = "collectd-" + if not collectd_tarball_fname.startswith(prefix): + log.cl_error("tarball [%s] does not start with prefix [%s]", + collectd_tarball_fname, prefix) + return None + collectd_version = collectd_tarball_fname[len(prefix):-len(suffix)] + return collectd_version + + +def build_collectd_debs(log, host, source_dir, type_cache, + packages_dir, collectd_build_dir, + collectd_src_dir, tarball_fpath, + collectd_version): + """ + Build Collectd debs on a host + """ + # pylint: disable=too-many-locals + collectd_version = collectd_tarball_fpath2version(log, tarball_fpath) + if collectd_version is None: + return -1 + + tarball_fname = os.path.basename(COLLECTD_DEBIAN_URL) + tarball_fpath = type_cache + "/" + tarball_fname + ret = host.sh_download_file(log, COLLECTD_DEBIAN_URL, tarball_fpath, + COLLECTD_DEBIAN_SHA1SUM) + if ret: + log.cl_error("failed to download Collectd debian tarball") + return -1 + + command = "tar xf %s -C %s" % (tarball_fpath, collectd_src_dir) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + patch_dir = source_dir + "/barreleye/collectd/ubuntu_build_patches" + rc = build_common.apply_patches(log, host, collectd_src_dir, + patch_dir) + if rc: + log.cl_error("failed to apply ubuntu build patches to [%s]", + collectd_src_dir) + return -1 + + command = "date -R" + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + changelog_date = retval.cr_stdout.strip() + + changelog_lines = ["collectd (%s) unstable; urgency=medium" % collectd_version, + "", + " * Automated changelog." + "" + "Coral Packaging Team %s" % changelog_date] + changelog_fpath = collectd_src_dir + "/debian/changelog" + command = "> %s" % changelog_fpath + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + for changelog_line in changelog_lines: + command = ("echo \"%s\" >> %s" % + (changelog_line, + changelog_fpath)) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + command = "cd %s && dpkg-buildpackage -us -uc -I.git -I.github" % collectd_src_dir + log.cl_info("running command [%s] on host [%s]", + command, host.sh_hostname) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + command = ("mv %s/*.deb %s" % + (collectd_build_dir, packages_dir)) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + return 0 + + +def collectd_build_and_check_ubuntu(log, host, source_dir, + type_cache, target_cpu, + packages_dir, + collectd_build_dir, + collectd_src_dir, + collectd_version, + tarball_fpath, + extra_package_fnames): + """ + If existing Collectd debs are not complete, build them. + """ + existing_deb_fnames = host.sh_get_dir_fnames(log, packages_dir) + if existing_deb_fnames is None: + log.cl_error("failed to get fnames under dir [%s] on host [%s]", + packages_dir, + host.sh_hostname) + return -1 + + ret = check_collectd_debs_integrity(log, existing_deb_fnames, + target_cpu, + collectd_version) + if ret == 0: + log.cl_debug("Collectd debs already exist") + collectd_deb_fnames = get_and_clean_collectd_debs(log, host, + packages_dir, + existing_deb_fnames, + target_cpu, + collectd_version) + if collectd_deb_fnames is None: + log.cl_error("failed to get the Collectd deb names") + return -1 + extra_package_fnames += collectd_deb_fnames + return 0 + + log.cl_debug("building Collectd debs") + ret = remove_collectd_packages(log, host, packages_dir) + if ret: + log.cl_error("failed to remove old Collectd debs") + return -1 + + ret = build_collectd_debs(log, host, source_dir, + type_cache, packages_dir, + collectd_build_dir, + collectd_src_dir, tarball_fpath, + collectd_version) + if ret: + log.cl_error("failed to build Collectd debs from src [%s]", + collectd_src_dir) + return -1 + + existing_deb_fnames = host.sh_get_dir_fnames(log, packages_dir) + if existing_deb_fnames is None: + log.cl_error("failed to get fnames under dir [%s] on host [%s]", + packages_dir, + host.sh_hostname) + return -1 + + ret = check_collectd_debs_integrity(log, existing_deb_fnames, + target_cpu, + collectd_version) + if ret == 0: + collectd_deb_fnames = get_and_clean_collectd_debs(log, host, + packages_dir, + existing_deb_fnames, + target_cpu, + collectd_version) + if collectd_deb_fnames is None: + log.cl_error("failed to get the Collectd deb names") + return -1 + extra_package_fnames += collectd_deb_fnames + return 0 + return 0 + + +def collectd_build_and_check(log, host, source_dir, type_cache, + target_cpu, packages_dir, + collectd_build_dir, + collectd_src_dir, collectd_version, + collectd_version_release, + tarball_fpath, extra_package_fnames): + """ + If existing Collectd packages are not complete, build them. + """ + distro = host.sh_distro(log) + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + return collectd_build_and_check_rhel(log, host, target_cpu, + packages_dir, + collectd_src_dir, + collectd_version, + collectd_version_release, + tarball_fpath, + extra_package_fnames) + if distro in (ssh_host.DISTRO_UBUNTU2204): + return collectd_build_and_check_ubuntu(log, host, + source_dir, + type_cache, + target_cpu, + packages_dir, + collectd_build_dir, + collectd_src_dir, + collectd_version, + tarball_fpath, + extra_package_fnames) + return 0 + + +def build_collectd_tarball(log, workspace, host, source_dir, type_cache, + target_cpu, packages_dir, tarball_fpath, extra_package_fnames, known_collectd_version=None): """ @@ -487,23 +915,29 @@ def build_collectd_tarball(log, workspace, host, target_cpu, packages_dir, "source dir") return -1 - ret = collectd_build_and_check(log, host, target_cpu, packages_dir, - collectd_src_dir, collectd_version, + ret = collectd_build_and_check(log, host, source_dir, + type_cache, + target_cpu, + packages_dir, + collectd_build_dir, + collectd_src_dir, + collectd_version, collectd_version_release, tarball_fpath, extra_package_fnames) if ret: - log.cl_error("failed to build and check Collectd RPMs") + log.cl_error("failed to build and check Collectd packages") return -1 return 0 -def download_and_build_collectd(log, workspace, host, type_cache, target_cpu, +def download_and_build_collectd(log, workspace, host, source_dir, + type_cache, target_cpu, packages_dir, collectd_url, expected_sha1sum, extra_package_fnames): """ Download Collectd source code tarball and build """ - log.cl_info("building Collectd RPMs from URL [%s] on host [%s]", + log.cl_info("building Collectd packages from URL [%s] on host [%s]", collectd_url, host.sh_hostname) tarball_fname = os.path.basename(collectd_url) tarball_fpath = type_cache + "/" + tarball_fname @@ -513,8 +947,8 @@ def download_and_build_collectd(log, workspace, host, type_cache, target_cpu, log.cl_error("failed to download Collectd sourcecode tarball") return -1 - ret = build_collectd_tarball(log, workspace, host, target_cpu, - packages_dir, tarball_fpath, + ret = build_collectd_tarball(log, workspace, host, source_dir, type_cache, + target_cpu, packages_dir, tarball_fpath, extra_package_fnames) if ret: log.cl_error("failed to build Collectd tarball [%s]", @@ -523,8 +957,10 @@ def download_and_build_collectd(log, workspace, host, type_cache, target_cpu, return 0 -def build_collectd_dir(log, workspace, host, target_cpu, packages_dir, - origin_collectd_dir, extra_package_fnames): +def build_collectd_dir(log, workspace, host, source_dir, + type_cache, target_cpu, + packages_dir, origin_collectd_dir, + extra_package_fnames): """ Build Collectd from src dir """ @@ -604,7 +1040,8 @@ def build_collectd_dir(log, workspace, host, target_cpu, packages_dir, return -1 collectd_tarball_fpath = collectd_dir + "/" + collectd_tarball_fname - ret = build_collectd_tarball(log, workspace, host, target_cpu, packages_dir, + ret = build_collectd_tarball(log, workspace, host, source_dir, + type_cache, target_cpu, packages_dir, collectd_tarball_fpath, extra_package_fnames, known_collectd_version=collectd_version) if ret: @@ -614,23 +1051,29 @@ def build_collectd_dir(log, workspace, host, target_cpu, packages_dir, return 0 -def build_collectd(log, workspace, host, type_cache, target_cpu, packages_dir, +def build_collectd(log, workspace, host, source_dir, + type_cache, target_cpu, packages_dir, collectd, extra_package_fnames): """ Build Collectd """ if collectd is None: - return download_and_build_collectd(log, workspace, host, type_cache, - target_cpu, packages_dir, COLLECTD_URL, + return download_and_build_collectd(log, workspace, host, + source_dir, + type_cache, + target_cpu, + packages_dir, + COLLECTD_URL, COLLECTD_SHA1SUM, extra_package_fnames) stat_result = host.sh_stat(log, collectd) if stat_result is not None: if stat.S_ISREG(stat_result.st_mode): - log.cl_info("building Collectd RPMs from tarball [%s] on host [%s]", + log.cl_info("building Collectd packages from tarball [%s] on host [%s]", collectd, host.sh_hostname) - ret = build_collectd_tarball(log, workspace, host, target_cpu, + ret = build_collectd_tarball(log, workspace, host, source_dir, + type_cache, target_cpu, packages_dir, collectd, extra_package_fnames) if ret: @@ -638,9 +1081,11 @@ def build_collectd(log, workspace, host, type_cache, target_cpu, packages_dir, collectd) return -1 elif stat.S_ISDIR(stat_result.st_mode): - log.cl_info("building Collectd RPMs from dir [%s] on host [%s]", + log.cl_info("building Collectd packages from dir [%s] on host [%s]", collectd, host.sh_hostname) - ret = build_collectd_dir(log, workspace, host, target_cpu, + ret = build_collectd_dir(log, workspace, host, source_dir, + type_cache, + target_cpu, packages_dir, collectd, extra_package_fnames) if ret: @@ -653,8 +1098,15 @@ def build_collectd(log, workspace, host, type_cache, target_cpu, packages_dir, return -1 return 0 - return download_and_build_collectd(log, workspace, host, type_cache, - target_cpu, packages_dir, collectd, None, + return download_and_build_collectd(log, + workspace, + host, + source_dir, + type_cache, + target_cpu, + packages_dir, + collectd, + None, extra_package_fnames) @@ -819,36 +1271,45 @@ def build_grafana_plugins(log, host, type_cache, iso_cache, extra_iso_fnames): return 0 -def build_barreleye(log, workspace, host, type_cache, target_cpu, iso_cache, +def build_barreleye(log, workspace, host, source_dir, + type_cache, target_cpu, iso_cache, packages_dir, collectd, extra_iso_fnames, - extra_package_fnames, extra_rpm_names): + extra_package_fnames, extra_package_names): """ Build barreleye """ - rc = build_collectd(log, workspace, host, type_cache, target_cpu, - packages_dir, collectd, extra_package_fnames) + rc = build_collectd(log, workspace, host, source_dir, type_cache, + target_cpu, packages_dir, collectd, + extra_package_fnames) if rc: log.cl_error("failed to build Collectd RPMs") return -1 - rc = build_grafana_plugins(log, host, type_cache, iso_cache, - extra_iso_fnames) - if rc: - log.cl_error("failed to download Grafana") - return -1 + distro = host.sh_distro(log) + if distro == ssh_host.DISTRO_UBUNTU2204: + log.cl_info("skip building server packages for Barreleye " + "since distro [%s] has no server support", + distro) + extra_package_names += barrele_constant.BARRELE_DOWNLOAD_DEPENDENT_DEBS + else: + rc = build_grafana_plugins(log, host, type_cache, iso_cache, + extra_iso_fnames) + if rc: + log.cl_error("failed to download Grafana") + return -1 - rc = build_grafana(log, host, target_cpu, packages_dir, extra_package_fnames) - if rc: - log.cl_error("failed to download Grafana") - return -1 + rc = build_grafana(log, host, target_cpu, packages_dir, extra_package_fnames) + if rc: + log.cl_error("failed to download Grafana") + return -1 - rc = build_influxdb(log, host, target_cpu, packages_dir, - extra_package_fnames) - if rc: - log.cl_error("failed to download Influxdb") - return -1 + rc = build_influxdb(log, host, target_cpu, packages_dir, + extra_package_fnames) + if rc: + log.cl_error("failed to download Influxdb") + return -1 - extra_rpm_names += barrele_constant.BARRELE_DOWNLOAD_DEPENDENT_RPMS + extra_package_names += barrele_constant.BARRELE_DOWNLOAD_DEPENDENT_RPMS return 0 @@ -863,28 +1324,32 @@ def __init__(self): is_devel=False, need_collectd=True) - def cpt_build_dependent_rpms(self, distro): + def cpt_build_dependent_packages(self, distro): """ Return the RPMs needed to install before building """ + # pylint: disable=no-self-use if distro == ssh_host.DISTRO_RHEL7: return COLLECTD_BUILD_DEPENDENT_RHEL7_RPMS if distro == ssh_host.DISTRO_RHEL8: return COLLECTD_BUILD_DEPENDENT_RHEL8_RPMS + if distro == ssh_host.DISTRO_UBUNTU2204: + return BARRELE_BUILD_DEPENDENT_UBUNTU2204_DEBS return None def cpt_build(self, log, workspace, local_host, source_dir, target_cpu, type_cache, iso_cache, packages_dir, extra_iso_fnames, - extra_package_fnames, extra_rpm_names, option_dict): + extra_package_fnames, extra_package_names, option_dict): """ Build the plugin """ # pylint: disable=unused-argument,no-self-use collectd = option_dict["collectd"] - ret = build_barreleye(log, workspace, local_host, type_cache, + ret = build_barreleye(log, workspace, local_host, source_dir, + type_cache, target_cpu, iso_cache, packages_dir, collectd, extra_iso_fnames, extra_package_fnames, - extra_rpm_names) + extra_package_names) if ret: log.cl_error("failed to build Barreleye") return -1 @@ -975,8 +1440,8 @@ def build(self, collectd=None): local_host.sh_hostname) cmd_general.cmd_exit(log, -1) - rc = build_collectd(log, workspace, local_host, type_cache, - target_cpu, packages_dir, collectd, + rc = build_collectd(log, workspace, local_host, source_dir, + type_cache, target_cpu, packages_dir, collectd, extra_package_fnames) if rc: log.cl_error("failed to build Collectd RPMs") diff --git a/pybuild/build_common.py b/pybuild/build_common.py index f862795..145e015 100644 --- a/pybuild/build_common.py +++ b/pybuild/build_common.py @@ -122,7 +122,7 @@ def __init__(self, plugin_name, else: self.cpt_plugins = plugins - def cpt_build_dependent_rpms(self, distro): + def cpt_build_dependent_packages(self, distro): """ Return the RPMs needed to install before building. Return None on failure. @@ -140,7 +140,7 @@ def cpt_install_build_dependency(self, log, workspace, host, def cpt_build(self, log, workspace, local_host, source_dir, target_cpu, type_cache, iso_cache, packages_dir, extra_iso_fnames, - extra_package_fnames, extra_rpm_names, option_dict): + extra_package_fnames, extra_package_names, option_dict): """ Build the plugin """ @@ -176,7 +176,7 @@ def __init__(self, package_name, depend_package_names=None): # useful when need to install other newly buildt packages. self.cpb_depend_package_names = depend_package_names - def cpb_build_dependent_rpms(self, distro): + def cpb_build_dependent_packages(self, distro): """ Return the RPMs needed to install before building """ @@ -430,3 +430,38 @@ def get_shared_build_cache(log, host, workspace, shared_cache): "than 10 minutes, aborting", lock_file) return ret + + +def apply_patches(log, host, target_source_dir, patch_dir): + """ + Apply a series of patches. The patches shall be in patch_dir. And they + shall have sorted file names like: + 0000-fname.patch + 0001-fname.patch + ... + """ + patch_fnames = host.sh_get_dir_fnames(log, patch_dir) + if patch_fnames is None: + log.cl_error("failed to get patches of Mpifileutils under dir [%s] " + "on host [%s]", + patch_dir, host.sh_hostname) + return -1 + + patch_fnames.sort() + + for patch_fname in patch_fnames: + if not patch_fname.endswith(".patch"): + continue + patch_fpath = patch_dir + "/" + patch_fname + command = "cd %s && patch -p1 < %s" % (target_source_dir, patch_fpath) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + return 0 diff --git a/pybuild/coral.py b/pybuild/coral.py index 9cf4532..30fa017 100644 --- a/pybuild/coral.py +++ b/pybuild/coral.py @@ -25,9 +25,9 @@ def coral_command_bootstrap(tsinghua_mirror=False): local_host.sh_hostname) sys.exit(-1) - missing_rpms, missing_pips = \ + missing_packages, missing_pips = \ install_common.command_missing_packages(distro) - if missing_rpms is None: + if missing_packages is None: log.cl_error("failed to get the missing packages of host [%s]", local_host.sh_hostname) sys.exit(-1) @@ -42,7 +42,8 @@ def coral_command_bootstrap(tsinghua_mirror=False): retval.cr_stderr) sys.exit(-1) - ret = install_common.bootstrap_from_internet(log, local_host, missing_rpms, + ret = install_common.bootstrap_from_internet(log, local_host, + missing_packages, missing_pips, constant.CORAL_BUILD_CACHE_PIP_DIR, tsinghua_mirror=tsinghua_mirror) diff --git a/pybuild/coral_build.py b/pybuild/coral_build.py index 48704d8..cd77904 100644 --- a/pybuild/coral_build.py +++ b/pybuild/coral_build.py @@ -25,6 +25,8 @@ PYINSTALLER_TARBALL_SHA1SUM = "60c595f5cbe66223d33c6edf1bb731ab9f02c3de" # "v4.10.tar.gz" is not a good name, specify the fname to save. PYINSTALLER_TABALL_FNAME = "pyinstaller-4.10.tar.gz" +REPLACE_DEB_DICT = {} +REPLACE_DEB_DICT["debconf-2.0"] = "debconf" def merge_list(list_x, list_y): @@ -212,7 +214,7 @@ def check_package_rpms(log, host, packages_dir, dependent_rpms, def download_dependent_rpms(log, host, distro, target_cpu, packages_dir, extra_package_fnames, - extra_rpm_names): + extra_package_names): """ Download dependent RPMs """ @@ -233,7 +235,7 @@ def download_dependent_rpms(log, host, distro, target_cpu, return -1 dependent_rpms = merge_list(constant.CORAL_DEPENDENT_RPMS, - extra_rpm_names) + extra_package_names) if distro == ssh_host.DISTRO_RHEL7: ret = download_dependent_rpms_rhel7(log, host, target_cpu, @@ -243,6 +245,9 @@ def download_dependent_rpms(log, host, distro, target_cpu, ret = download_dependent_rpms_rhel8(log, host, packages_dir, dependent_rpms, extra_package_fnames) + else: + log.cl_error("unsupported distro [%s]", distro) + return -1 if ret: log.cl_error("failed to download dependent RPMs on host [%s]", host.sh_hostname) @@ -257,6 +262,100 @@ def download_dependent_rpms(log, host, distro, target_cpu, return 0 +def download_dependent_debs(log, host, packages_dir, extra_package_fnames, + extra_package_names): + """ + Download dependent debs + """ + # pylint: disable=consider-using-get + if len(extra_package_names) == 0: + return 0 + log.cl_info("downloading dependency debs") + command = 'apt-rdepends' + for extra_package_name in extra_package_names: + command += " " + extra_package_name + command = command + ' | grep -v "^ "' + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + packages = retval.cr_stdout.splitlines() + + downloading_dir = packages_dir + "/downloading." + cmd_general.get_identity() + command = "mkdir %s" % downloading_dir + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + for package in packages: + if package in REPLACE_DEB_DICT: + package = REPLACE_DEB_DICT[package] + command = ("cd %s && apt download %s" % + (downloading_dir, package)) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + downloaded_fnames = host.sh_get_dir_fnames(log, downloading_dir) + if downloaded_fnames is None: + log.cl_error("failed to get the fnames under [%s]", + downloading_dir) + return -1 + extra_package_fnames += downloaded_fnames + + command = ("mv %s/* %s && rmdir %s" % + (downloading_dir, packages_dir, downloading_dir)) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + return 0 + + +def download_dependent_packages(log, host, distro, target_cpu, + packages_dir, extra_package_fnames, + extra_package_names): + """ + Download packages for Barreleye. + """ + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + return download_dependent_rpms(log, host, distro, target_cpu, + packages_dir, extra_package_fnames, + extra_package_names) + if distro in (ssh_host.DISTRO_UBUNTU2204): + return download_dependent_debs(log, host, + packages_dir, extra_package_fnames, + extra_package_names) + log.cl_error("unsupported distro [%s]", distro) + return -1 + + def install_pyinstaller(log, host, type_cache, tsinghua_mirror=False): """ Install pyinstaller @@ -299,9 +398,29 @@ def prepare_install_modulemd_tools(log, host): return ["modulemd-tools"] -def install_build_dependency(log, workspace, host, distro, target_cpu, - type_cache, plugins, package_dict, pip_dir, - tsinghua_mirror=False): +def get_build_dependent_packages(log, distro, plugins, package_dict): + """ + Return a list of dependent packages (RPMs/debs) and pips + """ + dependent_packages = [] + dependent_pips = [] + for plugin in plugins: + packages = plugin.cpt_build_dependent_packages(distro) + if packages is None: + log.cl_error("failed to get the dependet packages for building [%s]", + plugin.cpt_plugin_name) + return None, None + dependent_packages += packages + dependent_pips += plugin.cpt_build_dependent_pips + + for package in package_dict.values(): + dependent_packages += package.cpb_build_dependent_packages(distro) + return dependent_packages, dependent_pips + + +def install_build_dependency_rhel(log, workspace, host, distro, target_cpu, + type_cache, plugins, package_dict, pip_dir, + tsinghua_mirror=False): """ Install the dependency of building Coral """ @@ -366,20 +485,11 @@ def install_build_dependency(log, workspace, host, distro, target_cpu, retval.cr_stderr) return -1 - # We need all dependency for all plugins since no matter they - # are needed or not, the Python codes will be checked, and the Python - # codes might depend on the RPMs. - for plugin in build_common.CORAL_PLUGIN_DICT.values(): - rpms = plugin.cpt_build_dependent_rpms(distro) - if rpms is None: - log.cl_error("failed to get the dependet RPMs for building [%s]", - plugin.cpt_plugin_name) - return -1 - dependent_rpms += rpms - dependent_pips += plugin.cpt_build_dependent_pips - - for package in package_dict.values(): - dependent_rpms += package.cpb_build_dependent_rpms(distro) + rpms, pips = get_build_dependent_packages(log, distro, plugins, package_dict) + if rpms is None or pips is None: + return -1 + dependent_rpms += rpms + dependent_pips += pips ret = install_common.bootstrap_from_internet(log, host, dependent_rpms, dependent_pips, @@ -407,6 +517,74 @@ def install_build_dependency(log, workspace, host, distro, target_cpu, return 0 +def install_build_dependency_ubuntu(log, workspace, host, distro, + target_cpu, type_cache, plugins, + package_dict, pip_dir, + tsinghua_mirror=False): + """ + Install the dependency of building Coral for Ubuntu + """ + # pylint: disable=unused-argument,too-many-locals + if tsinghua_mirror: + ret = install_common.ubuntu2204_apt_mirror_replace_to_tsinghua(log, host) + if ret: + log.cl_error("failed to replace deb mirrors to Tsinghua University") + return -1 + + command = 'apt update' + log.cl_info("running command [%s] on host [%s]", + command, host.sh_hostname) + retval = host.sh_watched_run(log, command, None, None, + return_stdout=False, + return_stderr=False) + if retval.cr_exit_status != 0: + log.cl_error("failed to run command [%s] on host [%s]", + command, host.sh_hostname) + return -1 + + dependent_debs = ["libjson-c-dev", "apt-rdepends"] + dependent_pips = ["PyInstaller", "tinyaes", "pycryptodome"] + + debs, pips = get_build_dependent_packages(log, distro, plugins, + package_dict) + if debs is None or pips is None: + return -1 + dependent_pips += pips + dependent_debs += debs + + ret = install_common.bootstrap_from_internet(log, host, + dependent_debs, + dependent_pips, + pip_dir, + tsinghua_mirror=tsinghua_mirror) + if ret: + log.cl_error("failed to install missing packages on host [%s] " + "from Internet", host.sh_hostname) + return -1 + return 0 + + +def install_build_dependency(log, workspace, host, distro, target_cpu, + type_cache, plugins, package_dict, pip_dir, + tsinghua_mirror=False): + """ + Install the dependency of building Coral + """ + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + return install_build_dependency_rhel(log, workspace, host, distro, + target_cpu, type_cache, plugins, + package_dict, pip_dir, + tsinghua_mirror=tsinghua_mirror) + + if distro in (ssh_host.DISTRO_UBUNTU2204): + return install_build_dependency_ubuntu(log, workspace, host, distro, + target_cpu, type_cache, plugins, + package_dict, pip_dir, + tsinghua_mirror=tsinghua_mirror) + log.cl_error("unsupported distro [%s]", distro) + return -1 + + def sync_shared_build_cache(log, host, private_cache, shared_parent): """ Sync from the local cache to shared cache @@ -628,7 +806,7 @@ def get_needed_packages(log, plugins): def build_packages(log, workspace, local_host, source_dir, target_cpu, type_cache, iso_cache, packages_dir, extra_iso_fnames, - extra_package_fnames, extra_rpm_names, option_dict, + extra_package_fnames, extra_package_names, option_dict, package_dict): """ Build the packages in cpt_packages. @@ -650,7 +828,7 @@ def build_packages(log, workspace, local_host, source_dir, target_cpu, ret = package.cpb_build(log, workspace, local_host, source_dir, target_cpu, type_cache, iso_cache, packages_dir, extra_iso_fnames, - extra_package_fnames, extra_rpm_names, + extra_package_fnames, extra_package_names, option_dict) if ret: log.cl_error("failed to build package [%s]", @@ -754,7 +932,8 @@ def build(log, source_dir, workspace, local_host = ssh_host.get_local_host(ssh=False) distro = local_host.sh_distro(log) - if distro not in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + if distro not in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8, + ssh_host.DISTRO_UBUNTU2204): log.cl_error("build on distro [%s] is not supported yet", distro) return -1 @@ -762,7 +941,7 @@ def build(log, source_dir, workspace, # Shared cache for this build type shared_type_cache = shared_cache + "/" + type_fname # Extra RPMs to download - extra_rpm_names = [] + extra_package_names = [] # Extra RPM file names under package directory extra_package_fnames = [] # Extra file names under ISO directory @@ -851,7 +1030,7 @@ def build(log, source_dir, workspace, ret = build_packages(log, workspace, local_host, source_dir, target_cpu, type_cache, iso_cache, packages_dir, extra_iso_fnames, - extra_package_fnames, extra_rpm_names, option_dict, + extra_package_fnames, extra_package_names, option_dict, package_dict) if ret: log.cl_error("failed to build packages") @@ -861,18 +1040,19 @@ def build(log, source_dir, workspace, ret = plugin.cpt_build(log, workspace, local_host, source_dir, target_cpu, type_cache, iso_cache, packages_dir, extra_iso_fnames, - extra_package_fnames, extra_rpm_names, + extra_package_fnames, extra_package_names, option_dict) if ret: log.cl_error("failed to build plugin [%s]", plugin.cpt_plugin_name) return -1 - ret = download_dependent_rpms(log, local_host, distro, - target_cpu, packages_dir, - extra_package_fnames, extra_rpm_names) + ret = download_dependent_packages(log, local_host, distro, + target_cpu, packages_dir, + extra_package_fnames, + extra_package_names) if ret: - log.cl_error("failed to download dependent rpms") + log.cl_error("failed to download dependent packages") return -1 contents = ([constant.BUILD_PACKAGES] + diff --git a/pycheck/Makefile.am b/pycheck/Makefile.am index 061246d..2760b80 100644 --- a/pycheck/Makefile.am +++ b/pycheck/Makefile.am @@ -1,8 +1,23 @@ PYTHON_COMMANDS = \ - ../barrele \ ../coral -PYTHON_LIB_FILES = $(wildcard ../pybuild/*.py ../pybarrele/*.py ../pycoral/*.py) +PYTHON_LIB_FILES = $(wildcard ../pycoral/*.py) \ + ../pybuild/build_common.py \ + ../pybuild/build_constant.py \ + ../pybuild/build_doc.py \ + ../pybuild/build_release_info.py \ + ../pybuild/build_version.py \ + ../pybuild/coral_build.py \ + ../pybuild/coral_command.py \ + ../pybuild/coral.py \ + ../pybuild/__init__.py + +if ENABLE_BARRELE +PYTHON_COMMANDS += ../barrele +PYTHON_LIB_FILES += $(wildcard ../pybarrele/*.py) \ + ../pybuild/build_barrele.py +endif + PYTHON_FILES = $(PYTHON_LIB_FILES) $(PYTHON_COMMANDS) PYTHON_CHECKS = $(PYTHON_FILES:%=%.python_checked) diff --git a/pycoral/cmd_general.py b/pycoral/cmd_general.py index 1eb84ce..622c97d 100644 --- a/pycoral/cmd_general.py +++ b/pycoral/cmd_general.py @@ -115,7 +115,7 @@ def init_env_noconfig(logdir, log_to_file, logdir_is_default, if not isinstance(log_to_file, bool): print("ERROR: invalid debug option [%s], should be a bool type" % - (log_to_file), file=sys.stderr) + str(log_to_file), file=sys.stderr) sys.exit(1) if log_to_file: diff --git a/pycoral/install_common.py b/pycoral/install_common.py index db04fa2..99e8d3d 100644 --- a/pycoral/install_common.py +++ b/pycoral/install_common.py @@ -952,7 +952,7 @@ def cic_install(self, log, parallelism=10): def yum_install_rpm_from_internet(log, host, rpms, tsinghua_mirror=False): """ - Check whether a RPM installed or not. If not, use yum to install + Install RPMs by downloading it from Internet. """ # pylint: disable=too-many-branches if tsinghua_mirror: @@ -1101,15 +1101,113 @@ def yum_replace_to_tsinghua(log, host): return 0 -def bootstrap_from_internet(log, host, rpms, pip_packages, pip_dir, +def ubuntu2204_apt_mirror_replace_to_tsinghua(log, host): + """ + Replace apt mirror. + """ + # See https://mirror.tuna.tsinghua.edu.cn/help/ubuntu/ for more information. + source_list = "/etc/apt/sources.list" + command = "grep tsinghua %s" % source_list + retval = host.sh_run(log, command) + if retval.cr_exit_status == 0: + return 0 + + if retval.cr_exit_status != 1 or retval.cr_stdout != "": + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + command = "echo '# Configured by Coral' > %s" % source_list + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + + lines = ["deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy main restricted universe multiverse", + "deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse", + "deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse", + "deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse"] + for line in lines: + command = "sed -i '$a\%s' %s" % (line, source_list) + retval = host.sh_run(log, command) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + return 0 + + +def ubuntu2204_install_deb_from_internet(log, host, debs, tsinghua_mirror=False): + """ + Install debs by downloading it from Internet. + """ + # pylint: disable=too-many-branches + if tsinghua_mirror: + ret = ubuntu2204_apt_mirror_replace_to_tsinghua(log, host) + if ret: + log.cl_error("failed to replace apt mirror to Tsinghua " + "University on host [%s]", host.sh_hostname) + return -1 + + if len(debs) == 0: + return 0 + + command = "apt install -y" + for deb in debs: + command += " " + deb + + retval = host.sh_run(log, command, timeout=None) + if retval.cr_exit_status: + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 + return 0 + + +def bootstrap_from_internet(log, host, packages, pip_packages, pip_dir, tsinghua_mirror=False): """ Install the dependent RPMs and pip packages from Internet """ - ret = yum_install_rpm_from_internet(log, host, rpms, - tsinghua_mirror=tsinghua_mirror) + distro = host.sh_distro(log) + if distro is None: + log.cl_error("failed to get distro of host [%s]", + host.sh_hostname) + return -1 + + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + ret = yum_install_rpm_from_internet(log, host, packages, + tsinghua_mirror=tsinghua_mirror) + elif distro == ssh_host.DISTRO_UBUNTU2204: + ret = ubuntu2204_install_deb_from_internet(log, host, packages, + tsinghua_mirror=tsinghua_mirror) + else: + log.cl_error("unsupported distro [%s] of host [%s]", + distro, host.sh_hostname) + return -1 if ret: - log.cl_error("failed to install missing RPMs on host [%s]", + log.cl_error("failed to install missing packages on host [%s]", host.sh_hostname) return -1 @@ -1247,17 +1345,76 @@ def command_missing_packages_rhel7(): return missing_rpms, missing_pips +def command_missing_packages_ubuntu2204(): + """ + Add the missing debs and pip packages for RHEL7 + """ + # pylint: disable=unused-import,bad-option-value,import-outside-toplevel + # pylint: disable=unused-variable + missing_debs = [] + try: + import fire + except ImportError: + missing_debs.append("python3-fire") + + try: + import prettytable + except ImportError: + missing_debs.append("python3-prettytable") + + try: + import toml + except ImportError: + missing_debs.append("python3-toml") + + try: + import dateutil + except ImportError: + missing_debs.append("python3-dateutil") + + try: + import filelock + except ImportError: + missing_debs.append("python3-filelock") + + try: + import psutil + except ImportError: + missing_debs.append("python3-psutil") + + missing_pips = [] + return missing_debs, missing_pips + + def command_missing_packages(distro): """ - Add the missing RPMs and pip packages + Add the missing RPMs/debs and pip packages """ if distro == ssh_host.DISTRO_RHEL7: return command_missing_packages_rhel7() if distro == ssh_host.DISTRO_RHEL8: return command_missing_packages_rhel8() + if distro == ssh_host.DISTRO_UBUNTU2204: + return command_missing_packages_ubuntu2204() return None, None +def install_packages_from_internet(log, host, packages, tsinghua_mirror=False): + """ + Install packages from Internet. + """ + distro = host.sh_distro(log) + if distro in (ssh_host.DISTRO_RHEL7, ssh_host.DISTRO_RHEL8): + return yum_install_rpm_from_internet(log, host, packages, + tsinghua_mirror=tsinghua_mirror) + if distro == ssh_host.DISTRO_UBUNTU2204: + return ubuntu2204_install_deb_from_internet(log, host, packages, + tsinghua_mirror=tsinghua_mirror) + log.cl_error("unsupported distro [%s] of host [%s]", + distro, host.sh_hostname) + return -1 + + def download_pip3_packages(log, host, pip_dir, pip_packages, tsinghua_mirror=False): """ @@ -1271,10 +1428,11 @@ def download_pip3_packages(log, host, pip_dir, pip_packages, message = "" log.cl_info("downloading pip3 packages %s to dir [%s] on host [%s]%s", pip_packages, pip_dir, host.sh_hostname, message) - ret = yum_install_rpm_from_internet(log, host, ["python3-pip"], - tsinghua_mirror=tsinghua_mirror) + + ret = install_packages_from_internet(log, host, ["python3-pip"], + tsinghua_mirror=tsinghua_mirror) if ret: - log.cl_error("failed to install [python3-pip] RPM") + log.cl_error("failed to install [python3-pip] package") return -1 command = ("mkdir -p %s && cd %s && pip3 download" % (pip_dir, pip_dir)) diff --git a/pycoral/lustre_version.py b/pycoral/lustre_version.py index 9b7654f..c89c59f 100644 --- a/pycoral/lustre_version.py +++ b/pycoral/lustre_version.py @@ -302,3 +302,33 @@ def match_lustre_version_from_rpms(log, rpm_fnames, skip_kernel=False, definition_message, version_string) return None, None return matched_versions[0], matched_rpm_type_dicts[0] + +# See the commend of ES6_0_PATTERNS for the difference between es6.0 +# and es6.1. +DEB_ES6_0_PATTERN = r"^(2\.14\.[0]-ddn([0-9]|[1-7][0-9]|8[0-6])\D.+)$" +DEB_ES6_1_PATTERN = r"^(2\.14\.[0]-ddn.+)$" +DEB_2_15_PATTERN = r"^(2\.15\..+)$" + +def match_lustre_version_from_deb(log, deb_version): + """ + The version of deb package usually comes from command + apt list --installed | grep lustre-client-modules + or + apt show lustre-client-modules-5.15.0-69-generic | grep Version + Example: + 2.15.2-70-gb74560d-1 + """ + match = re.search(DEB_ES6_0_PATTERN, deb_version) + if match is not None: + return LUSTRE_VERSION_ES6_0 + + match = re.search(DEB_ES6_1_PATTERN, deb_version) + if match is not None: + return LUSTRE_VERSION_ES6_1 + + match = re.search(DEB_2_15_PATTERN, deb_version) + if match is not None: + return LUSTRE_VERSION_2_15 + + log.cl_error("unsupported Lustre version [%s]", deb_version) + return None diff --git a/pycoral/ssh_host.py b/pycoral/ssh_host.py index b10655a..796bba6 100644 --- a/pycoral/ssh_host.py +++ b/pycoral/ssh_host.py @@ -30,6 +30,8 @@ DISTRO_RHEL7 = "rhel7" # OS distribution RHEL8/CentOS8 DISTRO_RHEL8 = "rhel8" +# OS distribution ubuntu2204 +DISTRO_UBUNTU2204 = "ubuntu2204" # The shortest time that a reboot could finish. It is used to check whether # a host has actually rebooted or not. SHORTEST_TIME_REBOOT = 10 @@ -386,6 +388,8 @@ def sh_distro(self, log): log.cl_error("unsupported version [%s] of [%s] on host [%s]", version, "fc", self.sh_hostname) return None + if name == "Ubuntu": + return DISTRO_UBUNTU2204 log.cl_error("unsupported version [%s] of [%s] on host [%s]", version, name, self.sh_hostname) return None diff --git a/pycoral/utils.py b/pycoral/utils.py index b2e4c0d..95ffd80 100644 --- a/pycoral/utils.py +++ b/pycoral/utils.py @@ -363,6 +363,7 @@ def thread_start(target, args): """ Wrap the target function and start a thread to run it """ + # pylint: disable=deprecated-method run_thread = threading.Thread(target=target, args=args) run_thread.setDaemon(True) @@ -541,6 +542,7 @@ def lr_release(self, number, info_string): """ Release resource """ + # pylint: disable=deprecated-method self.lr_condition.acquire() self.lr_number += number self.lr_condition.notifyAll()