From 58f2c5288ba89fa8bac9cd945d1c9855b1817a0f Mon Sep 17 00:00:00 2001 From: Andy Lake Date: Wed, 20 Dec 2023 01:19:45 +0000 Subject: [PATCH] Adding a prometheus exporter for various perfsonar host metrics --- .../perfsonar-host-metrics/Makefile | 4 +- .../apache-perfsonar_host_exporter.conf | 16 ++ .../perfsonar-host-exporter.service | 12 ++ .../perfsonar_host_exporter | 149 ++++++++++++++++++ .../perfsonar_metrics.py | 16 -- .../unibuild-packaging/deb/control | 3 +- .../deb/perfsonar-host-metrics.install | 5 +- .../deb/perfsonar-host-metrics.postinst | 5 +- .../rpm/perfsonar-host-metrics.spec | 12 ++ 9 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 perfsonar-host-metrics/perfsonar-host-metrics/apache-perfsonar_host_exporter.conf create mode 100644 perfsonar-host-metrics/perfsonar-host-metrics/perfsonar-host-exporter.service create mode 100644 perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_host_exporter delete mode 100644 perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_metrics.py diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/Makefile b/perfsonar-host-metrics/perfsonar-host-metrics/Makefile index 8475a5d..f9ca2e9 100644 --- a/perfsonar-host-metrics/perfsonar-host-metrics/Makefile +++ b/perfsonar-host-metrics/perfsonar-host-metrics/Makefile @@ -15,4 +15,6 @@ endif mkdir -p ${PERFSONAR-ROOTPATH} mkdir -p ${HTTPD-CONFIGPATH} install -m 755 exporter_opts.sh ${PERFSONAR-ROOTPATH}/ - install -m 644 apache-node_exporter.conf ${HTTPD-CONFIGPATH}/ \ No newline at end of file + install -m 755 perfsonar_exporter ${PERFSONAR-ROOTPATH}/ + install -m 644 apache-node_exporter.conf ${HTTPD-CONFIGPATH}/ + install -m 644 apache-perfsonar_host_exporter.conf ${HTTPD-CONFIGPATH}/ \ No newline at end of file diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/apache-perfsonar_host_exporter.conf b/perfsonar-host-metrics/perfsonar-host-metrics/apache-perfsonar_host_exporter.conf new file mode 100644 index 0000000..c499b50 --- /dev/null +++ b/perfsonar-host-metrics/perfsonar-host-metrics/apache-perfsonar_host_exporter.conf @@ -0,0 +1,16 @@ + + ProxyRequests Off + + = 2.4> + Require all granted + + + Order deny,allow + Allow from all + + + + ProxyPass /perfsonar_exporter http://localhost:11284 status=+I + ProxyPassReverse /perfsonar_exporter http://localhost:11284 status=+I + ProxyPreserveHost On + diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar-host-exporter.service b/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar-host-exporter.service new file mode 100644 index 0000000..3b14580 --- /dev/null +++ b/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar-host-exporter.service @@ -0,0 +1,12 @@ +[Unit] +Description=perfSONAR Host Exporter +After=network.target + +[Service] +Type=simple +ExecStart=/usr/lib/perfsonar/host_metrics/perfsonar_host_exporter +Restart=always +RestartSec=30s + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_host_exporter b/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_host_exporter new file mode 100644 index 0000000..e04ecb5 --- /dev/null +++ b/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_host_exporter @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +import argparse +import os +import requests +import subprocess +from psconfig.utilities.metrics import PSConfigMetricCalculator +from psconfig.utilities.cli import PSCONFIG_CLI_AGENTS +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +class PSMetricsWebHandler(BaseHTTPRequestHandler): + LS_BASE_URL = "http://35.223.142.206:8090/lookup/records" + + def _read_one_liner(self, filename): + value = "" + if os.path.isfile(filename): + with open(filename, 'r') as file: + for line in file: + value = line.rstrip('\n') + if value: break + return value + + def pscheduler_metrics(self): + ## + # Exec pscheduler metrics command + ps_metric_output = "" + result = subprocess.run(["pscheduler", "metrics", "--format", "prometheus"], text=True, capture_output=True) + if result and result.returncode == 0 and result.stdout: + ps_metric_output = result.stdout + + return ps_metric_output + + def psconfig_metrics(self): + # Find agent + ps_metric_output = "" + for psconfig_agent in PSCONFIG_CLI_AGENTS: + #Try each agent, but don't exit if exception since agent may not be installed + try: + # Get metrics + agent_name = psconfig_agent["name"].lower() + psconfig_calculator = PSConfigMetricCalculator(agent_name) + psconfig_metrics = psconfig_calculator.run_metrics() + if psconfig_metrics.guid: + ps_metric_output += psconfig_metrics.to_prometheus() + # Get remotes + config = psconfig_agent["client_class"](url=psconfig_agent["config_file"]).get_config() + remote_fmt = 'perfsonar_psconfig_{}_remote{{url="{}",configure_archives="{}"}} 1\n' + if config.remotes(): + ps_metric_output += "# HELP perfsonar_psconfig_{0}_remote Information about pSConfig {0} agent remote configurations\n".format(agent_name) + ps_metric_output += "# TYPE perfsonar_psconfig_{0}_remote gauge\n".format(agent_name) + for remote in config.remotes(): + ps_metric_output += remote_fmt.format(agent_name, remote.url(), remote.configure_archives()) + # Get agent uuid + if agent_name == "pscheduler": + uuid_file = config.client_uuid_file() + if not uuid_file: + uuid_file = "/var/lib/perfsonar/psconfig/client_uuid" + uuid = self._read_one_liner(uuid_file) + if uuid: + ps_metric_output += "# HELP perfsonar_psconfig_{0}_agent Information about the pSConfig {0} agent\n".format(agent_name) + ps_metric_output += "# TYPE perfsonar_psconfig_{0}_agent gauge\n".format(agent_name) + ps_metric_output += 'perfsonar_psconfig_{}_agent{{client_uuid="{}"}} 1\n'.format(agent_name, uuid) + except: + pass + + return ps_metric_output + + def bundle_metrics(self): + ## + # Get bundle type and version + bundle_type = self._read_one_liner("/var/lib/perfsonar/bundles/bundle_type") + bundle_version = self._read_one_liner("/var/lib/perfsonar/bundles/bundle_version") + + ## + # Output prometheus if we at least have type + ps_metric_output = "" + if bundle_type: + ps_metric_output = "# HELP perfsonar_bundle Information about perfSONAR bundle installed\n" + ps_metric_output += "# TYPE perfsonar_bundle gauge\n" + ps_metric_output += 'perfsonar_bundle{{type="{}",version="{}"}} 1\n'.format(bundle_type, bundle_version) + + return ps_metric_output + + def _is_registered(self, uuid): + ls_url = "{}?type=host&client-uuid={}".format(self.LS_BASE_URL, uuid) + try: + r = requests.get(ls_url, timeout=3) + r.raise_for_status() + if r and r.json() and r.json()[0].get("client-uuid", None): + return 1 + except: + pass + + return 0 + + def lookup_svc_metrics(self): + ps_metric_output = "" + + ## + # Get client uuid + uuid = self._read_one_liner("/var/lib/perfsonar/lsregistrationdaemon/client_uuid") + if uuid: + ## + # Test if registered + ps_metric_output += "# HELP perfsonar_host_registered Indicates if perfSONAR host is registered in Lookup Service\n" + ps_metric_output += "# TYPE perfsonar_host_registered gauge\n" + ps_metric_output += 'perfsonar_host_registered{{uuid="{}"}} {}\n'.format(uuid, self._is_registered(uuid)) + + return ps_metric_output + + def do_GET(self): + ps_metric_output = "" + + ## + # toolkit/bundle metrics + ps_metric_output += self.bundle_metrics() + + ## + # pScheduler metrics + ps_metric_output += self.pscheduler_metrics() + + ## + # pSConfig Metrics + ps_metric_output += self.psconfig_metrics() + + ## + # LS client uuid + ps_metric_output += self.lookup_svc_metrics() + + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(ps_metric_output.encode("utf-8")) + + +if __name__ == "__main__": + ## + # Parse cli arguments + parser = argparse.ArgumentParser( + prog='perfsonar_exporter', + description='A web server that exports metrics about a perfSONAR host in Prometheus format' + ) + parser.add_argument('--host', dest='host', action='store', default='localhost', help='The host to listen for connections. 0.0.0.0 means all interfaces.') + parser.add_argument('--port', dest='port', action='store', type=int, default=11284, help='The port on which to listen for connections.') + args = parser.parse_args() + + #Build server + server = ThreadingHTTPServer((args.host, args.port), PSMetricsWebHandler) + server.serve_forever() \ No newline at end of file diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_metrics.py b/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_metrics.py deleted file mode 100644 index c87bf81..0000000 --- a/perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_metrics.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -from psconfig.utilities.metrics import PSConfigMetricCalculator -from psconfig.utilities.cli import PSCONFIG_CLI_AGENTS - -## -# pSConfig Metrics -for psconfig_agent in PSCONFIG_CLI_AGENTS: - #Try each agent, but don't exit if exception since agent may not be installed - try: - psconfig_calculator = PSConfigMetricCalculator(psconfig_agent["name"].lower()) - psconfig_metrics = psconfig_calculator.run_metrics() - if psconfig_metrics.guid: - print(psconfig_metrics.to_prometheus()) - except: - pass diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/control b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/control index 1e1482e..b49e594 100644 --- a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/control +++ b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/control @@ -9,7 +9,8 @@ Homepage: http://www.perfsonar.net Package: perfsonar-host-metrics Architecture: all Depends: ${shlibs:Depends}, ${misc:Depends}, prometheus-node-exporter, - openssl, apache2, apache2-ssl-dev, perfsonar-common, python-perfsonar-psconfig + openssl, apache2, apache2-ssl-dev, perfsonar-common, python-perfsonar-psconfig, + python-pscheduler Description: Prometheus node-exporter integration with perfSONAR A package that installs and sets-up Prometheus node_exporter for a perfSONAR to collect and report host metrics. diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.install b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.install index f2e28f8..e01cf0c 100644 --- a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.install +++ b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.install @@ -1,2 +1,5 @@ apache-node_exporter.conf /etc/apache2/conf-available/ -exporter_opts.sh /usr/lib/perfsonar/host_metrics/ \ No newline at end of file +apache-perfsonar_host_exporter.conf /etc/apache2/conf-available/ +exporter_opts.sh /usr/lib/perfsonar/host_metrics/ +perfsonar_host_exporter /usr/lib/perfsonar/host_metrics/ +perfsonar-host-exporter.service lib/systemd/system/ \ No newline at end of file diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.postinst b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.postinst index 9085a71..f6025bc 100644 --- a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.postinst +++ b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/deb/perfsonar-host-metrics.postinst @@ -13,10 +13,12 @@ case "$1" in #set node_exporter opts bash /usr/lib/perfsonar/host_metrics/exporter_opts.sh - #enable node_exporter - ignore errors for docker build env + #enable exporters - ignore errors for docker build env systemctl daemon-reload || : systemctl enable node_exporter.service || : systemctl restart node_exporter.service || : + systemctl enable perfsonar-host-exporter.service || : + systemctl restart perfsonar-host-exporter.service || : # Apache setup if [ -e /usr/share/apache2/apache2-maintscript-helper ]; then @@ -27,6 +29,7 @@ case "$1" in apache2_invoke enmod proxy apache2_invoke enmod proxy_http apache2_invoke enconf apache-node_exporter + apache2_invoke enconf apache-perfsonar_host_exporter fi fi ;; diff --git a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/rpm/perfsonar-host-metrics.spec b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/rpm/perfsonar-host-metrics.spec index ae95014..851dd08 100644 --- a/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/rpm/perfsonar-host-metrics.spec +++ b/perfsonar-host-metrics/perfsonar-host-metrics/unibuild-packaging/rpm/perfsonar-host-metrics.spec @@ -26,6 +26,8 @@ Requires: perfsonar-common Requires: openssl Requires: prometheus-node-exporter Requires: python-perfsonar-psconfig +Requires: python-pscheduler +Requires: python-requests Requires: httpd Requires: mod_ssl Requires: selinux-policy-%{selinuxtype} @@ -43,6 +45,8 @@ A package that installs and sets-up Prometheus node_exporter for a perfSONAR ins %install make PERFSONAR-ROOTPATH=%{buildroot}/%{pkg_install_base} HTTPD-CONFIGPATH=%{buildroot}/%{httpd_config_base} install +mkdir -p %{buildroot}/%{_unitdir}/ +install -m 644 *.service %{buildroot}/%{_unitdir}/ %clean rm -rf %{buildroot} @@ -50,6 +54,7 @@ rm -rf %{buildroot} %post #Restart/enable opensearch and logstash %systemd_post node_exporter.service +%systemd_post perfsonar-host-exporter.service if [ "$1" = "1" ]; then #set SELinux booleans to allow httpd proxy to work %selinux_set_booleans -s %{selinuxtype} %{selinuxbooleans} @@ -61,6 +66,8 @@ if [ "$1" = "1" ]; then systemctl daemon-reload systemctl enable node_exporter.service systemctl restart node_exporter.service + systemctl enable perfsonar-host-exporter.service + systemctl restart perfsonar-host-exporter.service #Enable and restart apache for reverse proxy systemctl enable httpd systemctl restart httpd @@ -68,9 +75,11 @@ fi %preun %systemd_preun node_exporter.service +%systemd_preun perfsonar-host-exporter.service %postun %systemd_postun_with_restart node_exporter.service +%systemd_postun_with_restart perfsonar-host-exporter.service if [ $1 -eq 0 ]; then %selinux_unset_booleans -s %{selinuxtype} %{selinuxbooleans} fi @@ -79,7 +88,10 @@ fi %defattr(0644,perfsonar,perfsonar,0755) %license LICENSE %attr(0755, perfsonar, perfsonar) %{pkg_install_base}/exporter_opts.sh +%attr(0755, perfsonar, perfsonar) %{pkg_install_base}/perfsonar_host_exporter %attr(0644, perfsonar, perfsonar) %{httpd_config_base}/apache-node_exporter.conf +%attr(0644, perfsonar, perfsonar) %{httpd_config_base}/apache-perfsonar_host_exporter.conf +%{_unitdir}/perfsonar-host-exporter.service %changelog * Tue Oct 24 2023 andy@es.net 5.0.5-0.0.a1