Skip to content

Commit

Permalink
Adding a prometheus exporter for various perfsonar host metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
arlake228 committed Dec 20, 2023
1 parent 441073e commit 58f2c52
Show file tree
Hide file tree
Showing 9 changed files with 202 additions and 20 deletions.
4 changes: 3 additions & 1 deletion perfsonar-host-metrics/perfsonar-host-metrics/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,6 @@ endif
mkdir -p ${PERFSONAR-ROOTPATH}
mkdir -p ${HTTPD-CONFIGPATH}
install -m 755 exporter_opts.sh ${PERFSONAR-ROOTPATH}/
install -m 644 apache-node_exporter.conf ${HTTPD-CONFIGPATH}/
install -m 755 perfsonar_exporter ${PERFSONAR-ROOTPATH}/
install -m 644 apache-node_exporter.conf ${HTTPD-CONFIGPATH}/
install -m 644 apache-perfsonar_host_exporter.conf ${HTTPD-CONFIGPATH}/
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<IfModule proxy_module>
ProxyRequests Off
<Proxy *>
<IfVersion >= 2.4>
Require all granted
</IfVersion>
<IfVersion < 2.4>
Order deny,allow
Allow from all
</IfVersion>
</Proxy>

ProxyPass /perfsonar_exporter http://localhost:11284 status=+I
ProxyPassReverse /perfsonar_exporter http://localhost:11284 status=+I
ProxyPreserveHost On
</IfModule>
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[Unit]
Description=perfSONAR Host Exporter
After=network.target

[Service]
Type=simple
ExecStart=/usr/lib/perfsonar/host_metrics/perfsonar_host_exporter
Restart=always
RestartSec=30s

[Install]
WantedBy=multi-user.target
149 changes: 149 additions & 0 deletions perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_host_exporter
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#!/usr/bin/env python3

import argparse
import os
import requests
import subprocess
from psconfig.utilities.metrics import PSConfigMetricCalculator
from psconfig.utilities.cli import PSCONFIG_CLI_AGENTS
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

class PSMetricsWebHandler(BaseHTTPRequestHandler):
LS_BASE_URL = "http://35.223.142.206:8090/lookup/records"

def _read_one_liner(self, filename):
value = ""
if os.path.isfile(filename):
with open(filename, 'r') as file:
for line in file:
value = line.rstrip('\n')
if value: break
return value

def pscheduler_metrics(self):
##
# Exec pscheduler metrics command
ps_metric_output = ""
result = subprocess.run(["pscheduler", "metrics", "--format", "prometheus"], text=True, capture_output=True)
if result and result.returncode == 0 and result.stdout:
ps_metric_output = result.stdout

return ps_metric_output

def psconfig_metrics(self):
# Find agent
ps_metric_output = ""
for psconfig_agent in PSCONFIG_CLI_AGENTS:
#Try each agent, but don't exit if exception since agent may not be installed
try:
# Get metrics
agent_name = psconfig_agent["name"].lower()
psconfig_calculator = PSConfigMetricCalculator(agent_name)
psconfig_metrics = psconfig_calculator.run_metrics()
if psconfig_metrics.guid:
ps_metric_output += psconfig_metrics.to_prometheus()
# Get remotes
config = psconfig_agent["client_class"](url=psconfig_agent["config_file"]).get_config()
remote_fmt = 'perfsonar_psconfig_{}_remote{{url="{}",configure_archives="{}"}} 1\n'
if config.remotes():
ps_metric_output += "# HELP perfsonar_psconfig_{0}_remote Information about pSConfig {0} agent remote configurations\n".format(agent_name)
ps_metric_output += "# TYPE perfsonar_psconfig_{0}_remote gauge\n".format(agent_name)
for remote in config.remotes():
ps_metric_output += remote_fmt.format(agent_name, remote.url(), remote.configure_archives())
# Get agent uuid
if agent_name == "pscheduler":
uuid_file = config.client_uuid_file()
if not uuid_file:
uuid_file = "/var/lib/perfsonar/psconfig/client_uuid"
uuid = self._read_one_liner(uuid_file)
if uuid:
ps_metric_output += "# HELP perfsonar_psconfig_{0}_agent Information about the pSConfig {0} agent\n".format(agent_name)
ps_metric_output += "# TYPE perfsonar_psconfig_{0}_agent gauge\n".format(agent_name)
ps_metric_output += 'perfsonar_psconfig_{}_agent{{client_uuid="{}"}} 1\n'.format(agent_name, uuid)
except:
pass

return ps_metric_output

def bundle_metrics(self):
##
# Get bundle type and version
bundle_type = self._read_one_liner("/var/lib/perfsonar/bundles/bundle_type")
bundle_version = self._read_one_liner("/var/lib/perfsonar/bundles/bundle_version")

##
# Output prometheus if we at least have type
ps_metric_output = ""
if bundle_type:
ps_metric_output = "# HELP perfsonar_bundle Information about perfSONAR bundle installed\n"
ps_metric_output += "# TYPE perfsonar_bundle gauge\n"
ps_metric_output += 'perfsonar_bundle{{type="{}",version="{}"}} 1\n'.format(bundle_type, bundle_version)

return ps_metric_output

def _is_registered(self, uuid):
ls_url = "{}?type=host&client-uuid={}".format(self.LS_BASE_URL, uuid)
try:
r = requests.get(ls_url, timeout=3)
r.raise_for_status()
if r and r.json() and r.json()[0].get("client-uuid", None):
return 1
except:
pass

return 0

def lookup_svc_metrics(self):
ps_metric_output = ""

##
# Get client uuid
uuid = self._read_one_liner("/var/lib/perfsonar/lsregistrationdaemon/client_uuid")
if uuid:
##
# Test if registered
ps_metric_output += "# HELP perfsonar_host_registered Indicates if perfSONAR host is registered in Lookup Service\n"
ps_metric_output += "# TYPE perfsonar_host_registered gauge\n"
ps_metric_output += 'perfsonar_host_registered{{uuid="{}"}} {}\n'.format(uuid, self._is_registered(uuid))

return ps_metric_output

def do_GET(self):
ps_metric_output = ""

##
# toolkit/bundle metrics
ps_metric_output += self.bundle_metrics()

##
# pScheduler metrics
ps_metric_output += self.pscheduler_metrics()

##
# pSConfig Metrics
ps_metric_output += self.psconfig_metrics()

##
# LS client uuid
ps_metric_output += self.lookup_svc_metrics()

self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.end_headers()
self.wfile.write(ps_metric_output.encode("utf-8"))


if __name__ == "__main__":
##
# Parse cli arguments
parser = argparse.ArgumentParser(
prog='perfsonar_exporter',
description='A web server that exports metrics about a perfSONAR host in Prometheus format'
)
parser.add_argument('--host', dest='host', action='store', default='localhost', help='The host to listen for connections. 0.0.0.0 means all interfaces.')
parser.add_argument('--port', dest='port', action='store', type=int, default=11284, help='The port on which to listen for connections.')
args = parser.parse_args()

#Build server
server = ThreadingHTTPServer((args.host, args.port), PSMetricsWebHandler)
server.serve_forever()
16 changes: 0 additions & 16 deletions perfsonar-host-metrics/perfsonar-host-metrics/perfsonar_metrics.py

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ Homepage: http://www.perfsonar.net
Package: perfsonar-host-metrics
Architecture: all
Depends: ${shlibs:Depends}, ${misc:Depends}, prometheus-node-exporter,
openssl, apache2, apache2-ssl-dev, perfsonar-common, python-perfsonar-psconfig
openssl, apache2, apache2-ssl-dev, perfsonar-common, python-perfsonar-psconfig,
python-pscheduler
Description: Prometheus node-exporter integration with perfSONAR
A package that installs and sets-up Prometheus node_exporter for a perfSONAR
to collect and report host metrics.
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
apache-node_exporter.conf /etc/apache2/conf-available/
exporter_opts.sh /usr/lib/perfsonar/host_metrics/
apache-perfsonar_host_exporter.conf /etc/apache2/conf-available/
exporter_opts.sh /usr/lib/perfsonar/host_metrics/
perfsonar_host_exporter /usr/lib/perfsonar/host_metrics/
perfsonar-host-exporter.service lib/systemd/system/
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@ case "$1" in
#set node_exporter opts
bash /usr/lib/perfsonar/host_metrics/exporter_opts.sh

#enable node_exporter - ignore errors for docker build env
#enable exporters - ignore errors for docker build env
systemctl daemon-reload || :
systemctl enable node_exporter.service || :
systemctl restart node_exporter.service || :
systemctl enable perfsonar-host-exporter.service || :
systemctl restart perfsonar-host-exporter.service || :

# Apache setup
if [ -e /usr/share/apache2/apache2-maintscript-helper ]; then
Expand All @@ -27,6 +29,7 @@ case "$1" in
apache2_invoke enmod proxy
apache2_invoke enmod proxy_http
apache2_invoke enconf apache-node_exporter
apache2_invoke enconf apache-perfsonar_host_exporter
fi
fi
;;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ Requires: perfsonar-common
Requires: openssl
Requires: prometheus-node-exporter
Requires: python-perfsonar-psconfig
Requires: python-pscheduler
Requires: python-requests
Requires: httpd
Requires: mod_ssl
Requires: selinux-policy-%{selinuxtype}
Expand All @@ -43,13 +45,16 @@ A package that installs and sets-up Prometheus node_exporter for a perfSONAR ins

%install
make PERFSONAR-ROOTPATH=%{buildroot}/%{pkg_install_base} HTTPD-CONFIGPATH=%{buildroot}/%{httpd_config_base} install
mkdir -p %{buildroot}/%{_unitdir}/
install -m 644 *.service %{buildroot}/%{_unitdir}/

%clean
rm -rf %{buildroot}

%post
#Restart/enable opensearch and logstash
%systemd_post node_exporter.service
%systemd_post perfsonar-host-exporter.service
if [ "$1" = "1" ]; then
#set SELinux booleans to allow httpd proxy to work
%selinux_set_booleans -s %{selinuxtype} %{selinuxbooleans}
Expand All @@ -61,16 +66,20 @@ if [ "$1" = "1" ]; then
systemctl daemon-reload
systemctl enable node_exporter.service
systemctl restart node_exporter.service
systemctl enable perfsonar-host-exporter.service
systemctl restart perfsonar-host-exporter.service
#Enable and restart apache for reverse proxy
systemctl enable httpd
systemctl restart httpd
fi

%preun
%systemd_preun node_exporter.service
%systemd_preun perfsonar-host-exporter.service

%postun
%systemd_postun_with_restart node_exporter.service
%systemd_postun_with_restart perfsonar-host-exporter.service
if [ $1 -eq 0 ]; then
%selinux_unset_booleans -s %{selinuxtype} %{selinuxbooleans}
fi
Expand All @@ -79,7 +88,10 @@ fi
%defattr(0644,perfsonar,perfsonar,0755)
%license LICENSE
%attr(0755, perfsonar, perfsonar) %{pkg_install_base}/exporter_opts.sh
%attr(0755, perfsonar, perfsonar) %{pkg_install_base}/perfsonar_host_exporter
%attr(0644, perfsonar, perfsonar) %{httpd_config_base}/apache-node_exporter.conf
%attr(0644, perfsonar, perfsonar) %{httpd_config_base}/apache-perfsonar_host_exporter.conf
%{_unitdir}/perfsonar-host-exporter.service

%changelog
* Tue Oct 24 2023 [email protected] 5.0.5-0.0.a1
Expand Down

0 comments on commit 58f2c52

Please sign in to comment.