From 222a2161bda82af6d3de12cadcbc534ee9ef0294 Mon Sep 17 00:00:00 2001 From: Nathan Scott Date: Tue, 31 Oct 2023 16:56:12 +1100 Subject: [PATCH] pcp-geolocate: setup standard labels for geolocation (optional) Provides a utility and systemd service to automate translation of IP address based latitude and longitude lookup, for storing as metric labels. Handy for building map-based tools. --- build/rpm/pcp.spec.in | 32 ++++++-- build/rpm/redhat.spec | 34 +++++++-- qa/1803 | 37 +++++++++ qa/1803.out | 5 ++ qa/group | 1 + src/pcp/GNUmakefile | 9 ++- src/pcp/geolocate/.gitignore | 2 + src/pcp/geolocate/GNUmakefile | 52 +++++++++++++ src/pcp/geolocate/pcp-geolocate.1 | 68 +++++++++++++++++ src/pcp/geolocate/pcp-geolocate.py | 89 ++++++++++++++++++++++ src/pcp/geolocate/pcp-geolocate.service.in | 15 ++++ 11 files changed, 328 insertions(+), 16 deletions(-) create mode 100755 qa/1803 create mode 100644 qa/1803.out create mode 100644 src/pcp/geolocate/.gitignore create mode 100644 src/pcp/geolocate/GNUmakefile create mode 100644 src/pcp/geolocate/pcp-geolocate.1 create mode 100755 src/pcp/geolocate/pcp-geolocate.py create mode 100644 src/pcp/geolocate/pcp-geolocate.service.in diff --git a/build/rpm/pcp.spec.in b/build/rpm/pcp.spec.in index 80280c2c27..793dc56bfe 100755 --- a/build/rpm/pcp.spec.in +++ b/build/rpm/pcp.spec.in @@ -598,6 +598,25 @@ See https://www.elastic.co/community for further details. %endif %if "@have_python@" == "true" +# +# pcp-geolocate +# +%package geolocate +License: GPL-2.0-or-later +Summary: Performance Co-Pilot geographical location metric labels +URL: https://pcp.io +Requires: pcp-libs >= %{version}-%{release} +%if "@enable_python3@" == "true" +Requires: python3-pcp = @package_version@ +%else +Requires: %{__python2}-pcp = @package_version@ +%endif + +%description geolocate +Performance Co-Pilot (PCP) tools that automatically apply metric labels +containing latitude and longitude, based on IP-address-based lookups. +Used with live maps to show metric values from different locations. + # # pcp-export-pcp2graphite # @@ -2280,11 +2299,12 @@ basic_manifest | keep "$PCP_GUI|pcp-gui|applications|pixmaps|hicolor" | cull 'pm basic_manifest | keep 'selinux' | cull 'tmp|testsuite' >pcp-selinux-files basic_manifest | keep 'zeroconf|daily[-_]report|/sa$' >pcp-zeroconf-files basic_manifest | grep -E -e 'pmiostat|pmrep|dstat|htop|pcp2csv' \ - -e 'pcp-atop|pcp-dmcache|pcp-dstat|pcp-free|pcp-htop' \ - -e 'pcp-ipcs|pcp-iostat|pcp-lvmcache|pcp-mpstat' \ - -e 'pcp-numastat|pcp-pidstat|pcp-shping|pcp-tapestat' \ - -e 'pcp-uptime|pcp-verify|pcp-ss|pcp-ps|pcp-meminfo' | \ + -e 'pcp-atop|pcp-dmcache|pcp-dstat|pcp-free' \ + -e 'pcp-htop|pcp-ipcs|pcp-iostat|pcp-lvmcache|pcp-mpstat' \ + -e 'pcp-numastat|pcp-pidstat|pcp-shping|pcp-ss' \ + -e 'pcp-tapestat|pcp-uptime|pcp-verify' | \ cull 'selinux|pmlogconf|pmieconf|pmrepconf' >pcp-system-tools-files +basic_manifest | keep 'geolocate' >pcp-geolocate-files basic_manifest | keep 'sar2pcp' >pcp-import-sar2pcp-files basic_manifest | keep 'iostat2pcp' >pcp-import-iostat2pcp-files basic_manifest | keep 'sheet2pcp' >pcp-import-sheet2pcp-files @@ -2417,7 +2437,7 @@ done for subpackage in \ pcp-conf pcp-gui pcp-doc pcp-libs pcp-devel pcp-libs-devel \ - pcp-selinux pcp-system-tools pcp-testsuite pcp-zeroconf \ + pcp-geolocate pcp-selinux pcp-system-tools pcp-testsuite pcp-zeroconf \ $pmda_packages $import_packages $export_packages ; \ do \ echo $subpackage >> packages.list; \ @@ -3088,6 +3108,8 @@ fi %endif %if "@have_python@" == "true" +%files geolocate -f pcp-geolocate-files.rpm + %files pmda-gluster -f pcp-pmda-gluster-files.rpm %files pmda-zswap -f pcp-pmda-zswap-files.rpm diff --git a/build/rpm/redhat.spec b/build/rpm/redhat.spec index ab6666378f..8544d7f0f9 100644 --- a/build/rpm/redhat.spec +++ b/build/rpm/redhat.spec @@ -738,6 +738,25 @@ Performance Co-Pilot (PCP) module for exporting metrics from PCP to Zabbix via the Zabbix agent - see zbxpcp(3) for further details. %if !%{disable_python2} || !%{disable_python3} +# +# pcp-geolocate +# +%package geolocate +License: GPL-2.0-or-later +Summary: Performance Co-Pilot geographical location metric labels +URL: https://pcp.io +Requires: pcp-libs >= %{version}-%{release} +%if "@enable_python3@" == "true" +Requires: python3-pcp = @package_version@ +%else +Requires: %{__python2}-pcp = @package_version@ +%endif + +%description geolocate +Performance Co-Pilot (PCP) tools that automatically apply metric labels +containing latitude and longitude, based on IP-address-based lookups. +Used with live maps to show metric values from different locations. + # # pcp-export-pcp2elasticsearch # @@ -2438,13 +2457,12 @@ basic_manifest | keep "$PCP_GUI|pcp-gui|applications|pixmaps|hicolor" | cull 'pm basic_manifest | keep 'selinux' | cull 'tmp|testsuite' >pcp-selinux-files basic_manifest | keep 'zeroconf|daily[-_]report|/sa$' >pcp-zeroconf-files basic_manifest | grep -E -e 'pmiostat|pmrep|dstat|htop|pcp2csv' \ - -e 'pcp-atop|pcp-dmcache|pcp-dstat|pcp-free|pcp-htop' \ - -e 'pcp-ipcs|pcp-iostat|pcp-lvmcache|pcp-mpstat|pcp-netstat' \ - -e 'pcp-buddyinfo|pcp-meminfo|pcp-slabinfo|pcp-zoneinfo' \ - -e 'pcp-numastat|pcp-pidstat|pcp-shping|pcp-tapestat' \ - -e 'pcp-uptime|pcp-verify|pcp-ss|pcp-ps' | \ + -e 'pcp-atop|pcp-dmcache|pcp-dstat|pcp-free' \ + -e 'pcp-htop|pcp-ipcs|pcp-iostat|pcp-lvmcache|pcp-mpstat' \ + -e 'pcp-numastat|pcp-pidstat|pcp-shping|pcp-ss' \ + -e 'pcp-tapestat|pcp-uptime|pcp-verify' | \ cull 'selinux|pmlogconf|pmieconf|pmrepconf' >pcp-system-tools-files - +basic_manifest | keep 'geolocate' >pcp-geolocate-files basic_manifest | keep 'sar2pcp' >pcp-import-sar2pcp-files basic_manifest | keep 'iostat2pcp' >pcp-import-iostat2pcp-files basic_manifest | keep 'sheet2pcp' >pcp-import-sheet2pcp-files @@ -2578,7 +2596,7 @@ done for subpackage in \ pcp-conf pcp-gui pcp-doc pcp-libs pcp-devel pcp-libs-devel \ - pcp-selinux pcp-system-tools pcp-testsuite pcp-zeroconf \ + pcp-geolocate pcp-selinux pcp-system-tools pcp-testsuite pcp-zeroconf \ $pmda_packages $import_packages $export_packages ; \ do \ echo $subpackage >> packages.list; \ @@ -3221,6 +3239,8 @@ fi %endif %if !%{disable_python2} || !%{disable_python3} +%files geolocate -f pcp-geolocate-files.rpm + %files pmda-gluster -f pcp-pmda-gluster-files.rpm %files pmda-zswap -f pcp-pmda-zswap-files.rpm diff --git a/qa/1803 b/qa/1803 new file mode 100755 index 0000000000..c4f4691b3d --- /dev/null +++ b/qa/1803 @@ -0,0 +1,37 @@ +#!/bin/sh +# PCP QA Test No. 1803 +# Exercise pcp-geolocate(1) metric labels. +# +# Copyright (c) 2023 Red Hat. All Rights Reserved. +# + +seq=`basename $0` +echo "QA output created by $seq" + +. ./common.python + +$sudo rm -rf $tmp $tmp.* $seq.full + +_cleanup() +{ + cd $here + $sudo rm -rf $tmp $tmp.* +} + +status=0 # success is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_filter() +{ + sed \ + -e 's/"latitude": .*,$/LATITUDE/g' \ + -e 's/"longitude": .*$/LONGITUDE/g' \ + # end +} + +# real QA test starts here +$PCP_BINADM_DIR/pcp-geolocate $tmp.geo +pmjson < $tmp.geo | _filter + +# success, all done +exit diff --git a/qa/1803.out b/qa/1803.out new file mode 100644 index 0000000000..6b98a9f1e5 --- /dev/null +++ b/qa/1803.out @@ -0,0 +1,5 @@ +QA output created by 1803 +{ + LATITUDE, + LONGITUDE +} diff --git a/qa/group b/qa/group index be1042c6ee..a6ce70eda7 100644 --- a/qa/group +++ b/qa/group @@ -2041,6 +2041,7 @@ x11 1793 pmrep pcp2xxx python local pmlogdump 1795 pmda.bpf local 1801 dstat python pcp local +1803 python geolocate labels local 1805 pmda.linux kernel local 1810 pmda.bpf local 1813 python labels local diff --git a/src/pcp/GNUmakefile b/src/pcp/GNUmakefile index a5db40c169..ff93a347bc 100644 --- a/src/pcp/GNUmakefile +++ b/src/pcp/GNUmakefile @@ -1,17 +1,17 @@ # -# Copyright (c) 2014-2020 Red Hat. +# Copyright (c) 2014-2022 Red Hat. # Copyright (c) 2000,2004 Silicon Graphics, Inc. All Rights Reserved. -# +# # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2 of the License, or (at your # option) any later version. -# +# # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License # for more details. -# +# TOPDIR = ../.. include $(TOPDIR)/src/include/builddefs @@ -22,6 +22,7 @@ SUBDIRS = \ dmcache \ dstat \ free \ + geolocate \ htop \ iostat \ ipcs \ diff --git a/src/pcp/geolocate/.gitignore b/src/pcp/geolocate/.gitignore new file mode 100644 index 0000000000..0ee788cc41 --- /dev/null +++ b/src/pcp/geolocate/.gitignore @@ -0,0 +1,2 @@ +pcp-geolocate.1.gz +pcp-geolocate.service diff --git a/src/pcp/geolocate/GNUmakefile b/src/pcp/geolocate/GNUmakefile new file mode 100644 index 0000000000..a659002a48 --- /dev/null +++ b/src/pcp/geolocate/GNUmakefile @@ -0,0 +1,52 @@ +# +# Copyright (c) 2022-2023 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# + +TOPDIR = ../../.. +include $(TOPDIR)/src/include/builddefs + +TARGET = pcp-geolocate +SCRIPT = $(TARGET).py +MAN_SECTION = 1 +MAN_PAGES = $(TARGET).$(MAN_SECTION) +MAN_DEST = $(PCP_MAN_DIR)/man$(MAN_SECTION) +LDIRT = $(TARGET).service + +default: $(SCRIPT) $(MAN_PAGES) $(TARGET).service + +include $(BUILDRULES) + +install: default +ifeq "$(HAVE_PYTHON)" "true" + $(INSTALL) -m 755 $(SCRIPT) $(PCP_BINADM_DIR)/$(TARGET) +ifeq ($(ENABLE_SYSTEMD),true) + $(INSTALL) -m 644 $(TARGET).service $(PCP_SYSTEMDUNIT_DIR)/$(TARGET).service +endif + @$(INSTALL_MAN) +endif + +default_pcp : default + +install_pcp : install + +$(TARGET).service: $(TARGET).service.in + $(SED) <$< >$@ \ + -e 's;@PCP_BINADM_DIR@;'$(PCP_BINADM_DIR)';' \ + -e 's;@PCP_SYSCONF_DIR@;'$(PCP_SYSCONF_DIR)';' \ + # END + +check :: $(SCRIPT) + $(PYLINT) $^ + +check :: $(MAN_PAGES) + $(MANLINT) $^ diff --git a/src/pcp/geolocate/pcp-geolocate.1 b/src/pcp/geolocate/pcp-geolocate.1 new file mode 100644 index 0000000000..4e6e7a1855 --- /dev/null +++ b/src/pcp/geolocate/pcp-geolocate.1 @@ -0,0 +1,68 @@ +'\"macro stdmacro +.\" +.\" Copyright (c) 2022-2023 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PCP-GEOLOCATE 1 "PCP" "Performance Co-Pilot" +.SH NAME +\f3pcp-geolocate\f1 \- discover collector system geographical labels +.SH SYNOPSIS +\f3pcp\f1 \f3geolocate\f1 +.SH DESCRIPTION +.B pcp-geolocate +reports the latitude and longitude for the local Performance Co-Pilot +collector host in JSON format. +This geolocation information is sourced from the cache file +.I $PCP_SYSCONF_DIR/labels/optional/geolocate +if it exists, else an attempt is made to perform geolocation based +on the host IP address, via several online sources (REST APIs). +.PP +The output from this command is suited for storing as metric labels by +saving it to the cache file mentioned above. +.PP +The opt-in +.BR systemd (1) +service unit file for this command provides an automated location +discovery for PCP metric labels. +.SH METRIC LABELS +The JSON output is the preferred format for storing geographical location +as PCP metric labels. +Every PCP metric available from this host will be tagged with the labels +for latitude and longitude, enabling tools that make use of this such as +the Grafana geomap panel. +.PP +For more information about metric labels refer to the +.BR pmcd (1), +.BR pminfo (1), +.BR pmlogger (1) +and +.BR pmLookupLabels (3) +manual entries. +.SH PCP ENVIRONMENT +Environment variables with the prefix \fBPCP_\fP are used to parameterize +the file and directory names used by PCP. +On each installation, the +file \fI/etc/pcp.conf\fP contains the local values for these variables. +The \fB$PCP_CONF\fP variable may be used to specify an alternative +configuration file, as described in \fBpcp.conf\fP(5). +.SH SEE ALSO +.BR PCPIntro (1), +.BR pmcd (1), +.BR pminfo (1), +.BR pmseries (1), +.BR pmlogger (1), +.BR systemd (1), +.BR pmLookupLabels (3), +.BR pcp.conf (5) +and +.BR pcp.env (5). diff --git a/src/pcp/geolocate/pcp-geolocate.py b/src/pcp/geolocate/pcp-geolocate.py new file mode 100755 index 0000000000..062e035310 --- /dev/null +++ b/src/pcp/geolocate/pcp-geolocate.py @@ -0,0 +1,89 @@ +#!/usr/bin/pmpython +# +# Copyright (C) 2022-2023 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# pylint: disable=broad-except, bare-except, protected-access +""" Display geographical location from local cache or IP lookup """ + +from pcp.pmapi import pmContext as PCP +try: + import urllib.request as httprequest +except Exception: + import urllib2 as httprequest +import threading +import time +import json +import sys +import os + +start = time.time() +output = '{"latitude":%s,"longitude":%s}' +writing = False +stdoutfd = sys.stdout + +# first extract from cached (labels) file +try: + if len(sys.argv) == 2: # use single argument if presented + path = sys.argv[1] + else: + path = PCP.pmGetConfig('PCP_SYSCONF_DIR') + path += '/labels/optional/geolocate' + cached = json.load(path) + print(output % (cached['latitude'], cached['longitude'])) +except: + try: + sys.stdout = open(path, 'w') # create (or overwrite) + writing = True + except PermissionError: + sys.stderr.write('No permission to write to %s\n' % path) + sys.stdout = stdoutfd + + +# setup threads for handling parallel REST API requests +threads = [] + +def finish(url): + if writing: + sys.stdout.close() + sys.stdout = stdoutfd + #print("'%s\' fetched in %ss" % (url, time.time() - start)) + os._exit(0) # not sys.exit, which awaits all threads + + +def ipinfo(url): + with httprequest.urlopen(url) as http: + data = json.load(http) + coords = data['loc'].split(',') + print(output % (coords[0], coords[1])) + finish(url) + +IPINFO = "https://ipinfo.io/json" +threads.append(threading.Thread(target = ipinfo, args = (IPINFO, ))) + + +def mozilla(url): + with httprequest.urlopen(url) as http: + data = json.load(http) + coords = data['location'] + print(output % (coords['lat'], coords['lng'])) + finish(url) + +MOZILLA = "https://location.services.mozilla.com/v1/geolocate?key=geoclue" +threads.append(threading.Thread(target = mozilla, args = (MOZILLA,))) + + +try: + for thread in threads: + thread.start() +except KeyboardInterrupt: + finish('(none)') diff --git a/src/pcp/geolocate/pcp-geolocate.service.in b/src/pcp/geolocate/pcp-geolocate.service.in new file mode 100644 index 0000000000..602fa5f344 --- /dev/null +++ b/src/pcp/geolocate/pcp-geolocate.service.in @@ -0,0 +1,15 @@ +[Unit] +Description=PCP geographical location label service +Documentation=man:pcp-geolocate(1) +ConditionPathExists=!@PCP_SYSCONF_DIR/labels/optional/geolocate +After=network-online.target +Before=pmcd.service + +[Service] +Type=oneshot +ExecStart=@PCP_BINADM_DIR/pcp-geolocate @PCP_SYSCONF_DIR/labels/optional/geolocate +Group=root +User=root + +[Install] +WantedBy=multi-user.target