Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add semantic versioning to pgbouncer_exporter #180

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add semantic versioning to pgbouncer_exporter #180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 83 additions & 62 deletions collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,61 +15,62 @@ package main

import (
"database/sql"
"errors"
"fmt"
"log/slog"
"math"
"os"
"regexp"
"strconv"
"time"
"unicode/utf8"

"github.com/blang/semver/v4"
_ "github.com/lib/pq"
"github.com/prometheus/client_golang/prometheus"
)

var (
metricMaps = map[string]map[string]ColumnMapping{
"databases": {
"name": {LABEL, "N/A", 1, "N/A"},
"host": {LABEL, "N/A", 1, "N/A"},
"port": {LABEL, "N/A", 1, "N/A"},
"database": {LABEL, "N/A", 1, "N/A"},
"force_user": {LABEL, "N/A", 1, "N/A"},
"pool_size": {GAUGE, "pool_size", 1, "Maximum number of server connections"},
"reserve_pool": {GAUGE, "reserve_pool", 1, "Maximum number of additional connections for this database"},
"pool_mode": {LABEL, "N/A", 1, "N/A"},
"max_connections": {GAUGE, "max_connections", 1, "Maximum number of allowed connections for this database"},
"current_connections": {GAUGE, "current_connections", 1, "Current number of connections for this database"},
"paused": {GAUGE, "paused", 1, "1 if this database is currently paused, else 0"},
"disabled": {GAUGE, "disabled", 1, "1 if this database is currently disabled, else 0"},
"name": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"host": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"port": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"database": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"force_user": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"pool_size": {GAUGE, "pool_size", 1, "Maximum number of server connections", semver.Version{}},
"reserve_pool": {GAUGE, "reserve_pool", 1, "Maximum number of additional connections for this database", semver.Version{}},
"pool_mode": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"max_connections": {GAUGE, "max_connections", 1, "Maximum number of allowed connections for this database", semver.Version{}},
"current_connections": {GAUGE, "current_connections", 1, "Current number of connections for this database", semver.Version{}},
"paused": {GAUGE, "paused", 1, "1 if this database is currently paused, else 0", semver.Version{}},
"disabled": {GAUGE, "disabled", 1, "1 if this database is currently disabled, else 0", semver.Version{}},
},
"stats": {
"database": {LABEL, "N/A", 1, "N/A"},
"total_query_count": {COUNTER, "queries_pooled_total", 1, "Total number of SQL queries pooled"},
"total_query_time": {COUNTER, "queries_duration_seconds_total", 1e-6, "Total number of seconds spent by pgbouncer when actively connected to PostgreSQL, executing queries"},
"total_received": {COUNTER, "received_bytes_total", 1, "Total volume in bytes of network traffic received by pgbouncer, shown as bytes"},
"total_requests": {COUNTER, "queries_total", 1, "Total number of SQL requests pooled by pgbouncer, shown as requests"},
"total_sent": {COUNTER, "sent_bytes_total", 1, "Total volume in bytes of network traffic sent by pgbouncer, shown as bytes"},
"total_wait_time": {COUNTER, "client_wait_seconds_total", 1e-6, "Time spent by clients waiting for a server in seconds"},
"total_xact_count": {COUNTER, "sql_transactions_pooled_total", 1, "Total number of SQL transactions pooled"},
"total_xact_time": {COUNTER, "server_in_transaction_seconds_total", 1e-6, "Total number of seconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries"},
"database": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"total_query_count": {COUNTER, "queries_pooled_total", 1, "Total number of SQL queries pooled", semver.Version{}},
"total_query_time": {COUNTER, "queries_duration_seconds_total", 1e-6, "Total number of seconds spent by pgbouncer when actively connected to PostgreSQL, executing queries", semver.Version{}},
"total_received": {COUNTER, "received_bytes_total", 1, "Total volume in bytes of network traffic received by pgbouncer, shown as bytes", semver.Version{}},
"total_requests": {COUNTER, "queries_total", 1, "Total number of SQL requests pooled by pgbouncer, shown as requests", semver.Version{}},
"total_sent": {COUNTER, "sent_bytes_total", 1, "Total volume in bytes of network traffic sent by pgbouncer, shown as bytes", semver.Version{}},
"total_wait_time": {COUNTER, "client_wait_seconds_total", 1e-6, "Time spent by clients waiting for a server in seconds", semver.Version{}},
"total_xact_count": {COUNTER, "sql_transactions_pooled_total", 1, "Total number of SQL transactions pooled", semver.Version{}},
"total_xact_time": {COUNTER, "server_in_transaction_seconds_total", 1e-6, "Total number of seconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries", semver.Version{}},
},
"pools": {
"database": {LABEL, "N/A", 1, "N/A"},
"user": {LABEL, "N/A", 1, "N/A"},
"cl_active": {GAUGE, "client_active_connections", 1, "Client connections linked to server connection and able to process queries, shown as connection"},
"cl_active_cancel_req": {GAUGE, "client_active_cancel_connections", 1, "Client connections that have forwarded query cancellations to the server and are waiting for the server response"},
"cl_waiting": {GAUGE, "client_waiting_connections", 1, "Client connections waiting on a server connection, shown as connection"},
"cl_waiting_cancel_req": {GAUGE, "client_waiting_cancel_connections", 1, "Client connections that have not forwarded query cancellations to the server yet"},
"sv_active": {GAUGE, "server_active_connections", 1, "Server connections linked to a client connection, shown as connection"},
"sv_active_cancel": {GAUGE, "server_active_cancel_connections", 1, "Server connections that are currently forwarding a cancel request."},
"sv_being_canceled": {GAUGE, "server_being_canceled_connections", 1, "Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server."},
"sv_idle": {GAUGE, "server_idle_connections", 1, "Server connections idle and ready for a client query, shown as connection"},
"sv_used": {GAUGE, "server_used_connections", 1, "Server connections idle more than server_check_delay, needing server_check_query, shown as connection"},
"sv_tested": {GAUGE, "server_testing_connections", 1, "Server connections currently running either server_reset_query or server_check_query, shown as connection"},
"sv_login": {GAUGE, "server_login_connections", 1, "Server connections currently in the process of logging in, shown as connection"},
"maxwait": {GAUGE, "client_maxwait_seconds", 1, "Age of oldest unserved client connection, shown as second"},
"database": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"user": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"cl_active": {GAUGE, "client_active_connections", 1, "Client connections linked to server connection and able to process queries, shown as connection", semver.Version{}},
"cl_active_cancel_req": {GAUGE, "client_active_cancel_connections", 1, "Client connections that have forwarded query cancellations to the server and are waiting for the server response", semver.Version{}},
"cl_waiting": {GAUGE, "client_waiting_connections", 1, "Client connections waiting on a server connection, shown as connection", semver.Version{}},
"cl_waiting_cancel_req": {GAUGE, "client_waiting_cancel_connections", 1, "Client connections that have not forwarded query cancellations to the server yet", semver.Version{}},
"sv_active": {GAUGE, "server_active_connections", 1, "Server connections linked to a client connection, shown as connection", semver.Version{}},
"sv_active_cancel": {GAUGE, "server_active_cancel_connections", 1, "Server connections that are currently forwarding a cancel request.", semver.Version{}},
"sv_being_canceled": {GAUGE, "server_being_canceled_connections", 1, "Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server.", semver.Version{}},
"sv_idle": {GAUGE, "server_idle_connections", 1, "Server connections idle and ready for a client query, shown as connection", semver.Version{}},
"sv_used": {GAUGE, "server_used_connections", 1, "Server connections idle more than server_check_delay, needing server_check_query, shown as connection", semver.Version{}},
"sv_tested": {GAUGE, "server_testing_connections", 1, "Server connections currently running either server_reset_query or server_check_query, shown as connection", semver.Version{}},
"sv_login": {GAUGE, "server_login_connections", 1, "Server connections currently in the process of logging in, shown as connection", semver.Version{}},
"maxwait": {GAUGE, "client_maxwait_seconds", 1, "Age of oldest unserved client connection, shown as second", semver.Version{}},
},
}

Expand Down Expand Up @@ -142,10 +143,17 @@ func NewExporter(connectionString string, namespace string, logger *slog.Logger)
os.Exit(1)
}

_, pgbouncerVersion, err := querySemver(db)
if err != nil {
// Don't fail on error, just log it
logger.Debug("error getting pgbouncer version", "err", err.Error())
}

return &Exporter{
metricMap: makeDescMap(metricMaps, namespace, logger),
metricMap: makeDescMap(metricMaps, namespace, logger, pgbouncerVersion),
db: db,
logger: logger,
version: pgbouncerVersion,
}
}

Expand Down Expand Up @@ -404,37 +412,37 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, db *sql.DB, metricMap m
return namespaceErrors
}

// Gather the pgbouncer version info.
func queryVersion(ch chan<- prometheus.Metric, db *sql.DB) error {
rows, err := db.Query("SHOW VERSION;")
if err != nil {
return fmt.Errorf("error getting pgbouncer version: %w", err)
}
defer rows.Close()
// Extract semver from `SHOW VERSION;` output
// Should be some variation of "PgBouncer 1.23.1"
var versionRegex = regexp.MustCompile(`^\w+ ((\d+)(\.\d+)?(\.\d+)?)`)

var columnNames []string
columnNames, err = rows.Columns()
func querySemver(db *sql.DB) (string, semver.Version, error) {
var version string
err := db.QueryRow("SHOW VERSION;").Scan(&version)
if err != nil {
return fmt.Errorf("error retrieving column list for version: %w", err)
return "", semver.Version{}, fmt.Errorf("error getting pgbouncer version: %w", err)
}
if len(columnNames) != 1 || columnNames[0] != "version" {
return errors.New("show version didn't return version column")
submatches := versionRegex.FindStringSubmatch(version)
if len(submatches) > 1 {
v, err := semver.ParseTolerant(submatches[1])
return version, v, err
}
return version, semver.Version{}, nil
}

var bouncerVersion string
// Gather the pgbouncer version info.
func queryVersion(ch chan<- prometheus.Metric, db *sql.DB) error {
bouncerVersion, _, err := querySemver(db)

for rows.Next() {
err := rows.Scan(&bouncerVersion)
if err != nil {
return err
}
ch <- prometheus.MustNewConstMetric(
bouncerVersionDesc,
prometheus.GaugeValue,
1.0,
bouncerVersion,
)
if err != nil {
return err
}
ch <- prometheus.MustNewConstMetric(
bouncerVersionDesc,
prometheus.GaugeValue,
1.0,
bouncerVersion,
)

return nil
}
Expand Down Expand Up @@ -502,7 +510,7 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
}

// Turn the MetricMap column mapping into a prometheus descriptor mapping.
func makeDescMap(metricMaps map[string]map[string]ColumnMapping, namespace string, logger *slog.Logger) map[string]MetricMapNamespace {
func makeDescMap(metricMaps map[string]map[string]ColumnMapping, namespace string, logger *slog.Logger, pgbouncerVersion semver.Version) map[string]MetricMapNamespace {
var metricMap = make(map[string]MetricMapNamespace)

for metricNamespace, mappings := range metricMaps {
Expand All @@ -511,6 +519,13 @@ func makeDescMap(metricMaps map[string]map[string]ColumnMapping, namespace strin

// First collect all the labels since the metrics will need them
for columnName, columnMapping := range mappings {
fmt.Println(columnName)
fmt.Println(columnMapping.minVersion)
if pgbouncerVersion.LT(columnMapping.minVersion) {
logger.Debug("Skipping column due to version", "column_name", columnName, "metric_namespace", metricNamespace, "min_version", columnMapping.minVersion, "pgbouncer_version", pgbouncerVersion)
continue
}

if columnMapping.usage == LABEL {
logger.Debug("Adding label", "column_name", columnName, "metric_namespace", metricNamespace)
labels = append(labels, columnName)
Expand All @@ -520,6 +535,12 @@ func makeDescMap(metricMaps map[string]map[string]ColumnMapping, namespace strin
for columnName, columnMapping := range mappings {
factor := columnMapping.factor

// Check semver compatibility
if pgbouncerVersion.LT(columnMapping.minVersion) {
logger.Debug("Skipping column due to version", "column_name", columnName, "metric_namespace", metricNamespace, "min_version", columnMapping.minVersion, "pgbouncer_version", pgbouncerVersion)
continue
}

// Determine how to convert the column based on its usage.
switch columnMapping.usage {
case COUNTER:
Expand Down
113 changes: 111 additions & 2 deletions collector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"log/slog"

"github.com/DATA-DOG/go-sqlmock"
"github.com/blang/semver/v4"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/smartystreets/goconvey/convey"
Expand Down Expand Up @@ -63,7 +64,7 @@ func TestQueryShowList(t *testing.T) {
AddRow("users", 2)

mock.ExpectQuery("SHOW LISTS;").WillReturnRows(rows)
logger := &slog.Logger{}
logger := slog.Default()

ch := make(chan prometheus.Metric)
go func() {
Expand Down Expand Up @@ -105,7 +106,7 @@ func TestQueryShowConfig(t *testing.T) {
AddRow("client_tls_ciphers", "default", "default", "yes")

mock.ExpectQuery("SHOW CONFIG;").WillReturnRows(rows)
logger := &slog.Logger{}
logger := slog.Default()

ch := make(chan prometheus.Metric)
go func() {
Expand All @@ -129,3 +130,111 @@ func TestQueryShowConfig(t *testing.T) {
t.Errorf("there were unfulfilled exceptions: %s", err)
}
}

func TestQueryVersion(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("Error opening a stub db connection: %s", err)
}
defer db.Close()

rows := sqlmock.NewRows([]string{"version"}).
AddRow("PgBouncer 1.23.1")

mock.ExpectQuery("SHOW VERSION;").WillReturnRows(rows)

ch := make(chan prometheus.Metric)
go func() {
defer close(ch)
err := queryVersion(ch, db)
if err != nil {
t.Errorf("Error running queryShowConfig: %s", err)
}
}()

expected := []MetricResult{
{labels: labelMap{"version": "PgBouncer 1.23.1"}, metricType: dto.MetricType_GAUGE, value: 1},
}

convey.Convey("Metrics comparison", t, func() {
for _, expect := range expected {
m := readMetric(<-ch)
convey.So(expect, convey.ShouldResemble, m)
}
})
if err := mock.ExpectationsWereMet(); err != nil {
t.Errorf("there were unfulfilled exceptions: %s", err)
}
}

func TestBadQueryVersion(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("Error opening a stub db connection: %s", err)
}
defer db.Close()

rows := sqlmock.NewRows([]string{"version"}).
AddRow("PgBouncer x.x.x")

mock.ExpectQuery("SHOW VERSION;").WillReturnRows(rows)

ch := make(chan prometheus.Metric)
go func() {
defer close(ch)
err := queryVersion(ch, db)
if err != nil {
t.Errorf("Error running queryShowConfig: %s", err)
}
}()

expected := []MetricResult{
{labels: labelMap{"version": "PgBouncer x.x.x"}, metricType: dto.MetricType_GAUGE, value: 1},
}

convey.Convey("Metrics comparison", t, func() {
for _, expect := range expected {
m := readMetric(<-ch)
convey.So(expect, convey.ShouldResemble, m)
}
})
if err := mock.ExpectationsWereMet(); err != nil {
t.Errorf("there were unfulfilled exceptions: %s", err)
}
}

func TestMakeDescMap(t *testing.T) {
currentVersion := semver.MustParse("1.20.1")
metricMap := map[string]ColumnMapping{
"name": {LABEL, "N/A", 1, "N/A", semver.Version{}},
"host": {LABEL, "N/A", 1, "N/A", semver.MustParse("1.21.0")},
"port": {LABEL, "N/A", 1, "N/A", semver.MustParse("1.9.0")},
"pool_size": {GAUGE, "pool_size", 1, "Maximum number of server connections", semver.MustParse("1.22.0")},
"reserve_pool": {GAUGE, "reserve_pool", 1, "Maximum number of additional connections for this database", semver.Version{}},
"current_connections": {GAUGE, "current_connections", 1e-6, "Current number of connections for this database", semver.MustParse("1.7.0")},
"total_query_count": {COUNTER, "queries_pooled_total", 1, "Total number of SQL queries pooled", semver.Version{}},
}
metricMaps := map[string]map[string]ColumnMapping{
"database": metricMap,
}
logger := slog.Default()

convey.Convey("Test makeDescMap", t, func() {
descMap := makeDescMap(metricMaps, "foo", logger, currentVersion)

convey.So(descMap, convey.ShouldContainKey, "database")
convey.So(descMap, convey.ShouldHaveLength, 1)

convey.So(descMap["database"].labels, convey.ShouldHaveLength, 2)
convey.So(descMap["database"].labels, convey.ShouldContain, "name")
convey.So(descMap["database"].labels, convey.ShouldContain, "port")

convey.So(descMap["database"].columnMappings, convey.ShouldHaveLength, 3)
convey.So(descMap["database"].columnMappings, convey.ShouldContainKey, "reserve_pool")
convey.So(descMap["database"].columnMappings["reserve_pool"].vtype, convey.ShouldEqual, prometheus.GaugeValue)
convey.So(descMap["database"].columnMappings, convey.ShouldContainKey, "current_connections")
convey.So(descMap["database"].columnMappings["current_connections"].vtype, convey.ShouldEqual, prometheus.GaugeValue)
convey.So(descMap["database"].columnMappings, convey.ShouldContainKey, "total_query_count")
convey.So(descMap["database"].columnMappings["total_query_count"].vtype, convey.ShouldEqual, prometheus.CounterValue)
})
}
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.22
require (
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/alecthomas/kingpin/v2 v2.4.0
github.com/blang/semver/v4 v4.0.0
github.com/lib/pq v1.10.9
github.com/prometheus/client_golang v1.20.4
github.com/prometheus/client_model v0.6.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
Expand Down
Loading