Feat(config): add option to export multiple pgbouncer instances with …

…one exporter instance.

Signed-off-by: Maikel Poot <[email protected]>

README.md

@@ -9,10 +9,21 @@ Exports metrics at `9127/metrics`
     make build
     ./pgbouncer_exporter <flags>
 
+## Exporter configuration
+
+### Command line flags
 To see all available configuration flags:
 
     ./pgbouncer_exporter -h
 
+### Export multiple PGBouncer instances
+
+If you want to export metrics for multiple PGBouncer instances without running multiple exporters you can use the config.file option
+
+    ./pgbouncer_exporter --config.file config.yaml
+
+For more information about the possibilities and requirements see [the example config.yaml file within this repo](config.yaml) 
+
 ## PGBouncer configuration
 
 The pgbouncer\_exporter requires a configuration change to pgbouncer to ignore a PostgreSQL driver connection parameter. In the `pgbouncer.ini` please include this option:

collector.go

@@ -133,9 +133,16 @@ var (
 	)
 )
 
-func NewExporter(connectionString string, namespace string, logger *slog.Logger) *Exporter {
+func NewExporter(connectionString string, namespace string, logger *slog.Logger, mustConnect bool) *Exporter {
 
-	db, err := getDB(connectionString)
+	var db *sql.DB
+	var err error
+
+	if mustConnect {
+		db, err = getDBWithTest(connectionString)
+	} else {
+		db, err = getDB(connectionString)
+	}
 
 	if err != nil {
 		logger.Error("error setting up DB connection", "err", err.Error())
@@ -337,15 +344,22 @@ func getDB(conn string) (*sql.DB, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	db.SetMaxOpenConns(1)
+	db.SetMaxIdleConns(1)
+
+	return db, nil
+}
+func getDBWithTest(conn string) (*sql.DB, error) {
+	db, err := getDB(conn)
+	if err != nil {
+		return nil, err
+	}
 	rows, err := db.Query("SHOW STATS")
 	if err != nil {
 		return nil, fmt.Errorf("error pinging pgbouncer: %w", err)
 	}
 	defer rows.Close()
-
-	db.SetMaxOpenConns(1)
-	db.SetMaxIdleConns(1)
-
 	return db, nil
 }
 

config.go

@@ -0,0 +1,133 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"gopkg.in/yaml.v3"
+	"maps"
+	"os"
+	"slices"
+	"strings"
+)
+
+var (
+	ErrNoPgbouncersConfigured = errors.New("no pgbouncer instances configured")
+	ErrEmptyPgbouncersDSN     = errors.New("atleast one pgbouncer instance has an empty dsn configured")
+)
+
+func NewDefaultConfig() *Config {
+	return &Config{
+		MustConnectOnStartup: true,
+		ExtraLabels:          map[string]string{},
+		MetricsPath:          "/metrics",
+		PgBouncers:           []PgBouncerConfig{},
+	}
+}
+
+func NewConfigFromFile(path string) (*Config, error) {
+	var err error
+	var data []byte
+	if path == "" {
+		return nil, nil
+	}
+	config := NewDefaultConfig()
+
+	data, err = os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(data, config)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(config.PgBouncers) == 0 {
+		return nil, ErrNoPgbouncersConfigured
+	}
+
+	for _, instance := range config.PgBouncers {
+		if strings.TrimSpace(instance.DSN) == "" {
+			return nil, ErrEmptyPgbouncersDSN
+		}
+	}
+
+	return config, nil
+
+}
+
+type Config struct {
+	MustConnectOnStartup bool              `yaml:"must_connect_on_startup"`
+	ExtraLabels          map[string]string `yaml:"extra_labels"`
+	PgBouncers           []PgBouncerConfig `yaml:"pgbouncers"`
+	MetricsPath          string            `yaml:"metrics_path"`
+}
+type PgBouncerConfig struct {
+	DSN         string            `yaml:"dsn"`
+	PidFile     string            `yaml:"pid-file"`
+	ExtraLabels map[string]string `yaml:"extra_labels"`
+}
+
+func (p *Config) AddPgbouncerConfig(dsn string, pidFilePath string, extraLabels map[string]string) {
+	p.PgBouncers = append(
+		p.PgBouncers,
+		PgBouncerConfig{
+			DSN:         dsn,
+			PidFile:     pidFilePath,
+			ExtraLabels: extraLabels,
+		},
+	)
+}
+
+func (p *Config) MergedExtraLabels(extraLabels map[string]string) map[string]string {
+	mergedLabels := make(map[string]string)
+	maps.Copy(mergedLabels, p.ExtraLabels)
+	maps.Copy(mergedLabels, extraLabels)
+
+	return mergedLabels
+}
+
+func (p Config) ValidateLabels() error {
+
+	var labels = make(map[string]int)
+	var keys = make(map[string]int)
+	for _, cfg := range p.PgBouncers {
+
+		var slabels []string
+
+		for k, v := range p.MergedExtraLabels(cfg.ExtraLabels) {
+			slabels = append(slabels, fmt.Sprintf("%s=%s", k, v))
+			keys[k]++
+		}
+		slices.Sort(slabels)
+		hash := strings.Join(slabels, ",")
+		if _, ok := labels[hash]; ok {
+			return fmt.Errorf("Every pgbouncer instance must have unique label values,"+
+				" found the following label=value combination multiple times: '%s'", hash)
+		}
+		labels[hash] = 1
+	}
+
+	for k, amount := range keys {
+		if amount != len(p.PgBouncers) {
+			return fmt.Errorf("Every pgbouncer instance must define the same extra labels,"+
+				" the label '%s' is only found on %d of the %d instances", k, amount, len(p.PgBouncers))
+		}
+	}
+
+	return nil
+
+}

config.yaml

@@ -0,0 +1,33 @@
+## must_connect_on_startup: true|false; Default: true
+## If true the exporter will fail to start if any connection fails to connect within the startup fase.
+## If false the exporter will start even if some connections fail.
+must_connect_on_startup: false
+
+## extra_labels: map of label:value; Default: empty
+## These common extra labels will be set on all metrics for all connections. The value can be overridden per connection
+## Note: Every connection MUST have the same set of labels but a unique set of values.
+extra_labels:
+  environment: test
+
+## metrics_path: /path/for/metrics; Default: /metrics
+metrics_path: /metrics
+
+## All the PGBouncers to scrape,
+## when multiple connections are used, extra_labels is required to give every connection a unique set of label values
+pgbouncers:
+  - dsn: postgres://postgres:@localhost:6543/pgbouncer?sslmode=disable    # Connection string for the pgbouncer instance (Required)
+    pid-file: /path/to/pidfile                                            # Add path to pgbouncer pid file to enable the process exporter metrics, Default: empty
+    extra_labels:                                                         # Extra labels to identify the metrics for each instance. As mentioned
+      pgbouncer_instance: set1-0                                          # Example: a unique identifier for each pgbouncer instance.
+      environment: prod                                                   # Example: a shared label for multiple pgbouncer instances
+  - dsn: postgres://postgres:@localhost:6544/pgbouncer?sslmode=disable
+    pid-file:
+    extra_labels:
+      pgbouncer_instance: set1-1
+      environment: prod
+  - dsn: postgres://postgres:@localhost:6545/pgbouncer?sslmode=disable
+    pid-file:
+    extra_labels:
+      pgbouncer_instance: set2-0
+      ## the metrics of this instance will have the additional labels: {environment: "test", pgbouncer_instance: "set2-0"}
+      ## as `environment: "test"` is inherited from common extra_labels

config_test.go

@@ -0,0 +1,156 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific langu
+package main
+
+import (
+	"errors"
+	"github.com/google/go-cmp/cmp"
+	"io/fs"
+	"maps"
+	"strings"
+	"testing"
+)
+
+func TestDefaultConfig(t *testing.T) {
+
+	config := NewDefaultConfig()
+
+	MustConnectOnStartupWant := true
+	if config.MustConnectOnStartup != MustConnectOnStartupWant {
+		t.Errorf("MustConnectOnStartup does not match. Want: %v, Got: %v", MustConnectOnStartupWant, config.MustConnectOnStartup)
+	}
+
+	MetricsPathWant := "/metrics"
+	if config.MetricsPath != MetricsPathWant {
+		t.Errorf("MustConnectOnStartup does not match. Want: %v, Got: %v", MetricsPathWant, config.MustConnectOnStartup)
+	}
+
+}
+
+func TestUnHappyFileConfig(t *testing.T) {
+
+	var config *Config
+	var err error
+
+	config, err = NewConfigFromFile("")
+	if config != nil || err != nil {
+		t.Errorf("NewConfigFromFile should return nil for config and error if path is empty. Got: %v", err)
+	}
+
+	_, err = NewConfigFromFile("./testdata/i-do-not-exist.yaml")
+	if errors.Is(err, fs.ErrNotExist) == false {
+		t.Errorf("NewConfigFromFile should return fs.ErrNotExist error. Got: %v", err)
+	}
+
+	_, err = NewConfigFromFile("./testdata/parse_error.yaml")
+	if err != nil && strings.Contains(err.Error(), "yaml: line") == false {
+		t.Errorf("NewConfigFromFile should return yaml parse error. Got: %v", err)
+	}
+
+	_, err = NewConfigFromFile("./testdata/empty.yaml")
+	if errors.Is(err, ErrNoPgbouncersConfigured) == false {
+		t.Errorf("NewConfigFromFile should return ErrNoPgbouncersConfigured error. Got: %v", err)
+	}
+
+	_, err = NewConfigFromFile("./testdata/no-dsn.yaml")
+	if errors.Is(err, ErrEmptyPgbouncersDSN) == false {
+		t.Errorf("NewConfigFromFile should return ErrEmptyPgbouncersDSN error. Got: %v", err)
+	}
+
+}
+
+func TestFileConfig(t *testing.T) {
+
+	var config *Config
+	var err error
+
+	config, err = NewConfigFromFile("./testdata/config.yaml")
+	if err != nil {
+		t.Errorf("NewConfigFromFile() should not throw an error: %v", err)
+	}
+
+	MustConnectOnStartupWant := false
+	if config.MustConnectOnStartup != MustConnectOnStartupWant {
+		t.Errorf("MustConnectOnStartup does not match. Want: %v, Got: %v", MustConnectOnStartupWant, config.MustConnectOnStartup)
+	}
+
+	MetricsPathWant := "/prom"
+	if config.MetricsPath != MetricsPathWant {
+		t.Errorf("MustConnectOnStartup does not match. Want: %v, Got: %v", MetricsPathWant, config.MustConnectOnStartup)
+	}
+
+	CommonExtraLabelsWant := map[string]string{"environment": "sandbox"}
+	if maps.Equal(config.ExtraLabels, CommonExtraLabelsWant) == false {
+		t.Errorf("ExtraLabels does not match. Want: %v, Got: %v", CommonExtraLabelsWant, config.ExtraLabels)
+	}
+
+	pgWants := []PgBouncerConfig{
+		{
+			DSN:         "postgres://postgres:@localhost:6543/pgbouncer?sslmode=disable",
+			PidFile:     "/var/run/pgbouncer1.pid",
+			ExtraLabels: map[string]string{"pgbouncer_instance": "set1-0", "environment": "prod"},
+		},
+		{
+			DSN:         "postgres://postgres:@localhost:6544/pgbouncer?sslmode=disable",
+			PidFile:     "/var/run/pgbouncer2.pid",
+			ExtraLabels: map[string]string{"pgbouncer_instance": "set1-1", "environment": "prod"},
+		},
+	}
+
+	for i := range pgWants {
+		if cmp.Equal(config.PgBouncers[i], pgWants[i]) == false {
+			t.Errorf("PGBouncer config %d does not match. Want: %v, Got: %v", i, pgWants[i], config.PgBouncers[i])
+		}
+	}
+
+	err = config.ValidateLabels()
+	if err != nil {
+		t.Errorf("ValidateLabels() throws an unexpected error: %v", err)
+	}
+
+}
+
+func TestNotUniqueLabels(t *testing.T) {
+
+	config := NewDefaultConfig()
+
+	config.AddPgbouncerConfig("", "", map[string]string{"pgbouncer_instance": "set1-0", "environment": "prod"})
+	config.AddPgbouncerConfig("", "", map[string]string{"pgbouncer_instance": "set1-0", "environment": "prod"})
+
+	err := config.ValidateLabels()
+	if err == nil {
+		t.Errorf("ValidateLabels() did not throw an error ")
+	}
+	errorWant := "Every pgbouncer instance must have unique label values, found the following label=value combination multiple times: 'environment=prod,pgbouncer_instance=set1-0'"
+	if err.Error() != errorWant {
+		t.Errorf("ValidateLabels() did not throw the expected error.\n- Want: %s\n- Got: %s", errorWant, err.Error())
+	}
+
+}
+
+func TestMissingLabels(t *testing.T) {
+
+	config := NewDefaultConfig()
+
+	config.AddPgbouncerConfig("", "", map[string]string{"pgbouncer_instance": "set1-0", "environment": "prod"})
+	config.AddPgbouncerConfig("", "", map[string]string{"pgbouncer_instance": "set1-0"})
+
+	err := config.ValidateLabels()
+	if err == nil {
+		t.Errorf("ValidateLabels() did not throw an error ")
+	}
+	errorWant := "Every pgbouncer instance must define the same extra labels, the label 'environment' is only found on 1 of the 2 instances"
+	if err.Error() != errorWant {
+		t.Errorf("ValidateLabels() did not throw the expected error.\n- Want: %s\n- Got: %s", errorWant, err.Error())
+	}
+
+}