feat: Tolerate collector failures

Signed-off-by: Jan-Otto Kröpke <[email protected]>
prometheus-community · Nov 20, 2024 · 976a64f · 976a64f
1 parent 5b7cd4a
commit 976a64f
Show file tree

Hide file tree

Showing 12 changed files with 96 additions and 24 deletions.
diff --git a/internal/collector/mssql/mssql_access_methods.go b/internal/collector/mssql/mssql_access_methods.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -110,6 +111,7 @@ func (c *Collector) buildAccessMethods() error {
 	var err error
 
 	c.accessMethodsPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		accessMethodsAUCleanupbatchesPerSec,
 		accessMethodsAUCleanupsPerSec,
@@ -160,7 +162,7 @@ func (c *Collector) buildAccessMethods() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.accessMethodsPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Access Methods"), nil, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create AccessMethods collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create AccessMethods collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -430,14 +432,18 @@ func (c *Collector) buildAccessMethods() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectAccessMethods(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorAccessMethods, c.accessMethodsPerfDataCollectors, c.collectAccessMethodsInstance)
 }
 
 func (c *Collector) collectAccessMethodsInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "AccessMethods"), err)

diff --git a/internal/collector/mssql/mssql_availability_replica.go b/internal/collector/mssql/mssql_availability_replica.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -41,6 +42,7 @@ func (c *Collector) buildAvailabilityReplica() error {
 	var err error
 
 	c.availabilityReplicaPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		availReplicaBytesReceivedFromReplicaPerSec,
 		availReplicaBytesSentToReplicaPerSec,
@@ -56,7 +58,7 @@ func (c *Collector) buildAvailabilityReplica() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.availabilityReplicaPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Availability Replica"), perfdata.InstanceAll, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create Availability Replica collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create Availability Replica collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -116,14 +118,18 @@ func (c *Collector) buildAvailabilityReplica() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectAvailabilityReplica(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorAvailabilityReplica, c.availabilityReplicaPerfDataCollectors, c.collectAvailabilityReplicaInstance)
 }
 
 func (c *Collector) collectAvailabilityReplicaInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "Availability Replica"), err)

diff --git a/internal/collector/mssql/mssql_buffer_manager.go b/internal/collector/mssql/mssql_buffer_manager.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -68,6 +69,7 @@ func (c *Collector) buildBufferManager() error {
 	var err error
 
 	c.bufManPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		bufManBackgroundWriterPagesPerSec,
 		bufManBufferCacheHitRatio,
@@ -97,7 +99,7 @@ func (c *Collector) buildBufferManager() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.bufManPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Buffer Manager"), nil, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create Buffer Manager collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create Buffer Manager collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -240,14 +242,18 @@ func (c *Collector) buildBufferManager() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectBufferManager(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorBufferManager, c.bufManPerfDataCollectors, c.collectBufferManagerInstance)
 }
 
 func (c *Collector) collectBufferManagerInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "Buffer Manager"), err)

diff --git a/internal/collector/mssql/mssql_database.go b/internal/collector/mssql/mssql_database.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -118,6 +119,7 @@ func (c *Collector) buildDatabases() error {
 	var err error
 
 	c.databasesPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		databasesActiveParallelRedoThreads,
 		databasesActiveTransactions,
@@ -172,7 +174,7 @@ func (c *Collector) buildDatabases() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.databasesPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Databases"), perfdata.InstanceAll, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create Databases collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create Databases collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -465,14 +467,18 @@ func (c *Collector) buildDatabases() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectDatabases(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorDatabases, c.databasesPerfDataCollectors, c.collectDatabasesInstance)
 }
 
 func (c *Collector) collectDatabasesInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "Databases"), err)

diff --git a/internal/collector/mssql/mssql_database_replica.go b/internal/collector/mssql/mssql_database_replica.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -70,6 +71,7 @@ func (c *Collector) buildDatabaseReplica() error {
 	var err error
 
 	c.dbReplicaPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		dbReplicaDatabaseFlowControlDelay,
 		dbReplicaDatabaseFlowControlsPerSec,
@@ -100,7 +102,7 @@ func (c *Collector) buildDatabaseReplica() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.dbReplicaPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Database Replica"), perfdata.InstanceAll, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create Database Replica collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create Database Replica collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -250,14 +252,18 @@ func (c *Collector) buildDatabaseReplica() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectDatabaseReplica(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorDatabaseReplica, c.dbReplicaPerfDataCollectors, c.collectDatabaseReplicaInstance)
 }
 
 func (c *Collector) collectDatabaseReplicaInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "Database Replica"), err)

diff --git a/internal/collector/mssql/mssql_general_statistics.go b/internal/collector/mssql/mssql_general_statistics.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -70,6 +71,7 @@ func (c *Collector) buildGeneralStatistics() error {
 	var err error
 
 	c.genStatsPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		genStatsActiveTempTables,
 		genStatsConnectionResetPerSec,
@@ -100,7 +102,7 @@ func (c *Collector) buildGeneralStatistics() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.genStatsPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "General Statistics"), nil, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create General Statistics collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create General Statistics collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -250,14 +252,18 @@ func (c *Collector) buildGeneralStatistics() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectGeneralStatistics(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorGeneralStatistics, c.genStatsPerfDataCollectors, c.collectGeneralStatisticsInstance)
 }
 
 func (c *Collector) collectGeneralStatisticsInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "General Statistics"), err)

diff --git a/internal/collector/mssql/mssql_locks.go b/internal/collector/mssql/mssql_locks.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -39,6 +40,7 @@ func (c *Collector) buildLocks() error {
 	var err error
 
 	c.locksPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		locksAverageWaitTimeMS,
 		locksAverageWaitTimeMSBase,
@@ -53,7 +55,7 @@ func (c *Collector) buildLocks() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.locksPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Locks"), perfdata.InstanceAll, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create Locks collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create Locks collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -106,14 +108,18 @@ func (c *Collector) buildLocks() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectLocks(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorLocks, c.locksPerfDataCollectors, c.collectLocksInstance)
 }
 
 func (c *Collector) collectLocksInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "Locks"), err)

diff --git a/internal/collector/mssql/mssql_memory_manager.go b/internal/collector/mssql/mssql_memory_manager.go
@@ -3,6 +3,7 @@
 package mssql
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/perfdata"
@@ -62,6 +63,7 @@ func (c *Collector) buildMemoryManager() error {
 	var err error
 
 	c.memMgrPerfDataCollectors = make(map[string]*perfdata.Collector, len(c.mssqlInstances))
+	errs := make([]error, 0, len(c.mssqlInstances))
 	counters := []string{
 		memMgrConnectionMemoryKB,
 		memMgrDatabaseCacheMemoryKB,
@@ -88,7 +90,7 @@ func (c *Collector) buildMemoryManager() error {
 	for sqlInstance := range c.mssqlInstances {
 		c.memMgrPerfDataCollectors[sqlInstance], err = perfdata.NewCollector(c.mssqlGetPerfObjectName(sqlInstance, "Memory Manager"), perfdata.InstanceAll, counters)
 		if err != nil {
-			return fmt.Errorf("failed to create Locks collector for instance %s: %w", sqlInstance, err)
+			errs = append(errs, fmt.Errorf("failed to create Memory Manager collector for instance %s: %w", sqlInstance, err))
 		}
 	}
 
@@ -213,14 +215,18 @@ func (c *Collector) buildMemoryManager() error {
 		nil,
 	)
 
-	return nil
+	return errors.Join(errs...)
 }
 
 func (c *Collector) collectMemoryManager(ch chan<- prometheus.Metric) error {
 	return c.collect(ch, subCollectorMemoryManager, c.memMgrPerfDataCollectors, c.collectMemoryManagerInstance)
 }
 
 func (c *Collector) collectMemoryManagerInstance(ch chan<- prometheus.Metric, sqlInstance string, perfDataCollector *perfdata.Collector) error {
+	if perfDataCollector == nil {
+		return types.ErrPerfCounterCollectorNotInitialized
+	}
+
 	perfData, err := perfDataCollector.Collect()
 	if err != nil {
 		return fmt.Errorf("failed to collect %s metrics: %w", c.mssqlGetPerfObjectName(sqlInstance, "Memory Manager"), err)