diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 1a1e0efab..696d8bb29 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -48,4 +48,6 @@ run2: build2 TRAFFIC_GENERATOR_NUM_WORKERS=4 \ TRAFFIC_GENERATOR_CHAIN_RPC=http://localhost:8545 \ TRAFFIC_GENERATOR_NUM_RETRIES=2 \ + TRAFFIC_GENERATOR_METRICS_FUZZY_BLACKLIST=_returned_chunk,recombination_success \ + TRAFFIC_GENERATOR_METRICS_BLACKLIST=get_status_CONFIRMED \ ./bin/server2 diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index 3f7f7d41e..2b147f1a4 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -106,6 +106,9 @@ func NewConfig(ctx *cli.Context) (*Config, error) { EigenDAServiceManager: retrieverConfig.EigenDAServiceManagerAddr, SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), CustomQuorums: customQuorumsUint8, + + MetricsBlacklist: ctx.StringSlice(MetricsBlacklistFlag.Name), + MetricsFuzzyBlacklist: ctx.StringSlice(MetricsFuzzyBlacklistFlag.Name), }, } diff --git a/tools/traffic/config/flags.go b/tools/traffic/config/flags.go index 20105de62..c4218e52c 100644 --- a/tools/traffic/config/flags.go +++ b/tools/traffic/config/flags.go @@ -80,6 +80,20 @@ var ( EnvVar: common.PrefixEnvVar(envPrefix, "INSTANCE_LAUNCH_INTERVAL"), } + MetricsBlacklistFlag = cli.StringSliceFlag{ + Name: common.PrefixFlag(FlagPrefix, "metrics-blacklist"), + Usage: "Any metric with a label exactly matching this string will not be sent to the metrics server.", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_BLACKLIST"), + } + + MetricsFuzzyBlacklistFlag = cli.StringSliceFlag{ + Name: common.PrefixFlag(FlagPrefix, "metrics-fuzzy-blacklist"), + Usage: "Any metric that contains any string in this list will not be sent to the metrics server.", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_FUZZY_BLACKLIST"), + } + /* Configuration for the blob writer. */ NumWriteInstancesFlag = cli.UintFlag{ @@ -238,6 +252,8 @@ var optionalFlags = []cli.Flag{ GetBlobStatusTimeoutFlag, WriteTimeoutFlag, VerificationChannelCapacityFlag, + MetricsBlacklistFlag, + MetricsFuzzyBlacklistFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/config/worker_config.go b/tools/traffic/config/worker_config.go index 08dead295..78eb7a6fc 100644 --- a/tools/traffic/config/worker_config.go +++ b/tools/traffic/config/worker_config.go @@ -2,7 +2,7 @@ package config import "time" -// Config configures the traffic generator workers. +// WorkerConfig configures the traffic generator workers. type WorkerConfig struct { // The number of worker threads that generate write traffic. NumWriteInstances uint @@ -42,4 +42,12 @@ type WorkerConfig struct { SignerPrivateKey string // Custom quorum numbers to use for the traffic generator. CustomQuorums []uint8 + + // Any metric with a label exactly matching one of the strings in this list will not be sent to the metrics server. + MetricsBlacklist []string + + // Any metric that contains any string in this list will not be sent to the metrics server. For example, + // including the string "_returned_chunk" will cause all metrics in the form of + // "operator_fb390a64122db3957fb220c3c42d5f71e97ab0c995da4e1e5cc3261602dac527_returned_chunk" to be omitted. + MetricsFuzzyBlacklist []string } diff --git a/tools/traffic/generator_v2.go b/tools/traffic/generator_v2.go index cb9165d9c..d5ce70ae6 100644 --- a/tools/traffic/generator_v2.go +++ b/tools/traffic/generator_v2.go @@ -77,7 +77,11 @@ func NewTrafficGeneratorV2(config *config.Config) (*Generator, error) { ctx, cancel := context.WithCancel(context.Background()) waitGroup := sync.WaitGroup{} - generatorMetrics := metrics.NewMetrics(config.MetricsHTTPPort, logger) + generatorMetrics := metrics.NewMetrics( + config.MetricsHTTPPort, + logger, + config.WorkerConfig.MetricsBlacklist, + config.WorkerConfig.MetricsFuzzyBlacklist) blobTable := table.NewBlobStore() diff --git a/tools/traffic/metrics/count_metric.go b/tools/traffic/metrics/count_metric.go index 0e9ba6e7d..daa508bb8 100644 --- a/tools/traffic/metrics/count_metric.go +++ b/tools/traffic/metrics/count_metric.go @@ -14,14 +14,19 @@ type CountMetric interface { type countMetric struct { metrics *metrics description string + // disabled specifies whether the metrics should behave as a no-op + disabled bool } // Increment increments the count of a type of event. func (metric *countMetric) Increment() { + if metric.disabled { + return + } metric.metrics.count.WithLabelValues(metric.description).Inc() } -// NewCountMetric creates a new prometheus collector for counting metrics. +// buildCounterCollector creates a new prometheus collector for counting metrics. func buildCounterCollector(namespace string, registry *prometheus.Registry) *prometheus.CounterVec { return promauto.With(registry).NewCounterVec( prometheus.CounterOpts{ diff --git a/tools/traffic/metrics/gauge_metric.go b/tools/traffic/metrics/gauge_metric.go index 5cace773d..aa2d5d641 100644 --- a/tools/traffic/metrics/gauge_metric.go +++ b/tools/traffic/metrics/gauge_metric.go @@ -15,14 +15,19 @@ type GaugeMetric interface { type gaugeMetric struct { metrics *metrics description string + // disabled specifies whether the metrics should behave as a no-op + disabled bool } // Set sets the value of a gauge metric. func (metric *gaugeMetric) Set(value float64) { + if metric.disabled { + return + } metric.metrics.gauge.WithLabelValues(metric.description).Set(value) } -// NewGaugeMetric creates a collector for gauge metrics. +// buildGaugeCollector creates a collector for gauge metrics. func buildGaugeCollector(namespace string, registry *prometheus.Registry) *prometheus.GaugeVec { return promauto.With(registry).NewGaugeVec( prometheus.GaugeOpts{ diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go index 892994079..79e1d12da 100644 --- a/tools/traffic/metrics/latency_metric.go +++ b/tools/traffic/metrics/latency_metric.go @@ -15,14 +15,19 @@ type LatencyMetric interface { type latencyMetric struct { metrics *metrics description string + // disabled specifies whether the metrics should behave as a no-op + disabled bool } // ReportLatency reports the latency of an operation. func (metric *latencyMetric) ReportLatency(latency time.Duration) { + if metric.disabled { + return + } metric.metrics.latency.WithLabelValues(metric.description).Observe(latency.Seconds()) } -// NewLatencyMetric creates a new prometheus collector for latency metrics. +// buildLatencyCollector creates a new prometheus collector for latency metrics. func buildLatencyCollector(namespace string, registry *prometheus.Registry) *prometheus.SummaryVec { return promauto.With(registry).NewSummaryVec( prometheus.SummaryOpts{ diff --git a/tools/traffic/metrics/metrics.go b/tools/traffic/metrics/metrics.go index 971dad248..e24a52dfc 100644 --- a/tools/traffic/metrics/metrics.go +++ b/tools/traffic/metrics/metrics.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" + "strings" ) // Metrics allows the creation of metrics for the traffic generator. @@ -31,22 +32,39 @@ type metrics struct { httpPort string logger logging.Logger + + metricsBlacklist []string + metricsFuzzyBlacklist []string } // NewMetrics creates a new Metrics instance. -func NewMetrics(httpPort string, logger logging.Logger) Metrics { +func NewMetrics( + httpPort string, + logger logging.Logger, + metricsBlacklist []string, + metricsFuzzyBlacklist []string) Metrics { + namespace := "eigenda_generator" reg := prometheus.NewRegistry() reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) reg.MustRegister(collectors.NewGoCollector()) + if metricsBlacklist == nil { + metricsBlacklist = []string{} + } + if metricsFuzzyBlacklist == nil { + metricsFuzzyBlacklist = []string{} + } + metrics := &metrics{ - count: buildCounterCollector(namespace, reg), - latency: buildLatencyCollector(namespace, reg), - gauge: buildGaugeCollector(namespace, reg), - registry: reg, - httpPort: httpPort, - logger: logger.With("component", "GeneratorMetrics"), + count: buildCounterCollector(namespace, reg), + latency: buildLatencyCollector(namespace, reg), + gauge: buildGaugeCollector(namespace, reg), + registry: reg, + httpPort: httpPort, + logger: logger.With("component", "GeneratorMetrics"), + metricsBlacklist: metricsBlacklist, + metricsFuzzyBlacklist: metricsFuzzyBlacklist, } return metrics } @@ -71,6 +89,7 @@ func (metrics *metrics) NewLatencyMetric(description string) LatencyMetric { return &latencyMetric{ metrics: metrics, description: description, + disabled: metrics.isBlacklisted(description), } } @@ -79,6 +98,7 @@ func (metrics *metrics) NewCountMetric(description string) CountMetric { return &countMetric{ metrics: metrics, description: description, + disabled: metrics.isBlacklisted(description), } } @@ -87,5 +107,21 @@ func (metrics *metrics) NewGaugeMetric(description string) GaugeMetric { return &gaugeMetric{ metrics: metrics, description: description, + disabled: metrics.isBlacklisted(description), + } +} + +// isBlacklisted returns true if the metric name is blacklisted. +func (metrics *metrics) isBlacklisted(metricName string) bool { + for _, blacklisted := range metrics.metricsBlacklist { + if metricName == blacklisted { + return true + } + } + for _, blacklisted := range metrics.metricsFuzzyBlacklist { + if strings.Contains(metricName, blacklisted) { + return true + } } + return false }