From 9b91bfce335780d8e32824d2d5fda1a77f39663b Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Wed, 27 Dec 2023 09:54:00 +0100 Subject: [PATCH] Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458) * use InitDefaults(); elastic-agent-autodiscover: fix default resource metadata config - https://github.com/elastic/elastic-agent-autodiscover/pull/72 Signed-off-by: Tetiana Kravchenko * revert unintended changes Signed-off-by: Tetiana Kravchenko * upgrade elastic-agent-autodiscover lib to v0.6.6 Signed-off-by: Tetiana Kravchenko * run make update; fix linter issues Signed-off-by: Tetiana Kravchenko * revert changes related to the 'commonMetaConfig'; add missing metadata configuration on the metricset level Signed-off-by: Tetiana Kravchenko * x-pack/metricbeat: run make update Signed-off-by: Tetiana Kravchenko --------- Signed-off-by: Tetiana Kravchenko (cherry picked from commit 8c4a40fdbfda1a8e5ff2ceae01930a9f3e629d97) # Conflicts: # NOTICE.txt # go.mod # go.sum --- NOTICE.txt | 8 +++++++ go.mod | 6 ++++- go.sum | 9 ++++++++ .../add_kubernetes_metadata/config.go | 18 +++++++-------- .../add_kubernetes_metadata/config_test.go | 10 ++++---- .../add_kubernetes_metadata/kubernetes.go | 3 +-- metricbeat/docs/modules/kubernetes.asciidoc | 23 +++++++++++++++---- metricbeat/metricbeat.reference.yml | 23 +++++++++++++++---- .../kubernetes/_meta/config.reference.yml | 23 +++++++++++++++---- metricbeat/module/kubernetes/_meta/config.yml | 12 +++++++--- .../module/kubernetes/util/kubernetes.go | 20 ++++++++-------- metricbeat/modules.d/kubernetes.yml.disabled | 12 +++++++--- x-pack/metricbeat/metricbeat.reference.yml | 23 +++++++++++++++---- 13 files changed, 141 insertions(+), 49 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 5b182ede53b..ccc72a51032 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12268,11 +12268,19 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover +<<<<<<< HEAD Version: v0.6.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.4/LICENSE: +======= +Version: v0.6.6 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE: +>>>>>>> 8c4a40fdbf (Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458)) Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index e7fa775b028..fdacdd61539 100644 --- a/go.mod +++ b/go.mod @@ -201,8 +201,13 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 +<<<<<<< HEAD github.com/elastic/elastic-agent-autodiscover v0.6.4 github.com/elastic/elastic-agent-libs v0.6.2 +======= + github.com/elastic/elastic-agent-autodiscover v0.6.6 + github.com/elastic/elastic-agent-libs v0.7.2 +>>>>>>> 8c4a40fdbf (Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458)) github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.8.2 github.com/elastic/go-elasticsearch/v8 v8.10.0 @@ -406,7 +411,6 @@ replace ( github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 - ) // Exclude this version because the version has an invalid checksum. diff --git a/go.sum b/go.sum index f56cb68e928..95d12a6c312 100644 --- a/go.sum +++ b/go.sum @@ -651,12 +651,21 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= +<<<<<<< HEAD github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA= github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3+oKIQDzdPDo= github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700= +======= +github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= +github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= +github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= +github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM= +github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o= +github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc= +>>>>>>> 8c4a40fdbf (Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458)) github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.8.2 h1:99H1o+WTD/g6jcQr1hUOZ+dsZkZF0SfNePmjGg7TnhU= diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index 0998a275ea4..7c74c82268d 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -50,15 +50,13 @@ type Enabled struct { type PluginConfig []map[string]config.C -func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig { - return kubeAnnotatorConfig{ - SyncPeriod: 10 * time.Minute, - CleanupTimeout: 60 * time.Second, - DefaultMatchers: Enabled{true}, - DefaultIndexers: Enabled{true}, - Scope: "node", - AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), - } +func (k *kubeAnnotatorConfig) InitDefaults() { + k.SyncPeriod = 10 * time.Minute + k.CleanupTimeout = 60 * time.Second + k.DefaultMatchers = Enabled{true} + k.DefaultIndexers = Enabled{true} + k.Scope = "node" + k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig() } func (k *kubeAnnotatorConfig) Validate() error { @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error { err := matcherCfg.Unpack(&logsPathMatcher) if err != nil { - return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err) + return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err) } if logsPathMatcher.LogsPath == "" { return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well") diff --git a/libbeat/processors/add_kubernetes_metadata/config_test.go b/libbeat/processors/add_kubernetes_metadata/config_test.go index e94089f388a..3857eb148fa 100644 --- a/libbeat/processors/add_kubernetes_metadata/config_test.go +++ b/libbeat/processors/add_kubernetes_metadata/config_test.go @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) { for _, test := range tests { cfg := config.MustNewConfigFrom(test.cfg) - c := defaultKubernetesAnnotatorConfig() + var c kubeAnnotatorConfig err := cfg.Unpack(&c) if test.error { @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) { for _, test := range tests { cfg, _ := config.NewConfigFrom(test.matcherConfig) - c := defaultKubernetesAnnotatorConfig() - c.DefaultMatchers = Enabled{false} + var c kubeAnnotatorConfig - err := cfg.Unpack(&c) + _ = cfg.Unpack(&c) + c.DefaultMatchers = Enabled{false} c.Matchers = PluginConfig{ { test.matcherName: *cfg, }, } - err = c.Validate() + err := c.Validate() if test.error { require.NotNil(t, err) } else { diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index a8667aef0a8..954a59ab3f1 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) { } func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) { - config := defaultKubernetesAnnotatorConfig() - + var config kubeAnnotatorConfig err := cfg.Unpack(&config) if err != nil { return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 1e02b3e9940..7df6f8fdaac 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -232,11 +232,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -276,13 +283,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index fc79ddb514c..93682b091a6 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -504,11 +504,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -548,13 +555,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index dcd59309119..23f5ce8dea6 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -18,11 +18,18 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -62,13 +69,21 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 44ef19c9786..1c56e57b167 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -16,15 +16,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 26728fccdae..60b3360ab89 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - // GetPodMetaGen requires cfg of type Config + // commonMetaConfig stores the metadata configuration of the resource itself commonMetaConfig := metadata.Config{} if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { logp.Err("Error initializing Kubernetes metadata enricher: %s", err) @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher( // update func(m map[string]mapstr.M, r kubernetes.Resource) { accessor, _ := meta.Accessor(r) - id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all + id := join(accessor.GetNamespace(), accessor.GetName()) switch r := r.(type) { case *kubernetes.Pod: @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher( return &nilEnricher{} } + // commonMetaConfig stores the metadata configuration of the resource itself + commonMetaConfig := metadata.Config{} + if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { + logp.Err("Error initializing Kubernetes metadata enricher: %s", err) + return &nilEnricher{} + } + cfg, _ := conf.NewConfigFrom(&commonMetaConfig) + // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher( } } - commonMetaConfig := metadata.Config{} - if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { - logp.Err("Error initializing Kubernetes metadata enricher: %s", err) - return &nilEnricher{} - } - cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata) enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { SyncPeriod: time.Minute * 10, AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), } + if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.New("error unpacking configs") } diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 7a1fb30fb8a..25f031971ad 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -19,15 +19,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 436693bdfbc..fe40dee0dec 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -901,11 +901,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -945,13 +952,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: