Skip to content

Commit

Permalink
Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to …
Browse files Browse the repository at this point in the history 
…init default configuration (#37458)

* use InitDefaults(); elastic-agent-autodiscover: fix default resource metadata config - elastic/elastic-agent-autodiscover#72

Signed-off-by: Tetiana Kravchenko <[email protected]>

* revert unintended changes

Signed-off-by: Tetiana Kravchenko <[email protected]>

* upgrade elastic-agent-autodiscover lib to v0.6.6

Signed-off-by: Tetiana Kravchenko <[email protected]>

* run make update; fix linter issues

Signed-off-by: Tetiana Kravchenko <[email protected]>

* revert changes related to the 'commonMetaConfig'; add missing metadata configuration on the metricset level

Signed-off-by: Tetiana Kravchenko <[email protected]>

* x-pack/metricbeat: run make update

Signed-off-by: Tetiana Kravchenko <[email protected]>

---------

Signed-off-by: Tetiana Kravchenko <[email protected]>
(cherry picked from commit 8c4a40f)

# Conflicts:
#	NOTICE.txt
#	go.mod
#	go.sum
  • Loading branch information
@tetianakravchenko @mergify
tetianakravchenko authored and mergify[bot] committed Dec 27, 2023
1 parent 24ef011 commit 9b91bfc
Show file tree
Hide file tree
Showing 13 changed files with 141 additions and 49 deletions.
8 changes: 8 additions & 0 deletions NOTICE.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12268,11 +12268,19 @@ SOFTWARE.

--------------------------------------------------------------------------------
Dependency : github.com/elastic/elastic-agent-autodiscover
<<<<<<< HEAD
Version: v0.6.4
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
=======
Version: v0.6.6
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
>>>>>>> 8c4a40fdbf (Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458))

Apache License
Version 2.0, January 2004
Expand Down
6 changes: 5 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,13 @@ require (
github.com/aws/smithy-go v1.13.5
github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
github.com/elastic/bayeux v1.0.5
<<<<<<< HEAD
github.com/elastic/elastic-agent-autodiscover v0.6.4
github.com/elastic/elastic-agent-libs v0.6.2
=======
github.com/elastic/elastic-agent-autodiscover v0.6.6
github.com/elastic/elastic-agent-libs v0.7.2
>>>>>>> 8c4a40fdbf (Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458))
github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
github.com/elastic/elastic-agent-system-metrics v0.8.2
github.com/elastic/go-elasticsearch/v8 v8.10.0
Expand Down Expand Up @@ -406,7 +411,6 @@ replace (
github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19
github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15

)

// Exclude this version because the version has an invalid checksum.
Expand Down
9 changes: 9 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,12 +651,21 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0=
github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68=
github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
<<<<<<< HEAD
github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA=
github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M=
github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo=
github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0=
github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3+oKIQDzdPDo=
github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700=
=======
github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A=
github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o=
github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc=
github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM=
github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o=
github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc=
>>>>>>> 8c4a40fdbf (Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458))
github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
github.com/elastic/elastic-agent-system-metrics v0.8.2 h1:99H1o+WTD/g6jcQr1hUOZ+dsZkZF0SfNePmjGg7TnhU=
Expand Down
18 changes: 8 additions & 10 deletions libbeat/processors/add_kubernetes_metadata/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,13 @@ type Enabled struct {

type PluginConfig []map[string]config.C

func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig {
return kubeAnnotatorConfig{
SyncPeriod: 10 * time.Minute,
CleanupTimeout: 60 * time.Second,
DefaultMatchers: Enabled{true},
DefaultIndexers: Enabled{true},
Scope: "node",
AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
}
func (k *kubeAnnotatorConfig) InitDefaults() {
k.SyncPeriod = 10 * time.Minute
k.CleanupTimeout = 60 * time.Second
k.DefaultMatchers = Enabled{true}
k.DefaultIndexers = Enabled{true}
k.Scope = "node"
k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig()
}

func (k *kubeAnnotatorConfig) Validate() error {
Expand All @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error {

err := matcherCfg.Unpack(&logsPathMatcher)
if err != nil {
return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err)
return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err)
}
if logsPathMatcher.LogsPath == "" {
return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well")
Expand Down
10 changes: 5 additions & 5 deletions libbeat/processors/add_kubernetes_metadata/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) {

for _, test := range tests {
cfg := config.MustNewConfigFrom(test.cfg)
c := defaultKubernetesAnnotatorConfig()
var c kubeAnnotatorConfig

err := cfg.Unpack(&c)
if test.error {
Expand Down Expand Up @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) {
for _, test := range tests {
cfg, _ := config.NewConfigFrom(test.matcherConfig)

c := defaultKubernetesAnnotatorConfig()
c.DefaultMatchers = Enabled{false}
var c kubeAnnotatorConfig

err := cfg.Unpack(&c)
_ = cfg.Unpack(&c)
c.DefaultMatchers = Enabled{false}
c.Matchers = PluginConfig{
{
test.matcherName: *cfg,
},
}
err = c.Validate()
err := c.Validate()
if test.error {
require.NotNil(t, err)
} else {
Expand Down
3 changes: 1 addition & 2 deletions libbeat/processors/add_kubernetes_metadata/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) {
}

func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) {
config := defaultKubernetesAnnotatorConfig()

var config kubeAnnotatorConfig
err := cfg.Unpack(&config)
if err != nil {
return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err)
Expand Down
23 changes: 19 additions & 4 deletions metricbeat/docs/modules/kubernetes.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,11 +232,18 @@ metricbeat.modules:
# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true
# When used outside the cluster:
#node: node_name
# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -276,13 +283,21 @@ metricbeat.modules:
# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true
# When used outside the cluster:
#node: node_name
# Set the namespace to watch for resources
#namespace: staging
# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
23 changes: 19 additions & 4 deletions metricbeat/metricbeat.reference.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,11 +504,18 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -548,13 +555,21 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
23 changes: 19 additions & 4 deletions metricbeat/module/kubernetes/_meta/config.reference.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,18 @@

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -62,13 +69,21 @@

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
12 changes: 9 additions & 3 deletions metricbeat/module/kubernetes/_meta/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,21 @@

# Enriching parameters:
#add_metadata: true
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
20 changes: 11 additions & 9 deletions metricbeat/module/kubernetes/util/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher(
return &nilEnricher{}
}

// GetPodMetaGen requires cfg of type Config
// commonMetaConfig stores the metadata configuration of the resource itself
commonMetaConfig := metadata.Config{}
if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
Expand Down Expand Up @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher(
// update
func(m map[string]mapstr.M, r kubernetes.Resource) {
accessor, _ := meta.Accessor(r)
id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all
id := join(accessor.GetNamespace(), accessor.GetName())

switch r := r.(type) {
case *kubernetes.Pod:
Expand Down Expand Up @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher(
return &nilEnricher{}
}

// commonMetaConfig stores the metadata configuration of the resource itself
commonMetaConfig := metadata.Config{}
if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
return &nilEnricher{}
}
cfg, _ := conf.NewConfigFrom(&commonMetaConfig)

// Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to
// in order to be able to retrieve 2nd layer Owner metadata like in case of:
// Deployment -> Replicaset -> Pod
Expand All @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher(
}
}

commonMetaConfig := metadata.Config{}
if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
return &nilEnricher{}
}
cfg, _ := conf.NewConfigFrom(&commonMetaConfig)

metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata)

enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher,
Expand Down Expand Up @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) {
SyncPeriod: time.Minute * 10,
AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
}

if err := base.Module().UnpackConfig(&config); err != nil {
return nil, errors.New("error unpacking configs")
}
Expand Down
12 changes: 9 additions & 3 deletions metricbeat/modules.d/kubernetes.yml.disabled
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,21 @@

# Enriching parameters:
#add_metadata: true
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
Loading

0 comments on commit 9b91bfc

Please sign in to comment.