feat: support multiple configurations via configmaps

controllers/apply.go

@@ -38,6 +38,7 @@ import (
 	"github.com/instana/instana-agent-operator/pkg/k8s/operator/operator_utils"
 	"github.com/instana/instana-agent-operator/pkg/k8s/operator/status"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
 func getDaemonSetBuilders(
@@ -87,14 +88,16 @@ func (r *InstanaAgentReconciler) applyResources(
 	statusManager status.AgentStatusManager,
 	keysSecret *corev1.Secret,
 	k8SensorBackends []backends.K8SensorBackend,
+	apiClient v1.CoreV1Interface,
 ) reconcileReturn {
 	log := r.loggerFor(ctx, agent)
 	log.V(1).Info("applying Kubernetes resources for agent")
+	configMerger := agentsecrets.NewConfigMergerBuilder(apiClient)
 
 	builders := append(
 		getDaemonSetBuilders(agent, isOpenShift, statusManager),
 		headlessservice.NewHeadlessServiceBuilder(agent),
-		agentsecrets.NewConfigBuilder(agent, statusManager, keysSecret, k8SensorBackends),
+		agentsecrets.NewConfigBuilder(agent, statusManager, keysSecret, k8SensorBackends, configMerger),
 		agentsecrets.NewContainerBuilder(agent, keysSecret),
 		tlssecret.NewSecretBuilder(agent),
 		service.NewServiceBuilder(agent),

controllers/instanaagent_controller.go

@@ -25,6 +25,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -38,6 +39,7 @@ import (
 	"github.com/instana/instana-agent-operator/pkg/k8s/operator/status"
 	"github.com/instana/instana-agent-operator/pkg/multierror"
 	"github.com/instana/instana-agent-operator/pkg/recovery"
+	"k8s.io/client-go/rest"
 )
 
 const (
@@ -132,6 +134,9 @@ func (r *InstanaAgentReconciler) reconcile(
 
 	k8SensorBackends := r.getK8SensorBackends(agent)
 
+	clientConfig, _ := rest.InClusterConfig()
+	clientSet, _ := kubernetes.NewForConfig(clientConfig)
+
 	if applyResourcesRes := r.applyResources(
 		ctx,
 		agent,
@@ -140,6 +145,7 @@ func (r *InstanaAgentReconciler) reconcile(
 		statusManager,
 		keysSecret,
 		k8SensorBackends,
+		clientSet.CoreV1(),
 	); applyResourcesRes.suppliesReconcileResult() {
 		return applyResourcesRes
 	}

pkg/k8s/object/builders/agent/secrets/config.go

@@ -30,19 +30,21 @@ type configBuilder struct {
 	keysSecret    *corev1.Secret
 	logger        logr.Logger
 	backends      []backends.K8SensorBackend
+	configMerger  ConfigMerger
 }
 
 func NewConfigBuilder(
 	agent *instanav1.InstanaAgent,
 	statusManager status.AgentStatusManager,
 	keysSecret *corev1.Secret,
-	backends []backends.K8SensorBackend) commonbuilder.ObjectBuilder {
+	backends []backends.K8SensorBackend, configMerger ConfigMerger) commonbuilder.ObjectBuilder {
 	return &configBuilder{
 		InstanaAgent:  agent,
 		statusManager: statusManager,
 		keysSecret:    keysSecret,
 		logger:        logf.Log.WithName("instana-agent-config-secret-builder"),
 		backends:      backends,
+		configMerger:  configMerger,
 	}
 }
 
@@ -85,7 +87,7 @@ func (c *configBuilder) data() (map[string][]byte, error) {
 		data["cluster_name"] = []byte(c.Spec.Cluster.Name)
 	}
 	if c.Spec.Agent.ConfigurationYaml != "" {
-		data["configuration.yaml"] = []byte(c.Spec.Agent.ConfigurationYaml)
+		data["configuration.yaml"] = c.configMerger.MergeConfigurationYaml(c.Spec.Agent.ConfigurationYaml)
 	}
 	if otlp := c.Spec.OpenTelemetry; otlp.IsEnabled() {
 		mrshl, _ := yaml.Marshal(map[string]instanav1.OpenTelemetry{"com.instana.plugin.opentelemetry": otlp})

pkg/k8s/object/builders/agent/secrets/config_merger.go

@@ -0,0 +1,87 @@
+/*
+(c) Copyright IBM Corp. 2024
+*/
+
+package secrets
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/go-logr/logr"
+	"gopkg.in/yaml.v3"
+	apiV1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const ConfigMapLabel = "instana.io/agent-config=true"
+
+type ConfigMerger interface {
+	MergeConfigurationYaml(string) []byte
+}
+
+type DefaultConfigMerger struct {
+	logger    logr.Logger
+	k8sClient v1.CoreV1Interface
+}
+
+func NewConfigMergerBuilder(client v1.CoreV1Interface) DefaultConfigMerger {
+	return DefaultConfigMerger{
+		logger:    logf.Log.WithName("instana-agent-config-merger"),
+		k8sClient: client,
+	}
+}
+
+func (c DefaultConfigMerger) MergeConfigurationYaml(agentConfiguration string) []byte {
+	agentData := make(map[string]interface{})
+	err := yaml.Unmarshal([]byte([]byte(agentConfiguration)), agentData)
+	config := []byte{}
+	if err != nil {
+		c.logger.Error(err, "Failed to load agent configuration")
+	} else {
+		configMaps := c.fetchConfigMaps()
+		for _, configMap := range configMaps {
+			configMapData := make(map[string]interface{})
+			err := yaml.Unmarshal([]byte(configMap.Data["configuration_yaml"]), &configMapData)
+			if err != nil {
+				c.logger.Error(err, "Failed to parse agent configuration YAML")
+			} else {
+				agentData = c.mergeConfig(agentData, configMapData)
+			}
+		}
+		config, err = yaml.Marshal(agentData)
+	}
+	return config
+}
+
+func (c DefaultConfigMerger) mergeConfig(agentData, configMapData map[string]interface{}) map[string]interface{} {
+	for key, configMapValue := range configMapData {
+		if agentValue, ok := agentData[key]; ok {
+			agentValueKind := reflect.TypeOf(agentValue).Kind()
+			if agentValueKind == reflect.Array || agentValueKind == reflect.Slice {
+				agentData[key] = append(agentValue.([]interface{}), configMapValue.([]interface{})...)
+			} else {
+				c.mergeConfig(agentData[key].(map[string]interface{}), configMapValue.(map[string]interface{}))
+			}
+		} else {
+			agentData[key] = configMapValue
+		}
+	}
+	return agentData
+}
+
+func (c DefaultConfigMerger) fetchConfigMaps() []apiV1.ConfigMap {
+	configMaps := []apiV1.ConfigMap{}
+	c.logger.Info(fmt.Sprintf("Fetching agent configmaps with label '%s'", ConfigMapLabel))
+	configMapList, err := c.k8sClient.ConfigMaps("").List(context.TODO(), metav1.ListOptions{LabelSelector: ConfigMapLabel})
+	if err != nil {
+		c.logger.Error(err, fmt.Sprintf("Failed to fetch agent configmaps with label '%s'", ConfigMapLabel))
+	} else {
+		configMaps = configMapList.Items
+		c.logger.Info(fmt.Sprintf("Found %d configmaps with label '%s'", len(configMaps), ConfigMapLabel))
+	}
+	return configMaps
+}

pkg/k8s/object/builders/agent/secrets/config_merger_test.go

@@ -0,0 +1,133 @@
+/*
+(c) Copyright IBM Corp. 2024
+*/
+
+package secrets
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	apiV1 "k8s.io/api/core/v1"
+	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	coreV1 "k8s.io/client-go/kubernetes/typed/core/v1"
+)
+
+type CoreV1Mock struct {
+	coreV1.CoreV1Interface
+	mock.Mock
+}
+
+type ConfigMapMock struct {
+	coreV1.ConfigMapInterface
+	mock.Mock
+}
+
+func (mock *CoreV1Mock) ConfigMaps(namespace string) coreV1.ConfigMapInterface {
+	args := mock.Called()
+	return args.Get(0).(*ConfigMapMock)
+}
+
+func (mock *ConfigMapMock) List(ctx context.Context, opts metaV1.ListOptions) (*apiV1.ConfigMapList, error) {
+	args := mock.Called()
+	return args.Get(0).(*apiV1.ConfigMapList), nil
+}
+
+func TestMergeConfigurationYamlWithNoConfigMaps(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMapsList := new(apiV1.ConfigMapList)
+	configMapsList.Items = []apiV1.ConfigMap{}
+	configMaps.On("List").Return(configMapsList).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("c: d\n")
+
+	assert.Equal(t, "c: d\n", string(config_bytes))
+}
+
+func TestMergeConfigurationYamlWithOtherConfigMapData(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMapsList := new(apiV1.ConfigMapList)
+	configMapsList.Items = []apiV1.ConfigMap{{Data: map[string]string{"a": "b"}}}
+	configMaps.On("List").Return(configMapsList).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("c: d\n")
+
+	assert.Equal(t, "c: d\n", string(config_bytes))
+}
+
+func TestMergeConfigurationYamlWithEmptyConfigMapData(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMapsList := new(apiV1.ConfigMapList)
+	configMapsList.Items = []apiV1.ConfigMap{{Data: map[string]string{"configuration_yaml": ""}}}
+	configMaps.On("List").Return(configMapsList).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("c: d\n")
+
+	assert.Equal(t, "c: d\n", string(config_bytes))
+}
+
+func TestMergeConfigurationYamlWithNewTopLevelKey(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMapsList := new(apiV1.ConfigMapList)
+	configMapsList.Items = []apiV1.ConfigMap{{Data: map[string]string{"configuration_yaml": "a:\n b:\n  - 1\n"}}}
+	configMaps.On("List").Return(configMapsList).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("c: d\n")
+
+	assert.Equal(t, "a:\n    b:\n        - 1\nc: d\n", string(config_bytes))
+}
+
+func TestMergeConfigurationYamlWithNewListItem(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMapsList := new(apiV1.ConfigMapList)
+	configMapsList.Items = []apiV1.ConfigMap{{Data: map[string]string{"configuration_yaml": "a:\n b:\n  - 2\n"}}}
+	configMaps.On("List").Return(configMapsList).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("a:\n b:\n  - 1\nc: d\n")
+
+	assert.Equal(t, "a:\n    b:\n        - 1\n        - 2\nc: d\n", string(config_bytes))
+}
+
+func TestMergeConfigurationYamlWithMultipleNewListItems(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMapsList := new(apiV1.ConfigMapList)
+	configMapsList.Items = []apiV1.ConfigMap{{Data: map[string]string{"configuration_yaml": "a:\n b:\n  - 2\n  - 3\n"}}}
+	configMaps.On("List").Return(configMapsList).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("a:\n b:\n  - 1\nc: d\n")
+
+	assert.Equal(t, "a:\n    b:\n        - 1\n        - 2\n        - 3\nc: d\n", string(config_bytes))
+}
+
+func TestMergeConfigurationYamlForFailedRetrieval(t *testing.T) {
+	client := new(CoreV1Mock)
+	configMaps := new(ConfigMapMock)
+	client.On("ConfigMaps").Return(configMaps).Once()
+	configMaps.On("List").Return(&apiV1.ConfigMapList{}, errors.New("Failed")).Once()
+
+	merger := NewConfigMergerBuilder(client)
+	config_bytes := merger.MergeConfigurationYaml("a:\n b:\n  - 1\nc: d\n")
+
+	assert.Equal(t, "a:\n    b:\n        - 1\nc: d\n", string(config_bytes))
+}

pkg/k8s/object/builders/agent/secrets/config_test.go

@@ -12,23 +12,41 @@ import (
 	backend "github.com/instana/instana-agent-operator/pkg/k8s/object/builders/common/backends"
 	"github.com/instana/instana-agent-operator/pkg/pointer"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	gomock "go.uber.org/mock/gomock"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const ConfigurationYamlValue = "configuration-yaml-value"
+
+type ConfigMergerMock struct {
+	ConfigMerger
+	mock.Mock
+}
+
+func (mock *ConfigMergerMock) MergeConfigurationYaml(agentConfig string) []byte {
+	args := mock.Called()
+	return args.Get(0).([]byte)
+}
+
+func mockConfigMerger() ConfigMerger {
+	configMerger := new(ConfigMergerMock)
+	configMerger.On("MergeConfigurationYaml").Return([]byte(ConfigurationYamlValue))
+	return configMerger
+}
+
 func TestConfigBuilderComponentName(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	statusManager := mocks.NewMockAgentStatusManager(ctrl)
-	s := NewConfigBuilder(&instanav1.InstanaAgent{}, statusManager, &corev1.Secret{}, make([]backend.K8SensorBackend, 0))
-
+	s := NewConfigBuilder(&instanav1.InstanaAgent{}, statusManager, &corev1.Secret{}, make([]backend.K8SensorBackend, 0), mockConfigMerger())
 	assert.True(t, s.IsNamespaced())
 }
 
 func TestConfigBuilderIsNamespaced(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	statusManager := mocks.NewMockAgentStatusManager(ctrl)
-	s := NewConfigBuilder(&instanav1.InstanaAgent{}, statusManager, &corev1.Secret{}, make([]backend.K8SensorBackend, 0))
+	s := NewConfigBuilder(&instanav1.InstanaAgent{}, statusManager, &corev1.Secret{}, make([]backend.K8SensorBackend, 0), mockConfigMerger())
 
 	assert.Equal(t, "instana-agent", s.ComponentName())
 }
@@ -93,7 +111,7 @@ func TestAgentSecretConfigBuild(t *testing.T) {
 						EndpointHost:      "main-backend-host",
 						EndpointPort:      "main-backend-port",
 						Key:               "main-backend-key",
-						ConfigurationYaml: "configuration-yaml-value",
+						ConfigurationYaml: ConfigurationYamlValue,
 						ProxyHost:         "proxy-host-value",
 						ProxyPort:         "proxy-port-value",
 						ProxyUser:         "proxy-user-value",
@@ -115,7 +133,7 @@ func TestAgentSecretConfigBuild(t *testing.T) {
 			keysSecret: &corev1.Secret{},
 			expected: map[string][]byte{
 				"cluster_name":                                 []byte(objectMeta.Name),
-				"configuration.yaml":                           []byte("configuration-yaml-value"),
+				"configuration.yaml":                           []byte(ConfigurationYamlValue),
 				"configuration-opentelemetry.yaml":             []byte("com.instana.plugin.opentelemetry:\n    grpc: {}\n"),
 				"configuration-prometheus-remote-write.yaml":   []byte("com.instana.plugin.prometheus:\n    remote_write:\n        enabled: true\n"),
 				"configuration-disable-kubernetes-sensor.yaml": []byte("com.instana.plugin.kubernetes:\n    enabled: false\n"),
@@ -386,7 +404,7 @@ func TestAgentSecretConfigBuild(t *testing.T) {
 				statusManager := mocks.NewMockAgentStatusManager(ctrl)
 				statusManager.EXPECT().SetAgentSecretConfig(gomock.Any()).AnyTimes()
 
-				builder := NewConfigBuilder(&test.agent, statusManager, test.keysSecret, test.k8sBackends)
+				builder := NewConfigBuilder(&test.agent, statusManager, test.keysSecret, test.k8sBackends, mockConfigMerger())
 
 				actual := builder.Build().Get()