diff --git a/apis/apps/v1alpha1/apimanager_types.go b/apis/apps/v1alpha1/apimanager_types.go index 9b74b0f6f..a78c0649b 100644 --- a/apis/apps/v1alpha1/apimanager_types.go +++ b/apis/apps/v1alpha1/apimanager_types.go @@ -1346,6 +1346,123 @@ func (apimanager *APIManager) IsAsyncDisableAnnotationPresent() bool { return asyncDisabledFound } +func (a *APIManager) GetApicastHTTPSCertSecretRefs() []*v1.LocalObjectReference { + secretRefs := []*v1.LocalObjectReference{} + + if a.Spec.Apicast != nil && a.Spec.Apicast.ProductionSpec != nil && a.Spec.Apicast.ProductionSpec.HTTPSCertificateSecretRef != nil { + secretRefs = append(secretRefs, a.Spec.Apicast.ProductionSpec.HTTPSCertificateSecretRef) + } + + if a.Spec.Apicast != nil && a.Spec.Apicast.StagingSpec != nil && a.Spec.Apicast.StagingSpec.HTTPSCertificateSecretRef != nil { + secretRefs = append(secretRefs, a.Spec.Apicast.StagingSpec.HTTPSCertificateSecretRef) + } + + return secretRefs +} + +func (a *APIManager) GetApicastOpenTelemetrySecretRefs() []*v1.LocalObjectReference { + secretRefs := []*v1.LocalObjectReference{} + + if a.Spec.Apicast != nil && a.Spec.Apicast.ProductionSpec != nil && a.Spec.Apicast.ProductionSpec.OpenTelemetry != nil && a.Spec.Apicast.ProductionSpec.OpenTelemetry.TracingConfigSecretRef != nil { + secretRefs = append(secretRefs, a.Spec.Apicast.ProductionSpec.OpenTelemetry.TracingConfigSecretRef) + } + + if a.Spec.Apicast != nil && a.Spec.Apicast.StagingSpec != nil && a.Spec.Apicast.StagingSpec.OpenTelemetry != nil && a.Spec.Apicast.StagingSpec.OpenTelemetry.TracingConfigSecretRef != nil { + secretRefs = append(secretRefs, a.Spec.Apicast.StagingSpec.OpenTelemetry.TracingConfigSecretRef) + } + + return secretRefs +} + +func (a *APIManager) GetApicastCustomEnvironmentsSecretRefs() []*v1.LocalObjectReference { + secretRefs := []*v1.LocalObjectReference{} + + if a.Spec.Apicast.ProductionSpec.CustomEnvironments != nil { + for _, env := range a.Spec.Apicast.ProductionSpec.CustomEnvironments { + if env.SecretRef != nil { + secretRefs = append(secretRefs, env.SecretRef) + } + } + } + + if a.Spec.Apicast.StagingSpec.CustomEnvironments != nil { + for _, env := range a.Spec.Apicast.StagingSpec.CustomEnvironments { + if env.SecretRef != nil { + secretRefs = append(secretRefs, env.SecretRef) + } + } + } + + return secretRefs +} + +func (a *APIManager) GetApicastCustomPoliciesSecretRefs() []*v1.LocalObjectReference { + secretRefs := []*v1.LocalObjectReference{} + + if a.Spec.Apicast.ProductionSpec.CustomPolicies != nil { + for _, policy := range a.Spec.Apicast.ProductionSpec.CustomPolicies { + if policy.SecretRef != nil { + secretRefs = append(secretRefs, policy.SecretRef) + } + } + } + + if a.Spec.Apicast.StagingSpec.CustomPolicies != nil { + for _, policy := range a.Spec.Apicast.StagingSpec.CustomPolicies { + if policy.SecretRef != nil { + secretRefs = append(secretRefs, policy.SecretRef) + } + } + } + + return secretRefs +} + +func (apimanager *APIManager) Get3scaleSecretRefs() []*v1.LocalObjectReference { + secretRefs := []*v1.LocalObjectReference{} + + // TODO: Add TLS Secrets and ACL Secrets once support for them is implemented + + apicastHTTPSCertSecretRefs := apimanager.GetApicastHTTPSCertSecretRefs() + if len(apicastHTTPSCertSecretRefs) > 0 { + secretRefs = append(secretRefs, apicastHTTPSCertSecretRefs...) + } + + apicastOpenTelemetrySecretRefs := apimanager.GetApicastOpenTelemetrySecretRefs() + if len(apicastOpenTelemetrySecretRefs) > 0 { + secretRefs = append(secretRefs, apicastOpenTelemetrySecretRefs...) + } + + apicastCustomEnvironmentSecretRefs := apimanager.GetApicastCustomEnvironmentsSecretRefs() + if len(apicastCustomEnvironmentSecretRefs) > 0 { + secretRefs = append(secretRefs, apicastCustomEnvironmentSecretRefs...) + } + + apicastCustomPoliciesSecretRefs := apimanager.GetApicastCustomPoliciesSecretRefs() + if len(apicastCustomPoliciesSecretRefs) > 0 { + secretRefs = append(secretRefs, apicastCustomPoliciesSecretRefs...) + } + + secretRefs = removeDuplicateSecretRefs(secretRefs) + + return secretRefs +} + +func removeDuplicateSecretRefs(refs []*v1.LocalObjectReference) []*v1.LocalObjectReference { + nameMap := make(map[string]bool) + uniqueRefs := make([]*v1.LocalObjectReference, 0) + + for _, ref := range refs { + if ref != nil { + if _, exists := nameMap[ref.Name]; !exists { + nameMap[ref.Name] = true + uniqueRefs = append(uniqueRefs, ref) + } + } + } + return uniqueRefs +} + func (apimanager *APIManager) Validate() field.ErrorList { fieldErrors := field.ErrorList{} diff --git a/apis/apps/v1alpha1/apimanager_types_test.go b/apis/apps/v1alpha1/apimanager_types_test.go index 2a0c5d07d..02b872ee4 100644 --- a/apis/apps/v1alpha1/apimanager_types_test.go +++ b/apis/apps/v1alpha1/apimanager_types_test.go @@ -7,6 +7,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/3scale/3scale-operator/version" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -162,3 +163,171 @@ func minimumAPIManagerTest() *APIManager { }, } } + +func TestRemoveDuplicateSecretRefs(t *testing.T) { + type args struct { + refs []*v1.LocalObjectReference + } + tests := []struct { + name string + args args + want []*v1.LocalObjectReference + }{ + { + name: "SecretRefs is nil", + args: args{ + refs: nil, + }, + want: []*v1.LocalObjectReference{}, + }, + { + name: "SecretRefs is empty", + args: args{ + refs: []*v1.LocalObjectReference{}, + }, + want: []*v1.LocalObjectReference{}, + }, + { + name: "SecretRefs has duplicates", + args: args{ + refs: []*v1.LocalObjectReference{ + { + Name: "ref1", + }, + { + Name: "ref1", + }, + { + Name: "ref2", + }, + }, + }, + want: []*v1.LocalObjectReference{ + { + Name: "ref1", + }, + { + Name: "ref2", + }, + }, + }, + { + name: "SecretRefs does not have duplicates", + args: args{ + refs: []*v1.LocalObjectReference{ + { + Name: "ref1", + }, + { + Name: "ref2", + }, + }, + }, + want: []*v1.LocalObjectReference{ + { + Name: "ref1", + }, + { + Name: "ref2", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := removeDuplicateSecretRefs(tt.args.refs); !reflect.DeepEqual(got, tt.want) { + t.Errorf("RemoveDuplicateSecretRefs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAPIManager_Get3scaleSecretRefs(t *testing.T) { + type fields struct { + Spec APIManagerSpec + } + tests := []struct { + name string + fields fields + want []*v1.LocalObjectReference + }{ + { + name: "No secret refs to gather", + fields: fields{ + Spec: APIManagerSpec{ + Apicast: &ApicastSpec{ + ProductionSpec: &ApicastProductionSpec{}, + StagingSpec: &ApicastStagingSpec{}, + }, + }, + }, + want: []*v1.LocalObjectReference{}, + }, + { + name: "Apicast has secret refs", + fields: fields{ + Spec: APIManagerSpec{ + Apicast: &ApicastSpec{ + ProductionSpec: &ApicastProductionSpec{ + HTTPSCertificateSecretRef: &v1.LocalObjectReference{ + Name: "https-cert-secret", + }, + OpenTelemetry: &OpenTelemetrySpec{ + TracingConfigSecretRef: &v1.LocalObjectReference{ + Name: "otel-secret", + }, + }, + CustomEnvironments: []CustomEnvironmentSpec{ + { + SecretRef: &v1.LocalObjectReference{ + Name: "custom-env-1-secret", + }, + }, + }, + }, + StagingSpec: &ApicastStagingSpec{ + CustomEnvironments: []CustomEnvironmentSpec{ + { + SecretRef: &v1.LocalObjectReference{ + Name: "custom-env-1-secret", + }, + }, + }, + CustomPolicies: []CustomPolicySpec{ + { + SecretRef: &v1.LocalObjectReference{ + Name: "custom-policy-1-secret", + }, + }, + }, + }, + }, + }, + }, + want: []*v1.LocalObjectReference{ + { + Name: "https-cert-secret", + }, + { + Name: "otel-secret", + }, + { + Name: "custom-env-1-secret", + }, + { + Name: "custom-policy-1-secret", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + apimanager := &APIManager{ + Spec: tt.fields.Spec, + } + if got := apimanager.Get3scaleSecretRefs(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Get3scaleSecretRefs() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index 5d1e24199..a75d7a1b3 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "time" "github.com/3scale/3scale-operator/pkg/upgrade" @@ -488,6 +489,12 @@ func (r *APIManagerReconciler) reconcileAPIManagerLogic(cr *appsv1alpha1.APIMana return result, err } + // Create the hashed secret to track watched secrets' changes + result, err = r.reconcileHashedSecret(cr) + if err != nil || result.Requeue { + return result, err + } + // 3scale 2.14 -> 2.15 err = upgrade.DeleteImageStreams(cr.Namespace, r.Client()) if err != nil { @@ -650,3 +657,25 @@ func (r *APIManagerReconciler) setRequirementsAnnotation(apim *appsv1alpha1.APIM return nil } + +func (r *APIManagerReconciler) reconcileHashedSecret(cr *appsv1alpha1.APIManager) (reconcile.Result, error) { + secretLabels := map[string]string{ + "app": *cr.Spec.AppLabel, + } + secret, err := component.HashedSecret(r.Context(), r.Client(), cr.Get3scaleSecretRefs(), cr.Namespace, secretLabels) + if err != nil { + r.Logger().Error(err, "failed to generate hashed-secret-data secret") + return reconcile.Result{}, err + } + + secretMutators := []reconcilers.SecretMutateFn{ + reconcilers.SecretStringDataMutator, + } + err = r.ReconcileResource(&v1.Secret{}, secret, reconcilers.DeploymentSecretMutator(secretMutators...)) + if err != nil { + r.Logger().Error(err, "failed to reconcile hashed-secret-data secret") + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} diff --git a/controllers/apps/apimanager_controller_test.go b/controllers/apps/apimanager_controller_test.go index 5f2291804..09747c974 100644 --- a/controllers/apps/apimanager_controller_test.go +++ b/controllers/apps/apimanager_controller_test.go @@ -3,8 +3,10 @@ package controllers import ( "context" "fmt" + "github.com/3scale/3scale-operator/pkg/3scale/amp/component" "io" "os" + "reflect" "time" "github.com/3scale/3scale-operator/apis/apps" @@ -112,6 +114,17 @@ var _ = Describe("APIManager controller", func() { return err == nil }, 5*time.Minute, 5*time.Second).Should(BeTrue()) + // Create custom environment secret + customEnvSecret := testGetCustomEnvironmentSecret(testNamespace) + + // Get the newly created custom environment secret for later + err = testK8sClient.Create(context.Background(), customEnvSecret) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() bool { + err := testK8sClient.Get(context.Background(), types.NamespacedName{Name: customEnvSecret.Name, Namespace: customEnvSecret.Namespace}, customEnvSecret) + return err == nil + }, 5*time.Minute, 5*time.Second).Should(BeTrue()) + enableResourceRequirements := false wildcardDomain := "test1.127.0.0.1.nip.io" apimanager := &appsv1alpha1.APIManager{ @@ -129,6 +142,26 @@ var _ = Describe("APIManager controller", func() { }, }, }, + Apicast: &appsv1alpha1.ApicastSpec{ + StagingSpec: &appsv1alpha1.ApicastStagingSpec{ + CustomEnvironments: []appsv1alpha1.CustomEnvironmentSpec{ + { + SecretRef: &v1.LocalObjectReference{ + Name: customEnvSecret.Name, + }, + }, + }, + }, + ProductionSpec: &appsv1alpha1.ApicastProductionSpec{ + CustomEnvironments: []appsv1alpha1.CustomEnvironmentSpec{ + { + SecretRef: &v1.LocalObjectReference{ + Name: customEnvSecret.Name, + }, + }, + }, + }, + }, }, ObjectMeta: metav1.ObjectMeta{ Name: "example-apimanager", @@ -154,6 +187,23 @@ var _ = Describe("APIManager controller", func() { Expect(err).ToNot(HaveOccurred()) fmt.Fprintf(GinkgoWriter, "All APIManager managed Routes are available\n") + fmt.Fprintf(GinkgoWriter, "Waiting until APIManager CR has the correct secret UIDs\n") + err = waitForAPIManagerLabels(testNamespace, 5*time.Second, 5*time.Minute, apimanager, customEnvSecret, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + fmt.Fprintf(GinkgoWriter, "APIManager CR has the correct secret UIDs\n") + + fmt.Fprintf(GinkgoWriter, "Waiting until hashed secret has been created and is accurate\n") + err = waitForHashedSecret(testNamespace, 5*time.Second, 5*time.Minute, customEnvSecret, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + fmt.Fprintf(GinkgoWriter, "Hashed secret has been created and is accurate\n") + + fmt.Fprintf(GinkgoWriter, "Waiting until apicast pod annotations have been verified\n") + err = waitForApicastPodAnnotations(testNamespace, 5*time.Second, 5*time.Minute, customEnvSecret, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + fmt.Fprintf(GinkgoWriter, "Apicast pod annotations have been verified\n") + + // TODO: Add code checking annotations on apicast pods + fmt.Fprintf(GinkgoWriter, "Waiting until APIManager's 'Available' condition is true\n") err = waitForAPIManagerAvailableCondition(testNamespace, 5*time.Second, 15*time.Minute, apimanager, GinkgoWriter) Expect(err).ToNot(HaveOccurred()) @@ -271,3 +321,118 @@ func waitForAPIManagerAvailableCondition(namespace string, retryInterval, timeou return nil } + +func waitForAPIManagerLabels(namespace string, retryInterval time.Duration, timeout time.Duration, apimanager *appsv1alpha1.APIManager, customEnvSecret *v1.Secret, w io.Writer) error { + Eventually(func() bool { + reconciledApimanager := &appsv1alpha1.APIManager{} + reconciledApimanagerKey := types.NamespacedName{Name: apimanager.Name, Namespace: namespace} + err := testK8sClient.Get(context.Background(), reconciledApimanagerKey, reconciledApimanager) + if err != nil { + fmt.Fprintf(w, "Error getting APIManager '%s': %v\n", apimanager.Name, err) + return false + } + + expectedLabels := map[string]string{ + fmt.Sprintf("%s%s", APImanagerSecretLabelPrefix, string(customEnvSecret.GetUID())): "true", + } + + // Then verify that the hash matches the hashed config secret + return reflect.DeepEqual(reconciledApimanager.Labels, expectedLabels) + }, timeout, retryInterval).Should(BeTrue()) + + return nil +} + +func waitForHashedSecret(namespace string, retryInterval time.Duration, timeout time.Duration, customEnvSecret *v1.Secret, w io.Writer) error { + Eventually(func() bool { + // First get the master hashed secret + hashedSecret := &v1.Secret{} + hashedSecretLookupKey := types.NamespacedName{Name: component.HashedSecretName, Namespace: namespace} + err := testK8sClient.Get(context.Background(), hashedSecretLookupKey, hashedSecret) + if err != nil { + fmt.Fprintf(w, "Error getting hashed secret '%s': %v\n", hashedSecretLookupKey.Name, err) + return false + } + + // Then verify that the hash matches the hashed custom environment secret + return helper.GetSecretStringDataFromData(hashedSecret.Data)[customEnvSecret.Name] == component.HashSecret(customEnvSecret.Data) + }, timeout, retryInterval).Should(BeTrue()) + + return nil +} + +func waitForApicastPodAnnotations(namespace string, retryInterval time.Duration, timeout time.Duration, customEnvSecret *v1.Secret, w io.Writer) error { + apicastDeploymentNames := []string{ + "apicast-production", + "apicast-staging", + } + + for _, dName := range apicastDeploymentNames { + apicastDeploymentLookupKey := types.NamespacedName{Name: dName, Namespace: namespace} + apicastDeployment := &k8sappsv1.Deployment{} + Eventually(func() bool { + err := testK8sClient.Get(context.Background(), apicastDeploymentLookupKey, apicastDeployment) + if err != nil { + return false + } + + for aKey, aValue := range apicastDeployment.Spec.Template.Annotations { + if aKey == fmt.Sprintf("%s%s", component.CustomEnvSecretResverAnnotationPrefix, customEnvSecret.Name) { + if aValue == customEnvSecret.ResourceVersion { + fmt.Fprintf(w, "Deployment '%s' has the custom env secret annotation and correct resourceVersion\n", dName) + return true + } + fmt.Fprintf(w, "Deployment '%s' has the custom env secret annotation but the resourceVersion '%s' doesn't match the expected value '%s'\n", dName, aValue, customEnvSecret.ResourceVersion) + return false + } + } + fmt.Fprintf(w, "Deployment '%s' doesn't have the custom env secret annotation\n", dName) + return false + }, timeout, retryInterval).Should(BeTrue()) + } + + return nil +} + +func testCustomEnvironmentContent() string { + return ` + local cjson = require('cjson') + local PolicyChain = require('apicast.policy_chain') + local policy_chain = context.policy_chain + + local logging_policy_config = cjson.decode([[ + { + "enable_access_logs": false, + "custom_logging": "\"{{request}}\" to service {{service.name}} and {{service.id}}" + } + ]]) + + policy_chain:insert( PolicyChain.load_policy('logging', 'builtin', logging_policy_config), 1) + + return { + policy_chain = policy_chain, + port = { metrics = 9421 }, + } +` +} + +func testGetCustomEnvironmentSecret(namespace string) *v1.Secret { + customEnvironmentSecret := v1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-env-1", + Namespace: namespace, + Labels: map[string]string{ + "apimanager.apps.3scale.net/watched-by": "apimanager", + }, + }, + StringData: map[string]string{ + "custom_env.lua": testCustomEnvironmentContent(), + "custom_env2.lua": testCustomEnvironmentContent(), + }, + } + return &customEnvironmentSecret +} diff --git a/controllers/apps/apimanager_status_reconciler.go b/controllers/apps/apimanager_status_reconciler.go index 2d7c92955..c5a883e06 100644 --- a/controllers/apps/apimanager_status_reconciler.go +++ b/controllers/apps/apimanager_status_reconciler.go @@ -3,6 +3,9 @@ package controllers import ( "context" "fmt" + "sort" + "strings" + appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" subController "github.com/3scale/3scale-operator/controllers/subscription" "github.com/3scale/3scale-operator/pkg/3scale/amp/component" @@ -10,9 +13,11 @@ import ( "github.com/3scale/3scale-operator/pkg/helper" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/version" + "github.com/RHsyseng/operator-utils/pkg/olm" "github.com/go-logr/logr" routev1 "github.com/openshift/api/route/v1" + k8sappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -20,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sort" ) type APIManagerStatusReconciler struct { @@ -90,7 +94,10 @@ func (s *APIManagerStatusReconciler) calculateStatus() (*appsv1alpha1.APIManager newStatus.Conditions = s.apimanagerResource.Status.Conditions.Copy() - availableCondition, err := s.apimanagerAvailableCondition(deployments) + // Check if any of the watched secrets are missing + watchedSecretsExist, watchedSecretsMessage := s.watchedSecretsExist(s.apimanagerResource) + + availableCondition, err := s.apimanagerAvailableCondition(deployments, watchedSecretsExist, watchedSecretsMessage) if err != nil { return nil, err } @@ -188,7 +195,7 @@ func (s *APIManagerStatusReconciler) existingDeployments() ([]k8sappsv1.Deployme return deployments, nil } -func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeployments []k8sappsv1.Deployment) (common.Condition, error) { +func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeployments []k8sappsv1.Deployment, watchedSecretsExist bool, missingSecretsMessage string) (common.Condition, error) { deploymentsAvailable := s.deploymentsAvailable(existingDeployments) defaultRoutesReady, err := s.defaultRoutesReady() @@ -201,11 +208,16 @@ func (s *APIManagerStatusReconciler) apimanagerAvailableCondition(existingDeploy Status: v1.ConditionFalse, } - s.logger.V(1).Info("Status apimanagerAvailableCondition", "deploymentsAvailable", deploymentsAvailable, "defaultRoutesReady", defaultRoutesReady) - if deploymentsAvailable && defaultRoutesReady { + s.logger.V(1).Info("Status apimanagerAvailableCondition", "deploymentsAvailable", deploymentsAvailable, "defaultRoutesReady", defaultRoutesReady, "watchedSecretsExist", watchedSecretsExist) + if deploymentsAvailable && defaultRoutesReady && watchedSecretsExist { newAvailableCondition.Status = v1.ConditionTrue } + if !watchedSecretsExist { + newAvailableCondition.Message = missingSecretsMessage + newAvailableCondition.Reason = "MissingWatchedSecrets" + } + return newAvailableCondition, nil } @@ -417,3 +429,32 @@ func (s *APIManagerStatusReconciler) reconcilePreflightsStatus(conditions *commo return nil } + +func (s *APIManagerStatusReconciler) watchedSecretsExist(cr *appsv1alpha1.APIManager) (bool, string) { + secretsToCheck := cr.Get3scaleSecretRefs() + if len(secretsToCheck) == 0 { + // Return because there are no watched secrets to check + return true, "" + } + + allWatchedSecretsExist := true + watchedSecretsMessage := "" + var missingSecretNames []string + + for _, secretRef := range secretsToCheck { + secret := &v1.Secret{} + secretKey := client.ObjectKey{Name: secretRef.Name, Namespace: cr.Namespace} + err := s.Client().Get(s.Context(), secretKey, secret) + if err != nil { + allWatchedSecretsExist = false + missingSecretNames = append(missingSecretNames, secretRef.Name) + } + } + + // If there are watched secrets that can't be found, add the warning condition + if len(missingSecretNames) > 0 { + watchedSecretsMessage = fmt.Sprintf("The following secret(s) could not be found: %s", strings.Join(missingSecretNames, ", ")) + } + + return allWatchedSecretsExist, watchedSecretsMessage +} diff --git a/doc/adding-apicast-custom-environments.md b/doc/adding-apicast-custom-environments.md index 8b09d532a..5f87402a7 100644 --- a/doc/adding-apicast-custom-environments.md +++ b/doc/adding-apicast-custom-environments.md @@ -1,8 +1,8 @@ ## Adding custom environments -Add custom environment loaded in all 3scale products. +Add custom environment(s) loaded in all 3scale products. -Here is an example of a policy that is loaded in all services: `custom_env.lua` +Here is an example of a environment that is loaded in all services: `custom_env.lua` ```lua local cjson = require('cjson') @@ -26,14 +26,24 @@ return { ### Prerequisites -* One or more custom environment in lua code. +* One or more custom environment(s) in lua code. ### Adding custom environment #### Create secret with the custom environment content ``` -oc create secret generic custom-env-1 --from-file=./env11.lua +oc create secret generic custom-env-1 --from-file=./custom_env.lua +``` + +By default, content changes in the secret will not be noticed by the 3scale operator. +The 3scale operator allows the monitoring of secret changes, this can be achieved by adding the +`apimanager.apps.3scale.net/watched-by=apimanager` label to the required secret. +With the label in place, when the content of the secret changes, the operator will update the deployment of the apicast +where that secret is used (staging or production). +The operator will not take *ownership* of the secret in any way. +``` +oc label secret custom-env-1 apimanager.apps.3scale.net/watched-by=apimanager ``` **NOTE**: a secret can host multiple custom environments. The operator will load each one of them. @@ -52,13 +62,13 @@ spec: productionSpec: customEnvironments: - secretRef: - name: env1 + name: custom-env-1 - secretRef: - name: env2 + name: custom-env-2 stagingSpec: customEnvironments: - secretRef: - name: env3 + name: custom-env-3 ``` **NOTE**: Multiple custom environment secrets can be added. The operator will load each one of them. @@ -71,13 +81,4 @@ oc apply -f apimanager.yaml The APIManager custom resource allows adding multiple custom environments per secret. -**NOTE**: If secret does not exist, the operator would mark the custom resource as failed. The Deployment object would fail if secret does not exist. - -*NOTE*: Once apicast has been deployed, the content of the secret should not be updated externally. -If the content of the secret is updated externally, after apicast has been deployed, the container can automatically see the changes. -However, apicast has the environment already loaded and it does not change the behavior. - -If the custom environment content needs to be changed, there are two options: - -* [**recommended way**] Create another secret with a different name and update the APIManager custom resource field `customEnvironments[].secretRef.name`. The operator will trigger a rolling update loading the new custom environment content. -* Update the existing secret content and redeploy apicast turning `spec.apicast.productionSpec.replicas` or `spec.apicast.stagingSpec.replicas` to 0 and then back to the previous value. +**NOTE**: If the referenced secret does not exist, the operator will mark the APIManager CustomResource as failed. The apicast Deployment object will also fail if the referenced secret does not exist. \ No newline at end of file diff --git a/doc/adding-custom-policies.md b/doc/adding-custom-policies.md index f7a2766f7..0349bc325 100644 --- a/doc/adding-custom-policies.md +++ b/doc/adding-custom-policies.md @@ -6,7 +6,7 @@ for more info about creating custom policies. ### Prerequisites -* Custom policy metadata included in `apicast-policy.json` +* Custom policy metadata included in `apicast-policy.json`: ```json { @@ -21,9 +21,9 @@ for more info about creating custom policies. } } ``` -* Custom policy lua code. `init.lua` file is required. Optionally, add more lua files. For this specific example, `init.lua` and `example.lua` files will be shown. +* Custom policy lua code. The `init.lua` file is required. Optionally, add more lua files. For this specific example, `init.lua` and `example.lua` files are used. -`init.lua` +`init.lua`: ```lua return require('example') ``` @@ -101,7 +101,7 @@ With the label in place, when the content of the secret changes, the operator wi where that secret is used (staging or production). The operator will not take *ownership* of the secret in any way. ``` -kubectl label secret custom-policy-example-1 apimanager.apps.3scale.net/watched-by=apimanager +oc label secret custom-policy-example-1 apimanager.apps.3scale.net/watched-by=apimanager ``` #### Configure and deploy APIManager CR with the custom policy @@ -120,21 +120,21 @@ spec: - name: custom-policy1 version: "0.1" secretRef: - name: custom-policy1-secret + name: custom-policy-example-1 - name: custom-policy2 version: "0.1" secretRef: - name: custom-policy2-secret + name: custom-policy-example-2 productionSpec: customPolicies: - name: custom-policy1 version: "0.1" secretRef: - name: custom-policy1-secret + name: custom-policy-example-1 - name: custom-policy2 version: "0.1" secretRef: - name: custom-policy2-secret + name: custom-policy-example-2 ``` ``` @@ -145,16 +145,7 @@ The APIManager custom resource allows adding multiple custom policies. **NOTE**: The tuple (`name`, `version`) has to be unique in the `customPolicies` array. -**NOTE**: If secret does not exist, the operator would mark the custom resource as failed. - -*NOTE*: Once the apicast has been deployed, the content of the secret should not be updated externally. -If the content of the secret is updated externally, after apicast has been deployed, the container can automatically see the changes. -However, apicast has the policy already loaded and it does not change the behavior. - -If the policy content needs to be changed, there are two options: - -* [recommended way] Create another secret with a different name and update the APIManager custom resource field `customPolicies[].secretRef.name`. The operator will trigger a rolling update loading the new policy content. -* Update the existing secret content and redeploy apicast turning `spec.apicast.productionSpec.replicas` (or `spec.apicast.stagingSpec.replicas`) to 0 and then back to the previous value. +**NOTE**: If secret does not exist, the operator will mark the APIManager CustomResource as failed. #### Add the custom policy metadata to 3scale policy registry @@ -219,7 +210,7 @@ spec: #### Adding the custom policy to a policy chain in 3scale Configure 3scale product to include the new custom policy in the gateway policy chain to be used by APIcast. -The custom policy needs to be added to the policy registry before adding it to some 3scale product's policy chain.. +The custom policy needs to be added to the policy registry before adding it to some 3scale product's policy chain. * Using the 3scale admin portal UI diff --git a/doc/apimanager-reference.md b/doc/apimanager-reference.md index f6dd9aca6..916a4fc54 100644 --- a/doc/apimanager-reference.md +++ b/doc/apimanager-reference.md @@ -214,7 +214,7 @@ By default, content changes in the secret will not be noticed by the 3scale oper The 3scale operator allows monitoring the secret for changes adding the `apimanager.apps.3scale.net/watched-by=apimanager` label. With that label in place, when the content of the secret is changed, the operator will get notified. Then, the operator will rollout apicast deployment to make the changes effective. -The operator will not take *ownership* of the secret in any way. +The operator will not take *ownership* of the watched secret in any way. ### APIcastTracingConfigSecret diff --git a/doc/development.md b/doc/development.md index 912f81627..6f542f4f4 100644 --- a/doc/development.md +++ b/doc/development.md @@ -19,7 +19,7 @@ * [Validate an operator bundle image](#validate-an-operator-bundle-image) * [Licenses management](#licenses-management) * [Adding manually a new license](#adding-manually-a-new-license) -* [Building and pushing 3scale component images](#building-and-pushing-3scale-component-images) +* [Adding new watched secrets](#adding-new-watched-secrets) Generated using [github-markdown-toc](https://github.com/ekalinin/github-markdown-toc) @@ -238,3 +238,22 @@ license_finder approval add github.com/golang/glog --decisions-file=doc/dependen [docker]:https://docs.docker.com/install/ [kubernetes]:https://kubernetes.io/ [oc]:https://github.com/openshift/origin/releases + +## Adding new watched secrets +After adding a new secret to the APIManager CRD make sure to also update the following files if you want the 3scale-operator to watch the new secret: +1. [apis/apps/v1alpha1/apimanager_types.go](../apis/apps/v1alpha1/apimanager_types.go) + - Add a new `GetXYZSecretRef()` function that returns the secret ref +2. [apis/apps/v1alpha1/apimanager_types.go](../apis/apps/v1alpha1/apimanager_types.go) + - Update the `Get3scaleSecretRefs()` to call the new `GetXYZSecretRef()` function from step 1 +3. [pkg/3scale/amp/operator/apicast_reconciler.go](../pkg/3scale/amp/operator/apicast_reconciler.go) + - Add the new secret to the `getSecretUIDs()` function +4. [pkg/3scale/amp/component/deployment_annotations.go](../pkg/3scale/amp/component/deployment_annotations.go) + - Add the new secret to the `getWatchedSecretAnnotations()` function +5. [pkg/3scale/amp/component/deployment_annotations.go](../pkg/3scale/amp/component/deployment_annotations.go) + - Add the new secret to the switch in the `HasSecretHashChanged()` function +6. pkg/3scale/amp/component/{component_name}.go + - Add a new const called `XYZSecretResverAnnotationPrefix` that can be referenced throughout the code + - The const should be in the `component` package but the exact file will depend on which deployment the new watched secret relates to. For example if the secret is relevant to the `apicast` deployments, the const belongs in [pkg/3scale/amp/component/apicast.go](../pkg/3scale/amp/component/apicast.go) +7. pkg/3scale/amp/component/{component_name}.go + - Add an annotation for the new watched secret to the relevant deployment's `.spec.template.metadata.annotations` + - The exact file that needs changing will depend on which deployment the new watched secret relates to. See the `StagingDeployment()` function in [pkg/3scale/amp/component/apicast.go](../pkg/3scale/amp/component/apicast.go) for an example diff --git a/pkg/3scale/amp/component/apicast.go b/pkg/3scale/amp/component/apicast.go index 42f579774..fa7685698 100644 --- a/pkg/3scale/amp/component/apicast.go +++ b/pkg/3scale/amp/component/apicast.go @@ -1,8 +1,10 @@ package component import ( + "context" "crypto/md5" "fmt" + "hash/fnv" "path" "sort" "strconv" @@ -16,6 +18,7 @@ import ( policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -48,6 +51,14 @@ const ( APIcastOpentelemetryConfigAnnotationPartialKey = "apps.3scale.net/" + OpentelemetryConfigurationVolumeName ) +const ( + APIcastEnvironmentCMAnnotation = "apimanager.apps.3scale.net/env-configmap-hash" + HttpsCertSecretResverAnnotationPrefix = "apimanager.apps.3scale.net/https-cert-secret-resource-version-" + OpenTelemetrySecretResverAnnotationPrefix = "apimanager.apps.3scale.net/opentelemetry-secret-resource-version-" + CustomEnvSecretResverAnnotationPrefix = "apimanager.apps.3scale.net/customenv-secret-resource-version-" + CustomPoliciesSecretResverAnnotationPrefix = "apimanager.apps.3scale.net/custompolicy-secret-resource-version-" +) + type Apicast struct { Options *ApicastOptions } @@ -90,7 +101,12 @@ func (apicast *Apicast) ProductionService() *v1.Service { } } -func (apicast *Apicast) StagingDeployment(containerImage string) *k8sappsv1.Deployment { +func (apicast *Apicast) StagingDeployment(ctx context.Context, k8sclient client.Client, containerImage string) (*k8sappsv1.Deployment, error) { + watchedSecretAnnotations, err := ComputeWatchedSecretAnnotations(ctx, k8sclient, ApicastStagingName, apicast.Options.Namespace, apicast) + if err != nil { + return nil, err + } + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ @@ -121,7 +137,7 @@ func (apicast *Apicast) StagingDeployment(containerImage string) *k8sappsv1.Depl Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: apicast.Options.StagingPodTemplateLabels, - Annotations: apicast.stagingPodAnnotations(), + Annotations: apicast.stagingPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ Affinity: apicast.Options.StagingAffinity, @@ -162,10 +178,15 @@ func (apicast *Apicast) StagingDeployment(containerImage string) *k8sappsv1.Depl }, }, }, - } + }, nil } -func (apicast *Apicast) ProductionDeployment(containerImage string) *k8sappsv1.Deployment { +func (apicast *Apicast) ProductionDeployment(ctx context.Context, k8sclient client.Client, containerImage string) (*k8sappsv1.Deployment, error) { + watchedSecretAnnotations, err := ComputeWatchedSecretAnnotations(ctx, k8sclient, ApicastStagingName, apicast.Options.Namespace, apicast) + if err != nil { + return nil, err + } + return &k8sappsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: reconcilers.DeploymentAPIVersion, Kind: reconcilers.DeploymentKind}, ObjectMeta: metav1.ObjectMeta{ @@ -196,7 +217,7 @@ func (apicast *Apicast) ProductionDeployment(containerImage string) *k8sappsv1.D Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: apicast.Options.ProductionPodTemplateLabels, - Annotations: apicast.productionPodAnnotations(), + Annotations: apicast.productionPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ Affinity: apicast.Options.ProductionAffinity, @@ -250,7 +271,7 @@ func (apicast *Apicast) ProductionDeployment(containerImage string) *k8sappsv1.D }, }, }, - } + }, nil } func (apicast *Apicast) buildApicastCommonEnv() []v1.EnvVar { @@ -826,13 +847,14 @@ func (apicast *Apicast) stagingServicePorts() []v1.ServicePort { return ports } -func (apicast *Apicast) stagingPodAnnotations() map[string]string { +func (apicast *Apicast) stagingPodAnnotations(watchedSecretAnnotations map[string]string) map[string]string { annotations := map[string]string{ - "prometheus.io/scrape": "true", - "prometheus.io/port": "9421", + "prometheus.io/scrape": "true", + "prometheus.io/port": "9421", + APIcastEnvironmentCMAnnotation: apicast.envConfigMapHash(), } - for key, val := range apicast.Options.StagingAdditionalPodAnnotations { + for key, val := range watchedSecretAnnotations { annotations[key] = val } @@ -843,13 +865,14 @@ func (apicast *Apicast) stagingPodAnnotations() map[string]string { return annotations } -func (apicast *Apicast) productionPodAnnotations() map[string]string { +func (apicast *Apicast) productionPodAnnotations(watchedSecretAnnotations map[string]string) map[string]string { annotations := map[string]string{ - "prometheus.io/scrape": "true", - "prometheus.io/port": "9421", + "prometheus.io/scrape": "true", + "prometheus.io/port": "9421", + APIcastEnvironmentCMAnnotation: apicast.envConfigMapHash(), } - for key, val := range apicast.Options.ProductionAdditionalPodAnnotations { + for key, val := range watchedSecretAnnotations { annotations[key] = val } @@ -888,3 +911,15 @@ func ApicastOpentelemetryConfigVolumeNamesFromAnnotations(annotations map[string func ApicastEnvVolumeNamesFromAnnotations(annotations map[string]string) []string { return AnnotationsValuesWithAnnotationKeyPrefix(annotations, CustomEnvironmentsAnnotationPartialKey) } + +// APIcast environment hash +// When any of the fields used to compute the hash change, the hash will change and the apicast deployment will rollout +func (apicast *Apicast) envConfigMapHash() string { + + h := fnv.New32a() + h.Write([]byte(apicast.Options.ManagementAPI)) + h.Write([]byte(apicast.Options.OpenSSLVerify)) + h.Write([]byte(apicast.Options.ResponseCodes)) + val := h.Sum32() + return fmt.Sprint(val) +} diff --git a/pkg/3scale/amp/component/apicast_options.go b/pkg/3scale/amp/component/apicast_options.go index d7eb35186..e609fa814 100644 --- a/pkg/3scale/amp/component/apicast_options.go +++ b/pkg/3scale/amp/component/apicast_options.go @@ -129,9 +129,6 @@ type ApicastOptions struct { ProductionServiceCacheSize *int32 StagingServiceCacheSize *int32 - - StagingAdditionalPodAnnotations map[string]string `validate:"required"` - ProductionAdditionalPodAnnotations map[string]string `validate:"required"` } func NewApicastOptions() *ApicastOptions { diff --git a/pkg/3scale/amp/component/deployment_annotations.go b/pkg/3scale/amp/component/deployment_annotations.go new file mode 100644 index 000000000..6d0cc40ec --- /dev/null +++ b/pkg/3scale/amp/component/deployment_annotations.go @@ -0,0 +1,246 @@ +package component + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/3scale/3scale-operator/pkg/helper" + + "github.com/go-logr/logr" + k8sappsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ComputeWatchedSecretAnnotations(ctx context.Context, client k8sclient.Client, deploymentName, watchNS string, component interface{}) (map[string]string, error) { + // First get the initial annotations + uncheckedAnnotations, err := getWatchedSecretAnnotations(ctx, client, deploymentName, component) + if err != nil { + return nil, err + } + + // Then get the deployment (if it exists) to compare the existing annotations + deployment := &k8sappsv1.Deployment{} + deploymentKey := k8sclient.ObjectKey{ + Name: deploymentName, + Namespace: watchNS, + } + err = client.Get(ctx, deploymentKey, deployment) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + // If the deployment doesn't exist yet then just return the uncheckedAnnotations because there's nothing to compare to + if apierrors.IsNotFound(err) { + return uncheckedAnnotations, nil + } + + // Next get the master hashed secret (if it exists) to compare the secret hashes + hashedSecret := &corev1.Secret{} + hashedSecretKey := k8sclient.ObjectKey{ + Name: HashedSecretName, + Namespace: watchNS, + } + err = client.Get(ctx, hashedSecretKey, hashedSecret) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + // If the master hashed secret doesn't exist yet then just return the uncheckedAnnotations because there's nothing to compare to + if apierrors.IsNotFound(err) { + return uncheckedAnnotations, nil + } + + existingPodAnnotations := deployment.Spec.Template.Annotations + + // First check if the annotations match, if they do then there's no need to check the hash + if !reflect.DeepEqual(existingPodAnnotations, uncheckedAnnotations) { + reconciledAnnotations := map[string]string{} + + // Loop through the annotations to see if the secret data has actually changed + for key, resourceVersion := range uncheckedAnnotations { + if existingPodAnnotations[key] == "" { + reconciledAnnotations[key] = resourceVersion // If this is the first time adding the annotation then use the new resourceVersion + } else if existingPodAnnotations[key] != resourceVersion && HasSecretHashChanged(ctx, client, key, hashedSecret, watchNS, component) { + reconciledAnnotations[key] = resourceVersion // Else if the resourceVersions don't match and the hash has changed then use the new resourceVersion + } else { + reconciledAnnotations[key] = existingPodAnnotations[key] // Otherwise keep the existing resourceVersion + } + } + + return reconciledAnnotations, nil + } + return uncheckedAnnotations, nil // No difference with existing annotations so can return uncheckedAnnotations +} + +func getWatchedSecretAnnotations(ctx context.Context, client k8sclient.Client, deploymentName string, component interface{}) (map[string]string, error) { + annotations := map[string]string{} + + switch c := component.(type) { + case *Apicast: + // HTTPs Certificate Secret + // OpenTelemetry Config Secret + // Custom Policy Secret(s) + // Custom Env Secret(s) + + apicast := c + + if deploymentName == ApicastProductionName { + if apicast.Options.ProductionHTTPSCertificateSecretName != nil && *apicast.Options.ProductionHTTPSCertificateSecretName != "" { + httpCertSecret := &corev1.Secret{} + httpCertSecretKey := k8sclient.ObjectKey{ + Name: *apicast.Options.ProductionHTTPSCertificateSecretName, + Namespace: apicast.Options.Namespace, + } + err := client.Get(ctx, httpCertSecretKey, httpCertSecret) + if err != nil { + return nil, err + } + if helper.IsSecretWatchedBy3scale(httpCertSecret) { + annotationKey := fmt.Sprintf("%s%s", HttpsCertSecretResverAnnotationPrefix, httpCertSecret.Name) + annotations[annotationKey] = httpCertSecret.ResourceVersion + } + } + + if &apicast.Options.ProductionOpentelemetry != nil && apicast.Options.ProductionOpentelemetry.Enabled { + if &apicast.Options.ProductionOpentelemetry.Secret != nil && apicast.Options.ProductionOpentelemetry.Secret.Name != "" { + telemetryConfigSecret := &corev1.Secret{} + telemetryConfigSecretKey := k8sclient.ObjectKey{ + Name: apicast.Options.ProductionOpentelemetry.Secret.Name, + Namespace: apicast.Options.Namespace, + } + err := client.Get(ctx, telemetryConfigSecretKey, telemetryConfigSecret) + if err != nil { + return nil, err + } + if helper.IsSecretWatchedBy3scale(telemetryConfigSecret) { + annotationKey := fmt.Sprintf("%s%s", OpenTelemetrySecretResverAnnotationPrefix, telemetryConfigSecret.Name) + annotations[annotationKey] = telemetryConfigSecret.ResourceVersion + } + + } + } + + for idx := range apicast.Options.ProductionCustomPolicies { + // Secrets must exist and have the watched-by label + // Annotation key includes the name of the secret + if helper.IsSecretWatchedBy3scale(apicast.Options.ProductionCustomPolicies[idx].Secret) { + annotationKey := fmt.Sprintf("%s%s", CustomPoliciesSecretResverAnnotationPrefix, apicast.Options.ProductionCustomPolicies[idx].Secret.Name) + annotations[annotationKey] = apicast.Options.ProductionCustomPolicies[idx].Secret.ResourceVersion + } + } + + for idx := range apicast.Options.ProductionCustomEnvironments { + // Secrets must exist and have the watched-by label + // Annotation key includes the name of the secret + if helper.IsSecretWatchedBy3scale(apicast.Options.ProductionCustomEnvironments[idx]) { + annotationKey := fmt.Sprintf("%s%s", CustomEnvSecretResverAnnotationPrefix, apicast.Options.ProductionCustomEnvironments[idx].Name) + annotations[annotationKey] = apicast.Options.ProductionCustomEnvironments[idx].ResourceVersion + } + } + } else if deploymentName == ApicastStagingName { + if apicast.Options.StagingHTTPSCertificateSecretName != nil && *apicast.Options.StagingHTTPSCertificateSecretName != "" { + httpCertSecret := &corev1.Secret{} + httpCertSecretKey := k8sclient.ObjectKey{ + Name: *apicast.Options.StagingHTTPSCertificateSecretName, + Namespace: apicast.Options.Namespace, + } + err := client.Get(ctx, httpCertSecretKey, httpCertSecret) + if err != nil { + return nil, err + } + if helper.IsSecretWatchedBy3scale(httpCertSecret) { + annotationKey := fmt.Sprintf("%s%s", HttpsCertSecretResverAnnotationPrefix, httpCertSecret.Name) + annotations[annotationKey] = httpCertSecret.ResourceVersion + } + } + + if &apicast.Options.StagingOpentelemetry != nil && apicast.Options.StagingOpentelemetry.Enabled { + if &apicast.Options.StagingOpentelemetry.Secret != nil && apicast.Options.StagingOpentelemetry.Secret.Name != "" { + telemetryConfigSecret := &corev1.Secret{} + telemetryConfigSecretKey := k8sclient.ObjectKey{ + Name: apicast.Options.StagingOpentelemetry.Secret.Name, + Namespace: apicast.Options.Namespace, + } + err := client.Get(ctx, telemetryConfigSecretKey, telemetryConfigSecret) + if err != nil { + return nil, err + } + if helper.IsSecretWatchedBy3scale(telemetryConfigSecret) { + annotationKey := fmt.Sprintf("%s%s", OpenTelemetrySecretResverAnnotationPrefix, telemetryConfigSecret.Name) + annotations[annotationKey] = telemetryConfigSecret.ResourceVersion + } + } + } + + for idx := range apicast.Options.StagingCustomPolicies { + // Secrets must exist and have the watched-by label + // Annotation key includes the name of the secret + if helper.IsSecretWatchedBy3scale(apicast.Options.StagingCustomPolicies[idx].Secret) { + annotationKey := fmt.Sprintf("%s%s", CustomPoliciesSecretResverAnnotationPrefix, apicast.Options.StagingCustomPolicies[idx].Secret.Name) + annotations[annotationKey] = apicast.Options.StagingCustomPolicies[idx].Secret.ResourceVersion + } + } + + for idx := range apicast.Options.StagingCustomEnvironments { + // Secrets must exist and have the watched-by label + // Annotation key includes the name of the secret + if helper.IsSecretWatchedBy3scale(apicast.Options.StagingCustomEnvironments[idx]) { + annotationKey := fmt.Sprintf("%s%s", CustomEnvSecretResverAnnotationPrefix, apicast.Options.StagingCustomEnvironments[idx].Name) + annotations[annotationKey] = apicast.Options.StagingCustomEnvironments[idx].ResourceVersion + } + } + } + + default: + return nil, fmt.Errorf("unrecognized component %s is not supported", deploymentName) + } + + return annotations, nil +} + +func HasSecretHashChanged(ctx context.Context, client k8sclient.Client, deploymentAnnotation string, hashedSecret *corev1.Secret, watchNS string, component interface{}) bool { + logger, _ := logr.FromContext(ctx) + + secretToCheck := &corev1.Secret{} + secretToCheckKey := k8sclient.ObjectKey{ + Namespace: watchNS, + } + + // Assign the name of the secret to check based on the component and secret type + switch c := component.(type) { + case *Apicast: + switch { + case strings.HasPrefix(deploymentAnnotation, HttpsCertSecretResverAnnotationPrefix): + secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, HttpsCertSecretResverAnnotationPrefix) + case strings.HasPrefix(deploymentAnnotation, OpenTelemetrySecretResverAnnotationPrefix): + secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, OpenTelemetrySecretResverAnnotationPrefix) + case strings.HasPrefix(deploymentAnnotation, CustomEnvSecretResverAnnotationPrefix): + secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, CustomEnvSecretResverAnnotationPrefix) + case strings.HasPrefix(deploymentAnnotation, CustomPoliciesSecretResverAnnotationPrefix): + secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, CustomPoliciesSecretResverAnnotationPrefix) + default: + return false + } + default: + logger.Info(fmt.Sprintf("unrecognized component %s is not supported", c)) + return false + } + + // Get latest version of the secret to check + err := client.Get(ctx, secretToCheckKey, secretToCheck) + if err != nil { + logger.Error(err, fmt.Sprintf("failed to get secret %s", secretToCheckKey.Name)) + return false + } + + // Compare the hash of the latest version of the secret's data to the reference in the hashed secret + if HashSecret(secretToCheck.Data) != helper.GetSecretStringDataFromData(hashedSecret.Data)[secretToCheckKey.Name] { + logger.V(1).Info(fmt.Sprintf("%s secret .data has changed - updating the resourceVersion in deployment's annotation", secretToCheckKey.Name)) + return true + } + + logger.V(1).Info(fmt.Sprintf("%s secret .data has not changed since last checked", secretToCheckKey.Name)) + return false +} diff --git a/pkg/3scale/amp/component/hash_secret.go b/pkg/3scale/amp/component/hash_secret.go new file mode 100644 index 000000000..cdc58bdf3 --- /dev/null +++ b/pkg/3scale/amp/component/hash_secret.go @@ -0,0 +1,81 @@ +package component + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "sort" + + "github.com/3scale/3scale-operator/pkg/helper" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + HashedSecretName = "hashed-secret-data" +) + +func HashedSecret(ctx context.Context, k8sclient client.Client, secretRefs []*v1.LocalObjectReference, ns string, hashSecretLabels map[string]string) (*v1.Secret, error) { + hashedSecretData, err := computeHashedSecretData(ctx, k8sclient, secretRefs, ns) + if err != nil { + return nil, err + } + + return &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: HashedSecretName, + Namespace: ns, + Labels: hashSecretLabels, + }, + StringData: hashedSecretData, + Type: v1.SecretTypeOpaque, + }, nil +} + +func computeHashedSecretData(ctx context.Context, k8sclient client.Client, secretRefs []*v1.LocalObjectReference, ns string) (map[string]string, error) { + data := make(map[string]string) + + for _, secretRef := range secretRefs { + secret := &v1.Secret{} + key := client.ObjectKey{ + Name: secretRef.Name, + Namespace: ns, + } + err := k8sclient.Get(ctx, key, secret) + if err != nil { + return nil, err + } + + if helper.IsSecretWatchedBy3scale(secret) { + data[secretRef.Name] = HashSecret(secret.Data) + } + } + + return data, nil +} + +func HashSecret(data map[string][]byte) string { + hash := sha256.New() + + sortedKeys := make([]string, 0, len(data)) + for k := range data { + sortedKeys = append(sortedKeys, k) + } + sort.Strings(sortedKeys) + + for _, key := range sortedKeys { + value := data[key] + combinedKeyValue := append([]byte(key), value...) + hash.Write(combinedKeyValue) + } + + hashBytes := hash.Sum(nil) + + return hex.EncodeToString(hashBytes) +} diff --git a/pkg/3scale/amp/operator/apicast_options_provider.go b/pkg/3scale/amp/operator/apicast_options_provider.go index b9b660795..a084477b9 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider.go +++ b/pkg/3scale/amp/operator/apicast_options_provider.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "hash/fnv" "path" "sort" "strconv" @@ -29,13 +28,6 @@ type ApicastOptionsProvider struct { secretSource *helper.SecretSource } -const ( - APIcastEnvironmentCMAnnotation = "apps.3scale.net/env-configmap-hash" - PodPrioritySystemNodeCritical = "system-node-critical" - CustomPoliciesSecretResverAnnotationPrefix = "apimanager.apps.3scale.net/custompolicy-secret-resource-version-" - OpentelemetrySecretResverAnnotationPrefix = "apimanager.apps.3scale.net/opentelemtry-secret-resource-version-" -) - func NewApicastOptionsProvider(apimanager *appsv1alpha1.APIManager, client client.Client) *ApicastOptionsProvider { return &ApicastOptionsProvider{ apimanager: apimanager, @@ -127,10 +119,6 @@ func (a *ApicastOptionsProvider) GetApicastOptions() (*component.ApicastOptions, a.setProxyConfigurations() - // Pod Annotations. Used to rollout apicast deployment if any secrets/configmap changes - a.apicastOptions.StagingAdditionalPodAnnotations = a.stagingAdditionalPodAnnotations() - a.apicastOptions.ProductionAdditionalPodAnnotations = a.productionAdditionalPodAnnotations() - err = a.apicastOptions.Validate() if err != nil { return nil, fmt.Errorf("GetApicastOptions validating: %w", err) @@ -493,74 +481,6 @@ func (a *ApicastOptionsProvider) setProductionProxyConfigurations() { a.apicastOptions.ProductionNoProxy = a.apimanager.Spec.Apicast.ProductionSpec.NoProxy } -func (a *ApicastOptionsProvider) stagingAdditionalPodAnnotations() map[string]string { - annotations := map[string]string{ - APIcastEnvironmentCMAnnotation: a.envConfigMapHash(), - } - - for idx := range a.apicastOptions.StagingCustomPolicies { - // Secrets must exist - // Annotation key includes the name of the secret - annotationKey := fmt.Sprintf("%s%s", CustomPoliciesSecretResverAnnotationPrefix, a.apicastOptions.StagingCustomPolicies[idx].Secret.Name) - annotations[annotationKey] = a.apicastOptions.StagingCustomPolicies[idx].Secret.ResourceVersion - } - - if a.apimanager.OpenTelemetryEnabledForStaging() && a.isOpentelemetryPodAnnotationRequired(&a.apicastOptions.StagingOpentelemetry.Secret) { - if a.apicastOptions.StagingOpentelemetry.Secret.Name != "" { - annotationKey := fmt.Sprintf("%s%s", OpentelemetrySecretResverAnnotationPrefix, a.apicastOptions.StagingOpentelemetry.Secret.Name) - annotations[annotationKey] = a.apicastOptions.StagingOpentelemetry.Secret.ResourceVersion - } - } - - return annotations -} - -func (a *ApicastOptionsProvider) productionAdditionalPodAnnotations() map[string]string { - annotations := map[string]string{ - APIcastEnvironmentCMAnnotation: a.envConfigMapHash(), - } - - for idx := range a.apicastOptions.ProductionCustomPolicies { - // Secrets must exist - // Annotation key includes the name of the secret - annotationKey := fmt.Sprintf("%s%s", CustomPoliciesSecretResverAnnotationPrefix, a.apicastOptions.ProductionCustomPolicies[idx].Secret.Name) - annotations[annotationKey] = a.apicastOptions.ProductionCustomPolicies[idx].Secret.ResourceVersion - } - - if a.apimanager.OpenTelemetryEnabledForProduction() && a.isOpentelemetryPodAnnotationRequired(&a.apicastOptions.ProductionOpentelemetry.Secret) { - if a.apicastOptions.ProductionOpentelemetry.Secret.Name != "" { - annotationKey := fmt.Sprintf("%s%s", OpentelemetrySecretResverAnnotationPrefix, a.apicastOptions.ProductionOpentelemetry.Secret.Name) - annotations[annotationKey] = a.apicastOptions.ProductionOpentelemetry.Secret.ResourceVersion - } - } - - return annotations -} - -func (a *ApicastOptionsProvider) isOpentelemetryPodAnnotationRequired(secret *v1.Secret) bool { - existingLabels := secret.Labels - - if existingLabels != nil { - if _, ok := existingLabels["apimanager.apps.3scale.net/watched-by"]; ok { - return true - } - } - - return false -} - -// APIcast environment hash -// When any of the fields used to compute the hash change the value, the hash will change -// and the apicast deployment will rollout -func (a *ApicastOptionsProvider) envConfigMapHash() string { - h := fnv.New32a() - h.Write([]byte(a.apicastOptions.ManagementAPI)) - h.Write([]byte(a.apicastOptions.OpenSSLVerify)) - h.Write([]byte(a.apicastOptions.ResponseCodes)) - val := h.Sum32() - return fmt.Sprint(val) -} - func (a *ApicastOptionsProvider) setPriorityClassNames() { if a.apimanager.Spec.Apicast.StagingSpec.PriorityClassName != nil { a.apicastOptions.PriorityClassNameStaging = *a.apimanager.Spec.Apicast.StagingSpec.PriorityClassName diff --git a/pkg/3scale/amp/operator/apicast_options_provider_test.go b/pkg/3scale/amp/operator/apicast_options_provider_test.go index 77109bb9d..afba0f25f 100644 --- a/pkg/3scale/amp/operator/apicast_options_provider_test.go +++ b/pkg/3scale/amp/operator/apicast_options_provider_test.go @@ -144,27 +144,25 @@ func basicApimanagerTestApicastOptions() *appsv1alpha1.APIManager { func defaultApicastOptions() *component.ApicastOptions { return &component.ApicastOptions{ - ManagementAPI: apicastManagementAPI, - OpenSSLVerify: strconv.FormatBool(openSSLVerify), - ResponseCodes: strconv.FormatBool(responseCodes), - ImageTag: version.ThreescaleVersionMajorMinor(), - ExtendedMetrics: true, - ProductionResourceRequirements: component.DefaultProductionResourceRequirements(), - StagingResourceRequirements: component.DefaultStagingResourceRequirements(), - ProductionReplicas: int32(productionReplicaCount), - StagingReplicas: int32(stagingReplicaCount), - CommonLabels: testApicastCommonLabels(), - CommonStagingLabels: testApicastStagingLabels(), - CommonProductionLabels: testApicastProductionLabels(), - StagingPodTemplateLabels: testApicastStagingPodLabels(), - ProductionPodTemplateLabels: testApicastProductionPodLabels(), - Namespace: namespace, - ProductionTracingConfig: &component.APIcastTracingConfig{TracingLibrary: apps.APIcastDefaultTracingLibrary}, - StagingTracingConfig: &component.APIcastTracingConfig{TracingLibrary: apps.APIcastDefaultTracingLibrary}, - StagingOpentelemetry: component.OpentelemetryConfig{}, - ProductionOpentelemetry: component.OpentelemetryConfig{}, - StagingAdditionalPodAnnotations: map[string]string{APIcastEnvironmentCMAnnotation: "788712912"}, - ProductionAdditionalPodAnnotations: map[string]string{APIcastEnvironmentCMAnnotation: "788712912"}, + ManagementAPI: apicastManagementAPI, + OpenSSLVerify: strconv.FormatBool(openSSLVerify), + ResponseCodes: strconv.FormatBool(responseCodes), + ImageTag: version.ThreescaleVersionMajorMinor(), + ExtendedMetrics: true, + ProductionResourceRequirements: component.DefaultProductionResourceRequirements(), + StagingResourceRequirements: component.DefaultStagingResourceRequirements(), + ProductionReplicas: int32(productionReplicaCount), + StagingReplicas: int32(stagingReplicaCount), + CommonLabels: testApicastCommonLabels(), + CommonStagingLabels: testApicastStagingLabels(), + CommonProductionLabels: testApicastProductionLabels(), + StagingPodTemplateLabels: testApicastStagingPodLabels(), + ProductionPodTemplateLabels: testApicastProductionPodLabels(), + Namespace: namespace, + ProductionTracingConfig: &component.APIcastTracingConfig{TracingLibrary: apps.APIcastDefaultTracingLibrary}, + StagingTracingConfig: &component.APIcastTracingConfig{TracingLibrary: apps.APIcastDefaultTracingLibrary}, + StagingOpentelemetry: component.OpentelemetryConfig{}, + ProductionOpentelemetry: component.OpentelemetryConfig{}, } } diff --git a/pkg/3scale/amp/operator/apicast_reconciler.go b/pkg/3scale/amp/operator/apicast_reconciler.go index 2bfe65831..ef2a40083 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler.go +++ b/pkg/3scale/amp/operator/apicast_reconciler.go @@ -102,7 +102,11 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // Staging Deployment - err = r.ReconcileDeployment(apicast.StagingDeployment(ampImages.Options.ApicastImage), reconcilers.DeploymentMutator(stagingMutators...)) + stagingDeployment, err := apicast.StagingDeployment(r.Context(), r.Client(), ampImages.Options.ApicastImage) + if err != nil { + return reconcile.Result{}, err + } + err = r.ReconcileDeployment(stagingDeployment, reconcilers.DeploymentMutator(stagingMutators...)) if err != nil { return reconcile.Result{}, err } @@ -151,7 +155,11 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) { } // Production Deployment - err = r.ReconcileDeployment(apicast.ProductionDeployment(ampImages.Options.ApicastImage), reconcilers.DeploymentMutator(productionMutators...)) + productionDeployment, err := apicast.ProductionDeployment(r.Context(), r.Client(), ampImages.Options.ApicastImage) + if err != nil { + return reconcile.Result{}, err + } + err = r.ReconcileDeployment(productionDeployment, reconcilers.DeploymentMutator(productionMutators...)) if err != nil { return reconcile.Result{}, err } @@ -639,18 +647,18 @@ func apicastCustomEnvAnnotationsMutator(desired, existing *k8sappsv1.Deployment) func apicastPodTemplateEnvConfigMapAnnotationsMutator(desired, existing *k8sappsv1.Deployment) (bool, error) { // Only reconcile the pod annotation regarding apicast-environment hash - desiredVal, ok := desired.Spec.Template.Annotations[APIcastEnvironmentCMAnnotation] + desiredVal, ok := desired.Spec.Template.Annotations[component.APIcastEnvironmentCMAnnotation] if !ok { return false, nil } updated := false - existingVal, ok := existing.Spec.Template.Annotations[APIcastEnvironmentCMAnnotation] + existingVal, ok := existing.Spec.Template.Annotations[component.APIcastEnvironmentCMAnnotation] if !ok || existingVal != desiredVal { if existing.Spec.Template.Annotations == nil { existing.Spec.Template.Annotations = map[string]string{} } - existing.Spec.Template.Annotations[APIcastEnvironmentCMAnnotation] = desiredVal + existing.Spec.Template.Annotations[component.APIcastEnvironmentCMAnnotation] = desiredVal updated = true } @@ -684,15 +692,41 @@ func (r *ApicastReconciler) reconcileApimanagerSecretLabels(ctx context.Context) return replaceAPIManagerSecretLabels(r.apiManager, secretUIDs), nil } -func (r *ApicastReconciler) getSecretUIDs(ctx context.Context) ([]string, error) { - // production custom policy - // staging custom policy +func (r *ApicastReconciler) getSecretUIDs(ctx context.Context) (map[string]string, error) { + // HTTPs Certificate Secret + // OpenTelemetry Config Secret + // Custom Policy Secret(s) + // Custom Env Secret(s) secretKeys := []client.ObjectKey{} - if r.apiManager.Spec.Apicast.ProductionSpec.CustomPolicies != nil { - for _, customPolicy := range r.apiManager.Spec.Apicast.ProductionSpec.CustomPolicies { + + if r.apiManager.Spec.Apicast.StagingSpec.HTTPSCertificateSecretRef != nil && r.apiManager.Spec.Apicast.StagingSpec.HTTPSCertificateSecretRef.Name != "" { + secretKeys = append(secretKeys, client.ObjectKey{ + Name: r.apiManager.Spec.Apicast.StagingSpec.HTTPSCertificateSecretRef.Name, + Namespace: r.apiManager.Namespace, + }) + } + + if r.apiManager.Spec.Apicast.ProductionSpec.HTTPSCertificateSecretRef != nil && r.apiManager.Spec.Apicast.ProductionSpec.HTTPSCertificateSecretRef.Name != "" { + secretKeys = append(secretKeys, client.ObjectKey{ + Name: r.apiManager.Spec.Apicast.ProductionSpec.HTTPSCertificateSecretRef.Name, + Namespace: r.apiManager.Namespace, + }) + } + + if r.apiManager.OpenTelemetryEnabledForStaging() { + if r.apiManager.Spec.Apicast.StagingSpec.OpenTelemetry.TracingConfigSecretRef != nil { secretKeys = append(secretKeys, client.ObjectKey{ - Name: customPolicy.SecretRef.Name, + Name: r.apiManager.Spec.Apicast.StagingSpec.OpenTelemetry.TracingConfigSecretRef.Name, + Namespace: r.apiManager.Namespace, + }) + } + } + + if r.apiManager.OpenTelemetryEnabledForProduction() { + if r.apiManager.Spec.Apicast.ProductionSpec.OpenTelemetry.TracingConfigSecretRef != nil { + secretKeys = append(secretKeys, client.ObjectKey{ + Name: r.apiManager.Spec.Apicast.ProductionSpec.OpenTelemetry.TracingConfigSecretRef.Name, Namespace: r.apiManager.Namespace, }) } @@ -707,37 +741,48 @@ func (r *ApicastReconciler) getSecretUIDs(ctx context.Context) ([]string, error) } } - if r.apiManager.OpenTelemetryEnabledForStaging() { - if r.apiManager.Spec.Apicast.StagingSpec.OpenTelemetry.TracingConfigSecretRef != nil { + if r.apiManager.Spec.Apicast.ProductionSpec.CustomPolicies != nil { + for _, customPolicy := range r.apiManager.Spec.Apicast.ProductionSpec.CustomPolicies { secretKeys = append(secretKeys, client.ObjectKey{ - Name: r.apiManager.Spec.Apicast.StagingSpec.OpenTelemetry.TracingConfigSecretRef.Name, + Name: customPolicy.SecretRef.Name, Namespace: r.apiManager.Namespace, }) } } - if r.apiManager.OpenTelemetryEnabledForProduction() { - if r.apiManager.Spec.Apicast.ProductionSpec.OpenTelemetry.TracingConfigSecretRef != nil { + if r.apiManager.Spec.Apicast.StagingSpec.CustomEnvironments != nil { + for _, customEnv := range r.apiManager.Spec.Apicast.StagingSpec.CustomEnvironments { secretKeys = append(secretKeys, client.ObjectKey{ - Name: r.apiManager.Spec.Apicast.ProductionSpec.OpenTelemetry.TracingConfigSecretRef.Name, + Name: customEnv.SecretRef.Name, + Namespace: r.apiManager.Namespace, + }) + } + } + + if r.apiManager.Spec.Apicast.ProductionSpec.CustomEnvironments != nil { + for _, customEnv := range r.apiManager.Spec.Apicast.ProductionSpec.CustomEnvironments { + secretKeys = append(secretKeys, client.ObjectKey{ + Name: customEnv.SecretRef.Name, Namespace: r.apiManager.Namespace, }) } } - uids := []string{} + uidMap := map[string]string{} for idx := range secretKeys { secret := &v1.Secret{} secretKey := secretKeys[idx] err := r.Client().Get(ctx, secretKey, secret) - r.Logger().V(1).Info("read secret", "objectKey", secretKey, "error", err) + r.Logger().V(1).Info("reading secret", "objectKey", secretKey, "error", err) if err != nil { return nil, err } - uids = append(uids, string(secret.GetUID())) + + watchedByVal := fmt.Sprintf("%t", helper.IsSecretWatchedBy3scale(secret)) + uidMap[string(secret.GetUID())] = watchedByVal } - return uids, nil + return uidMap, nil } func Apicast(apimanager *appsv1alpha1.APIManager, cl client.Client) (*component.Apicast, error) { diff --git a/pkg/3scale/amp/operator/apicast_reconciler_test.go b/pkg/3scale/amp/operator/apicast_reconciler_test.go index d7d886c64..b91f52d5f 100644 --- a/pkg/3scale/amp/operator/apicast_reconciler_test.go +++ b/pkg/3scale/amp/operator/apicast_reconciler_test.go @@ -231,7 +231,10 @@ func TestApicastReconcilerCustomPolicyParts(t *testing.T) { ProductionTracingConfig: &component.APIcastTracingConfig{}, } apicast := component.NewApicast(apicastOptions) - existingProdDeployment := apicast.ProductionDeployment(ampImages.Options.ApicastImage) + existingProdDeployment, err := apicast.ProductionDeployment(context.TODO(), fake.NewFakeClient(), ampImages.Options.ApicastImage) + if err != nil { + t.Fatal(err) + } existingProdDeployment.Namespace = namespace // - Policy annotation for P1 added @@ -441,7 +444,10 @@ func TestApicastReconcilerTracingConfigParts(t *testing.T) { ProductionTracingConfig: &existingTracingConfig1, } apicast := component.NewApicast(apicastOptions) - existingProdDeployment := apicast.ProductionDeployment(ampImages.Options.ApicastImage) + existingProdDeployment, err := apicast.ProductionDeployment(context.TODO(), fake.NewFakeClient(), ampImages.Options.ApicastImage) + if err != nil { + t.Fatal(err) + } existingProdDeployment.Namespace = namespace // - Tracing Configuration 1 added into the Production Deployment with the expected key diff --git a/pkg/3scale/amp/operator/apimanager_utils.go b/pkg/3scale/amp/operator/apimanager_utils.go index 3d0415328..92e234cb5 100644 --- a/pkg/3scale/amp/operator/apimanager_utils.go +++ b/pkg/3scale/amp/operator/apimanager_utils.go @@ -10,14 +10,13 @@ import ( const ( APIManagerSecretLabelPrefix = "secret.apimanager.apps.3scale.net/" - APIManagerSecretLabelValue = "true" ) func apimanagerSecretLabelKey(uid string) string { return fmt.Sprintf("%s%s", APIManagerSecretLabelPrefix, uid) } -func replaceAPIManagerSecretLabels(apimanager *appsv1alpha1.APIManager, desiredSecretUIDs []string) bool { +func replaceAPIManagerSecretLabels(apimanager *appsv1alpha1.APIManager, desiredSecretUIDs map[string]string) bool { existingLabels := apimanager.GetLabels() @@ -28,18 +27,18 @@ func replaceAPIManagerSecretLabels(apimanager *appsv1alpha1.APIManager, desiredS existingSecretLabels := map[string]string{} // existing Secret UIDs not included in desiredAPIUIDs are deleted - for k := range existingLabels { - if strings.HasPrefix(k, APIManagerSecretLabelPrefix) { - existingSecretLabels[k] = APIManagerSecretLabelValue + for key, value := range existingLabels { + if strings.HasPrefix(key, APIManagerSecretLabelPrefix) { + existingSecretLabels[key] = value // it is safe to remove keys while looping in range - delete(existingLabels, k) + delete(existingLabels, key) } } desiredSecretLabels := map[string]string{} - for _, uid := range desiredSecretUIDs { - desiredSecretLabels[apimanagerSecretLabelKey(uid)] = APIManagerSecretLabelValue - existingLabels[apimanagerSecretLabelKey(uid)] = APIManagerSecretLabelValue + for uid, watchedByStatus := range desiredSecretUIDs { + desiredSecretLabels[apimanagerSecretLabelKey(uid)] = watchedByStatus + existingLabels[apimanagerSecretLabelKey(uid)] = watchedByStatus } apimanager.SetLabels(existingLabels) diff --git a/pkg/3scale/amp/prometheusrules/apicast_rules.go b/pkg/3scale/amp/prometheusrules/apicast_rules.go index 3e2be6ec9..9b371c563 100644 --- a/pkg/3scale/amp/prometheusrules/apicast_rules.go +++ b/pkg/3scale/amp/prometheusrules/apicast_rules.go @@ -53,9 +53,6 @@ func apicastOptions(ns string) (*component.ApicastOptions, error) { o.StagingTracingConfig = &component.APIcastTracingConfig{TracingLibrary: apps.APIcastDefaultTracingLibrary} o.ProductionTracingConfig = &component.APIcastTracingConfig{TracingLibrary: apps.APIcastDefaultTracingLibrary} - o.StagingAdditionalPodAnnotations = map[string]string{} - o.ProductionAdditionalPodAnnotations = map[string]string{} - return o, o.Validate() } diff --git a/pkg/helper/secretutils.go b/pkg/helper/secretutils.go index aab24b83b..69b4005d5 100644 --- a/pkg/helper/secretutils.go +++ b/pkg/helper/secretutils.go @@ -181,3 +181,19 @@ func ValidateTLSSecret(nn types.NamespacedName, client k8sclient.Client) error { return nil } + +func IsSecretWatchedBy3scale(secret *v1.Secret) bool { + if secret == nil { + return false + } + + existingLabels := secret.Labels + + if existingLabels != nil { + if _, ok := existingLabels["apimanager.apps.3scale.net/watched-by"]; ok { + return true + } + } + + return false +} diff --git a/pkg/helper/secretutils_test.go b/pkg/helper/secretutils_test.go new file mode 100644 index 000000000..e1737dcd1 --- /dev/null +++ b/pkg/helper/secretutils_test.go @@ -0,0 +1,64 @@ +package helper + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIsSecretWatchedBy3scale(t *testing.T) { + labeledSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "labeled-secret", + Namespace: "test-namespace", + Labels: map[string]string{ + "apimanager.apps.3scale.net/watched-by": "apimanager", + }, + }, + } + unlabeledSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unlabeled-secret", + Namespace: "test-namespace", + }, + } + + type args struct { + secret *v1.Secret + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Secret doesn't have watched-by label", + args: args{ + secret: unlabeledSecret, + }, + want: false, + }, + { + name: "Secret has watched-by label", + args: args{ + secret: labeledSecret, + }, + want: true, + }, + { + name: "Secret doesn't exist", + args: args{ + secret: nil, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsSecretWatchedBy3scale(tt.args.secret); got != tt.want { + t.Errorf("IsSecretWatchedBy3scale() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/reconcilers/secret.go b/pkg/reconcilers/secret.go index d68dba497..6bb0abfe7 100644 --- a/pkg/reconcilers/secret.go +++ b/pkg/reconcilers/secret.go @@ -2,6 +2,7 @@ package reconcilers import ( "fmt" + "reflect" "github.com/3scale/3scale-operator/pkg/common" @@ -89,3 +90,21 @@ func SecretReconcileField(fieldName string) func(desired, existing *v1.Secret) b return updated } } + +func SecretStringDataMutator(desired, existing *v1.Secret) bool { + updated := false + + // StringData is merged to Data on write, so we need to compare the existing Data to the desired StringData + // Before we can do this we need to convert the existing Data to StringData + existingStringData := make(map[string]string) + for key, bytes := range existing.Data { + existingStringData[key] = string(bytes) + } + if !reflect.DeepEqual(existingStringData, desired.StringData) { + updated = true + existing.Data = nil // Need to clear the existing.Data because of how StringData is converted to Data + existing.StringData = desired.StringData + } + + return updated +}