From 6c3f69e8ebb99a12e2db67c71a94981d96c8d529 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 22 Aug 2024 13:34:39 +0200 Subject: [PATCH] Keep hyperkube image mutation for safety, but we can definitely removed after next release. --- go.mod | 1 + go.sum | 2 + pkg/webhook/shoot/mutator.go | 157 +++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) diff --git a/go.mod b/go.mod index 52522f8d..042dacc5 100644 --- a/go.mod +++ b/go.mod @@ -158,6 +158,7 @@ require ( istio.io/api v1.19.2-0.20231011000955-f3015ebb5bd4 // indirect istio.io/client-go v1.19.3 // indirect k8s.io/apiserver v0.28.9 // indirect + k8s.io/cluster-bootstrap v0.28.3 // indirect k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect k8s.io/helm v2.17.0+incompatible // indirect k8s.io/klog v1.0.0 // indirect diff --git a/go.sum b/go.sum index 44066c6b..a04da9b3 100644 --- a/go.sum +++ b/go.sum @@ -2023,6 +2023,8 @@ k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0 h1:y0TgWoHaeYEv3L1MfLC+D2WVxyN1 k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0/go.mod h1:w6/LjLR3DPQd57vlgvgbpzpuJKsCiily0+OzQI+nyfI= k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/cluster-bootstrap v0.28.3 h1:hGK3mJsmVGGvRJ61nyQcYNR9g/IYax75TbJcylTmZts= +k8s.io/cluster-bootstrap v0.28.3/go.mod h1:s1B3FTw713b9iw67yGFiVF3zCfw5obrZXWl3EMelvdg= k8s.io/code-generator v0.28.3 h1:I847QvdpYx7xKiG2KVQeCSyNF/xU9TowaDAg601mvlw= k8s.io/code-generator v0.28.3/go.mod h1:A2EAHTRYvCvBrb/MM2zZBNipeCk3f8NtpdNIKawC43M= k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= diff --git a/pkg/webhook/shoot/mutator.go b/pkg/webhook/shoot/mutator.go index 9cac9c02..51b28dd3 100644 --- a/pkg/webhook/shoot/mutator.go +++ b/pkg/webhook/shoot/mutator.go @@ -2,10 +2,25 @@ package shoot import ( "context" + "errors" "fmt" + "net/url" + "slices" + "strings" extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component/extensions/operatingsystemconfig/downloader" + "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/helper" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + + metalv1alpha1 "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/v1alpha1" + + extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + "github.com/gardener/gardener/pkg/utils" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -49,6 +64,14 @@ func (m *mutator) Mutate(ctx context.Context, new, _ client.Object) error { extensionswebhook.LogMutation(logger, x.Kind, x.Namespace, x.Name) return m.mutateVPNShootDeployment(ctx, x) } + case *corev1.Secret: + // TODO: remove this once gardener-node-agent is in use + // the purpose of this hack is to enable the cloud-config-downloader to pull the hyperkube image from + // a registry mirror in case this shoot cluster is configured with networkaccesstype restricted/forbidden + err = m.mutateCloudConfigDownloaderHyperkubeImage(ctx, x) + if err != nil { + return fmt.Errorf("mutating cloud config downlader secret failed %w", err) + } } return nil } @@ -76,3 +99,137 @@ func (m *mutator) mutateVPNShootDeployment(_ context.Context, deployment *appsv1 return nil } + +const ( + gardenerRegistry = "eu.gcr.io" + hyperkubeImage = "/gardener-project/hyperkube" + + // this should be the final destination + newGardenerRegistry = "europe-docker.pkg.dev" + newHyperkubeImage = "/gardener-project/releases/hyperkube" +) + +func (m *mutator) mutateCloudConfigDownloaderHyperkubeImage(ctx context.Context, secret *corev1.Secret) error { + if secret.Labels["gardener.cloud/role"] != "cloud-config" { + return nil + } + + shootName, err := extractShootNameFromSecret(secret) + if err != nil { + return err + } + + cluster := &extensionsv1alpha1.Cluster{} + if err := m.client.Get(ctx, kutil.Key(shootName), cluster); err != nil { + return err + } + + shoot, err := extensionscontroller.ShootFromCluster(cluster) + if err != nil { + return fmt.Errorf("unable to decode cluster.Spec.Shoot.Raw %w", err) + } + + if len(shoot.Spec.Provider.Workers) == 0 { + m.logger.Info("workerless shoot, nothing to do here", "shoot", shootName) + return nil + } + + cloudProfile := &gardencorev1beta1.CloudProfile{} + err = helper.DecodeRawExtension(&cluster.Spec.CloudProfile, cloudProfile, m.decoder) + if err != nil { + return err + } + + cloudProfileConfig, err := helper.DecodeCloudProfileConfig(cloudProfile) + if err != nil { + return err + } + + infrastructureConfig := &metalv1alpha1.InfrastructureConfig{} + err = helper.DecodeRawExtension(shoot.Spec.Provider.InfrastructureConfig, infrastructureConfig, m.decoder) + if err != nil { + return err + } + + _, p, err := helper.FindMetalControlPlane(cloudProfileConfig, infrastructureConfig.PartitionID) + if err != nil { + return err + } + + controlPlaneConfig := &metalv1alpha1.ControlPlaneConfig{} + err = helper.DecodeRawExtension(shoot.Spec.Provider.ControlPlaneConfig, controlPlaneConfig, m.decoder) + if err != nil { + return err + } + + if controlPlaneConfig.NetworkAccessType == nil || *controlPlaneConfig.NetworkAccessType == metalv1alpha1.NetworkAccessBaseline { + // this shoot does not have networkaccesstype restricted or forbidden specified, nothing to do here + return nil + } + + if p.NetworkIsolation == nil || len(p.NetworkIsolation.RegistryMirrors) == 0 { + m.logger.Info("no registry mirrors specified in this shoot, nothing to do here", "shoot", shootName) + return nil + } + + var ( + networkIsolation = p.NetworkIsolation + destinationRegistry string + ) + + for _, registry := range networkIsolation.RegistryMirrors { + if slices.Contains(registry.MirrorOf, gardenerRegistry) { + parsed, err := url.Parse(registry.Endpoint) + if err != nil { + return fmt.Errorf("unable to parse registry endpoint:%w", err) + } + destinationRegistry = parsed.Host + break + } + } + if destinationRegistry == "" { + err := errors.New("no matching destination registry detected for the hyperkube image") + m.logger.Error(err, "please check the networkisolation configuration", "shoot", shootName) + return err + } + + m.logger.Info("mutate secret", "shoot", shootName, "secret", secret.Name) + + raw, ok := secret.Data[downloader.DataKeyScript] + if ok { + script := string(raw) + newScript := strings.ReplaceAll(script, gardenerRegistry+hyperkubeImage, destinationRegistry+hyperkubeImage) + newScript = strings.ReplaceAll(newScript, newGardenerRegistry+newHyperkubeImage, destinationRegistry+newHyperkubeImage) + secret.Data[downloader.DataKeyScript] = []byte(newScript) + secret.Annotations[downloader.AnnotationKeyChecksum] = utils.ComputeChecksum(newScript) + } + return nil +} + +func extractShootNameFromSecret(secret *corev1.Secret) (string, error) { + // resources.gardener.cloud/origin: shoot--test--fra-equ01-8fef639c-bbe4-4c6f-9656-617dc4a4efd8-gardener-soil-test:shoot--pjb9j2--forbidden/shoot-cloud-config-execution + origin, ok := secret.Annotations[resourcesv1alpha1.OriginAnnotation] + if !ok { + return "", fmt.Errorf("no matching annotation found to identify the shoot namespace") + } + + // does not work + // shootName, _, err := resourcesv1alpha1helper.SplitOrigin(origin) + // if err != nil { + // return "", fmt.Errorf("no matching content found in origin annotation to get shoot namespace %w", err) + // } + + // resources.gardener.cloud/origin: shoot--test--fra-equ01-8fef639c-bbe4-4c6f-9656-617dc4a4efd8-gardener-soil-test:shoot--pjb9j2--forbidden/shoot-cloud-config-execution + _, firstpart, found := strings.Cut(origin, ":") + if !found { + return "", fmt.Errorf("no matching content found in origin annotation to get shoot namespace") + } + shootName, _, found := strings.Cut(firstpart, "/") + if !found { + return "", fmt.Errorf("no matching content found in origin annotation to get shoot namespace") + } + if len(shootName) == 0 { + return "", fmt.Errorf("could not find shoot name for webhook request") + } + return shootName, nil +}