Adding support for mutliple oci namespaces in BR OS vsphere (#7876)

aws · Mar 23, 2024 · cec82a8 · cec82a8
1 parent a29b916
commit cec82a8
Show file tree

Hide file tree

Showing 12 changed files with 1,079 additions and 16 deletions.
diff --git a/internal/pkg/api/cluster.go b/internal/pkg/api/cluster.go
@@ -271,7 +271,7 @@ func WithProxyConfig(httpProxy, httpsProxy string, noProxy []string) ClusterFill
 }
 
 // WithRegistryMirror adds a registry mirror configuration.
-func WithRegistryMirror(endpoint, port string, caCert string, authenticate bool, insecureSkipVerify bool) ClusterFiller {
+func WithRegistryMirror(endpoint, port string, caCert string, authenticate bool, insecureSkipVerify bool, ociNamespaces ...anywherev1.OCINamespace) ClusterFiller {
 	return func(c *anywherev1.Cluster) {
 		if c.Spec.RegistryMirrorConfiguration == nil {
 			c.Spec.RegistryMirrorConfiguration = &anywherev1.RegistryMirrorConfiguration{}
@@ -281,6 +281,10 @@ func WithRegistryMirror(endpoint, port string, caCert string, authenticate bool,
 		c.Spec.RegistryMirrorConfiguration.CACertContent = caCert
 		c.Spec.RegistryMirrorConfiguration.Authenticate = authenticate
 		c.Spec.RegistryMirrorConfiguration.InsecureSkipVerify = insecureSkipVerify
+
+		if len(ociNamespaces) != 0 {
+			c.Spec.RegistryMirrorConfiguration.OCINamespaces = ociNamespaces
+		}
 	}
 }
 

diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml
@@ -133,11 +133,21 @@ spec:
 {{- end }}
 {{- if and .registryMirrorMap (eq .format "bottlerocket") }}
       registryMirror:
-        endpoint: {{ .publicMirror }}
+        {{- if .publicECRMirror }}
+        endpoint: {{ .publicECRMirror }}
+        {{- end }}
         {{- if .registryCACert }}
         caCert: |
 {{ .registryCACert | indent 10 }}
         {{- end }}
+        {{- if not .publicECRMirror }}
+        mirrors:
+        {{- range $orig, $mirror := .registryMirrorMap }}
+          - registry: "{{ $orig }}"
+            endpoints:
+            - {{ $mirror }}
+        {{- end }}
+        {{- end }}
 {{- end }}
 {{- if .bottlerocketSettings }}
 {{ .bottlerocketSettings | indent 6 }}
@@ -422,11 +432,21 @@ spec:
 {{- end }}
 {{- if and .registryMirrorMap (eq .format "bottlerocket") }}
       registryMirror:
-        endpoint: {{ .publicMirror }}
+        {{- if .publicECRMirror }}
+        endpoint: {{ .publicECRMirror }}
+        {{- end }}       
         {{- if .registryCACert }}
         caCert: |
 {{ .registryCACert | indent 10 }}
         {{- end }}
+        {{- if not .publicECRMirror }}
+        mirrors:
+        {{- range $orig, $mirror := .registryMirrorMap }}
+          - registry: "{{ $orig }}"
+            endpoints:
+            - {{ $mirror }}
+        {{- end }}
+        {{- end }}
 {{- end }}
 {{- if .bottlerocketSettings }}
 {{ .bottlerocketSettings | indent 6 }}

diff --git a/pkg/providers/vsphere/config/template-md.yaml b/pkg/providers/vsphere/config/template-md.yaml
@@ -24,11 +24,21 @@ spec:
 {{- end }}
 {{- if and .registryMirrorMap (eq .format "bottlerocket") }}
         registryMirror:
-          endpoint: {{ .publicMirror }}
+          {{- if .publicECRMirror }}
+          endpoint: {{ .publicECRMirror }}
+          {{- end }}
           {{- if .registryCACert }}
           caCert: |
 {{ .registryCACert | indent 12 }}
           {{- end }}
+          {{- if not .publicECRMirror }}
+          mirrors:
+          {{- range $orig, $mirror := .registryMirrorMap }}
+            - registry: "{{ $orig }}"
+              endpoints:
+              - {{ $mirror }}
+          {{- end }}
+          {{- end }}
 {{- end }}
 {{- if .bottlerocketSettings }}
 {{ .bottlerocketSettings | indent 8 }}

diff --git a/pkg/providers/vsphere/template.go b/pkg/providers/vsphere/template.go
@@ -230,6 +230,12 @@ func buildTemplateMapCP(
 			values["registryCACert"] = registryMirror.CACertContent
 		}
 
+		if controlPlaneMachineSpec.OSFamily == anywherev1.Bottlerocket &&
+			len(registryMirror.NamespacedRegistryMap) == 1 &&
+			registryMirror.CoreEKSAMirror() != "" {
+			values["publicECRMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror())
+		}
+
 		if registryMirror.Auth {
 			values["registryAuth"] = registryMirror.Auth
 			username, password, err := config.ReadCredentials()
@@ -419,6 +425,12 @@ func buildTemplateMapMD(
 			values["registryCACert"] = registryMirror.CACertContent
 		}
 
+		if workerNodeGroupMachineSpec.OSFamily == anywherev1.Bottlerocket &&
+			len(registryMirror.NamespacedRegistryMap) == 1 &&
+			registryMirror.CoreEKSAMirror() != "" {
+			values["publicECRMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror())
+		}
+
 		if registryMirror.Auth {
 			values["registryAuth"] = registryMirror.Auth
 			username, password, err := config.ReadCredentials()

diff --git a/...providers/vsphere/testdata/cluster_bottlerocket_mirror_config_multiple_ocinamespaces.yaml b/...providers/vsphere/testdata/cluster_bottlerocket_mirror_config_multiple_ocinamespaces.yaml
@@ -0,0 +1,114 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: test
+spec:
+  controlPlaneConfiguration:
+    count: 3
+    endpoint:
+      host: 1.2.3.4
+    machineGroupRef:
+      name: test-cp
+      kind: VSphereMachineConfig
+  kubernetesVersion: "1.21"
+  workerNodeGroupConfigurations:
+    - count: 3
+      machineGroupRef:
+        name: test-wn
+        kind: VSphereMachineConfig
+      name: md-0
+  externalEtcdConfiguration:
+    count: 3
+    machineGroupRef:
+      name: test-etcd
+      kind: VSphereMachineConfig
+  datacenterRef:
+    kind: VSphereDatacenterConfig
+    name: test
+  clusterNetwork:
+    cni: "cilium"
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+  registryMirrorConfiguration:
+    endpoint: 1.2.3.4
+    port: 1234
+    ociNamespaces:
+      - registry: "public.ecr.aws"
+        namespace: "eks-anywhere"
+      - registry: "docker.io"
+        namespace: "eks-anywhere"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereMachineConfig
+metadata:
+  name: test-cp
+spec:
+  diskGiB: 25
+  cloneMode: linkedClone
+  datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore"
+  folder: "/SDDC-Datacenter/vm"
+  memoryMiB: 8192
+  numCPUs: 2
+  osFamily: bottlerocket
+  resourcePool: "*/Resources"
+  storagePolicyName: "vSAN Default Storage Policy"
+  template: "/SDDC-Datacenter/vm/Templates/bottlerocket-1804-kube-v1.19.6"
+  users:
+    - name: ec2-user
+      sshAuthorizedKeys:
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereMachineConfig
+metadata:
+  name: test-wn
+spec:
+  diskGiB: 25
+  cloneMode: linkedClone
+  datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore"
+  folder: "/SDDC-Datacenter/vm"
+  memoryMiB: 4096
+  numCPUs: 3
+  osFamily: bottlerocket
+  resourcePool: "*/Resources"
+  storagePolicyName: "vSAN Default Storage Policy"
+  template: "/SDDC-Datacenter/vm/Templates/bottlerocket-1804-kube-v1.19.6"
+  users:
+    - name: ec2-user
+      sshAuthorizedKeys:
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereMachineConfig
+metadata:
+  name: test-etcd
+spec:
+  diskGiB: 25
+  cloneMode: linkedClone
+  datastore: "/SDDC-Datacenter/datastore/WorkloadDatastore"
+  folder: "/SDDC-Datacenter/vm"
+  memoryMiB: 4096
+  numCPUs: 3
+  osFamily: bottlerocket
+  resourcePool: "*/Resources"
+  storagePolicyName: "vSAN Default Storage Policy"
+  template: "/SDDC-Datacenter/vm/Templates/bottlerocket-1804-kube-v1.19.6"
+  users:
+    - name: ec2-user
+      sshAuthorizedKeys:
+       - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: VSphereDatacenterConfig
+metadata:
+  name: test
+spec:
+  datacenter: "SDDC-Datacenter"
+  network: "/SDDC-Datacenter/network/sddc-cgw-network-1"
+  server: "vsphere_server"
+  thumbprint: "ABCDEFG"
+  insecure: false