diff --git a/addons/controllers/addon_controller.go b/addons/controllers/addon_controller.go index 53eb7782a94..be1a95f3a3c 100644 --- a/addons/controllers/addon_controller.go +++ b/addons/controllers/addon_controller.go @@ -10,6 +10,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -592,7 +593,7 @@ func (r *AddonReconciler) GetAddonKappResourceReconciler( } // GetExternalCRDs returns all external custom resources that addon controller depends on -func GetExternalCRDs() map[schema.GroupVersion]*sets.String { +func GetExternalCRDs() (map[schema.GroupVersion]*sets.String, []client.Object) { var crds = map[schema.GroupVersion]*sets.String{} // cluster-api clusterapiv1alpha3Resources := sets.NewString("clusters") @@ -612,5 +613,40 @@ func GetExternalCRDs() map[schema.GroupVersion]*sets.String { kapppkgv1alpha1Resources := sets.NewString("packageinstalls", "packagerepositories") crds[kapppkg.SchemeGroupVersion] = &kapppkgv1alpha1Resources - return crds + // Add CustomResourceDefinition for the above apis + clusterCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}}, + } + + kubeadmcontrolplaneCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "kubeadmcontrolplanes.controlplane.cluster.x-k8s.io"}, + } + + PackageInstallCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "packageinstalls.packaging.carvel.dev"}, + } + + tanzukubernetesreleaseCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "tanzukubernetesreleases.run.tanzu.vmware.com"}, + } + + packagerepositorieCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "packagerepositories.packaging.carvel.dev"}, + } + + appCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "apps.kappctrl.k14s.io"}, + } + + initObjs := []client.Object{ + clusterCRD.DeepCopy(), + kubeadmcontrolplaneCRD.DeepCopy(), + PackageInstallCRD.DeepCopy(), + tanzukubernetesreleaseCRD.DeepCopy(), + packagerepositorieCRD.DeepCopy(), + appCRD.DeepCopy(), + } + + return crds, initObjs } diff --git a/addons/controllers/clusterbootstrap_controller.go b/addons/controllers/clusterbootstrap_controller.go index 66602b0c1f0..017580b688e 100644 --- a/addons/controllers/clusterbootstrap_controller.go +++ b/addons/controllers/clusterbootstrap_controller.go @@ -48,6 +48,7 @@ import ( "github.com/vmware-tanzu/tanzu-framework/addons/pkg/util" "github.com/vmware-tanzu/tanzu-framework/addons/pkg/util/clusterbootstrapclone" "github.com/vmware-tanzu/tanzu-framework/addons/predicates" + "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3" runtanzuv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3" ) @@ -577,7 +578,19 @@ func (r *ClusterBootstrapReconciler) mergeClusterBootstrapPackagesWithTemplate( // Find the one to one match for additional package in new ClusterBootstrapTemplate and old ClusterBootstrap and update if pkg, ok := additionalPackageMap[packageRefName]; ok { + oldCBTPkg, err := r.getPackageFromCBTInResolvedTKR(pkg.RefName, cluster, updatedClusterBootstrap, log) + if err != nil { + return nil, err + } pkg.RefName = templatePkg.RefName + + if isCBPackageValuesFromNil(pkg) && oldCBTPkg != nil && isCBPackageValuesFromNil(oldCBTPkg) { + if templatePkg.ValuesFrom != nil { + pkg.ValuesFrom = templatePkg.ValuesFrom.DeepCopy() + packages = append(packages, pkg) + } + } + } else { // If new additional package is added in ClusterBootstrapTemplate, just add it to updated ClusterBootstrap newPkg := templatePkg.DeepCopy() @@ -604,6 +617,32 @@ func (r *ClusterBootstrapReconciler) mergeClusterBootstrapPackagesWithTemplate( return packages, nil } +func (r *ClusterBootstrapReconciler) getPackageFromCBTInResolvedTKR( + packageRefName string, + cluster *clusterapiv1beta1.Cluster, + updatedClusterBootstrap *runtanzuv1alpha3.ClusterBootstrap, + log logr.Logger) (*runtanzuv1alpha3.ClusterBootstrapPackage, error) { + + clusterBootstrapTemplate := &runtanzuv1alpha3.ClusterBootstrapTemplate{} + key := client.ObjectKey{Namespace: r.Config.SystemNamespace, Name: updatedClusterBootstrap.Status.ResolvedTKR} + if err := r.Client.Get(r.context, key, clusterBootstrapTemplate); err != nil { + log.Error(err, "unable to fetch ClusterBootstrapTemplate", "objectkey", key) + return nil, err + } + for _, pkg := range clusterBootstrapTemplate.Spec.AdditionalPackages { + if packageRefName == pkg.RefName { + return pkg, nil + } + } + + // If the specified package is not found, it is not an error condition. So return error as nil + return nil, nil +} + +func isCBPackageValuesFromNil(cbpkg *v1alpha3.ClusterBootstrapPackage) bool { + return cbpkg.ValuesFrom == nil || (cbpkg.ValuesFrom.ProviderRef == nil && cbpkg.ValuesFrom.SecretRef == "" && cbpkg.ValuesFrom.Inline == nil) +} + // createOrPatchKappPackageInstall contains the logic that create/update PackageInstall CR for kapp-controller on // mgmt cluster. The kapp-controller running on mgmt cluster reconciles the PackageInstall CR and creates kapp-controller resources // on remote workload cluster. This is required for a workload cluster and its corresponding package installations to be functional. diff --git a/addons/controllers/clusterbootstrap_controller_test.go b/addons/controllers/clusterbootstrap_controller_test.go index 61a220b768a..59b09bdee76 100644 --- a/addons/controllers/clusterbootstrap_controller_test.go +++ b/addons/controllers/clusterbootstrap_controller_test.go @@ -1524,6 +1524,118 @@ var _ = Describe("ClusterBootstrap Reconciler", func() { }) }) }) + + When("cluster upgrade with values change", func() { + BeforeEach(func() { + clusterName = "test-cluster-10" + clusterNamespace = "cluster-namespace-10" + clusterResourceFilePath = "testdata/test-cluster-bootstrap-10.yaml" + }) + + Context("cluster with no inline values are set", func() { + It("should work when no inline values are set", func() { + + By("filling ClusterBootstrap correctly", func() { + Eventually(func(g Gomega) { + clusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{} + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, clusterBootstrap) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(clusterBootstrap.Spec.CNI.RefName).To(BeEquivalentTo("antrea.tanzu.vmware.com.1.11.7--vmware.1-tkg.2")) + g.Expect(clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil()) + g.Expect(*clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com")) + g.Expect(clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.Kind).To(Equal("AntreaConfig")) + g.Expect(clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-antrea-package", clusterName))) + + // Validate Kapp + g.Expect(clusterBootstrap.Spec.Kapp.RefName).To(Equal("kapp-controller.tanzu.vmware.com.0.32.11")) + g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom).NotTo(BeNil()) + g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil()) + g.Expect(*clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.APIGroup).To(Equal("run.tanzu.vmware.com")) + g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.Kind).To(Equal("KappControllerConfig")) + g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-kapp-controller-package", clusterName))) + + // Validate additional packages + g.Expect(len(clusterBootstrap.Spec.AdditionalPackages)).To(Equal(2)) + + g.Expect(clusterBootstrap.Spec.AdditionalPackages[0].RefName).To(BeEquivalentTo("pinniped.tanzu.vmware.com.0.13.3--vmware.1-tkg.1")) + g.Expect(clusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom).NotTo(BeNil()) + g.Expect(clusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom.SecretRef).To(BeEquivalentTo(fmt.Sprintf("%s-pinniped-package", clusterName))) + + g.Expect(clusterBootstrap.Spec.AdditionalPackages[1].RefName).To(BeEquivalentTo("foobar.tanzu.vmware.com.0.2.4--vmware.1-tkg.1")) + //g.Expect(clusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom).To(BeNil()) + }, waitTimeout, pollingInterval).Should(Succeed()) + }) + + By("setting cluster phase to provisioned", func() { + cluster := &clusterapiv1beta1.Cluster{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed()) + cluster.Status.Phase = string(clusterapiv1beta1.ClusterPhaseProvisioned) + Expect(k8sClient.Status().Update(ctx, cluster)).To(Succeed()) + }) + + By("upgrading Cluster to newer TKR", func() { + cluster := &clusterapiv1beta1.Cluster{} + newTKRVersion := "v1.25.6-custom" + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed()) + cluster.Labels[constants.TKRLabelClassyClusters] = newTKRVersion + + // Mock cluster pause mutating webhook + cluster.Spec.Paused = true + if cluster.Annotations == nil { + cluster.Annotations = map[string]string{} + } + cluster.Annotations[constants.ClusterPauseLabel] = newTKRVersion + Expect(k8sClient.Update(ctx, cluster)).To(Succeed()) + + Eventually(func(g Gomega) { + upgradedClusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{} + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), upgradedClusterBootstrap) + + g.Expect(err).To(BeNil()) + g.Expect(upgradedClusterBootstrap.Status.ResolvedTKR).To(BeEquivalentTo(newTKRVersion)) + + // Validate CNI + cni := upgradedClusterBootstrap.Spec.CNI + // Note: The value of CNI has been bumped to the one in TKR after cluster upgrade + g.Expect(cni.RefName).To(Equal("antrea.tanzu.vmware.com.1.11.8--vmware.1-tkg.2")) + g.Expect(cni.ValuesFrom).NotTo(BeNil()) + g.Expect(cni.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil()) + g.Expect(*cni.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com")) + g.Expect(cni.ValuesFrom.ProviderRef.Kind).To(Equal("AntreaConfig")) + g.Expect(cni.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-antrea-package", clusterName))) + + // Validate Kapp + kapp := upgradedClusterBootstrap.Spec.Kapp + g.Expect(kapp.RefName).To(Equal("kapp-controller.tanzu.vmware.com.0.32.12")) + g.Expect(kapp.ValuesFrom).NotTo(BeNil()) + g.Expect(kapp.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil()) + g.Expect(*kapp.ValuesFrom.ProviderRef.APIGroup).To(Equal("run.tanzu.vmware.com")) + g.Expect(kapp.ValuesFrom.ProviderRef.Kind).To(Equal("KappControllerConfig")) + g.Expect(kapp.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-kapp-controller-package", clusterName))) + + // Validate additional packages + g.Expect(len(upgradedClusterBootstrap.Spec.AdditionalPackages)).To(Equal(2)) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[0].RefName).To(BeEquivalentTo("pinniped.tanzu.vmware.com.0.13.4--vmware.1-tkg.1")) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom).NotTo(BeNil()) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom.SecretRef).To(BeEquivalentTo(fmt.Sprintf("%s-pinniped-package", clusterName))) + + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].RefName).To(BeEquivalentTo("foobar.tanzu.vmware.com.0.2.5--vmware.1-tkg.1")) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom).NotTo(BeNil()) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom.Inline).NotTo(BeNil()) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom.Inline["key1"]).To(Equal("value1")) + g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom.Inline["key2"].(map[string]interface{})["key3"]).To(Equal("value3")) + + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed()) + g.Expect(cluster.Spec.Paused).ToNot(BeTrue()) + if cluster.Annotations != nil { + _, ok := cluster.Annotations[constants.ClusterPauseLabel] + g.Expect(ok).ToNot(BeTrue()) + } + }, waitTimeout, pollingInterval).Should(Succeed()) + }) + }) + }) + }) }) func assertSecretContains(ctx context.Context, k8sClient client.Client, namespace, name string, secretContent map[string][]byte) { diff --git a/addons/controllers/suite_test.go b/addons/controllers/suite_test.go index d4522cd40e9..230c9bc42dc 100644 --- a/addons/controllers/suite_test.go +++ b/addons/controllers/suite_test.go @@ -18,12 +18,14 @@ import ( . "github.com/onsi/gomega" capociv1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" capvv1beta1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" @@ -33,6 +35,7 @@ import ( controlplanev1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" @@ -234,6 +237,9 @@ var _ = BeforeSuite(func(done Done) { err = topologyv1alpha1.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred()) + err = apiextensionsv1.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).ToNot(HaveOccurred()) Expect(k8sClient).ToNot(BeNil()) @@ -262,18 +268,24 @@ var _ = BeforeSuite(func(done Done) { Expect(err).ToNot(HaveOccurred()) ctx, cancel = context.WithCancel(ctx) + + initObjs := []client.Object{} + fakeApiReader := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + crdwaiter := crdwait.CRDWaiter{ - Ctx: ctx, - ClientSetFn: func() (kubernetes.Interface, error) { - return kubernetes.NewForConfig(cfg) - }, + Ctx: ctx, + ClientSet: fake.NewSimpleClientset(), + APIReader: fakeApiReader, Logger: setupLog, Scheme: scheme, PollInterval: constants.CRDWaitPollInterval, PollTimeout: constants.CRDWaitPollTimeout, } - if err := crdwaiter.WaitForCRDs(GetExternalCRDs(), + crds, initObjs := GetExternalCRDs() + crdwaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + + if err := crdwaiter.WaitForCRDs(crds, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}}, constants.AddonControllerName, ); err != nil { diff --git a/addons/controllers/testdata/test-cluster-bootstrap-10.yaml b/addons/controllers/testdata/test-cluster-bootstrap-10.yaml new file mode 100644 index 00000000000..465b9034321 --- /dev/null +++ b/addons/controllers/testdata/test-cluster-bootstrap-10.yaml @@ -0,0 +1,966 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cluster-namespace-10 +--- +apiVersion: run.tanzu.vmware.com/v1alpha3 +kind: TanzuKubernetesRelease +metadata: + name: v1.25.5-custom +spec: + version: v1.25.5 + kubernetes: + version: v1.25.5 + imageRepository: foo + osImages: [] + bootstrapPackages: + - name: kapp-controller.tanzu.vmware.com.0.32.11 + - name: antrea.tanzu.vmware.com.1.11.7--vmware.1-tkg.2 + - name: calico.tanzu.vmware.com.1.3.7--vmware.1-tkg.1 + - name: pinniped.tanzu.vmware.com.0.13.3--vmware.1-tkg.1 + - name: foobar.tanzu.vmware.com.0.2.4--vmware.1-tkg.1 +--- +apiVersion: run.tanzu.vmware.com/v1alpha3 +kind: ClusterBootstrapTemplate +metadata: + name: v1.25.5-custom + namespace: tkg-system +spec: + kapp: + refName: kapp-controller.tanzu.vmware.com.0.32.11 + valuesFrom: + providerRef: + apiGroup: run.tanzu.vmware.com + kind: KappControllerConfig + name: kapp-config-custom-1 + cni: + refName: antrea.tanzu.vmware.com.1.11.7--vmware.1-tkg.2 + valuesFrom: + providerRef: + apiGroup: cni.tanzu.vmware.com + kind: AntreaConfig + name: antrea-config-custom-1 + additionalPackages: + - refName: pinniped.tanzu.vmware.com.0.13.3--vmware.1-tkg.1 + valuesFrom: + secretRef: default-pinniped-config-v1.23.6---vmware.1-tkg.1 + - refName: foobar.tanzu.vmware.com.0.2.4--vmware.1-tkg.1 +--- +apiVersion: run.tanzu.vmware.com/v1alpha3 +kind: TanzuKubernetesRelease +metadata: + name: v1.25.6-custom +spec: + version: v1.25.6 + kubernetes: + version: v1.25.6 + imageRepository: foo + osImages: [] + bootstrapPackages: + - name: kapp-controller.tanzu.vmware.com.0.32.12 + - name: antrea.tanzu.vmware.com.1.11.8--vmware.1-tkg.2 + - name: calico.tanzu.vmware.com.1.3.8--vmware.1-tkg.1 + - name: pinniped.tanzu.vmware.com.0.13.4--vmware.1-tkg.1 + - name: foobar.tanzu.vmware.com.0.2.5--vmware.1-tkg.1 +--- +apiVersion: run.tanzu.vmware.com/v1alpha3 +kind: ClusterBootstrapTemplate +metadata: + name: v1.25.6-custom + namespace: tkg-system +spec: + kapp: + refName: kapp-controller.tanzu.vmware.com.0.32.12 + valuesFrom: + providerRef: + apiGroup: run.tanzu.vmware.com + kind: KappControllerConfig + name: kapp-config-custom-1 + cni: + refName: antrea.tanzu.vmware.com.1.11.8--vmware.1-tkg.2 + valuesFrom: + providerRef: + apiGroup: cni.tanzu.vmware.com + kind: AntreaConfig + name: antrea-config-custom-1 + additionalPackages: + - refName: pinniped.tanzu.vmware.com.0.13.4--vmware.1-tkg.1 + valuesFrom: + secretRef: default-pinniped-config-v1.23.7---vmware.1-tkg.1 + - refName: foobar.tanzu.vmware.com.0.2.5--vmware.1-tkg.1 + valuesFrom: + inline: + key1: value1 + key2: + key3: value3 +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: kapp-controller.tanzu.vmware.com.0.32.11 + namespace: tkg-system +spec: + refName: kapp-controller.tanzu.vmware.com + version: 0.32.11 + releaseNotes: kapp-controller 0.32.11 https://github.com/vmware-tanzu/carvel-kapp-controller + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.32.11_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-30T10:59:32Z" + valuesSchema: + openAPIv3: + title: kapp-controller.tanzu.vmware.com.0.32.11+vmware.1-tkg.1 values schema +--- +# manually sync package to required namespace (done by kapp-controller on a real cluster) +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: kapp-controller.tanzu.vmware.com.0.32.11 + namespace: cluster-namespace-10 +spec: + refName: kapp-controller.tanzu.vmware.com + version: 0.32.11 + releaseNotes: kapp-controller 0.32.11 https://github.com/vmware-tanzu/carvel-kapp-controller + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.32.11_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-30T10:59:32Z" + valuesSchema: + openAPIv3: + title: kapp-controller.tanzu.vmware.com.0.32.11+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: kapp-controller.tanzu.vmware.com.0.32.12 + namespace: tkg-system +spec: + refName: kapp-controller.tanzu.vmware.com + version: 0.32.12 + releaseNotes: kapp-controller 0.32.12 https://github.com/vmware-tanzu/carvel-kapp-controller + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.32.12_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-30T10:59:32Z" + valuesSchema: + openAPIv3: + title: kapp-controller.tanzu.vmware.com.0.32.12+vmware.1-tkg.1 values schema +--- +# manually sync package to required namespace (done by kapp-controller on a real cluster) +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: kapp-controller.tanzu.vmware.com.0.32.12 + namespace: cluster-namespace-10 +spec: + refName: kapp-controller.tanzu.vmware.com + version: 0.32.12 + releaseNotes: kapp-controller 0.32.12 https://github.com/vmware-tanzu/carvel-kapp-controller + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/kapp-controller:v0.32.12_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-30T10:59:32Z" + valuesSchema: + openAPIv3: + title: kapp-controller.tanzu.vmware.com.0.32.12+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: antrea.tanzu.vmware.com.1.11.7--vmware.1-tkg.2 + namespace: tkg-system +spec: + refName: antrea.tanzu.vmware.com + version: 1.11.7+vmware.1-tkg.1 + releaseNotes: antrea 1.11.7 https://github.com/antrea-io/antrea/releases/tag/v1.11.7 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.11.7_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: antrea.tanzu.vmware.com.1.11.7+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: antrea.tanzu.vmware.com.1.11.7--vmware.1-tkg.2 + namespace: cluster-namespace-10 +spec: + refName: antrea.tanzu.vmware.com + version: 1.11.7+vmware.1-tkg.1 + releaseNotes: antrea 1.11.7 https://github.com/antrea-io/antrea/releases/tag/v1.11.7 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.11.7_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: antrea.tanzu.vmware.com.1.11.7+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: antrea.tanzu.vmware.com.1.11.8--vmware.1-tkg.2 + namespace: tkg-system +spec: + refName: antrea.tanzu.vmware.com + version: 1.11.8+vmware.1-tkg.1 + releaseNotes: antrea 1.11.8 https://github.com/antrea-io/antrea/releases/tag/v1.11.8 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.11.8_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: antrea.tanzu.vmware.com.1.11.8+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: antrea.tanzu.vmware.com.1.11.8--vmware.1-tkg.2 + namespace: cluster-namespace-10 +spec: + refName: antrea.tanzu.vmware.com + version: 1.11.8+vmware.1-tkg.1 + releaseNotes: antrea 1.11.8 https://github.com/antrea-io/antrea/releases/tag/v1.11.8 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/antrea:v1.11.8_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: antrea.tanzu.vmware.com.1.11.8+vmware.1-tkg.1 values schema +--- +# manually sync package to required namespace (done by kapp-controller on a real cluster) +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: calico.tanzu.vmware.com.1.3.7--vmware.1-tkg.1 + namespace: tkg-system +spec: + refName: calico.tanzu.vmware.com + version: 1.3.7+vmware.1-tkg.1 + releaseNotes: calico 1.3.7 https://github.com/calico-io/calico/releases/tag/v1.3.7 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v1.3.7_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: calico.tanzu.vmware.com.1.3.7+vmware.1-tkg.1 values schema +--- +# manually sync package to required namespace (done by kapp-controller on a real cluster) +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: calico.tanzu.vmware.com.1.3.7--vmware.1-tkg.1 + namespace: cluster-namespace-10 +spec: + refName: calico.tanzu.vmware.com + version: 1.3.7+vmware.1-tkg.1 + releaseNotes: calico 1.3.7 https://github.com/calico-io/calico/releases/tag/v1.3.7 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v1.3.7_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: calico.tanzu.vmware.com.1.3.7+vmware.1-tkg.1 values schema +--- +# manually sync package to required namespace (done by kapp-controller on a real cluster) +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: calico.tanzu.vmware.com.1.3.8--vmware.1-tkg.1 + namespace: tkg-system +spec: + refName: calico.tanzu.vmware.com + version: 1.3.8+vmware.1-tkg.1 + releaseNotes: calico 1.3.8 https://github.com/calico-io/calico/releases/tag/v1.3.8 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v1.3.8_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: calico.tanzu.vmware.com.1.3.8+vmware.1-tkg.1 values schema +--- +# manually sync package to required namespace (done by kapp-controller on a real cluster) +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: calico.tanzu.vmware.com.1.3.8--vmware.1-tkg.1 + namespace: cluster-namespace-10 +spec: + refName: calico.tanzu.vmware.com + version: 1.3.7+vmware.1-tkg.1 + releaseNotes: calico 1.3.7 https://github.com/calico-io/calico/releases/tag/v1.3.7 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/tkgextensions-dev/packages/core/calico:v1.3.7_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2021-12-20T10:59:32Z" + valuesSchema: + openAPIv3: + title: calico.tanzu.vmware.com.1.3.8+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: pinniped.tanzu.vmware.com.0.13.3--vmware.1-tkg.1 + namespace: tkg-system +spec: + refName: pinniped.tanzu.vmware.com + version: 0.13.3+vmware.1-tkg.1 + releaseNotes: pinniped 0.13.3 https://github.com/vmware-tanzu/pinniped/releases/tag/v0.13.3 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/pinniped:v0.13.3_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: pinniped.tanzu.vmware.com.0.13.3+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: pinniped.tanzu.vmware.com.0.13.3--vmware.1-tkg.1 + namespace: cluster-namespace-10 +spec: + refName: pinniped.tanzu.vmware.com + version: 0.13.3+vmware.1-tkg.1 + releaseNotes: pinniped 0.13.3 https://github.com/vmware-tanzu/pinniped/releases/tag/v0.13.3 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/pinniped:v0.13.3_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: pinniped.tanzu.vmware.com.0.13.3+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: pinniped.tanzu.vmware.com.0.13.4--vmware.1-tkg.1 + namespace: tkg-system +spec: + refName: pinniped.tanzu.vmware.com + version: 0.13.4+vmware.1-tkg.1 + releaseNotes: pinniped 0.13.4 https://github.com/vmware-tanzu/pinniped/releases/tag/v0.13.4 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/pinniped:v0.13.4_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: pinniped.tanzu.vmware.com.0.13.4+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: pinniped.tanzu.vmware.com.0.13.4--vmware.1-tkg.1 + namespace: cluster-namespace-10 +spec: + refName: pinniped.tanzu.vmware.com + version: 0.13.4+vmware.1-tkg.1 + releaseNotes: pinniped 0.13.4 https://github.com/vmware-tanzu/pinniped/releases/tag/v0.13.4 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/pinniped:v0.13.4_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: pinniped.tanzu.vmware.com.0.13.4+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: foobar.tanzu.vmware.com.0.2.4--vmware.1-tkg.1 + namespace: tkg-system +spec: + refName: foobar.tanzu.vmware.com + version: 0.2.4+vmware.1-tkg.1 + releaseNotes: foobar 0.2.4 https://github.com/vmware-tanzu/foobar/releases/tag/v0.2.4 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/foobar:v0.2.4_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: foobar.tanzu.vmware.com.0.2.4+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: foobar.tanzu.vmware.com.0.2.4--vmware.1-tkg.1 + namespace: cluster-namespace-10 +spec: + refName: foobar.tanzu.vmware.com + version: 0.2.4+vmware.1-tkg.1 + releaseNotes: foobar 0.2.4 https://github.com/vmware-tanzu/foobar/releases/tag/v0.2.4 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/foobar:v0.2.4_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: foobar.tanzu.vmware.com.0.2.4+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: foobar.tanzu.vmware.com.0.2.5--vmware.1-tkg.1 + namespace: tkg-system +spec: + refName: foobar.tanzu.vmware.com + version: 0.2.5+vmware.1-tkg.1 + releaseNotes: foobar 0.2.5 https://github.com/vmware-tanzu/foobar/releases/tag/v0.2.5 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/foobar:v0.2.5_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: foobar.tanzu.vmware.com.0.2.5+vmware.1-tkg.1 values schema +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: foobar.tanzu.vmware.com.0.2.5--vmware.1-tkg.1 + namespace: cluster-namespace-10 +spec: + refName: foobar.tanzu.vmware.com + version: 0.2.5+vmware.1-tkg.1 + releaseNotes: foobar 0.2.5 https://github.com/vmware-tanzu/foobar/releases/tag/v0.2.5 + licenses: + - 'VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)' + template: + spec: + fetch: + - imgpkgBundle: + image: projects-stg.registry.vmware.com/tkg/packages/core/foobar:v0.2.5_vmware.1-tkg.1 + template: + - ytt: + paths: + - config/ + ignoreUnknownComments: true + - kbld: + paths: + - '-' + - kbld-config.yaml + - .imgpkg/images.yml + deploy: + - kapp: + rawOptions: + - --dangerous-allow-empty-list-of-resources=true + - --wait-timeout=30s + - --kube-api-qps=20 + - --kube-api-burst=30 + releasedAt: "2022-06-21T16:00:43Z" + valuesSchema: + openAPIv3: + title: foobar.tanzu.vmware.com.0.2.5+vmware.1-tkg.1 values schema +--- +apiVersion: cni.tanzu.vmware.com/v1alpha2 +kind: AntreaConfig +metadata: + name: antrea-config-custom-1 + namespace: tkg-system +spec: + antrea: + config: + trafficEncapMode: encap + noSNAT: false + tlsCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384 + disableUdpTunnelOffload: false + featureGates: + AntreaProxy: true + EndpointSlice: true + AntreaPolicy: true + NodePortLocal: false + AntreaTraceflow: true + Egress: false + FlowExporter: false +--- +apiVersion: run.tanzu.vmware.com/v1alpha3 +kind: KappControllerConfig +metadata: + name: kapp-config-custom-1 + namespace: tkg-system +spec: + namespace: test-ns + kappController: + createNamespace: true + globalNamespace: tanzu-package-repo-global + deployment: + concurrency: 4 + hostNetwork: true + priorityClassName: system-cluster-critical + apiPort: 10100 + metricsBindAddress: "0" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" +--- +apiVersion: v1 +kind: Secret +metadata: + name: default-pinniped-config-v1.23.6---vmware.1-tkg.1 + namespace: tkg-system +stringData: + values.yaml: | + infrastructure_provider: vsphere + identity_management_type: none + tkg_cluster_role: workload +--- +apiVersion: v1 +kind: Secret +metadata: + name: default-pinniped-config-v1.23.7---vmware.1-tkg.1 + namespace: cluster-namespace-10 +stringData: + values.yaml: | + infrastructure_provider: vsphere + identity_management_type: none + tkg_cluster_role: workload +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test-cluster-10 + namespace: cluster-namespace-10 + labels: + tkg.tanzu.vmware.com/cluster-name: test-cluster-10 + run.tanzu.vmware.com/tkr: v1.25.5-custom +spec: + infrastructureRef: + kind: VSphereCluster + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: test-cluster-10 + namespace: cluster-namespace-10 + clusterNetwork: + pods: + cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ] + services: + cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ] +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereCluster +metadata: + name: test-cluster-10 + namespace: cluster-namespace-10 +spec: + identityRef: + kind: Secret + name: test-cluster-tcbt-5 + thumbprint: test-thumbprint + server: vsphere-server.local +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: test-cluster-10 + namespace: cluster-namespace-10 +spec: + template: + spec: + datacenter: dc0 + template: /dc0/vm/photon-3-kube-v1.22.3+vmware.1-tkg.2 + network: + devices: + - networkName: test-network + dhcp6: true + dhcp4: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-cluster-tcbt-5 + namespace: cluster-namespace-10 +data: + password: QWRtaW4hMjM= # Admin!23 + username: YWRtaW5pc3RyYXRvckB2c3BoZXJlLmxvY2Fs # administrator@vsphere.local diff --git a/addons/controllers/webhooks_test.go b/addons/controllers/webhooks_test.go index 3ef6880d9c0..70d2901e10c 100644 --- a/addons/controllers/webhooks_test.go +++ b/addons/controllers/webhooks_test.go @@ -65,7 +65,7 @@ var _ = Describe("when webhook TLS is being continuously managed", func() { }) }) Context("if rotation time is longer than one week", func() { - It("should set rotation to one week", func() { + It("should set rotation to one week minus required minimum grace period", func() { rotationTime := oneWeek + oneDay managementFrequency := rotationTime / 2 webhookTLS := webhooks.WebhookTLS{ @@ -82,7 +82,7 @@ var _ = Describe("when webhook TLS is being continuously managed", func() { } err := webhookTLS.ManageCertificates(managementFrequency) Expect(err).ToNot(HaveOccurred()) - Expect(webhookTLS.RotationTime).To(Equal(oneWeek)) + Expect(webhookTLS.RotationTime).To(Equal(oneWeek - webhooks.RequireMinGracePeriod)) }) }) @@ -108,6 +108,27 @@ var _ = Describe("when webhook TLS is being continuously managed", func() { }) }) + Context("if rotation times is less than minGracePeriod", func() { + It("should set rotation to match minGracePeriod", func() { + rotationTime := oneWeek - webhooks.RequireMinGracePeriod + time.Second*1 + managementFrequency := oneDay // any value larger than 0 will work here + webhookTLS := webhooks.WebhookTLS{ + Ctx: context.Background(), + K8sConfig: k8sConfig, + CertPath: certPath, + KeyPath: keyPath, + Name: constants.WebhookScrtName, + ServiceName: webhookServiceName, + LabelSelector: constants.AddonWebhookLabelKey, + Logger: setupLog, + Namespace: addonNamespace, + RotationTime: rotationTime, + } + err := webhookTLS.ManageCertificates(managementFrequency) + Expect(err).ToNot(HaveOccurred()) + Expect(webhookTLS.RotationTime).To(Equal(oneWeek - webhooks.RequireMinGracePeriod)) + }) + }) Context("if rotation time is one week", func() { It("tls certificates should be validated and not rotated until one week", func() { privCtx := context.Background() diff --git a/addons/main.go b/addons/main.go index be00f507e3a..3c9b88a686b 100644 --- a/addons/main.go +++ b/addons/main.go @@ -187,23 +187,6 @@ func main() { }() } - crdwaiter := crdwait.CRDWaiter{ - Ctx: ctx, - ClientSetFn: func() (kubernetes.Interface, error) { - return kubernetes.NewForConfig(ctrl.GetConfigOrDie()) - }, - Logger: setupLog, - Scheme: scheme, - PollInterval: constants.CRDWaitPollInterval, - PollTimeout: constants.CRDWaitPollTimeout, - } - if err := crdwaiter.WaitForCRDs(controllers.GetExternalCRDs(), - &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: os.Getenv("POD_NAME"), Namespace: os.Getenv("POD_NAMESPACE")}}, - constants.AddonControllerName, - ); err != nil { - setupLog.Error(err, "unable to wait for CRDs") - os.Exit(1) - } if flags.featureGateClusterBootstrap { if err := os.MkdirAll(constants.WebhookCertDir, 0755); err != nil { setupLog.Error(err, "unable to create directory for webhook certificates", "directory", constants.WebhookCertDir) @@ -223,6 +206,32 @@ func main() { SyncPeriod: &flags.syncPeriod, HealthProbeBindAddress: flags.healthdAddr, }) + + clientSet, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + setupLog.Error(err, "unable to create kubernetes clientset") + os.Exit(1) + } + + crdwaiter := crdwait.CRDWaiter{ + Ctx: ctx, + APIReader: mgr.GetAPIReader(), + ClientSet: clientSet, + Logger: setupLog, + Scheme: scheme, + PollInterval: constants.CRDWaitPollInterval, + PollTimeout: constants.CRDWaitPollTimeout, + } + + crds, _ := controllers.GetExternalCRDs() + if err := crdwaiter.WaitForCRDs(crds, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: os.Getenv("POD_NAME"), Namespace: os.Getenv("POD_NAMESPACE")}}, + constants.AddonControllerName, + ); err != nil { + setupLog.Error(err, "unable to wait for CRDs") + os.Exit(1) + } + if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) diff --git a/addons/pkg/crdwait/crd_wait.go b/addons/pkg/crdwait/crd_wait.go index 01fd5a4df7e..6ec9efe5467 100644 --- a/addons/pkg/crdwait/crd_wait.go +++ b/addons/pkg/crdwait/crd_wait.go @@ -10,6 +10,9 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" @@ -17,14 +20,14 @@ import ( "k8s.io/client-go/kubernetes" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" ) -type clientSetProvider func() (kubernetes.Interface, error) - // CRDWaiter waits for api-resources and emits logs and events for missing resources type CRDWaiter struct { Ctx context.Context - ClientSetFn clientSetProvider + ClientSet kubernetes.Interface + APIReader client.Reader Logger logr.Logger Scheme *runtime.Scheme PollInterval time.Duration @@ -34,12 +37,7 @@ type CRDWaiter struct { // WaitForCRDs checks if CRDs are available by polling for resources. func (c *CRDWaiter) WaitForCRDs(crds map[schema.GroupVersion]*sets.String, object runtime.Object, controllerName string) error { - cs, err := c.ClientSetFn() - if err != nil { - return err - } - - eventRecorder, eventBroadcaster := c.getEventRecorder(cs, controllerName) + eventRecorder, eventBroadcaster := c.getEventRecorder(c.ClientSet, controllerName) go func() { <-c.Ctx.Done() eventBroadcaster.Shutdown() @@ -47,11 +45,6 @@ func (c *CRDWaiter) WaitForCRDs(crds map[schema.GroupVersion]*sets.String, objec poller := func() (done bool, err error) { // Create new clientset for every invocation to avoid caching of resources - cs, err := c.ClientSetFn() - if err != nil { - return false, err - } - allFound := true for gv, resources := range crds { // All resources found, do nothing @@ -59,21 +52,31 @@ func (c *CRDWaiter) WaitForCRDs(crds map[schema.GroupVersion]*sets.String, objec delete(crds, gv) continue } - - // Get the Resources for this GroupVersion groupVersion := gv.String() - - resourceList, err := cs.Discovery().ServerResourcesForGroupVersion(groupVersion) - if err != nil { - c.Logger.Info("error retrieving GroupVersion", "GroupVersion", groupVersion) - eventRecorder.Eventf(object, corev1.EventTypeWarning, - "Polling for GroupVersion", "The GroupVersion '%s' is not available yet", groupVersion) - return false, nil - } - - // Remove each found resource from the resources set that we are waiting for - for i := 0; i < len(resourceList.APIResources); i++ { - resources.Delete(resourceList.APIResources[i].Name) + for _, resource := range resources.List() { + // Get the Resources for this GroupVersion + + res := gv.WithResource(resource) + crd := unstructured.Unstructured{} + crd.SetGroupVersionKind(apiextensions.SchemeGroupVersion.WithKind("CustomResourceDefinition")) + name := res.GroupResource().String() + err := c.APIReader.Get(c.Ctx, + client.ObjectKey{Name: name}, + &crd) + + if apierrors.IsNotFound(err) { + c.Logger.Info("CRD not available yet", "name", name) + eventRecorder.Eventf(object, corev1.EventTypeWarning, + "Polling for CRD", "The CRD '%s' is not available yet", name) + return false, nil + } + if err != nil { + c.Logger.Error(err, "error retrieving CRD", "name", name) + eventRecorder.Eventf(object, corev1.EventTypeWarning, + "Error retrieving CRD %s", "Error trying to read CRD '%s'. Exiting", name) + return false, err + } + resources.Delete(resource) } // Still waiting on some resources in this group version diff --git a/addons/pkg/crdwait/crd_wait_test.go b/addons/pkg/crdwait/crd_wait_test.go index ee89f0c19c2..59b950a9816 100644 --- a/addons/pkg/crdwait/crd_wait_test.go +++ b/addons/pkg/crdwait/crd_wait_test.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -23,6 +24,8 @@ import ( "k8s.io/klog/v2/klogr" clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" controlplanev1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" kappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" kapppkg "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" @@ -30,6 +33,7 @@ import ( ) var fakeClientSet *fake.Clientset +var fakeApiReader client.Reader var _ = Describe("WaitForCRDs", func() { @@ -37,7 +41,10 @@ var _ = Describe("WaitForCRDs", func() { var crdWaiter CRDWaiter var fakeRecorder *record.FakeRecorder scheme = runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) + err := clientgoscheme.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + err = apiextensionsv1.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", @@ -46,11 +53,16 @@ var _ = Describe("WaitForCRDs", func() { BeforeEach(func() { fakeClientSet = fake.NewSimpleClientset(pod) + + initObjs := []client.Object{pod.DeepCopy()} + fakeApiReader := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + crdWaiter = CRDWaiter{ - Ctx: ctx, - ClientSetFn: getFakeClientSet, - Logger: klogr.New(), - Scheme: scheme, + Ctx: ctx, + ClientSet: fakeClientSet, + APIReader: fakeApiReader, + Logger: klogr.New(), + Scheme: scheme, } fakeRecorder = record.NewFakeRecorder(50) crdWaiter.eventRecorder = fakeRecorder @@ -95,8 +107,14 @@ var _ = Describe("WaitForCRDs", func() { crdWaiter.PollInterval = time.Second crdWaiter.PollTimeout = time.Second - Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).NotTo(HaveOccurred()) + clusterCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}}} + + initObjs := []client.Object{clusterCRD.DeepCopy(), pod.DeepCopy()} + crdWaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).NotTo(HaveOccurred()) }) }) @@ -138,6 +156,42 @@ var _ = Describe("WaitForCRDs", func() { crdWaiter.PollInterval = time.Second crdWaiter.PollTimeout = time.Second + clusterCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}}, + } + + kubeadmcontrolplaneCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "kubeadmcontrolplanes.controlplane.cluster.x-k8s.io"}, + } + + PackageInstallCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "packageinstalls.packaging.carvel.dev"}, + } + + tanzukubernetesreleaseCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "tanzukubernetesreleases.run.tanzu.vmware.com"}, + } + + packagerepositorieCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "packagerepositories.packaging.carvel.dev"}, + } + + appCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "apps.kappctrl.k14s.io"}, + } + + initObjs := []client.Object{ + clusterCRD.DeepCopy(), + kubeadmcontrolplaneCRD.DeepCopy(), + PackageInstallCRD.DeepCopy(), + tanzukubernetesreleaseCRD.DeepCopy(), + packagerepositorieCRD.DeepCopy(), + appCRD.DeepCopy(), + pod.DeepCopy(), + } + crdWaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + Expect(crdWaiter.WaitForCRDs(getCRDs(), pod, "foo")).NotTo(HaveOccurred()) }) }) @@ -153,14 +207,28 @@ var _ = Describe("WaitForCRDs", func() { return true, nil, nil }) + clusterCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}}, + } + initObjs := []client.Object{clusterCRD.DeepCopy(), pod.DeepCopy()} + client := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + err = client.Delete(ctx, clusterCRD) + Expect(err).ToNot(HaveOccurred()) + go func() { time.Sleep(time.Second * 3) + fakeClientSet.Resources = append(fakeClientSet.Resources, &metav1.APIResourceList{GroupVersion: clusterapiv1beta1.GroupVersion.String(), APIResources: []metav1.APIResource{ {Name: "clusters", Namespaced: true, Kind: "Cluster"}, }, }) + + err = client.Create(ctx, clusterCRD) + Expect(err).ToNot(HaveOccurred()) + crdWaiter.APIReader = client }() crdWaiter.PollInterval = time.Second @@ -182,29 +250,7 @@ var _ = Describe("WaitForCRDs", func() { crdWaiter.PollTimeout = time.Second * 2 Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).To(HaveOccurred()) - Expect(<-fakeRecorder.Events).To(ContainSubstring(fmt.Sprintf("The GroupVersion '%s' is not available yet", clusterapiv1beta1.GroupVersion.String()))) - - }) - }) - - Context("when GroupVersion exists but not all api-resources exist ", func() { - It("should fail and emit events for missing api-resources", func() { - var crds = map[schema.GroupVersion]*sets.String{} - // cluster-api - clusterapiv1alpha3Resources := sets.NewString("clusters") - crds[clusterapiv1beta1.GroupVersion] = &clusterapiv1alpha3Resources - - fakeClientSet.Resources = append(fakeClientSet.Resources, - &metav1.APIResourceList{GroupVersion: clusterapiv1beta1.GroupVersion.String(), - APIResources: []metav1.APIResource{ - {Name: "foo", Namespaced: true, Kind: "Cluster"}, - }, - }) - crdWaiter.PollInterval = time.Second - crdWaiter.PollTimeout = time.Second * 2 - - Expect(crdWaiter.WaitForCRDs(crds, pod, "foo")).To(HaveOccurred()) - Expect(<-fakeRecorder.Events).To(ContainSubstring(fmt.Sprintf("The api-resources '[clusters]' in GroupVersion '%s' are not available yet", clusterapiv1beta1.GroupVersion.String()))) + Expect(<-fakeRecorder.Events).To(ContainSubstring(fmt.Sprintf("The CRD 'clusters.cluster.x-k8s.io' is not available yet"))) }) }) diff --git a/addons/pkg/webhooks/webhooks.go b/addons/pkg/webhooks/webhooks.go index ceb4af73a7a..239659f1dcc 100644 --- a/addons/pkg/webhooks/webhooks.go +++ b/addons/pkg/webhooks/webhooks.go @@ -32,14 +32,19 @@ type WebhookTLS struct { } const ( - oneWeek = time.Hour * 24 * 7 - oneDay = time.Hour * 24 + oneWeek = time.Hour * 24 * 7 + oneDay = time.Hour * 24 + RequireMinGracePeriod = time.Hour * 24 ) func (w *WebhookTLS) UpdateOrCreate() error { - if w.RotationTime > oneWeek { - w.Logger.Info("rotation time will be set to maximum allowed value of one Week") - w.RotationTime = oneWeek + // We need to have enough time (RequireMinGracePeriod) to rotate the certificates before their end of life (one week) to avoid race conditions. + // The function we are solving for is "RotationTime + RequiredMinGracePeriod <= oneWeek && RotationTime > 0" + maxAllowedRotationTime := oneWeek - RequireMinGracePeriod // ensures gracePeriod > 0 + + if w.RotationTime > maxAllowedRotationTime { + w.Logger.Info("rotation time will be set to maximum allowed value of " + maxAllowedRotationTime.String()) + w.RotationTime = maxAllowedRotationTime } if w.RotationTime <= time.Second*0 { w.Logger.Info("rotation may not be 0 or less than 0, setting rotation to one day") diff --git a/apis/core/v1alpha2/featuregate_webhook.go b/apis/core/v1alpha2/featuregate_webhook.go index 704fef47c6d..ba5be35d206 100644 --- a/apis/core/v1alpha2/featuregate_webhook.go +++ b/apis/core/v1alpha2/featuregate_webhook.go @@ -56,6 +56,16 @@ func (r *FeatureGate) getClient() (client.Client, error) { // SetupWebhookWithManager adds the webhook to the manager. func (r *FeatureGate) SetupWebhookWithManager(mgr ctrl.Manager) error { + s, err := getScheme() + if err != nil { + return err + } + + cl, err = client.New(mgr.GetConfig(), client.Options{Scheme: s}) + if err != nil { + return err + } + return ctrl.NewWebhookManagedBy(mgr). For(r). Complete() diff --git a/featuregates/Makefile b/featuregates/Makefile index 2cc6505d635..c75f518cd41 100644 --- a/featuregates/Makefile +++ b/featuregates/Makefile @@ -19,10 +19,24 @@ CRD_OPTIONS ?= "crd" all: manager + + +LOCALBIN ?= $(shell pwd)/../hack/tools/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +ENVTEST ?= $(LOCALBIN)/setup-envtest +ENVTEST_K8S_VERSION = 1.26.1 + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + # Run tests -test: +test: envtest cd client && go test ./... -coverprofile cover.out - cd controller && go test ./... -coverprofile cover.out + cd controller && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -v -coverprofile cover.out # Build manager binary manager: diff --git a/featuregates/controller/Dockerfile b/featuregates/controller/Dockerfile index 0ae6ad1f433..c497aa5eeac 100644 --- a/featuregates/controller/Dockerfile +++ b/featuregates/controller/Dockerfile @@ -20,6 +20,7 @@ COPY featuregates/client featuregates/client COPY apis/ apis/ COPY cli/runtime cli/runtime COPY capabilities/client capabilities/client +COPY util/go.mod util/go.mod WORKDIR featuregates/controller # Setting default GOPROXY to https://proxy.golang.org,direct and GOSUMDB to sum.golang.org which can be override by Makefile diff --git a/featuregates/controller/go.mod b/featuregates/controller/go.mod index 41a0e567a3a..0c4e3132c8b 100644 --- a/featuregates/controller/go.mod +++ b/featuregates/controller/go.mod @@ -13,8 +13,8 @@ replace ( require ( github.com/go-logr/logr v1.2.3 - github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.2 + github.com/onsi/ginkgo/v2 v2.6.0 + github.com/onsi/gomega v1.24.1 github.com/vmware-tanzu/tanzu-framework/apis/config v0.0.0-20220824221239-af5a644ffef7 github.com/vmware-tanzu/tanzu-framework/apis/core v0.0.0-00010101000000-000000000000 github.com/vmware-tanzu/tanzu-framework/cli/runtime v0.0.0-00010101000000-000000000000 @@ -27,7 +27,6 @@ require ( ) require ( - github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -54,7 +53,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -75,7 +73,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.24.4 // indirect diff --git a/featuregates/controller/go.sum b/featuregates/controller/go.sum index b159bfeb4b3..733ad6bfb68 100644 --- a/featuregates/controller/go.sum +++ b/featuregates/controller/go.sum @@ -65,9 +65,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -172,7 +171,6 @@ github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -365,20 +363,19 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= +github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= -github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -695,7 +692,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -791,7 +787,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -970,8 +965,9 @@ k8s.io/client-go v0.24.4/go.mod h1:+AxlPWw/H6f+EJhRSjIeALaJT4tbeB/8g9BNvXGPd0Y= k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/code-generator v0.24.4/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.24.4 h1:WEGRp06GBYVwxp5JdiRaJ1zkdOhrqucxRv/8IrABLG0= k8s.io/component-base v0.24.4/go.mod h1:sWxkgcMfbYHadw0OJ0N+vIscd14/nqSIM2veCdg843o= +k8s.io/component-base v0.24.6 h1:LqSZunOeX1l6BpsMaD+iiJJSxZYl7IlVFW6qQu52ikA= +k8s.io/component-base v0.24.6/go.mod h1:YwPBIsm9c6VONDa6vBW4WjDMyvQgrLMHNoUokOkOdzU= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= diff --git a/featuregates/controller/pkg/feature/feature_suite_test.go b/featuregates/controller/pkg/feature/feature_suite_test.go new file mode 100644 index 00000000000..64d34556b2a --- /dev/null +++ b/featuregates/controller/pkg/feature/feature_suite_test.go @@ -0,0 +1,452 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package feature + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "log" + "math/big" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + corev1alpha2 "github.com/vmware-tanzu/tanzu-framework/apis/core/v1alpha2" + testutil "github.com/vmware-tanzu/tanzu-framework/featuregates/controller/pkg/test" +) + +func TestFeaturegate(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Featuregate Suite") +} + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + setupLog = ctrl.Log.WithName("controllers").WithName("Features") + + timeout = 5 * time.Second + interval = 100 * time.Millisecond + + tmpDir string + generatedWebhookManifestBytes []byte +) + +func generateCertificateAndManifests() error { + key, cert, err := testutil.GenerateSelfSignedCertsForTest("tanzu-featuregates-webhook-service.tkg-system.svc") + if err != nil { + return err + } + + tmpDir, err = os.MkdirTemp("/tmp", "featuregate-test") + if err != nil { + return err + } + err = os.WriteFile(path.Join(tmpDir, "feature-webhook.crt"), cert, 0644) + if err != nil { + return err + } + + err = os.WriteFile(path.Join(tmpDir, "feature-webhook.key"), key, 0644) + if err != nil { + return err + } + + input, err := os.ReadFile("testdata/webhook.yaml") + if err != nil { + log.Fatalln(err) + } + + lines := strings.Split(string(input), "\n") + + for i, line := range lines { + if strings.Contains(line, "Cg==") { + lines[i] = strings.Replace(lines[i], "Cg==", base64.StdEncoding.EncodeToString(cert), 1) + } + } + generatedWebhookManifestBytes = []byte(strings.Join(lines, "\n")) + return nil +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "apis", "core", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + testEnv.ControlPlane.APIServer.Configure().Append("admission-control", "ValidatingAdmissionWebhook") + + err = corev1alpha2.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + MetricsBindAddress: "0", + Host: "127.0.0.1", + Port: 9443, + }) + Expect(err).ToNot(HaveOccurred()) + + err = generateCertificateAndManifests() + Expect(err).ToNot(HaveOccurred()) + + k8sManager.GetWebhookServer().TLSMinVersion = "1.2" + k8sManager.GetWebhookServer().CertDir = tmpDir + k8sManager.GetWebhookServer().CertName = "feature-webhook.crt" + k8sManager.GetWebhookServer().KeyName = "feature-webhook.key" + k8sManager.GetWebhookServer().Port = 9443 + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + dynamicClient, err := dynamic.NewForConfig(cfg) + Expect(err).ToNot(HaveOccurred()) + Expect(dynamicClient).ToNot(BeNil()) + + err = (&FeatureReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Log: setupLog, + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tkg-system"}} + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + + err = (&corev1alpha2.FeatureGate{}).SetupWebhookWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() + + err = testutil.CreateResourcesFromManifest(generatedWebhookManifestBytes, cfg, dynamicClient) + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) + + err = os.RemoveAll(tmpDir) + Expect(err).NotTo(HaveOccurred()) +}) + +func getTestFeatureGate() *corev1alpha2.FeatureGate { + randomNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) + if err != nil { + return nil + } + + return &corev1alpha2.FeatureGate{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("featuregate-%v", randomNumber), + }, + Spec: corev1alpha2.FeatureGateSpec{ + Features: []corev1alpha2.FeatureReference{}, + }, + Status: corev1alpha2.FeatureGateStatus{}, + } +} + +func getTestFeature(stability corev1alpha2.StabilityLevel) *corev1alpha2.Feature { + randomNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) + if err != nil { + return nil + } + + return &corev1alpha2.Feature{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("feature-%v", randomNumber), + }, + Spec: corev1alpha2.FeatureSpec{ + Stability: stability, + }, + } +} + +var _ = Describe("Featuregate controller", func() { + It("Should not activate experimental features by default", func() { + feature := getTestFeature(corev1alpha2.Experimental) + Expect(k8sClient.Create(ctx, feature)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil + }, timeout, interval).Should(BeTrue()) + + featureGate := getTestFeatureGate() + featureGate.Spec.Features = append(featureGate.Spec.Features, corev1alpha2.FeatureReference{ + Name: feature.Name, + Activate: false, + }) + Expect(k8sClient.Create(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil && len(featureGate.Status.FeatureReferenceResults) == 1 + }, timeout, interval).Should(BeTrue()) + + Expect(featureGate.Status.FeatureReferenceResults[0].Status).Should(Equal(corev1alpha2.FeatureReferenceStatus("Applied"))) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature)).Should(BeNil()) + Expect(feature.Status.Activated).Should(Equal(false)) + + featureGate.Spec.Features[0].Activate = true + Expect(k8sClient.Update(ctx, featureGate)).ShouldNot(Succeed()) + + Expect(k8sClient.Delete(ctx, feature)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, featureGate)).Should(BeNil()) + }) + + It("Should activate and de-activate experimental features when permanently voided all support guarantees", func() { + feature := getTestFeature(corev1alpha2.Experimental) + Expect(k8sClient.Create(ctx, feature)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil + }, timeout, interval).Should(BeTrue()) + + featureGate := getTestFeatureGate() + featureGate.Spec.Features = append(featureGate.Spec.Features, corev1alpha2.FeatureReference{ + Name: feature.Name, + Activate: false, + PermanentlyVoidAllSupportGuarantees: true, + }) + Expect(k8sClient.Create(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil && len(featureGate.Status.FeatureReferenceResults) == 1 + }, timeout, interval).Should(BeTrue()) + + Expect(featureGate.Status.FeatureReferenceResults[0].Status).Should(Equal(corev1alpha2.FeatureReferenceStatus("Applied"))) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature)).Should(BeNil()) + Expect(feature.Status.Activated).Should(Equal(false)) + + featureGate.Spec.Features[0].Activate = true + Expect(k8sClient.Update(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == true + }, timeout, interval).Should(BeTrue()) + + featureGate.Spec.Features[0].Activate = false + Expect(k8sClient.Update(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == false + }, timeout, interval).Should(BeTrue()) + + // PermanentlyVoidAllSupportGuarantees once set to true, should not be set to false + featureGate.Spec.Features[0].PermanentlyVoidAllSupportGuarantees = false + Expect(k8sClient.Update(ctx, featureGate)).ShouldNot(Succeed()) + + Expect(k8sClient.Delete(ctx, feature)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, featureGate)).Should(BeNil()) + }) + + It("Should activate preview features irrespective of permanently voiding all support guarantees", func() { + feature := getTestFeature(corev1alpha2.TechnicalPreview) + Expect(k8sClient.Create(ctx, feature)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil + }, timeout, interval).Should(BeTrue()) + + featureGate := getTestFeatureGate() + featureGate.Spec.Features = append(featureGate.Spec.Features, corev1alpha2.FeatureReference{ + Name: feature.Name, + Activate: false, + }) + Expect(k8sClient.Create(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil && len(featureGate.Status.FeatureReferenceResults) == 1 + }, timeout, interval).Should(BeTrue()) + + Expect(featureGate.Status.FeatureReferenceResults[0].Status).Should(Equal(corev1alpha2.FeatureReferenceStatus("Applied"))) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature)).Should(BeNil()) + Expect(feature.Status.Activated).Should(Equal(false)) + + featureGate.Spec.Features[0].Activate = true + Expect(k8sClient.Update(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == true + }, timeout, interval).Should(BeTrue()) + + featureGate.Spec.Features[0].Activate = false + Expect(k8sClient.Update(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == false + }, timeout, interval).Should(BeTrue()) + + Expect(k8sClient.Delete(ctx, feature)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, featureGate)).Should(BeNil()) + }) + + It("Should activate stable features by default and stable features should be immutable", func() { + feature := getTestFeature(corev1alpha2.Stable) + Expect(k8sClient.Create(ctx, feature)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == true + }, timeout, interval).Should(BeTrue()) + + featureGate := getTestFeatureGate() + featureGate.Spec.Features = append(featureGate.Spec.Features, corev1alpha2.FeatureReference{ + Name: feature.Name, + Activate: false, + }) + Expect(k8sClient.Create(ctx, featureGate)).ShouldNot(Succeed()) + + Expect(k8sClient.Delete(ctx, feature)).Should(BeNil()) + }) + + It("Should activate deprecated features by default and deprected features should be mutable", func() { + feature := getTestFeature(corev1alpha2.Deprecated) + Expect(k8sClient.Create(ctx, feature)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == true + }, timeout, interval).Should(BeTrue()) + + featureGate := getTestFeatureGate() + featureGate.Spec.Features = append(featureGate.Spec.Features, corev1alpha2.FeatureReference{ + Name: feature.Name, + Activate: false, + }) + Expect(k8sClient.Create(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil && len(featureGate.Status.FeatureReferenceResults) == 1 + }, timeout, interval).Should(BeTrue()) + + Expect(featureGate.Status.FeatureReferenceResults[0].Status).Should(Equal(corev1alpha2.FeatureReferenceStatus("Applied"))) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == false + }, timeout, interval).Should(BeTrue()) + + Expect(k8sClient.Delete(ctx, feature)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, featureGate)).Should(BeNil()) + }) + + It("Should remove feature from featuregate status when reference is removed", func() { + feature := getTestFeature(corev1alpha2.Deprecated) + Expect(k8sClient.Create(ctx, feature)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: feature.Name}, feature) + return err == nil && feature.Status.Activated == true + }, timeout, interval).Should(BeTrue()) + + featureGate := getTestFeatureGate() + featureGate.Spec.Features = append(featureGate.Spec.Features, corev1alpha2.FeatureReference{ + Name: feature.Name, + Activate: false, + }) + Expect(k8sClient.Create(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil && len(featureGate.Status.FeatureReferenceResults) == 1 + }, timeout, interval).Should(BeTrue()) + + featureGate.Spec.Features = []corev1alpha2.FeatureReference{} + Expect(k8sClient.Update(ctx, featureGate)).Should(Succeed()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: featureGate.Name}, featureGate) + return err == nil && len(featureGate.Status.FeatureReferenceResults) == 0 + }, timeout, interval).Should(BeTrue()) + + Expect(k8sClient.Delete(ctx, feature)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, featureGate)).Should(BeNil()) + }) +}) diff --git a/featuregates/controller/pkg/feature/testdata/webhook.yaml b/featuregates/controller/pkg/feature/testdata/webhook.yaml new file mode 100644 index 00000000000..abf7b708a5d --- /dev/null +++ b/featuregates/controller/pkg/feature/testdata/webhook.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: tanzu-featuregates-webhook-service + namespace: tkg-system +spec: + type: ExternalName + externalName: 127.0.0.1 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: tanzu-featuregates-validating-webhook-core + annotations: +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: tanzu-featuregates-webhook-service + namespace: tkg-system + path: /validate-core-tanzu-vmware-com-v1alpha2-featuregate + port: 9443 + failurePolicy: Fail + name: featuregate.core.tanzu.vmware.com + rules: + - apiGroups: + - core.tanzu.vmware.com + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - featuregates + sideEffects: None diff --git a/featuregates/controller/pkg/featuregate/suite_test.go b/featuregates/controller/pkg/featuregate/suite_test.go deleted file mode 100644 index 4a456b666e0..00000000000 --- a/featuregates/controller/pkg/featuregate/suite_test.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build envtest -// +build envtest - -// Copyright 2021 VMware, Inc. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package featuregate - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - // +kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecsWithDefaultAndCustomReporters(t, - "Controller Suite", - []Reporter{printer.NewlineReporter{}}) -} diff --git a/featuregates/controller/pkg/test/cert.go b/featuregates/controller/pkg/test/cert.go new file mode 100644 index 00000000000..660294ba98e --- /dev/null +++ b/featuregates/controller/pkg/test/cert.go @@ -0,0 +1,67 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package test has helper functions for featuregates unit tests +package test + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" +) + +func GenerateSelfSignedCertsForTest(dnsName string) (keyBytes, certBytes []byte, err error) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Company, INC."}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"Golden Gate Bridge"}, + PostalCode: []string{"94016"}, + }, + DNSNames: []string{dnsName}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: false, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + key, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return []byte{}, []byte{}, err + } + + certBytes, err = x509.CreateCertificate(rand.Reader, cert, cert, &key.PublicKey, key) + if err != nil { + return []byte{}, []byte{}, err + } + + certPem := new(bytes.Buffer) + err = pem.Encode(certPem, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + return []byte{}, []byte{}, err + } + + keyPem := new(bytes.Buffer) + err = pem.Encode(keyPem, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + if err != nil { + return []byte{}, []byte{}, err + } + + return keyPem.Bytes(), certPem.Bytes(), nil +} diff --git a/featuregates/controller/pkg/test/kube.go b/featuregates/controller/pkg/test/kube.go new file mode 100644 index 00000000000..9ca97e3880b --- /dev/null +++ b/featuregates/controller/pkg/test/kube.go @@ -0,0 +1,79 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "bytes" + "context" + "io" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +func CreateResourcesFromManifest(manifestBytes []byte, cfg *rest.Config, dynamicClient dynamic.Interface) error { + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(manifestBytes), 100) + mapper, err := apiutil.NewDiscoveryRESTMapper(cfg) + if err != nil { + return err + } + for { + resource, unstructuredObj, err := getResource(decoder, mapper, dynamicClient) + if err != nil { + if err == io.EOF { + break + } else { + return err + } + } + _, err = resource.Create(context.Background(), unstructuredObj, metav1.CreateOptions{}) + if err != nil { + return err + } + } + return nil +} + +func getResource(decoder *yamlutil.YAMLOrJSONDecoder, mapper meta.RESTMapper, dynamicClient dynamic.Interface) ( + dynamic.ResourceInterface, *unstructured.Unstructured, error) { // nolint:whitespace + var rawObj runtime.RawExtension + if err := decoder.Decode(&rawObj); err != nil { + return nil, nil, err + } + + obj, gvk, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) + if err != nil { + return nil, nil, err + } + + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, nil, err + } + + unstructuredObj := &unstructured.Unstructured{Object: unstructuredMap} + + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, nil, err + } + + var resource dynamic.ResourceInterface + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + if unstructuredObj.GetNamespace() == "" { + unstructuredObj.SetNamespace("default") + } + resource = dynamicClient.Resource(mapping.Resource).Namespace(unstructuredObj.GetNamespace()) + } else { + resource = dynamicClient.Resource(mapping.Resource) + } + return resource, unstructuredObj, nil +} diff --git a/packages/tkr-source-controller/bundle/config/upstream/default-compatible-tkr.yaml b/packages/tkr-source-controller/bundle/config/upstream/default-compatible-tkr.yaml index fdcf73a4acc..0080e970456 100644 --- a/packages/tkr-source-controller/bundle/config/upstream/default-compatible-tkr.yaml +++ b/packages/tkr-source-controller/bundle/config/upstream/default-compatible-tkr.yaml @@ -13,6 +13,8 @@ metadata: run.tanzu.vmware.com/additional-compatible-tkrs: "" name: tkg-compatibility-versions namespace: #@ data.values.namespace + annotations: + kapp.k14s.io/change-group: "tkr-source-controller.tanzu.vmware.com/ConfigMap" data: tkrVersions: #@ yaml.encode(getCompatibleTKRs()) @@ -22,6 +24,8 @@ kind: ConfigMap metadata: name: tkr-controller-config namespace: #@ data.values.namespace + annotations: + kapp.k14s.io/change-group: "tkr-source-controller.tanzu.vmware.com/ConfigMap" data: caCerts: "" imageRepository: #@ data.values.imageRepository diff --git a/packages/tkr-source-controller/bundle/config/upstream/tkr-source-controller-deployment.yaml b/packages/tkr-source-controller/bundle/config/upstream/tkr-source-controller-deployment.yaml index 4af14c67ef8..a19524bb086 100644 --- a/packages/tkr-source-controller/bundle/config/upstream/tkr-source-controller-deployment.yaml +++ b/packages/tkr-source-controller/bundle/config/upstream/tkr-source-controller-deployment.yaml @@ -17,6 +17,7 @@ metadata: annotations: kapp.k14s.io/change-rule.0: "upsert after upserting tkr-source-controller.tanzu.vmware.com/ClusterRoleBinding" kapp.k14s.io/change-rule.1: "delete before deleting tkr-source-controller.tanzu.vmware.com/ClusterRoleBinding" + kapp.k14s.io/change-rule.3: "upsert after upserting tkr-source-controller.tanzu.vmware.com/ConfigMap" spec: replicas: 1 selector: diff --git a/providers/ytt/03_customizations/autoscaler/autoscaler_overlay.yaml b/providers/ytt/03_customizations/autoscaler/autoscaler_overlay.yaml index 9e5dfe3e5b8..4d3080d3955 100644 --- a/providers/ytt/03_customizations/autoscaler/autoscaler_overlay.yaml +++ b/providers/ytt/03_customizations/autoscaler/autoscaler_overlay.yaml @@ -10,8 +10,8 @@ kind: MachineDeployment metadata: #@overlay/match missing_ok=True annotations: - cluster.k8s.io/cluster-api-autoscaler-node-group-min-size: #@ "{}".format(data.values.AUTOSCALER_MIN_SIZE_0 or data.values.WORKER_MACHINE_COUNT_0) - cluster.k8s.io/cluster-api-autoscaler-node-group-max-size: #@ "{}".format(data.values.AUTOSCALER_MAX_SIZE_0 or data.values.WORKER_MACHINE_COUNT_0) + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: #@ "{}".format(data.values.AUTOSCALER_MIN_SIZE_0 or data.values.WORKER_MACHINE_COUNT_0) + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: #@ "{}".format(data.values.AUTOSCALER_MAX_SIZE_0 or data.values.WORKER_MACHINE_COUNT_0) #@overlay/match by=overlay.subset({"kind":"MachineDeployment", "metadata": {"name": data.values.CLUSTER_NAME + "-md-1"}}), missing_ok=True --- @@ -20,8 +20,8 @@ kind: MachineDeployment metadata: #@overlay/match missing_ok=True annotations: - cluster.k8s.io/cluster-api-autoscaler-node-group-min-size: #@ "{}".format(data.values.AUTOSCALER_MIN_SIZE_1 or data.values.WORKER_MACHINE_COUNT_1) - cluster.k8s.io/cluster-api-autoscaler-node-group-max-size: #@ "{}".format(data.values.AUTOSCALER_MAX_SIZE_1 or data.values.WORKER_MACHINE_COUNT_1) + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: #@ "{}".format(data.values.AUTOSCALER_MIN_SIZE_1 or data.values.WORKER_MACHINE_COUNT_1) + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: #@ "{}".format(data.values.AUTOSCALER_MAX_SIZE_1 or data.values.WORKER_MACHINE_COUNT_1) #@overlay/match by=overlay.subset({"kind":"MachineDeployment", "metadata": {"name": data.values.CLUSTER_NAME + "-md-2"}}), missing_ok=True --- @@ -30,7 +30,7 @@ kind: MachineDeployment metadata: #@overlay/match missing_ok=True annotations: - cluster.k8s.io/cluster-api-autoscaler-node-group-min-size: #@ "{}".format(data.values.AUTOSCALER_MIN_SIZE_2 or data.values.WORKER_MACHINE_COUNT_2) - cluster.k8s.io/cluster-api-autoscaler-node-group-max-size: #@ "{}".format(data.values.AUTOSCALER_MAX_SIZE_2 or data.values.WORKER_MACHINE_COUNT_2) + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: #@ "{}".format(data.values.AUTOSCALER_MIN_SIZE_2 or data.values.WORKER_MACHINE_COUNT_2) + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: #@ "{}".format(data.values.AUTOSCALER_MAX_SIZE_2 or data.values.WORKER_MACHINE_COUNT_2) #@ end diff --git a/providers/ytt/03_customizations/autoscaler/enable_autoscaler.yaml b/providers/ytt/03_customizations/autoscaler/enable_autoscaler.yaml index cf84b9f9428..aaed8e56ea8 100644 --- a/providers/ytt/03_customizations/autoscaler/enable_autoscaler.yaml +++ b/providers/ytt/03_customizations/autoscaler/enable_autoscaler.yaml @@ -38,7 +38,7 @@ #@ args.append("--v=4") #@ args.append("--clusterapi-cloud-config-authoritative") #@ args.append("--kubeconfig=" + autoscaler_kubeconfig_mount_path() + "/value") -#@ args.append("--node-group-auto-discovery=clusterapi:clusterName=" + data.values.CLUSTER_NAME) +#@ args.append("--node-group-auto-discovery=clusterapi:clusterName=" + data.values.CLUSTER_NAME + ",namespace=" +data.values.NAMESPACE) #@ args.append("--scale-down-delay-after-add=" + data.values.AUTOSCALER_SCALE_DOWN_DELAY_AFTER_ADD) #@ args.append("--scale-down-delay-after-delete=" + data.values.AUTOSCALER_SCALE_DOWN_DELAY_AFTER_DELETE) #@ args.append("--scale-down-delay-after-failure=" + data.values.AUTOSCALER_SCALE_DOWN_DELAY_AFTER_FAILURE) @@ -234,5 +234,12 @@ rules: verbs: - get - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - '*' + verbs: + - get + - list #@ end diff --git a/providers/yttcc/02_customizations/autoscaler/enable_autoscaler.yaml b/providers/yttcc/02_customizations/autoscaler/enable_autoscaler.yaml index 1437fd03303..82d6e19e26e 100644 --- a/providers/yttcc/02_customizations/autoscaler/enable_autoscaler.yaml +++ b/providers/yttcc/02_customizations/autoscaler/enable_autoscaler.yaml @@ -38,7 +38,7 @@ #@ args.append("--v=4") #@ args.append("--clusterapi-cloud-config-authoritative") #@ args.append("--kubeconfig=" + autoscaler_kubeconfig_mount_path() + "/value") -#@ args.append("--node-group-auto-discovery=clusterapi:clusterName=" + data.values.CLUSTER_NAME) +#@ args.append("--node-group-auto-discovery=clusterapi:clusterName=" + data.values.CLUSTER_NAME +",namespace=" +data.values.NAMESPACE) #@ args.append("--scale-down-delay-after-add=" + data.values.AUTOSCALER_SCALE_DOWN_DELAY_AFTER_ADD) #@ args.append("--scale-down-delay-after-delete=" + data.values.AUTOSCALER_SCALE_DOWN_DELAY_AFTER_DELETE) #@ args.append("--scale-down-delay-after-failure=" + data.values.AUTOSCALER_SCALE_DOWN_DELAY_AFTER_FAILURE) @@ -234,5 +234,12 @@ rules: verbs: - get - update + - apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - '*' + verbs: + - get + - list #@ end diff --git a/tkg/client/upgrade_cluster_clusterclass.go b/tkg/client/upgrade_cluster_clusterclass.go index 2b0840f33aa..5688f39a6fa 100644 --- a/tkg/client/upgrade_cluster_clusterclass.go +++ b/tkg/client/upgrade_cluster_clusterclass.go @@ -5,6 +5,7 @@ package client import ( "fmt" + "time" "github.com/pkg/errors" @@ -21,7 +22,10 @@ func (c *TkgClient) DoClassyClusterUpgrade(regionalClusterClient clusterclient.C log.Infof("Upgrading kubernetes cluster to `%v` version, tkr version: `%s`", kubernetesVersion, tkrVersion) patchJSONString := fmt.Sprintf(`{"spec": {"topology": {"version": "%v"}}}`, tkrVersion) - err := regionalClusterClient.PatchClusterObject(options.ClusterName, options.Namespace, patchJSONString) + // Timeout set to 15 minutes because the continuousTKRDiscoverFreq for tkr-source-controller's fetcher is 10 minutes. + // Wait time should be longer than the fetcher's frequency of pulling tkrs. + pollOptions := &clusterclient.PollOptions{Interval: upgradePatchInterval, Timeout: 15 * time.Minute} + err := regionalClusterClient.PatchClusterObjectWithPollOptions(options.ClusterName, options.Namespace, patchJSONString, pollOptions) if err != nil { return errors.Wrap(err, "unable to patch kubernetes version to cluster") } diff --git a/tkg/client/upgrade_cluster_test.go b/tkg/client/upgrade_cluster_test.go index 4803a42767a..c86aa98e481 100644 --- a/tkg/client/upgrade_cluster_test.go +++ b/tkg/client/upgrade_cluster_test.go @@ -982,7 +982,7 @@ var _ = Describe("Unit tests for clusterclass-based upgrade", func() { }) Context("When cluster patch fails", func() { BeforeEach(func() { - regionalClusterClient.PatchClusterObjectReturns(errors.New("fake-patch-error")) + regionalClusterClient.PatchClusterObjectWithPollOptionsReturns(errors.New("fake-patch-error")) }) It("should return an error", func() { Expect(err).To(HaveOccurred()) @@ -991,7 +991,7 @@ var _ = Describe("Unit tests for clusterclass-based upgrade", func() { }) Context("When failure happens while waiting for control-plane node upgrade", func() { BeforeEach(func() { - regionalClusterClient.PatchClusterObjectReturns(nil) + regionalClusterClient.PatchClusterObjectWithPollOptionsReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForCPNodesReturns(errors.New("fake-error-kcp-upgrade")) }) It("should return an error", func() { @@ -1001,7 +1001,7 @@ var _ = Describe("Unit tests for clusterclass-based upgrade", func() { }) Context("When failure happens while waiting for worker node upgrade", func() { BeforeEach(func() { - regionalClusterClient.PatchClusterObjectReturns(nil) + regionalClusterClient.PatchClusterObjectWithPollOptionsReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForCPNodesReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForWorkerNodesReturns(errors.New("fake-error-worker-upgrade")) }) @@ -1012,7 +1012,7 @@ var _ = Describe("Unit tests for clusterclass-based upgrade", func() { }) Context("When failure happens while applyPatch for autoscaler upgrade", func() { BeforeEach(func() { - regionalClusterClient.PatchClusterObjectReturns(nil) + regionalClusterClient.PatchClusterObjectWithPollOptionsReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForCPNodesReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForWorkerNodesReturns(nil) regionalClusterClient.ApplyPatchForAutoScalerDeploymentReturns(errors.Errorf("autoscaler image not available for kubernetes minor version %s", k8sVersionPrefix)) @@ -1024,7 +1024,7 @@ var _ = Describe("Unit tests for clusterclass-based upgrade", func() { }) Context("When cluster patch is successful and cluster get's upgraded successfully", func() { BeforeEach(func() { - regionalClusterClient.PatchClusterObjectReturns(nil) + regionalClusterClient.PatchClusterObjectWithPollOptionsReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForCPNodesReturns(nil) regionalClusterClient.WaitK8sVersionUpdateForWorkerNodesReturns(nil) regionalClusterClient.ApplyPatchForAutoScalerDeploymentReturns(nil) diff --git a/tkg/client/upgrade_region.go b/tkg/client/upgrade_region.go index 448e35826d3..d86ab4244fb 100644 --- a/tkg/client/upgrade_region.go +++ b/tkg/client/upgrade_region.go @@ -149,6 +149,17 @@ func (c *TkgClient) UpgradeManagementCluster(options *UpgradeClusterOptions) err } log.Info("Management cluster providers upgraded successfully...") + // Patch management cluster with the TKG version + // The TKG version should be patched before tkr-source-controller(in tkg-pkg) upgrade. + // So that tkr-source-controller's initial fetch can download the new tkrs. + // This will prevent the circular dependency that: + // A. Tanzu CLI updates TKG version after cluster upgraded to the desired tkr. + // B. tkr-source-controller downloads the desired tkr for cluster upgrade based on the updated TKG version + err = regionalClusterClient.PatchClusterObjectWithTKGVersion(options.ClusterName, options.Namespace, c.tkgBomClient.GetCurrentTKGVersion()) + if err != nil { + return err + } + // If clusterclass feature flag is enabled then deploy management components if config.IsFeatureActivated(constants.FeatureFlagPackageBasedCC) { log.Info("Preparing addons manager for upgrade") @@ -191,12 +202,6 @@ func (c *TkgClient) UpgradeManagementCluster(options *UpgradeClusterOptions) err return err } - // Patch management cluster with the TKG version - err = regionalClusterClient.PatchClusterObjectWithTKGVersion(options.ClusterName, options.Namespace, c.tkgBomClient.GetCurrentTKGVersion()) - if err != nil { - return err - } - if config.IsFeatureActivated(constants.FeatureFlagPackageBasedCC) { log.Info("Creating tkg-bom versioned ConfigMaps...") if err := c.CreateOrUpdateVerisionedTKGBom(regionalClusterClient); err != nil {