Skip to content
This repository has been archived by the owner on Oct 10, 2023. It is now read-only.

Commit

Permalink
Merge branch 'main' into cluster-essential
Browse files Browse the repository at this point in the history
  • Loading branch information
@Vandy-P
Vandy-P authored Mar 23, 2023
2 parents cfaea2c + 893310a commit 948c2f5
Show file tree
Hide file tree
Showing 28 changed files with 2,056 additions and 160 deletions.
40 changes: 38 additions & 2 deletions addons/controllers/addon_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -592,7 +593,7 @@ func (r *AddonReconciler) GetAddonKappResourceReconciler(
}

// GetExternalCRDs returns all external custom resources that addon controller depends on
func GetExternalCRDs() map[schema.GroupVersion]*sets.String {
func GetExternalCRDs() (map[schema.GroupVersion]*sets.String, []client.Object) {
var crds = map[schema.GroupVersion]*sets.String{}
// cluster-api
clusterapiv1alpha3Resources := sets.NewString("clusters")
Expand All @@ -612,5 +613,40 @@ func GetExternalCRDs() map[schema.GroupVersion]*sets.String {
kapppkgv1alpha1Resources := sets.NewString("packageinstalls", "packagerepositories")
crds[kapppkg.SchemeGroupVersion] = &kapppkgv1alpha1Resources

return crds
// Add CustomResourceDefinition for the above apis
clusterCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "clusters.cluster.x-k8s.io"},
Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "cluster.x-k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta1"}}},
}

kubeadmcontrolplaneCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "kubeadmcontrolplanes.controlplane.cluster.x-k8s.io"},
}

PackageInstallCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "packageinstalls.packaging.carvel.dev"},
}

tanzukubernetesreleaseCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "tanzukubernetesreleases.run.tanzu.vmware.com"},
}

packagerepositorieCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "packagerepositories.packaging.carvel.dev"},
}

appCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "apps.kappctrl.k14s.io"},
}

initObjs := []client.Object{
clusterCRD.DeepCopy(),
kubeadmcontrolplaneCRD.DeepCopy(),
PackageInstallCRD.DeepCopy(),
tanzukubernetesreleaseCRD.DeepCopy(),
packagerepositorieCRD.DeepCopy(),
appCRD.DeepCopy(),
}

return crds, initObjs
}
39 changes: 39 additions & 0 deletions addons/controllers/clusterbootstrap_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util/clusterbootstrapclone"
"github.com/vmware-tanzu/tanzu-framework/addons/predicates"
"github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
runtanzuv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
)

Expand Down Expand Up @@ -577,7 +578,19 @@ func (r *ClusterBootstrapReconciler) mergeClusterBootstrapPackagesWithTemplate(

// Find the one to one match for additional package in new ClusterBootstrapTemplate and old ClusterBootstrap and update
if pkg, ok := additionalPackageMap[packageRefName]; ok {
oldCBTPkg, err := r.getPackageFromCBTInResolvedTKR(pkg.RefName, cluster, updatedClusterBootstrap, log)
if err != nil {
return nil, err
}
pkg.RefName = templatePkg.RefName

if isCBPackageValuesFromNil(pkg) && oldCBTPkg != nil && isCBPackageValuesFromNil(oldCBTPkg) {
if templatePkg.ValuesFrom != nil {
pkg.ValuesFrom = templatePkg.ValuesFrom.DeepCopy()
packages = append(packages, pkg)
}
}

} else {
// If new additional package is added in ClusterBootstrapTemplate, just add it to updated ClusterBootstrap
newPkg := templatePkg.DeepCopy()
Expand All @@ -604,6 +617,32 @@ func (r *ClusterBootstrapReconciler) mergeClusterBootstrapPackagesWithTemplate(
return packages, nil
}

func (r *ClusterBootstrapReconciler) getPackageFromCBTInResolvedTKR(
packageRefName string,
cluster *clusterapiv1beta1.Cluster,
updatedClusterBootstrap *runtanzuv1alpha3.ClusterBootstrap,
log logr.Logger) (*runtanzuv1alpha3.ClusterBootstrapPackage, error) {

clusterBootstrapTemplate := &runtanzuv1alpha3.ClusterBootstrapTemplate{}
key := client.ObjectKey{Namespace: r.Config.SystemNamespace, Name: updatedClusterBootstrap.Status.ResolvedTKR}
if err := r.Client.Get(r.context, key, clusterBootstrapTemplate); err != nil {
log.Error(err, "unable to fetch ClusterBootstrapTemplate", "objectkey", key)
return nil, err
}
for _, pkg := range clusterBootstrapTemplate.Spec.AdditionalPackages {
if packageRefName == pkg.RefName {
return pkg, nil
}
}

// If the specified package is not found, it is not an error condition. So return error as nil
return nil, nil
}

func isCBPackageValuesFromNil(cbpkg *v1alpha3.ClusterBootstrapPackage) bool {
return cbpkg.ValuesFrom == nil || (cbpkg.ValuesFrom.ProviderRef == nil && cbpkg.ValuesFrom.SecretRef == "" && cbpkg.ValuesFrom.Inline == nil)
}

// createOrPatchKappPackageInstall contains the logic that create/update PackageInstall CR for kapp-controller on
// mgmt cluster. The kapp-controller running on mgmt cluster reconciles the PackageInstall CR and creates kapp-controller resources
// on remote workload cluster. This is required for a workload cluster and its corresponding package installations to be functional.
Expand Down
112 changes: 112 additions & 0 deletions addons/controllers/clusterbootstrap_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1524,6 +1524,118 @@ var _ = Describe("ClusterBootstrap Reconciler", func() {
})
})
})

When("cluster upgrade with values change", func() {
BeforeEach(func() {
clusterName = "test-cluster-10"
clusterNamespace = "cluster-namespace-10"
clusterResourceFilePath = "testdata/test-cluster-bootstrap-10.yaml"
})

Context("cluster with no inline values are set", func() {
It("should work when no inline values are set", func() {

By("filling ClusterBootstrap correctly", func() {
Eventually(func(g Gomega) {
clusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{}
err := k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, clusterBootstrap)
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(clusterBootstrap.Spec.CNI.RefName).To(BeEquivalentTo("antrea.tanzu.vmware.com.1.11.7--vmware.1-tkg.2"))
g.Expect(clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil())
g.Expect(*clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com"))
g.Expect(clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.Kind).To(Equal("AntreaConfig"))
g.Expect(clusterBootstrap.Spec.CNI.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-antrea-package", clusterName)))

// Validate Kapp
g.Expect(clusterBootstrap.Spec.Kapp.RefName).To(Equal("kapp-controller.tanzu.vmware.com.0.32.11"))
g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom).NotTo(BeNil())
g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil())
g.Expect(*clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.APIGroup).To(Equal("run.tanzu.vmware.com"))
g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.Kind).To(Equal("KappControllerConfig"))
g.Expect(clusterBootstrap.Spec.Kapp.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-kapp-controller-package", clusterName)))

// Validate additional packages
g.Expect(len(clusterBootstrap.Spec.AdditionalPackages)).To(Equal(2))

g.Expect(clusterBootstrap.Spec.AdditionalPackages[0].RefName).To(BeEquivalentTo("pinniped.tanzu.vmware.com.0.13.3--vmware.1-tkg.1"))
g.Expect(clusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom).NotTo(BeNil())
g.Expect(clusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom.SecretRef).To(BeEquivalentTo(fmt.Sprintf("%s-pinniped-package", clusterName)))

g.Expect(clusterBootstrap.Spec.AdditionalPackages[1].RefName).To(BeEquivalentTo("foobar.tanzu.vmware.com.0.2.4--vmware.1-tkg.1"))
//g.Expect(clusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom).To(BeNil())
}, waitTimeout, pollingInterval).Should(Succeed())
})

By("setting cluster phase to provisioned", func() {
cluster := &clusterapiv1beta1.Cluster{}
Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed())
cluster.Status.Phase = string(clusterapiv1beta1.ClusterPhaseProvisioned)
Expect(k8sClient.Status().Update(ctx, cluster)).To(Succeed())
})

By("upgrading Cluster to newer TKR", func() {
cluster := &clusterapiv1beta1.Cluster{}
newTKRVersion := "v1.25.6-custom"
Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed())
cluster.Labels[constants.TKRLabelClassyClusters] = newTKRVersion

// Mock cluster pause mutating webhook
cluster.Spec.Paused = true
if cluster.Annotations == nil {
cluster.Annotations = map[string]string{}
}
cluster.Annotations[constants.ClusterPauseLabel] = newTKRVersion
Expect(k8sClient.Update(ctx, cluster)).To(Succeed())

Eventually(func(g Gomega) {
upgradedClusterBootstrap := &runtanzuv1alpha3.ClusterBootstrap{}
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), upgradedClusterBootstrap)

g.Expect(err).To(BeNil())
g.Expect(upgradedClusterBootstrap.Status.ResolvedTKR).To(BeEquivalentTo(newTKRVersion))

// Validate CNI
cni := upgradedClusterBootstrap.Spec.CNI
// Note: The value of CNI has been bumped to the one in TKR after cluster upgrade
g.Expect(cni.RefName).To(Equal("antrea.tanzu.vmware.com.1.11.8--vmware.1-tkg.2"))
g.Expect(cni.ValuesFrom).NotTo(BeNil())
g.Expect(cni.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil())
g.Expect(*cni.ValuesFrom.ProviderRef.APIGroup).To(Equal("cni.tanzu.vmware.com"))
g.Expect(cni.ValuesFrom.ProviderRef.Kind).To(Equal("AntreaConfig"))
g.Expect(cni.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-antrea-package", clusterName)))

// Validate Kapp
kapp := upgradedClusterBootstrap.Spec.Kapp
g.Expect(kapp.RefName).To(Equal("kapp-controller.tanzu.vmware.com.0.32.12"))
g.Expect(kapp.ValuesFrom).NotTo(BeNil())
g.Expect(kapp.ValuesFrom.ProviderRef.APIGroup).NotTo(BeNil())
g.Expect(*kapp.ValuesFrom.ProviderRef.APIGroup).To(Equal("run.tanzu.vmware.com"))
g.Expect(kapp.ValuesFrom.ProviderRef.Kind).To(Equal("KappControllerConfig"))
g.Expect(kapp.ValuesFrom.ProviderRef.Name).To(Equal(fmt.Sprintf("%s-kapp-controller-package", clusterName)))

// Validate additional packages
g.Expect(len(upgradedClusterBootstrap.Spec.AdditionalPackages)).To(Equal(2))
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[0].RefName).To(BeEquivalentTo("pinniped.tanzu.vmware.com.0.13.4--vmware.1-tkg.1"))
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom).NotTo(BeNil())
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[0].ValuesFrom.SecretRef).To(BeEquivalentTo(fmt.Sprintf("%s-pinniped-package", clusterName)))

g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].RefName).To(BeEquivalentTo("foobar.tanzu.vmware.com.0.2.5--vmware.1-tkg.1"))
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom).NotTo(BeNil())
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom.Inline).NotTo(BeNil())
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom.Inline["key1"]).To(Equal("value1"))
g.Expect(upgradedClusterBootstrap.Spec.AdditionalPackages[1].ValuesFrom.Inline["key2"].(map[string]interface{})["key3"]).To(Equal("value3"))

g.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: clusterNamespace, Name: clusterName}, cluster)).To(Succeed())
g.Expect(cluster.Spec.Paused).ToNot(BeTrue())
if cluster.Annotations != nil {
_, ok := cluster.Annotations[constants.ClusterPauseLabel]
g.Expect(ok).ToNot(BeTrue())
}
}, waitTimeout, pollingInterval).Should(Succeed())
})
})
})
})
})

func assertSecretContains(ctx context.Context, k8sClient client.Client, namespace, name string, secretContent map[string][]byte) {
Expand Down
22 changes: 17 additions & 5 deletions addons/controllers/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ import (
. "github.com/onsi/gomega"
capociv1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
capvv1beta1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
Expand All @@ -33,6 +35,7 @@ import (
controlplanev1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
Expand Down Expand Up @@ -234,6 +237,9 @@ var _ = BeforeSuite(func(done Done) {
err = topologyv1alpha1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())

err = apiextensionsv1.AddToScheme(scheme)
Expect(err).ToNot(HaveOccurred())

k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
Expect(err).ToNot(HaveOccurred())
Expect(k8sClient).ToNot(BeNil())
Expand Down Expand Up @@ -262,18 +268,24 @@ var _ = BeforeSuite(func(done Done) {
Expect(err).ToNot(HaveOccurred())

ctx, cancel = context.WithCancel(ctx)

initObjs := []client.Object{}
fakeApiReader := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()

crdwaiter := crdwait.CRDWaiter{
Ctx: ctx,
ClientSetFn: func() (kubernetes.Interface, error) {
return kubernetes.NewForConfig(cfg)
},
Ctx: ctx,
ClientSet: fake.NewSimpleClientset(),
APIReader: fakeApiReader,
Logger: setupLog,
Scheme: scheme,
PollInterval: constants.CRDWaitPollInterval,
PollTimeout: constants.CRDWaitPollTimeout,
}

if err := crdwaiter.WaitForCRDs(GetExternalCRDs(),
crds, initObjs := GetExternalCRDs()
crdwaiter.APIReader = ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()

if err := crdwaiter.WaitForCRDs(crds,
&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}},
constants.AddonControllerName,
); err != nil {
Expand Down
Loading

0 comments on commit 948c2f5

Please sign in to comment.