TEST ONLY - testing e2e tests

.github/workflows/build_test.yml

@@ -89,6 +89,8 @@ jobs:
         run: |
           make hmc-chart-release
           make helm-push
+          make set-hmc-version test-hmc-version 
+          make helm-push
 
   controller-e2etest:
     name: E2E Controller

.github/workflows/test_e2e.yml

@@ -0,0 +1,136 @@
+name: CI
+on:
+  pull_request:
+    types:
+      - labeled
+      - opened
+      - synchronize
+      - reopened
+    paths-ignore:
+      - 'config/**'
+      - '**.md'
+  push:
+    tags:
+      - '*'
+
+env:
+  GO_VERSION: '1.22'
+  REGISTRY_REPO: 'oci://ghcr.io/mirantis/hmc/charts-ci'
+
+jobs:
+  build:
+    concurrency:
+      group: build-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    name: Build and Unit Test
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.vars.outputs.version }}
+      cutversion: ${{ steps.vars.outputs.cutversion }}
+      clustername: ${{ steps.vars.outputs.clustername }}
+      pr: ${{ steps.pr.outputs.result }}
+    steps:
+      - name: Get PR ref
+        uses: actions/github-script@v7
+        id: pr
+        with:
+          script: |
+            const { data: pullRequest } = await github.rest.pulls.get({
+              ...context.repo,
+              pull_number: context.payload.pull_request.number,
+            });
+            return pullRequest
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}}
+          fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GHCR
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get outputs
+        id: vars
+        run: |
+          GIT_VERSION=$(git describe --tags --always)
+          echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT
+          echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT
+          echo "cutversion=$(echo ${GIT_VERSION:1} | sed 's/.*-//g')" >> $GITHUB_OUTPUT
+      - name: Build and push HMC controller image
+        uses: docker/build-push-action@v6
+        with:
+          build-args: |
+            LD_FLAGS=-s -w -X github.com/Mirantis/hmc/internal/build.Version=${{ steps.vars.outputs.version }}
+          context: .
+          platforms: linux/amd64
+          tags: |
+            ghcr.io/mirantis/hmc/controller-ci:${{ steps.vars.outputs.version }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Prepare and push HMC template charts
+        run: |
+          make hmc-chart-release
+          make helm-push
+
+  provider-cloud-e2etest:
+    name: E2E Cloud Providers
+    runs-on: ubuntu-latest
+    if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
+    needs: build
+    outputs:
+      clustername: ${{ needs.build.outputs.clustername }}
+      version: ${{ needs.build.outputs.version }}
+      pr: ${{ needs.build.outputs.pr }}
+    env:
+      AWS_REGION: us-west-2
+      AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
+      AZURE_REGION: westus2
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }}
+      AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
+      AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
+      AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
+      BUILD_VERSION: ${{ needs.build.outputs.cutversion }}
+    steps:
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        with:
+          detached: true
+        timeout-minutes: 45
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v4
+      - uses: actions/checkout@v4
+      - name: Run E2E tests
+        env:
+          GINKGO_LABEL_FILTER: 'provider:cloud'
+          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
+          VERSION: ${{ needs.build.outputs.version }}
+        run: |
+          make test-e2e
+      - name: Archive test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: cloud-e2etest-logs
+          path: |
+            test/e2e/*.log

Makefile

@@ -72,8 +72,32 @@ set-hmc-version: yq
 	$(YQ) eval '.metadata.name = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml
 	$(YQ) eval '.spec.hmc.template = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml
 
+.PHONY: test-hmc-version
+test-hmc-version: yq
+	@TEMPLATE_DIR=templates/provider/hmc-templates/files/templates/*; \
+	cutversion=$$(echo $(VERSION) | sed "s/.*-//g"); \
+	for template in $$TEMPLATE_DIR; do \
+  		if  [ $$(yq eval '.kind' $$template) = 'ClusterTemplate' ]; then \
+  			if grep --quiet "chartVersion:.*-"$$cutversion $$template; then \
+              echo "skipping already processed templates $$template"; \
+  			else \
+  			    echo $$template; \
+  				sed -i "s/chartVersion:.*[^$$cutversion].*/&-$$cutversion/g" "$$template"; \
+  			fi \
+  		fi; \
+	done; \
+	CHART_DIR=templates/cluster/**/Chart.yaml; \
+	for chart in $$CHART_DIR; do \
+	  if grep --quiet "version:.*-$$cutversion" $$chart; then \
+		echo "skipping already processed templates $$chart"; \
+		else \
+			echo $$chart; \
+			sed -i "s/version:.*[^$$cutversion]*./&-$$cutversion/g" "$$chart"; \
+		fi \
+	done
+
 .PHONY: hmc-chart-release
-hmc-chart-release: set-hmc-version templates-generate ## Generate hmc helm chart
+hmc-chart-release: set-hmc-version test-hmc-version templates-generate ## Generate hmc helm chart
 
 .PHONY: hmc-dist-release
 hmc-dist-release: $(HELM) $(YQ)
@@ -110,7 +134,7 @@ test: generate-all fmt vet envtest tidy external-crd ## Run tests.
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling
 # compatibility with other vendors.
-.PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster.
+.PHONY: test-e2e test-hmc-version# Run the e2e tests using a Kind k8s instance as the management cluster.
 test-e2e: cli-install
 	@if [ "$$GINKGO_LABEL_FILTER" ]; then \
 		ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \
@@ -347,7 +371,7 @@ dev-eks-creds: dev-aws-creds
 dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates dev-release
 
 .PHONY: test-apply
-test-apply: set-hmc-version helm-package dev-deploy dev-templates dev-release
+test-apply: set-hmc-version test-hmc-version helm-package dev-deploy dev-templates dev-release
 
 .PHONY: dev-destroy
 dev-destroy: kind-undeploy registry-undeploy ## Destroy the development environment by deleting the kind cluster and local registry.

test/e2e/e2e_suite_test.go

@@ -66,6 +66,15 @@ var _ = BeforeSuite(func() {
 		}
 		return nil
 	}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+	Eventually(func() error {
+		err = managedcluster.ValidateClusterTemplates(context.Background(), kc)
+		if err != nil {
+			_, _ = fmt.Fprintf(GinkgoWriter, "Controller validation failed: %v\n", err)
+			return err
+		}
+		return nil
+	}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 })
 
 var _ = AfterSuite(func() {
@@ -93,6 +102,9 @@ func verifyControllersUp(kc *kubeclient.KubeClient) error {
 		managedcluster.ProviderAWS,
 		managedcluster.ProviderAzure,
 		managedcluster.ProviderVSphere,
+		managedcluster.ProviderK0smotron,
+		managedcluster.ProviderK0smotronBootstrap,
+		managedcluster.ProviderK0smotronControlPlane,
 	}
 
 	for _, provider := range providers {

test/e2e/kubeclient/kubeclient.go

@@ -243,6 +243,21 @@ func (kc *KubeClient) listResource(
 	return resources.Items, nil
 }
 
+func (kc *KubeClient) ListClusterTemplates(ctx context.Context) ([]unstructured.Unstructured, error) {
+	client := kc.GetDynamicClient(schema.GroupVersionResource{
+		Group:    "hmc.mirantis.com",
+		Version:  "v1alpha1",
+		Resource: "clustertemplates",
+	}, true)
+
+	resources, err := client.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list cluster templates")
+	}
+
+	return resources.Items, nil
+}
+
 // ListMachines returns a list of Machine resources for the given cluster.
 func (kc *KubeClient) ListMachines(ctx context.Context, clusterName string) ([]unstructured.Unstructured, error) {
 	GinkgoHelper()

test/e2e/managedcluster/clusteridentity/clusteridentity.go

@@ -134,7 +134,7 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust
 	ci.createSecret(kc)
 	ci.createClusterIdentity(kc)
 	ci.createCredential(kc)
-
+	ci.waitForCredentialReady(kc)
 	return &ci
 }
 
@@ -252,3 +252,40 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) {
 
 	kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced)
 }
+
+func (ci *ClusterIdentity) waitForCredentialReady(kc *kubeclient.KubeClient) {
+	GinkgoHelper()
+
+	By(fmt.Sprintf("waiting for %s credential to be ready", ci.Kind))
+
+	ctx := context.Background()
+	credName := fmt.Sprintf("%s-cred", ci.IdentityName)
+
+	gvr := schema.GroupVersionResource{
+		Group:    "hmc.mirantis.com",
+		Version:  "v1alpha1",
+		Resource: "credentials",
+	}
+
+	Eventually(func() error {
+		creds, err := kc.GetDynamicClient(gvr, true).Get(ctx, credName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		ready, found, err := unstructured.NestedBool(creds.Object, "status", "ready")
+		if err != nil {
+			return err
+		}
+
+		if !found {
+			return fmt.Errorf("ready status not found on credential %s", credName)
+		}
+
+		if !ready {
+			return fmt.Errorf("credential %s not ready", credName)
+		}
+
+		return nil
+	}).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed())
+}

test/e2e/managedcluster/managedcluster.go

@@ -15,6 +15,7 @@
 package managedcluster
 
 import (
+	"context"
 	_ "embed"
 	"fmt"
 	"os"
@@ -27,16 +28,20 @@ import (
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
+	"github.com/Mirantis/hmc/test/e2e/kubeclient"
 	"github.com/Mirantis/hmc/test/utils"
 )
 
 type ProviderType string
 
 const (
-	ProviderCAPI    ProviderType = "cluster-api"
-	ProviderAWS     ProviderType = "infrastructure-aws"
-	ProviderAzure   ProviderType = "infrastructure-azure"
-	ProviderVSphere ProviderType = "infrastructure-vsphere"
+	ProviderCAPI                  ProviderType = "cluster-api"
+	ProviderAWS                   ProviderType = "infrastructure-aws"
+	ProviderAzure                 ProviderType = "infrastructure-azure"
+	ProviderVSphere               ProviderType = "infrastructure-vsphere"
+	ProviderK0smotron             ProviderType = "infrastructure-k0sproject-k0smotron"
+	ProviderK0smotronBootstrap    ProviderType = "bootstrap-k0sproject-k0smotron"
+	ProviderK0smotronControlPlane ProviderType = "control-plane-k0sproject-k0smotron"
 
 	providerLabel = "cluster.x-k8s.io/provider"
 )
@@ -142,13 +147,37 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 	Expect(err).NotTo(HaveOccurred(), "failed to substitute environment variables")
 
 	var managedClusterConfig map[string]any
-
+	By(fmt.Sprintf("Cluster being applied\n %s", managedClusterConfigBytes))
 	err = yaml.Unmarshal(managedClusterConfigBytes, &managedClusterConfig)
 	Expect(err).NotTo(HaveOccurred(), "failed to unmarshal deployment config")
 
 	return &unstructured.Unstructured{Object: managedClusterConfig}
 }
 
+func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error {
+	templates, err := client.ListClusterTemplates(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list cluster templates: %w", err)
+	}
+
+	for _, template := range templates {
+		valid, found, err := unstructured.NestedBool(template.Object, "status", "valid")
+		if err != nil {
+			return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err)
+		}
+
+		if !found {
+			return fmt.Errorf("valid flag for template %s not found", template.GetName())
+		}
+
+		if !valid {
+			return fmt.Errorf("template %s is still invalid", template.GetName())
+		}
+	}
+
+	return nil
+}
+
 func ValidateDeploymentVars(v []string) {
 	GinkgoHelper()
 

test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl

@@ -3,7 +3,7 @@ kind: ManagedCluster
 metadata:
   name: ${MANAGED_CLUSTER_NAME}
 spec:
-  template: aws-hosted-cp-0-0-3
+  template: aws-hosted-cp-0-0-3-${BUILD_VERSION}
   credential: ${AWS_CLUSTER_IDENTITY}-cred
   config:
     clusterIdentity:

test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl

@@ -3,7 +3,7 @@ kind: ManagedCluster
 metadata:
   name: ${MANAGED_CLUSTER_NAME}
 spec:
-  template: aws-standalone-cp-0-0-3
+  template: aws-standalone-cp-0-0-3-${BUILD_VERSION}
   credential: ${AWS_CLUSTER_IDENTITY}-cred
   config:
     clusterIdentity:
@@ -15,5 +15,7 @@ spec:
     workersNumber: ${WORKERS_NUMBER:=1}
     controlPlane:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+      rootVolumeSize: 32
     worker:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+      rootVolumeSize: 32