From 21b415f61b8b110bedca0071eb27bbe68f28d192 Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Wed, 8 Nov 2023 11:55:45 +0100 Subject: [PATCH 1/5] chore: use a license during e2e tests in order to capture problems with the embedded license we need to use a version of the embedded cluster binary with an embedded kots license. --- .gitignore | 3 ++- Makefile | 18 +++++++++++------- e2e/README.md | 11 +++++++++++ e2e/kots-release/kots-app.yaml | 15 +++++++++++++++ e2e/kots-release/license.yaml | 27 +++++++++++++++++++++++++++ e2e/scripts/embedded-preflight.sh | 2 +- pkg/kubeutils/kubeutils.go | 4 ++-- 7 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 e2e/kots-release/kots-app.yaml create mode 100644 e2e/kots-release/license.yaml diff --git a/.gitignore b/.gitignore index d14580a86..a62e950e7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ bundle pkg/goods/bins pkg/goods/images *tgz -license.yaml +release.tar.gz +release.o diff --git a/Makefile b/Makefile index febfb4903..531578164 100644 --- a/Makefile +++ b/Makefile @@ -92,12 +92,16 @@ pkg/goods/bins/embedded-cluster/kubectl-preflight: tar -xzf output/tmp/preflight/preflight.tar.gz -C output/tmp/preflight mv output/tmp/preflight/preflight pkg/goods/bins/embedded-cluster/kubectl-preflight -.PHONY: embed-license -embed-license: embedded-cluster-linux-amd64 license.yaml +release.tar.gz: mkdir -p output/tmp - tar -czvf output/tmp/license.tar.gz license.yaml - objcopy --input-target binary --output-target binary --rename-section .data=sec_bundle output/tmp/license.tar.gz output/tmp/license.tar.gz.o - objcopy --add-section sec_bundle=output/tmp/license.tar.gz.o output/bin/embedded-cluster + tar -czf release.tar.gz -C e2e/kots-release . + +.PHONY: embedded-release +embedded-release: embedded-cluster-linux-amd64 release.tar.gz + objcopy --input-target binary --output-target binary --rename-section .data=sec_bundle release.tar.gz release.o + @if ! objcopy --update-section sec_bundle=release.o output/bin/embedded-cluster ; then \ + objcopy --add-section sec_bundle=release.o output/bin/embedded-cluster ; \ + fi .PHONY: static static: pkg/goods/bins/embedded-cluster/kubectl-preflight \ @@ -133,14 +137,14 @@ vet: static-linux-amd64 static go vet ./... .PHONY: e2e-tests -e2e-tests: embedded-cluster-linux-amd64 +e2e-tests: embedded-release mkdir -p output/tmp rm -rf output/tmp/id_rsa* ssh-keygen -t rsa -N "" -C "Integration Test Key" -f output/tmp/id_rsa go test -timeout 45m -parallel 1 -failfast -v ./e2e .PHONY: e2e-test -e2e-test: embedded-cluster-linux-amd64 +e2e-test: embedded-release mkdir -p output/tmp rm -rf output/tmp/id_rsa* ssh-keygen -t rsa -N "" -C "Integration Test Key" -f output/tmp/id_rsa diff --git a/e2e/README.md b/e2e/README.md index e3fa991c2..3c49450de 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -54,3 +54,14 @@ TestSomething is the name of the test function you want to run. To add more tests you just need to create one inside this directory and then add it to the `.github/workflows/e2e.yaml` file. + + +### Kots test application + +During end to end tests we embed a license for a smoke test kots app, +this app can be found in: + +https://vendor.replicated.com/apps/embedded-cluster-smoke-test-app + +Make sure to update the application yaml files under kots-release +directory if you create a new release of the remote application. diff --git a/e2e/kots-release/kots-app.yaml b/e2e/kots-release/kots-app.yaml new file mode 100644 index 000000000..cb1cf2d3b --- /dev/null +++ b/e2e/kots-release/kots-app.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: nginx +spec: + title: Embedded Cluster Smoke Test App + icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/icon/color/kubernetes-icon-color.png + statusInformers: + - deployment/nginx + ports: + - serviceName: "nginx" + servicePort: 80 + localPort: 8888 + applicationUrl: "http://nginx" diff --git a/e2e/kots-release/license.yaml b/e2e/kots-release/license.yaml new file mode 100644 index 000000000..20f93c6d3 --- /dev/null +++ b/e2e/kots-release/license.yaml @@ -0,0 +1,27 @@ +apiVersion: kots.io/v1beta1 +kind: License +metadata: + name: embeddedclusterintegrationtests +spec: + appSlug: embedded-cluster-smoke-test-app + channelID: 2Xt5IoImUY2W8eo5DWZEdtA5Yr1 + channelName: Stable + customerName: Embedded Cluster Integration Tests + endpoint: https://replicated.app + entitlements: + expires_at: + description: License Expiration + signature: {} + title: Expiration + value: "" + valueType: String + isGitOpsSupported: true + isHelmVmDownloadEnabled: true + isKotsInstallEnabled: true + isNewKotsUiEnabled: true + isSemverRequired: true + isSnapshotSupported: true + licenseID: 2Xt5q15MDpLvaO15UZ5XhKxBqB1 + licenseSequence: 1 + licenseType: prod + signature: eyJsaWNlbnNlRGF0YSI6ImV5SmhjR2xXWlhKemFXOXVJam9pYTI5MGN5NXBieTkyTVdKbGRHRXhJaXdpYTJsdVpDSTZJa3hwWTJWdWMyVWlMQ0p0WlhSaFpHRjBZU0k2ZXlKdVlXMWxJam9pWlcxaVpXUmtaV1JqYkhWemRHVnlhVzUwWldkeVlYUnBiMjUwWlhOMGN5SjlMQ0p6Y0dWaklqcDdJbXhwWTJWdWMyVkpSQ0k2SWpKWWREVnhNVFZOUkhCTWRtRlBNVFZWV2pWWWFFdDRRbkZDTVNJc0lteHBZMlZ1YzJWVWVYQmxJam9pY0hKdlpDSXNJbU4xYzNSdmJXVnlUbUZ0WlNJNklrVnRZbVZrWkdWa0lFTnNkWE4wWlhJZ1NXNTBaV2R5WVhScGIyNGdWR1Z6ZEhNaUxDSmhjSEJUYkhWbklqb2laVzFpWldSa1pXUXRZMngxYzNSbGNpMXpiVzlyWlMxMFpYTjBMV0Z3Y0NJc0ltTm9ZVzV1Wld4SlJDSTZJakpZZERWSmIwbHRWVmt5VnpobGJ6VkVWMXBGWkhSQk5WbHlNU0lzSW1Ob1lXNXVaV3hPWVcxbElqb2lVM1JoWW14bElpd2liR2xqWlc1elpWTmxjWFZsYm1ObElqb3hMQ0psYm1Sd2IybHVkQ0k2SW1oMGRIQnpPaTh2Y21Wd2JHbGpZWFJsWkM1aGNIQWlMQ0psYm5ScGRHeGxiV1Z1ZEhNaU9uc2laWGh3YVhKbGMxOWhkQ0k2ZXlKMGFYUnNaU0k2SWtWNGNHbHlZWFJwYjI0aUxDSmtaWE5qY21sd2RHbHZiaUk2SWt4cFkyVnVjMlVnUlhod2FYSmhkR2x2YmlJc0luWmhiSFZsSWpvaUlpd2lkbUZzZFdWVWVYQmxJam9pVTNSeWFXNW5JaXdpYzJsbmJtRjBkWEpsSWpwN2ZYMTlMQ0pwYzBkcGRFOXdjMU4xY0hCdmNuUmxaQ0k2ZEhKMVpTd2lhWE5UYm1Gd2MyaHZkRk4xY0hCdmNuUmxaQ0k2ZEhKMVpTd2lhWE5PWlhkTGIzUnpWV2xGYm1GaWJHVmtJanAwY25WbExDSnBjMGhsYkcxV2JVUnZkMjVzYjJGa1JXNWhZbXhsWkNJNmRISjFaU3dpYVhOTGIzUnpTVzV6ZEdGc2JFVnVZV0pzWldRaU9uUnlkV1VzSW1selUyVnRkbVZ5VW1WeGRXbHlaV1FpT25SeWRXVjlmUT09IiwiaW5uZXJTaWduYXR1cmUiOiJleUpzYVdObGJuTmxVMmxuYm1GMGRYSmxJam9pVTNGNWJHbHBUM1YyZUhGSWNIRjJOMVl5YkV4eGNFUkhOMnB0VWtwbVZVbDJaR016Wld0RFNuTjZSeXRLTWxSdVdXSjFZMUoyV0UxTmRHMXBTR1J1WlU1NGRFSldaMHB5U0dKMEsxVTFPRXc1TTJZME1XRjBRbnBKWW1KVGIwWkpPR0ZzVUhadFYwYzRVa1JyU1VOVFlWTmxNSGRpWWpKaFR6Um9kMlp1UkZKaFVHRTJaRFJEVEZRclJISXhSazl2Y21SaGIwbGhSVlZLUjBkME9Xd3ZXVlZFWWpKcFlXSkdiVzVYTWxnclNrWXJNVm93U0dJM1EwSndhSEUzVDNSTGIwTlBNRzFWYVc0eWVtdE1ZbFk1ZUV0cmFUSmlUemxIZVhRNU4xRjRiMDVKWlhwRGVqWm1ZWGt3YkZsWkwwRkZVMjV3YWl0aUsxbEtNelJCWTFWSVkxTlZRbFIwVmpKMFVtTjJhQ3RxTDBWTUsxUlRPVkF4WjNkaVZFMHdlbnA1UmxSWlNtNVhZbFZpZFRkbWJtNTBUV2N5VlVsRlZFUmliMFV5U0VaM1ozUk9MMmN5VVNzNFVrYzVhVmwyV0c0eVVUUlhNMUZCUFQwaUxDSndkV0pzYVdOTFpYa2lPaUl0TFMwdExVSkZSMGxPSUZCVlFreEpReUJMUlZrdExTMHRMVnh1VFVsSlFrbHFRVTVDWjJ0eGFHdHBSemwzTUVKQlVVVkdRVUZQUTBGUk9FRk5TVWxDUTJkTFEwRlJSVUY2UVdGNGRWZDNjREF2UVdkd05VOTZjVzUzVTF4dVVUWXlSVE5sZFhVclVWVmpXRk1yYTFOTVZDOHhLMjU1TUdGMU9FRXhVVVZWUkV3eFMzWmlkV0l5TVZGaFZqaFNhbWhyTUVGS1QyaERPRkF5TjNJdlJGeHVXSGh4V25oWWRVbDNhRUZyTVUxMWNIVmhWSEpzYldKMFduVkpjV1ZCYUZsbmFqbGhOM0JHTTFFd1VreEVWbWhJYVdkWmFXeHZjRlpNY2pGeVVIUXJjRnh1WVhsdFdYYzRaVFp5YjFGaFUyVTBTbHB1VWxWdlNEVkVNVWh2ZEhobFUwNUZiQzlaYWxoNFdXZDFNVWhrV1ZsaGNHSldObFp4VjBaSlZXSnpOMk5PYlZ4dU1VOXBPSHB2ZHpCdFVqQnlMM0JGUkdsd1VHVndabVJLU3paa2VGbzNUazUzY1VoWVQxVmFiRkZuWjFoRFVYSklTVWRzZDBwd1ZIQlNjbU5pWm0wNFdGeHVRVGxyY1hGeFZEbHNSWEVyUkVaUVJrdFBaMVJEV25OM2RFNVlZbFJ5UkhkcU5uTnRNV2RpVlhsS2IyNXJVVk5GVm5CcmFUQlhNblpQVDBOWlpESXllRnh1VEZGSlJFRlJRVUpjYmkwdExTMHRSVTVFSUZCVlFreEpReUJMUlZrdExTMHRMVnh1SWl3aWEyVjVVMmxuYm1GMGRYSmxJam9pWlhsS2VtRlhaSFZaV0ZJeFkyMVZhVTlwU2sxVWJFWk9ZMVpzVTAxdFVrNWpNbFkxVDFkME1XRlVVa1JPVmxaUFlqSnNZVTVVUWs1aU1YQjJXakpLUjJWdGNHRk5SRUp3VERCT1YyUllSa1JOYVhSS1dsaGtiVkZXYTNoTU1qaDVWa1pvY1ZWSVVrZFRSbFp4VGpOc1VsWXpRakJaZWxKMVZXMDFWR1ZyVmtaWlZscFdZVlV4VEU5SWNHaGxSM2d4Vmxac2FXVkdUVFJqVlZaTFVUSndjVnBIVm5SYVJXaHpVVlZrYkZOWGQzWmliV3h6WkdwS2VGUlVaSEZhYWxKVVYxaHNiMVl4YkROTlNGcFJaVlYzY21GcldubFBSa3BKVTBNNWEySkhjRkJQVTNSclRtdGFiRlJJVGxsVmJXeElUVVprYUZWc1JtMVBXRW8xVkc1U05sbHVaREJPTTNCRFZubDBkRTFIVGtWVFIwNHdZMGh3TTJKdFNteFZla3BKV1ROa2JXUkZhSFpWVm1zelYxUlpNRlpFVmxCTlJuQlFUMVpTYm1OcGRHdGFTRXBQVGxaa1JVOVhOVnBoVjFaSlltNUdSR05zVmxOV2JrSm9aREJvVjAxcGRFUlJhMFp0VDBSQmVWb3dXbFJPU0ZKR1V6SlNTbUpHVWtkU1ZrWnZWbFpTUkdOclJscFZNbEp4WkRKS1IxSXdZekJQUnpGb1RXNUdNbVJyZEhoVFIwNXdWakZ3Y2xORk1IaE9NVlpLVEROYWRrNHlTbkZYVkZwUlV6SldjRll5ZUZKU1ZuQnVXakpaTVZkSVVraFpWMHBaVWtSTmNsUkhZemxRVTBselNXMWtjMkl5U21oaVJYUnNaVlZzYTBscWIybFpiVkpzV2xSVk1rNVVXWGRaTWxwcFRrUk9hazlYU1hsUFIwcHRUMVJvYkZsWFRtaGFiVVV5VGtSWmFXWlJQVDBpZlE9PSJ9 diff --git a/e2e/scripts/embedded-preflight.sh b/e2e/scripts/embedded-preflight.sh index 8f31e8f2c..ac3c6b7b3 100644 --- a/e2e/scripts/embedded-preflight.sh +++ b/e2e/scripts/embedded-preflight.sh @@ -121,7 +121,7 @@ embed_preflight() { objcopy --input-target binary --output-target binary --rename-section .data=sec_bundle /root/preflight.tar.gz /root/preflight.o rm -rf /usr/local/bin/embedded-cluster cp -Rfp /usr/local/bin/embedded-cluster-copy /usr/local/bin/embedded-cluster - objcopy --add-section sec_bundle=/root/preflight.o /usr/local/bin/embedded-cluster + objcopy --update-section sec_bundle=/root/preflight.o /usr/local/bin/embedded-cluster } has_applied_host_preflight() { diff --git a/pkg/kubeutils/kubeutils.go b/pkg/kubeutils/kubeutils.go index 935b24f8f..448361697 100644 --- a/pkg/kubeutils/kubeutils.go +++ b/pkg/kubeutils/kubeutils.go @@ -29,12 +29,12 @@ func WaitForDeployment(ctx context.Context, cli client.Client, ns, name string) if err := wait.ExponentialBackoff(backoff, func() (bool, error) { ready, err := IsDeploymentReady(ctx, cli, ns, name) if err != nil { - lasterr = fmt.Errorf("error checking status of kotsadm: %v", err) + lasterr = fmt.Errorf("unable to get deploy %s status: %v", name, err) return false, nil } return ready, nil }); err != nil { - return fmt.Errorf("timed out waiting for admin console: %v", lasterr) + return fmt.Errorf("timed out waiting for deploy %s: %v", name, lasterr) } return nil } From 35e02070604283c1ea14d9eb5afb81bda8ebca68 Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Wed, 8 Nov 2023 14:46:09 +0100 Subject: [PATCH 2/5] chore: fix wrong e2e test name --- e2e/embed_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/embed_test.go b/e2e/embed_test.go index 719171690..747cf84be 100644 --- a/e2e/embed_test.go +++ b/e2e/embed_test.go @@ -6,7 +6,7 @@ import ( "github.com/replicatedhq/embedded-cluster/e2e/cluster" ) -func AndInstall(t *testing.T) { +func TestEmbedAndInstall(t *testing.T) { t.Parallel() tc := cluster.NewTestCluster(&cluster.Input{ T: t, From 75aa64dee5cf0a8196a6149948680ceb76074bdc Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Wed, 8 Nov 2023 16:44:29 +0100 Subject: [PATCH 3/5] chore: error out if message is not found --- e2e/scripts/interactive-multi-node-install.exp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/scripts/interactive-multi-node-install.exp b/e2e/scripts/interactive-multi-node-install.exp index d3603d3b9..91ed44d4c 100644 --- a/e2e/scripts/interactive-multi-node-install.exp +++ b/e2e/scripts/interactive-multi-node-install.exp @@ -21,4 +21,8 @@ expect -re "Add another node?.*:" { send "n\r" } set timeout 600 expect "Enter a new Admin Console password:" { send "password\r" } expect "Confirm password:" { send "password\r" } -expect "Admin Console is ready!" +expect "Confirm password:" { send "password\r" } +expect { + "Admin Console is ready!" { exit 0 } + eof { exit 1 } +} From d894ce5c05fe17f25ac0543f8acceafb6d8b15c1 Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Wed, 8 Nov 2023 20:43:04 +0100 Subject: [PATCH 4/5] chore: remove surplus password confirmation --- e2e/scripts/interactive-multi-node-install.exp | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/scripts/interactive-multi-node-install.exp b/e2e/scripts/interactive-multi-node-install.exp index 91ed44d4c..dca3f1ab1 100644 --- a/e2e/scripts/interactive-multi-node-install.exp +++ b/e2e/scripts/interactive-multi-node-install.exp @@ -21,7 +21,6 @@ expect -re "Add another node?.*:" { send "n\r" } set timeout 600 expect "Enter a new Admin Console password:" { send "password\r" } expect "Confirm password:" { send "password\r" } -expect "Confirm password:" { send "password\r" } expect { "Admin Console is ready!" { exit 0 } eof { exit 1 } From 4f02f76658013ca6c032bfef26181c754a81819a Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Wed, 8 Nov 2023 23:37:05 +0100 Subject: [PATCH 5/5] chore: bump admin console version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 531578164..8b490e263 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ ARCH := $(shell uname -m) APP_NAME = embedded-cluster ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library ADMIN_CONSOLE_CHART_NAME = admin-console -ADMIN_CONSOLE_CHART_VERSION = 1.104.0 +ADMIN_CONSOLE_CHART_VERSION = 1.104.0-build.1 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator EMBEDDED_OPERATOR_CHART_VERSION = 0.2.0