From 83369ea831df4870f26a9e23434dd70eb1749cda Mon Sep 17 00:00:00 2001 From: Tao Yi Date: Tue, 10 Sep 2024 18:25:44 +0800 Subject: [PATCH] test(envtest): Setup envtest for reconciler (#555) * setup envtest for reconciler * move new reconciler tests to dedicated envtest * add envtest workflow in tests per PR * address comments --- .github/workflows/tests.yaml | 33 ++++ .tools_versions.yaml | 2 + Makefile | 39 +++++ controller/konnect/reconciler_generic_test.go | 56 ------- pkg/utils/test/setup_helpers.go | 7 +- .../reconciler_setupwithmanager_test.go | 146 ++++++++++++++++++ test/envtest/setup.go | 85 ++++++++++ 7 files changed, 309 insertions(+), 59 deletions(-) delete mode 100644 controller/konnect/reconciler_generic_test.go create mode 100644 test/envtest/reconciler_setupwithmanager_test.go create mode 100644 test/envtest/setup.go diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 24728ca49..d2bd6a290 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -173,6 +173,39 @@ jobs: with: name: tests-report path: unit-tests.xml + + envtest-tests: + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@v4 + + - name: setup golang + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: jdx/mise-action@v2 + with: + install: false + + - name: run envtest tests + run: make test.envtest + env: + GOTESTSUM_JUNITFILE: "envtest-tests.xml" + + - name: collect test coverage + uses: actions/upload-artifact@v3 + with: + name: coverage-envtest + path: coverage.envtest.out + + - name: collect test report + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: tests-report + path: envtest-tests.xml conformance-tests: runs-on: ubuntu-latest diff --git a/.tools_versions.yaml b/.tools_versions.yaml index 9a65076cd..8fd3ecd39 100644 --- a/.tools_versions.yaml +++ b/.tools_versions.yaml @@ -16,3 +16,5 @@ gotestsum: "1.12.0" crd-ref-docs: "0.1.0" # renovate: datasource=github-releases depName=vektra/mockery mockery: "2.45.0" +# renovate: datasource=github-releases depName=kubernetes-sigs/controller-runtime +setup-envtest: "0.19.0" diff --git a/Makefile b/Makefile index 1d5b4c9fb..5bae84a23 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,9 @@ MISE := $(shell which mise) mise: @mise -V >/dev/null || (echo "mise - https://github.com/jdx/mise - not found. Please install it." && exit 1) +mise-plugin-install: mise + @$(MISE) plugin install --yes -q $(DEP) $(URL) + KIC_ROLE_GENERATOR = $(PROJECT_DIR)/bin/kic-role-generator .PHONY: kic-role-generator kic-role-generator: @@ -133,6 +136,17 @@ mockery: mise yq ## Download mockery locally if necessary. @$(MISE) plugin install --yes -q mockery https://github.com/cabify/asdf-mockery.git @$(MISE) install -q mockery@$(MOCKERY_VERSION) +SETUP_ENVTEST_VERSION = $(shell $(YQ) -r '.setup-envtest' < $(TOOLS_VERSIONS_FILE)) +SETUP_ENVTEST = $(PROJECT_DIR)/bin/installs/setup-envtest/$(SETUP_ENVTEST_VERSION)/bin/setup-envtest +.PHONY: setup-envtest +setup-envtest: mise ## Download setup-envtest locally if necessary. + @$(MAKE) mise-plugin-install DEP=setup-envtest URL=https://github.com/pmalek/mise-setup-envtest.git + @$(MISE) install setup-envtest@$(SETUP_ENVTEST_VERSION) + +.PHONY: use-setup-envtest +use-setup-envtest: + $(SETUP_ENVTEST) use + # ------------------------------------------------------------------------------ # Build # ------------------------------------------------------------------------------ @@ -328,6 +342,31 @@ test.unit: test.unit.pretty: @$(MAKE) _test.unit GOTESTSUM_FORMAT=pkgname GOTESTFLAGS="$(GOTESTFLAGS)" UNIT_TEST_PATHS="$(UNIT_TEST_PATHS)" +ENVTEST_TEST_PATHS := ./test/envtest/... +ENVTEST_TIMEOUT ?= 5m +PKG_LIST=./controller/...,./internal/...,./pkg/...,./modules/... + +.PHONY: _test.envtest +_test.envtest: gotestsum setup-envtest use-setup-envtest + KUBEBUILDER_ASSETS="$(shell $(SETUP_ENVTEST) use -p path)" \ + GOTESTSUM_FORMAT=$(GOTESTSUM_FORMAT) \ + $(GOTESTSUM) -- $(GOTESTFLAGS) \ + -race \ + -timeout $(ENVTEST_TIMEOUT) \ + -covermode=atomic \ + -coverpkg=$(PKG_LIST) \ + -coverprofile=coverage.envtest.out \ + -ldflags "$(LDFLAGS_COMMON) $(LDFLAGS)" \ + $(ENVTEST_TEST_PATHS) + +.PHONY: test.envtest +test.envtest: + $(MAKE) _test.envtest GOTESTSUM_FORMAT=standard-verbose + +.PHONY: test.envtest.pretty +test.envtest.pretty: + $(MAKE) _test.envtest GOTESTSUM_FORMAT=testname + .PHONY: _test.integration _test.integration: webhook-certs-dir gotestsum GOFLAGS=$(GOFLAGS) \ diff --git a/controller/konnect/reconciler_generic_test.go b/controller/konnect/reconciler_generic_test.go deleted file mode 100644 index 838039b97..000000000 --- a/controller/konnect/reconciler_generic_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package konnect - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/kong/gateway-operator/controller/konnect/constraints" - "github.com/kong/gateway-operator/controller/konnect/ops" - "github.com/kong/gateway-operator/modules/manager/scheme" - - configurationv1 "github.com/kong/kubernetes-configuration/api/configuration/v1" - configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" - configurationv1beta1 "github.com/kong/kubernetes-configuration/api/configuration/v1beta1" - konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" -) - -func TestNewKonnectEntityReconciler(t *testing.T) { - testNewKonnectEntityReconciler(t, konnectv1alpha1.KonnectGatewayControlPlane{}) - testNewKonnectEntityReconciler(t, configurationv1alpha1.KongService{}) - testNewKonnectEntityReconciler(t, configurationv1.KongConsumer{}) - testNewKonnectEntityReconciler(t, configurationv1alpha1.KongRoute{}) - testNewKonnectEntityReconciler(t, configurationv1beta1.KongConsumerGroup{}) - // TODO: Reconcilers setting index require a real k8s API server: - // https://github.com/kubernetes-sigs/controller-runtime/issues/657 - // Maybe we should import envtest. - // testNewKonnectEntityReconciler(t, configurationv1alpha1.KongPluginBinding{}) -} - -func testNewKonnectEntityReconciler[ - T constraints.SupportedKonnectEntityType, - TEnt constraints.EntityType[T], -]( - t *testing.T, - ent T, -) { - t.Helper() - - // TODO: use a mock Konnect SDK factory here and use envtest to trigger real reconciliations and Konnect requests - sdkFactory := &ops.MockSDKFactory{} - - t.Run(ent.GetTypeName(), func(t *testing.T) { - cl := fakectrlruntimeclient.NewFakeClient() - mgr, err := ctrl.NewManager(&rest.Config{}, ctrl.Options{ - Scheme: scheme.Get(), - }) - require.NoError(t, err) - - reconciler := NewKonnectEntityReconciler[T, TEnt](sdkFactory, false, cl) - require.NoError(t, reconciler.SetupWithManager(context.Background(), mgr)) - }) -} diff --git a/pkg/utils/test/setup_helpers.go b/pkg/utils/test/setup_helpers.go index 4ecbc3853..82585e0c5 100644 --- a/pkg/utils/test/setup_helpers.go +++ b/pkg/utils/test/setup_helpers.go @@ -31,7 +31,8 @@ import ( ) const ( - kubernetesConfigurationModuleName = "github.com/kong/kubernetes-configuration" + // KubernetesConfigurationModuleName is the name of the module where we import and install Kong configuration CRDs from. + KubernetesConfigurationModuleName = "github.com/kong/kubernetes-configuration" ) func noOpClose() error { @@ -227,9 +228,9 @@ func DeployCRDs(ctx context.Context, crdPath string, operatorClient *operatorcli // CRDs for Kong configuration // First extract version of `kong/kubernetes-configuration` module used - kongCRDVersion, err := ExtractModuleVersion(kubernetesConfigurationModuleName) + kongCRDVersion, err := ExtractModuleVersion(KubernetesConfigurationModuleName) if err != nil { - return fmt.Errorf("failed to extract Kong CRDs (%s) module's version: %w", kubernetesConfigurationModuleName, err) + return fmt.Errorf("failed to extract Kong CRDs (%s) module's version: %w", KubernetesConfigurationModuleName, err) } // Then install CRDs from the module found in `$GOPATH`. kongCRDPath := filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "kong", diff --git a/test/envtest/reconciler_setupwithmanager_test.go b/test/envtest/reconciler_setupwithmanager_test.go new file mode 100644 index 000000000..c6f46497f --- /dev/null +++ b/test/envtest/reconciler_setupwithmanager_test.go @@ -0,0 +1,146 @@ +package envtest + +import ( + "context" + "testing" + "time" + + "github.com/samber/lo" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/kong/gateway-operator/controller/konnect" + "github.com/kong/gateway-operator/controller/konnect/conditions" + "github.com/kong/gateway-operator/controller/konnect/constraints" + "github.com/kong/gateway-operator/controller/konnect/ops" + "github.com/kong/gateway-operator/modules/manager/scheme" + + configurationv1 "github.com/kong/kubernetes-configuration/api/configuration/v1" + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + configurationv1beta1 "github.com/kong/kubernetes-configuration/api/configuration/v1beta1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +func TestNewKonnectEntityReconciler(t *testing.T) { + testNewKonnectEntityReconciler(t, konnectv1alpha1.KonnectGatewayControlPlane{}, konnectGatewayControlPlaneTestCases) + testNewKonnectEntityReconciler(t, configurationv1alpha1.KongService{}, nil) + testNewKonnectEntityReconciler(t, configurationv1.KongConsumer{}, nil) + testNewKonnectEntityReconciler(t, configurationv1alpha1.KongRoute{}, nil) + testNewKonnectEntityReconciler(t, configurationv1beta1.KongConsumerGroup{}, nil) + testNewKonnectEntityReconciler(t, configurationv1alpha1.KongPluginBinding{}, nil) +} + +const ( + testNamespaceName = "test" + envTestWaitDuration = time.Second + envTestWaitTick = 20 * time.Millisecond +) + +type konnectEntityReconcilerTestCase struct { + name string + objectOps func(ctx context.Context, t *testing.T, cl client.Client) + eventuallyPredicate func(ctx context.Context, t *testing.T, cl client.Client) bool +} + +var konnectGatewayControlPlaneTestCases = []konnectEntityReconcilerTestCase{ + { + name: "should resolve auth", + objectOps: func(ctx context.Context, t *testing.T, cl client.Client) { + auth := &konnectv1alpha1.KonnectAPIAuthConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "auth", + Namespace: testNamespaceName, + }, + Spec: konnectv1alpha1.KonnectAPIAuthConfigurationSpec{ + Type: konnectv1alpha1.KonnectAPIAuthTypeToken, + Token: "kpat_test", + }, + } + require.NoError(t, cl.Create(ctx, auth)) + cp := &konnectv1alpha1.KonnectGatewayControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-1", + Namespace: testNamespaceName, + }, + Spec: konnectv1alpha1.KonnectGatewayControlPlaneSpec{ + KonnectConfiguration: konnectv1alpha1.KonnectConfiguration{ + APIAuthConfigurationRef: konnectv1alpha1.KonnectAPIAuthConfigurationRef{ + Name: "auth", + }, + }, + }, + } + require.NoError(t, cl.Create(ctx, cp)) + }, + eventuallyPredicate: func(ctx context.Context, t *testing.T, cl client.Client) bool { + cp := &konnectv1alpha1.KonnectGatewayControlPlane{} + err := cl.Get(ctx, k8stypes.NamespacedName{Namespace: testNamespaceName, Name: "cp-1"}, cp) + require.NoError(t, err) + // TODO: setup mock Konnect SDK and verify that Konnect CP is "Created". + return lo.ContainsBy(cp.Status.Conditions, func(condition metav1.Condition) bool { + return condition.Type == conditions.KonnectEntityAPIAuthConfigurationResolvedRefConditionType && condition.Status == metav1.ConditionTrue + }) + }, + }, +} + +func testNewKonnectEntityReconciler[ + T constraints.SupportedKonnectEntityType, + TEnt constraints.EntityType[T], +]( + t *testing.T, + ent T, + testCases []konnectEntityReconcilerTestCase, +) { + t.Helper() + + sdkFactory := &ops.MockSDKFactory{} + + t.Run(ent.GetTypeName(), func(t *testing.T) { + s := scheme.Get() + cfg := Setup(t, s) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: s, + Metrics: metricsserver.Options{ + // We do not need metrics server so we set BindAddress to 0 to disable it. + BindAddress: "0", + }, + }) + require.NoError(t, err) + + cl := mgr.GetClient() + reconciler := konnect.NewKonnectEntityReconciler[T, TEnt](sdkFactory, false, cl) + require.NoError(t, reconciler.SetupWithManager(ctx, mgr)) + + err = cl.Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }) + require.NoError(t, err) + + t.Logf("Starting manager for test case %s", t.Name()) + go func() { + err := mgr.Start(ctx) + require.NoError(t, err) + }() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.objectOps(ctx, t, cl) + require.Eventually(t, func() bool { + return tc.eventuallyPredicate(ctx, t, cl) + }, envTestWaitDuration, envTestWaitTick) + }) + } + }) +} diff --git a/test/envtest/setup.go b/test/envtest/setup.go new file mode 100644 index 000000000..30ec2a4d7 --- /dev/null +++ b/test/envtest/setup.go @@ -0,0 +1,85 @@ +package envtest + +import ( + "go/build" + "os" + "os/signal" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + testutil "github.com/kong/gateway-operator/pkg/utils/test" +) + +// Setup sets up a test k8s API server environment and returned the configuration. +func Setup(t *testing.T, scheme *k8sruntime.Scheme) *rest.Config { + t.Helper() + + testEnv := &envtest.Environment{ + ControlPlaneStopTimeout: time.Second * 60, + } + + t.Logf("starting envtest environment for test %s...", t.Name()) + cfg, err := testEnv.Start() + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + done := make(chan struct{}) + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + defer wg.Done() + select { + case <-ch: + _ = testEnv.Stop() + case <-done: + _ = testEnv.Stop() + } + }() + + kongConfVersion, err := testutil.ExtractModuleVersion(testutil.KubernetesConfigurationModuleName) + require.NoError(t, err) + kongCRDPath := filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "kong", "kubernetes-configuration@"+kongConfVersion, "config", "crd") + kongBaseCRDPath := kongCRDPath + "/bases" + // we do not deal with incubator resources here, so we only install base CRDs. + t.Logf("install Kong CRDs from path %s", kongCRDPath) + _, err = envtest.InstallCRDs(cfg, envtest.CRDInstallOptions{ + Scheme: scheme, + Paths: []string{kongBaseCRDPath}, + ErrorIfPathMissing: true, + }) + require.NoError(t, err) + + config, err := clientcmd.BuildConfigFromFlags(cfg.Host, "") + require.NoError(t, err) + config.CertData = cfg.CertData + config.CAData = cfg.CAData + config.KeyData = cfg.KeyData + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + require.NoError(t, err) + + i, err := discoveryClient.ServerVersion() + require.NoError(t, err) + + t.Logf("envtest environment (%s) started at %s", i, cfg.Host) + + t.Cleanup(func() { + t.Helper() + t.Logf("stopping envtest environment for test %s", t.Name()) + close(done) + wg.Wait() + }) + + return cfg +}