Skip to content

Commit

Permalink
tests: add Konnect entities integration test (#537)
Browse files Browse the repository at this point in the history
* tests: add Konnect entities integration test

* address review comments
  • Loading branch information
czeslavo authored Sep 3, 2024
1 parent 4ea8ea2 commit c1fdd11
Show file tree
Hide file tree
Showing 13 changed files with 173 additions and 8 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,8 @@ jobs:
KONG_CONTROLLER_OUT: stdout
GOTESTSUM_JUNITFILE: integration-tests-webhook-enabled-${{ matrix.webhook-enabled }}.xml
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
KONG_TEST_KONNECT_ACCESS_TOKEN: ${{ secrets.KONG_TEST_KONNECT_ACCESS_TOKEN }}
KONG_TEST_KONNECT_SERVER_URL: us.api.konghq.tech

- name: upload diagnostics
if: always()
Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,7 @@ _run:
-enable-controller-controlplane \
-enable-controller-gateway \
-enable-controller-aigateway \
-enable-controller-konnect \
-zap-time-encoding iso8601 \
-zap-log-level 2 \
-zap-devel true
Expand Down
5 changes: 3 additions & 2 deletions config/debug/manager_debug.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ spec:
- -zap-time-encoding=iso8601
- -cluster-ca-secret-namespace=kong-system
- -zap-log-level=debug
- -enable-controller-kongplugininstallation=true
- -enable-validating-webhook=true
- -enable-controller-kongplugininstallation
- -enable-controller-konnect
- -enable-validating-webhook
name: manager
env:
- name: GATEWAY_OPERATOR_DEVELOPMENT_MODE
Expand Down
5 changes: 3 additions & 2 deletions config/dev/manager_dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ spec:
- -cluster-ca-secret-namespace=kong-system
- -zap-log-level=debug
- -zap-devel=true
- -enable-controller-kongplugininstallation=true
- -enable-validating-webhook=true
- -enable-controller-kongplugininstallation
- -enable-validating-webhook
- -enable-controller-konnect
name: manager
env:
- name: GATEWAY_OPERATOR_DEVELOPMENT_MODE
Expand Down
2 changes: 2 additions & 0 deletions config/rbac/role/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ rules:
- konglicenses/status
- kongpluginbindings/status
- kongplugins/status
- kongroutes/status
- kongservices/status
- kongupstreampolicies/status
- kongvaults/status
Expand All @@ -164,6 +165,7 @@ rules:
- configuration.konghq.com
resources:
- kongpluginbindings
- kongroutes
- kongservices
verbs:
- get
Expand Down
2 changes: 1 addition & 1 deletion controller/konnect/index_kongpluginbinding.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func IndexOptionsForKongPluginBinding() []ReconciliationIndexOption {
return []ReconciliationIndexOption{
{
IndexObject: &configurationv1alpha1.KongPluginBinding{},
IndexField: IndexFieldKongPluginBindingKongClusterPluginReference,
IndexField: IndexFieldKongPluginBindingKongPluginReference,
ExtractValue: kongPluginReferencesFromKongPluginBinding,
},
{
Expand Down
3 changes: 3 additions & 0 deletions controller/konnect/reconciler_generic_rbac.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,7 @@ package konnect
//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongpluginbindings,verbs=get;list;watch;update;patch
//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongpluginbindings/status,verbs=get;update;patch

//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongroutes,verbs=get;list;watch;update;patch
//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongroutes/status,verbs=get;update;patch

//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
9 changes: 9 additions & 0 deletions pkg/utils/test/clients.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import (
operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1"
operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1"
operatorclient "github.com/kong/gateway-operator/pkg/clientset"

configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1"
konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1"
)

// K8sClients is a struct that contains all the Kubernetes clients needed by the tests.
Expand Down Expand Up @@ -51,6 +54,12 @@ func NewK8sClients(env environments.Environment) (K8sClients, error) {
if err := gatewayv1beta1.Install(clients.MgrClient.Scheme()); err != nil {
return clients, err
}
if err := konnectv1alpha1.AddToScheme(clients.MgrClient.Scheme()); err != nil {
return clients, err
}
if err := configurationv1alpha1.AddToScheme(clients.MgrClient.Scheme()); err != nil {
return clients, err
}

// TODO: remove this when support for v1alpha2 is dropped in GW API. For now
// we need to add it to the scheme so that we can pass conformance tests.
Expand Down
6 changes: 3 additions & 3 deletions pkg/utils/test/setup_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
operatorclient "github.com/kong/gateway-operator/pkg/clientset"
)

const kicCRDsKustomizeURL = "https://github.com/Kong/kubernetes-ingress-controller/config/crd"
const kongCRDsKustomizeURL = "https://github.com/Kong/kubernetes-configuration/config/crd"

func noOpClose() error {
return nil
Expand Down Expand Up @@ -197,8 +197,8 @@ func DeployCRDs(ctx context.Context, crdPath string, operatorClient *operatorcli
return err
}

fmt.Printf("INFO: deploying KIC CRDs: %s\n", kicCRDsKustomizeURL)
if err := clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL); err != nil {
fmt.Printf("INFO: deploying Kong (kubernetes-configuration) CRDs: %s\n", kongCRDsKustomizeURL)
if err := clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kongCRDsKustomizeURL); err != nil {
return err
}

Expand Down
10 changes: 10 additions & 0 deletions test/envvars.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,13 @@ func IsMetalLBDisabled() bool {
}
return ret
}

// KonnectAccessToken returns the Konnect access token for the test environment.
func KonnectAccessToken() string {
return os.Getenv("KONG_TEST_KONNECT_ACCESS_TOKEN")
}

// KonnectServerURL returns the Konnect server URL for the test environment.
func KonnectServerURL() string {
return os.Getenv("KONG_TEST_KONNECT_SERVER_URL")
}
1 change: 1 addition & 0 deletions test/integration/suite.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ func DefaultControllerConfigForTests() manager.Config {
cfg.AIGatewayControllerEnabled = true
cfg.ValidatingWebhookEnabled = webhookEnabled
cfg.AnonymousReports = false
cfg.KonnectControllersEnabled = true

cfg.NewClientFunc = func(config *rest.Config, options client.Options) (client.Client, error) {
// always hijack and impersonate the system service account here so that the manager
Expand Down
134 changes: 134 additions & 0 deletions test/integration/test_konnect_entities.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package integration

import (
"testing"
"time"

"github.com/Kong/sdk-konnect-go/models/components"
"github.com/google/uuid"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

testutils "github.com/kong/gateway-operator/pkg/utils/test"
"github.com/kong/gateway-operator/test"
"github.com/kong/gateway-operator/test/helpers"

configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1"
konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1"
)

func TestKonnectEntities(t *testing.T) {
// A cleaner is created underneath anyway, and a whole namespace is deleted eventually.
// We can't use a cleaner to delete objects because it handles deletes in FIFO order and that won't work in this
// case: KonnectAPIAuthConfiguration shouldn't be deleted before any other object as that is required for others to
// complete their finalizer which is deleting a reflecting entity in Konnect. That's why we're only cleaning up a
// KonnectControlPlane and waiting for its deletion synchronously with deleteObjectAndWaitForDeletionFn to ensure it
// was successfully deleted along with its children. The KonnectAPIAuthConfiguration is implicitly deleted along
// with the namespace.
ns, _ := helpers.SetupTestEnv(t, GetCtx(), GetEnv())

// Let's generate a unique test ID that we can refer to in Konnect entities.
// Using only the first 8 characters of the UUID to keep the ID short enough for Konnect to accept it as a part
// of an entity name.
testID := uuid.NewString()[:8]
t.Logf("Running Konnect entities test with ID: %s", testID)

t.Logf("Creating KonnectAPIAuthConfiguration")
authCfg := &konnectv1alpha1.KonnectAPIAuthConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "auth-" + testID,
Namespace: ns.Name,
},
Spec: konnectv1alpha1.KonnectAPIAuthConfigurationSpec{
Type: konnectv1alpha1.KonnectAPIAuthTypeToken,
Token: test.KonnectAccessToken(),
ServerURL: test.KonnectServerURL(),
},
}
err := GetClients().MgrClient.Create(GetCtx(), authCfg)
require.NoError(t, err)

cpName := "cp-" + testID
t.Logf("Creating KonnectControlPlane %s", cpName)
cp := &konnectv1alpha1.KonnectControlPlane{
ObjectMeta: metav1.ObjectMeta{
Name: cpName,
Namespace: ns.Name,
},
Spec: konnectv1alpha1.KonnectControlPlaneSpec{
CreateControlPlaneRequest: components.CreateControlPlaneRequest{
Name: cpName,
ClusterType: lo.ToPtr(components.ClusterTypeClusterTypeControlPlane),
Labels: map[string]string{"test_id": testID},
},
KonnectConfiguration: konnectv1alpha1.KonnectConfiguration{
APIAuthConfigurationRef: konnectv1alpha1.KonnectAPIAuthConfigurationRef{
Name: authCfg.Name,
},
},
},
}
err = GetClients().MgrClient.Create(GetCtx(), cp)
require.NoError(t, err)
t.Cleanup(deleteObjectAndWaitForDeletionFn(t, cp))

t.Logf("Waiting for Konnect ID to be assigned to ControlPlane %s/%s", cp.Namespace, cp.Name)
require.EventuallyWithT(t, func(t *assert.CollectT) {
err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, cp)
require.NoError(t, err)
assert.NotEmpty(t, cp.Status.KonnectEntityStatus.GetKonnectID())
assert.NotEmpty(t, cp.Status.KonnectEntityStatus.GetOrgID())
assert.NotEmpty(t, cp.Status.KonnectEntityStatus.GetServerURL())
}, testutils.ObjectUpdateTimeout, time.Second)

t.Logf("Creating KongService")
ksName := "ks-" + testID
ks := &configurationv1alpha1.KongService{
ObjectMeta: metav1.ObjectMeta{
Name: "ks-" + testID,
Namespace: ns.Name,
},
Spec: configurationv1alpha1.KongServiceSpec{
ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{
Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef,
KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{Name: cp.Name},
},
KongServiceAPISpec: configurationv1alpha1.KongServiceAPISpec{
Name: lo.ToPtr(ksName),
URL: lo.ToPtr("http://example.com"),
},
},
}
err = GetClients().MgrClient.Create(GetCtx(), ks)
require.NoError(t, err)

t.Logf("Waiting for KongService to be updated with Konnect ID")
require.EventuallyWithT(t, func(t *assert.CollectT) {
err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: ks.Name, Namespace: ks.Namespace}, ks)
require.NoError(t, err)
assert.NotEmpty(t, ks.Status.Konnect.KonnectEntityStatus.GetKonnectID())
assert.NotEmpty(t, ks.Status.Konnect.KonnectEntityStatus.GetOrgID())
assert.NotEmpty(t, ks.Status.Konnect.KonnectEntityStatus.GetServerURL())
}, testutils.ObjectUpdateTimeout, time.Second)

// TODO(czeslavo): test all supported entities here
}

// deleteObjectAndWaitForDeletionFn returns a function that deletes the given object and waits for it to be gone.
// It's designed to be used with t.Cleanup() to ensure the object is properly deleted (it's not stuck with finalizers, etc.).
func deleteObjectAndWaitForDeletionFn(t *testing.T, obj client.Object) func() {
return func() {
err := GetClients().MgrClient.Delete(GetCtx(), obj)
require.NoError(t, err)

require.EventuallyWithT(t, func(t *assert.CollectT) {
err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, obj)
assert.True(t, k8serrors.IsNotFound(err))
}, testutils.ObjectUpdateTimeout, time.Second)
}
}
1 change: 1 addition & 0 deletions test/integration/zz_generated_registered_testcases.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit c1fdd11

Please sign in to comment.