From 64bc260d261475a6d781c93546387850bde4f20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Tou=C5=A1ek?= Date: Fri, 6 Dec 2024 10:40:01 +0100 Subject: [PATCH] test: Fix stream tests --- internal/pkg/service/stream/config/config_test.go | 4 +++- internal/pkg/service/stream/dependencies/mocked.go | 2 ++ .../sink/type/tablesink/keboola/bridge/token_test.go | 9 +++++++++ test/stream/api/api_test.go | 3 ++- test/stream/bridge/keboola/guest_test.go | 2 ++ test/stream/bridge/keboola/keboola_test.go | 2 ++ 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/pkg/service/stream/config/config_test.go b/internal/pkg/service/stream/config/config_test.go index dd2535955a..ea6caeda39 100644 --- a/internal/pkg/service/stream/config/config_test.go +++ b/internal/pkg/service/stream/config/config_test.go @@ -15,6 +15,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/common/configpatch" "github.com/keboola/keboola-as-code/internal/pkg/service/common/ptr" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/encryption" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level" local "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/config" @@ -302,7 +303,7 @@ storage: expiration: 30m0s encryption: # Encryption provider. Validation rules: required,oneof=none native gcp aws azure - provider: native + provider: none native: # Secret key for local encryption. Do not use in production. secretKey: '*****' @@ -329,6 +330,7 @@ encryption: cfg.Source.HTTP.PublicURL, _ = url.Parse("https://stream-in.keboola.local") cfg.Etcd.Endpoint = "test-etcd" cfg.Etcd.Namespace = "test-namespace" + cfg.Encryption.Provider = encryption.ProviderNative cfg.Encryption.Native.SecretKey = []byte("12345678901234567890123456789012") cfg.Encryption.Normalize() require.NoError(t, validator.New().Validate(context.Background(), cfg)) diff --git a/internal/pkg/service/stream/dependencies/mocked.go b/internal/pkg/service/stream/dependencies/mocked.go index 32a46ee6d3..b897f3422e 100644 --- a/internal/pkg/service/stream/dependencies/mocked.go +++ b/internal/pkg/service/stream/dependencies/mocked.go @@ -13,6 +13,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/encryption" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/dummy" "github.com/keboola/keboola-as-code/internal/pkg/utils/netutils" ) @@ -66,6 +67,7 @@ func testConfig(tb testing.TB, d dependencies.Mocked) config.Config { secretKey := make([]byte, 32) _, err := rand.Read(secretKey) require.NoError(tb, err) + cfg.Encryption.Provider = encryption.ProviderNative cfg.Encryption.Native.SecretKey = secretKey // Validate configuration diff --git a/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/token_test.go b/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/token_test.go index 5d552771c2..c5368082a2 100644 --- a/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/token_test.go +++ b/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/token_test.go @@ -3,6 +3,7 @@ package bridge_test import ( "bytes" "context" + "crypto/rand" "testing" "time" @@ -22,6 +23,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/encryption" keboolasink "github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/type/tablesink/keboola" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/model/schema" bridgeTest "github.com/keboola/keboola-as-code/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/test" @@ -48,6 +50,9 @@ func TestBridge_MigrateTokens(t *testing.T) { sourceKey := key.SourceKey{BranchKey: branchKey, SourceID: "my-source"} sinkKey := key.SinkKey{SourceKey: sourceKey, SinkID: "my-sink"} ignoredKeys := etcdhelper.WithIgnoredKeyPattern("^definition/|storage/file/all/|storage/slice/all/|storage/volume/") + secretKey := make([]byte, 32) + _, err := rand.Read(secretKey) + require.NoError(t, err) // Get services d, mocked := dependencies.NewMockedAPIScopeWithConfig(t, ctx, func(cfg *config.Config) { @@ -64,6 +69,10 @@ func TestBridge_MigrateTokens(t *testing.T) { SlicesCount: 100, Expiration: duration.From(30 * time.Minute), } + cfg.Encryption.Provider = encryption.ProviderNative + cfg.Encryption.Native = &encryption.NativeConfig{ + SecretKey: secretKey, + } }, deps.WithClock(clk)) client := mocked.TestEtcdClient() defRepo := d.DefinitionRepository() diff --git a/test/stream/api/api_test.go b/test/stream/api/api_test.go index 0dae0f2223..3f4d6b69d8 100644 --- a/test/stream/api/api_test.go +++ b/test/stream/api/api_test.go @@ -4,6 +4,7 @@ package api import ( "context" "crypto/rand" + "encoding/base64" "fmt" "os" "path/filepath" @@ -78,7 +79,7 @@ func TestStreamApiE2E(t *testing.T) { "STREAM_ETCD_USERNAME": etcdCfg.Username, "STREAM_ETCD_PASSWORD": etcdCfg.Password, "STREAM_ENCRYPTION_PROVIDER": "native", - "STREAM_ENCRYPTION_NATIVE_SECRET_KEY": string(secretKey), + "STREAM_ENCRYPTION_NATIVE_SECRET_KEY": base64.StdEncoding.EncodeToString(secretKey), }) // Run the test diff --git a/test/stream/bridge/keboola/guest_test.go b/test/stream/bridge/keboola/guest_test.go index 932712690c..8d124925df 100644 --- a/test/stream/bridge/keboola/guest_test.go +++ b/test/stream/bridge/keboola/guest_test.go @@ -16,6 +16,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/stream/api" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/encryption" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" "github.com/keboola/keboola-as-code/internal/pkg/utils/netutils" ) @@ -33,6 +34,7 @@ func TestGuestUserWorkflow(t *testing.T) { require.NoError(t, err) modifyConfig := func(cfg *config.Config) { + cfg.Encryption.Provider = encryption.ProviderNative cfg.Encryption.Native.SecretKey = secretKey apiPort := netutils.FreePortForTest(t) cfg.API.Listen = "0.0.0.0:" + strconv.FormatInt(int64(apiPort), 10) diff --git a/test/stream/bridge/keboola/keboola_test.go b/test/stream/bridge/keboola/keboola_test.go index 44553886d9..e18369a14c 100644 --- a/test/stream/bridge/keboola/keboola_test.go +++ b/test/stream/bridge/keboola/keboola_test.go @@ -19,6 +19,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/encoding/json" "github.com/keboola/keboola-as-code/internal/pkg/service/common/duration" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/encryption" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/local/diskwriter/network" stagingConfig "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/staging/config" targetConfig "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/level/target/config" @@ -40,6 +41,7 @@ func TestKeboolaBridgeWorkflow(t *testing.T) { // Update configuration to make the cluster testable configFn := func(cfg *config.Config) { // Setup encryption + cfg.Encryption.Provider = encryption.ProviderNative cfg.Encryption.Native.SecretKey = secretKey // Enable metadata cleanup for removing storage jobs cfg.Storage.MetadataCleanup.Enabled = true