From 251aef8147691f5b0f65568e576d28c9ba74c174 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Thu, 27 Oct 2022 13:36:24 -0400 Subject: [PATCH 1/2] allow migration phase to be manually set --- pkg/config/config.go | 36 ++++++++++++++----------------- pkg/controller/validate_config.go | 9 +++++--- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 8cb9d59e..9e443999 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -49,6 +49,7 @@ var ( spiceDBCmdKey = newKey("cmd", "spicedb") skipMigrationsKey = newBoolOrStringKey("skipMigrations", false) targetMigrationKey = newStringKey("targetMigration") + targetPhase = newStringKey("datastoreMigrationPhase") logLevelKey = newKey("logLevel", "info") migrationLogLevelKey = newKey("migrationLogLevel", "debug") spannerCredentialsKey = newStringKey("spannerCredentials") @@ -160,6 +161,7 @@ func NewConfig(nn types.NamespacedName, uid types.UID, currentState *SpiceDBMigr SpiceDBCmd: spiceConfig.SpiceDBCmd, DatastoreTLSSecretName: datastoreTLSSecretKey.pop(config), TargetMigration: targetMigrationKey.pop(config), + TargetPhase: targetPhase.pop(config), } datastoreEngine := datastoreEngineKey.pop(config) @@ -172,10 +174,14 @@ func NewConfig(nn types.NamespacedName, uid types.UID, currentState *SpiceDBMigr image := imageKey.pop(config) image, imgWarnings := validateImage(image, globalConfig) - var err error - migrationConfig.TargetSpiceDBImage, migrationConfig.TargetMigration, migrationConfig.TargetPhase, err = computeTargets(image, datastoreEngine, currentState, globalConfig, rolling) - if err != nil { - imgWarnings = append(imgWarnings, err) + if len(migrationConfig.TargetMigration) == 0 && len(migrationConfig.TargetPhase) == 0 { + var err error + migrationConfig.TargetSpiceDBImage, migrationConfig.TargetMigration, migrationConfig.TargetPhase, err = computeTargets(image, datastoreEngine, currentState, globalConfig, rolling) + if err != nil { + imgWarnings = append(imgWarnings, err) + } + } else { + migrationConfig.TargetSpiceDBImage = image } if !globalConfig.DisableImageValidation { @@ -361,19 +367,7 @@ func validateImage(image string, globalConfig *OperatorConfig) (string, []error) func computeTargets(image, engine string, currentState *SpiceDBMigrationState, globalConfig *OperatorConfig, rolling bool) (targetImage, targetMigration, targetPhase string, err error) { specBaseImage, tag, _ := ExplodeImage(image) targetImage = image - - // if head migration, look up the actual migration name - if headMigration, ok := globalConfig.HeadMigrations[SpiceDBDatastoreState{Tag: tag, Datastore: engine}.String()]; ok { - targetMigration = headMigration - } else { - targetMigration = "head" - } - - // don't look for required edges if the currently running image is - // the desired one (only check when the image is changed) - if currentState == nil || currentState.Tag == tag { - return - } + targetMigration = "head" // if already migrating or rolling, use current state if rolling { @@ -383,11 +377,13 @@ func computeTargets(image, engine string, currentState *SpiceDBMigrationState, g return } - // if head migration, look up the actual migration name (if present) + // look up the actual head migration name, if any if currentState.Migration == "" || currentState.Migration == "head" { - if headMigration, ok := globalConfig.HeadMigrations[SpiceDBDatastoreState{Tag: currentState.Tag, Datastore: engine}.String()]; ok { - currentState.Migration = headMigration + headMigrationName, ok := globalConfig.HeadMigrations[SpiceDBDatastoreState{Tag: tag, Datastore: engine}.String()] + if !ok { + headMigrationName = "head" } + currentState.Migration = headMigrationName } // if there's a required edge, take it diff --git a/pkg/controller/validate_config.go b/pkg/controller/validate_config.go index df25a924..131808c5 100644 --- a/pkg/controller/validate_config.go +++ b/pkg/controller/validate_config.go @@ -34,13 +34,16 @@ func (c *ValidateConfigHandler) Handle(ctx context.Context) { secret := CtxSecret.Value(ctx) operatorConfig := CtxOperatorConfig.MustValue(ctx) status := CtxClusterStatus.MustValue(ctx).Status - _, tag, _ := config.ExplodeImage(status.Image) + _, statusTag, _ := config.ExplodeImage(status.Image) currentState := &config.SpiceDBMigrationState{ - Tag: tag, + Tag: statusTag, Phase: status.Phase, Migration: status.Migration, } - validatedConfig, warning, err := config.NewConfig(nn, cluster.UID, currentState, operatorConfig, rawConfig, secret, currentStatus.IsStatusConditionTrue(v1alpha1.ConditionTypeMigrating) || currentStatus.IsStatusConditionTrue(v1alpha1.ConditionTypeRolling)) + rolloutInProgress := currentStatus.IsStatusConditionTrue(v1alpha1.ConditionTypeMigrating) || + currentStatus.IsStatusConditionTrue(v1alpha1.ConditionTypeRolling) || + currentStatus.Status.CurrentMigrationHash != currentStatus.Status.TargetMigrationHash + validatedConfig, warning, err := config.NewConfig(nn, cluster.UID, currentState, operatorConfig, rawConfig, secret, rolloutInProgress) if err != nil { failedCondition := v1alpha1.NewInvalidConfigCondition(CtxSecretHash.Value(ctx), err) if existing := currentStatus.FindStatusCondition(v1alpha1.ConditionValidatingFailed); existing != nil && existing.Message == failedCondition.Message { From 8adf54ddf623daa9f4ad7b8a38eaf13fd37fec9a Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Thu, 27 Oct 2022 17:09:44 -0400 Subject: [PATCH 2/2] always update from a required node previous behavior would leave a running cluster alone even if it matched a required edge, and only move it if it was changed. this led to a couple of edge cases where it was possible to not detect required edges. instead, if the current state matches a required edge, the update is always taken. also adds helpers for building update graphs --- .github/workflows/build-test.yaml | 5 + e2e/cluster_test.go | 243 ++----------------------- e2e/spanner.yaml | 2 +- pkg/config/config.go | 17 +- pkg/config/config_test.go | 39 ++-- pkg/config/global.go | 82 +++++++-- pkg/controller/validate_config_test.go | 2 +- tools/go.mod | 13 +- tools/go.sum | 26 +-- 9 files changed, 142 insertions(+), 287 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 97d5aa80..f6548404 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -75,11 +75,16 @@ jobs: file: "spicedb/Dockerfile" tags: "spicedb:dev,spicedb:updated" outputs: "type=docker,dest=/tmp/image.tar" + - name: "Tag Test v1.13.0 image" + run: | + docker pull ghcr.io/authzed/spicedb:v1.13.0 + docker tag ghcr.io/authzed/spicedb:v1.13.0 spicedb:v1.13.0 - name: "Run Ginkgo Tests" run: "go run github.com/onsi/ginkgo/v2/ginkgo --skip-package ./spicedb --tags=e2e -r --procs=2 -v --randomize-all --randomize-suites --fail-on-pending --fail-fast --no-color --race --trace --json-report=report.json -- -v=4" env: PROVISION: "true" ARCHIVES: "/tmp/image.tar" + IMAGES: "spicedb:v1.13.0" - uses: "docker/build-push-action@v2" with: context: "./" diff --git a/e2e/cluster_test.go b/e2e/cluster_test.go index 06d95e12..3683fa63 100644 --- a/e2e/cluster_test.go +++ b/e2e/cluster_test.go @@ -374,7 +374,6 @@ var _ = Describe("SpiceDBClusters", func() { } // found a matching job - TailF(job) // wait for job to succeed @@ -898,193 +897,6 @@ var _ = Describe("SpiceDBClusters", func() { }) }) }) - - When("a valid SpiceDBCluster with required upgrade edges", Ordered, func() { - var spiceCluster *v1alpha1.SpiceDBCluster - var migration string - - BeforeAll(func() { - ctx, cancel := context.WithCancel(context.Background()) - DeferCleanup(cancel) - - switch dsDef.datastoreEngine { - case "spanner": - migration = "add-metadata-and-counters" - case "cockroachdb": - migration = "add-metadata-and-counters" - case "postgres": - migration = "drop-bigserial-ids" - case "mysql": - migration = "add_ns_config_id" - } - - dev := config.SpiceDBMigrationState{Tag: "dev", Migration: "head"} - updated := config.SpiceDBMigrationState{Tag: "updated", Migration: migration} - newConfig := config.OperatorConfig{ - AllowedTags: []string{"dev", "updated", "next"}, - AllowedImages: []string{"spicedb"}, - RequiredEdges: map[string]string{ - dev.String(): updated.String(), - }, - Nodes: map[string]config.SpiceDBMigrationState{ - dev.String(): dev, - updated.String(): updated, - }, - } - WriteConfig(newConfig) - - config := map[string]any{ - "logLevel": "debug", - "datastoreEngine": dsDef.datastoreEngine, - "envPrefix": spicedbEnvPrefix, - "cmd": spicedbCmd, - "tlsSecretName": "spicedb-grpc-tls", - "dispatchUpstreamCASecretName": "spicedb-grpc-tls", - "image": "spicedb:dev", - } - for k, v := range dsDef.passthroughConfig { - config[k] = v - } - jsonConfig, err := json.Marshal(config) - Expect(err).To(BeNil()) - spiceCluster = &v1alpha1.SpiceDBCluster{ - TypeMeta: metav1.TypeMeta{ - Kind: v1alpha1.SpiceDBClusterKind, - APIVersion: v1alpha1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("test2-%s", dsDef.label), - Namespace: testNamespace, - }, - Spec: v1alpha1.ClusterSpec{ - Config: jsonConfig, - SecretRef: "spicedb3", - }, - } - - tlsSecret := GenerateCertManagerCompliantTLSSecretForService( - ktypes.NamespacedName{Name: spiceCluster.Name, Namespace: spiceCluster.Namespace}, - ktypes.NamespacedName{Name: "spicedb-grpc-tls", Namespace: spiceCluster.Namespace}, - ) - _, err = kclient.CoreV1().Secrets(spiceCluster.Namespace).Create(ctx, tlsSecret, metav1.CreateOptions{}) - Expect(err).To(Succeed()) - DeferCleanup(kclient.CoreV1().Secrets(spiceCluster.Namespace).Delete, ctx, tlsSecret.Name, metav1.DeleteOptions{}) - - secret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spicedb3", - Namespace: spiceCluster.Namespace, - }, - StringData: map[string]string{ - "datastore_uri": dsDef.datastoreUri, - "migration_secrets": "kaitain-bootstrap-token=testtesttesttest,sharewith-bootstrap-token=testtesttesttest,thumper-bootstrap-token=testtesttesttest,metrics-proxy-token=testtesttesttest", - "preshared_key": "testtesttesttest", - }, - } - _, err = kclient.CoreV1().Secrets(spiceCluster.Namespace).Create(ctx, &secret, metav1.CreateOptions{}) - Expect(err).To(Succeed()) - DeferCleanup(kclient.CoreV1().Secrets(spiceCluster.Namespace).Delete, ctx, secret.Name, metav1.DeleteOptions{}) - - u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(spiceCluster) - Expect(err).To(Succeed()) - _, err = client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Create(ctx, &unstructured.Unstructured{Object: u}, metav1.CreateOptions{}) - Expect(err).To(Succeed()) - - AssertMigrationsCompleted("spicedb:dev", "", "", - func() (string, string, string) { - return spiceCluster.Namespace, spiceCluster.Name, dsDef.datastoreEngine - }) - }) - - AfterAll(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - newConfig := config.OperatorConfig{ - ImageName: "spicedb", - ImageTag: "dev", - } - WriteConfig(newConfig) - - Expect(client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Delete(ctx, spiceCluster.Name, metav1.DeleteOptions{})).To(Succeed()) - - AssertDependentResourceCleanup(spiceCluster.Namespace, spiceCluster.Name, "spicedb3") - }) - - Describe("with a migrated datastore", Ordered, func() { - BeforeAll(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - AssertHealthySpiceDBCluster("spicedb:dev", - func() (string, string) { - return testNamespace, spiceCluster.Name - }, Not(ContainSubstring("ERROR: kuberesolver"))) - - Eventually(func(g Gomega) { - clusterUnst, err := client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Get(ctx, spiceCluster.Name, metav1.GetOptions{}) - g.Expect(err).To(Succeed()) - cluster, err := typed.UnstructuredObjToTypedObj[*v1alpha1.SpiceDBCluster](clusterUnst) - g.Expect(err).To(Succeed()) - GinkgoWriter.Println(cluster.Status.Conditions) - g.Expect(len(cluster.Status.Conditions)).To(BeZero()) - }).Should(Succeed()) - }) - - When("the image is updated but there is a required edge", func() { - BeforeEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - DeferCleanup(cancel) - - Eventually(func(g Gomega) { - clusterUnst, err := client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Get(ctx, spiceCluster.Name, metav1.GetOptions{}) - g.Expect(err).To(Succeed()) - cluster, err := typed.UnstructuredObjToTypedObj[*v1alpha1.SpiceDBCluster](clusterUnst) - g.Expect(err).To(Succeed()) - - config := map[string]any{ - "datastoreEngine": dsDef.datastoreEngine, - "envPrefix": spicedbEnvPrefix, - "cmd": spicedbCmd, - "tlsSecretName": "spicedb-grpc-tls", - "dispatchUpstreamCASecretName": "spicedb-grpc-tls", - "image": "spicedb:next", - } - for k, v := range dsDef.passthroughConfig { - config[k] = v - } - jsonConfig, err := json.Marshal(config) - g.Expect(err).To(Succeed()) - cluster.Spec.Config = jsonConfig - u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cluster) - g.Expect(err).To(Succeed()) - _, err = client.Resource(v1alpha1ClusterGVR).Namespace(cluster.Namespace).Update(ctx, &unstructured.Unstructured{Object: u}, metav1.UpdateOptions{}) - g.Expect(err).To(Succeed()) - }).Should(Succeed()) - }) - - It("migrates to the required edge", func() { - AssertMigrationsCompleted("spicedb:updated", migration, "", - func() (string, string, string) { - return spiceCluster.Namespace, spiceCluster.Name, dsDef.datastoreEngine - }) - }) - - It("migrates to the desired image", func() { - Eventually(func(g Gomega) { - ctx, cancel := context.WithCancel(context.Background()) - DeferCleanup(cancel) - clusterUnst, err := client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Get(ctx, spiceCluster.Name, metav1.GetOptions{}) - g.Expect(err).To(Succeed()) - cluster, err := typed.UnstructuredObjToTypedObj[*v1alpha1.SpiceDBCluster](clusterUnst) - g.Expect(err).To(Succeed()) - GinkgoWriter.Println(cluster.Status) - g.Expect(cluster.Status.Image).To(Equal("spicedb:next")) - }).Should(Succeed()) - }) - }) - }) - }) }) } @@ -1139,11 +951,6 @@ var _ = Describe("SpiceDBClusters", func() { ctx, cancel := context.WithCancel(context.Background()) DeferCleanup(cancel) - init := config.SpiceDBMigrationState{Tag: "v1.13.0", Migration: "add-ns-config-id"} - phase1 := config.SpiceDBMigrationState{Tag: "dev", Migration: "add-xid-columns", Phase: "write-both-read-old"} - phase2 := config.SpiceDBMigrationState{Tag: "dev", Migration: "add-xid-constraints", Phase: "write-both-read-new"} - phase3 := config.SpiceDBMigrationState{Tag: "dev", Migration: "drop-id-constraints"} - phase4 := config.SpiceDBMigrationState{Tag: "updated"} newConfig := config.OperatorConfig{ AllowedTags: []string{"v1.13.0", "dev", "updated"}, @@ -1151,35 +958,20 @@ var _ = Describe("SpiceDBClusters", func() { "ghcr.io/authzed/spicedb", "spicedb", }, - HeadMigrations: map[string]string{ - config.SpiceDBDatastoreState{Tag: "v1.13.0", Datastore: "postgres"}.String(): "add-ns-config-id", - }, - RequiredEdges: map[string]string{ - init.String(): phase1.String(), - phase1.String(): phase2.String(), - phase2.String(): phase3.String(), - phase3.String(): phase4.String(), - }, - Nodes: map[string]config.SpiceDBMigrationState{ - init.String(): init, - phase1.String(): phase1, - phase2.String(): phase2, - phase3.String(): phase3, - phase4.String(): phase4, - }, + UpdateGraph: config.NewUpdateGraph(), } WriteConfig(newConfig) - config := map[string]any{ + classConfig := map[string]any{ "logLevel": "debug", "datastoreEngine": "postgres", "envPrefix": spicedbEnvPrefix, "cmd": spicedbCmd, "tlsSecretName": "spicedb4-grpc-tls", "dispatchUpstreamCASecretName": "spicedb4-grpc-tls", - "image": "ghcr.io/authzed/spicedb:v1.13.0", + "image": "spicedb:v1.13.0", } - jsonConfig, err := json.Marshal(config) + jsonConfig, err := json.Marshal(classConfig) Expect(err).To(BeNil()) spiceCluster = &v1alpha1.SpiceDBCluster{ TypeMeta: metav1.TypeMeta{ @@ -1224,7 +1016,7 @@ var _ = Describe("SpiceDBClusters", func() { _, err = client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Create(ctx, &unstructured.Unstructured{Object: u}, metav1.CreateOptions{}) Expect(err).To(Succeed()) - AssertMigrationsCompleted("ghcr.io/authzed/spicedb:v1.13.0", "add-ns-config-id", "", + AssertMigrationsCompleted("spicedb:v1.13.0", "head", "", func() (string, string, string) { return spiceCluster.Namespace, spiceCluster.Name, "postgres" }) @@ -1238,19 +1030,18 @@ var _ = Describe("SpiceDBClusters", func() { }).Should(Succeed()) // once the cluster is running at the initial version, - // edit the target version to trigger the migration flow - clusterUnst, err := client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Get(ctx, spiceCluster.Name, metav1.GetOptions{}) - Expect(err).To(Succeed()) - fetched, err := typed.UnstructuredObjToTypedObj[*v1alpha1.SpiceDBCluster](clusterUnst) - Expect(err).To(Succeed()) - config["image"] = "spicedb:updated" - jsonConfig, err = json.Marshal(config) - Expect(err).To(BeNil()) - fetched.Spec.Config = jsonConfig - updated, err := runtime.DefaultUnstructuredConverter.ToUnstructured(fetched) - Expect(err).To(Succeed()) - _, err = client.Resource(v1alpha1ClusterGVR).Namespace(spiceCluster.Namespace).Update(ctx, &unstructured.Unstructured{Object: updated}, metav1.UpdateOptions{}) - Expect(err).To(Succeed()) + init := config.SpiceDBMigrationState{Tag: "v1.13.0", Migration: "add-ns-config-id"} + phase1 := config.SpiceDBMigrationState{Tag: "dev", Migration: "add-xid-columns", Phase: "write-both-read-old"} + phase2 := config.SpiceDBMigrationState{Tag: "dev", Migration: "add-xid-constraints", Phase: "write-both-read-new"} + phase3 := config.SpiceDBMigrationState{Tag: "dev", Migration: "drop-id-constraints"} + phase4 := config.SpiceDBMigrationState{Tag: "updated"} + + newConfig.AddHeadMigration(config.SpiceDBDatastoreState{Tag: "v1.13.0", Datastore: "postgres"}, "add-ns-config-id") + newConfig.AddEdge(init, phase1) + newConfig.AddEdge(phase1, phase2) + newConfig.AddEdge(phase2, phase3) + newConfig.AddEdge(phase3, phase4) + WriteConfig(newConfig) }) AfterAll(func() { diff --git a/e2e/spanner.yaml b/e2e/spanner.yaml index 0866387c..d7c9720a 100644 --- a/e2e/spanner.yaml +++ b/e2e/spanner.yaml @@ -35,7 +35,7 @@ spec: spec: containers: - name: spanner - image: gcr.io/cloud-spanner-emulator/emulator:1.4.2 + image: gcr.io/cloud-spanner-emulator/emulator:1.4.6 imagePullPolicy: IfNotPresent ports: - containerPort: 9010 diff --git a/pkg/config/config.go b/pkg/config/config.go index 9e443999..e7f4a46c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -378,7 +378,7 @@ func computeTargets(image, engine string, currentState *SpiceDBMigrationState, g } // look up the actual head migration name, if any - if currentState.Migration == "" || currentState.Migration == "head" { + if currentState != nil && (currentState.Migration == "" || currentState.Migration == "head") { headMigrationName, ok := globalConfig.HeadMigrations[SpiceDBDatastoreState{Tag: tag, Datastore: engine}.String()] if !ok { headMigrationName = "head" @@ -387,19 +387,26 @@ func computeTargets(image, engine string, currentState *SpiceDBMigrationState, g } // if there's a required edge, take it + if globalConfig.RequiredEdges == nil || globalConfig.Nodes == nil { + return + } + requiredEdge, ok := globalConfig.RequiredEdges[currentState.String()] if !ok { return } - reguiredNode, ok := globalConfig.Nodes[requiredEdge] + requiredNode, ok := globalConfig.Nodes[requiredEdge] if !ok { err = fmt.Errorf("required edge defined but not found: %s -> %s", currentState.String(), requiredEdge) return } + targetImage = specBaseImage + ":" + requiredNode.Tag + targetMigration = requiredNode.Migration + targetPhase = requiredNode.Phase - targetImage = specBaseImage + ":" + reguiredNode.Tag - targetMigration = reguiredNode.Migration - targetPhase = reguiredNode.Phase + if targetMigration == "" { + targetMigration = "head" + } return } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 069e4eef..cb17fe50 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -930,26 +930,15 @@ func TestNewConfig(t *testing.T) { ImageName: "image", ImageTag: "init", AllowedImages: []string{"image"}, - AllowedTags: []string{"tag", "tag2"}, - RequiredEdges: map[string]string{ - SpiceDBMigrationState{ - Tag: "init", - }.String(): SpiceDBMigrationState{ - Tag: "tag", - }.String(), - }, - Nodes: map[string]SpiceDBMigrationState{ - SpiceDBMigrationState{ - Tag: "init", - }.String(): { - Tag: "init", - }, - SpiceDBMigrationState{ - Tag: "tag", - }.String(): { - Tag: "tag", - }, - }, + AllowedTags: []string{"init", "tag", "tag2"}, + UpdateGraph: NewUpdateGraph().AddEdge(SpiceDBMigrationState{ + Tag: "init", + Migration: "head", + }, SpiceDBMigrationState{ + Tag: "tag", + Migration: "migration", + Phase: "phase", + }), }, currentState: &SpiceDBMigrationState{ Tag: "init", @@ -957,7 +946,7 @@ func TestNewConfig(t *testing.T) { rawConfig: json.RawMessage(` { "logLevel": "debug", - "image": "image:tag2", + "image": "image:init", "migrationLogLevel": "info", "datastoreEngine": "cockroachdb", "skipMigrations": "true" @@ -979,7 +968,8 @@ func TestNewConfig(t *testing.T) { EnvPrefix: "SPICEDB", SpiceDBCmd: "spicedb", DatastoreTLSSecretName: "", - TargetMigration: "", + TargetMigration: "migration", + TargetPhase: "phase", }, SpiceConfig: SpiceConfig{ LogLevel: "debug", @@ -992,8 +982,9 @@ func TestNewConfig(t *testing.T) { EnvPrefix: "SPICEDB", SpiceDBCmd: "spicedb", Passthrough: map[string]string{ - "datastoreEngine": "cockroachdb", - "dispatchClusterEnabled": "true", + "datastoreEngine": "cockroachdb", + "datastoreMigrationPhase": "phase", + "dispatchClusterEnabled": "true", }, }, }, diff --git a/pkg/config/global.go b/pkg/config/global.go index 0d29da62..d21c5635 100644 --- a/pkg/config/global.go +++ b/pkg/config/global.go @@ -39,15 +39,21 @@ func (s SpiceDBDatastoreState) String() string { // OperatorConfig holds operator-wide config that is used across all objects type OperatorConfig struct { - DisableImageValidation bool `json:"disableImageValidation"` - ImageName string `json:"imageName"` - ImageTag string `json:"imageTag"` - ImageDigest string `json:"imageDigest,omitempty"` - AllowedTags []string `json:"allowedTags"` - AllowedImages []string `json:"allowedImages"` - HeadMigrations map[string]string `json:"headMigrations"` - RequiredEdges map[string]string `json:"requiredEdges"` - Nodes map[string]SpiceDBMigrationState `json:"nodes"` + DisableImageValidation bool `json:"disableImageValidation"` + ImageName string `json:"imageName,omitempty"` + ImageTag string `json:"imageTag,omitempty"` + ImageDigest string `json:"imageDigest,omitempty"` + AllowedTags []string `json:"allowedTags,omitempty"` + AllowedImages []string `json:"allowedImages,omitempty"` + UpdateGraph +} + +func NewOperatorConfig() OperatorConfig { + return OperatorConfig{ + AllowedTags: make([]string, 0), + AllowedImages: make([]string, 0), + UpdateGraph: NewUpdateGraph(), + } } func (o OperatorConfig) DefaultImage() string { @@ -68,8 +74,60 @@ func (o OperatorConfig) Copy() OperatorConfig { ImageDigest: o.ImageDigest, AllowedTags: slices.Clone(o.AllowedTags), AllowedImages: slices.Clone(o.AllowedImages), - HeadMigrations: maps.Clone(o.HeadMigrations), - RequiredEdges: maps.Clone(o.RequiredEdges), - Nodes: maps.Clone(o.Nodes), + UpdateGraph: o.UpdateGraph.Copy(), + } +} + +// UpdateGraph holds a graph of required update edges +type UpdateGraph struct { + HeadMigrations map[string]string `json:"headMigrations,omitempty"` + RequiredEdges map[string]string `json:"requiredEdges,omitempty"` + Nodes map[string]SpiceDBMigrationState `json:"nodes,omitempty"` +} + +func NewUpdateGraph() UpdateGraph { + return UpdateGraph{ + HeadMigrations: make(map[string]string, 0), + RequiredEdges: make(map[string]string, 0), + Nodes: make(map[string]SpiceDBMigrationState, 0), + } +} + +func (o UpdateGraph) GetHeadMigration(state SpiceDBDatastoreState) string { + if m, ok := o.HeadMigrations[state.String()]; ok { + return m + } + return "head" +} + +func (o UpdateGraph) GetRequiredEdge(state SpiceDBMigrationState) (*SpiceDBMigrationState, error) { + requiredEdge, ok := o.RequiredEdges[state.String()] + if !ok { + return nil, nil + } + requiredNode, ok := o.Nodes[requiredEdge] + if !ok { + return nil, fmt.Errorf("required edge defined but not found: %s -> %s", state.String(), requiredEdge) + } + return &requiredNode, nil +} + +func (o UpdateGraph) AddHeadMigration(state SpiceDBDatastoreState, migrationName string) UpdateGraph { + o.HeadMigrations[state.String()] = migrationName + return o +} + +func (o UpdateGraph) AddEdge(from, to SpiceDBMigrationState) UpdateGraph { + o.Nodes[from.String()] = from + o.Nodes[to.String()] = to + o.RequiredEdges[from.String()] = to.String() + return o +} + +func (o UpdateGraph) Copy() UpdateGraph { + return UpdateGraph{ + HeadMigrations: maps.Clone(o.HeadMigrations), + RequiredEdges: maps.Clone(o.RequiredEdges), + Nodes: maps.Clone(o.Nodes), } } diff --git a/pkg/controller/validate_config_test.go b/pkg/controller/validate_config_test.go index a6173d55..69d163c6 100644 --- a/pkg/controller/validate_config_test.go +++ b/pkg/controller/validate_config_test.go @@ -59,7 +59,7 @@ func TestValidateConfigHandler(t *testing.T) { { name: "valid config, new target migrationhash", currentStatus: &v1alpha1.SpiceDBCluster{Status: v1alpha1.ClusterStatus{ - Image: "image", + Image: "image:tag", CurrentMigrationHash: "old", }}, rawConfig: json.RawMessage(`{ diff --git a/tools/go.mod b/tools/go.mod index 82e5da1e..278bef79 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -5,12 +5,14 @@ go 1.19 require ( github.com/authzed/spicedb-operator v0.0.0-00010101000000-000000000000 github.com/google/go-github/v43 v43.0.0 + sigs.k8s.io/yaml v1.3.0 ) require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/authzed/controller-idioms v0.3.0 // indirect + github.com/authzed/controller-idioms v0.5.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/fatih/camelcase v1.0.0 // indirect @@ -32,9 +34,9 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect golang.org/x/exp v0.0.0-20220823124025-807a23277127 // indirect - golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect - golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect + golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect + golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect @@ -47,12 +49,11 @@ require ( k8s.io/api v0.25.0 // indirect k8s.io/apimachinery v0.25.0 // indirect k8s.io/client-go v0.25.0 // indirect - k8s.io/klog/v2 v2.70.1 // indirect + k8s.io/klog/v2 v2.80.0 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) replace github.com/authzed/spicedb-operator => ../ diff --git a/tools/go.sum b/tools/go.sum index 65e4ddbf..a6e100db 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -4,9 +4,11 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/authzed/controller-idioms v0.3.0 h1:8veMtvxO67jqwFzxVradPrcZDPBUGmhwwmXAVuPuPus= -github.com/authzed/controller-idioms v0.3.0/go.mod h1:zfp2hcC4pzMukaQWYZ72jGBMEGpKUV3Y1JGHcCbxwJY= +github.com/authzed/controller-idioms v0.5.0 h1:ZJQR43vrX5tPO9fuJZrCLX2MdeuiJC1toWujKu4b00g= +github.com/authzed/controller-idioms v0.5.0/go.mod h1:Ov4P7cMA/CQF7ojBx1Z9dNUp7pAuJviMerp+aRUtUTc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -93,8 +95,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -132,11 +134,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY= -golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -146,8 +148,8 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -217,8 +219,8 @@ k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.0 h1:lyJt0TWMPaGoODa8B8bUuxgHS3W/m/bNr2cca3brA/g= +k8s.io/klog/v2 v2.80.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsMoD+Izn93E/hm8U=